diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index e8907fac04e..7ea65aeec11 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -12,7 +12,7 @@ impl vortex_alp::ALP pub fn vortex_alp::ALP::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_alp::ALPArray -pub unsafe fn vortex_alp::ALP::new_unchecked(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_alp::ALPArray +pub unsafe fn vortex_alp::ALP::new_unchecked(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option, dtype: vortex_array::dtype::DType) -> vortex_alp::ALPArray pub fn vortex_alp::ALP::try_new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> vortex_error::VortexResult @@ -28,6 +28,8 @@ impl vortex_array::array::vtable::VTable for vortex_alp::ALP pub type vortex_alp::ALP::ArrayData = vortex_alp::ALPData +pub type vortex_alp::ALP::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_alp::ALP::OperationsVTable = vortex_alp::ALP pub type vortex_alp::ALP::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -40,7 +42,11 @@ pub fn vortex_alp::ALP::buffer(_array: vortex_array::array::view::ArrayView<'_, pub fn vortex_alp::ALP::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_alp::ALP::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_alp::ALP::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_alp::ALP::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_alp::ALP::dtype(array: &vortex_alp::ALPData) -> &vortex_array::dtype::DType pub fn vortex_alp::ALP::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -48,17 +54,23 @@ pub fn vortex_alp::ALP::execute_parent(array: vortex_array::array::view::ArrayVi pub fn vortex_alp::ALP::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_alp::ALP::len(array: &vortex_alp::ALPData) -> usize + +pub fn vortex_alp::ALP::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_alp::ALP::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_alp::ALP::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_alp::ALP::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_alp::ALP::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_alp::ALP::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_alp::ALP::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_alp::ALP::validate(&self, data: &vortex_alp::ALPData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_alp::ALP::stats(array: &vortex_alp::ALPData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_alp::ALP::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_alp::ALP::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -106,14 +118,22 @@ pub struct vortex_alp::ALPData impl vortex_alp::ALPData +pub fn vortex_alp::ALPData::dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_alp::ALPData::encoded(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_alp::ALPData::exponents(&self) -> vortex_alp::Exponents -pub fn vortex_alp::ALPData::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option) +pub fn vortex_alp::ALPData::into_parts(self) -> (vortex_array::array::erased::ArrayRef, vortex_alp::Exponents, core::option::Option, vortex_array::dtype::DType) + +pub fn vortex_alp::ALPData::is_empty(&self) -> bool + +pub fn vortex_alp::ALPData::len(&self) -> usize pub fn vortex_alp::ALPData::patches(&self) -> core::option::Option +pub fn vortex_alp::ALPData::ptype(&self) -> vortex_array::dtype::ptype::PType + impl vortex_alp::ALPData pub fn vortex_alp::ALPData::new(encoded: vortex_array::array::erased::ArrayRef, exponents: vortex_alp::Exponents, patches: core::option::Option) -> Self @@ -124,10 +144,18 @@ impl core::clone::Clone for vortex_alp::ALPData pub fn vortex_alp::ALPData::clone(&self) -> vortex_alp::ALPData +impl core::convert::From for vortex_array::array::erased::ArrayRef + +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_alp::ALPData) -> vortex_array::array::erased::ArrayRef + impl core::fmt::Debug for vortex_alp::ALPData pub fn vortex_alp::ALPData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::array::IntoArray for vortex_alp::ALPData + +pub fn vortex_alp::ALPData::into_array(self) -> vortex_array::array::erased::ArrayRef + pub struct vortex_alp::ALPMetadata impl core::clone::Clone for vortex_alp::ALPMetadata @@ -170,6 +198,8 @@ impl vortex_array::array::vtable::VTable for vortex_alp::ALPRD pub type vortex_alp::ALPRD::ArrayData = vortex_alp::ALPRDData +pub type vortex_alp::ALPRD::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_alp::ALPRD::OperationsVTable = vortex_alp::ALPRD pub type vortex_alp::ALPRD::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -182,7 +212,11 @@ pub fn vortex_alp::ALPRD::buffer(_array: vortex_array::array::view::ArrayView<'_ pub fn vortex_alp::ALPRD::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_alp::ALPRD::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_alp::ALPRD::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_alp::ALPRD::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_alp::ALPRD::dtype(array: &vortex_alp::ALPRDData) -> &vortex_array::dtype::DType pub fn vortex_alp::ALPRD::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -190,17 +224,23 @@ pub fn vortex_alp::ALPRD::execute_parent(array: vortex_array::array::view::Array pub fn vortex_alp::ALPRD::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_alp::ALPRD::len(array: &vortex_alp::ALPRDData) -> usize + +pub fn vortex_alp::ALPRD::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_alp::ALPRD::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_alp::ALPRD::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_alp::ALPRD::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_alp::ALPRD::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_alp::ALPRD::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_alp::ALPRD::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_alp::ALPRD::validate(&self, data: &vortex_alp::ALPRDData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_alp::ALPRD::stats(array: &vortex_alp::ALPRDData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_alp::ALPRD::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_alp::ALPRD::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -232,11 +272,37 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_alp::ALPR pub fn vortex_alp::ALPRD::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult> +pub struct vortex_alp::ALPRDArrayParts + +pub vortex_alp::ALPRDArrayParts::dtype: vortex_array::dtype::DType + +pub vortex_alp::ALPRDArrayParts::left_parts: vortex_array::array::erased::ArrayRef + +pub vortex_alp::ALPRDArrayParts::left_parts_dictionary: vortex_buffer::buffer::Buffer + +pub vortex_alp::ALPRDArrayParts::left_parts_patches: core::option::Option + +pub vortex_alp::ALPRDArrayParts::right_parts: vortex_array::array::erased::ArrayRef + +impl core::clone::Clone for vortex_alp::ALPRDArrayParts + +pub fn vortex_alp::ALPRDArrayParts::clone(&self) -> vortex_alp::ALPRDArrayParts + +impl core::fmt::Debug for vortex_alp::ALPRDArrayParts + +pub fn vortex_alp::ALPRDArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub struct vortex_alp::ALPRDData impl vortex_alp::ALPRDData -pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDDataParts +pub fn vortex_alp::ALPRDData::dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_alp::ALPRDData::into_parts(self) -> vortex_alp::ALPRDArrayParts + +pub fn vortex_alp::ALPRDData::is_empty(&self) -> bool + +pub fn vortex_alp::ALPRDData::is_f32(&self) -> bool pub fn vortex_alp::ALPRDData::left_parts(&self) -> &vortex_array::array::erased::ArrayRef @@ -244,6 +310,8 @@ pub fn vortex_alp::ALPRDData::left_parts_dictionary(&self) -> &vortex_buffer::bu pub fn vortex_alp::ALPRDData::left_parts_patches(&self) -> core::option::Option +pub fn vortex_alp::ALPRDData::len(&self) -> usize + pub fn vortex_alp::ALPRDData::replace_left_parts_patches(&mut self, patches: core::option::Option) pub fn vortex_alp::ALPRDData::right_bit_width(&self) -> u8 @@ -256,27 +324,17 @@ impl core::clone::Clone for vortex_alp::ALPRDData pub fn vortex_alp::ALPRDData::clone(&self) -> vortex_alp::ALPRDData -impl core::fmt::Debug for vortex_alp::ALPRDData - -pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -pub struct vortex_alp::ALPRDDataParts +impl core::convert::From for vortex_array::array::erased::ArrayRef -pub vortex_alp::ALPRDDataParts::left_parts: vortex_array::array::erased::ArrayRef +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_alp::ALPRDData) -> vortex_array::array::erased::ArrayRef -pub vortex_alp::ALPRDDataParts::left_parts_dictionary: vortex_buffer::buffer::Buffer - -pub vortex_alp::ALPRDDataParts::left_parts_patches: core::option::Option - -pub vortex_alp::ALPRDDataParts::right_parts: vortex_array::array::erased::ArrayRef - -impl core::clone::Clone for vortex_alp::ALPRDDataParts +impl core::fmt::Debug for vortex_alp::ALPRDData -pub fn vortex_alp::ALPRDDataParts::clone(&self) -> vortex_alp::ALPRDDataParts +pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl core::fmt::Debug for vortex_alp::ALPRDDataParts +impl vortex_array::array::IntoArray for vortex_alp::ALPRDData -pub fn vortex_alp::ALPRDDataParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_alp::ALPRDData::into_array(self) -> vortex_array::array::erased::ArrayRef pub struct vortex_alp::ALPRDMetadata diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 93dfb8d3932..a5930c9b1ce 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -4,18 +4,19 @@ use std::fmt::Debug; use std::hash::Hash; -use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; +use vortex_array::SerializeMetadata; use vortex_array::arrays::Primitive; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; @@ -25,6 +26,7 @@ use vortex_array::patches::PatchesMetadata; use vortex_array::require_child; use vortex_array::require_patches; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; @@ -47,15 +49,28 @@ vtable!(ALP, ALP, ALPData); impl VTable for ALP { type ArrayData = ALPData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; + fn vtable(_array: &Self::ArrayData) -> &Self { + &ALP + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &ALPData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) + fn len(array: &ALPData) -> usize { + array.encoded().len() + } + + fn dtype(array: &ALPData) -> &DType { + &array.dtype + } + + fn stats(array: &ALPData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &ALPData, state: &mut H, precision: Precision) { @@ -82,31 +97,41 @@ impl VTable for ALP { None } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { let exponents = array.exponents(); - Ok(Some( - ALPMetadata { - exp_e: exponents.e as u32, - exp_f: exponents.f as u32, - patches: array - .patches() - .map(|p| p.to_metadata(array.len(), array.dtype())) - .transpose()?, - } - .encode_to_vec(), - )) + Ok(ProstMetadata(ALPMetadata { + exp_e: exponents.e as u32, + exp_f: exponents.f as u32, + patches: array + .patches() + .map(|p| p.to_metadata(array.len(), array.dtype())) + .transpose()?, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata( + as DeserializeMetadata>::deserialize(bytes)?, + )) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = ALPMetadata::decode(metadata)?; let encoded_ptype = match &dtype { DType::Primitive(PType::F32, n) => DType::Primitive(PType::I32, *n), DType::Primitive(PType::F64, n) => DType::Primitive(PType::I64, *n), @@ -218,7 +243,9 @@ pub struct ALPData { pub(super) slots: Vec>, patch_offset: Option, patch_offset_within_chunk: Option, + dtype: DType, exponents: Exponents, + stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -239,7 +266,7 @@ pub struct ALPMetadata { } impl ALPData { - fn validate_components( + fn validate( encoded: &ArrayRef, exponents: Exponents, patches: Option<&Patches>, @@ -290,38 +317,6 @@ impl ALPData { Ok(()) } - fn logical_dtype(encoded: &ArrayRef) -> VortexResult { - match encoded.dtype() { - DType::Primitive(PType::I32, nullability) => { - Ok(DType::Primitive(PType::F32, *nullability)) - } - DType::Primitive(PType::I64, nullability) => { - Ok(DType::Primitive(PType::F64, *nullability)) - } - _ => vortex_bail!("ALP encoded ints have invalid DType {}", encoded.dtype(),), - } - } - - fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { - let patches = self.patches(); - let logical_dtype = Self::logical_dtype(self.encoded())?; - Self::validate_components(self.encoded(), self.exponents, patches.as_ref())?; - - vortex_ensure!( - self.encoded().len() == len, - "ALP encoded len {} != outer len {len}", - self.encoded().len(), - ); - vortex_ensure!( - &logical_dtype == dtype, - "ALP dtype {} does not match encoded logical dtype {}", - dtype, - logical_dtype, - ); - - Ok(()) - } - /// Validate that any patches provided are valid for the ALPArray. fn validate_patches(patches: &Patches, encoded: &ArrayRef) -> VortexResult<()> { vortex_ensure!( @@ -373,7 +368,7 @@ impl ALPData { /// # Examples /// /// ``` - /// # use vortex_alp::{ALP, ALPData, Exponents}; + /// # use vortex_alp::{ALPData, ALPArray, Exponents}; /// # use vortex_array::IntoArray; /// # use vortex_buffer::buffer; /// @@ -394,11 +389,11 @@ impl ALPData { /// assert!(result.is_err()); /// /// // Success! - /// let value = ALP::try_new( + /// let value = ALPArray::try_from_data(ALPData::try_new( /// buffer![0i32].into_array(), /// Exponents { e: 1, f: 1 }, /// None - /// ).unwrap(); + /// ).unwrap()).unwrap(); /// /// assert_eq!(value.scalar_at(0).unwrap(), 0f32.into()); /// ``` @@ -407,7 +402,13 @@ impl ALPData { exponents: Exponents, patches: Option, ) -> VortexResult { - Self::validate_components(&encoded, exponents, patches.as_ref())?; + Self::validate(&encoded, exponents, patches.as_ref())?; + + let dtype = match encoded.dtype() { + DType::Primitive(PType::I32, nullability) => DType::Primitive(PType::F32, *nullability), + DType::Primitive(PType::I64, nullability) => DType::Primitive(PType::F64, *nullability), + _ => unreachable!(), + }; let slots = Self::make_slots(&encoded, &patches); let (patch_offset, patch_offset_within_chunk) = match &patches { @@ -416,10 +417,12 @@ impl ALPData { }; Ok(Self { + dtype, slots, + exponents, patch_offset, patch_offset_within_chunk, - exponents, + stats_set: Default::default(), }) } @@ -431,6 +434,7 @@ impl ALPData { encoded: ArrayRef, exponents: Exponents, patches: Option, + dtype: DType, ) -> Self { let slots = Self::make_slots(&encoded, &patches); let (patch_offset, patch_offset_within_chunk) = match &patches { @@ -439,10 +443,12 @@ impl ALPData { }; Self { + dtype, slots, + exponents, patch_offset, patch_offset_within_chunk, - exponents, + stats_set: Default::default(), } } } @@ -450,16 +456,8 @@ impl ALPData { /// Constructors for [`ALPArray`]. impl ALP { pub fn new(encoded: ArrayRef, exponents: Exponents, patches: Option) -> ALPArray { - let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); - let len = encoded.len(); - unsafe { - Array::from_parts_unchecked(ArrayParts::new( - ALP, - dtype, - len, - ALPData::new(encoded, exponents, patches), - )) - } + Array::try_from_data(ALPData::new(encoded, exponents, patches)) + .vortex_expect("ALPData is always valid") } pub fn try_new( @@ -467,10 +465,7 @@ impl ALP { exponents: Exponents, patches: Option, ) -> VortexResult { - let dtype = ALPData::logical_dtype(&encoded)?; - let len = encoded.len(); - let data = ALPData::try_new(encoded, exponents, patches)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data)) }) + Array::try_from_data(ALPData::try_new(encoded, exponents, patches)?) } /// # Safety @@ -479,15 +474,29 @@ impl ALP { encoded: ArrayRef, exponents: Exponents, patches: Option, + dtype: DType, ) -> ALPArray { - let dtype = ALPData::logical_dtype(&encoded).vortex_expect("ALP encoded dtype"); - let len = encoded.len(); - let data = unsafe { ALPData::new_unchecked(encoded, exponents, patches) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(ALP, dtype, len, data)) } + Array::try_from_data(unsafe { ALPData::new_unchecked(encoded, exponents, patches, dtype) }) + .vortex_expect("ALPData is always valid") } } impl ALPData { + /// Returns the number of elements in the array. + pub fn len(&self) -> usize { + self.encoded().len() + } + + /// Returns `true` if the array contains no elements. + pub fn is_empty(&self) -> bool { + self.encoded().len() == 0 + } + + /// Returns the logical data type of the array. + pub fn dtype(&self) -> &DType { + &self.dtype + } + fn make_slots(encoded: &ArrayRef, patches: &Option) -> Vec> { let (patch_indices, patch_values, patch_chunk_offsets) = match patches { Some(p) => ( @@ -505,6 +514,10 @@ impl ALPData { ] } + pub fn ptype(&self) -> PType { + self.dtype.as_ptype() + } + pub fn encoded(&self) -> &ArrayRef { self.slots[ENCODED_SLOT] .as_ref() @@ -542,12 +555,12 @@ impl ALPData { /// Consumes the array and returns its parts. #[inline] - pub fn into_parts(mut self) -> (ArrayRef, Exponents, Option) { + pub fn into_parts(mut self) -> (ArrayRef, Exponents, Option, DType) { let patches = self.patches(); let encoded = self.slots[ENCODED_SLOT] .take() .vortex_expect("ALPArray encoded slot"); - (encoded, self.exponents, patches) + (encoded, self.exponents, patches, self.dtype) } } @@ -871,11 +884,12 @@ mod tests { .unwrap(); // Build a new ALPArray with the same encoded data but patches without chunk_offsets. - let alp_without_chunk_offsets = ALP::new( + let alp_without_chunk_offsets = ALPArray::try_from_data(ALPData::new( normally_encoded.encoded().clone(), normally_encoded.exponents(), Some(patches_without_chunk_offsets), - ); + )) + .vortex_expect("ALPData is always valid"); // The legacy decompress_into_array path should work correctly. let result_legacy = decompress_into_array( diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index a55baf02698..e8992c7a2a4 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -47,7 +47,14 @@ pub fn alp_encode(parray: &PrimitiveArray, exponents: Option) -> Vort }; // SAFETY: alp_encode_components_typed must return well-formed components - unsafe { Ok(ALP::new_unchecked(encoded, exponents, patches)) } + unsafe { + Ok(ALP::new_unchecked( + encoded, + exponents, + patches, + parray.dtype().clone(), + )) + } } #[expect( diff --git a/encodings/alp/src/alp/compute/between.rs b/encodings/alp/src/alp/compute/between.rs index efaaf747768..ff0b4da8943 100644 --- a/encodings/alp/src/alp/compute/between.rs +++ b/encodings/alp/src/alp/compute/between.rs @@ -38,7 +38,7 @@ impl BetweenReduce for ALP { let nullability = array.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); - match_each_alp_float_ptype!(array.dtype().as_ptype(), |F| { + match_each_alp_float_ptype!(array.ptype(), |F| { between_impl::( array, F::try_from(&lower)?, diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index bd8a8b38381..ebca5e3fe0f 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -45,7 +45,8 @@ impl CastReduce for ALP { // SAFETY: casting nullability doesn't alter the invariants unsafe { Ok(Some( - ALP::new_unchecked(new_encoded, array.exponents(), new_patches).into_array(), + ALP::new_unchecked(new_encoded, array.exponents(), new_patches, dtype.clone()) + .into_array(), )) } } else { diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index 3daa6d53d7b..a4e3c4371d9 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -30,6 +30,7 @@ impl FilterKernel for ALP { array.encoded().filter(mask.clone())?, array.exponents(), patches, + array.dtype().clone(), ) .into_array(), )) diff --git a/encodings/alp/src/alp/decompress.rs b/encodings/alp/src/alp/decompress.rs index e118b9705cd..0e7c1a5e7a0 100644 --- a/encodings/alp/src/alp/decompress.rs +++ b/encodings/alp/src/alp/decompress.rs @@ -27,8 +27,7 @@ pub fn decompress_into_array( array: ALPArray, ctx: &mut ExecutionCtx, ) -> VortexResult { - let dtype = array.dtype().clone(); - let (encoded, exponents, patches) = array.into_data().into_parts(); + let (encoded, exponents, patches, dtype) = array.into_data().into_parts(); if let Some(ref patches) = patches && let Some(chunk_offsets) = patches.chunk_offsets() { @@ -60,8 +59,7 @@ pub fn decompress_into_array( /// /// A `PrimitiveArray` containing the decompressed floating-point values with all patches applied. pub fn execute_decompress(array: ALPArray, ctx: &mut ExecutionCtx) -> VortexResult { - let dtype = array.dtype().clone(); - let (encoded, exponents, patches) = array.into_data().into_parts(); + let (encoded, exponents, patches, dtype) = array.into_data().into_parts(); if let Some(ref patches) = patches && let Some(chunk_offsets) = patches.chunk_offsets() { diff --git a/encodings/alp/src/alp/mod.rs b/encodings/alp/src/alp/mod.rs index 7d5049b59e3..b68d58b2ad3 100644 --- a/encodings/alp/src/alp/mod.rs +++ b/encodings/alp/src/alp/mod.rs @@ -22,7 +22,7 @@ mod rules; #[cfg(test)] mod tests { - use prost::Message; + use vortex_array::ProstMetadata; use vortex_array::dtype::PType; use vortex_array::patches::PatchesMetadata; use vortex_array::test_harness::check_metadata; @@ -34,7 +34,7 @@ mod tests { fn test_alp_metadata() { check_metadata( "alp.metadata", - &ALPMetadata { + ProstMetadata(ALPMetadata { patches: Some(PatchesMetadata::new( usize::MAX, usize::MAX, @@ -45,8 +45,7 @@ mod tests { )), exp_e: u32::MAX, exp_f: u32::MAX, - } - .encode_to_vec(), + }), ); } } diff --git a/encodings/alp/src/alp/ops.rs b/encodings/alp/src/alp/ops.rs index 25cf19a3e55..9538e3c60a1 100644 --- a/encodings/alp/src/alp/ops.rs +++ b/encodings/alp/src/alp/ops.rs @@ -26,7 +26,7 @@ impl OperationsVTable for ALP { let encoded_val = array.encoded().scalar_at(index)?; - Ok(match_each_alp_float_ptype!(array.dtype().as_ptype(), |T| { + Ok(match_each_alp_float_ptype!(array.ptype(), |T| { let encoded_val: ::ALPInt = (&encoded_val).try_into().vortex_expect("invalid ALPInt"); Scalar::primitive( diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index b16470372bc..eca56a97a37 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -5,18 +5,19 @@ use std::fmt::Debug; use std::hash::Hash; use itertools::Itertools; -use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; +use vortex_array::SerializeMetadata; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -28,6 +29,7 @@ use vortex_array::patches::PatchesMetadata; use vortex_array::require_child; use vortex_array::require_patches; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -65,15 +67,28 @@ pub struct ALPRDMetadata { impl VTable for ALPRD { type ArrayData = ALPRDData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; + fn vtable(_array: &Self::ArrayData) -> &Self { + &ALPRD + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &ALPRDData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) + fn len(array: &ALPRDData) -> usize { + array.left_parts().len() + } + + fn dtype(array: &ALPRDData) -> &DType { + &array.dtype + } + + fn stats(array: &ALPRDData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &ALPRDData, state: &mut H, precision: Precision) { @@ -108,38 +123,48 @@ impl VTable for ALPRD { None } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { let dict = array .left_parts_dictionary() .iter() .map(|&i| i as u32) .collect::>(); - Ok(Some( - ALPRDMetadata { - right_bit_width: array.right_bit_width() as u32, - dict_len: array.left_parts_dictionary().len() as u32, - dict, - left_parts_ptype: array.left_parts().dtype().as_ptype() as i32, - patches: array - .left_parts_patches() - .map(|p| p.to_metadata(array.len(), array.left_parts().dtype())) - .transpose()?, - } - .encode_to_vec(), - )) + Ok(ProstMetadata(ALPRDMetadata { + right_bit_width: array.right_bit_width() as u32, + dict_len: array.left_parts_dictionary().len() as u32, + dict, + left_parts_ptype: array.left_parts().dtype().as_ptype() as i32, + patches: array + .left_parts_patches() + .map(|p| p.to_metadata(array.len(), array.left_parts().dtype())) + .transpose()?, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata( + as DeserializeMetadata>::deserialize(bytes)?, + )) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = ALPRDMetadata::decode(metadata)?; if children.len() < 2 { vortex_bail!( "Expected at least 2 children for ALPRD encoding, found {}", @@ -147,10 +172,10 @@ impl VTable for ALPRD { ); } - let left_parts_dtype = DType::Primitive(metadata.left_parts_ptype(), dtype.nullability()); + let left_parts_dtype = DType::Primitive(metadata.0.left_parts_ptype(), dtype.nullability()); let left_parts = children.get(0, &left_parts_dtype, len)?; - let left_parts_dictionary: Buffer = metadata.dict.as_slice() - [0..metadata.dict_len as usize] + let left_parts_dictionary: Buffer = metadata.0.dict.as_slice() + [0..metadata.0.dict_len as usize] .iter() .map(|&i| { u16::try_from(i) @@ -170,6 +195,7 @@ impl VTable for ALPRD { let right_parts = children.get(1, &right_parts_dtype, len)?; let left_parts_patches = metadata + .0 .patches .map(|p| { let indices = children.get(2, &p.indices_dtype()?, p.len()?)?; @@ -191,10 +217,10 @@ impl VTable for ALPRD { left_parts, left_parts_dictionary, right_parts, - u8::try_from(metadata.right_bit_width).map_err(|_| { + u8::try_from(metadata.0.right_bit_width).map_err(|_| { vortex_err!( "right_bit_width {} out of u8 range", - metadata.right_bit_width + metadata.0.right_bit_width ) })?, left_parts_patches, @@ -250,13 +276,14 @@ impl VTable for ALPRD { LP_PATCH_CHUNK_OFFSETS_SLOT ); - let dtype = array.dtype().clone(); let right_bit_width = array.right_bit_width(); - let ALPRDDataParts { + let ALPRDArrayParts { left_parts, right_parts, left_parts_dictionary, left_parts_patches, + dtype, + .. } = array.into_data().into_parts(); let ptype = dtype.as_ptype(); @@ -341,14 +368,17 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ #[derive(Clone, Debug)] pub struct ALPRDData { + dtype: DType, slots: Vec>, left_parts_patches: Option, left_parts_dictionary: Buffer, right_bit_width: u8, + stats_set: ArrayStats, } #[derive(Clone, Debug)] -pub struct ALPRDDataParts { +pub struct ALPRDArrayParts { + pub dtype: DType, pub left_parts: ArrayRef, pub left_parts_patches: Option, pub left_parts_dictionary: Buffer, @@ -369,21 +399,14 @@ impl ALPRD { right_bit_width: u8, left_parts_patches: Option, ) -> VortexResult { - let len = left_parts.len(); - let logical_dtype = dtype.clone(); - let data = ALPRDData::try_new( + Array::try_from_data(ALPRDData::try_new( dtype, left_parts, left_parts_dictionary, right_parts, right_bit_width, left_parts_patches, - )?; - Ok( - unsafe { - Array::from_parts_unchecked(ArrayParts::new(ALPRD, logical_dtype, len, data)) - }, - ) + )?) } /// # Safety @@ -396,9 +419,7 @@ impl ALPRD { right_bit_width: u8, left_parts_patches: Option, ) -> ALPRDArray { - let len = left_parts.len(); - let logical_dtype = dtype.clone(); - let data = unsafe { + Array::try_from_data(unsafe { ALPRDData::new_unchecked( dtype, left_parts, @@ -407,33 +428,33 @@ impl ALPRD { right_bit_width, left_parts_patches, ) - }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(ALPRD, logical_dtype, len, data)) } + }) + .vortex_expect("ALPRDData is always valid") } } impl ALPRDData { - fn validate_parts( - dtype: &DType, - len: usize, - left_parts: &ArrayRef, - right_parts: &ArrayRef, - left_parts_patches: Option<&Patches>, - ) -> VortexResult<()> { + /// Build a new `ALPRDArray` from components. + pub fn try_new( + dtype: DType, + left_parts: ArrayRef, + left_parts_dictionary: Buffer, + right_parts: ArrayRef, + right_bit_width: u8, + left_parts_patches: Option, + ) -> VortexResult { if !dtype.is_float() { vortex_bail!("ALPRDArray given invalid DType ({dtype})"); } - vortex_ensure!( - left_parts.len() == len, - "left_parts len {} != outer len {len}", - left_parts.len(), - ); - vortex_ensure!( - right_parts.len() == len, - "right_parts len {} != outer len {len}", - right_parts.len(), - ); + let len = left_parts.len(); + if right_parts.len() != len { + vortex_bail!( + "left_parts (len {}) and right_parts (len {}) must be of same length", + len, + right_parts.len() + ); + } if !left_parts.dtype().is_unsigned_int() { vortex_bail!("left_parts dtype must be uint"); @@ -447,64 +468,11 @@ impl ALPRDData { ); } - let expected_right_parts_dtype = match dtype { - DType::Primitive(PType::F32, _) => { - DType::Primitive(PType::U32, Nullability::NonNullable) - } - DType::Primitive(PType::F64, _) => { - DType::Primitive(PType::U64, Nullability::NonNullable) - } - _ => vortex_bail!("Expected f32 or f64 dtype, got {:?}", dtype), - }; - vortex_ensure!( - right_parts.dtype() == &expected_right_parts_dtype, - "right_parts dtype {} does not match expected {}", - right_parts.dtype(), - expected_right_parts_dtype, - ); - - if let Some(patches) = left_parts_patches { - vortex_ensure!( - patches.array_len() == len, - "patches array_len {} != outer len {len}", - patches.array_len(), - ); - vortex_ensure!( - patches.dtype() == left_parts.dtype(), - "patches dtype {} does not match left_parts dtype {}", - patches.dtype(), - left_parts.dtype(), - ); - vortex_ensure!( - patches.values().all_valid()?, - "patches must be all valid: {}", - patches.values() - ); + // we enforce right_parts to be non-nullable uint + if !right_parts.dtype().is_unsigned_int() || right_parts.dtype().is_nullable() { + vortex_bail!(MismatchedTypes: "non-nullable uint", right_parts.dtype()); } - Ok(()) - } - - fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { - Self::validate_parts( - dtype, - len, - self.left_parts(), - self.right_parts(), - self.left_parts_patches.as_ref(), - ) - } - - /// Build a new `ALPRDArray` from components. - pub fn try_new( - dtype: DType, - left_parts: ArrayRef, - left_parts_dictionary: Buffer, - right_parts: ArrayRef, - right_bit_width: u8, - left_parts_patches: Option, - ) -> VortexResult { - let len = left_parts.len(); let left_parts_patches = left_parts_patches .map(|patches| { if !patches.values().all_valid()? { @@ -520,28 +488,22 @@ impl ALPRDData { }) .transpose()?; - Self::validate_parts( - &dtype, - len, - &left_parts, - &right_parts, - left_parts_patches.as_ref(), - )?; - let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); Ok(Self { + dtype, slots, - left_parts_patches, left_parts_dictionary, right_bit_width, + left_parts_patches, + stats_set: Default::default(), }) } /// Build a new `ALPRDArray` from components. This does not perform any validation, and instead /// it constructs it from parts. pub(crate) unsafe fn new_unchecked( - _dtype: DType, + dtype: DType, left_parts: ArrayRef, left_parts_dictionary: Buffer, right_parts: ArrayRef, @@ -551,10 +513,12 @@ impl ALPRDData { let slots = Self::make_slots(&left_parts, &right_parts, &left_parts_patches); Self { + dtype, slots, left_parts_patches, left_parts_dictionary, right_bit_width, + stats_set: Default::default(), } } @@ -581,14 +545,15 @@ impl ALPRDData { } /// Return all the owned parts of the array - pub fn into_parts(mut self) -> ALPRDDataParts { + pub fn into_parts(mut self) -> ALPRDArrayParts { let left_parts = self.slots[LEFT_PARTS_SLOT] .take() .vortex_expect("ALPRDArray left_parts slot"); let right_parts = self.slots[RIGHT_PARTS_SLOT] .take() .vortex_expect("ALPRDArray right_parts slot"); - ALPRDDataParts { + ALPRDArrayParts { + dtype: self.dtype, left_parts, left_parts_patches: self.left_parts_patches, left_parts_dictionary: self.left_parts_dictionary, @@ -596,6 +561,29 @@ impl ALPRDData { } } + /// Returns the number of elements in the array. + pub fn len(&self) -> usize { + self.left_parts().len() + } + + /// Returns `true` if the array contains no elements. + pub fn is_empty(&self) -> bool { + self.left_parts().len() == 0 + } + + /// Returns the logical data type of the array. + pub fn dtype(&self) -> &DType { + &self.dtype + } + + /// Returns true if logical type of the array values is f32. + /// + /// Returns false if the logical type of the array values is f64. + #[inline] + pub fn is_f32(&self) -> bool { + matches!(&self.dtype, DType::Primitive(PType::F32, _)) + } + /// The leftmost (most significant) bits of the floating point values stored in the array. /// /// These are bit-packed and dictionary encoded, and cannot directly be interpreted without @@ -654,8 +642,8 @@ impl ValidityChild for ALPRD { #[cfg(test)] mod test { - use prost::Message; use rstest::rstest; + use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -699,7 +687,7 @@ mod test { fn test_alprd_metadata() { check_metadata( "alprd.metadata", - &ALPRDMetadata { + ProstMetadata(ALPRDMetadata { right_bit_width: u32::MAX, patches: Some(PatchesMetadata::new( usize::MAX, @@ -712,8 +700,7 @@ mod test { dict: Vec::new(), left_parts_ptype: PType::U64 as i32, dict_len: 8, - } - .encode_to_vec(), + }), ); } } diff --git a/encodings/alp/src/alp_rd/ops.rs b/encodings/alp/src/alp_rd/ops.rs index 6f77fdbd843..63dc3fc5d89 100644 --- a/encodings/alp/src/alp_rd/ops.rs +++ b/encodings/alp/src/alp_rd/ops.rs @@ -3,7 +3,6 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; -use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_array::vtable::OperationsVTable; use vortex_error::VortexExpect; @@ -40,7 +39,7 @@ impl OperationsVTable for ALPRD { }; // combine left and right values - Ok(if array.dtype().as_ptype() == PType::F32 { + Ok(if array.is_f32() { let right: u32 = array .right_parts() .scalar_at(index)? diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 74eed2ce798..5f05421a6d6 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -6,12 +6,8 @@ impl vortex_bytebool::ByteBool pub const vortex_bytebool::ByteBool::ID: vortex_array::array::ArrayId -pub fn vortex_bytebool::ByteBool::from_option_vec(data: alloc::vec::Vec>) -> vortex_bytebool::ByteBoolArray - pub fn vortex_bytebool::ByteBool::from_vec>(data: alloc::vec::Vec, validity: V) -> vortex_bytebool::ByteBoolArray -pub fn vortex_bytebool::ByteBool::new(buffer: vortex_array::buffer::BufferHandle, validity: vortex_array::validity::Validity) -> vortex_bytebool::ByteBoolArray - impl core::clone::Clone for vortex_bytebool::ByteBool pub fn vortex_bytebool::ByteBool::clone(&self) -> vortex_bytebool::ByteBool @@ -24,6 +20,8 @@ impl vortex_array::array::vtable::VTable for vortex_bytebool::ByteBool pub type vortex_bytebool::ByteBool::ArrayData = vortex_bytebool::ByteBoolData +pub type vortex_bytebool::ByteBool::Metadata = vortex_array::metadata::EmptyMetadata + pub type vortex_bytebool::ByteBool::OperationsVTable = vortex_bytebool::ByteBool pub type vortex_bytebool::ByteBool::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValidityHelper @@ -36,7 +34,11 @@ pub fn vortex_bytebool::ByteBool::buffer(array: vortex_array::array::view::Array pub fn vortex_bytebool::ByteBool::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_bytebool::ByteBool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_bytebool::ByteBool::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_bytebool::ByteBool::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_bytebool::ByteBool::dtype(array: &vortex_bytebool::ByteBoolData) -> &vortex_array::dtype::DType pub fn vortex_bytebool::ByteBool::execute(array: vortex_array::array::typed::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -44,17 +46,23 @@ pub fn vortex_bytebool::ByteBool::execute_parent(array: vortex_array::array::vie pub fn vortex_bytebool::ByteBool::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_bytebool::ByteBool::len(array: &vortex_bytebool::ByteBoolData) -> usize + +pub fn vortex_bytebool::ByteBool::metadata(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_bytebool::ByteBool::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_bytebool::ByteBool::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_bytebool::ByteBool::serialize(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_bytebool::ByteBool::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_bytebool::ByteBool::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_bytebool::ByteBool::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_bytebool::ByteBool::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_bytebool::ByteBool::stats(array: &vortex_bytebool::ByteBoolData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_bytebool::ByteBool::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_bytebool::ByteBool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -86,6 +94,8 @@ pub fn vortex_bytebool::ByteBoolData::as_slice(&self) -> &[bool] pub fn vortex_bytebool::ByteBoolData::buffer(&self) -> &vortex_array::buffer::BufferHandle +pub fn vortex_bytebool::ByteBoolData::dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_bytebool::ByteBoolData::from_vec>(data: alloc::vec::Vec, validity: V) -> Self pub fn vortex_bytebool::ByteBoolData::is_empty(&self) -> bool @@ -94,8 +104,6 @@ pub fn vortex_bytebool::ByteBoolData::len(&self) -> usize pub fn vortex_bytebool::ByteBoolData::new(buffer: vortex_array::buffer::BufferHandle, validity: vortex_array::validity::Validity) -> Self -pub fn vortex_bytebool::ByteBoolData::validate(buffer: &vortex_array::buffer::BufferHandle, validity: &vortex_array::validity::Validity, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - pub fn vortex_bytebool::ByteBoolData::validity_mask(&self) -> vortex_mask::Mask impl core::clone::Clone for vortex_bytebool::ByteBoolData @@ -110,10 +118,18 @@ impl core::convert::From>> for vortex pub fn vortex_bytebool::ByteBoolData::from(value: alloc::vec::Vec>) -> Self +impl core::convert::From for vortex_array::array::erased::ArrayRef + +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_bytebool::ByteBoolData) -> vortex_array::array::erased::ArrayRef + impl core::fmt::Debug for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::array::IntoArray for vortex_bytebool::ByteBoolData + +pub fn vortex_bytebool::ByteBoolData::into_array(self) -> vortex_array::array::erased::ArrayRef + impl vortex_array::array::vtable::validity::ValidityHelper for vortex_bytebool::ByteBoolData pub fn vortex_bytebool::ByteBoolData::validity(&self) -> &vortex_array::validity::Validity diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 48448ba7b57..04744355a11 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -7,9 +7,9 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; @@ -19,6 +19,7 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; @@ -28,6 +29,7 @@ use vortex_array::vtable::ValidityVTableFromValidityHelper; use vortex_array::vtable::validity_to_child; use vortex_buffer::BitBuffer; use vortex_buffer::ByteBuffer; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -42,15 +44,28 @@ vtable!(ByteBool, ByteBool, ByteBoolData); impl VTable for ByteBool { type ArrayData = ByteBoolData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValidityHelper; + fn vtable(_array: &Self::ArrayData) -> &Self { + &ByteBool + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - ByteBoolData::validate(data.buffer(), data.validity(), dtype, len) + fn len(array: &ByteBoolData) -> usize { + array.buffer.len() + } + + fn dtype(array: &ByteBoolData) -> &DType { + &array.dtype + } + + fn stats(array: &ByteBoolData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &ByteBoolData, state: &mut H, precision: Precision) { @@ -81,25 +96,31 @@ impl VTable for ByteBool { } } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + _metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - if !metadata.is_empty() { - vortex_bail!( - "ByteBoolArray expects empty metadata, got {} bytes", - metadata.len() - ); - } let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { @@ -134,7 +155,7 @@ impl VTable for ByteBool { ); array.validity = match &slots[VALIDITY_SLOT] { Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.validity.nullability()), + None => Validity::from(array.dtype.nullability()), }; array.slots = slots; Ok(()) @@ -173,9 +194,11 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct ByteBoolData { + dtype: DType, buffer: BufferHandle, validity: Validity, pub(super) slots: Vec>, + stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -184,53 +207,14 @@ pub struct ByteBool; impl ByteBool { pub const ID: ArrayId = ArrayId::new_ref("vortex.bytebool"); - pub fn new(buffer: BufferHandle, validity: Validity) -> ByteBoolArray { - let dtype = DType::Bool(validity.nullability()); - let data = ByteBoolData::new(buffer, validity); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } - } - /// Construct a [`ByteBoolArray`] from a `Vec` and validity. pub fn from_vec>(data: Vec, validity: V) -> ByteBoolArray { - let data = ByteBoolData::from_vec(data, validity); - let dtype = DType::Bool(data.validity.nullability()); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } - } - - /// Construct a [`ByteBoolArray`] from optional bools. - pub fn from_option_vec(data: Vec>) -> ByteBoolArray { - let data = ByteBoolData::from(data); - let dtype = DType::Bool(data.validity.nullability()); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(ByteBool, dtype, len, data)) } + Array::try_from_data(ByteBoolData::from_vec(data, validity)) + .vortex_expect("ByteBoolData is always valid") } } impl ByteBoolData { - pub fn validate( - buffer: &BufferHandle, - validity: &Validity, - dtype: &DType, - len: usize, - ) -> VortexResult<()> { - let expected_dtype = DType::Bool(validity.nullability()); - vortex_ensure!( - dtype == &expected_dtype, - "expected dtype {expected_dtype}, got {dtype}" - ); - vortex_ensure!( - buffer.len() == len, - "expected len {len}, got {}", - buffer.len() - ); - if let Some(vlen) = validity.maybe_len() { - vortex_ensure!(vlen == len, "expected validity len {len}, got {vlen}"); - } - Ok(()) - } - fn make_slots(validity: &Validity, len: usize) -> Vec> { vec![validity_to_child(validity, len)] } @@ -248,9 +232,11 @@ impl ByteBoolData { } let slots = Self::make_slots(&validity, length); Self { + dtype: DType::Bool(validity.nullability()), buffer, validity, slots, + stats_set: Default::default(), } } @@ -264,6 +250,11 @@ impl ByteBoolData { self.buffer.len() == 0 } + /// Returns the logical data type of the array. + pub fn dtype(&self) -> &DType { + &self.dtype + } + /// Returns the validity mask for this array. pub fn validity_mask(&self) -> Mask { self.validity.to_mask(self.len()) @@ -328,8 +319,8 @@ mod tests { use vortex_array::ArrayContext; use vortex_array::IntoArray; use vortex_array::assert_arrays_eq; + use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; - use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::ByteBufferMut; @@ -343,7 +334,8 @@ mod tests { let v = vec![true, false]; let v_len = v.len(); - let arr = ByteBool::from_vec(v, Validity::AllValid); + let arr = ByteBoolArray::try_from_data(ByteBoolData::from(v)) + .vortex_expect("ByteBoolData is always valid"); assert_eq!(v_len, arr.len()); for idx in 0..arr.len() { @@ -351,7 +343,8 @@ mod tests { } let v = vec![Some(true), None, Some(false)]; - let arr = ByteBool::from_option_vec(v); + let arr = ByteBoolArray::try_from_data(ByteBoolData::from(v)) + .vortex_expect("ByteBoolData is always valid"); assert!(arr.is_valid(0).unwrap()); assert!(!arr.is_valid(1).unwrap()); assert!(arr.is_valid(2).unwrap()); @@ -360,7 +353,8 @@ mod tests { let v: Vec> = vec![None, None]; let v_len = v.len(); - let arr = ByteBool::from_option_vec(v); + let arr = ByteBoolArray::try_from_data(ByteBoolData::from(v)) + .vortex_expect("ByteBoolData is always valid"); assert_eq!(v_len, arr.len()); for idx in 0..arr.len() { @@ -371,7 +365,13 @@ mod tests { #[test] fn test_nullable_bytebool_serde_roundtrip() { - let array = ByteBool::from_option_vec(vec![Some(true), None, Some(false), None]); + let array = ByteBoolArray::try_from_data(ByteBoolData::from(vec![ + Some(true), + None, + Some(false), + None, + ])) + .unwrap(); let dtype = array.dtype().clone(); let len = array.len(); let session = VortexSession::empty().with::(); @@ -389,7 +389,7 @@ mod tests { concat.extend_from_slice(buf.as_ref()); } - let parts = SerializedArray::try_from(concat.freeze()).unwrap(); + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); let decoded = parts .decode(&dtype, len, &ReadContext::new(ctx.to_ids()), &session) .unwrap(); diff --git a/encodings/bytebool/src/compute.rs b/encodings/bytebool/src/compute.rs index 904c8e5cf93..a44f30cce0c 100644 --- a/encodings/bytebool/src/compute.rs +++ b/encodings/bytebool/src/compute.rs @@ -16,6 +16,7 @@ use vortex_array::validity::Validity; use vortex_error::VortexResult; use super::ByteBool; +use super::ByteBoolData; impl CastReduce for ByteBool { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -31,7 +32,7 @@ impl CastReduce for ByteBool { .cast_nullability(dtype.nullability(), array.len())?; return Ok(Some( - ByteBool::new(array.buffer().clone(), new_validity).into_array(), + ByteBoolData::new(array.buffer().clone(), new_validity).into_array(), )); } @@ -43,7 +44,7 @@ impl CastReduce for ByteBool { impl MaskReduce for ByteBool { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { Ok(Some( - ByteBool::new( + ByteBoolData::new( array.buffer().clone(), array .validity() @@ -78,7 +79,9 @@ impl TakeExecute for ByteBool { .collect::>() }); - Ok(Some(ByteBool::from_vec(taken_bools, validity).into_array())) + Ok(Some( + ByteBoolData::from_vec(taken_bools, validity).into_array(), + )) } } @@ -95,16 +98,19 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_error::VortexExpect; use super::*; use crate::ByteBoolArray; fn bb(v: Vec) -> ByteBoolArray { - ByteBool::from_vec(v, Validity::AllValid) + ByteBoolArray::try_from_data(ByteBoolData::from(v)) + .vortex_expect("ByteBoolData is always valid") } fn bb_opt(v: Vec>) -> ByteBoolArray { - ByteBool::from_option_vec(v) + ByteBoolArray::try_from_data(ByteBoolData::from(v)) + .vortex_expect("ByteBoolData is always valid") } #[test] diff --git a/encodings/bytebool/src/slice.rs b/encodings/bytebool/src/slice.rs index 4c55f5bbd40..b5926602808 100644 --- a/encodings/bytebool/src/slice.rs +++ b/encodings/bytebool/src/slice.rs @@ -10,11 +10,12 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::ByteBool; +use crate::ByteBoolData; impl SliceReduce for ByteBool { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { Ok(Some( - ByteBool::new( + ByteBoolData::new( array.buffer().slice(range.clone()), array.validity().slice(range)?, ) diff --git a/encodings/datetime-parts/public-api.lock b/encodings/datetime-parts/public-api.lock index a06b945b65d..3073939b292 100644 --- a/encodings/datetime-parts/public-api.lock +++ b/encodings/datetime-parts/public-api.lock @@ -22,6 +22,8 @@ impl vortex_array::array::vtable::VTable for vortex_datetime_parts::DateTimePart pub type vortex_datetime_parts::DateTimeParts::ArrayData = vortex_datetime_parts::DateTimePartsData +pub type vortex_datetime_parts::DateTimeParts::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_datetime_parts::DateTimeParts::OperationsVTable = vortex_datetime_parts::DateTimeParts pub type vortex_datetime_parts::DateTimeParts::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -34,7 +36,11 @@ pub fn vortex_datetime_parts::DateTimeParts::buffer(_array: vortex_array::array: pub fn vortex_datetime_parts::DateTimeParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_datetime_parts::DateTimeParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_datetime_parts::DateTimeParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_datetime_parts::DateTimeParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_datetime_parts::DateTimeParts::dtype(array: &vortex_datetime_parts::DateTimePartsData) -> &vortex_array::dtype::DType pub fn vortex_datetime_parts::DateTimeParts::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -42,17 +48,23 @@ pub fn vortex_datetime_parts::DateTimeParts::execute_parent(array: vortex_array: pub fn vortex_datetime_parts::DateTimeParts::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_datetime_parts::DateTimeParts::len(array: &vortex_datetime_parts::DateTimePartsData) -> usize + +pub fn vortex_datetime_parts::DateTimeParts::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_datetime_parts::DateTimeParts::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_datetime_parts::DateTimeParts::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_datetime_parts::DateTimeParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_datetime_parts::DateTimeParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_datetime_parts::DateTimeParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_datetime_parts::DateTimeParts::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_datetime_parts::DateTimeParts::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_datetime_parts::DateTimeParts::stats(array: &vortex_datetime_parts::DateTimePartsData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_datetime_parts::DateTimeParts::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_datetime_parts::DateTimeParts::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -88,12 +100,34 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_datetime_ pub fn vortex_datetime_parts::DateTimeParts::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult> +pub struct vortex_datetime_parts::DateTimePartsArrayParts + +pub vortex_datetime_parts::DateTimePartsArrayParts::days: vortex_array::array::erased::ArrayRef + +pub vortex_datetime_parts::DateTimePartsArrayParts::dtype: vortex_array::dtype::DType + +pub vortex_datetime_parts::DateTimePartsArrayParts::seconds: vortex_array::array::erased::ArrayRef + +pub vortex_datetime_parts::DateTimePartsArrayParts::subseconds: vortex_array::array::erased::ArrayRef + +impl core::clone::Clone for vortex_datetime_parts::DateTimePartsArrayParts + +pub fn vortex_datetime_parts::DateTimePartsArrayParts::clone(&self) -> vortex_datetime_parts::DateTimePartsArrayParts + +impl core::fmt::Debug for vortex_datetime_parts::DateTimePartsArrayParts + +pub fn vortex_datetime_parts::DateTimePartsArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub struct vortex_datetime_parts::DateTimePartsData impl vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::days(&self) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_datetime_parts::DateTimePartsData::dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_datetime_parts::DateTimePartsData::into_parts(self) -> vortex_datetime_parts::DateTimePartsArrayParts + pub fn vortex_datetime_parts::DateTimePartsData::is_empty(&self) -> bool pub fn vortex_datetime_parts::DateTimePartsData::len(&self) -> usize @@ -102,12 +136,16 @@ pub fn vortex_datetime_parts::DateTimePartsData::seconds(&self) -> &vortex_array pub fn vortex_datetime_parts::DateTimePartsData::subseconds(&self) -> &vortex_array::array::erased::ArrayRef -pub fn vortex_datetime_parts::DateTimePartsData::validate(dtype: &vortex_array::dtype::DType, days: &vortex_array::array::erased::ArrayRef, seconds: &vortex_array::array::erased::ArrayRef, subseconds: &vortex_array::array::erased::ArrayRef, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_datetime_parts::DateTimePartsData::try_new(dtype: vortex_array::dtype::DType, days: vortex_array::array::erased::ArrayRef, seconds: vortex_array::array::erased::ArrayRef, subseconds: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult impl core::clone::Clone for vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::clone(&self) -> vortex_datetime_parts::DateTimePartsData +impl core::convert::From for vortex_array::array::erased::ArrayRef + +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_datetime_parts::DateTimePartsData) -> vortex_array::array::erased::ArrayRef + impl core::convert::TryFrom for vortex_datetime_parts::DateTimePartsData pub type vortex_datetime_parts::DateTimePartsData::Error = vortex_error::VortexError @@ -118,6 +156,10 @@ impl core::fmt::Debug for vortex_datetime_parts::DateTimePartsData pub fn vortex_datetime_parts::DateTimePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::array::IntoArray for vortex_datetime_parts::DateTimePartsData + +pub fn vortex_datetime_parts::DateTimePartsData::into_array(self) -> vortex_array::array::erased::ArrayRef + #[repr(C)] pub struct vortex_datetime_parts::DateTimePartsMetadata pub vortex_datetime_parts::DateTimePartsMetadata::days_ptype: i32 diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index f59cec3dd11..54cbb6528e3 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -3,24 +3,26 @@ use std::fmt::Debug; -use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; +use vortex_array::SerializeMetadata; use vortex_array::arrays::TemporalArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; @@ -72,15 +74,28 @@ impl DateTimePartsMetadata { impl VTable for DateTimeParts { type ArrayData = DateTimePartsData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; + fn vtable(_array: &Self::ArrayData) -> &Self { + &DateTimeParts + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - DateTimePartsData::validate(dtype, data.days(), data.seconds(), data.subseconds(), len) + fn len(array: &DateTimePartsData) -> usize { + array.days().len() + } + + fn dtype(array: &DateTimePartsData) -> &DType { + &array.dtype + } + + fn stats(array: &DateTimePartsData) -> &ArrayStats { + &array.stats_set } fn array_hash( @@ -115,27 +130,37 @@ impl VTable for DateTimeParts { vortex_panic!("DateTimePartsArray buffer_name index {idx} out of bounds") } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - DateTimePartsMetadata { - days_ptype: PType::try_from(array.days().dtype())? as i32, - seconds_ptype: PType::try_from(array.seconds().dtype())? as i32, - subseconds_ptype: PType::try_from(array.subseconds().dtype())? as i32, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(DateTimePartsMetadata { + days_ptype: PType::try_from(array.days().dtype())? as i32, + seconds_ptype: PType::try_from(array.seconds().dtype())? as i32, + subseconds_ptype: PType::try_from(array.subseconds().dtype())? as i32, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata( + as DeserializeMetadata>::deserialize(bytes)?, + )) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = DateTimePartsMetadata::decode(metadata)?; if children.len() != 3 { vortex_bail!( "Expected 3 children for datetime-parts encoding, found {}", @@ -145,17 +170,17 @@ impl VTable for DateTimeParts { let days = children.get( 0, - &DType::Primitive(metadata.get_days_ptype()?, dtype.nullability()), + &DType::Primitive(metadata.0.get_days_ptype()?, dtype.nullability()), len, )?; let seconds = children.get( 1, - &DType::Primitive(metadata.get_seconds_ptype()?, Nullability::NonNullable), + &DType::Primitive(metadata.0.get_seconds_ptype()?, Nullability::NonNullable), len, )?; let subseconds = children.get( 2, - &DType::Primitive(metadata.get_subseconds_ptype()?, Nullability::NonNullable), + &DType::Primitive(metadata.0.get_subseconds_ptype()?, Nullability::NonNullable), len, )?; @@ -216,7 +241,17 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["days", "seconds", "subseconds #[derive(Clone, Debug)] pub struct DateTimePartsData { + dtype: DType, pub(super) slots: Vec>, + stats_set: ArrayStats, +} + +#[derive(Clone, Debug)] +pub struct DateTimePartsArrayParts { + pub dtype: DType, + pub days: ArrayRef, + pub seconds: ArrayRef, + pub subseconds: ArrayRef, } #[derive(Clone, Debug)] @@ -232,38 +267,24 @@ impl DateTimeParts { seconds: ArrayRef, subseconds: ArrayRef, ) -> VortexResult { - let data = DateTimePartsData::try_new(dtype.clone(), days, seconds, subseconds)?; - let len = data.len(); - Ok( - unsafe { - Array::from_parts_unchecked(ArrayParts::new(DateTimeParts, dtype, len, data)) - }, - ) + Array::try_from_data(DateTimePartsData::try_new( + dtype, days, seconds, subseconds, + )?) } /// Construct a [`DateTimePartsArray`] from a [`TemporalArray`]. pub fn try_from_temporal(temporal: TemporalArray) -> VortexResult { - let dtype = temporal.dtype().clone(); - let data = DateTimePartsData::try_from(temporal)?; - let len = data.len(); - Ok( - unsafe { - Array::from_parts_unchecked(ArrayParts::new(DateTimeParts, dtype, len, data)) - }, - ) + Array::try_from_data(DateTimePartsData::try_from(temporal)?) } } impl DateTimePartsData { - pub fn validate( - dtype: &DType, - days: &ArrayRef, - seconds: &ArrayRef, - subseconds: &ArrayRef, - len: usize, - ) -> VortexResult<()> { - vortex_ensure!(days.len() == len, "expected len {len}, got {}", days.len()); - + pub fn try_new( + dtype: DType, + days: ArrayRef, + seconds: ArrayRef, + subseconds: ArrayRef, + ) -> VortexResult { if !days.dtype().is_int() || (dtype.is_nullable() != days.dtype().is_nullable()) { vortex_bail!( "Expected integer with nullability {}, got {}", @@ -278,7 +299,8 @@ impl DateTimePartsData { vortex_bail!(MismatchedTypes: "non-nullable integer", subseconds.dtype()); } - if len != seconds.len() || len != subseconds.len() { + let length = days.len(); + if length != seconds.len() || length != subseconds.len() { vortex_bail!( "Mismatched lengths {} {} {}", days.len(), @@ -287,19 +309,39 @@ impl DateTimePartsData { ); } - Ok(()) + Ok(Self { + dtype, + slots: vec![Some(days), Some(seconds), Some(subseconds)], + stats_set: Default::default(), + }) } - pub(crate) fn try_new( + pub(crate) unsafe fn new_unchecked( dtype: DType, days: ArrayRef, seconds: ArrayRef, subseconds: ArrayRef, - ) -> VortexResult { - Self::validate(&dtype, &days, &seconds, &subseconds, days.len())?; - Ok(Self { + ) -> Self { + Self { + dtype, slots: vec![Some(days), Some(seconds), Some(subseconds)], - }) + stats_set: Default::default(), + } + } + + pub fn into_parts(mut self) -> DateTimePartsArrayParts { + DateTimePartsArrayParts { + dtype: self.dtype, + days: self.slots[DAYS_SLOT] + .take() + .vortex_expect("DateTimePartsArray days slot"), + seconds: self.slots[SECONDS_SLOT] + .take() + .vortex_expect("DateTimePartsArray seconds slot"), + subseconds: self.slots[SUBSECONDS_SLOT] + .take() + .vortex_expect("DateTimePartsArray subseconds slot"), + } } /// Returns the number of elements in the array. @@ -312,6 +354,11 @@ impl DateTimePartsData { self.days().len() == 0 } + /// Returns the logical data type of the array. + pub fn dtype(&self) -> &DType { + &self.dtype + } + pub fn days(&self) -> &ArrayRef { self.slots[DAYS_SLOT] .as_ref() diff --git a/encodings/datetime-parts/src/canonical.rs b/encodings/datetime-parts/src/canonical.rs index daecaecfd3f..99337f898a5 100644 --- a/encodings/datetime-parts/src/canonical.rs +++ b/encodings/datetime-parts/src/canonical.rs @@ -119,7 +119,8 @@ mod test { use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::DateTimeParts; + use crate::DateTimePartsArray; + use crate::DateTimePartsData; use crate::canonical::decode_to_temporal; #[rstest] @@ -139,11 +140,14 @@ mod test { ], validity.clone(), ); - let date_times = DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( - milliseconds.clone().into_array(), - TimeUnit::Milliseconds, - Some("UTC".into()), - ))?; + let date_times = DateTimePartsArray::try_from_data( + DateTimePartsData::try_from(TemporalArray::new_timestamp( + milliseconds.clone().into_array(), + TimeUnit::Milliseconds, + Some("UTC".into()), + )) + .unwrap(), + )?; let mut ctx = ExecutionCtx::new(VortexSession::empty()); diff --git a/encodings/datetime-parts/src/compute/cast.rs b/encodings/datetime-parts/src/compute/cast.rs index ca846bbd424..9c0c405827d 100644 --- a/encodings/datetime-parts/src/compute/cast.rs +++ b/encodings/datetime-parts/src/compute/cast.rs @@ -10,6 +10,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::DateTimeParts; +use crate::DateTimePartsData; impl CastReduce for DateTimeParts { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if !array.dtype().eq_ignore_nullability(dtype) { @@ -17,7 +18,7 @@ impl CastReduce for DateTimeParts { }; Ok(Some( - DateTimeParts::try_new( + DateTimePartsData::try_new( dtype.clone(), array .days() @@ -44,11 +45,11 @@ mod tests { use vortex_array::validity::Validity; use vortex_buffer::buffer; - use crate::DateTimeParts; use crate::DateTimePartsArray; + use crate::DateTimePartsData; fn date_time_array(validity: Validity) -> ArrayRef { - DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + DateTimePartsData::try_from(TemporalArray::new_timestamp( PrimitiveArray::new( buffer![ 86_400i64, // element with only day component @@ -103,7 +104,7 @@ mod tests { } #[rstest] - #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![ 0i64, 86_400_000, // 1 day in ms @@ -113,8 +114,8 @@ mod tests { ].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] - #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + )).unwrap()).unwrap())] + #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([ Some(0i64), None, @@ -124,12 +125,12 @@ mod tests { ]).into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] - #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + )).unwrap()).unwrap())] + #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![86_400_000_000_000i64].into_array(), // 1 day in ns TimeUnit::Nanoseconds, Some("UTC".into()) - )).unwrap())] + )).unwrap()).unwrap())] fn test_cast_datetime_parts_conformance(#[case] array: DateTimePartsArray) { use vortex_array::compute::conformance::cast::test_cast_conformance; test_cast_conformance(&array.into_array()); diff --git a/encodings/datetime-parts/src/compute/compare.rs b/encodings/datetime-parts/src/compute/compare.rs index 76250bd320e..0c92e6a5168 100644 --- a/encodings/datetime-parts/src/compute/compare.rs +++ b/encodings/datetime-parts/src/compute/compare.rs @@ -205,14 +205,13 @@ mod test { use vortex_buffer::buffer; use super::*; - use crate::DateTimeParts; - use crate::DateTimePartsArray; + use crate::DateTimePartsData; fn dtp_array_from_timestamp( value: T, validity: Validity, - ) -> DateTimePartsArray { - DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + ) -> DateTimePartsData { + DateTimePartsData::try_from(TemporalArray::new_timestamp( PrimitiveArray::new(buffer![value], validity).into_array(), TimeUnit::Seconds, Some("UTC".into()), @@ -313,7 +312,7 @@ mod test { Some("UTC".into()), ); - let lhs = DateTimeParts::try_new( + let lhs = DateTimePartsData::try_new( DType::Extension(temporal_array.ext_dtype()), PrimitiveArray::new(buffer![0i32], lhs_validity).into_array(), PrimitiveArray::new(buffer![0u32], Validity::NonNullable).into_array(), diff --git a/encodings/datetime-parts/src/compute/filter.rs b/encodings/datetime-parts/src/compute/filter.rs index da825d8a95e..ba339063540 100644 --- a/encodings/datetime-parts/src/compute/filter.rs +++ b/encodings/datetime-parts/src/compute/filter.rs @@ -9,10 +9,11 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::DateTimeParts; +use crate::DateTimePartsData; impl FilterReduce for DateTimeParts { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { Ok(Some( - DateTimeParts::try_new( + DateTimePartsData::try_new( array.dtype().clone(), array.days().filter(mask.clone())?, array.seconds().filter(mask.clone())?, @@ -32,7 +33,7 @@ mod test { use vortex_array::extension::datetime::TimeUnit; use vortex_buffer::buffer; - use crate::DateTimeParts; + use crate::DateTimePartsData; #[test] fn test_filter_datetime_parts() { @@ -49,7 +50,7 @@ mod test { let temporal = TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); - let array = DateTimeParts::try_from_temporal(temporal).unwrap(); + let array = DateTimePartsData::try_from(temporal).unwrap(); test_filter_conformance(&array.into_array()); // Test with nullable values @@ -65,7 +66,7 @@ mod test { let temporal = TemporalArray::new_timestamp(timestamps, TimeUnit::Milliseconds, Some("UTC".into())); - let array = DateTimeParts::try_from_temporal(temporal).unwrap(); + let array = DateTimePartsData::try_from(temporal).unwrap(); test_filter_conformance(&array.into_array()); } } diff --git a/encodings/datetime-parts/src/compute/mask.rs b/encodings/datetime-parts/src/compute/mask.rs index 290707ef8ef..023831d251a 100644 --- a/encodings/datetime-parts/src/compute/mask.rs +++ b/encodings/datetime-parts/src/compute/mask.rs @@ -9,12 +9,13 @@ use vortex_array::scalar_fn::fns::mask::MaskReduce; use vortex_error::VortexResult; use crate::DateTimeParts; +use crate::DateTimePartsData; impl MaskReduce for DateTimeParts { fn mask(array: ArrayView<'_, Self>, mask: &ArrayRef) -> VortexResult> { let masked_days = array.days().clone().mask(mask.clone())?; Ok(Some( - DateTimeParts::try_new( + DateTimePartsData::try_new( array.dtype().as_nullable(), masked_days, array.seconds().clone(), diff --git a/encodings/datetime-parts/src/compute/mod.rs b/encodings/datetime-parts/src/compute/mod.rs index 53717a52b0c..4ca8398fd6b 100644 --- a/encodings/datetime-parts/src/compute/mod.rs +++ b/encodings/datetime-parts/src/compute/mod.rs @@ -21,59 +21,55 @@ mod tests { use vortex_array::extension::datetime::TimeUnit; use vortex_buffer::buffer; - use crate::DateTimeParts; use crate::DateTimePartsArray; - - fn dtp_from_temporal(temporal: TemporalArray) -> DateTimePartsArray { - DateTimeParts::try_from_temporal(temporal).unwrap() - } + use crate::DateTimePartsData; #[rstest] // Basic datetime arrays - #[case::datetime_seconds(dtp_from_temporal(TemporalArray::new_timestamp( + #[case::datetime_seconds(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![0i64, 86400, 172800, 259200, 345600].into_array(), TimeUnit::Seconds, Some("UTC".into()), - )))] - #[case::datetime_millis(dtp_from_temporal(TemporalArray::new_timestamp( + )).unwrap()).unwrap())] + #[case::datetime_millis(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![0i64, 86400000, 172800000].into_array(), TimeUnit::Milliseconds, Some("UTC".into()), - )))] - #[case::datetime_micros(dtp_from_temporal(TemporalArray::new_timestamp( + )).unwrap()).unwrap())] + #[case::datetime_micros(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![0i64, 86400000000, 172800000000].into_array(), TimeUnit::Microseconds, Some("UTC".into()), - )))] - #[case::datetime_nanos(dtp_from_temporal(TemporalArray::new_timestamp( + )).unwrap()).unwrap())] + #[case::datetime_nanos(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![0i64, 86400000000000].into_array(), TimeUnit::Nanoseconds, Some("UTC".into()), - )))] + )).unwrap()).unwrap())] // Nullable arrays - #[case::datetime_nullable_seconds(dtp_from_temporal(TemporalArray::new_timestamp( + #[case::datetime_nullable_seconds(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([Some(0i64), None, Some(86400), Some(172800), None]).into_array(), TimeUnit::Seconds, Some("UTC".into()), - )))] + )).unwrap()).unwrap())] // Edge cases - #[case::datetime_single(dtp_from_temporal(TemporalArray::new_timestamp( + #[case::datetime_single(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![1234567890i64].into_array(), TimeUnit::Seconds, Some("UTC".into()), - )))] + )).unwrap()).unwrap())] // Large arrays (> 1024 elements) - #[case::datetime_large(dtp_from_temporal(TemporalArray::new_timestamp( + #[case::datetime_large(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( PrimitiveArray::from_iter((0..1500).map(|i| i as i64 * 86400)).into_array(), TimeUnit::Seconds, Some("UTC".into()), - )))] + )).unwrap()).unwrap())] // Different time patterns - #[case::datetime_with_subseconds(dtp_from_temporal(TemporalArray::new_timestamp( + #[case::datetime_with_subseconds(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![123456789i64, 234567890, 345678901, 456789012, 567890123].into_array(), TimeUnit::Milliseconds, Some("UTC".into()), - )))] + )).unwrap()).unwrap())] fn test_datetime_parts_consistency(#[case] array: DateTimePartsArray) { test_array_consistency(&array.into_array()); diff --git a/encodings/datetime-parts/src/compute/rules.rs b/encodings/datetime-parts/src/compute/rules.rs index 92c1f877c08..dedc87caa13 100644 --- a/encodings/datetime-parts/src/compute/rules.rs +++ b/encodings/datetime-parts/src/compute/rules.rs @@ -26,6 +26,7 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DateTimeParts; +use crate::DateTimePartsData; use crate::timestamp; pub(crate) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&DTPFilterPushDownRule), @@ -56,7 +57,7 @@ impl ArrayParentReduceRule for DTPFilterPushDownRule { return Ok(None); } - DateTimeParts::try_new( + DateTimePartsData::try_new( child.dtype().clone(), child.days().clone().filter(parent.filter_mask().clone())?, ConstantArray::new( @@ -191,7 +192,6 @@ mod tests { use vortex_buffer::Buffer; use super::*; - use crate::DateTimeParts; use crate::DateTimePartsArray; const SECONDS_PER_DAY: i64 = 86400; @@ -215,8 +215,8 @@ mod tests { time_unit, None, ); - DateTimeParts::try_from_temporal(temporal) - .vortex_expect("TemporalArray must produce valid DateTimeParts") + DateTimePartsArray::try_from_data(DateTimePartsData::try_from(temporal).unwrap()) + .vortex_expect("DateTimePartsData is always valid") } /// Create a constant timestamp scalar at midnight for the given day. @@ -348,7 +348,7 @@ mod tests { TimeUnit::Seconds, None, ); - let dtp = DateTimeParts::try_from_temporal(temporal).unwrap(); + let dtp = DateTimePartsData::try_from(temporal).unwrap(); let len = dtp.len(); // Compare against midnight constant diff --git a/encodings/datetime-parts/src/compute/slice.rs b/encodings/datetime-parts/src/compute/slice.rs index 5bad980d8dc..b63693980f7 100644 --- a/encodings/datetime-parts/src/compute/slice.rs +++ b/encodings/datetime-parts/src/compute/slice.rs @@ -10,18 +10,19 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::DateTimeParts; +use crate::DateTimePartsData; impl SliceReduce for DateTimeParts { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // SAFETY: slicing all components preserves values - Ok(Some( - DateTimeParts::try_new( + Ok(Some(unsafe { + DateTimePartsData::new_unchecked( array.dtype().clone(), array.days().slice(range.clone())?, array.seconds().slice(range.clone())?, array.subseconds().slice(range)?, - )? - .into_array(), - )) + ) + .into_array() + })) } } diff --git a/encodings/datetime-parts/src/compute/take.rs b/encodings/datetime-parts/src/compute/take.rs index 4cd214663c1..b804752657b 100644 --- a/encodings/datetime-parts/src/compute/take.rs +++ b/encodings/datetime-parts/src/compute/take.rs @@ -16,6 +16,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_panic; use crate::DateTimeParts; +use crate::DateTimePartsData; fn take_datetime_parts( array: ArrayView, indices: &ArrayRef, @@ -38,7 +39,7 @@ fn take_datetime_parts( if !taken_seconds.dtype().is_nullable() && !taken_subseconds.dtype().is_nullable() { return Ok( - DateTimeParts::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? + DateTimePartsData::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? .into_array(), ); } @@ -77,7 +78,10 @@ fn take_datetime_parts( .cast(array.subseconds().dtype())?; let taken_subseconds = taken_subseconds.fill_null(subseconds_fill)?; - Ok(DateTimeParts::try_new(dtype, taken_days, taken_seconds, taken_subseconds)?.into_array()) + Ok( + DateTimePartsData::try_new(dtype, taken_days, taken_seconds, taken_subseconds)? + .into_array(), + ) } impl TakeExecute for DateTimeParts { @@ -100,11 +104,11 @@ mod tests { use vortex_array::extension::datetime::TimeUnit; use vortex_buffer::buffer; - use crate::DateTimeParts; use crate::DateTimePartsArray; + use crate::DateTimePartsData; #[rstest] - #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![ 0i64, 86_400_000, // 1 day in ms @@ -114,8 +118,8 @@ mod tests { ].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] - #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + )).unwrap()).unwrap())] + #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( PrimitiveArray::from_option_iter([ Some(0i64), None, @@ -125,12 +129,12 @@ mod tests { ]).into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] - #[case(DateTimeParts::try_from_temporal(TemporalArray::new_timestamp( + )).unwrap()).unwrap())] + #[case(DateTimePartsArray::try_from_data(DateTimePartsData::try_from(TemporalArray::new_timestamp( buffer![86_400_000i64].into_array(), TimeUnit::Milliseconds, Some("UTC".into()) - )).unwrap())] + )).unwrap()).unwrap())] fn test_take_datetime_parts_conformance(#[case] array: DateTimePartsArray) { test_take_conformance(&array.into_array()); } diff --git a/encodings/datetime-parts/src/lib.rs b/encodings/datetime-parts/src/lib.rs index 1babf4abaab..7a7c2fc0ef5 100644 --- a/encodings/datetime-parts/src/lib.rs +++ b/encodings/datetime-parts/src/lib.rs @@ -30,7 +30,7 @@ pub fn initialize(session: &VortexSession) { #[cfg(test)] mod test { - use prost::Message; + use vortex_array::ProstMetadata; use vortex_array::dtype::PType; use vortex_array::test_harness::check_metadata; @@ -41,12 +41,11 @@ mod test { fn test_datetimeparts_metadata() { check_metadata( "datetimeparts.metadata", - &DateTimePartsMetadata { + ProstMetadata(DateTimePartsMetadata { days_ptype: PType::I64 as i32, seconds_ptype: PType::I64 as i32, subseconds_ptype: PType::I64 as i32, - } - .encode_to_vec(), + }), ); } } diff --git a/encodings/decimal-byte-parts/public-api.lock b/encodings/decimal-byte-parts/public-api.lock index 268967616c6..9dbb6a4ba82 100644 --- a/encodings/decimal-byte-parts/public-api.lock +++ b/encodings/decimal-byte-parts/public-api.lock @@ -20,6 +20,8 @@ impl vortex_array::array::vtable::VTable for vortex_decimal_byte_parts::DecimalB pub type vortex_decimal_byte_parts::DecimalByteParts::ArrayData = vortex_decimal_byte_parts::DecimalBytePartsData +pub type vortex_decimal_byte_parts::DecimalByteParts::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_decimal_byte_parts::DecimalByteParts::OperationsVTable = vortex_decimal_byte_parts::DecimalByteParts pub type vortex_decimal_byte_parts::DecimalByteParts::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -32,7 +34,11 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer(_array: vortex_array: pub fn vortex_decimal_byte_parts::DecimalByteParts::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_decimal_byte_parts::DecimalByteParts::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_decimal_byte_parts::DecimalByteParts::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_decimal_byte_parts::DecimalByteParts::dtype(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> &vortex_array::dtype::DType pub fn vortex_decimal_byte_parts::DecimalByteParts::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -40,17 +46,23 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::execute_parent(array: vortex pub fn vortex_decimal_byte_parts::DecimalByteParts::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_decimal_byte_parts::DecimalByteParts::len(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> usize + +pub fn vortex_decimal_byte_parts::DecimalByteParts::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_decimal_byte_parts::DecimalByteParts::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_decimal_byte_parts::DecimalByteParts::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_decimal_byte_parts::DecimalByteParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_decimal_byte_parts::DecimalByteParts::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_decimal_byte_parts::DecimalByteParts::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_decimal_byte_parts::DecimalByteParts::stats(array: &vortex_decimal_byte_parts::DecimalBytePartsData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_decimal_byte_parts::DecimalByteParts::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_decimal_byte_parts::DecimalByteParts::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -86,29 +98,43 @@ impl vortex_array::scalar_fn::fns::mask::kernel::MaskReduce for vortex_decimal_b pub fn vortex_decimal_byte_parts::DecimalByteParts::mask(array: vortex_array::array::view::ArrayView<'_, Self>, mask: &vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult> +pub struct vortex_decimal_byte_parts::DecimalBytePartsArrayParts + +pub vortex_decimal_byte_parts::DecimalBytePartsArrayParts::dtype: vortex_array::dtype::DType + +pub vortex_decimal_byte_parts::DecimalBytePartsArrayParts::msp: vortex_array::array::erased::ArrayRef + pub struct vortex_decimal_byte_parts::DecimalBytePartsData impl vortex_decimal_byte_parts::DecimalBytePartsData -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_parts(self) -> vortex_decimal_byte_parts::DecimalBytePartsDataParts +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::decimal_dtype(&self) -> &vortex_array::dtype::decimal::DecimalDType + +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_parts(self) -> vortex_decimal_byte_parts::DecimalBytePartsArrayParts pub fn vortex_decimal_byte_parts::DecimalBytePartsData::is_empty(&self) -> bool pub fn vortex_decimal_byte_parts::DecimalBytePartsData::len(&self) -> usize -pub fn vortex_decimal_byte_parts::DecimalBytePartsData::validate(msp: &vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::try_new(msp: vortex_array::array::erased::ArrayRef, decimal_dtype: vortex_array::dtype::decimal::DecimalDType) -> vortex_error::VortexResult impl core::clone::Clone for vortex_decimal_byte_parts::DecimalBytePartsData pub fn vortex_decimal_byte_parts::DecimalBytePartsData::clone(&self) -> vortex_decimal_byte_parts::DecimalBytePartsData +impl core::convert::From for vortex_array::array::erased::ArrayRef + +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_decimal_byte_parts::DecimalBytePartsData) -> vortex_array::array::erased::ArrayRef + impl core::fmt::Debug for vortex_decimal_byte_parts::DecimalBytePartsData pub fn vortex_decimal_byte_parts::DecimalBytePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct vortex_decimal_byte_parts::DecimalBytePartsDataParts +impl vortex_array::array::IntoArray for vortex_decimal_byte_parts::DecimalBytePartsData -pub vortex_decimal_byte_parts::DecimalBytePartsDataParts::msp: vortex_array::array::erased::ArrayRef +pub fn vortex_decimal_byte_parts::DecimalBytePartsData::into_array(self) -> vortex_array::array::erased::ArrayRef pub struct vortex_decimal_byte_parts::DecimalBytesPartsMetadata diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs index d78c6e2a96d..5314dd8845d 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/cast.rs @@ -7,7 +7,6 @@ use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::DecimalByteParts; @@ -20,11 +19,7 @@ impl CastReduce for DecimalByteParts { }; // Check if this is just a nullability change - if array - .dtype() - .as_decimal_opt() - .vortex_expect("must be a decimal dtype") - == target_decimal + if array.decimal_dtype() == target_decimal && array.dtype().nullability() != *target_nullability { // Cast the msp array to handle nullability change diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs index b0afad10cfa..b1be5b1b54d 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/compare.rs @@ -38,7 +38,7 @@ impl CompareKernel for DecimalByteParts { return Ok(None); }; - let nullability = lhs.dtype().nullability() | rhs.dtype().nullability(); + let nullability = lhs.dtype.nullability() | rhs.dtype().nullability(); let scalar_type = lhs.msp().dtype().with_nullability(nullability); let rhs_decimal = rhs_const diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs index 49435845476..638c167e9ed 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/filter.rs @@ -5,21 +5,14 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::arrays::filter::FilterReduce; -use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; use crate::DecimalByteParts; impl FilterReduce for DecimalByteParts { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - DecimalByteParts::try_new( - array.msp().filter(mask.clone())?, - *array - .dtype() - .as_decimal_opt() - .vortex_expect("must be a decimal dtype"), - ) - .map(|d| Some(d.into_array())) + DecimalByteParts::try_new(array.msp().filter(mask.clone())?, *array.decimal_dtype()) + .map(|d| Some(d.into_array())) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs index 4c1b1a36c64..f3d5068faa0 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mask.rs @@ -8,9 +8,9 @@ use vortex_array::arrays::scalar_fn::ScalarFnArrayExt; use vortex_array::scalar_fn::EmptyOptions; use vortex_array::scalar_fn::fns::mask::Mask as MaskExpr; use vortex_array::scalar_fn::fns::mask::MaskReduce; -use vortex_error::VortexExpect; use vortex_error::VortexResult; +use super::DecimalBytePartsData; use crate::DecimalByteParts; impl MaskReduce for DecimalByteParts { @@ -21,14 +21,7 @@ impl MaskReduce for DecimalByteParts { [array.msp().clone(), mask.clone()], )?; Ok(Some( - DecimalByteParts::try_new( - masked_msp, - *array - .dtype() - .as_decimal_opt() - .vortex_expect("must be a decimal dtype"), - )? - .into_array(), + DecimalBytePartsData::try_new(masked_msp, *array.decimal_dtype())?.into_array(), )) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs index 35da5be40b6..ef271694aea 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use super::DecimalBytePartsData; mod cast; mod compare; mod filter; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs index 3c7f7ecfabf..9aa64aa8d37 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/compute/take.rs @@ -6,9 +6,9 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::dict::TakeExecute; -use vortex_error::VortexExpect; use vortex_error::VortexResult; +use super::DecimalBytePartsData; use crate::DecimalByteParts; impl TakeExecute for DecimalByteParts { @@ -17,13 +17,7 @@ impl TakeExecute for DecimalByteParts { indices: &ArrayRef, _ctx: &mut ExecutionCtx, ) -> VortexResult> { - DecimalByteParts::try_new( - array.msp().take(indices.clone())?, - *array - .dtype() - .as_decimal_opt() - .vortex_expect("must be a decimal dtype"), - ) - .map(|a| Some(a.into_array())) + DecimalBytePartsData::try_new(array.msp().take(indices.clone())?, *array.decimal_dtype()) + .map(|a| Some(a.into_array())) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index 8773b2e8184..5fff2a47bf5 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_array::Array; -use vortex_array::ArrayParts; use vortex_array::ArrayView; pub(crate) mod compute; mod rules; @@ -17,6 +16,8 @@ use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; +use vortex_array::SerializeMetadata; use vortex_array::arrays::DecimalArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; @@ -28,6 +29,7 @@ use vortex_array::scalar::DecimalValue; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; @@ -56,18 +58,28 @@ pub struct DecimalBytesPartsMetadata { impl VTable for DecimalByteParts { type ArrayData = DecimalBytePartsData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; + fn vtable(_array: &Self::ArrayData) -> &Self { + &DecimalByteParts + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - let Some(decimal_dtype) = dtype.as_decimal_opt() else { - vortex_bail!("expected decimal dtype, got {}", dtype) - }; - DecimalBytePartsData::validate(data.msp(), *decimal_dtype, dtype, len) + fn len(array: &DecimalBytePartsData) -> usize { + array.msp().len() + } + + fn dtype(array: &DecimalBytePartsData) -> &DType { + &array.dtype + } + + fn stats(array: &DecimalBytePartsData) -> &ArrayStats { + &array.stats_set } fn array_hash( @@ -98,26 +110,34 @@ impl VTable for DecimalByteParts { vortex_panic!("DecimalBytePartsArray buffer_name index {idx} out of bounds") } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - DecimalBytesPartsMetadata { - zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32, - lower_part_count: 0, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(DecimalBytesPartsMetadata { + zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32, + lower_part_count: 0, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata(DecimalBytesPartsMetadata::decode(bytes)?)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = DecimalBytesPartsMetadata::decode(metadata)?; let Some(decimal_dtype) = dtype.as_decimal_opt() else { vortex_bail!("decoding decimal but given non decimal dtype {}", dtype) }; @@ -191,42 +211,52 @@ pub struct DecimalBytePartsData { // NOTE: the lower_parts is currently unused, we reserve this field so that it is properly // read/written during serde, but provide no constructor to initialize this to anything // other than the empty Vec. + // Must update `DecimalBytePartsArrayParts` too. _lower_parts: Vec, + dtype: DType, + stats_set: ArrayStats, } -pub struct DecimalBytePartsDataParts { +pub struct DecimalBytePartsArrayParts { pub msp: ArrayRef, + pub dtype: DType, } impl DecimalBytePartsData { - pub fn validate( - msp: &ArrayRef, - decimal_dtype: DecimalDType, - dtype: &DType, - len: usize, - ) -> VortexResult<()> { + pub fn try_new(msp: ArrayRef, decimal_dtype: DecimalDType) -> VortexResult { if !msp.dtype().is_signed_int() { vortex_bail!("decimal bytes parts, first part must be a signed array") } - let expected_dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); - vortex_ensure!( - dtype == &expected_dtype, - "expected dtype {expected_dtype}, got {dtype}" - ); - vortex_ensure!(msp.len() == len, "expected len {len}, got {}", msp.len()); - Ok(()) - } - - pub(crate) fn try_new(msp: ArrayRef, decimal_dtype: DecimalDType) -> VortexResult { - let dtype = DType::Decimal(decimal_dtype, msp.dtype().nullability()); - Self::validate(&msp, decimal_dtype, &dtype, msp.len())?; + let nullable = msp.dtype().nullability(); Ok(Self { slots: vec![Some(msp)], _lower_parts: Vec::new(), + dtype: DType::Decimal(decimal_dtype, nullable), + stats_set: Default::default(), }) } + pub(crate) unsafe fn new_unchecked(msp: ArrayRef, decimal_dtype: DecimalDType) -> Self { + let nullable = msp.dtype().nullability(); + Self { + slots: vec![Some(msp)], + _lower_parts: Vec::new(), + dtype: DType::Decimal(decimal_dtype, nullable), + stats_set: Default::default(), + } + } + + /// If `_lower_parts` is supported check all calls use this correctly. + pub fn into_parts(mut self) -> DecimalBytePartsArrayParts { + DecimalBytePartsArrayParts { + msp: self.slots[MSP_SLOT] + .take() + .vortex_expect("DecimalBytePartsArray msp slot"), + dtype: self.dtype, + } + } + /// Returns the number of elements in the array. pub fn len(&self) -> usize { self.msp().len() @@ -237,19 +267,22 @@ impl DecimalBytePartsData { self.msp().len() == 0 } + /// Returns the logical data type of the array. + pub fn dtype(&self) -> &DType { + &self.dtype + } + + pub fn decimal_dtype(&self) -> &DecimalDType { + self.dtype + .as_decimal_opt() + .vortex_expect("must be a decimal dtype") + } + pub(crate) fn msp(&self) -> &ArrayRef { self.slots[MSP_SLOT] .as_ref() .vortex_expect("DecimalBytePartsArray msp slot") } - - pub fn into_parts(mut self) -> DecimalBytePartsDataParts { - DecimalBytePartsDataParts { - msp: self.slots[MSP_SLOT] - .take() - .vortex_expect("DecimalBytePartsArray msp slot"), - } - } } #[derive(Clone, Debug)] @@ -263,12 +296,7 @@ impl DecimalByteParts { msp: ArrayRef, decimal_dtype: DecimalDType, ) -> VortexResult { - let data = DecimalBytePartsData::try_new(msp, decimal_dtype)?; - let dtype = DType::Decimal(decimal_dtype, data.msp().dtype().nullability()); - let len = data.len(); - Ok(unsafe { - Array::from_parts_unchecked(ArrayParts::new(DecimalByteParts, dtype, len, data)) - }) + Array::try_from_data(DecimalBytePartsData::try_new(msp, decimal_dtype)?) } } @@ -288,10 +316,7 @@ fn to_canonical_decimal( unsafe { DecimalArray::new_unchecked( prim.to_buffer::

(), - *array - .dtype() - .as_decimal_opt() - .vortex_expect("must be a decimal dtype"), + *array.decimal_dtype(), prim.validity(), ) } @@ -313,7 +338,7 @@ impl OperationsVTable for DecimalByteParts { // TODO(joe): extend this to support multiple parts. let value = primitive_scalar.as_::().vortex_expect("non-null"); Scalar::try_new( - array.dtype().clone(), + array.dtype.clone(), Some(ScalarValue::Decimal(DecimalValue::I64(value))), ) } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs index 0eeaeed5716..f9b4422f7a6 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/rules.rs @@ -11,9 +11,9 @@ use vortex_array::optimizer::rules::ArrayParentReduceRule; use vortex_array::optimizer::rules::ParentRuleSet; use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; use vortex_array::scalar_fn::fns::mask::MaskReduceAdaptor; -use vortex_error::VortexExpect; use vortex_error::VortexResult; +use super::DecimalBytePartsData; use crate::DecimalByteParts; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ @@ -43,14 +43,8 @@ impl ArrayParentReduceRule for DecimalBytePartsFilterPushDownR } let new_msp = child.msp().filter(parent.filter_mask().clone())?; - let new_child = DecimalByteParts::try_new( - new_msp, - *child - .dtype() - .as_decimal_opt() - .vortex_expect("must be a decimal dtype"), - )? - .into_array(); + let new_child = + DecimalBytePartsData::try_new(new_msp, *child.decimal_dtype())?.into_array(); Ok(Some(new_child)) } } diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs index 89275570a53..051d80ff016 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/slice.rs @@ -7,22 +7,17 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; use vortex_array::arrays::slice::SliceReduce; -use vortex_error::VortexExpect; use vortex_error::VortexResult; +use super::DecimalBytePartsData; use crate::DecimalByteParts; impl SliceReduce for DecimalByteParts { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some( - DecimalByteParts::try_new( - array.msp().slice(range)?, - *array - .dtype() - .as_decimal_opt() - .vortex_expect("must be a decimal dtype"), - )? - .into_array(), - )) + // SAFETY: slicing encoded MSP does not change the encoded values + Ok(Some(unsafe { + DecimalBytePartsData::new_unchecked(array.msp().slice(range)?, *array.decimal_dtype()) + .into_array() + })) } } diff --git a/encodings/fastlanes/benches/bitpacking_take.rs b/encodings/fastlanes/benches/bitpacking_take.rs index 6129f479d47..e0325b7a481 100644 --- a/encodings/fastlanes/benches/bitpacking_take.rs +++ b/encodings/fastlanes/benches/bitpacking_take.rs @@ -160,9 +160,9 @@ fn patched_take_10_stratified(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); assert_eq!( - packed.patches(packed.len()).unwrap().num_patches(), + packed.patches().unwrap().num_patches(), NUM_EXCEPTIONS as usize ); @@ -185,9 +185,9 @@ fn patched_take_10_contiguous(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); assert_eq!( - packed.patches(packed.len()).unwrap().num_patches(), + packed.patches().unwrap().num_patches(), NUM_EXCEPTIONS as usize ); @@ -249,9 +249,9 @@ fn patched_take_10k_contiguous_patches(bencher: Bencher) { let uncompressed = PrimitiveArray::new(values, Validity::NonNullable); let packed = bitpack_to_best_bit_width(&uncompressed).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); assert_eq!( - packed.patches(packed.len()).unwrap().num_patches(), + packed.patches().unwrap().num_patches(), NUM_EXCEPTIONS as usize ); diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index 98354d33f0b..08373b0092e 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -40,11 +40,11 @@ pub fn vortex_fastlanes::bitpack_decompress::apply_patches_to_uninit_range_fn usize -pub fn vortex_fastlanes::bitpack_decompress::unpack_array(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::bitpack_decompress::unpack_array(array: &vortex_fastlanes::BitPackedData, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_fastlanes::bitpack_decompress::unpack_primitive_array(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_fastlanes::bitpack_decompress::unpack_primitive_array(array: &vortex_fastlanes::BitPackedData, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_fastlanes::bitpack_decompress::unpack_single(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>, index: usize) -> vortex_array::scalar::Scalar +pub fn vortex_fastlanes::bitpack_decompress::unpack_single(array: &vortex_fastlanes::BitPackedData, index: usize) -> vortex_array::scalar::Scalar pub unsafe fn vortex_fastlanes::bitpack_decompress::unpack_single_primitive(packed: &[T], bit_width: usize, index_to_decode: usize) -> T @@ -78,15 +78,15 @@ pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::decode_into(&mut sel pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::initial(&mut self) -> core::option::Option<&mut [T]> -pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::trailer(&mut self) -> core::option::Option<&mut [T]> +pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::new_with_strategy(strategy: S, packed: vortex_buffer::ByteBuffer, bit_width: usize, offset: usize, len: usize) -> Self -pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::try_new_with_strategy(strategy: S, packed: vortex_buffer::ByteBuffer, bit_width: usize, offset: usize, len: usize) -> vortex_error::VortexResult +pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::trailer(&mut self) -> core::option::Option<&mut [T]> impl vortex_fastlanes::unpack_iter::UnpackedChunks pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::full_chunks(&mut self) -> vortex_fastlanes::unpack_iter::BitUnpackIterator<'_, T> -pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::try_new(array: &vortex_fastlanes::BitPackedData, len: usize) -> vortex_error::VortexResult +pub fn vortex_fastlanes::unpack_iter::UnpackedChunks::new(array: &vortex_fastlanes::BitPackedData) -> Self pub trait vortex_fastlanes::unpack_iter::BitPacked: vortex_array::dtype::ptype::PhysicalPType @@ -124,10 +124,6 @@ pub const vortex_fastlanes::BitPacked::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::BitPacked::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPacked::into_parts(array: vortex_fastlanes::BitPackedArray) -> vortex_fastlanes::BitPackedDataParts - -pub fn vortex_fastlanes::BitPacked::try_new(packed: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::ptype::PType, validity: vortex_array::validity::Validity, patches: core::option::Option, bit_width: u8, len: usize, offset: u16) -> vortex_error::VortexResult - impl core::clone::Clone for vortex_fastlanes::BitPacked pub fn vortex_fastlanes::BitPacked::clone(&self) -> vortex_fastlanes::BitPacked @@ -140,6 +136,8 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::BitPacked pub type vortex_fastlanes::BitPacked::ArrayData = vortex_fastlanes::BitPackedData +pub type vortex_fastlanes::BitPacked::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_fastlanes::BitPacked::OperationsVTable = vortex_fastlanes::BitPacked pub type vortex_fastlanes::BitPacked::ValidityVTable = vortex_fastlanes::BitPacked @@ -154,7 +152,11 @@ pub fn vortex_fastlanes::BitPacked::buffer(array: vortex_array::array::view::Arr pub fn vortex_fastlanes::BitPacked::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_fastlanes::BitPacked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fastlanes::BitPacked::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_fastlanes::BitPacked::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_fastlanes::BitPacked::dtype(array: &vortex_fastlanes::BitPackedData) -> &vortex_array::dtype::DType pub fn vortex_fastlanes::BitPacked::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -162,17 +164,23 @@ pub fn vortex_fastlanes::BitPacked::execute_parent(array: vortex_array::array::v pub fn vortex_fastlanes::BitPacked::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_fastlanes::BitPacked::len(array: &vortex_fastlanes::BitPackedData) -> usize + +pub fn vortex_fastlanes::BitPacked::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_fastlanes::BitPacked::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::BitPacked::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::BitPacked::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::BitPacked::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::BitPacked::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::BitPacked::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::BitPacked::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::BitPacked::stats(array: &vortex_fastlanes::BitPackedData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_fastlanes::BitPacked::vtable(_array: &vortex_fastlanes::BitPackedData) -> &Self pub fn vortex_fastlanes::BitPacked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -200,15 +208,35 @@ impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_fastlanes pub fn vortex_fastlanes::BitPacked::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> +pub struct vortex_fastlanes::BitPackedArrayParts + +pub vortex_fastlanes::BitPackedArrayParts::bit_width: u8 + +pub vortex_fastlanes::BitPackedArrayParts::len: usize + +pub vortex_fastlanes::BitPackedArrayParts::offset: u16 + +pub vortex_fastlanes::BitPackedArrayParts::packed: vortex_array::buffer::BufferHandle + +pub vortex_fastlanes::BitPackedArrayParts::patches: core::option::Option + +pub vortex_fastlanes::BitPackedArrayParts::validity: vortex_array::validity::Validity + pub struct vortex_fastlanes::BitPackedData impl vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::bit_width(&self) -> u8 +pub fn vortex_fastlanes::BitPackedData::dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_fastlanes::BitPackedData::encode(array: &vortex_array::array::erased::ArrayRef, bit_width: u8) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPackedData::into_parts(self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_fastlanes::BitPackedDataParts +pub fn vortex_fastlanes::BitPackedData::into_parts(self) -> vortex_fastlanes::BitPackedArrayParts + +pub fn vortex_fastlanes::BitPackedData::is_empty(&self) -> bool + +pub fn vortex_fastlanes::BitPackedData::len(&self) -> usize pub fn vortex_fastlanes::BitPackedData::max_packed_value(&self) -> usize @@ -218,49 +246,35 @@ pub fn vortex_fastlanes::BitPackedData::packed(&self) -> &vortex_array::buffer:: pub fn vortex_fastlanes::BitPackedData::packed_slice(&self) -> &[T] -pub fn vortex_fastlanes::BitPackedData::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedData::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedData::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedData::patches(&self, len: usize) -> core::option::Option +pub fn vortex_fastlanes::BitPackedData::patches(&self) -> core::option::Option -pub fn vortex_fastlanes::BitPackedData::ptype(&self, dtype: &vortex_array::dtype::DType) -> vortex_array::dtype::ptype::PType +pub fn vortex_fastlanes::BitPackedData::ptype(&self) -> vortex_array::dtype::ptype::PType pub fn vortex_fastlanes::BitPackedData::replace_patches(&mut self, patches: core::option::Option) pub fn vortex_fastlanes::BitPackedData::try_new(packed: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::ptype::PType, validity: vortex_array::validity::Validity, patches: core::option::Option, bit_width: u8, length: usize, offset: u16) -> vortex_error::VortexResult -pub fn vortex_fastlanes::BitPackedData::unpacked_chunks(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult> +pub fn vortex_fastlanes::BitPackedData::unpacked_chunks(&self) -> vortex_fastlanes::unpack_iter::BitUnpackedChunks -pub fn vortex_fastlanes::BitPackedData::validity(&self, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_array::validity::Validity +pub fn vortex_fastlanes::BitPackedData::validity(&self) -> vortex_array::validity::Validity -pub fn vortex_fastlanes::BitPackedData::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedData::validity_mask(&self, len: usize, nullability: vortex_array::dtype::nullability::Nullability) -> vortex_mask::Mask +pub fn vortex_fastlanes::BitPackedData::validity_mask(&self) -> vortex_mask::Mask impl core::clone::Clone for vortex_fastlanes::BitPackedData pub fn vortex_fastlanes::BitPackedData::clone(&self) -> vortex_fastlanes::BitPackedData -impl core::fmt::Debug for vortex_fastlanes::BitPackedData - -pub fn vortex_fastlanes::BitPackedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::convert::From for vortex_array::array::erased::ArrayRef -pub struct vortex_fastlanes::BitPackedDataParts +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::BitPackedData) -> vortex_array::array::erased::ArrayRef -pub vortex_fastlanes::BitPackedDataParts::bit_width: u8 - -pub vortex_fastlanes::BitPackedDataParts::len: usize - -pub vortex_fastlanes::BitPackedDataParts::offset: u16 +impl core::fmt::Debug for vortex_fastlanes::BitPackedData -pub vortex_fastlanes::BitPackedDataParts::packed: vortex_array::buffer::BufferHandle +pub fn vortex_fastlanes::BitPackedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_fastlanes::BitPackedDataParts::patches: core::option::Option +impl vortex_array::array::IntoArray for vortex_fastlanes::BitPackedData -pub vortex_fastlanes::BitPackedDataParts::validity: vortex_array::validity::Validity +pub fn vortex_fastlanes::BitPackedData::into_array(self) -> vortex_array::array::erased::ArrayRef pub struct vortex_fastlanes::Delta @@ -270,8 +284,6 @@ pub const vortex_fastlanes::Delta::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::Delta::try_from_primitive_array(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -pub fn vortex_fastlanes::Delta::try_new(bases: vortex_array::array::erased::ArrayRef, deltas: vortex_array::array::erased::ArrayRef, offset: usize, len: usize) -> vortex_error::VortexResult - impl core::clone::Clone for vortex_fastlanes::Delta pub fn vortex_fastlanes::Delta::clone(&self) -> vortex_fastlanes::Delta @@ -284,6 +296,8 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::Delta pub type vortex_fastlanes::Delta::ArrayData = vortex_fastlanes::DeltaData +pub type vortex_fastlanes::Delta::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_fastlanes::Delta::OperationsVTable = vortex_fastlanes::Delta pub type vortex_fastlanes::Delta::ValidityVTable = vortex_fastlanes::Delta @@ -296,23 +310,33 @@ pub fn vortex_fastlanes::Delta::buffer(_array: vortex_array::array::view::ArrayV pub fn vortex_fastlanes::Delta::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::Delta::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fastlanes::Delta::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_fastlanes::Delta::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_fastlanes::Delta::dtype(array: &vortex_fastlanes::DeltaData) -> &vortex_array::dtype::DType pub fn vortex_fastlanes::Delta::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_fastlanes::Delta::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_fastlanes::Delta::len(array: &vortex_fastlanes::DeltaData) -> usize + +pub fn vortex_fastlanes::Delta::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_fastlanes::Delta::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::Delta::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::Delta::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::Delta::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::Delta::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::Delta::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::Delta::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::Delta::stats(array: &vortex_fastlanes::DeltaData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_fastlanes::Delta::vtable(_array: &vortex_fastlanes::DeltaData) -> &Self pub fn vortex_fastlanes::Delta::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -340,18 +364,34 @@ pub fn vortex_fastlanes::DeltaData::bases(&self) -> &vortex_array::array::erased pub fn vortex_fastlanes::DeltaData::deltas(&self) -> &vortex_array::array::erased::ArrayRef +pub fn vortex_fastlanes::DeltaData::dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_fastlanes::DeltaData::is_empty(&self) -> bool + +pub fn vortex_fastlanes::DeltaData::len(&self) -> usize + pub fn vortex_fastlanes::DeltaData::offset(&self) -> usize +pub fn vortex_fastlanes::DeltaData::try_from_primitive_array(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + pub fn vortex_fastlanes::DeltaData::try_new(bases: vortex_array::array::erased::ArrayRef, deltas: vortex_array::array::erased::ArrayRef, offset: usize, len: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_fastlanes::DeltaData pub fn vortex_fastlanes::DeltaData::clone(&self) -> vortex_fastlanes::DeltaData +impl core::convert::From for vortex_array::array::erased::ArrayRef + +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::DeltaData) -> vortex_array::array::erased::ArrayRef + impl core::fmt::Debug for vortex_fastlanes::DeltaData pub fn vortex_fastlanes::DeltaData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::array::IntoArray for vortex_fastlanes::DeltaData + +pub fn vortex_fastlanes::DeltaData::into_array(self) -> vortex_array::array::erased::ArrayRef + pub struct vortex_fastlanes::FoR impl vortex_fastlanes::FoR @@ -374,6 +414,8 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::FoR pub type vortex_fastlanes::FoR::ArrayData = vortex_fastlanes::FoRData +pub type vortex_fastlanes::FoR::Metadata = vortex_array::scalar::Scalar + pub type vortex_fastlanes::FoR::OperationsVTable = vortex_fastlanes::FoR pub type vortex_fastlanes::FoR::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -386,7 +428,11 @@ pub fn vortex_fastlanes::FoR::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_fastlanes::FoR::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::FoR::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fastlanes::FoR::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_fastlanes::FoR::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_fastlanes::FoR::dtype(array: &vortex_fastlanes::FoRData) -> &vortex_array::dtype::DType pub fn vortex_fastlanes::FoR::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -394,17 +440,23 @@ pub fn vortex_fastlanes::FoR::execute_parent(array: vortex_array::array::view::A pub fn vortex_fastlanes::FoR::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_fastlanes::FoR::len(array: &vortex_fastlanes::FoRData) -> usize + +pub fn vortex_fastlanes::FoR::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_fastlanes::FoR::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::FoR::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::FoR::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::FoR::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::FoR::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::FoR::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::FoR::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::FoR::stats(array: &vortex_fastlanes::FoRData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_fastlanes::FoR::vtable(_array: &vortex_fastlanes::FoRData) -> &Self pub fn vortex_fastlanes::FoR::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -452,6 +504,8 @@ pub fn vortex_fastlanes::FoRData::ptype(&self) -> vortex_array::dtype::ptype::PT pub fn vortex_fastlanes::FoRData::reference_scalar(&self) -> &vortex_array::scalar::Scalar +pub fn vortex_fastlanes::FoRData::try_new(encoded: vortex_array::array::erased::ArrayRef, reference: vortex_array::scalar::Scalar) -> vortex_error::VortexResult + impl vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::encode(array: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult @@ -460,10 +514,18 @@ impl core::clone::Clone for vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::clone(&self) -> vortex_fastlanes::FoRData +impl core::convert::From for vortex_array::array::erased::ArrayRef + +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::FoRData) -> vortex_array::array::erased::ArrayRef + impl core::fmt::Debug for vortex_fastlanes::FoRData pub fn vortex_fastlanes::FoRData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::array::IntoArray for vortex_fastlanes::FoRData + +pub fn vortex_fastlanes::FoRData::into_array(self) -> vortex_array::array::erased::ArrayRef + pub struct vortex_fastlanes::RLE impl vortex_fastlanes::RLE @@ -472,9 +534,7 @@ pub const vortex_fastlanes::RLE::ID: vortex_array::array::ArrayId pub fn vortex_fastlanes::RLE::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult -pub unsafe fn vortex_fastlanes::RLE::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_fastlanes::RLEArray - -pub fn vortex_fastlanes::RLE::try_new(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult +pub unsafe fn vortex_fastlanes::RLE::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, dtype: vortex_array::dtype::DType, offset: usize, length: usize) -> vortex_fastlanes::RLEArray impl core::clone::Clone for vortex_fastlanes::RLE @@ -488,9 +548,11 @@ impl vortex_array::array::vtable::VTable for vortex_fastlanes::RLE pub type vortex_fastlanes::RLE::ArrayData = vortex_fastlanes::RLEData +pub type vortex_fastlanes::RLE::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_fastlanes::RLE::OperationsVTable = vortex_fastlanes::RLE -pub type vortex_fastlanes::RLE::ValidityVTable = vortex_fastlanes::RLE +pub type vortex_fastlanes::RLE::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChildSliceHelper pub fn vortex_fastlanes::RLE::array_eq(array: &vortex_fastlanes::RLEData, other: &vortex_fastlanes::RLEData, precision: vortex_array::hash::Precision) -> bool @@ -500,7 +562,11 @@ pub fn vortex_fastlanes::RLE::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_fastlanes::RLE::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, _idx: usize) -> core::option::Option -pub fn vortex_fastlanes::RLE::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLE::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_fastlanes::RLE::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_fastlanes::RLE::dtype(array: &vortex_fastlanes::RLEData) -> &vortex_array::dtype::DType pub fn vortex_fastlanes::RLE::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -508,17 +574,23 @@ pub fn vortex_fastlanes::RLE::execute_parent(array: vortex_array::array::view::A pub fn vortex_fastlanes::RLE::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_fastlanes::RLE::len(array: &vortex_fastlanes::RLEData) -> usize + +pub fn vortex_fastlanes::RLE::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_fastlanes::RLE::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fastlanes::RLE::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fastlanes::RLE::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_fastlanes::RLE::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_fastlanes::RLE::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fastlanes::RLE::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fastlanes::RLE::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_fastlanes::RLE::stats(array: &vortex_fastlanes::RLEData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_fastlanes::RLE::vtable(_array: &vortex_fastlanes::RLEData) -> &Self pub fn vortex_fastlanes::RLE::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -526,9 +598,9 @@ impl vortex_array::array::vtable::operations::OperationsVTable, index: usize, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult -impl vortex_array::array::vtable::validity::ValidityVTable for vortex_fastlanes::RLE +impl vortex_array::array::vtable::validity::ValidityChild for vortex_fastlanes::RLE -pub fn vortex_fastlanes::RLE::validity(array: vortex_array::array::view::ArrayView<'_, vortex_fastlanes::RLE>) -> vortex_error::VortexResult +pub fn vortex_fastlanes::RLE::validity_child(array: &vortex_fastlanes::RLEData) -> &vortex_array::array::erased::ArrayRef impl vortex_array::arrays::slice::SliceKernel for vortex_fastlanes::RLE @@ -542,13 +614,15 @@ pub struct vortex_fastlanes::RLEData impl vortex_fastlanes::RLEData -pub fn vortex_fastlanes::RLEData::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult - -impl vortex_fastlanes::RLEData +pub fn vortex_fastlanes::RLEData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_fastlanes::RLEData::indices(&self) -> &vortex_array::array::erased::ArrayRef -pub unsafe fn vortex_fastlanes::RLEData::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, offset: usize) -> Self +pub fn vortex_fastlanes::RLEData::is_empty(&self) -> bool + +pub fn vortex_fastlanes::RLEData::len(&self) -> usize + +pub unsafe fn vortex_fastlanes::RLEData::new_unchecked(values: vortex_array::array::erased::ArrayRef, indices: vortex_array::array::erased::ArrayRef, values_idx_offsets: vortex_array::array::erased::ArrayRef, dtype: vortex_array::dtype::DType, offset: usize, length: usize) -> Self pub fn vortex_fastlanes::RLEData::offset(&self) -> usize @@ -558,109 +632,29 @@ pub fn vortex_fastlanes::RLEData::values(&self) -> &vortex_array::array::erased: pub fn vortex_fastlanes::RLEData::values_idx_offsets(&self) -> &vortex_array::array::erased::ArrayRef -impl core::clone::Clone for vortex_fastlanes::RLEData - -pub fn vortex_fastlanes::RLEData::clone(&self) -> vortex_fastlanes::RLEData - -impl core::fmt::Debug for vortex_fastlanes::RLEData - -pub fn vortex_fastlanes::RLEData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -pub trait vortex_fastlanes::BitPackedArrayExt - -pub fn vortex_fastlanes::BitPackedArrayExt::bit_width(&self) -> u8 - -pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData - -pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_fastlanes::BitPackedArrayExt::bitpacked_len(&self) -> usize - -pub fn vortex_fastlanes::BitPackedArrayExt::offset(&self) -> u16 - -pub fn vortex_fastlanes::BitPackedArrayExt::packed(&self) -> &vortex_array::buffer::BufferHandle - -pub fn vortex_fastlanes::BitPackedArrayExt::packed_slice(&self) -> &[T] - -pub fn vortex_fastlanes::BitPackedArrayExt::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedArrayExt::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedArrayExt::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedArrayExt::patches(&self) -> core::option::Option - -pub fn vortex_fastlanes::BitPackedArrayExt::unpacked_chunks(&self) -> vortex_error::VortexResult> - -pub fn vortex_fastlanes::BitPackedArrayExt::validity(&self) -> vortex_array::validity::Validity - -pub fn vortex_fastlanes::BitPackedArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_fastlanes::BitPackedArrayExt::validity_mask(&self) -> vortex_mask::Mask - -impl vortex_fastlanes::BitPackedArrayExt for vortex_array::array::typed::Array - -pub fn vortex_array::array::typed::Array::bit_width(&self) -> u8 - -pub fn vortex_array::array::typed::Array::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData - -pub fn vortex_array::array::typed::Array::bitpacked_dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_array::array::typed::Array::bitpacked_len(&self) -> usize - -pub fn vortex_array::array::typed::Array::offset(&self) -> u16 - -pub fn vortex_array::array::typed::Array::packed(&self) -> &vortex_array::buffer::BufferHandle - -pub fn vortex_array::array::typed::Array::packed_slice(&self) -> &[T] - -pub fn vortex_array::array::typed::Array::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_array::array::typed::Array::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_array::array::typed::Array::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_array::array::typed::Array::patches(&self) -> core::option::Option - -pub fn vortex_array::array::typed::Array::unpacked_chunks(&self) -> vortex_error::VortexResult> - -pub fn vortex_array::array::typed::Array::validity(&self) -> vortex_array::validity::Validity - -pub fn vortex_array::array::typed::Array::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> - -pub fn vortex_array::array::typed::Array::validity_mask(&self) -> vortex_mask::Mask - -impl vortex_fastlanes::BitPackedArrayExt for vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked> - -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bit_width(&self) -> u8 - -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_data(&self) -> &vortex_fastlanes::BitPackedData - -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_dtype(&self) -> &vortex_array::dtype::DType - -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::bitpacked_len(&self) -> usize +impl vortex_fastlanes::RLEData -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::offset(&self) -> u16 +pub fn vortex_fastlanes::RLEData::encode(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::packed(&self) -> &vortex_array::buffer::BufferHandle +impl core::clone::Clone for vortex_fastlanes::RLEData -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::packed_slice(&self) -> &[T] +pub fn vortex_fastlanes::RLEData::clone(&self) -> vortex_fastlanes::RLEData -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +impl core::convert::From for vortex_array::array::erased::ArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_indices(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fastlanes::RLEData) -> vortex_array::array::erased::ArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patch_values(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +impl core::fmt::Debug for vortex_fastlanes::RLEData -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::patches(&self) -> core::option::Option +pub fn vortex_fastlanes::RLEData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::unpacked_chunks(&self) -> vortex_error::VortexResult> +impl vortex_array::array::IntoArray for vortex_fastlanes::RLEData -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity(&self) -> vortex_array::validity::Validity +pub fn vortex_fastlanes::RLEData::into_array(self) -> vortex_array::array::erased::ArrayRef -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity_child(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> +impl vortex_array::array::vtable::validity::ValidityChildSliceHelper for vortex_fastlanes::RLEData -pub fn vortex_array::array::view::ArrayView<'_, vortex_fastlanes::BitPacked>::validity_mask(&self) -> vortex_mask::Mask +pub fn vortex_fastlanes::RLEData::unsliced_child_and_slice(&self) -> (&vortex_array::array::erased::ArrayRef, usize, usize) pub fn vortex_fastlanes::delta_compress(array: &vortex_array::arrays::primitive::vtable::PrimitiveArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult<(vortex_array::arrays::primitive::vtable::PrimitiveArray, vortex_array::arrays::primitive::vtable::PrimitiveArray)> diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index 4290e5cb44a..43ae4e410c5 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -23,8 +23,8 @@ use vortex_error::vortex_bail; use vortex_mask::AllOr; use vortex_mask::Mask; -use crate::BitPacked; use crate::BitPackedArray; +use crate::BitPackedData; use crate::bitpack_decompress; pub fn bitpack_to_best_bit_width(array: &PrimitiveArray) -> VortexResult { @@ -71,16 +71,27 @@ pub fn bitpack_encode( .transpose()? .flatten(); - let bitpacked = BitPacked::try_new( - BufferHandle::new_host(packed), - array.ptype(), - array.validity(), - patches, - bit_width, - array.len(), - 0, - )?; - bitpacked.statistics().inherit_from(array.statistics()); + // SAFETY: all components validated above + let bitpacked = unsafe { + BitPackedData::new_unchecked( + BufferHandle::new_host(packed), + array.dtype().clone(), + array.validity(), + patches, + bit_width, + array.len(), + 0, + ) + }; + let bitpacked = + BitPackedArray::try_from_data(bitpacked).vortex_expect("BitPackedData is always valid"); + { + let bp_ref = bitpacked.clone().into_array(); + bitpacked + .stats_set + .to_ref(&bp_ref) + .inherit_from(array.statistics()); + } Ok(bitpacked) } @@ -99,18 +110,28 @@ pub unsafe fn bitpack_encode_unchecked( // SAFETY: non-negativity of input checked by caller. let packed = unsafe { bitpack_unchecked(&array, bit_width) }; - let arr_ref = array.clone().into_array(); - let bitpacked = BitPacked::try_new( - BufferHandle::new_host(packed), - array.ptype(), - array.validity(), - None, - bit_width, - array.len(), - 0, - ) - .vortex_expect("bitpacked array construction should succeed"); - bitpacked.statistics().inherit_from(arr_ref.statistics()); + // SAFETY: checked by bitpack_unchecked + let data = unsafe { + BitPackedData::new_unchecked( + BufferHandle::new_host(packed), + array.dtype().clone(), + array.validity(), + None, + bit_width, + array.len(), + 0, + ) + }; + let bitpacked = + BitPackedArray::try_from_data(data).vortex_expect("BitPackedData is always valid"); + { + let bp_ref = bitpacked.clone().into_array(); + let arr_ref = array.into_array(); + bitpacked + .stats_set + .to_ref(&bp_ref) + .inherit_from(arr_ref.statistics()); + } Ok(bitpacked) } @@ -423,7 +444,6 @@ mod test { use vortex_session::VortexSession; use super::*; - use crate::BitPackedData; use crate::bitpack_compress::test_harness::make_array; static SESSION: LazyLock = @@ -449,7 +469,7 @@ mod test { ); assert!(values.ptype().is_unsigned_int()); let compressed = BitPackedData::encode(&values.into_array(), 4).unwrap(); - assert!(compressed.patches(compressed.len()).is_none()); + assert!(compressed.patches().is_none()); assert_eq!( (0..(1 << 4)).collect::>(), compressed @@ -511,7 +531,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches(bitpacked.len()).unwrap(); + let patches = bitpacked.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -534,7 +554,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches(bitpacked.len()).unwrap(); + let patches = bitpacked.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); assert_arrays_eq!(chunk_offsets, PrimitiveArray::from_iter([0u64, 2, 2])); @@ -553,7 +573,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches(bitpacked.len()).unwrap(); + let patches = bitpacked.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // chunk 0 (0-1023): patches at 100, 200 -> starts at patch index 0 @@ -577,7 +597,7 @@ mod test { let array = PrimitiveArray::from_iter(values); let bitpacked = bitpack_encode(&array, 4, None).unwrap(); - let patches = bitpacked.patches(bitpacked.len()).unwrap(); + let patches = bitpacked.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().as_ref().unwrap().to_primitive(); // Single chunk starting at patch index 0. diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 5ed8d6b89fa..4f31154d598 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -4,7 +4,6 @@ use fastlanes::BitPacking; use itertools::Itertools; use num_traits::AsPrimitive; -use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::arrays::PrimitiveArray; use vortex_array::builders::ArrayBuilder; @@ -18,22 +17,18 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use crate::BitPacked; -use crate::BitPackedArrayExt; -use crate::unpack_iter::BitPacked as BitPackedUnpack; +use crate::BitPackedData; +use crate::unpack_iter::BitPacked; /// Unpacks a bit-packed array into a primitive array. -pub fn unpack_array( - array: ArrayView<'_, BitPacked>, - ctx: &mut ExecutionCtx, -) -> VortexResult { - match_each_integer_ptype!(array.dtype().as_ptype(), |P| { +pub fn unpack_array(array: &BitPackedData, ctx: &mut ExecutionCtx) -> VortexResult { + match_each_integer_ptype!(array.ptype(), |P| { unpack_primitive_array::

(array, ctx) }) } -pub fn unpack_primitive_array( - array: ArrayView<'_, BitPacked>, +pub fn unpack_primitive_array( + array: &BitPackedData, ctx: &mut ExecutionCtx, ) -> VortexResult { let mut builder = PrimitiveBuilder::with_capacity(array.dtype().nullability(), array.len()); @@ -42,8 +37,8 @@ pub fn unpack_primitive_array( Ok(builder.finish_into_primitive()) } -pub(crate) fn unpack_into_primitive_builder( - array: ArrayView<'_, BitPacked>, +pub(crate) fn unpack_into_primitive_builder( + array: &BitPackedData, // TODO(ngates): do we want to use fastlanes alignment for this buffer? builder: &mut PrimitiveBuilder, ctx: &mut ExecutionCtx, @@ -64,7 +59,7 @@ pub(crate) fn unpack_into_primitive_builder( // SAFETY: `decode_into` will initialize all values in this range. let uninit_slice = unsafe { uninit_range.slice_uninit_mut(0, array.len()) }; - let mut bit_packed_iter = array.unpacked_chunks()?; + let mut bit_packed_iter = array.unpacked_chunks(); bit_packed_iter.decode_into(uninit_slice); if let Some(ref patches) = array.patches() { @@ -111,9 +106,9 @@ pub fn apply_patches_to_uninit_range_fn T>( Ok(()) } -pub fn unpack_single(array: ArrayView<'_, BitPacked>, index: usize) -> Scalar { +pub fn unpack_single(array: &BitPackedData, index: usize) -> Scalar { let bit_width = array.bit_width() as usize; - let ptype = array.dtype().as_ptype(); + let ptype = array.ptype(); // let packed = array.packed().into_primitive()?; let index_in_encoded = index + array.offset() as usize; let scalar: Scalar = match_each_unsigned_integer_ptype!(ptype.to_unsigned(), |P| { @@ -183,10 +178,6 @@ mod tests { static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); - fn unpack(bitpacked: &BitPackedArray) -> VortexResult { - unpack_array(bitpacked.as_view(), &mut SESSION.create_execution_ctx()) - } - fn compression_roundtrip(n: usize) { let values = PrimitiveArray::from_iter((0..n).map(|i| (i % 2047) as u16)); let compressed = BitPackedData::encode(&values.clone().into_array(), 11).unwrap(); @@ -197,9 +188,7 @@ mod tests { .iter() .enumerate() .for_each(|(i, v)| { - let scalar: u16 = (&unpack_single(compressed.as_view(), i)) - .try_into() - .unwrap(); + let scalar: u16 = (&unpack_single(&compressed, i)).try_into().unwrap(); assert_eq!(scalar, *v); }); } @@ -221,7 +210,7 @@ mod tests { fn test_all_zeros() -> VortexResult<()> { let zeros = buffer![0u16, 0, 0, 0].into_array().to_primitive(); let bitpacked = encode(&zeros, 0); - let actual = unpack(&bitpacked)?; + let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!(actual, PrimitiveArray::from_iter([0u16, 0, 0, 0])); Ok(()) } @@ -230,7 +219,7 @@ mod tests { fn test_simple_patches() -> VortexResult<()> { let zeros = buffer![0u16, 1, 0, 1].into_array().to_primitive(); let bitpacked = encode(&zeros, 0); - let actual = unpack(&bitpacked)?; + let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!(actual, PrimitiveArray::from_iter([0u16, 1, 0, 1])); Ok(()) } @@ -239,7 +228,7 @@ mod tests { fn test_one_full_chunk() -> VortexResult<()> { let zeros = BufferMut::from_iter(0u16..1024).into_array().to_primitive(); let bitpacked = encode(&zeros, 10); - let actual = unpack(&bitpacked)?; + let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(0u16..1024)); Ok(()) } @@ -251,7 +240,7 @@ mod tests { .to_primitive(); let bitpacked = encode(&zeros, 10); assert!(bitpacked.patches().is_some()); - let actual = unpack(&bitpacked)?; + let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!( actual, PrimitiveArray::from_iter((5u16..1029).chain(5u16..1029).chain(5u16..1029)) @@ -264,7 +253,7 @@ mod tests { let zeros = BufferMut::from_iter(0u16..1025).into_array().to_primitive(); let bitpacked = encode(&zeros, 11); assert!(bitpacked.patches().is_none()); - let actual = unpack(&bitpacked)?; + let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(0u16..1025)); Ok(()) } @@ -277,7 +266,7 @@ mod tests { let bitpacked = encode(&zeros, 10); assert_eq!(bitpacked.len(), 1025); assert!(bitpacked.patches().is_some()); - let actual = unpack(&bitpacked)?; + let actual = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!(actual, PrimitiveArray::from_iter(512u16..1537)); Ok(()) } @@ -332,7 +321,7 @@ mod tests { let mut builder = PrimitiveBuilder::::new(Nullability::NonNullable); unpack_into_primitive_builder( - bitpacked.as_view(), + &bitpacked, &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -360,7 +349,7 @@ mod tests { // Unpack into a new builder. let mut builder = PrimitiveBuilder::::with_capacity(Nullability::Nullable, 5); unpack_into_primitive_builder( - bitpacked.as_view(), + &bitpacked, &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -396,7 +385,7 @@ mod tests { // Unpack into a new builder. let mut builder = PrimitiveBuilder::::with_capacity(Nullability::NonNullable, 100); unpack_into_primitive_builder( - bitpacked.as_view(), + &bitpacked, &mut builder, &mut SESSION.create_execution_ctx(), )?; @@ -437,7 +426,7 @@ mod tests { let large_bitpacked = encode(&large_array, 8); assert!(large_bitpacked.patches().is_some()); - let large_result = unpack(&large_bitpacked)?; + let large_result = unpack_array(&large_bitpacked, &mut SESSION.create_execution_ctx())?; assert_eq!(large_result.len(), 3072); Ok(()) } @@ -451,7 +440,8 @@ mod tests { let array = PrimitiveArray::new(values, validity); let bitpacked = encode(&array, 9); - let result = unpack(&bitpacked).vortex_expect("unpack"); + let result = + unpack_array(&bitpacked, &mut SESSION.create_execution_ctx()).vortex_expect("unpack"); // Verify length. assert_eq!(result.len(), 7); @@ -468,7 +458,8 @@ mod tests { let patch_bitpacked = encode(&patch_array, 5); assert!(patch_bitpacked.patches().is_some()); - let patch_result = unpack(&patch_bitpacked).vortex_expect("unpack"); + let patch_result = unpack_array(&patch_bitpacked, &mut SESSION.create_execution_ctx()) + .vortex_expect("unpack"); assert_eq!(patch_result.len(), 7); // Test all nulls edge case. @@ -477,7 +468,8 @@ mod tests { Validity::from_iter([false, false, false, false]), ); let all_nulls_bp = encode(&all_nulls, 0); - let all_nulls_result = unpack(&all_nulls_bp).vortex_expect("unpack"); + let all_nulls_result = unpack_array(&all_nulls_bp, &mut SESSION.create_execution_ctx()) + .vortex_expect("unpack"); assert_eq!(all_nulls_result.len(), 4); } @@ -488,7 +480,7 @@ mod tests { let test_consistency = |array: &PrimitiveArray, bit_width: u8| -> VortexResult<()> { let bitpacked = encode(array, bit_width); - let unpacked_array = unpack(&bitpacked)?; + let unpacked_array = unpack_array(&bitpacked, &mut SESSION.create_execution_ctx())?; let executed = { let mut ctx = SESSION.create_execution_ctx(); @@ -592,23 +584,23 @@ mod tests { // Empty array. let empty: PrimitiveArray = PrimitiveArray::from_iter(Vec::::new()); let empty_bp = encode(&empty, 0); - let empty_result = unpack(&empty_bp)?; + let empty_result = unpack_array(&empty_bp, &mut SESSION.create_execution_ctx())?; assert_eq!(empty_result.len(), 0); // All zeros (bit_width = 0). let zeros = PrimitiveArray::from_iter([0u32; 100]); let zeros_bp = encode(&zeros, 0); - let zeros_result = unpack(&zeros_bp)?; + let zeros_result = unpack_array(&zeros_bp, &mut SESSION.create_execution_ctx())?; assert_eq!(zeros_result.len(), 100); // Verify consistency with unpack_array. - let zeros_array = unpack(&zeros_bp)?; + let zeros_array = unpack_array(&zeros_bp, &mut SESSION.create_execution_ctx())?; assert_eq!(zeros_result.len(), zeros_array.len()); assert_arrays_eq!(zeros_result, zeros_array); // Maximum bit width for u16 (15 bits, since bitpacking requires bit_width < type bit width). let max_values = PrimitiveArray::from_iter([32767u16; 50]); // 2^15 - 1 let max_bp = encode(&max_values, 15); - let max_result = unpack(&max_bp)?; + let max_result = unpack_array(&max_bp, &mut SESSION.create_execution_ctx())?; assert_eq!(max_result.len(), 50); // Exactly 3072 elements with patches across chunks. @@ -625,17 +617,17 @@ mod tests { let boundary_bp = encode(&boundary_array, 7); assert!(boundary_bp.patches().is_some()); - let boundary_result = unpack(&boundary_bp)?; + let boundary_result = unpack_array(&boundary_bp, &mut SESSION.create_execution_ctx())?; assert_eq!(boundary_result.len(), 3072); // Verify consistency. - let boundary_unpacked = unpack(&boundary_bp)?; + let boundary_unpacked = unpack_array(&boundary_bp, &mut SESSION.create_execution_ctx())?; assert_eq!(boundary_result.len(), boundary_unpacked.len()); assert_arrays_eq!(boundary_result, boundary_unpacked); // Single element. let single = PrimitiveArray::from_iter([42u8]); let single_bp = encode(&single, 6); - let single_result = unpack(&single_bp)?; + let single_result = unpack_array(&single_bp, &mut SESSION.create_execution_ctx())?; assert_eq!(single_result.len(), 1); Ok(()) } diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index 9e8c98b1c04..f54e788650b 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -2,17 +2,15 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use fastlanes::BitPacking; -use vortex_array::Array; use vortex_array::ArrayRef; -use vortex_array::ArrayView; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; -use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::patches::Patches; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable::child_to_validity; use vortex_array::vtable::validity_to_child; @@ -46,7 +44,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = [ "validity", ]; -pub struct BitPackedDataParts { +pub struct BitPackedArrayParts { pub offset: u16, pub bit_width: u8, pub len: usize, @@ -61,12 +59,15 @@ pub struct BitPackedData { /// The offset within the first block (created with a slice). /// 0 <= offset < 1024 pub(super) offset: u16, + pub(super) len: usize, + pub(super) dtype: DType, pub(super) bit_width: u8, pub(super) packed: BufferHandle, /// The offset metadata from patches, needed to reconstruct Patches from slots. pub(super) patch_offset: Option, /// The offset_within_chunk metadata from patches. pub(super) patch_offset_within_chunk: Option, + pub(super) stats_set: ArrayStats, } impl BitPackedData { @@ -92,6 +93,7 @@ impl BitPackedData { /// bit-packed array. pub(crate) unsafe fn new_unchecked( packed: BufferHandle, + dtype: DType, validity: Validity, patches: Option, bit_width: u8, @@ -107,10 +109,13 @@ impl BitPackedData { Self { slots, offset, + len, + dtype, bit_width, packed, patch_offset, patch_offset_within_chunk, + stats_set: Default::default(), } } @@ -171,28 +176,16 @@ impl BitPackedData { offset, )?; + let dtype = DType::Primitive(ptype, validity.nullability()); + // SAFETY: all components validated above unsafe { Ok(Self::new_unchecked( - packed, validity, patches, bit_width, length, offset, + packed, dtype, validity, patches, bit_width, length, offset, )) } } - pub(crate) fn validate_against_outer(&self, dtype: &DType, len: usize) -> VortexResult<()> { - let validity = self.validity(dtype.nullability()); - let patches = self.patches(len); - Self::validate( - &self.packed, - dtype.as_ptype(), - &validity, - patches.as_ref(), - self.bit_width, - len, - self.offset, - ) - } - fn validate( packed: &BufferHandle, ptype: PType, @@ -254,13 +247,31 @@ impl BitPackedData { Ok(()) } + /// Returns the length of the array. + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Returns `true` if the array is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Returns the dtype of the array. + #[inline] + pub fn dtype(&self) -> &DType { + &self.dtype + } + /// Returns the validity as a [`Mask`](vortex_mask::Mask). - pub fn validity_mask(&self, len: usize, nullability: Nullability) -> vortex_mask::Mask { - self.validity(nullability).to_mask(len) + pub fn validity_mask(&self) -> vortex_mask::Mask { + self.validity().to_mask(self.len()) } - pub fn ptype(&self, dtype: &DType) -> PType { - dtype.as_ptype() + pub fn ptype(&self) -> PType { + self.dtype.as_ptype() } /// Underlying bit packed values as byte array @@ -283,38 +294,14 @@ impl BitPackedData { unsafe { std::slice::from_raw_parts(packed_ptr, packed_len) } } - #[inline] - pub fn patch_indices(&self) -> Option<&ArrayRef> { - self.slots[PATCH_INDICES_SLOT].as_ref() - } - - #[inline] - pub fn patch_values(&self) -> Option<&ArrayRef> { - self.slots[PATCH_VALUES_SLOT].as_ref() - } - - #[inline] - pub fn patch_chunk_offsets(&self) -> Option<&ArrayRef> { - self.slots[PATCH_CHUNK_OFFSETS_SLOT].as_ref() - } - - #[inline] - pub fn validity_child(&self) -> Option<&ArrayRef> { - self.slots[VALIDITY_SLOT].as_ref() - } - /// Accessor for bit unpacked chunks - pub fn unpacked_chunks( - &self, - dtype: &DType, - len: usize, - ) -> VortexResult> { + pub fn unpacked_chunks(&self) -> BitUnpackedChunks { assert_eq!( T::PTYPE, - self.ptype(dtype), + self.ptype(), "Requested type doesn't match the array ptype" ); - BitUnpackedChunks::try_new(self, len) + BitUnpackedChunks::new(self) } /// Bit-width of the packed values @@ -328,19 +315,22 @@ impl BitPackedData { /// Reconstructs a `Patches` from the stored slots and patch metadata. /// If present, patches MUST be a `SparseArray` with equal-length to this array, and whose /// indices indicate the locations of patches. The indices must have non-zero length. - pub fn patches(&self, len: usize) -> Option { - match (self.patch_indices(), self.patch_values()) { + pub fn patches(&self) -> Option { + match ( + &self.slots[PATCH_INDICES_SLOT], + &self.slots[PATCH_VALUES_SLOT], + ) { (Some(indices), Some(values)) => { let patch_offset = self .patch_offset .vortex_expect("has patch slots but no patch_offset"); Some(unsafe { Patches::new_unchecked( - len, + self.len, patch_offset, indices.clone(), values.clone(), - self.patch_chunk_offsets().cloned(), + self.slots[PATCH_CHUNK_OFFSETS_SLOT].clone(), self.patch_offset_within_chunk, ) }) @@ -350,8 +340,8 @@ impl BitPackedData { } /// Returns the validity, reconstructed from the stored slot. - pub fn validity(&self, nullability: Nullability) -> Validity { - child_to_validity(&self.validity_child().cloned(), nullability) + pub fn validity(&self) -> Validity { + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } pub fn replace_patches(&mut self, patches: Option) { @@ -402,13 +392,13 @@ impl BitPackedData { (1 << self.bit_width()) - 1 } - pub fn into_parts(self, len: usize, nullability: Nullability) -> BitPackedDataParts { - let patches = self.patches(len); - let validity = self.validity(nullability); - BitPackedDataParts { + pub fn into_parts(self) -> BitPackedArrayParts { + let patches = self.patches(); + let validity = self.validity(); + BitPackedArrayParts { offset: self.offset, bit_width: self.bit_width, - len, + len: self.len, packed: self.packed, patches, validity, @@ -416,102 +406,6 @@ impl BitPackedData { } } -pub trait BitPackedArrayExt { - fn bitpacked_data(&self) -> &BitPackedData; - fn bitpacked_dtype(&self) -> &DType; - fn bitpacked_len(&self) -> usize; - - #[inline] - fn packed(&self) -> &BufferHandle { - self.bitpacked_data().packed() - } - - #[inline] - fn bit_width(&self) -> u8 { - self.bitpacked_data().bit_width() - } - - #[inline] - fn offset(&self) -> u16 { - self.bitpacked_data().offset() - } - - #[inline] - fn patch_indices(&self) -> Option<&ArrayRef> { - self.bitpacked_data().patch_indices() - } - - #[inline] - fn patch_values(&self) -> Option<&ArrayRef> { - self.bitpacked_data().patch_values() - } - - #[inline] - fn patch_chunk_offsets(&self) -> Option<&ArrayRef> { - self.bitpacked_data().patch_chunk_offsets() - } - - #[inline] - fn validity_child(&self) -> Option<&ArrayRef> { - self.bitpacked_data().validity_child() - } - - #[inline] - fn patches(&self) -> Option { - self.bitpacked_data().patches(self.bitpacked_len()) - } - - #[inline] - fn validity(&self) -> Validity { - self.bitpacked_data() - .validity(self.bitpacked_dtype().nullability()) - } - - #[inline] - fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.bitpacked_len()) - } - - #[inline] - fn packed_slice(&self) -> &[T] { - self.bitpacked_data().packed_slice::() - } - - #[inline] - fn unpacked_chunks(&self) -> VortexResult> { - self.bitpacked_data() - .unpacked_chunks::(self.bitpacked_dtype(), self.bitpacked_len()) - } -} - -impl BitPackedArrayExt for Array { - fn bitpacked_data(&self) -> &BitPackedData { - self.data() - } - - fn bitpacked_dtype(&self) -> &DType { - self.dtype() - } - - fn bitpacked_len(&self) -> usize { - self.len() - } -} - -impl BitPackedArrayExt for ArrayView<'_, crate::BitPacked> { - fn bitpacked_data(&self) -> &BitPackedData { - self.data() - } - - fn bitpacked_dtype(&self) -> &DType { - self.dtype() - } - - fn bitpacked_len(&self) -> usize { - self.len() - } -} - #[cfg(test)] mod test { use vortex_array::IntoArray; @@ -555,11 +449,7 @@ mod test { let parray = values.clone().into_array(); let packed_with_patches = BitPackedData::encode(&parray, 9).unwrap(); - assert!( - packed_with_patches - .patches(packed_with_patches.len()) - .is_some() - ); + assert!(packed_with_patches.patches().is_some()); assert_arrays_eq!( packed_with_patches.as_array().to_primitive(), PrimitiveArray::new(values, vortex_array::validity::Validity::NonNullable) diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index 80a73f82678..d14d218401c 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -11,8 +11,6 @@ use lending_iterator::prelude::Item; use lending_iterator::prelude::LendingIterator; use vortex_array::dtype::PhysicalPType; use vortex_buffer::ByteBuffer; -use vortex_error::VortexResult; -use vortex_error::vortex_ensure; use crate::BitPackedData; @@ -57,11 +55,10 @@ impl> UnpackStrategy for BitPackingStr /// use vortex_array::IntoArray; /// use vortex_buffer::buffer; /// use vortex_fastlanes::BitPackedData; -/// use vortex_fastlanes::BitPackedArrayExt; /// use vortex_fastlanes::unpack_iter::BitUnpackedChunks; /// /// let array = BitPackedData::encode(&buffer![2, 3, 4, 5].into_array(), 2).unwrap(); -/// let mut unpacked_chunks: BitUnpackedChunks = array.unpacked_chunks().unwrap(); +/// let mut unpacked_chunks: BitUnpackedChunks = array.unpacked_chunks(); /// /// if let Some(header) = unpacked_chunks.initial() { /// // handle partial initial chunk @@ -92,13 +89,13 @@ pub struct UnpackedChunks> { pub type BitUnpackedChunks = UnpackedChunks; impl BitUnpackedChunks { - pub fn try_new(array: &BitPackedData, len: usize) -> VortexResult { - Self::try_new_with_strategy( + pub fn new(array: &BitPackedData) -> Self { + Self::new_with_strategy( BitPackingStrategy, array.packed().clone().unwrap_host(), array.bit_width() as usize, array.offset() as usize, - len, + array.len(), ) } @@ -118,29 +115,26 @@ impl BitUnpackedChunks { } impl> UnpackedChunks { - pub fn try_new_with_strategy( + pub fn new_with_strategy( strategy: S, packed: ByteBuffer, bit_width: usize, offset: usize, len: usize, - ) -> VortexResult { - vortex_ensure!( - offset < CHUNK_SIZE, - "Invalid bit-packed offset {offset}, expected < {CHUNK_SIZE}" - ); + ) -> Self { let elems_per_chunk = 128 * bit_width / size_of::(); let num_chunks = (offset + len).div_ceil(CHUNK_SIZE); - vortex_ensure!( - packed.len() / size_of::() == num_chunks * elems_per_chunk, + assert_eq!( + packed.len() / size_of::(), + num_chunks * elems_per_chunk, "Invalid packed length: got {}, expected {}", packed.len() / size_of::(), num_chunks * elems_per_chunk ); let last_chunk_length = (offset + len) % CHUNK_SIZE; - Ok(Self { + Self { strategy, bit_width, offset, @@ -149,7 +143,7 @@ impl> UnpackedChunks { buffer: [const { MaybeUninit::::uninit() }; CHUNK_SIZE], num_chunks, last_chunk_length, - }) + } } #[inline(always)] diff --git a/encodings/fastlanes/src/bitpacking/compute/cast.rs b/encodings/fastlanes/src/bitpacking/compute/cast.rs index b7e6a7b63da..43cbc38d455 100644 --- a/encodings/fastlanes/src/bitpacking/compute/cast.rs +++ b/encodings/fastlanes/src/bitpacking/compute/cast.rs @@ -10,20 +10,21 @@ use vortex_array::patches::Patches; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; +use crate::BitPackedData; use crate::bitpacking::BitPacked; impl CastReduce for BitPacked { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if array.dtype().eq_ignore_nullability(dtype) { let new_validity = array - .validity(array.dtype().nullability()) + .validity() .cast_nullability(dtype.nullability(), array.len())?; return Ok(Some( - BitPacked::try_new( + BitPackedData::try_new( array.packed().clone(), dtype.as_ptype(), new_validity, array - .patches(array.len()) + .patches() .map(|patches| { let new_values = patches.values().cast(dtype.clone())?; Patches::new( diff --git a/encodings/fastlanes/src/bitpacking/compute/filter.rs b/encodings/fastlanes/src/bitpacking/compute/filter.rs index 5ebeaa4705c..a2887615698 100644 --- a/encodings/fastlanes/src/bitpacking/compute/filter.rs +++ b/encodings/fastlanes/src/bitpacking/compute/filter.rs @@ -25,7 +25,6 @@ use vortex_mask::MaskValues; use super::chunked_indices; use super::take::UNPACK_CHUNK_THRESHOLD; use crate::BitPacked; -use crate::BitPackedArrayExt; use crate::BitPackedData; /// The threshold over which it is faster to fully unpack the entire [`BitPackedArray`] and then @@ -59,26 +58,16 @@ impl FilterKernel for BitPacked { // If the density is high enough, then we would rather decompress the whole array and then apply // a filter over decompressing values one by one. - if values.density() > unpack_then_filter_threshold(array.dtype().as_ptype()) { + if values.density() > unpack_then_filter_threshold(array.ptype()) { return Ok(None); } // Filter and patch using the correct unsigned type for FastLanes, then cast to signed if needed. - let primitive = - match_each_unsigned_integer_ptype!(array.dtype().as_ptype().to_unsigned(), |U| { - let (buffer, validity) = filter_primitive_without_patches::(array, values)?; - // reinterpret_cast for signed types. - let primitive = PrimitiveArray::new(buffer, validity); - if array.dtype().as_ptype().is_signed_int() { - PrimitiveArray::from_buffer_handle( - primitive.buffer_handle().clone(), - array.dtype().as_ptype(), - primitive.validity(), - ) - } else { - primitive - } - }); + let primitive = match_each_unsigned_integer_ptype!(array.ptype().to_unsigned(), |U| { + let (buffer, validity) = filter_primitive_without_patches::(&array, values)?; + // reinterpret_cast for signed types. + PrimitiveArray::new(buffer, validity).reinterpret_cast(array.ptype()) + }); let patches = array .patches() @@ -87,12 +76,12 @@ impl FilterKernel for BitPacked { .flatten(); if let Some(patches) = patches { - let mut prim_array = primitive; + let mut prim_array = PrimitiveArray::try_from_data(primitive)?; prim_array = prim_array.patch(&patches, ctx)?; return Ok(Some(prim_array.into_array())); } - Ok(Some(primitive.into_array())) + Ok(Some(PrimitiveArray::try_from_data(primitive)?.into_array())) } } @@ -108,10 +97,10 @@ impl FilterKernel for BitPacked { /// /// Returns a tuple of (values buffer, validity mask). fn filter_primitive_without_patches( - array: ArrayView<'_, BitPacked>, + array: &BitPackedData, selection: &Arc, ) -> VortexResult<(Buffer, Validity)> { - let values = filter_with_indices(array.data(), selection.indices()); + let values = filter_with_indices(array, selection.indices()); let validity = array.validity().filter(&Mask::Values(selection.clone()))?; Ok((values.freeze(), validity)) @@ -275,7 +264,7 @@ mod test { let unpacked = PrimitiveArray::from_iter(values.clone()); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); assert!( - bitpacked.patches(bitpacked.len()).is_some(), + bitpacked.patches().is_some(), "Expected patches for values exceeding bit width" ); @@ -307,7 +296,7 @@ mod test { let unpacked = PrimitiveArray::from_iter(values.clone()); let bitpacked = BitPackedData::encode(&unpacked.into_array(), 7).unwrap(); assert!( - bitpacked.patches(bitpacked.len()).is_some(), + bitpacked.patches().is_some(), "Expected patches for values exceeding bit width" ); diff --git a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs index 900284e926d..314e4418ea1 100644 --- a/encodings/fastlanes/src/bitpacking/compute/is_constant.rs +++ b/encodings/fastlanes/src/bitpacking/compute/is_constant.rs @@ -22,7 +22,6 @@ use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use crate::BitPacked; -use crate::BitPackedArrayExt; use crate::unpack_iter::BitPacked as BitPackedUnpack; /// BitPacked-specific is_constant kernel with SIMD support. @@ -44,7 +43,7 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { return Ok(None); }; - let result = match_each_integer_ptype!(array.dtype().as_ptype(), |P| { + let result = match_each_integer_ptype!(array.ptype(), |P| { bitpacked_is_constant::() }>(array)? }); @@ -55,7 +54,7 @@ impl DynAggregateKernel for BitPackedIsConstantKernel { fn bitpacked_is_constant( array: ArrayView<'_, BitPacked>, ) -> VortexResult { - let mut bit_unpack_iterator = array.unpacked_chunks::()?; + let mut bit_unpack_iterator = array.unpacked_chunks::(); let patches = array.patches().map(|p| { let values = p.values().to_primitive(); let indices = p.indices().to_primitive(); diff --git a/encodings/fastlanes/src/bitpacking/compute/slice.rs b/encodings/fastlanes/src/bitpacking/compute/slice.rs index cc9394953f4..fe702e524b0 100644 --- a/encodings/fastlanes/src/bitpacking/compute/slice.rs +++ b/encodings/fastlanes/src/bitpacking/compute/slice.rs @@ -11,6 +11,7 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::BitPacked; +use crate::BitPackedData; impl SliceReduce for BitPacked { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { @@ -23,24 +24,24 @@ impl SliceReduce for BitPacked { let encoded_start = (block_start / 8) * array.bit_width() as usize; let encoded_stop = (block_stop / 8) * array.bit_width() as usize; - Ok(Some( - BitPacked::try_new( + // slice the buffer using the encoded start/stop values + // SAFETY: slicing packed values without decoding preserves invariants + Ok(Some(unsafe { + BitPackedData::new_unchecked( array.packed().slice(encoded_start..encoded_stop), - array.dtype().as_ptype(), + array.dtype().clone(), + array.validity().slice(range.clone())?, array - .validity(array.dtype().nullability()) - .slice(range.clone())?, - array - .patches(array.len()) + .patches() .map(|p| p.slice(range.clone())) .transpose()? .flatten(), array.bit_width(), range.len(), offset as u16, - )? - .into_array(), - )) + ) + .into_array() + })) } } diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 6d652c42b42..bf73dd1ba05 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -24,7 +24,7 @@ use vortex_error::VortexResult; use super::chunked_indices; use crate::BitPacked; -use crate::BitPackedArrayExt; +use crate::BitPackedData; use crate::bitpack_decompress; // TODO(connor): This is duplicated in `encodings/fastlanes/src/bitpacking/kernels/mod.rs`. @@ -54,24 +54,15 @@ impl TakeExecute for BitPacked { let indices = indices.clone().execute::(ctx)?; let taken = match_each_unsigned_integer_ptype!(ptype.to_unsigned(), |T| { match_each_integer_ptype!(indices.ptype(), |I| { - take_primitive::(array, &indices, taken_validity, ctx)? + take_primitive::(&array, &indices, taken_validity, ctx)? }) }); - let taken = if ptype.is_signed_int() { - PrimitiveArray::from_buffer_handle( - taken.buffer_handle().clone(), - ptype, - taken.validity(), - ) - } else { - taken - }; - Ok(Some(taken.into_array())) + Ok(Some(taken.reinterpret_cast(ptype).into_array())) } } fn take_primitive( - array: ArrayView<'_, BitPacked>, + array: &BitPackedData, indices: &PrimitiveArray, taken_validity: Validity, ctx: &mut ExecutionCtx, @@ -137,13 +128,11 @@ fn take_primitive( } }); - let unpatched_taken = if array.dtype().as_ptype().is_signed_int() { - let primitive = PrimitiveArray::new(output, taken_validity); - PrimitiveArray::from_buffer_handle( - primitive.buffer_handle().clone(), - array.dtype().as_ptype(), - primitive.validity(), - ) + let unpatched_taken = if array.ptype().is_signed_int() { + // Flip back to signed type before patching. + PrimitiveArray::try_from_data( + PrimitiveArray::new(output, taken_validity).reinterpret_cast(array.ptype()), + )? } else { PrimitiveArray::new(output, taken_validity) }; @@ -224,7 +213,7 @@ mod test { let values = (0..u16::MAX as u32 + num_patches as u32).collect::>(); let uncompressed = PrimitiveArray::new(values.clone(), Validity::NonNullable); let packed = BitPackedData::encode(&uncompressed.into_array(), 16).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); let rng = rng(); let range = Uniform::new(0, values.len()).unwrap(); @@ -256,7 +245,7 @@ mod test { BitPackedData::encode(&buffer![1i32, 2i32, 3i32, 4i32].into_array(), 1).unwrap(); let taken_primitive = take_primitive::( - start.as_view(), + &start, &PrimitiveArray::from_iter([0u64, 1, 2, 3]), Validity::NonNullable, &mut LEGACY_SESSION.create_execution_ctx(), diff --git a/encodings/fastlanes/src/bitpacking/mod.rs b/encodings/fastlanes/src/bitpacking/mod.rs index 95d8ef8c608..5a3a6ed0200 100644 --- a/encodings/fastlanes/src/bitpacking/mod.rs +++ b/encodings/fastlanes/src/bitpacking/mod.rs @@ -2,9 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; -pub use array::BitPackedArrayExt; +pub use array::BitPackedArrayParts; pub use array::BitPackedData; -pub use array::BitPackedDataParts; pub use array::bitpack_compress; pub use array::bitpack_decompress; pub use array::unpack_iter; diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 317788ce9c5..1e7ccd08665 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -3,19 +3,20 @@ use std::hash::Hash; -use prost::Message; use vortex_array::AnyCanonical; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; +use vortex_array::SerializeMetadata; use vortex_array::buffer::BufferHandle; use vortex_array::builders::ArrayBuilder; use vortex_array::dtype::DType; @@ -26,6 +27,7 @@ use vortex_array::patches::PatchesMetadata; use vortex_array::require_patches; use vortex_array::require_validity; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -37,9 +39,7 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; -use crate::BitPackedArrayExt; use crate::BitPackedData; -use crate::BitPackedDataParts; use crate::bitpack_decompress::unpack_array; use crate::bitpack_decompress::unpack_into_primitive_builder; use crate::bitpacking::array::NUM_SLOTS; @@ -70,15 +70,29 @@ pub struct BitPackedMetadata { impl VTable for BitPacked { type ArrayData = BitPackedData; + type Metadata = ProstMetadata; + type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &BitPackedData) -> &Self { + &BitPacked + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) + fn len(array: &BitPackedData) -> usize { + array.len + } + + fn dtype(array: &BitPackedData) -> &DType { + &array.dtype + } + + fn stats(array: &BitPackedData) -> &ArrayStats { + &array.stats_set } fn array_hash( @@ -89,64 +103,16 @@ impl VTable for BitPacked { array.offset.hash(state); array.bit_width.hash(state); array.packed.array_hash(state, precision); - match array.patch_indices() { - Some(indices) => { - true.hash(state); - indices.array_hash(state, precision); - } - None => false.hash(state), - } - match array.patch_values() { - Some(values) => { - true.hash(state); - values.array_hash(state, precision); - } - None => false.hash(state), - } - match array.patch_chunk_offsets() { - Some(offsets) => { - true.hash(state); - offsets.array_hash(state, precision); - } - None => false.hash(state), - } - match array.validity_child() { - Some(validity) => { - true.hash(state); - validity.array_hash(state, precision); - } - None => false.hash(state), - } - array.patch_offset.hash(state); - array.patch_offset_within_chunk.hash(state); + array.patches().array_hash(state, precision); + array.validity().array_hash(state, precision); } fn array_eq(array: &BitPackedData, other: &BitPackedData, precision: Precision) -> bool { array.offset == other.offset && array.bit_width == other.bit_width && array.packed.array_eq(&other.packed, precision) - && match (array.patch_indices(), other.patch_indices()) { - (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), - (None, None) => true, - _ => false, - } - && match (array.patch_values(), other.patch_values()) { - (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), - (None, None) => true, - _ => false, - } - && match (array.patch_chunk_offsets(), other.patch_chunk_offsets()) { - (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), - (None, None) => true, - _ => false, - } - && match (array.validity_child(), other.validity_child()) { - (Some(lhs), Some(rhs)) => lhs.array_eq(rhs, precision), - (None, None) => true, - _ => false, - } - && array.patch_offset == other.patch_offset - && array.patch_offset_within_chunk == other.patch_offset_within_chunk + && array.patches().array_eq(&other.patches(), precision) + && array.validity().array_eq(&other.validity(), precision) } fn nbuffers(_array: ArrayView<'_, Self>) -> usize { @@ -201,30 +167,44 @@ impl VTable for BitPacked { Ok(()) } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - BitPackedMetadata { - bit_width: array.bit_width() as u32, - offset: array.offset() as u32, - patches: array - .patches() - .map(|p| p.to_metadata(array.len(), array.dtype())) - .transpose()?, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(BitPackedMetadata { + bit_width: array.bit_width() as u32, + offset: array.offset() as u32, + patches: array + .patches() + .map(|p| p.to_metadata(array.len(), array.dtype())) + .transpose()?, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + let inner = as DeserializeMetadata>::deserialize(bytes)?; + Ok(ProstMetadata(inner)) + } + + /// Deserialize a BitPackedArray from its components. + /// + /// Note that the layout depends on whether patches and chunk_offsets are present: + /// - No patches: `[validity?]` + /// - With patches: `[patch_indices, patch_values, chunk_offsets?, validity?]` + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = BitPackedMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } @@ -294,9 +274,9 @@ impl VTable for BitPacked { builder: &mut dyn ArrayBuilder, ctx: &mut ExecutionCtx, ) -> VortexResult<()> { - match_each_integer_ptype!(array.dtype().as_ptype(), |T| { + match_each_integer_ptype!(array.ptype(), |T| { unpack_into_primitive_builder::( - array, + &array, builder .as_any_mut() .downcast_mut() @@ -314,14 +294,10 @@ impl VTable for BitPacked { PATCH_VALUES_SLOT, PATCH_CHUNK_OFFSETS_SLOT ); - require_validity!( - array, - &array.validity(), - VALIDITY_SLOT => AnyCanonical - ); + require_validity!(array, &array.validity(), VALIDITY_SLOT => AnyCanonical); Ok(ExecutionResult::done( - unpack_array(array.as_view(), ctx)?.into_array(), + unpack_array(&array, ctx)?.into_array(), )) } @@ -341,27 +317,6 @@ pub struct BitPacked; impl BitPacked { pub const ID: ArrayId = ArrayId::new_ref("fastlanes.bitpacked"); - pub fn try_new( - packed: BufferHandle, - ptype: PType, - validity: Validity, - patches: Option, - bit_width: u8, - len: usize, - offset: u16, - ) -> VortexResult { - let dtype = DType::Primitive(ptype, validity.nullability()); - let data = - BitPackedData::try_new(packed, ptype, validity, patches, bit_width, len, offset)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(BitPacked, dtype, len, data)) }) - } - - pub fn into_parts(array: BitPackedArray) -> BitPackedDataParts { - let len = array.len(); - let nullability = array.dtype().nullability(); - array.into_data().into_parts(len, nullability) - } - /// Encode an array into a bitpacked representation with the given bit width. pub fn encode(array: &ArrayRef, bit_width: u8) -> VortexResult { BitPackedData::encode(array, bit_width) diff --git a/encodings/fastlanes/src/bitpacking/vtable/operations.rs b/encodings/fastlanes/src/bitpacking/vtable/operations.rs index 49984c80ade..8dcfa18d08d 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/operations.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/operations.rs @@ -16,12 +16,12 @@ impl OperationsVTable for BitPacked { _ctx: &mut ExecutionCtx, ) -> VortexResult { Ok( - if let Some(patches) = array.patches(array.len()) + if let Some(patches) = array.patches() && let Some(patch) = patches.get_patched(index)? { patch } else { - bitpack_decompress::unpack_single(array, index) + bitpack_decompress::unpack_single(&array, index) }, ) } @@ -141,14 +141,14 @@ mod test { // We create an array that has 1 element that does not fit in the 6-bit range. let array = BitPackedData::encode(&buffer![0u32..=64].into_array(), 6).unwrap(); - assert!(array.patches(array.len()).is_some()); + assert!(array.patches().is_some()); - let patch_indices = array.patches(array.len()).unwrap().indices().clone(); + let patch_indices = array.patches().unwrap().indices().clone(); assert_eq!(patch_indices.len(), 1); // Slicing drops the empty patches array. let sliced_bp = slice_via_reduce(&array, 0..64); - assert!(sliced_bp.patches(sliced_bp.len()).is_none()); + assert!(sliced_bp.patches().is_none()); } #[test] @@ -177,29 +177,30 @@ mod test { #[test] fn scalar_at_invalid_patches() { - let packed_array = BitPacked::try_new( - BufferHandle::new_host(ByteBuffer::copy_from_aligned( - [0u8; 128], - Alignment::of::(), - )), - PType::U32, - Validity::AllInvalid, - Some( - Patches::new( - 8, - 0, - buffer![1u32].into_array(), - PrimitiveArray::new(buffer![999u32], Validity::AllValid).into_array(), - None, - ) - .unwrap(), - ), - 1, - 8, - 0, - ) - .unwrap() - .into_array(); + let packed_array = unsafe { + BitPackedData::new_unchecked( + BufferHandle::new_host(ByteBuffer::copy_from_aligned( + [0u8; 128], + Alignment::of::(), + )), + DType::Primitive(PType::U32, true.into()), + Validity::AllInvalid, + Some( + Patches::new( + 8, + 0, + buffer![1u32].into_array(), + PrimitiveArray::new(buffer![999u32], Validity::AllValid).into_array(), + None, + ) + .unwrap(), + ), + 1, + 8, + 0, + ) + .into_array() + }; assert_eq!( packed_array.scalar_at(1).unwrap(), Scalar::null(DType::Primitive(PType::U32, Nullability::Nullable)) @@ -211,9 +212,9 @@ mod test { let values = (0u32..257).collect::>(); let uncompressed = values.clone().into_array(); let packed = BitPackedData::encode(&uncompressed, 8).unwrap(); - assert!(packed.patches(packed.len()).is_some()); + assert!(packed.patches().is_some()); - let patches = packed.patches(packed.len()).unwrap().indices().clone(); + let patches = packed.patches().unwrap().indices().clone(); assert_eq!( usize::try_from(&patches.scalar_at(0).unwrap()).unwrap(), 256 diff --git a/encodings/fastlanes/src/bitpacking/vtable/validity.rs b/encodings/fastlanes/src/bitpacking/vtable/validity.rs index 2313e0112eb..c9b555eb92c 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/validity.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/validity.rs @@ -10,6 +10,6 @@ use crate::BitPacked; impl ValidityVTable for BitPacked { fn validity(array: ArrayView<'_, BitPacked>) -> VortexResult { - Ok(array.data().validity(array.dtype().nullability())) + Ok(array.data().validity()) } } diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index 0b79489246d..3f7a75d3a82 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -104,7 +104,8 @@ mod tests { use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::Delta; + use crate::DeltaArray; + use crate::DeltaData; use crate::bitpack_compress::bitpack_encode; use crate::delta::array::delta_decompress::delta_decompress; use crate::delta_compress; @@ -119,7 +120,10 @@ mod tests { (0u32..10_000).map(|i| (i % 2 == 0).then_some(i)), ))] fn test_compress(#[case] array: PrimitiveArray) -> VortexResult<()> { - let delta = Delta::try_from_primitive_array(&array, &mut SESSION.create_execution_ctx())?; + let delta = DeltaArray::try_from_data(DeltaData::try_from_primitive_array( + &array, + &mut SESSION.create_execution_ctx(), + )?)?; assert_eq!(delta.len(), array.len()); let decompressed = delta_decompress(&delta, &mut SESSION.create_execution_ctx())?; assert_arrays_eq!(decompressed, array); @@ -136,13 +140,16 @@ mod tests { ); let (bases, deltas) = delta_compress(&array, &mut SESSION.create_execution_ctx()).unwrap(); let bitpacked_deltas = bitpack_encode(&deltas, 1, None).unwrap(); - let packed_delta = Delta::try_new( - bases.into_array(), - bitpacked_deltas.into_array(), - 0, - array.len(), + let packed_delta = DeltaArray::try_from_data( + DeltaData::try_new( + bases.into_array(), + bitpacked_deltas.into_array(), + 0, + array.len(), + ) + .unwrap(), ) - .vortex_expect("Delta array construction should succeed"); + .vortex_expect("DeltaData is always valid"); assert_arrays_eq!(packed_delta.as_array().to_primitive(), array); } } diff --git a/encodings/fastlanes/src/delta/array/mod.rs b/encodings/fastlanes/src/delta/array/mod.rs index 05d9dfbbadb..048b860e000 100644 --- a/encodings/fastlanes/src/delta/array/mod.rs +++ b/encodings/fastlanes/src/delta/array/mod.rs @@ -3,9 +3,13 @@ use fastlanes::FastLanes; use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; +use vortex_array::stats::ArrayStats; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -33,11 +37,11 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; /// use vortex_array::VortexSessionExecute; /// use vortex_array::session::ArraySession; /// use vortex_session::VortexSession; -/// use vortex_fastlanes::Delta; +/// use vortex_fastlanes::DeltaData; /// /// let session = VortexSession::empty().with::(); /// let primitive = PrimitiveArray::from_iter([1_u32, 2, 3, 5, 10, 11]); -/// let array = Delta::try_from_primitive_array(&primitive, &mut session.create_execution_ctx()).unwrap(); +/// let array = DeltaData::try_from_primitive_array(&primitive, &mut session.create_execution_ctx()).unwrap(); /// ``` /// /// # Details @@ -62,10 +66,23 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; #[derive(Clone, Debug)] pub struct DeltaData { pub(super) offset: usize, + pub(super) len: usize, + pub(super) dtype: DType, pub(super) slots: Vec>, + pub(super) stats_set: ArrayStats, } impl DeltaData { + pub fn try_from_primitive_array( + array: &PrimitiveArray, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let logical_len = array.len(); + let (bases, deltas) = delta_compress::delta_compress(array, ctx)?; + + Self::try_new(bases.into_array(), deltas.into_array(), 0, logical_len) + } + /// Create a DeltaArray from the given `bases` and `deltas` arrays /// with given `offset` into first chunk and `logical_len` length. pub fn try_new( @@ -74,31 +91,6 @@ impl DeltaData { offset: usize, len: usize, ) -> VortexResult { - Self::validate_parts(&bases, &deltas, offset, len)?; - - // SAFETY: validation done above - Ok(unsafe { Self::new_unchecked(bases, deltas, offset) }) - } - - pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { - Self::validate_parts(self.bases(), self.deltas(), self.offset, len)?; - let expected_dtype = self - .bases() - .dtype() - .with_nullability(self.deltas().dtype().nullability()); - vortex_ensure!( - dtype == &expected_dtype, - "DeltaArray dtype mismatch: expected {expected_dtype}, got {dtype}" - ); - Ok(()) - } - - fn validate_parts( - bases: &ArrayRef, - deltas: &ArrayRef, - offset: usize, - len: usize, - ) -> VortexResult<()> { vortex_ensure!(offset < 1024, "offset must be less than 1024: {offset}"); vortex_ensure!( offset + len <= deltas.len(), @@ -113,8 +105,8 @@ impl DeltaData { ); vortex_ensure!( - bases.dtype().is_unsigned_int(), - "DeltaArray: dtype must be an unsigned integer, got {}", + bases.dtype().is_int(), + "DeltaArray: dtype must be an integer, got {}", bases.dtype() ); @@ -130,13 +122,24 @@ impl DeltaData { "bases length ({}) must be a multiple of LANES ({lanes})", bases.len(), ); - Ok(()) + + // SAFETY: validation done above + Ok(unsafe { Self::new_unchecked(bases, deltas, offset, len) }) } - pub(crate) unsafe fn new_unchecked(bases: ArrayRef, deltas: ArrayRef, offset: usize) -> Self { + pub(crate) unsafe fn new_unchecked( + bases: ArrayRef, + deltas: ArrayRef, + offset: usize, + logical_len: usize, + ) -> Self { + let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); Self { offset, + len: logical_len, + dtype, slots: vec![Some(bases), Some(deltas)], + stats_set: Default::default(), } } @@ -154,6 +157,25 @@ impl DeltaData { .vortex_expect("DeltaArray deltas slot") } + pub(crate) fn lanes(&self) -> usize { + lane_count(self.dtype().as_ptype()) + } + + #[inline] + pub fn len(&self) -> usize { + self.len + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + #[inline] + pub fn dtype(&self) -> &DType { + &self.dtype + } + #[inline] /// The logical offset into the first chunk of [`Self::deltas`]. pub fn offset(&self) -> usize { @@ -167,6 +189,10 @@ impl DeltaData { pub(crate) fn deltas_len(&self) -> usize { self.deltas().len() } + + pub(crate) fn stats_set(&self) -> &ArrayStats { + &self.stats_set + } } pub(crate) fn lane_count(ptype: PType) -> usize { diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index a9af6d09d60..46575c5dc6b 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -11,6 +11,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use vortex_error::vortex_panic; +use crate::DeltaData; use crate::delta::Delta; impl CastReduce for Delta { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -37,7 +38,8 @@ impl CastReduce for Delta { // Create a new DeltaArray with the casted components, preserving offset and logical length Ok(Some( - Delta::try_new(casted_bases, casted_deltas, array.offset(), array.len())?.into_array(), + DeltaData::try_new(casted_bases, casted_deltas, array.offset(), array.len())? + .into_array(), )) } } @@ -60,7 +62,7 @@ mod tests { use vortex_buffer::buffer; use vortex_session::VortexSession; - use crate::Delta; + use crate::DeltaData; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); @@ -68,7 +70,7 @@ mod tests { fn test_cast_delta_u8_to_u32() { let primitive = PrimitiveArray::from_iter([10u8, 20, 30, 40, 50]); let array = - Delta::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) + DeltaData::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) .unwrap(); let casted = array @@ -93,7 +95,8 @@ mod tests { vortex_array::validity::Validity::NonNullable, ); let array = - Delta::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx()).unwrap(); + DeltaData::try_from_primitive_array(&values, &mut SESSION.create_execution_ctx()) + .unwrap(); let casted = array .into_array() @@ -132,7 +135,7 @@ mod tests { )] fn test_cast_delta_conformance(#[case] primitive: PrimitiveArray) { let delta_array = - Delta::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) + DeltaData::try_from_primitive_array(&primitive, &mut SESSION.create_execution_ctx()) .unwrap(); test_cast_conformance(&delta_array.into_array()); } diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index 746221a5b1e..887de327a82 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -9,19 +9,20 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_error::VortexResult; @@ -54,15 +55,29 @@ pub struct DeltaMetadata { impl VTable for Delta { type ArrayData = DeltaData; + type Metadata = ProstMetadata; + type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &DeltaData) -> &Self { + &Delta + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) + fn len(array: &DeltaData) -> usize { + array.len() + } + + fn dtype(array: &DeltaData) -> &DType { + array.dtype() + } + + fn stats(array: &DeltaData) -> &ArrayStats { + array.stats_set() } fn array_hash(array: &DeltaData, state: &mut H, precision: Precision) { @@ -116,42 +131,41 @@ impl VTable for Delta { Ok(()) } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - DeltaMetadata { - deltas_len: array.deltas().len() as u64, - offset: array.offset() as u32, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(DeltaMetadata { + deltas_len: array.deltas().len() as u64, + offset: array.offset() as u32, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.0.encode_to_vec())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata(DeltaMetadata::decode(bytes)?)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - buffers: &[BufferHandle], + metadata: &Self::Metadata, + _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - vortex_ensure!( - buffers.is_empty(), - "DeltaArray expects 0 buffers, got {}", - buffers.len() - ); - vortex_ensure!( - children.len() == 2, - "DeltaArray expects 2 children, got {}", - children.len() - ); - let metadata = DeltaMetadata::decode(metadata)?; + assert_eq!(children.len(), 2); let ptype = PType::try_from(dtype)?; let lanes = match_each_unsigned_integer_ptype!(ptype, |T| { ::LANES }); // Compute the length of the bases array - let deltas_len = usize::try_from(metadata.deltas_len) - .map_err(|_| vortex_err!("deltas_len {} overflowed usize", metadata.deltas_len))?; + let deltas_len = usize::try_from(metadata.0.deltas_len) + .map_err(|_| vortex_err!("deltas_len {} overflowed usize", metadata.0.deltas_len))?; let num_chunks = deltas_len / 1024; let remainder_base_size = if deltas_len % 1024 > 0 { 1 } else { 0 }; let bases_len = num_chunks * lanes + remainder_base_size; @@ -159,7 +173,7 @@ impl VTable for Delta { let bases = children.get(0, dtype, bases_len)?; let deltas = children.get(1, dtype, deltas_len)?; - DeltaData::try_new(bases, deltas, metadata.offset as usize, len) + DeltaData::try_new(bases, deltas, metadata.0.offset as usize, len) } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { @@ -175,45 +189,31 @@ pub struct Delta; impl Delta { pub const ID: ArrayId = ArrayId::new_ref("fastlanes.delta"); - pub fn try_new( - bases: ArrayRef, - deltas: ArrayRef, - offset: usize, - len: usize, - ) -> VortexResult { - let dtype = bases.dtype().with_nullability(deltas.dtype().nullability()); - let data = DeltaData::try_new(bases, deltas, offset, len)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Delta, dtype, len, data)) }) - } - /// Compress a primitive array using Delta encoding. pub fn try_from_primitive_array( array: &PrimitiveArray, ctx: &mut ExecutionCtx, ) -> VortexResult { - let logical_len = array.len(); - let (bases, deltas) = crate::delta::array::delta_compress::delta_compress(array, ctx)?; - Self::try_new(bases.into_array(), deltas.into_array(), 0, logical_len) + Array::try_from_data(DeltaData::try_from_primitive_array(array, ctx)?) } } #[cfg(test)] mod tests { - use prost::Message; use vortex_array::test_harness::check_metadata; use super::DeltaMetadata; + use super::ProstMetadata; #[cfg_attr(miri, ignore)] #[test] fn test_delta_metadata() { check_metadata( "delta.metadata", - &DeltaMetadata { + ProstMetadata(DeltaMetadata { offset: u32::MAX, deltas_len: u64::MAX, - } - .encode_to_vec(), + }), ); } } diff --git a/encodings/fastlanes/src/delta/vtable/operations.rs b/encodings/fastlanes/src/delta/vtable/operations.rs index 731588a67bb..cae9b701ee9 100644 --- a/encodings/fastlanes/src/delta/vtable/operations.rs +++ b/encodings/fastlanes/src/delta/vtable/operations.rs @@ -38,15 +38,18 @@ mod tests { use vortex_error::VortexExpect; use vortex_session::VortexSession; - use crate::Delta; use crate::DeltaArray; + use crate::DeltaData; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); fn da(array: &PrimitiveArray) -> DeltaArray { - Delta::try_from_primitive_array(array, &mut SESSION.create_execution_ctx()) - .vortex_expect("Delta array construction should succeed") + DeltaArray::try_from_data( + DeltaData::try_from_primitive_array(array, &mut SESSION.create_execution_ctx()) + .unwrap(), + ) + .vortex_expect("DeltaData is always valid") } #[test] diff --git a/encodings/fastlanes/src/delta/vtable/slice.rs b/encodings/fastlanes/src/delta/vtable/slice.rs index c836bed33fd..3068569d3c0 100644 --- a/encodings/fastlanes/src/delta/vtable/slice.rs +++ b/encodings/fastlanes/src/delta/vtable/slice.rs @@ -10,7 +10,7 @@ use vortex_array::IntoArray; use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; -use crate::delta::array::lane_count; +use crate::DeltaData; use crate::delta::vtable::Delta; impl SliceReduce for Delta { @@ -23,7 +23,7 @@ impl SliceReduce for Delta { let bases = array.bases(); let deltas = array.deltas(); - let lanes = lane_count(array.dtype().as_ptype()); + let lanes = array.lanes(); let new_bases = bases.slice( min(start_chunk * lanes, array.bases_len())..min(stop_chunk * lanes, array.bases_len()), @@ -33,8 +33,10 @@ impl SliceReduce for Delta { min(start_chunk * 1024, array.deltas_len())..min(stop_chunk * 1024, array.deltas_len()), )?; - Ok(Some( - Delta::try_new(new_bases, new_deltas, physical_start % 1024, range.len())?.into_array(), - )) + // SAFETY: slicing valid bases/deltas preserves correctness + Ok(Some(unsafe { + DeltaData::new_unchecked(new_bases, new_deltas, physical_start % 1024, range.len()) + .into_array() + })) } } diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index 86bb6de2947..b911e20fa17 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -5,18 +5,20 @@ use num_traits::PrimInt; use num_traits::WrappingSub; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::primitive::PrimitiveData; use vortex_array::dtype::NativePType; use vortex_array::expr::stats::Stat; use vortex_array::match_each_integer_ptype; +use vortex_array::stats::ArrayStats; use vortex_error::VortexResult; use vortex_error::vortex_err; -use crate::FoR; use crate::FoRArray; use crate::FoRData; impl FoRData { pub fn encode(array: PrimitiveArray) -> VortexResult { let array_ref = array.clone().into_array(); + let stats = ArrayStats::from(array_ref.statistics().to_owned()); let min = array_ref .statistics() .compute_stat(Stat::Min)? @@ -25,17 +27,24 @@ impl FoRData { let encoded = match_each_integer_ptype!(array.ptype(), |T| { compress_primitive::(array, T::try_from(&min)?)?.into_array() }); - FoR::try_new(encoded, min) + let for_data = FoRData::try_new(encoded, min)?; + let for_array = FoRArray::try_from_data(for_data)?; + let for_ref = for_array.clone().into_array(); + for_array + .stats_set() + .to_ref(&for_ref) + .inherit_from(stats.to_ref(&for_ref)); + Ok(for_array) } } fn compress_primitive( parray: PrimitiveArray, min: T, -) -> VortexResult { +) -> VortexResult { // Set null values to the min value, ensuring that decompress into a value in the primitive // range (and stop them wrapping around). - let encoded = parray + parray .into_data() .map_each_with_validity::(|(v, bool)| { if bool { @@ -43,12 +52,7 @@ fn compress_primitive( } else { T::zero() } - })?; - Ok(PrimitiveArray::from_buffer_handle( - encoded.buffer_handle().clone(), - encoded.ptype(), - encoded.validity(), - )) + }) } #[cfg(test)] @@ -69,6 +73,7 @@ mod test { use super::*; use crate::BitPackedData; + use crate::FoRArray; use crate::r#for::array::for_decompress::decompress; use crate::r#for::array::for_decompress::fused_decompress; @@ -132,7 +137,7 @@ mod test { let expect = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7 + 10)); let array = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7)); let bp = BitPackedData::encode(&array.into_array(), 3).unwrap(); - let compressed = FoR::try_new(bp.into_array(), 10u32.into()).unwrap(); + let compressed = FoRData::try_new(bp.into_array(), 10u32.into()).unwrap(); assert_arrays_eq!(compressed, expect); } @@ -142,7 +147,9 @@ mod test { let expect = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7 + 10)); let array = PrimitiveArray::from_iter((0u32..1024).map(|x| x % 7)); let bp = BitPackedData::encode(&array.into_array(), 2).unwrap(); - let compressed = FoR::try_new(bp.clone().into_array(), 10u32.into())?; + let compressed = FoRArray::try_from_data( + FoRData::try_new(bp.clone().into_array(), 10u32.into()).unwrap(), + )?; let decompressed = fused_decompress::( &compressed, bp.as_view(), @@ -171,7 +178,7 @@ mod test { .reinterpret_cast(PType::U8); let unsigned: Vec = (0..=u8::MAX).collect_vec(); let expected_unsigned = PrimitiveArray::from_iter(unsigned); - assert_eq!(encoded.as_slice::(), expected_unsigned.as_slice::()); + assert_arrays_eq!(encoded, expected_unsigned); let decompressed = decompress(&compressed, &mut SESSION.create_execution_ctx())?; array diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index 82a84b73fa4..13de40e6365 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -18,7 +18,6 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use crate::BitPacked; -use crate::BitPackedArrayExt; use crate::FoRArray; use crate::bitpack_decompress; use crate::unpack_iter::UnpackStrategy; @@ -93,13 +92,13 @@ pub(crate) fn fused_decompress< let strategy = FoRStrategy { reference: ref_ }; // Create [`UnpackedChunks`] with FoR strategy. - let mut unpacked = UnpackedChunks::try_new_with_strategy( + let mut unpacked = UnpackedChunks::new_with_strategy( strategy, bp.packed().as_host().clone(), bp.bit_width() as usize, bp.offset() as usize, bp.len(), - )?; + ); let mut builder = PrimitiveBuilder::::with_capacity( for_.reference_scalar().dtype().nullability(), diff --git a/encodings/fastlanes/src/for/array/mod.rs b/encodings/fastlanes/src/for/array/mod.rs index f2cd3108250..74d88580bd3 100644 --- a/encodings/fastlanes/src/for/array/mod.rs +++ b/encodings/fastlanes/src/for/array/mod.rs @@ -5,9 +5,10 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; +use vortex_array::stats::ArrayStats; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; -use vortex_error::vortex_ensure; +use vortex_error::vortex_bail; pub mod for_compress; pub mod for_decompress; @@ -25,46 +26,33 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; pub struct FoRData { pub(super) slots: Vec>, pub(super) reference: Scalar, + pub(super) stats_set: ArrayStats, } impl FoRData { - pub(crate) fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { - Self::validate_parts(&encoded, &reference, reference.dtype(), encoded.len())?; + pub fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { + if reference.is_null() { + vortex_bail!("Reference value cannot be null"); + } + let reference = reference.cast( + &reference + .dtype() + .with_nullability(encoded.dtype().nullability()), + )?; Ok(Self { slots: vec![Some(encoded)], reference, + stats_set: Default::default(), }) } - pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { - Self::validate_parts(self.encoded(), &self.reference, dtype, len) - } - - fn validate_parts( - encoded: &ArrayRef, - reference: &Scalar, - dtype: &DType, - len: usize, - ) -> VortexResult<()> { - vortex_ensure!(!reference.is_null(), "Reference value cannot be null"); - vortex_ensure!(dtype.is_int(), "FoR requires an integer dtype, got {dtype}"); - vortex_ensure!( - reference.dtype() == dtype, - "FoR reference dtype mismatch: expected {dtype}, got {}", - reference.dtype() - ); - vortex_ensure!( - encoded.dtype() == dtype, - "FoR encoded dtype mismatch: expected {dtype}, got {}", - encoded.dtype() - ); - vortex_ensure!( - encoded.len() == len, - "FoR encoded length mismatch: expected {len}, got {}", - encoded.len() - ); - Ok(()) + pub(crate) unsafe fn new_unchecked(encoded: ArrayRef, reference: Scalar) -> Self { + Self { + slots: vec![Some(encoded)], + reference, + stats_set: Default::default(), + } } /// Returns the length of the array. @@ -101,4 +89,8 @@ impl FoRData { pub fn reference_scalar(&self) -> &Scalar { &self.reference } + + pub(crate) fn stats_set(&self) -> &ArrayStats { + &self.stats_set + } } diff --git a/encodings/fastlanes/src/for/compute/cast.rs b/encodings/fastlanes/src/for/compute/cast.rs index e37b25b64a1..884a0a8b94f 100644 --- a/encodings/fastlanes/src/for/compute/cast.rs +++ b/encodings/fastlanes/src/for/compute/cast.rs @@ -9,6 +9,7 @@ use vortex_array::dtype::DType; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; +use crate::FoRData; use crate::r#for::FoR; impl CastReduce for FoR { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { @@ -22,7 +23,7 @@ impl CastReduce for FoR { let casted_reference = array.reference_scalar().cast(dtype)?; Ok(Some( - FoR::try_new(casted_child, casted_reference)?.into_array(), + FoRData::try_new(casted_child, casted_reference)?.into_array(), )) } } @@ -43,11 +44,12 @@ mod tests { use vortex_buffer::buffer; use vortex_error::VortexExpect; - use crate::FoR; use crate::FoRArray; + use crate::FoRData; fn for_arr(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") + FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) + .vortex_expect("FoRData is always valid") } #[test] diff --git a/encodings/fastlanes/src/for/compute/compare.rs b/encodings/fastlanes/src/for/compute/compare.rs index 853bf4c2955..f23ec8c0256 100644 --- a/encodings/fastlanes/src/for/compute/compare.rs +++ b/encodings/fastlanes/src/for/compute/compare.rs @@ -23,6 +23,7 @@ use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use crate::FoR; +use crate::FoRData; impl CompareKernel for FoR { fn compare( @@ -36,7 +37,7 @@ impl CompareKernel for FoR { { match_each_integer_ptype!(constant.ptype(), |T| { return compare_constant( - lhs, + &lhs, constant .typed_value::() .vortex_expect("null scalar handled in adaptor"), @@ -51,7 +52,7 @@ impl CompareKernel for FoR { } fn compare_constant( - lhs: ArrayView<'_, FoR>, + lhs: &FoRData, mut rhs: T, nullability: Nullability, operator: CompareOperator, @@ -98,11 +99,12 @@ mod tests { use vortex_buffer::buffer; use super::*; - use crate::FoR; use crate::FoRArray; + use crate::FoRData; fn for_arr(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") + FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) + .vortex_expect("FoRData is always valid") } #[test] @@ -114,18 +116,13 @@ mod tests { reference, ); - let result = compare_constant( - lhs.as_view(), - 30i32, - Nullability::NonNullable, - CompareOperator::Eq, - ) - .unwrap() - .unwrap(); + let result = compare_constant(&lhs, 30i32, Nullability::NonNullable, CompareOperator::Eq) + .unwrap() + .unwrap(); assert_arrays_eq!(result, BoolArray::from_iter([false, true, false].map(Some))); let result = compare_constant( - lhs.as_view(), + &lhs, 12i32, Nullability::NonNullable, CompareOperator::NotEq, @@ -141,7 +138,7 @@ mod tests { CompareOperator::Gte, ] { assert!( - compare_constant(lhs.as_view(), 30i32, Nullability::NonNullable, op) + compare_constant(&lhs, 30i32, Nullability::NonNullable, op) .unwrap() .is_none() ); @@ -158,27 +155,17 @@ mod tests { ); assert_eq!( - compare_constant( - lhs.as_view(), - 30i32, - Nullability::Nullable, - CompareOperator::Eq, - ) - .unwrap() - .unwrap() - .dtype(), + compare_constant(&lhs, 30i32, Nullability::Nullable, CompareOperator::Eq) + .unwrap() + .unwrap() + .dtype(), &DType::Bool(Nullability::Nullable) ); assert_eq!( - compare_constant( - lhs.as_view(), - 30i32, - Nullability::NonNullable, - CompareOperator::Eq, - ) - .unwrap() - .unwrap() - .dtype(), + compare_constant(&lhs, 30i32, Nullability::NonNullable, CompareOperator::Eq) + .unwrap() + .unwrap() + .dtype(), &DType::Bool(Nullability::NonNullable) ); } @@ -192,21 +179,16 @@ mod tests { reference, ); - let result = compare_constant( - lhs.as_view(), - -1i32, - Nullability::NonNullable, - CompareOperator::Eq, - ) - .unwrap() - .unwrap(); + let result = compare_constant(&lhs, -1i32, Nullability::NonNullable, CompareOperator::Eq) + .unwrap() + .unwrap(); assert_arrays_eq!( result, BoolArray::from_iter([false, false, false].map(Some)) ); let result = compare_constant( - lhs.as_view(), + &lhs, -1i32, Nullability::NonNullable, CompareOperator::NotEq, @@ -230,7 +212,7 @@ mod tests { ); let result = compare_constant( - lhs.as_view(), + &lhs, 435090932899640449i64, Nullability::Nullable, CompareOperator::Eq, @@ -240,7 +222,7 @@ mod tests { assert_arrays_eq!(result, BoolArray::from_iter([Some(false), Some(true)])); let result = compare_constant( - lhs.as_view(), + &lhs, 435090932899640449i64, Nullability::Nullable, CompareOperator::NotEq, diff --git a/encodings/fastlanes/src/for/compute/is_sorted.rs b/encodings/fastlanes/src/for/compute/is_sorted.rs index 95504cdb7c7..05d5e31836e 100644 --- a/encodings/fastlanes/src/for/compute/is_sorted.rs +++ b/encodings/fastlanes/src/for/compute/is_sorted.rs @@ -10,7 +10,6 @@ use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; use vortex_array::aggregate_fn::fns::is_sorted::is_sorted; use vortex_array::aggregate_fn::fns::is_sorted::is_strict_sorted; use vortex_array::aggregate_fn::kernels::DynAggregateKernel; -use vortex_array::arrays::PrimitiveArray; use vortex_array::scalar::Scalar; use vortex_error::VortexResult; @@ -35,12 +34,9 @@ impl DynAggregateKernel for FoRIsSortedKernel { }; let encoded = array.encoded().to_primitive(); - let unsigned_array = PrimitiveArray::from_buffer_handle( - encoded.buffer_handle().clone(), - encoded.ptype().to_unsigned(), - encoded.validity(), - ) - .into_array(); + let unsigned_array = encoded + .reinterpret_cast(encoded.ptype().to_unsigned()) + .into_array(); let result = if options.strict { is_strict_sorted(&unsigned_array, ctx)? diff --git a/encodings/fastlanes/src/for/compute/mod.rs b/encodings/fastlanes/src/for/compute/mod.rs index 8bf7e93d029..7704dc02ff5 100644 --- a/encodings/fastlanes/src/for/compute/mod.rs +++ b/encodings/fastlanes/src/for/compute/mod.rs @@ -16,6 +16,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::FoR; +use crate::FoRData; impl TakeExecute for FoR { fn take( @@ -24,7 +25,7 @@ impl TakeExecute for FoR { _ctx: &mut ExecutionCtx, ) -> VortexResult> { Ok(Some( - FoR::try_new( + FoRData::try_new( array.encoded().take(indices.clone())?, array.reference_scalar().clone(), )? @@ -35,7 +36,7 @@ impl TakeExecute for FoR { impl FilterReduce for FoR { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { - FoR::try_new( + FoRData::try_new( array.encoded().filter(mask.clone())?, array.reference_scalar().clone(), ) @@ -54,11 +55,12 @@ mod test { use vortex_buffer::buffer; use vortex_error::VortexExpect; - use crate::FoR; use crate::FoRArray; + use crate::FoRData; fn fa(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") + FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) + .vortex_expect("FoRData is always valid") } #[test] @@ -108,11 +110,12 @@ mod tests { use vortex_buffer::buffer; use vortex_error::VortexExpect; - use crate::FoR; use crate::FoRArray; + use crate::FoRData; fn fa(encoded: ArrayRef, reference: Scalar) -> FoRArray { - FoR::try_new(encoded, reference).vortex_expect("FoR array construction should succeed") + FoRArray::try_from_data(FoRData::try_new(encoded, reference).unwrap()) + .vortex_expect("FoRData is always valid") } #[rstest] diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index 157645c410f..80a2b165b91 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -8,7 +8,6 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -21,6 +20,7 @@ use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTableFromChild; @@ -48,15 +48,29 @@ vtable!(FoR, FoR, FoRData); impl VTable for FoR { type ArrayData = FoRData; + type Metadata = Scalar; + type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; + fn vtable(_array: &FoRData) -> &Self { + &FoR + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) + fn len(array: &FoRData) -> usize { + array.encoded().len() + } + + fn dtype(array: &FoRData) -> &DType { + array.reference_scalar().dtype() + } + + fn stats(array: &FoRData) -> &ArrayStats { + array.stats_set() } fn array_hash(array: &FoRData, state: &mut H, precision: Precision) { @@ -100,27 +114,33 @@ impl VTable for FoR { Ok(()) } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(array.reference_scalar().clone()) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { // Note that we **only** serialize the optional scalar value (not including the dtype). - Ok(Some(ScalarValue::to_proto_bytes( - array.reference_scalar().value(), - ))) + Ok(Some(ScalarValue::to_proto_bytes(metadata.value()))) } fn deserialize( - &self, + bytes: &[u8], + dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + session: &VortexSession, + ) -> VortexResult { + let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?; + Scalar::try_new(dtype.clone(), scalar_value) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - buffers: &[BufferHandle], + metadata: &Self::Metadata, + _buffers: &[BufferHandle], children: &dyn ArrayChildren, - session: &VortexSession, ) -> VortexResult { - vortex_ensure!( - buffers.is_empty(), - "FoRArray expects 0 buffers, got {}", - buffers.len() - ); if children.len() != 1 { vortex_bail!( "Expected 1 child for FoR encoding, found {}", @@ -128,11 +148,9 @@ impl VTable for FoR { ) } - let scalar_value = ScalarValue::from_proto_bytes(metadata, dtype, session)?; - let reference = Scalar::try_new(dtype.clone(), scalar_value)?; let encoded = children.get(0, dtype, len)?; - FoRData::try_new(encoded, reference) + FoRData::try_new(encoded, metadata.clone()) } fn reduce_parent( @@ -165,14 +183,7 @@ impl FoR { /// Construct a new FoR array from an encoded array and a reference scalar. pub fn try_new(encoded: ArrayRef, reference: Scalar) -> VortexResult { - vortex_ensure!(!reference.is_null(), "Reference value cannot be null"); - let dtype = reference - .dtype() - .with_nullability(encoded.dtype().nullability()); - let reference = reference.cast(&dtype)?; - let len = encoded.len(); - let data = FoRData::try_new(encoded, reference)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(FoR, dtype, len, data)) }) + Array::try_from_data(FoRData::try_new(encoded, reference)?) } /// Encode a primitive array using Frame of Reference encoding. @@ -180,16 +191,3 @@ impl FoR { FoRData::encode(array) } } - -#[cfg(test)] -mod tests { - use vortex_array::scalar::ScalarValue; - use vortex_array::test_harness::check_metadata; - - #[cfg_attr(miri, ignore)] - #[test] - fn test_for_metadata() { - let metadata: Vec = ScalarValue::to_proto_bytes(Some(&ScalarValue::from(i64::MAX))); - check_metadata("for.metadata", &metadata); - } -} diff --git a/encodings/fastlanes/src/for/vtable/rules.rs b/encodings/fastlanes/src/for/vtable/rules.rs index 3f839fe74dd..38d4b4186f2 100644 --- a/encodings/fastlanes/src/for/vtable/rules.rs +++ b/encodings/fastlanes/src/for/vtable/rules.rs @@ -13,6 +13,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduceAdaptor; use vortex_error::VortexResult; use crate::FoR; +use crate::FoRData; pub(super) const PARENT_RULES: ParentRuleSet = ParentRuleSet::new(&[ // TODO: add BetweenReduceAdaptor(FoR) @@ -34,12 +35,12 @@ impl ArrayParentReduceRule for FoRFilterPushDownRule { parent: ArrayView<'_, Filter>, _child_idx: usize, ) -> VortexResult> { - Ok(Some( - FoR::try_new( + let new_array = unsafe { + FoRData::new_unchecked( child.encoded().filter(parent.filter_mask().clone())?, - child.reference_scalar().clone(), - )? - .into_array(), - )) + child.reference.clone(), + ) + }; + Ok(Some(new_array.into_array())) } } diff --git a/encodings/fastlanes/src/for/vtable/slice.rs b/encodings/fastlanes/src/for/vtable/slice.rs index acd8a4d0254..c7571ac274a 100644 --- a/encodings/fastlanes/src/for/vtable/slice.rs +++ b/encodings/fastlanes/src/for/vtable/slice.rs @@ -10,15 +10,17 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::FoR; +use crate::FoRData; impl SliceReduce for FoR { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some( - FoR::try_new( + // SAFETY: Just slicing encoded data does not affect FOR. + Ok(Some(unsafe { + FoRData::new_unchecked( array.encoded().slice(range)?, array.reference_scalar().clone(), - )? - .into_array(), - )) + ) + .into_array() + })) } } diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index 2ee4a082dfa..d0d207c1795 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -3,8 +3,8 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; +use vortex_array::stats::ArrayStats; use vortex_error::VortexExpect as _; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -33,9 +33,12 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["values", "indices", "values_i #[derive(Clone, Debug)] pub struct RLEData { + pub(super) dtype: DType, pub(super) slots: Vec>, + pub(super) stats_set: ArrayStats, // Offset relative to the start of the chunk. pub(super) offset: usize, + pub(super) length: usize, } impl RLEData { @@ -44,37 +47,6 @@ impl RLEData { indices: &ArrayRef, value_idx_offsets: &ArrayRef, offset: usize, - length: usize, - ) -> VortexResult<()> { - Self::validate_parts(values, indices, value_idx_offsets, offset, length)?; - Ok(()) - } - - pub(crate) fn validate_against_outer(&self, dtype: &DType, length: usize) -> VortexResult<()> { - Self::validate_parts( - self.values(), - self.indices(), - self.values_idx_offsets(), - self.offset, - length, - )?; - let expected_dtype = DType::Primitive( - self.values().dtype().as_ptype(), - self.indices().dtype().nullability(), - ); - vortex_ensure!( - dtype == &expected_dtype, - "RLE dtype mismatch: expected {expected_dtype}, got {dtype}" - ); - Ok(()) - } - - fn validate_parts( - values: &ArrayRef, - indices: &ArrayRef, - value_idx_offsets: &ArrayRef, - offset: usize, - length: usize, ) -> VortexResult<()> { vortex_ensure!( offset < 1024, @@ -83,11 +55,8 @@ impl RLEData { ); vortex_ensure!( - matches!( - values.dtype(), - DType::Primitive(_, Nullability::NonNullable) - ), - "RLE values must be a non-nullable primitive type, got {}", + values.dtype().is_primitive(), + "RLE values must be a primitive type, got {}", values.dtype() ); @@ -102,16 +71,6 @@ impl RLEData { "RLE value idx offsets must be non-nullable unsigned integer, got {}", value_idx_offsets.dtype() ); - vortex_ensure!( - indices.len().is_multiple_of(FL_CHUNK_SIZE), - "RLE indices length must be a multiple of {FL_CHUNK_SIZE}, got {}", - indices.len() - ); - vortex_ensure!( - offset + length <= indices.len(), - "RLE offset + length, {offset} + {length}, must not exceed the indices length {}", - indices.len() - ); vortex_ensure!( indices.len().div_ceil(FL_CHUNK_SIZE) == value_idx_offsets.len(), @@ -145,11 +104,18 @@ impl RLEData { offset: usize, length: usize, ) -> VortexResult { - Self::validate(&values, &indices, &values_idx_offsets, offset, length)?; + assert_eq!(indices.len() % FL_CHUNK_SIZE, 0); + Self::validate(&values, &indices, &values_idx_offsets, offset)?; + + // Ensure that the DType has the same nullability as the indices array. + let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); Ok(Self { + dtype, slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], + stats_set: ArrayStats::default(), offset, + length, }) } @@ -158,20 +124,42 @@ impl RLEData { /// # Safety /// The caller must ensure that: /// - `offset + length` does not exceed the length of the indices array + /// - The `dtype` is consistent with the values array's primitive type and validity nullability /// - The `indices` array contains valid indices into chunks of the `values` array /// - The `values_idx_offsets` array contains valid chunk start offsets + /// - The `validity` array has the same length as `length` pub unsafe fn new_unchecked( values: ArrayRef, indices: ArrayRef, values_idx_offsets: ArrayRef, + dtype: DType, offset: usize, + length: usize, ) -> Self { Self { + dtype, slots: vec![Some(values), Some(indices), Some(values_idx_offsets)], + stats_set: ArrayStats::default(), offset, + length, } } + #[inline] + pub fn len(&self) -> usize { + self.length + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.length == 0 + } + + #[inline] + pub fn dtype(&self) -> &DType { + &self.dtype + } + #[inline] pub fn values(&self) -> &ArrayRef { self.slots[VALUES_SLOT] @@ -223,6 +211,10 @@ impl RLEData { pub fn offset(&self) -> usize { self.offset } + + pub(crate) fn stats_set(&self) -> &ArrayStats { + &self.stats_set + } } #[cfg(test)] @@ -237,8 +229,8 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; + use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; - use vortex_array::serde::SerializedArray; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::ByteBufferMut; @@ -247,7 +239,7 @@ mod tests { use vortex_session::registry::ReadContext; use crate::FL_CHUNK_SIZE; - use crate::RLE; + use crate::RLEArray; use crate::RLEData; use crate::test::SESSION; @@ -260,8 +252,10 @@ mod tests { PrimitiveArray::from_iter([0u16, 0, 1, 1, 2].iter().cycle().take(1024).copied()) .into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64]).into_array(); - let rle_array = RLE::try_new(values, indices, values_idx_offsets, 0, 5) - .vortex_expect("RLEData is always valid"); + let rle_array = RLEArray::try_from_data( + RLEData::try_new(values, indices, values_idx_offsets, 0, 5).unwrap(), + ) + .vortex_expect("RLEData is always valid"); assert_eq!(rle_array.len(), 5); assert_eq!(rle_array.values().len(), 3); @@ -288,8 +282,10 @@ mod tests { ) .into_array(); - let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 3) - .vortex_expect("RLEData is always valid"); + let rle_array = RLEArray::try_from_data( + RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 3).unwrap(), + ) + .vortex_expect("RLEData is always valid"); assert_eq!(rle_array.len(), 3); assert_eq!(rle_array.values().len(), 2); @@ -318,8 +314,10 @@ mod tests { ) .into_array(); - let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 5) - .vortex_expect("RLEData is always valid"); + let rle_array = RLEArray::try_from_data( + RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 5).unwrap(), + ) + .vortex_expect("RLEData is always valid"); let valid_slice = rle_array.slice(0..3).unwrap().to_primitive(); // TODO(joe): replace with compute null count @@ -349,8 +347,10 @@ mod tests { ) .into_array(); - let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 5) - .vortex_expect("RLEData is always valid"); + let rle_array = RLEArray::try_from_data( + RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 5).unwrap(), + ) + .vortex_expect("RLEData is always valid"); // TODO(joe): replace with compute null count let invalid_slice = rle_array @@ -385,8 +385,10 @@ mod tests { ) .into_array(); - let rle_array = RLE::try_new(values, indices_with_validity, values_idx_offsets, 0, 4) - .vortex_expect("RLEData is always valid"); + let rle_array = RLEArray::try_from_data( + RLEData::try_new(values, indices_with_validity, values_idx_offsets, 0, 4).unwrap(), + ) + .vortex_expect("RLEData is always valid"); let sliced_array = rle_array.slice(1..4).unwrap(); let validity_mask = sliced_array.validity_mask().unwrap(); @@ -406,12 +408,15 @@ mod tests { let values = PrimitiveArray::from_iter(Vec::::new()).into_array(); let indices = PrimitiveArray::from_iter(Vec::::new()).into_array(); let values_idx_offsets = PrimitiveArray::from_iter(Vec::::new()).into_array(); - let rle_array = RLE::try_new( - values, - indices.clone(), - values_idx_offsets, - 0, - indices.len(), + let rle_array = RLEArray::try_from_data( + RLEData::try_new( + values, + indices.clone(), + values_idx_offsets, + 0, + indices.len(), + ) + .unwrap(), ) .vortex_expect("RLEData is always valid"); @@ -424,8 +429,10 @@ mod tests { let values = PrimitiveArray::from_iter([10u32, 20, 30, 40]).into_array(); let indices = PrimitiveArray::from_iter([0u16, 1].repeat(1024)).into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64, 2]).into_array(); - let rle_array = RLE::try_new(values, indices, values_idx_offsets, 0, 2048) - .vortex_expect("RLEData is always valid"); + let rle_array = RLEArray::try_from_data( + RLEData::try_new(values, indices, values_idx_offsets, 0, 2048).unwrap(), + ) + .vortex_expect("RLEData is always valid"); assert_eq!(rle_array.len(), 2048); assert_eq!(rle_array.values().len(), 4); @@ -454,7 +461,7 @@ mod tests { } let concat = concat.freeze(); - let parts = SerializedArray::try_from(concat).unwrap(); + let parts = ArrayParts::try_from(concat).unwrap(); let decoded = parts .decode( &DType::Primitive(PType::U32, Nullability::NonNullable), @@ -474,12 +481,15 @@ mod tests { let primitive = PrimitiveArray::from_iter((0..2048).map(|i| (i / 100) as u32)); let rle_array = RLEData::encode(&primitive).unwrap(); - let sliced = RLE::try_new( - rle_array.values().clone(), - rle_array.indices().clone(), - rle_array.values_idx_offsets().clone(), - 100, - 100, + let sliced = RLEArray::try_from_data( + RLEData::try_new( + rle_array.values().clone(), + rle_array.indices().clone(), + rle_array.values_idx_offsets().clone(), + 100, + 100, + ) + .unwrap(), ) .vortex_expect("RLEData is always valid"); assert_eq!(sliced.len(), 100); @@ -497,7 +507,7 @@ mod tests { } let concat = concat.freeze(); - let parts = SerializedArray::try_from(concat).unwrap(); + let parts = ArrayParts::try_from(concat).unwrap(); let decoded = parts .decode( sliced.dtype(), @@ -540,15 +550,16 @@ mod tests { // Reconstruct the outer RLE with re-encoded indices. // SAFETY: we only replace the indices child; all other invariants hold. - let reconstructed = unsafe { - RLE::new_unchecked( + let reconstructed = RLEArray::try_from_data(unsafe { + RLEData::new_unchecked( rle.values().clone(), re_encoded.into_array(), rle.values_idx_offsets().clone(), + rle.dtype().clone(), rle.offset(), rle.len(), ) - }; + })?; // Decompress — panicked before the fill_forward_nulls chunk-boundary fix. let decoded = reconstructed.as_array().to_primitive(); diff --git a/encodings/fastlanes/src/rle/array/rle_compress.rs b/encodings/fastlanes/src/rle/array/rle_compress.rs index 11270da5486..a432097b04e 100644 --- a/encodings/fastlanes/src/rle/array/rle_compress.rs +++ b/encodings/fastlanes/src/rle/array/rle_compress.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use arrayref::array_mut_ref; -use fastlanes::RLE as FastLanesRLE; +use fastlanes::RLE; use vortex_array::IntoArray; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; @@ -15,7 +15,6 @@ use vortex_buffer::BufferMut; use vortex_error::VortexResult; use crate::FL_CHUNK_SIZE; -use crate::RLE; use crate::RLEArray; use crate::RLEData; use crate::fill_forward_nulls; @@ -32,8 +31,8 @@ impl RLEData { /// In case the input array length is % 1024 != 0, the last chunk is padded. fn rle_encode_typed(array: &PrimitiveArray) -> VortexResult where - T: NativePType + FastLanesRLE, - NativeValue: FastLanesRLE, + T: NativePType + RLE, + NativeValue: RLE, { // Fill-forward null values so the RLE encoder doesn't see garbage at null positions, // which would create spurious run boundaries and inflate the dictionary. @@ -99,13 +98,13 @@ where // SAFETY: NativeValue is repr(transparent) to T. let values_buf = unsafe { values_buf.transmute::().freeze() }; - RLE::try_new( + RLEArray::try_from_data(RLEData::try_new( values_buf.into_array(), PrimitiveArray::new(indices_buf.freeze(), padded_validity(array)).into_array(), values_idx_offsets.into_array(), 0, array.len(), - ) + )?) } /// Returns validity padded to the next 1024 chunk for a given array. @@ -265,7 +264,7 @@ mod tests { #[case(vec![f16::ZERO, f16::NEG_ZERO])] #[case(vec![0f32, -0f32])] #[case(vec![0f64, -0f64])] - fn test_float_zeros(#[case] values: Vec) { + fn test_float_zeros(#[case] values: Vec) { let primitive = PrimitiveArray::from_iter(values); let rle = RLEData::encode(&primitive).unwrap(); let decoded = rle.as_array().to_primitive(); diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index c3ae7d42e81..88078862816 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -3,20 +3,17 @@ use vortex_array::ArrayRef; use vortex_array::ArrayView; -use vortex_array::IntoArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; -use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; +use crate::RLEData; use crate::rle::RLE; impl CastReduce for RLE { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Cast RLE values. - let casted_values = array - .values() - .cast(DType::Primitive(dtype.as_ptype(), Nullability::NonNullable))?; + let casted_values = array.values().cast(dtype.clone())?; // Cast RLE indices such that validity matches the target dtype. let casted_indices = if array.indices().dtype().nullability() != dtype.nullability() { @@ -28,16 +25,17 @@ impl CastReduce for RLE { array.indices().clone() }; - Ok(Some( - RLE::try_new( + Ok(Some(unsafe { + RLEData::new_unchecked( casted_values, casted_indices, array.values_idx_offsets().clone(), + dtype.clone(), array.offset(), array.len(), - )? - .into_array(), - )) + ) + .into() + })) } } diff --git a/encodings/fastlanes/src/rle/kernel.rs b/encodings/fastlanes/src/rle/kernel.rs index 65f8bfdb907..1da88206913 100644 --- a/encodings/fastlanes/src/rle/kernel.rs +++ b/encodings/fastlanes/src/rle/kernel.rs @@ -14,6 +14,7 @@ use vortex_error::VortexResult; use crate::FL_CHUNK_SIZE; use crate::RLE; +use crate::RLEData; pub(crate) static PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ParentKernelSet::lift(&SliceExecuteAdaptor(RLE))]); @@ -45,16 +46,18 @@ impl SliceKernel for RLE { .indices() .slice(chunk_start_idx * FL_CHUNK_SIZE..chunk_end_idx * FL_CHUNK_SIZE)?; - Ok(Some( - RLE::try_new( + // SAFETY: Slicing preserves all invariants. + Ok(Some(unsafe { + RLEData::new_unchecked( sliced_values, sliced_indices, sliced_values_idx_offsets, + array.dtype().clone(), // Keep the offset relative to the first chunk. (array.offset() + range.start) % FL_CHUNK_SIZE, range.len(), - )? - .into_array(), - )) + ) + .into_array() + })) } } diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 776fa1b84ad..63eeb7b5706 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -8,21 +8,24 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; use vortex_array::arrays::PrimitiveArray; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::VTable; +use vortex_array::vtable::ValidityVTableFromChildSliceHelper; +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; @@ -58,15 +61,29 @@ pub struct RLEMetadata { impl VTable for RLE { type ArrayData = RLEData; + type Metadata = ProstMetadata; + type OperationsVTable = Self; - type ValidityVTable = Self; + type ValidityVTable = ValidityVTableFromChildSliceHelper; + + fn vtable(_array: &RLEData) -> &Self { + &RLE + } fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate_against_outer(dtype, len) + fn len(array: &RLEData) -> usize { + array.len() + } + + fn dtype(array: &RLEData) -> &DType { + array.dtype() + } + + fn stats(array: &RLEData) -> &ArrayStats { + array.stats_set() } fn array_hash(array: &RLEData, state: &mut H, precision: Precision) { @@ -124,36 +141,39 @@ impl VTable for RLE { Ok(()) } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - RLEMetadata { - values_len: array.values().len() as u64, - indices_len: array.indices().len() as u64, - indices_ptype: PType::try_from(array.indices().dtype())? as i32, - values_idx_offsets_len: array.values_idx_offsets().len() as u64, - values_idx_offsets_ptype: PType::try_from(array.values_idx_offsets().dtype())? - as i32, - offset: array.offset() as u64, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(RLEMetadata { + values_len: array.values().len() as u64, + indices_len: array.indices().len() as u64, + indices_ptype: PType::try_from(array.indices().dtype())? as i32, + values_idx_offsets_len: array.values_idx_offsets().len() as u64, + values_idx_offsets_ptype: PType::try_from(array.values_idx_offsets().dtype())? as i32, + offset: array.offset() as u64, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.0.encode_to_vec())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata(RLEMetadata::decode(bytes)?)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - buffers: &[BufferHandle], + metadata: &Self::Metadata, + _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - vortex_ensure!( - buffers.is_empty(), - "RLEArray expects 0 buffers, got {}", - buffers.len() - ); - let metadata = RLEMetadata::decode(metadata)?; + let metadata = &metadata.0; let values = children.get( 0, &DType::Primitive(dtype.as_ptype(), Nullability::NonNullable), @@ -206,18 +226,6 @@ pub struct RLE; impl RLE { pub const ID: ArrayId = ArrayId::new_ref("fastlanes.rle"); - pub fn try_new( - values: ArrayRef, - indices: ArrayRef, - values_idx_offsets: ArrayRef, - offset: usize, - length: usize, - ) -> VortexResult { - let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); - let data = RLEData::try_new(values, indices, values_idx_offsets, offset, length)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RLE, dtype, length, data)) }) - } - /// Create a new RLE array without validation. /// /// # Safety @@ -226,12 +234,14 @@ impl RLE { values: ArrayRef, indices: ArrayRef, values_idx_offsets: ArrayRef, + dtype: DType, offset: usize, length: usize, ) -> RLEArray { - let dtype = DType::Primitive(values.dtype().as_ptype(), indices.dtype().nullability()); - let data = unsafe { RLEData::new_unchecked(values, indices, values_idx_offsets, offset) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(RLE, dtype, length, data)) } + Array::try_from_data(unsafe { + RLEData::new_unchecked(values, indices, values_idx_offsets, dtype, offset, length) + }) + .vortex_expect("RLEData is always valid") } /// Encode a primitive array using FastLanes RLE. @@ -242,9 +252,9 @@ impl RLE { #[cfg(test)] mod tests { - use prost::Message; use vortex_array::test_harness::check_metadata; + use super::ProstMetadata; use super::RLEMetadata; #[cfg_attr(miri, ignore)] @@ -252,15 +262,14 @@ mod tests { fn test_rle_metadata() { check_metadata( "rle.metadata", - &RLEMetadata { + ProstMetadata(RLEMetadata { values_len: u64::MAX, indices_len: u64::MAX, indices_ptype: i32::MAX, values_idx_offsets_len: u64::MAX, values_idx_offsets_ptype: i32::MAX, offset: u64::MAX, - } - .encode_to_vec(), + }), ); } } diff --git a/encodings/fastlanes/src/rle/vtable/operations.rs b/encodings/fastlanes/src/rle/vtable/operations.rs index e8dfc159f70..8804d30d178 100644 --- a/encodings/fastlanes/src/rle/vtable/operations.rs +++ b/encodings/fastlanes/src/rle/vtable/operations.rs @@ -47,7 +47,6 @@ mod tests { use vortex_buffer::buffer; use super::*; - use crate::RLE; use crate::RLEArray; use crate::RLEData; @@ -66,12 +65,15 @@ mod tests { .into_array(); let values_idx_offsets = PrimitiveArray::from_iter([0u64]).into_array(); - RLE::try_new( - values, - indices.clone(), - values_idx_offsets, - 0, - indices.len(), + RLEArray::try_from_data( + RLEData::try_new( + values, + indices.clone(), + values_idx_offsets, + 0, + indices.len(), + ) + .unwrap(), ) .vortex_expect("RLEData is always valid") } @@ -101,12 +103,15 @@ mod tests { ) .into_array(); - RLE::try_new( - values, - indices.clone(), - values_idx_offsets, - 0, - indices.len(), + RLEArray::try_from_data( + RLEData::try_new( + values, + indices.clone(), + values_idx_offsets, + 0, + indices.len(), + ) + .unwrap(), ) .vortex_expect("RLEData is always valid") } diff --git a/encodings/fastlanes/src/rle/vtable/validity.rs b/encodings/fastlanes/src/rle/vtable/validity.rs index ecef8047cb1..f46998774cd 100644 --- a/encodings/fastlanes/src/rle/vtable/validity.rs +++ b/encodings/fastlanes/src/rle/vtable/validity.rs @@ -1,16 +1,22 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_array::ArrayView; -use vortex_array::vtable::ValidityVTable; -use vortex_error::VortexResult; +use vortex_array::ArrayRef; +use vortex_array::vtable::ValidityChild; +use vortex_array::vtable::ValidityChildSliceHelper; -use crate::rle::RLE; +use super::RLE; +use crate::RLEData; -impl ValidityVTable for RLE { - fn validity(array: ArrayView<'_, RLE>) -> VortexResult { - let start = array.offset(); - let stop = start + array.len(); - array.indices().slice(start..stop)?.validity() +impl ValidityChild for RLE { + fn validity_child(array: &RLEData) -> &ArrayRef { + array.indices() + } +} + +impl ValidityChildSliceHelper for RLEData { + fn unsliced_child_and_slice(&self) -> (&ArrayRef, usize, usize) { + let (start, len) = (self.offset(), self.len()); + (self.indices(), start, start + len) } } diff --git a/encodings/fsst/public-api.lock b/encodings/fsst/public-api.lock index 4989f13dca7..32522dd3f04 100644 --- a/encodings/fsst/public-api.lock +++ b/encodings/fsst/public-api.lock @@ -20,6 +20,8 @@ impl vortex_array::array::vtable::VTable for vortex_fsst::FSST pub type vortex_fsst::FSST::ArrayData = vortex_fsst::FSSTData +pub type vortex_fsst::FSST::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_fsst::FSST::OperationsVTable = vortex_fsst::FSST pub type vortex_fsst::FSST::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -34,7 +36,11 @@ pub fn vortex_fsst::FSST::buffer(array: vortex_array::array::view::ArrayView<'_, pub fn vortex_fsst::FSST::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_fsst::FSST::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_fsst::FSST::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_fsst::FSST::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_fsst::FSST::dtype(array: &vortex_fsst::FSSTData) -> &vortex_array::dtype::DType pub fn vortex_fsst::FSST::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -42,17 +48,23 @@ pub fn vortex_fsst::FSST::execute_parent(array: vortex_array::array::view::Array pub fn vortex_fsst::FSST::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_fsst::FSST::len(array: &vortex_fsst::FSSTData) -> usize + +pub fn vortex_fsst::FSST::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_fsst::FSST::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_fsst::FSST::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_fsst::FSST::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_fsst::FSST::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_fsst::FSST::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_fsst::FSST::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_fsst::FSST::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_fsst::FSST::stats(array: &vortex_fsst::FSSTData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_fsst::FSST::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_fsst::FSST::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -100,6 +112,8 @@ pub fn vortex_fsst::FSSTData::compressor(&self) -> &fsst::Compressor pub fn vortex_fsst::FSSTData::decompressor(&self) -> fsst::Decompressor<'_> +pub fn vortex_fsst::FSSTData::dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_fsst::FSSTData::is_empty(&self) -> bool pub fn vortex_fsst::FSSTData::len(&self) -> usize @@ -108,22 +122,28 @@ pub fn vortex_fsst::FSSTData::symbol_lengths(&self) -> &vortex_buffer::buffer::B pub fn vortex_fsst::FSSTData::symbols(&self) -> &vortex_buffer::buffer::Buffer -pub fn vortex_fsst::FSSTData::try_new(symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult +pub fn vortex_fsst::FSSTData::try_new(dtype: vortex_array::dtype::DType, symbols: vortex_buffer::buffer::Buffer, symbol_lengths: vortex_buffer::buffer::Buffer, codes: vortex_array::arrays::varbin::vtable::VarBinArray, uncompressed_lengths: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult pub fn vortex_fsst::FSSTData::uncompressed_lengths(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_fsst::FSSTData::uncompressed_lengths_dtype(&self) -> &vortex_array::dtype::DType -pub fn vortex_fsst::FSSTData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - impl core::clone::Clone for vortex_fsst::FSSTData pub fn vortex_fsst::FSSTData::clone(&self) -> vortex_fsst::FSSTData +impl core::convert::From for vortex_array::array::erased::ArrayRef + +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_fsst::FSSTData) -> vortex_array::array::erased::ArrayRef + impl core::fmt::Debug for vortex_fsst::FSSTData pub fn vortex_fsst::FSSTData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::array::IntoArray for vortex_fsst::FSSTData + +pub fn vortex_fsst::FSSTData::into_array(self) -> vortex_array::array::erased::ArrayRef + pub struct vortex_fsst::FSSTMetadata impl vortex_fsst::FSSTMetadata diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index d716bc7608b..b98d5d9edd6 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -9,19 +9,20 @@ use std::sync::LazyLock; use fsst::Compressor; use fsst::Decompressor; use fsst::Symbol; -use prost::Message as _; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; +use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; +use vortex_array::SerializeMetadata; use vortex_array::arrays::VarBin; use vortex_array::arrays::VarBinArray; use vortex_array::buffer::BufferHandle; @@ -31,6 +32,7 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -72,15 +74,29 @@ impl FSSTMetadata { impl VTable for FSST { type ArrayData = FSSTData; + + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; + fn vtable(_array: &Self::ArrayData) -> &Self { + &FSST + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) + fn len(array: &FSSTData) -> usize { + array.codes().len() + } + + fn dtype(array: &FSSTData) -> &DType { + &array.dtype + } + + fn stats(array: &FSSTData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &FSSTData, state: &mut H, precision: Precision) { @@ -131,26 +147,60 @@ impl VTable for FSST { } } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - FSSTMetadata { - uncompressed_lengths_ptype: array.uncompressed_lengths().dtype().as_ptype().into(), - codes_offsets_ptype: array.codes.offsets().dtype().as_ptype().into(), - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(FSSTMetadata { + uncompressed_lengths_ptype: array.uncompressed_lengths().dtype().as_ptype().into(), + codes_offsets_ptype: array.codes.offsets().dtype().as_ptype().into(), + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata( + as DeserializeMetadata>::deserialize(bytes)?, + )) + } + + fn append_to_builder( + array: ArrayView<'_, Self>, + builder: &mut dyn ArrayBuilder, + ctx: &mut ExecutionCtx, + ) -> VortexResult<()> { + let Some(builder) = builder.as_any_mut().downcast_mut::() else { + builder.extend_from_array( + &array + .array() + .clone() + .execute::(ctx)? + .into_array(), + ); + return Ok(()); + }; + + // Decompress the whole block of data into a new buffer, and create some views + // from it instead. + let (buffers, views) = fsst_decode_views(&array, builder.completed_block_count(), ctx)?; + + builder.push_buffer_and_adjusted_views(&buffers, &views, array.array().validity_mask()?); + Ok(()) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = FSSTMetadata::decode(metadata)?; let symbols = Buffer::::from_byte_buffer(buffers[0].clone().try_to_host_sync()?); let symbol_lengths = Buffer::::from_byte_buffer(buffers[1].clone().try_to_host_sync()?); @@ -172,13 +222,19 @@ impl VTable for FSST { let uncompressed_lengths = children.get( 1, &DType::Primitive( - metadata.get_uncompressed_lengths_ptype()?, + metadata.0.get_uncompressed_lengths_ptype()?, Nullability::NonNullable, ), len, )?; - return FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, dtype); + return FSSTData::try_new( + dtype.clone(), + symbols, + symbol_lengths, + codes, + uncompressed_lengths, + ); } // Check for the current deserialization path. @@ -186,7 +242,7 @@ impl VTable for FSST { let uncompressed_lengths = children.get( 0, &DType::Primitive( - metadata.get_uncompressed_lengths_ptype()?, + metadata.0.get_uncompressed_lengths_ptype()?, Nullability::NonNullable, ), len, @@ -219,7 +275,13 @@ impl VTable for FSST { codes_validity, )?; - return FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, dtype); + return FSSTData::try_new( + dtype.clone(), + symbols, + symbol_lengths, + codes, + uncompressed_lengths, + ); } vortex_bail!( @@ -250,7 +312,7 @@ impl VTable for FSST { .vortex_expect("FSSTArray requires codes_offsets slot"); let codes_validity = match &slots[CODES_VALIDITY_SLOT] { Some(v) => Validity::Array(v.clone()), - None => Validity::from(array.codes.dtype().nullability()), + None => Validity::from(array.dtype.nullability()), }; let codes = VarBinArray::try_new( codes_offsets, @@ -266,31 +328,7 @@ impl VTable for FSST { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - canonicalize_fsst(array.as_view(), ctx).map(ExecutionResult::done) - } - - fn append_to_builder( - array: ArrayView<'_, Self>, - builder: &mut dyn ArrayBuilder, - ctx: &mut ExecutionCtx, - ) -> VortexResult<()> { - let Some(builder) = builder.as_any_mut().downcast_mut::() else { - builder.extend_from_array( - &array - .array() - .clone() - .execute::(ctx)? - .into_array(), - ); - return Ok(()); - }; - - // Decompress the whole block of data into a new buffer, and create some views - // from it instead. - let (buffers, views) = fsst_decode_views(&array, builder.completed_block_count(), ctx)?; - - builder.push_buffer_and_adjusted_views(&buffers, &views, array.array().validity_mask()?); - Ok(()) + canonicalize_fsst(&array, ctx).map(ExecutionResult::done) } fn execute_parent( @@ -323,6 +361,7 @@ pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = #[derive(Clone)] pub struct FSSTData { + dtype: DType, symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, @@ -330,6 +369,7 @@ pub struct FSSTData { codes_array: ArrayRef, /// Lengths of the original values before compression, can be compressed. slots: Vec>, + stats_set: ArrayStats, /// Memoized compressor used for push-down of compute by compressing the RHS. compressor: Arc Compressor + Send>>>, @@ -338,6 +378,7 @@ pub struct FSSTData { impl Debug for FSSTData { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("FSSTArray") + .field("dtype", &self.dtype) .field("symbols", &self.symbols) .field("symbol_lengths", &self.symbol_lengths) .field("codes", &self.codes) @@ -360,23 +401,13 @@ impl FSST { codes: VarBinArray, uncompressed_lengths: ArrayRef, ) -> VortexResult { - let len = codes.len(); - let data = FSSTData::try_new(symbols, symbol_lengths, codes, uncompressed_lengths, &dtype)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data)) }) - } - - pub(crate) unsafe fn new_unchecked( - dtype: DType, - symbols: Buffer, - symbol_lengths: Buffer, - codes: VarBinArray, - uncompressed_lengths: ArrayRef, - ) -> FSSTArray { - let len = codes.len(); - let data = unsafe { - FSSTData::new_unchecked(symbols, symbol_lengths, codes, uncompressed_lengths) - }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(FSST, dtype, len, data)) } + Array::try_from_data(FSSTData::try_new( + dtype, + symbols, + symbol_lengths, + codes, + uncompressed_lengths, + )?) } } @@ -390,56 +421,12 @@ impl FSSTData { /// Each code corresponds either to a symbol, or to the "escape code", /// which tells the decoder to emit the following byte without doing a table lookup. pub fn try_new( + dtype: DType, symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, uncompressed_lengths: ArrayRef, - dtype: &DType, ) -> VortexResult { - Self::validate_parts( - &symbols, - &symbol_lengths, - &codes, - &uncompressed_lengths, - dtype, - codes.len(), - )?; - - // SAFETY: all components validated above - unsafe { - Ok(Self::new_unchecked( - symbols, - symbol_lengths, - codes, - uncompressed_lengths, - )) - } - } - - pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { - Self::validate_parts( - &self.symbols, - &self.symbol_lengths, - &self.codes, - self.uncompressed_lengths(), - dtype, - len, - ) - } - - fn validate_parts( - symbols: &Buffer, - symbol_lengths: &Buffer, - codes: &VarBinArray, - uncompressed_lengths: &ArrayRef, - dtype: &DType, - len: usize, - ) -> VortexResult<()> { - vortex_ensure!( - matches!(dtype, DType::Binary(_) | DType::Utf8(_)), - "FSST arrays must be Binary or Utf8, found {dtype}" - ); - // Check: symbols must not have length > MAX_CODE if symbols.len() > 255 { vortex_bail!(InvalidArgument: "symbols array must have length <= 255"); @@ -448,11 +435,7 @@ impl FSSTData { vortex_bail!(InvalidArgument: "symbols and symbol_lengths arrays must have same length"); } - if codes.len() != len { - vortex_bail!(InvalidArgument: "codes must have same len as outer array"); - } - - if uncompressed_lengths.len() != len { + if uncompressed_lengths.len() != codes.len() { vortex_bail!(InvalidArgument: "uncompressed_lengths must be same len as codes"); } @@ -464,14 +447,21 @@ impl FSSTData { if !matches!(codes.dtype(), DType::Binary(_)) { vortex_bail!(InvalidArgument: "codes array must be DType::Binary type"); } - if codes.dtype().nullability() != dtype.nullability() { - vortex_bail!(InvalidArgument: "codes nullability must match outer dtype nullability"); - } - Ok(()) + // SAFETY: all components validated above + unsafe { + Ok(Self::new_unchecked( + dtype, + symbols, + symbol_lengths, + codes, + uncompressed_lengths, + )) + } } pub(crate) unsafe fn new_unchecked( + dtype: DType, symbols: Buffer, symbol_lengths: Buffer, codes: VarBinArray, @@ -488,6 +478,7 @@ impl FSSTData { let codes_validity_slot = validity_to_child(&codes.validity(), codes.len()); Self { + dtype, symbols, symbol_lengths, codes, @@ -497,6 +488,7 @@ impl FSSTData { codes_offsets_slot, codes_validity_slot, ], + stats_set: Default::default(), compressor, } } @@ -511,6 +503,11 @@ impl FSSTData { self.codes.len() == 0 } + /// Returns the logical data type of the array. + pub fn dtype(&self) -> &DType { + &self.dtype + } + /// Access the symbol table array pub fn symbols(&self) -> &Buffer { &self.symbols @@ -567,10 +564,9 @@ impl ValidityChild for FSST { mod test { use fsst::Compressor; use fsst::Symbol; - use prost::Message; - use vortex_array::DynVTable; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; + use vortex_array::ProstMetadata; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::VarBinViewArray; @@ -579,6 +575,7 @@ mod test { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::test_harness::check_metadata; + use vortex_array::vtable::VTable; use vortex_buffer::Buffer; use vortex_error::VortexError; @@ -591,11 +588,10 @@ mod test { fn test_fsst_metadata() { check_metadata( "fsst.metadata", - &FSSTMetadata { + ProstMetadata(FSSTMetadata { uncompressed_lengths_ptype: PType::U64 as i32, codes_offsets_ptype: PType::I32 as i32, - } - .encode_to_vec(), + }), ); } @@ -640,12 +636,10 @@ mod test { fsst_array.uncompressed_lengths().clone(), ]; - let fsst = DynVTable::build( - &FSST, - FSST::ID, + let fsst = FSST::build( &DType::Utf8(Nullability::NonNullable), 2, - &FSSTMetadata { + &ProstMetadata(FSSTMetadata { uncompressed_lengths_ptype: fsst_array .uncompressed_lengths() .dtype() @@ -653,11 +647,9 @@ mod test { .into(), // Legacy array did not store this field, use Protobuf default of 0. codes_offsets_ptype: 0, - } - .encode_to_vec(), + }), &buffers, &children.as_slice(), - &LEGACY_SESSION, ) .unwrap(); diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index 64fb8ea2117..41df50f913b 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use vortex_array::ArrayRef; -use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; @@ -18,14 +17,13 @@ use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; -use crate::FSST; use crate::FSSTData; pub(super) fn canonicalize_fsst( - array: ArrayView<'_, FSST>, + array: &FSSTData, ctx: &mut ExecutionCtx, ) -> VortexResult { - let (buffers, views) = fsst_decode_views(array.data(), 0, ctx)?; + let (buffers, views) = fsst_decode_views(array, 0, ctx)?; // SAFETY: FSST already validates the bytes for binary/UTF-8. We build views directly on // top of them, so the view pointers will all be valid. Ok(unsafe { diff --git a/encodings/fsst/src/compress.rs b/encodings/fsst/src/compress.rs index 53772e82636..eb016789584 100644 --- a/encodings/fsst/src/compress.rs +++ b/encodings/fsst/src/compress.rs @@ -14,8 +14,8 @@ use vortex_buffer::BufferMut; use vortex_error::VortexExpect; /// Compress a string array using FSST. -use crate::FSST; use crate::FSSTArray; +use crate::FSSTData; pub fn fsst_compress>( strings: A, len: usize, @@ -102,8 +102,11 @@ where let uncompressed_lengths = uncompressed_lengths.into_array(); - FSST::try_new(dtype, symbols, symbol_lengths, codes, uncompressed_lengths) - .vortex_expect("FSST parts must be valid") + FSSTArray::try_from_data( + FSSTData::try_new(dtype, symbols, symbol_lengths, codes, uncompressed_lengths) + .vortex_expect("building FSSTArray from parts"), + ) + .vortex_expect("FSSTData is always valid") } #[cfg(test)] diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs index 82034daab21..496d5bf3f92 100644 --- a/encodings/fsst/src/compute/cast.rs +++ b/encodings/fsst/src/compute/cast.rs @@ -11,6 +11,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::FSST; +use crate::FSSTData; impl CastReduce for FSST { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // FSST is a string compression encoding. @@ -24,7 +25,7 @@ impl CastReduce for FSST { .cast(array.codes().dtype().with_nullability(dtype.nullability()))?; Ok(Some( - FSST::try_new( + FSSTData::try_new( dtype.clone(), array.symbols().clone(), array.symbol_lengths().clone(), diff --git a/encodings/fsst/src/compute/compare.rs b/encodings/fsst/src/compute/compare.rs index 14ea2cbe4d6..cc0c563a887 100644 --- a/encodings/fsst/src/compute/compare.rs +++ b/encodings/fsst/src/compute/compare.rs @@ -21,6 +21,8 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use crate::FSST; +use crate::FSSTData; + impl CompareKernel for FSST { fn compare( lhs: ArrayView<'_, Self>, @@ -29,7 +31,7 @@ impl CompareKernel for FSST { ctx: &mut ExecutionCtx, ) -> VortexResult> { match rhs.as_constant() { - Some(constant) => compare_fsst_constant(lhs, &constant, operator, ctx), + Some(constant) => compare_fsst_constant(&lhs, &constant, operator, ctx), // Otherwise, fall back to the default comparison behavior. _ => Ok(None), } @@ -38,7 +40,7 @@ impl CompareKernel for FSST { /// Specialized compare function implementation used when performing against a constant fn compare_fsst_constant( - left: ArrayView<'_, FSST>, + left: &FSSTData, right: &Scalar, operator: CompareOperator, ctx: &mut ExecutionCtx, @@ -76,7 +78,7 @@ fn compare_fsst_constant( return Ok(Some( BoolArray::new( buffer, - Validity::copy_from_array(left.array())? + Validity::copy_from_array(&left.clone().into_array())? .union_nullability(right.dtype().nullability()), ) .into_array(), diff --git a/encodings/fsst/src/compute/filter.rs b/encodings/fsst/src/compute/filter.rs index 0a2aa130d7a..de58a83d5c4 100644 --- a/encodings/fsst/src/compute/filter.rs +++ b/encodings/fsst/src/compute/filter.rs @@ -12,6 +12,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::FSST; +use crate::FSSTData; impl FilterKernel for FSST { fn filter( @@ -30,7 +31,7 @@ impl FilterKernel for FSST { .vortex_expect("must be VarBin"); Ok(Some( - FSST::try_new( + FSSTData::try_new( array.dtype().clone(), array.symbols().clone(), array.symbol_lengths().clone(), diff --git a/encodings/fsst/src/compute/mod.rs b/encodings/fsst/src/compute/mod.rs index 1bc1df71a29..7ca91c4c331 100644 --- a/encodings/fsst/src/compute/mod.rs +++ b/encodings/fsst/src/compute/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use crate::FSSTData; mod cast; mod compare; mod filter; @@ -27,7 +28,7 @@ impl TakeExecute for FSST { _ctx: &mut ExecutionCtx, ) -> VortexResult> { Ok(Some( - FSST::try_new( + FSSTData::try_new( array .dtype() .clone() diff --git a/encodings/fsst/src/slice.rs b/encodings/fsst/src/slice.rs index 5e0dd21a9f3..b30e574e9bd 100644 --- a/encodings/fsst/src/slice.rs +++ b/encodings/fsst/src/slice.rs @@ -12,13 +12,14 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use crate::FSST; +use crate::FSSTData; impl SliceReduce for FSST { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // SAFETY: slicing the `codes` leaves the symbol table intact Ok(Some( unsafe { - FSST::new_unchecked( + FSSTData::new_unchecked( array.dtype().clone(), array.symbols().clone(), array.symbol_lengths().clone(), diff --git a/encodings/parquet-variant/src/array.rs b/encodings/parquet-variant/src/array.rs index df872889391..f07ae0e847f 100644 --- a/encodings/parquet-variant/src/array.rs +++ b/encodings/parquet-variant/src/array.rs @@ -7,8 +7,6 @@ use arrow_array::Array as ArrowArray; use arrow_array::ArrayRef as ArrowArrayRef; use arrow_schema::Field; use parquet_variant_compute::VariantArray as ArrowVariantArray; -use vortex_array::Array; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; @@ -17,6 +15,7 @@ use vortex_array::arrow::ArrowArrayExecutor; use vortex_array::arrow::FromArrowArray; use vortex_array::arrow::to_arrow_null_buffer; use vortex_array::dtype::DType; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable::validity_to_child; use vortex_buffer::BitBuffer; @@ -25,8 +24,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_ensure_eq; -use crate::ParquetVariant; - /// The validity bitmap indicating which elements are non-null. pub(crate) const VALIDITY_SLOT: usize = 0; /// The metadata array for the Parquet variant values. @@ -66,24 +63,10 @@ pub(crate) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity", "metadata", "value /// (which includes nullability). #[derive(Clone, Debug)] pub struct ParquetVariantData { + pub(crate) dtype: DType, pub(crate) validity: Validity, pub(crate) slots: Vec>, -} - -impl ParquetVariant { - pub fn try_new( - validity: Validity, - metadata: ArrayRef, - value: Option, - typed_value: Option, - ) -> VortexResult> { - let len = metadata.len(); - let dtype = DType::Variant(validity.nullability()); - let data = ParquetVariantData::try_new(validity, metadata, value, typed_value)?; - Ok(unsafe { - Array::from_parts_unchecked(ArrayParts::new(ParquetVariant, dtype, len, data)) - }) - } + pub(crate) stats_set: ArrayStats, } impl ParquetVariantData { @@ -94,82 +77,39 @@ impl ParquetVariantData { value: Option, typed_value: Option, ) -> VortexResult { - let len = metadata.len(); - let dtype = DType::Variant(validity.nullability()); - Self::validate_parts( - &validity, - &metadata, - value.as_ref(), - typed_value.as_ref(), - &dtype, - len, - )?; - - let validity_child = validity_to_child(&validity, len); - let slots = vec![validity_child, Some(metadata), value, typed_value]; - - Ok(Self { validity, slots }) - } - - pub(crate) fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { - Self::validate_parts( - &self.validity, - self.metadata_array(), - self.value_array(), - self.typed_value_array(), - dtype, - len, - ) - } - - fn validate_parts( - validity: &Validity, - metadata: &ArrayRef, - value: Option<&ArrayRef>, - typed_value: Option<&ArrayRef>, - dtype: &DType, - len: usize, - ) -> VortexResult<()> { - vortex_ensure!( - matches!(dtype, DType::Variant(_)), - "Expected Variant DType, found {dtype}" - ); vortex_ensure!( value.is_some() || typed_value.is_some(), "at least one of value or typed_value must be present" ); - - vortex_ensure_eq!( - dtype.nullability(), - validity.nullability(), - "variant dtype nullability must match validity nullability" - ); - vortex_ensure_eq!( - metadata.dtype(), - &DType::Binary(vortex_array::dtype::Nullability::NonNullable), - "metadata dtype must be non-nullable binary" - ); - vortex_ensure_eq!( - metadata.len(), - len, - "metadata length must match array length" - ); - + let len = metadata.len(); if let Some(validity_len) = validity.maybe_len() { - vortex_ensure_eq!(validity_len, len, "validity length must match array length"); - } - if let Some(v) = value { - vortex_ensure!( - matches!(v.dtype(), DType::Binary(_)), - "value dtype must be binary, found {}", - v.dtype() + vortex_ensure_eq!( + validity_len, + len, + "validity length must match metadata length" ); - vortex_ensure_eq!(v.len(), len, "value length must match array length"); } - if let Some(tv) = typed_value { - vortex_ensure_eq!(tv.len(), len, "typed_value length must match array length"); + if let Some(ref v) = value { + vortex_ensure_eq!(v.len(), len, "value length must match metadata length"); } - Ok(()) + if let Some(ref tv) = typed_value { + vortex_ensure_eq!( + tv.len(), + len, + "typed_value length must match metadata length" + ); + } + let nullability = validity.nullability(); + + let validity_child = validity_to_child(&validity, len); + let slots = vec![validity_child, Some(metadata), value, typed_value]; + + Ok(Self { + dtype: DType::Variant(nullability), + validity, + slots, + stats_set: ArrayStats::default(), + }) } /// Returns a reference to the metadata child array. @@ -228,7 +168,7 @@ impl ParquetVariantData { .map(|tv| ArrayRef::from_arrow(tv.as_ref(), typed_value_nullable)) .transpose()?; - let pv = ParquetVariant::try_new(validity, metadata, value, typed_value)?; + let pv = ParquetVariantData::try_new(validity, metadata, value, typed_value)?; Ok(VariantArray::new(pv.into_array()).into_array()) } diff --git a/encodings/parquet-variant/src/kernel.rs b/encodings/parquet-variant/src/kernel.rs index 69e922e1548..d921f97a325 100644 --- a/encodings/parquet-variant/src/kernel.rs +++ b/encodings/parquet-variant/src/kernel.rs @@ -18,6 +18,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::ParquetVariant; +use crate::array::ParquetVariantData; pub(crate) static PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ ParentKernelSet::lift(&FilterExecuteAdaptor(ParquetVariant)), @@ -42,7 +43,7 @@ impl SliceKernel for ParquetVariant { .map(|tv| tv.slice(range)) .transpose()?; Ok(Some( - ParquetVariant::try_new(validity, metadata, value, typed_value)?.into_array(), + ParquetVariantData::try_new(validity, metadata, value, typed_value)?.into_array(), )) } } @@ -64,7 +65,7 @@ impl FilterKernel for ParquetVariant { .map(|tv| tv.filter(mask.clone())) .transpose()?; Ok(Some( - ParquetVariant::try_new(validity, metadata, value, typed_value)?.into_array(), + ParquetVariantData::try_new(validity, metadata, value, typed_value)?.into_array(), )) } } @@ -86,7 +87,7 @@ impl TakeExecute for ParquetVariant { .map(|tv| tv.take(indices.clone())) .transpose()?; Ok(Some( - ParquetVariant::try_new(validity, metadata, value, typed_value)?.into_array(), + ParquetVariantData::try_new(validity, metadata, value, typed_value)?.into_array(), )) } } diff --git a/encodings/parquet-variant/src/lib.rs b/encodings/parquet-variant/src/lib.rs index 42c97d28870..3f1f7cb68a6 100644 --- a/encodings/parquet-variant/src/lib.rs +++ b/encodings/parquet-variant/src/lib.rs @@ -33,3 +33,4 @@ mod vtable; pub use array::ParquetVariantData; pub use vtable::ParquetVariant; pub use vtable::ParquetVariantArray; +pub use vtable::ParquetVariantMetadata; diff --git a/encodings/parquet-variant/src/operations.rs b/encodings/parquet-variant/src/operations.rs index 4b2d26b77a4..6ccbfeb6d75 100644 --- a/encodings/parquet-variant/src/operations.rs +++ b/encodings/parquet-variant/src/operations.rs @@ -61,7 +61,7 @@ impl OperationsVTable for ParquetVariant { )?; Scalar::try_new( - array.dtype().clone(), + array.dtype.clone(), Some(ScalarValue::Variant(Box::new(inner))), ) } diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index 083cfc42261..8815734e0c9 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -20,6 +20,7 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -45,6 +46,23 @@ impl ParquetVariant { pub const ID: ArrayId = ArrayId::new_ref("vortex.parquet.variant"); } +/// Serialized metadata for a [`ParquetVariantArray`]. +#[derive(Clone, Debug)] +pub struct ParquetVariantMetadata { + /// Whether the un-shredded `value` child is present. + pub has_value: bool, + /// Whether the `value` child is nullable. + /// + /// In partially-shredded layouts, rows whose data lives entirely in `typed_value` have a + /// null `value` slot, so the Arrow field is marked nullable. This flag preserves that + /// distinction across serialization round-trips. + pub value_nullable: bool, + /// DType of the shredded `typed_value`, if present. + /// + /// This is required to deserialize non-variant shredded children. + pub typed_value_dtype: Option, +} + #[derive(Clone, prost::Message)] struct ParquetVariantMetadataProto { /// Whether the un-shredded `value` child is present. @@ -62,15 +80,28 @@ vtable!(ParquetVariant, ParquetVariant, ParquetVariantData); impl VTable for ParquetVariant { type ArrayData = ParquetVariantData; + type Metadata = ParquetVariantMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValidityHelper; + fn vtable(_array: &Self::ArrayData) -> &Self { + &ParquetVariant + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) + fn len(array: &ParquetVariantData) -> usize { + array.metadata_array().len() + } + + fn dtype(array: &ParquetVariantData) -> &DType { + &array.dtype + } + + fn stats(array: &ParquetVariantData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &ParquetVariantData, state: &mut H, precision: Precision) { @@ -136,50 +167,64 @@ impl VTable for ParquetVariant { SLOT_NAMES[idx].to_string() } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - let typed_value_dtype = array - .typed_value_array() - .map(|tv| tv.dtype().try_into()) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ParquetVariantMetadata { + has_value: array.value_array().is_some(), + value_nullable: array.value_array().is_some_and(|v| v.dtype().is_nullable()), + typed_value_dtype: array.typed_value_array().map(|tv| tv.dtype().clone()), + }) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + let typed_value_dtype = metadata + .typed_value_dtype + .as_ref() + .map(|dtype| dtype.try_into()) .transpose()?; Ok(Some( ParquetVariantMetadataProto { - has_value: array.value_array().is_some(), + has_value: metadata.has_value, typed_value_dtype, - value_nullable: array.value_array().is_some_and(|v| v.dtype().is_nullable()), + value_nullable: metadata.value_nullable, } .encode_to_vec(), )) } fn deserialize( - &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], session: &VortexSession, - ) -> VortexResult { - vortex_ensure!( - buffers.is_empty(), - "ParquetVariantArray expects 0 buffers, got {}", - buffers.len() - ); - - let proto = ParquetVariantMetadataProto::decode(metadata)?; + ) -> VortexResult { + let proto = ParquetVariantMetadataProto::decode(bytes)?; let typed_value_dtype = match proto.typed_value_dtype.as_ref() { Some(dtype) => Some(DType::from_proto(dtype, session)?), None => None, }; + Ok(ParquetVariantMetadata { + has_value: proto.has_value, + value_nullable: proto.value_nullable, + typed_value_dtype, + }) + } + fn build( + dtype: &DType, + len: usize, + metadata: &Self::Metadata, + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + ) -> VortexResult { vortex_ensure!(matches!(dtype, DType::Variant(_)), "Expected Variant DType"); - let has_typed_value = typed_value_dtype.is_some(); + let has_typed_value = metadata.typed_value_dtype.is_some(); vortex_ensure!( - proto.has_value || has_typed_value, + metadata.has_value || has_typed_value, "At least one of value or typed_value must be present" ); - let expected_children = 1 + proto.has_value as usize + has_typed_value as usize; + let expected_children = 1 + metadata.has_value as usize + has_typed_value as usize; vortex_ensure!( children.len() == expected_children || children.len() == expected_children + 1, "Expected {} or {} children, got {}", @@ -197,8 +242,12 @@ impl VTable for ParquetVariant { children.get(child_idx, &DType::Binary(Nullability::NonNullable), len)?; child_idx += 1; - let value = if proto.has_value { - let v = children.get(child_idx, &DType::Binary(proto.value_nullable.into()), len)?; + let value = if metadata.has_value { + let v = children.get( + child_idx, + &DType::Binary(metadata.value_nullable.into()), + len, + )?; child_idx += 1; Some(v) } else { @@ -207,7 +256,9 @@ impl VTable for ParquetVariant { let typed_value = if has_typed_value { // typed_value can be any type — primitive, list, struct, etc. - let dtype = typed_value_dtype + let dtype = metadata + .typed_value_dtype + .clone() .ok_or_else(|| vortex_err!("typed_value_dtype missing for typed_value child"))?; let tv = children.get(child_idx, &dtype, len)?; Some(tv) @@ -262,8 +313,8 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; + use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; - use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; @@ -273,6 +324,8 @@ mod tests { use vortex_session::registry::ReadContext; use crate::ParquetVariant; + use crate::ParquetVariantData; + fn roundtrip(array: ArrayRef) -> ArrayRef { let dtype = array.dtype().clone(); let len = array.len(); @@ -290,7 +343,7 @@ mod tests { session.arrays().register(ParquetVariant); session.arrays().register(Variant); - let parts = SerializedArray::try_from(concat).unwrap(); + let parts = ArrayParts::try_from(concat).unwrap(); parts .decode(&dtype, len, &ReadContext::new(ctx.to_ids()), &session) .unwrap() @@ -304,7 +357,7 @@ mod tests { let inner_metadata = VarBinViewArray::from_iter_bin([b"\x01\x00", b"\x01\x00", b"\x01\x00"]).into_array(); let inner_value = VarBinViewArray::from_iter_bin([b"\x02", b"\x03", b"\x04"]).into_array(); - let inner_pv = ParquetVariant::try_new( + let inner_pv = ParquetVariantData::try_new( Validity::NonNullable, inner_metadata, Some(inner_value), @@ -313,7 +366,7 @@ mod tests { .unwrap(); let typed_value = VariantArray::new(inner_pv.into_array()).into_array(); - let outer_pv = ParquetVariant::try_new( + let outer_pv = ParquetVariantData::try_new( Validity::NonNullable, outer_metadata, None, @@ -336,7 +389,7 @@ mod tests { let value = VarBinViewArray::from_iter_bin([b"\x10", b"\x11", b"\x12"]).into_array(); let validity = Validity::from(BitBuffer::from_iter([true, false, true])); - let pv = ParquetVariant::try_new(validity, metadata, Some(value), None).unwrap(); + let pv = ParquetVariantData::try_new(validity, metadata, Some(value), None).unwrap(); let array = pv.into_array(); let decoded = roundtrip(array.clone()); @@ -353,7 +406,7 @@ mod tests { VarBinViewArray::from_iter_bin([b"\x01\x00", b"\x01\x00", b"\x01\x00"]).into_array(); let typed_value = buffer![10i32, 20, 30].into_array(); - let outer_pv = ParquetVariant::try_new( + let outer_pv = ParquetVariantData::try_new( Validity::NonNullable, outer_metadata, None, diff --git a/encodings/pco/public-api.lock b/encodings/pco/public-api.lock index 4513321e259..21d9c41cec2 100644 --- a/encodings/pco/public-api.lock +++ b/encodings/pco/public-api.lock @@ -20,6 +20,8 @@ impl vortex_array::array::vtable::VTable for vortex_pco::Pco pub type vortex_pco::Pco::ArrayData = vortex_pco::PcoData +pub type vortex_pco::Pco::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_pco::Pco::OperationsVTable = vortex_pco::Pco pub type vortex_pco::Pco::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValiditySliceHelper @@ -32,23 +34,33 @@ pub fn vortex_pco::Pco::buffer(array: vortex_array::array::view::ArrayView<'_, S pub fn vortex_pco::Pco::buffer_name(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_pco::Pco::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_pco::Pco::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_pco::Pco::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_pco::Pco::dtype(array: &vortex_pco::PcoData) -> &vortex_array::dtype::DType pub fn vortex_pco::Pco::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_pco::Pco::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_pco::Pco::len(array: &vortex_pco::PcoData) -> usize + +pub fn vortex_pco::Pco::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_pco::Pco::nbuffers(array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_pco::Pco::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_pco::Pco::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_pco::Pco::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_pco::Pco::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_pco::Pco::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_pco::Pco::validate(&self, data: &vortex_pco::PcoData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_pco::Pco::stats(array: &vortex_pco::PcoData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_pco::Pco::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_pco::Pco::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -92,6 +104,8 @@ impl vortex_pco::PcoData pub fn vortex_pco::PcoData::decompress(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_pco::PcoData::dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_pco::PcoData::from_array(array: vortex_array::array::erased::ArrayRef, level: usize, nums_per_page: usize) -> vortex_error::VortexResult pub fn vortex_pco::PcoData::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: usize, values_per_page: usize) -> vortex_error::VortexResult @@ -100,18 +114,24 @@ pub fn vortex_pco::PcoData::is_empty(&self) -> bool pub fn vortex_pco::PcoData::len(&self) -> usize -pub fn vortex_pco::PcoData::new(chunk_metas: alloc::vec::Vec, pages: alloc::vec::Vec, ptype: vortex_array::dtype::ptype::PType, metadata: vortex_pco::PcoMetadata, len: usize, validity: vortex_array::validity::Validity) -> Self - -pub fn vortex_pco::PcoData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_pco::PcoData::new(chunk_metas: alloc::vec::Vec, pages: alloc::vec::Vec, dtype: vortex_array::dtype::DType, metadata: vortex_pco::PcoMetadata, len: usize, validity: vortex_array::validity::Validity) -> Self impl core::clone::Clone for vortex_pco::PcoData pub fn vortex_pco::PcoData::clone(&self) -> vortex_pco::PcoData +impl core::convert::From for vortex_array::array::erased::ArrayRef + +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_pco::PcoData) -> vortex_array::array::erased::ArrayRef + impl core::fmt::Debug for vortex_pco::PcoData pub fn vortex_pco::PcoData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::array::IntoArray for vortex_pco::PcoData + +pub fn vortex_pco::PcoData::into_array(self) -> vortex_array::array::erased::ArrayRef + impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_pco::PcoData pub fn vortex_pco::PcoData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 265566ec35e..1365d8ad087 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -19,7 +19,6 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -27,6 +26,7 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::Precision; +use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; @@ -37,6 +37,7 @@ use vortex_array::dtype::PType; use vortex_array::dtype::half; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; @@ -83,15 +84,28 @@ vtable!(Pco, Pco, PcoData); impl VTable for Pco { type ArrayData = PcoData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValiditySliceHelper; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Pco + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &PcoData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) + fn len(array: &PcoData) -> usize { + array.slice_stop - array.slice_start + } + + fn dtype(array: &PcoData) -> &DType { + &array.dtype + } + + fn stats(array: &PcoData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &PcoData, state: &mut H, precision: Precision) { @@ -154,20 +168,31 @@ impl VTable for Pco { } } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(array.metadata.clone().encode_to_vec())) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(array.metadata.clone())) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.0.encode_to_vec())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata(PcoMetadata::decode(bytes)?)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = PcoMetadata::decode(metadata)?; let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { @@ -177,17 +202,18 @@ impl VTable for Pco { vortex_bail!("PcoArray expected 0 or 1 child, got {}", children.len()); }; - vortex_ensure!(buffers.len() >= metadata.chunks.len()); - let chunk_metas = buffers[..metadata.chunks.len()] + vortex_ensure!(buffers.len() >= metadata.0.chunks.len()); + let chunk_metas = buffers[..metadata.0.chunks.len()] .iter() .map(|b| b.clone().try_to_host_sync()) .collect::>>()?; - let pages = buffers[metadata.chunks.len()..] + let pages = buffers[metadata.0.chunks.len()..] .iter() .map(|b| b.clone().try_to_host_sync()) .collect::>>()?; let expected_n_pages = metadata + .0 .chunks .iter() .map(|info| info.pages.len()) @@ -197,8 +223,8 @@ impl VTable for Pco { Ok(PcoData::new( chunk_metas, pages, - dtype.as_ptype(), - metadata, + dtype.clone(), + metadata.0.clone(), len, validity, )) @@ -221,7 +247,7 @@ impl VTable for Pco { ); array.unsliced_validity = match &slots[VALIDITY_SLOT] { Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.unsliced_validity.nullability()), + None => Validity::from(array.dtype.nullability()), }; array.slots = slots; Ok(()) @@ -241,10 +267,7 @@ impl VTable for Pco { } pub(crate) fn number_type_from_dtype(dtype: &DType) -> NumberType { - number_type_from_ptype(dtype.as_ptype()) -} - -pub(crate) fn number_type_from_ptype(ptype: PType) -> NumberType { + let ptype = dtype.as_ptype(); match ptype { PType::F16 => NumberType::F16, PType::F32 => NumberType::F32, @@ -279,21 +302,13 @@ pub struct Pco; impl Pco { pub const ID: ArrayId = ArrayId::new_ref("vortex.pco"); - pub(crate) fn try_new(dtype: DType, data: PcoData) -> VortexResult { - let len = data.len(); - data.validate(&dtype, len)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Pco, dtype, len, data)) }) - } - /// Compress a primitive array using pcodec. pub fn from_primitive( parray: &PrimitiveArray, level: usize, values_per_page: usize, ) -> VortexResult { - let dtype = parray.dtype().clone(); - let data = PcoData::from_primitive(parray, level, values_per_page)?; - Self::try_new(dtype, data) + Array::try_from_data(PcoData::from_primitive(parray, level, values_per_page)?) } } @@ -307,72 +322,20 @@ pub struct PcoData { pub(crate) chunk_metas: Vec, pub(crate) pages: Vec, pub(crate) metadata: PcoMetadata, - ptype: PType, + dtype: DType, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, pub(super) slots: Vec>, + stats_set: ArrayStats, slice_start: usize, slice_stop: usize, } impl PcoData { - pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { - let _ = number_type_from_ptype(self.ptype); - vortex_ensure!( - dtype.as_ptype() == self.ptype, - "expected ptype {}, got {}", - self.ptype, - dtype.as_ptype() - ); - vortex_ensure!( - dtype.nullability() == self.unsliced_validity.nullability(), - "expected nullability {}, got {}", - self.unsliced_validity.nullability(), - dtype.nullability() - ); - vortex_ensure!( - self.slice_start <= self.slice_stop && self.slice_stop <= self.unsliced_n_rows, - "invalid slice range {}..{} for {} rows", - self.slice_start, - self.slice_stop, - self.unsliced_n_rows - ); - vortex_ensure!( - self.slice_stop - self.slice_start == len, - "expected len {len}, got {}", - self.slice_stop - self.slice_start - ); - if let Some(validity_len) = self.unsliced_validity.maybe_len() { - vortex_ensure!( - validity_len == self.unsliced_n_rows, - "expected validity len {}, got {}", - self.unsliced_n_rows, - validity_len - ); - } - vortex_ensure!( - self.chunk_metas.len() == self.metadata.chunks.len(), - "expected {} chunk metas, got {}", - self.metadata.chunks.len(), - self.chunk_metas.len() - ); - vortex_ensure!( - self.pages.len() - == self - .metadata - .chunks - .iter() - .map(|chunk| chunk.pages.len()) - .sum::(), - "page count does not match metadata" - ); - Ok(()) - } - pub fn new( chunk_metas: Vec, pages: Vec, - ptype: PType, + dtype: DType, metadata: PcoMetadata, len: usize, validity: Validity, @@ -383,10 +346,11 @@ impl PcoData { chunk_metas, pages, metadata, - ptype, + dtype, unsliced_validity: validity, unsliced_n_rows: len, slots: vec![validity_slot], + stats_set: Default::default(), slice_start: 0, slice_stop: len, } @@ -466,7 +430,7 @@ impl PcoData { Ok(PcoData::new( chunk_meta_buffers, page_buffers, - parray.dtype().as_ptype(), + parray.dtype().clone(), metadata, parray.len(), parray.validity(), @@ -486,7 +450,7 @@ impl PcoData { pub fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { // To start, we figure out which chunks and pages we need to decompress, and with // what value offset into the first such page. - let number_type = number_type_from_ptype(self.ptype); + let number_type = number_type_from_dtype(&self.dtype); let values_byte_buffer = match_number_enum!( number_type, NumberType => { @@ -496,7 +460,7 @@ impl PcoData { Ok(PrimitiveArray::from_values_byte_buffer( values_byte_buffer, - self.ptype, + self.dtype.as_ptype(), self.unsliced_validity .slice(self.slice_start..self.slice_stop)?, self.slice_stop - self.slice_start, @@ -583,6 +547,7 @@ impl PcoData { PcoData { slice_start: self.slice_start + start, slice_stop: self.slice_start + stop, + stats_set: Default::default(), ..self.clone() } } @@ -597,6 +562,11 @@ impl PcoData { self.slice_stop == self.slice_start } + /// Returns the logical data type of the array. + pub fn dtype(&self) -> &DType { + &self.dtype + } + pub(crate) fn slice_start(&self) -> usize { self.slice_start } diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index 877535bbd97..378f07254ac 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -29,17 +29,18 @@ impl CastReduce for Pco { .clone() .cast_nullability(dtype.nullability(), array.len())?; - let data = PcoData::new( - array.chunk_metas.clone(), - array.pages.clone(), - dtype.as_ptype(), - array.metadata.clone(), - array.unsliced_n_rows(), - new_validity, - ) - ._slice(array.slice_start(), array.slice_stop()); - - return Ok(Some(Pco::try_new(dtype.clone(), data)?.into_array())); + return Ok(Some( + PcoData::new( + array.chunk_metas.clone(), + array.pages.clone(), + dtype.clone(), + array.metadata.clone(), + array.unsliced_n_rows(), + new_validity, + ) + ._slice(array.slice_start(), array.slice_stop()) + .into_array(), + )); } // For other casts (e.g., numeric type changes), decode to canonical and let PrimitiveArray handle it diff --git a/encodings/pco/src/slice.rs b/encodings/pco/src/slice.rs index a53a1e39898..bdaa1006294 100644 --- a/encodings/pco/src/slice.rs +++ b/encodings/pco/src/slice.rs @@ -13,8 +13,6 @@ use crate::Pco; impl SliceReduce for Pco { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some( - Pco::try_new(array.dtype().clone(), array._slice(range.start, range.end))?.into_array(), - )) + Ok(Some(array._slice(range.start, range.end).into_array())) } } diff --git a/encodings/pco/src/test.rs b/encodings/pco/src/test.rs index 0bcbd8837ee..bfadb843fd4 100644 --- a/encodings/pco/src/test.rs +++ b/encodings/pco/src/test.rs @@ -17,8 +17,8 @@ use vortex_array::assert_nth_scalar; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; +use vortex_array::serde::ArrayParts; use vortex_array::serde::SerializeOptions; -use vortex_array::serde::SerializedArray; use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; @@ -39,6 +39,8 @@ static SESSION: LazyLock = LazyLock::new(|| { }); use crate::Pco; +use crate::PcoArray; + #[test] fn test_compress_decompress() { let data: Vec = (0..200).collect(); @@ -99,8 +101,7 @@ fn test_validity_and_multiple_chunks_and_pages() { let compression_level = 3; let values_per_chunk = 33; let values_per_page = 10; - let compressed = Pco::try_new( - array.dtype().clone(), + let compressed = PcoArray::try_from_data( PcoData::from_primitive_with_values_per_chunk( &array, compression_level, @@ -178,7 +179,7 @@ fn test_serde() -> VortexResult<()> { .collect::>() .freeze(); - let parts = SerializedArray::try_from(bytes)?; + let parts = ArrayParts::try_from(bytes)?; let decoded = parts.decode( &DType::Primitive(PType::I32, Nullability::NonNullable), 1_000_000, diff --git a/encodings/runend/public-api.lock b/encodings/runend/public-api.lock index 975353acb75..a1ded74c85e 100644 --- a/encodings/runend/public-api.lock +++ b/encodings/runend/public-api.lock @@ -44,6 +44,8 @@ impl vortex_array::array::vtable::VTable for vortex_runend::RunEnd pub type vortex_runend::RunEnd::ArrayData = vortex_runend::RunEndData +pub type vortex_runend::RunEnd::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_runend::RunEnd::OperationsVTable = vortex_runend::RunEnd pub type vortex_runend::RunEnd::ValidityVTable = vortex_runend::RunEnd @@ -56,7 +58,11 @@ pub fn vortex_runend::RunEnd::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_runend::RunEnd::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_runend::RunEnd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_runend::RunEnd::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_runend::RunEnd::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_runend::RunEnd::dtype(array: &vortex_runend::RunEndData) -> &vortex_array::dtype::DType pub fn vortex_runend::RunEnd::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -64,17 +70,23 @@ pub fn vortex_runend::RunEnd::execute_parent(array: vortex_array::array::view::A pub fn vortex_runend::RunEnd::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_runend::RunEnd::len(array: &vortex_runend::RunEndData) -> usize + +pub fn vortex_runend::RunEnd::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_runend::RunEnd::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_runend::RunEnd::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_runend::RunEnd::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_runend::RunEnd::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_runend::RunEnd::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_runend::RunEnd::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_runend::RunEnd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_runend::RunEnd::stats(array: &vortex_runend::RunEndData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_runend::RunEnd::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_runend::RunEnd::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -106,6 +118,12 @@ impl vortex_array::scalar_fn::fns::fill_null::kernel::FillNullReduce for vortex_ pub fn vortex_runend::RunEnd::fill_null(array: vortex_array::array::view::ArrayView<'_, Self>, fill_value: &vortex_array::scalar::Scalar) -> vortex_error::VortexResult> +pub struct vortex_runend::RunEndArrayParts + +pub vortex_runend::RunEndArrayParts::ends: vortex_array::array::erased::ArrayRef + +pub vortex_runend::RunEndArrayParts::values: vortex_array::array::erased::ArrayRef + pub struct vortex_runend::RunEndData impl vortex_runend::RunEndData @@ -118,35 +136,43 @@ pub fn vortex_runend::RunEndData::ends(&self) -> &vortex_array::array::erased::A pub fn vortex_runend::RunEndData::find_physical_index(&self, index: usize) -> vortex_error::VortexResult -pub fn vortex_runend::RunEndData::into_parts(self) -> vortex_runend::RunEndDataParts +pub fn vortex_runend::RunEndData::into_parts(self) -> vortex_runend::RunEndArrayParts + +pub fn vortex_runend::RunEndData::is_empty(&self) -> bool + +pub fn vortex_runend::RunEndData::len(&self) -> usize pub fn vortex_runend::RunEndData::new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> Self -pub unsafe fn vortex_runend::RunEndData::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, _length: usize) -> Self +pub unsafe fn vortex_runend::RunEndData::new_unchecked(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> Self pub fn vortex_runend::RunEndData::offset(&self) -> usize +pub fn vortex_runend::RunEndData::try_new(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef) -> vortex_error::VortexResult + +pub fn vortex_runend::RunEndData::try_new_offset_length(ends: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, offset: usize, length: usize) -> vortex_error::VortexResult + pub fn vortex_runend::RunEndData::values(&self) -> &vortex_array::array::erased::ArrayRef impl core::clone::Clone for vortex_runend::RunEndData pub fn vortex_runend::RunEndData::clone(&self) -> vortex_runend::RunEndData -impl core::fmt::Debug for vortex_runend::RunEndData +impl core::convert::From for vortex_array::array::erased::ArrayRef -pub fn vortex_runend::RunEndData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_runend::RunEndData) -> vortex_array::array::erased::ArrayRef -impl vortex_array::arrow::FromArrowArray<&arrow_array::array::run_array::RunArray> for vortex_runend::RunEndData where ::Native: vortex_array::dtype::ptype::NativePType +impl core::fmt::Debug for vortex_runend::RunEndData -pub fn vortex_runend::RunEndData::from_arrow(array: &arrow_array::array::run_array::RunArray, nullable: bool) -> vortex_error::VortexResult +pub fn vortex_runend::RunEndData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct vortex_runend::RunEndDataParts +impl vortex_array::array::IntoArray for vortex_runend::RunEndData -pub vortex_runend::RunEndDataParts::ends: vortex_array::array::erased::ArrayRef +pub fn vortex_runend::RunEndData::into_array(self) -> vortex_array::array::erased::ArrayRef -pub vortex_runend::RunEndDataParts::offset: usize +impl vortex_array::arrow::FromArrowArray<&arrow_array::array::run_array::RunArray> for vortex_runend::RunEndData where ::Native: vortex_array::dtype::ptype::NativePType -pub vortex_runend::RunEndDataParts::values: vortex_array::array::erased::ArrayRef +pub fn vortex_runend::RunEndData::from_arrow(array: &arrow_array::array::run_array::RunArray, nullable: bool) -> vortex_error::VortexResult pub struct vortex_runend::RunEndMetadata diff --git a/encodings/runend/src/arbitrary.rs b/encodings/runend/src/arbitrary.rs index 5347935c578..a63fa9bb2ed 100644 --- a/encodings/runend/src/arbitrary.rs +++ b/encodings/runend/src/arbitrary.rs @@ -14,12 +14,11 @@ use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexExpect; -use crate::RunEnd; -use crate::RunEndArray; +use crate::RunEndData; /// A wrapper type to implement `Arbitrary` for `RunEndArray`. #[derive(Clone, Debug)] -pub struct ArbitraryRunEndArray(pub RunEndArray); +pub struct ArbitraryRunEndArray(pub RunEndData); impl<'a> Arbitrary<'a> for ArbitraryRunEndArray { fn arbitrary(u: &mut Unstructured<'a>) -> Result { @@ -43,7 +42,7 @@ impl ArbitraryRunEndArray { // Empty RunEndArray let ends = PrimitiveArray::from_iter(Vec::::new()).into_array(); let values = ArbitraryArray::arbitrary_with(u, Some(0), dtype)?.0; - let runend_array = RunEnd::try_new(ends, values) + let runend_array = RunEndData::try_new(ends, values) .vortex_expect("Empty RunEndArray creation should succeed"); return Ok(ArbitraryRunEndArray(runend_array)); } @@ -55,7 +54,7 @@ impl ArbitraryRunEndArray { // Each end must be > previous end, and first end must be >= 1 let ends = random_strictly_sorted_ends(u, num_runs, len)?; - let runend_array = RunEnd::try_new(ends, values) + let runend_array = RunEndData::try_new(ends, values) .vortex_expect("RunEndArray creation should succeed in arbitrary impl"); Ok(ArbitraryRunEndArray(runend_array)) diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index a9fe74e7d36..081b48ab678 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -4,18 +4,19 @@ use std::fmt::Debug; use std::hash::Hash; -use prost::Message; use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; +use vortex_array::SerializeMetadata; use vortex_array::arrays::Primitive; use vortex_array::arrays::VarBinViewArray; use vortex_array::buffer::BufferHandle; @@ -26,6 +27,7 @@ use vortex_array::scalar::PValue; use vortex_array::search_sorted::SearchSorted; use vortex_array::search_sorted::SearchSortedSide; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -59,22 +61,28 @@ pub struct RunEndMetadata { impl VTable for RunEnd { type ArrayData = RunEndData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &RunEnd + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - RunEndData::validate(data.ends(), data.values(), data.offset, len)?; - vortex_ensure!( - data.values().dtype() == dtype, - "expected dtype {}, got {}", - dtype, - data.values().dtype() - ); - Ok(()) + fn len(array: &RunEndData) -> usize { + array.length + } + + fn dtype(array: &RunEndData) -> &DType { + array.values().dtype() + } + + fn stats(array: &RunEndData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &RunEndData, state: &mut H, precision: Precision) { @@ -101,28 +109,37 @@ impl VTable for RunEnd { vortex_panic!("RunEndArray buffer_name index {idx} out of bounds") } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - RunEndMetadata { - ends_ptype: PType::try_from(array.ends().dtype()) - .vortex_expect("Must be a valid PType") as i32, - num_runs: array.ends().len() as u64, - offset: array.offset() as u64, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(RunEndMetadata { + ends_ptype: PType::try_from(array.ends().dtype()).vortex_expect("Must be a valid PType") + as i32, + num_runs: array.ends().len() as u64, + offset: array.offset() as u64, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + let inner = as DeserializeMetadata>::deserialize(bytes)?; + Ok(ProstMetadata(inner)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = RunEndMetadata::decode(metadata)?; let ends_dtype = DType::Primitive(metadata.ends_ptype(), Nullability::NonNullable); let runs = usize::try_from(metadata.num_runs).vortex_expect("Must be a valid usize"); let ends = children.get(0, &ends_dtype, runs)?; @@ -189,12 +206,13 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["ends", "values"]; pub struct RunEndData { pub(super) slots: Vec>, offset: usize, + length: usize, + stats_set: ArrayStats, } -pub struct RunEndDataParts { +pub struct RunEndArrayParts { pub ends: ArrayRef, pub values: ArrayRef, - pub offset: usize, } #[derive(Clone, Debug)] @@ -213,17 +231,13 @@ impl RunEnd { offset: usize, length: usize, ) -> RunEndArray { - let data = unsafe { RunEndData::new_unchecked(ends, values, offset, length) }; - let dtype = data.dtype().clone(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, length, data)) } + Array::try_from_data(unsafe { RunEndData::new_unchecked(ends, values, offset, length) }) + .vortex_expect("RunEndData is always valid") } /// Build a new [`RunEndArray`] from ends and values. pub fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { - let len = RunEndData::logical_len_from_ends(&ends)?; - let data = RunEndData::try_new_offset_length(ends, values, 0, len)?; - let dtype = data.dtype().clone(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, len, data)) }) + Array::try_from_data(RunEndData::try_new(ends, values)?) } /// Build a new [`RunEndArray`] from ends, values, offset, and length. @@ -233,34 +247,24 @@ impl RunEnd { offset: usize, length: usize, ) -> VortexResult { - let data = RunEndData::try_new_offset_length(ends, values, offset, length)?; - let dtype = data.dtype().clone(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, length, data)) }) + Array::try_from_data(RunEndData::try_new_offset_length( + ends, values, offset, length, + )?) } /// Build a new [`RunEndArray`] from ends and values (panics on invalid input). pub fn new(ends: ArrayRef, values: ArrayRef) -> RunEndArray { - Self::try_new(ends, values).vortex_expect("RunEndData is always valid") + Array::try_from_data(RunEndData::new(ends, values)) + .vortex_expect("RunEndData is always valid") } /// Run the array through run-end encoding. pub fn encode(array: ArrayRef) -> VortexResult { - let len = array.len(); - let data = RunEndData::encode(array)?; - let dtype = data.dtype().clone(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(RunEnd, dtype, len, data)) }) + Array::try_from_data(RunEndData::encode(array)?) } } impl RunEndData { - fn logical_len_from_ends(ends: &ArrayRef) -> VortexResult { - if ends.is_empty() { - Ok(0) - } else { - usize::try_from(&ends.scalar_at(ends.len() - 1)?) - } - } - fn validate( ends: &ArrayRef, values: &ArrayRef, @@ -289,8 +293,12 @@ impl RunEndData { return Ok(()); } - // Zero-length logical slices may retain run metadata from the source array. + // Avoid building a non-empty array with zero logical length. if length == 0 { + vortex_ensure!( + ends.is_empty(), + "run ends must be empty when length is zero" + ); return Ok(()); } @@ -317,8 +325,8 @@ impl RunEndData { // Validate the offset and length are valid for the given ends and values if offset != 0 && length != 0 { let first_run_end = usize::try_from(&ends.scalar_at(0)?)?; - if first_run_end < offset { - vortex_bail!("First run end {first_run_end} must be >= offset {offset}"); + if first_run_end <= offset { + vortex_bail!("First run end {first_run_end} must be bigger than offset {offset}"); } } @@ -335,7 +343,7 @@ impl RunEndData { impl RunEndData { /// Build a new `RunEndArray` from an array of run `ends` and an array of `values`. /// - /// Panics if any of the validation conditions described in [`RunEnd::try_new`] is + /// Panics if any of the validation conditions described in [`RunEndData::try_new`] is /// not satisfied. /// /// # Examples @@ -367,15 +375,20 @@ impl RunEndData { /// # Validation /// /// The `ends` must be non-nullable unsigned integers. - pub(crate) fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { - let length = Self::logical_len_from_ends(&ends)?; + pub fn try_new(ends: ArrayRef, values: ArrayRef) -> VortexResult { + let length: usize = if ends.is_empty() { + 0 + } else { + usize::try_from(&ends.scalar_at(ends.len() - 1)?)? + }; + Self::try_new_offset_length(ends, values, 0, length) } /// Construct a new sliced `RunEndArray` with the provided offset and length. /// - /// This performs all the same validation as [`RunEnd::try_new_offset_length`]. - pub(crate) fn try_new_offset_length( + /// This performs all the same validation as [`RunEndData::try_new`]. + pub fn try_new_offset_length( ends: ArrayRef, values: ArrayRef, offset: usize, @@ -386,6 +399,8 @@ impl RunEndData { Ok(Self { slots: vec![Some(ends), Some(values)], offset, + length, + stats_set: Default::default(), }) } @@ -393,20 +408,21 @@ impl RunEndData { /// /// # Safety /// - /// The caller must ensure that all the validation performed in - /// [`RunEnd::try_new_offset_length`] is + /// The caller must ensure that all the validation performed in [`RunEndData::try_new`] is /// satisfied before calling this function. /// - /// See [`RunEnd::try_new_offset_length`] for the preconditions needed to build a new array. + /// See [`RunEndData::try_new`] for the preconditions needed to build a new array. pub unsafe fn new_unchecked( ends: ArrayRef, values: ArrayRef, offset: usize, - _length: usize, + length: usize, ) -> Self { Self { slots: vec![Some(ends), Some(values)], offset, + length, + stats_set: Default::default(), } } @@ -440,6 +456,18 @@ impl RunEndData { } } + /// Returns the length of the array. + #[inline] + pub fn len(&self) -> usize { + self.length + } + + /// Returns whether the array is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.length == 0 + } + /// Returns the logical data type of the array. #[inline] pub fn dtype(&self) -> &DType { @@ -476,15 +504,16 @@ impl RunEndData { .vortex_expect("RunEndArray values slot") } - pub fn into_parts(mut self) -> RunEndDataParts { - RunEndDataParts { + /// Split an `RunEndArray` into parts. + #[inline] + pub fn into_parts(mut self) -> RunEndArrayParts { + RunEndArrayParts { ends: self.slots[ENDS_SLOT] .take() .vortex_expect("RunEndArray ends slot"), values: self.slots[VALUES_SLOT] .take() .vortex_expect("RunEndArray values slot"), - offset: self.offset, } } } @@ -495,7 +524,7 @@ impl ValidityVTable for RunEnd { Validity::NonNullable | Validity::AllValid => Validity::AllValid, Validity::AllInvalid => Validity::AllInvalid, Validity::Array(values_validity) => Validity::Array(unsafe { - RunEnd::new_unchecked( + RunEndData::new_unchecked( array.ends().clone(), values_validity, array.offset(), diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index 8d9e674f778..82eb837f48f 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -30,12 +30,7 @@ where .reinterpret_cast(R::Native::PTYPE.to_unsigned()); let values = ArrayRef::from_arrow(array.values().as_ref(), nullable)?; - let ends_array = PrimitiveArray::from_buffer_handle( - ends.buffer_handle().clone(), - ends.ptype(), - ends.validity(), - ) - .into_array(); + let ends_array = ends.into_array(); let (ends_slice, values_slice) = if offset == 0 && len == array.run_ends().max_value() { (ends_array, values) } else { @@ -51,7 +46,7 @@ where ) }; - // SAFETY: arrow-rs enforces the RunEndArray invariants, we inherit their guarantees. + // SAFETY: arrow-rs enforces the RunEndArray invariants, we inherit their guarantees Ok(unsafe { RunEndData::new_unchecked(ends_slice, values_slice, offset, len) }) } } @@ -61,14 +56,12 @@ mod tests { use std::sync::Arc; use std::sync::LazyLock; - use arrow_array::Array as _; use arrow_array::Float64Array; use arrow_array::Int32Array; use arrow_array::Int64Array; use arrow_array::RunArray; use arrow_array::types::Int32Type; use arrow_array::types::Int64Type; - use arrow_array::types::RunEndIndexType; use arrow_schema::DataType; use arrow_schema::Field; use rstest::rstest; @@ -79,7 +72,6 @@ mod tests { use vortex_array::arrow::FromArrowArray; use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; - use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::session::ArraySession; @@ -87,28 +79,11 @@ mod tests { use vortex_error::VortexResult; use vortex_session::VortexSession; - use crate::RunEnd; use crate::RunEndData; static SESSION: LazyLock = LazyLock::new(|| VortexSession::empty().with::()); - fn decode_run_array( - array: &RunArray, - nullable: bool, - ) -> VortexResult - where - R::Native: NativePType, - { - let data = RunEndData::from_arrow(array, nullable)?; - RunEnd::try_new_offset_length( - data.ends().clone(), - data.values().clone(), - data.offset(), - array.len(), - ) - } - #[test] fn test_arrow_run_array_to_vortex() -> VortexResult<()> { // Create an Arrow RunArray with UInt32 run ends and Int32 values @@ -119,7 +94,7 @@ mod tests { let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); // Convert to Vortex - let vortex_array = decode_run_array(&arrow_run_array, false)?; + let vortex_array = RunEndData::from_arrow(&arrow_run_array, false)?; assert_arrays_eq!( vortex_array.into_array(), @@ -136,7 +111,7 @@ mod tests { let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); // Convert to Vortex with nullable=true - let vortex_array = decode_run_array(&arrow_run_array, true)?; + let vortex_array = RunEndData::from_arrow(&arrow_run_array, true)?; assert_arrays_eq!( vortex_array.into_array(), @@ -160,7 +135,7 @@ mod tests { let arrow_run_array = RunArray::::try_new(&run_ends, &values).unwrap(); // Convert to Vortex - let vortex_array = decode_run_array(&arrow_run_array, false)?; + let vortex_array = RunEndData::from_arrow(&arrow_run_array, false)?; assert_arrays_eq!(vortex_array, buffer![1.5f64, 2.5, 2.5, 3.5].into_array()); Ok(()) @@ -180,7 +155,7 @@ mod tests { let sliced_array = arrow_run_array.slice(1, 6); // Convert the sliced array to Vortex - let vortex_array = decode_run_array(&sliced_array, false)?; + let vortex_array = RunEndData::from_arrow(&sliced_array, false)?; assert_arrays_eq!( vortex_array, buffer![100, 200, 200, 200, 300, 300].into_array() @@ -203,7 +178,7 @@ mod tests { let sliced_array = arrow_run_array.slice(4, 6); // Convert to Vortex with nullable=true - let vortex_array = decode_run_array(&sliced_array, true)?; + let vortex_array = RunEndData::from_arrow(&sliced_array, true)?; assert_arrays_eq!( vortex_array, @@ -234,7 +209,7 @@ mod tests { let sliced_array = arrow_run_array.slice(4, 0); // Convert to Vortex with nullable=true - let vortex_array = decode_run_array(&sliced_array, true)?; + let vortex_array = RunEndData::from_arrow(&sliced_array, true)?; // Verify properties assert_eq!(vortex_array.len(), 0); @@ -265,7 +240,7 @@ mod tests { &Int32Array::from(vec![3i32, 5, 8]), &Int32Array::from(vec![10, 20, 30]), )?; - let vortex_array = decode_run_array(&original, false)?; + let vortex_array = RunEndData::from_arrow(&original, false)?; let target = ree_type(DataType::Int32, DataType::Int32); let result = execute(vortex_array.into_array(), &target)?; @@ -298,7 +273,8 @@ mod tests { #[case] expected_ends: &[i32], #[case] expected_values: &[i32], ) -> VortexResult<()> { - let array = RunEnd::encode(PrimitiveArray::from_iter(input.iter().copied()).into_array())?; + let array = + RunEndData::encode(PrimitiveArray::from_iter(input.iter().copied()).into_array())?; let sliced = array.into_array().slice(slice_range.clone())?; let target = ree_type(DataType::Int32, DataType::Int32); let result = execute(sliced, &target)?; diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index d9ecb3a0baf..a5fec75d802 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -10,6 +10,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::RunEnd; +use crate::RunEndData; impl CastReduce for RunEnd { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // Cast the values array to the target type @@ -18,7 +19,7 @@ impl CastReduce for RunEnd { // SAFETY: casting does not affect the ends being valid unsafe { Ok(Some( - RunEnd::new_unchecked( + RunEndData::new_unchecked( array.ends().clone(), casted_values, array.offset(), diff --git a/encodings/runend/src/compute/fill_null.rs b/encodings/runend/src/compute/fill_null.rs index abe0ef44f09..bb138e406d6 100644 --- a/encodings/runend/src/compute/fill_null.rs +++ b/encodings/runend/src/compute/fill_null.rs @@ -10,6 +10,7 @@ use vortex_array::scalar_fn::fns::fill_null::FillNullReduce; use vortex_error::VortexResult; use crate::RunEnd; +use crate::RunEndData; impl FillNullReduce for RunEnd { fn fill_null( @@ -20,7 +21,7 @@ impl FillNullReduce for RunEnd { // SAFETY: modifying values only, does not affect ends Ok(Some( unsafe { - RunEnd::new_unchecked( + RunEndData::new_unchecked( array.ends().clone(), new_values, array.offset(), diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs index e0ef5382d40..92c3ef40b8e 100644 --- a/encodings/runend/src/compute/filter.rs +++ b/encodings/runend/src/compute/filter.rs @@ -21,6 +21,7 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::RunEnd; +use crate::RunEndData; use crate::compute::take::take_indices_unchecked; const FILTER_TAKE_THRESHOLD: f64 = 0.1; @@ -58,7 +59,7 @@ impl FilterKernel for RunEnd { // SAFETY: guaranteed by implementation of filter_run_end_primitive unsafe { Ok(Some( - RunEnd::new_unchecked( + RunEndData::new_unchecked( run_ends.into_array(), values, 0, diff --git a/encodings/runend/src/kernel.rs b/encodings/runend/src/kernel.rs index 97f17c17857..2ca1586bb51 100644 --- a/encodings/runend/src/kernel.rs +++ b/encodings/runend/src/kernel.rs @@ -17,6 +17,7 @@ use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; use vortex_error::VortexResult; use crate::RunEnd; +use crate::RunEndData; use crate::compute::take_from::RunEndTakeFrom; pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ @@ -44,11 +45,11 @@ impl ExecuteParentKernel for RunEndSliceKernel { _child_idx: usize, _ctx: &mut ExecutionCtx, ) -> VortexResult> { - slice(array, parent.slice_range().clone()).map(Some) + slice(&array, parent.slice_range().clone()).map(Some) } } -fn slice(array: ArrayView<'_, RunEnd>, range: Range) -> VortexResult { +fn slice(array: &RunEndData, range: Range) -> VortexResult { let new_length = range.len(); let slice_begin = array.find_physical_index(range.start)?; @@ -62,7 +63,7 @@ fn slice(array: ArrayView<'_, RunEnd>, range: Range) -> VortexResult = ParentRuleSet::new(&[ // CastReduceAdaptor must come before RunEndScalarFnRule so that cast operations are executed @@ -80,7 +81,7 @@ impl ArrayParentReduceRule for RunEndScalarFnRule { Ok(Some( unsafe { - RunEnd::new_unchecked( + RunEndData::new_unchecked( run_end.ends().clone(), new_values, run_end.offset(), diff --git a/encodings/sequence/public-api.lock b/encodings/sequence/public-api.lock index cbbbd492efb..381df9583e5 100644 --- a/encodings/sequence/public-api.lock +++ b/encodings/sequence/public-api.lock @@ -22,6 +22,8 @@ impl vortex_array::array::vtable::VTable for vortex_sequence::Sequence pub type vortex_sequence::Sequence::ArrayData = vortex_sequence::SequenceData +pub type vortex_sequence::Sequence::Metadata = vortex_sequence::array::SequenceMetadata + pub type vortex_sequence::Sequence::OperationsVTable = vortex_sequence::Sequence pub type vortex_sequence::Sequence::ValidityVTable = vortex_sequence::Sequence @@ -34,7 +36,11 @@ pub fn vortex_sequence::Sequence::buffer(_array: vortex_array::array::view::Arra pub fn vortex_sequence::Sequence::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_sequence::Sequence::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_sequence::Sequence::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_sequence::Sequence::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_sequence::Sequence::dtype(array: &vortex_sequence::SequenceData) -> &vortex_array::dtype::DType pub fn vortex_sequence::Sequence::execute(array: vortex_array::array::typed::Array, _ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -42,17 +48,23 @@ pub fn vortex_sequence::Sequence::execute_parent(array: vortex_array::array::vie pub fn vortex_sequence::Sequence::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_sequence::Sequence::len(array: &vortex_sequence::SequenceData) -> usize + +pub fn vortex_sequence::Sequence::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_sequence::Sequence::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_sequence::Sequence::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_sequence::Sequence::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_sequence::Sequence::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_sequence::Sequence::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_sequence::Sequence::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_sequence::Sequence::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_sequence::Sequence::stats(array: &vortex_sequence::SequenceData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_sequence::Sequence::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_sequence::Sequence::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -88,35 +100,57 @@ impl vortex_array::scalar_fn::fns::list_contains::kernel::ListContainsElementRed pub fn vortex_sequence::Sequence::list_contains(list: &vortex_array::array::erased::ArrayRef, element: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult> +pub struct vortex_sequence::SequenceArrayParts + +pub vortex_sequence::SequenceArrayParts::base: vortex_array::scalar::typed_view::primitive::pvalue::PValue + +pub vortex_sequence::SequenceArrayParts::len: usize + +pub vortex_sequence::SequenceArrayParts::multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue + +pub vortex_sequence::SequenceArrayParts::nullability: vortex_array::dtype::nullability::Nullability + +pub vortex_sequence::SequenceArrayParts::ptype: vortex_array::dtype::ptype::PType + pub struct vortex_sequence::SequenceData impl vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::base(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue -pub fn vortex_sequence::SequenceData::into_parts(self) -> vortex_sequence::SequenceDataParts +pub fn vortex_sequence::SequenceData::dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_sequence::SequenceData::into_parts(self) -> vortex_sequence::SequenceArrayParts + +pub fn vortex_sequence::SequenceData::is_empty(&self) -> bool + +pub fn vortex_sequence::SequenceData::last(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue + +pub fn vortex_sequence::SequenceData::len(&self) -> usize pub fn vortex_sequence::SequenceData::multiplier(&self) -> vortex_array::scalar::typed_view::primitive::pvalue::PValue pub fn vortex_sequence::SequenceData::ptype(&self) -> vortex_array::dtype::ptype::PType -pub fn vortex_sequence::SequenceData::validate(base: vortex_array::scalar::typed_view::primitive::pvalue::PValue, multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue, dtype: &vortex_array::dtype::DType, length: usize) -> vortex_error::VortexResult<()> +pub fn vortex_sequence::SequenceData::try_new(base: vortex_array::scalar::typed_view::primitive::pvalue::PValue, multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue, ptype: vortex_array::dtype::ptype::PType, nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_error::VortexResult + +pub fn vortex_sequence::SequenceData::try_new_typed>(base: T, multiplier: T, nullability: vortex_array::dtype::nullability::Nullability, length: usize) -> vortex_error::VortexResult impl core::clone::Clone for vortex_sequence::SequenceData pub fn vortex_sequence::SequenceData::clone(&self) -> vortex_sequence::SequenceData -impl core::fmt::Debug for vortex_sequence::SequenceData +impl core::convert::From for vortex_array::array::erased::ArrayRef -pub fn vortex_sequence::SequenceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_sequence::SequenceData) -> vortex_array::array::erased::ArrayRef -pub struct vortex_sequence::SequenceDataParts +impl core::fmt::Debug for vortex_sequence::SequenceData -pub vortex_sequence::SequenceDataParts::base: vortex_array::scalar::typed_view::primitive::pvalue::PValue +pub fn vortex_sequence::SequenceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_sequence::SequenceDataParts::multiplier: vortex_array::scalar::typed_view::primitive::pvalue::PValue +impl vortex_array::array::IntoArray for vortex_sequence::SequenceData -pub vortex_sequence::SequenceDataParts::ptype: vortex_array::dtype::ptype::PType +pub fn vortex_sequence::SequenceData::into_array(self) -> vortex_array::array::erased::ArrayRef pub fn vortex_sequence::initialize(session: &vortex_session::VortexSession) diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index a3d50a420e1..4b38b5f9124 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -4,15 +4,16 @@ use std::hash::Hash; use num_traits::cast::FromPrimitive; -use prost::Message; use vortex_array::Array; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::Precision; +use vortex_array::ProstMetadata; +use vortex_array::SerializeMetadata; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; @@ -28,6 +29,7 @@ use vortex_array::scalar::PValue; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSet; use vortex_array::validity::Validity; use vortex_array::vtable; @@ -48,14 +50,29 @@ use crate::rules::RULES; vtable!(Sequence, Sequence, SequenceData); -#[derive(Clone, prost::Message)] +#[derive(Debug, Clone, Copy)] pub struct SequenceMetadata { + base: PValue, + multiplier: PValue, +} + +#[derive(Clone, prost::Message)] +pub struct ProstSequenceMetadata { #[prost(message, tag = "1")] base: Option, #[prost(message, tag = "2")] multiplier: Option, } +/// Components of [`SequenceArray`]. +pub struct SequenceArrayParts { + pub base: PValue, + pub multiplier: PValue, + pub len: usize, + pub ptype: PType, + pub nullability: Nullability, +} + pub(super) const SLOT_NAMES: [&str; 0] = []; #[derive(Clone, Debug)] @@ -63,17 +80,14 @@ pub(super) const SLOT_NAMES: [&str; 0] = []; pub struct SequenceData { base: PValue, multiplier: PValue, + dtype: DType, + pub(crate) len: usize, pub(super) slots: Vec>, -} - -pub struct SequenceDataParts { - pub base: PValue, - pub multiplier: PValue, - pub ptype: PType, + stats_set: ArrayStats, } impl SequenceData { - pub(crate) fn try_new_typed>( + pub fn try_new_typed>( base: T, multiplier: T, nullability: Nullability, @@ -89,70 +103,99 @@ impl SequenceData { } /// Constructs a sequence array using two integer values (with the same ptype). - pub(crate) fn try_new( + pub fn try_new( base: PValue, multiplier: PValue, ptype: PType, nullability: Nullability, length: usize, ) -> VortexResult { - let dtype = DType::Primitive(ptype, nullability); - Self::validate(base, multiplier, &dtype, length)?; - let (base, multiplier) = Self::normalize(base, multiplier, ptype)?; - - Ok(unsafe { Self::new_unchecked(base, multiplier) }) - } - - pub fn validate( - base: PValue, - multiplier: PValue, - dtype: &DType, - length: usize, - ) -> VortexResult<()> { - let DType::Primitive(ptype, _) = dtype else { - vortex_bail!("only primitive dtypes are supported in SequenceArray currently"); - }; - if !ptype.is_int() { vortex_bail!("only integer ptype are supported in SequenceArray currently") } - vortex_ensure!(length > 0, "SequenceArray length must be greater than zero"); - Self::try_last(base, multiplier, *ptype, length).map_err(|e| { + Self::try_last(base, multiplier, ptype, length).map_err(|e| { e.with_context(format!( "final value not expressible, base = {base:?}, multiplier = {multiplier:?}, len = {length} ", )) })?; - Ok(()) + // SAFETY: we just validated that `ptype` is an integer and that the final + // element is representable via `try_last`. + Ok(unsafe { Self::new_unchecked(base, multiplier, ptype, nullability, length) }) } - fn normalize(base: PValue, multiplier: PValue, ptype: PType) -> VortexResult<(PValue, PValue)> { - match_each_integer_ptype!(ptype, |P| { - Ok(( - PValue::from(base.cast::

()?), - PValue::from(multiplier.cast::

()?), - )) - }) - } - - /// Constructs a [`SequenceArray`] payload without validation. + /// Constructs a [`SequenceArray`] without validating that the `ptype` is an integer + /// type or that the final element is representable. /// /// # Safety /// /// The caller must ensure that: - /// - `base` and `multiplier` are both normalized to the same integer `ptype`. - /// - they are logically compatible with the outer dtype and len. - pub(crate) unsafe fn new_unchecked(base: PValue, multiplier: PValue) -> Self { + /// - `ptype` is an integer type (i.e., `ptype.is_int()` returns `true`). + /// - `base + (length - 1) * multiplier` does not overflow the range of `ptype`. + /// + /// Violating the first invariant will cause a panic. Violating the second will + /// cause silent wraparound when materializing elements, producing incorrect values. + pub(crate) unsafe fn new_unchecked( + base: PValue, + multiplier: PValue, + ptype: PType, + nullability: Nullability, + length: usize, + ) -> Self { + let dtype = DType::Primitive(ptype, nullability); + + // A sequence A[i] = base + i * multiplier is sorted iff multiplier >= 0, + // and strictly sorted iff multiplier > 0. + + let (is_sorted, is_strict_sorted) = match_each_pvalue!( + multiplier, + uint: |v| { (true, v> 0) }, + int: |v| { (v >= 0, v > 0) }, + float: |_v| { unreachable!("float multiplier not supported") } + ); + + // SAFETY: we don't have duplicate stats + let stats_set = unsafe { + StatsSet::new_unchecked(vec![ + (Stat::IsSorted, StatPrecision::Exact(is_sorted.into())), + ( + Stat::IsStrictSorted, + StatPrecision::Exact(is_strict_sorted.into()), + ), + ]) + }; + Self { base, multiplier, + dtype, + len: length, slots: vec![], + stats_set: ArrayStats::from(stats_set), } } + /// Returns the length of the array. + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Returns whether the array is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Returns the logical data type of the array. + #[inline] + pub fn dtype(&self) -> &DType { + &self.dtype + } + pub fn ptype(&self) -> PType { - self.base.ptype() + self.dtype.as_ptype() } pub fn base(&self) -> PValue { @@ -163,14 +206,6 @@ impl SequenceData { self.multiplier } - pub fn into_parts(self) -> SequenceDataParts { - SequenceDataParts { - base: self.base, - multiplier: self.multiplier, - ptype: self.base.ptype(), - } - } - pub(crate) fn try_last( base: PValue, multiplier: PValue, @@ -192,6 +227,8 @@ impl SequenceData { } pub(crate) fn index_value(&self, idx: usize) -> PValue { + assert!(idx < self.len, "index_value({idx}): index out of bounds"); + match_each_native_ptype!(self.ptype(), |P| { let base = self.base.cast::

().vortex_expect("must be able to cast"); let multiplier = self @@ -203,20 +240,49 @@ impl SequenceData { PValue::from(value) }) } + + /// Returns the validated final value of a sequence array + pub fn last(&self) -> PValue { + Self::try_last(self.base, self.multiplier, self.ptype(), self.len) + .vortex_expect("validated array") + } + + pub fn into_parts(self) -> SequenceArrayParts { + SequenceArrayParts { + base: self.base, + multiplier: self.multiplier, + len: self.len, + ptype: self.dtype.as_ptype(), + nullability: self.dtype.nullability(), + } + } } impl VTable for Sequence { type ArrayData = SequenceData; + type Metadata = SequenceMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Sequence + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - SequenceData::validate(data.base, data.multiplier, dtype, len) + fn len(array: &SequenceData) -> usize { + array.len + } + + fn dtype(array: &SequenceData) -> &DType { + &array.dtype + } + + fn stats(array: &SequenceData) -> &ArrayStats { + &array.stats_set } fn array_hash( @@ -244,41 +310,37 @@ impl VTable for Sequence { vortex_panic!("SequenceArray buffer_name index {idx} out of bounds") } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - let metadata = SequenceMetadata { - base: Some((&array.base()).into()), - multiplier: Some((&array.multiplier()).into()), - }; + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(SequenceMetadata { + base: array.base(), + multiplier: array.multiplier(), + }) + } - Ok(Some(metadata.encode_to_vec())) + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + let prost = ProstMetadata(ProstSequenceMetadata { + base: Some((&metadata.base).into()), + multiplier: Some((&metadata.multiplier).into()), + }); + + Ok(Some(prost.serialize())) } fn deserialize( - &self, + bytes: &[u8], dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, + _len: usize, + _buffers: &[BufferHandle], session: &VortexSession, - ) -> VortexResult { - vortex_ensure!( - buffers.is_empty(), - "SequenceArray expects 0 buffers, got {}", - buffers.len() - ); - vortex_ensure!( - children.is_empty(), - "SequenceArray expects 0 children, got {}", - children.len() - ); - let metadata = SequenceMetadata::decode(metadata)?; + ) -> VortexResult { + let prost = + as DeserializeMetadata>::deserialize(bytes)?; let ptype = dtype.as_ptype(); // We go via Scalar to validate that the value is valid for the ptype. let base = Scalar::from_proto_value( - metadata + prost .base .as_ref() .ok_or_else(|| vortex_err!("base required"))?, @@ -290,7 +352,7 @@ impl VTable for Sequence { .vortex_expect("sequence array base should be a non-nullable primitive"); let multiplier = Scalar::from_proto_value( - metadata + prost .multiplier .as_ref() .ok_or_else(|| vortex_err!("multiplier required"))?, @@ -301,7 +363,23 @@ impl VTable for Sequence { .pvalue() .vortex_expect("sequence array multiplier should be a non-nullable primitive"); - SequenceData::try_new(base, multiplier, ptype, dtype.nullability(), len) + Ok(SequenceMetadata { base, multiplier }) + } + + fn build( + dtype: &DType, + len: usize, + metadata: &Self::Metadata, + _buffers: &[BufferHandle], + _children: &dyn ArrayChildren, + ) -> VortexResult { + SequenceData::try_new( + metadata.base, + metadata.multiplier, + dtype.as_ptype(), + dtype.nullability(), + len, + ) } fn slots(array: ArrayView<'_, Self>) -> &[Option] { @@ -369,49 +447,6 @@ pub struct Sequence; impl Sequence { pub const ID: ArrayId = ArrayId::new_ref("vortex.sequence"); - fn stats(multiplier: PValue) -> StatsSet { - // A sequence A[i] = base + i * multiplier is sorted iff multiplier >= 0, - // and strictly sorted iff multiplier > 0. - let (is_sorted, is_strict_sorted) = match_each_pvalue!( - multiplier, - uint: |v| { (true, v > 0) }, - int: |v| { (v >= 0, v > 0) }, - float: |_v| { unreachable!("float multiplier not supported") } - ); - - // SAFETY: we don't have duplicate stats. - unsafe { - StatsSet::new_unchecked(vec![ - (Stat::IsSorted, StatPrecision::Exact(is_sorted.into())), - ( - Stat::IsStrictSorted, - StatPrecision::Exact(is_strict_sorted.into()), - ), - ]) - } - } - - /// Construct a new [`SequenceArray`] from pre-validated parts. - /// - /// # Safety - /// - /// Caller must ensure the sequence is logically compatible with the provided dtype and len. - pub(crate) unsafe fn new_unchecked( - base: PValue, - multiplier: PValue, - ptype: PType, - nullability: Nullability, - length: usize, - ) -> SequenceArray { - let dtype = DType::Primitive(ptype, nullability); - let (base, multiplier) = SequenceData::normalize(base, multiplier, ptype) - .vortex_expect("SequenceArray parts must be normalized to the target ptype"); - let stats = Self::stats(multiplier); - let data = unsafe { SequenceData::new_unchecked(base, multiplier) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Sequence, dtype, length, data)) } - .with_stats_set(stats) - } - /// Construct a new [`SequenceArray`] from its components. pub fn try_new( base: PValue, @@ -420,13 +455,13 @@ impl Sequence { nullability: Nullability, length: usize, ) -> VortexResult { - let dtype = DType::Primitive(ptype, nullability); - let data = SequenceData::try_new(base, multiplier, ptype, nullability, length)?; - let stats = Self::stats(data.multiplier()); - Ok( - unsafe { Array::from_parts_unchecked(ArrayParts::new(Sequence, dtype, length, data)) } - .with_stats_set(stats), - ) + Array::try_from_data(SequenceData::try_new( + base, + multiplier, + ptype, + nullability, + length, + )?) } /// Construct a new typed [`SequenceArray`] from base/multiplier values. @@ -436,14 +471,12 @@ impl Sequence { nullability: Nullability, length: usize, ) -> VortexResult { - let ptype = T::PTYPE; - let dtype = DType::Primitive(ptype, nullability); - let data = SequenceData::try_new_typed(base, multiplier, nullability, length)?; - let stats = Self::stats(data.multiplier()); - Ok( - unsafe { Array::from_parts_unchecked(ArrayParts::new(Sequence, dtype, length, data)) } - .with_stats_set(stats), - ) + Array::try_from_data(SequenceData::try_new_typed( + base, + multiplier, + nullability, + length, + )?) } } diff --git a/encodings/sequence/src/compress.rs b/encodings/sequence/src/compress.rs index 6321aa3df78..b073ace8dc2 100644 --- a/encodings/sequence/src/compress.rs +++ b/encodings/sequence/src/compress.rs @@ -18,10 +18,9 @@ use vortex_buffer::BufferMut; use vortex_buffer::trusted_len::TrustedLen; use vortex_error::VortexResult; -use crate::Sequence; use crate::SequenceArray; -use crate::SequenceData; /// An iterator that yields `base, base + step, base + 2*step, ...` via repeated addition. +use crate::SequenceData; struct SequenceIter { acc: T, step: T, @@ -113,7 +112,7 @@ fn encode_primitive_array + CheckedAdd + CheckedSu ) -> VortexResult> { if slice.len() == 1 { // The multiplier here can be any value, zero is chosen - return Sequence::try_new_typed(slice[0], P::zero(), nullability, 1) + return SequenceData::try_new_typed(slice[0], P::zero(), nullability, 1) .map(|a| Some(a.into_array())); } let base = slice[0]; @@ -134,7 +133,7 @@ fn encode_primitive_array + CheckedAdd + CheckedSu .windows(2) .all(|w| Some(w[1]) == w[0].checked_add(&multiplier)) .then_some( - Sequence::try_new_typed(base, multiplier, nullability, slice.len()) + SequenceData::try_new_typed(base, multiplier, nullability, slice.len()) .map(|a| a.into_array()), ) .transpose() diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs index 1eb9ddf47d8..54f3393f16e 100644 --- a/encodings/sequence/src/compute/cast.rs +++ b/encodings/sequence/src/compute/cast.rs @@ -13,6 +13,7 @@ use vortex_error::VortexResult; use vortex_error::vortex_err; use crate::Sequence; +use crate::SequenceData; impl CastReduce for Sequence { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { // SequenceArray represents arithmetic sequences (base + i * multiplier) which @@ -31,7 +32,7 @@ impl CastReduce for Sequence { // For SequenceArray, we can just create a new one with the same parameters // but different nullability return Ok(Some( - Sequence::try_new( + SequenceData::try_new( array.base(), array.multiplier(), *target_ptype, @@ -70,7 +71,7 @@ impl CastReduce for Sequence { .ok_or_else(|| vortex_err!("Cast resulted in null multiplier value"))?; return Ok(Some( - Sequence::try_new( + SequenceData::try_new( new_base, new_multiplier, *target_ptype, diff --git a/encodings/sequence/src/compute/min_max.rs b/encodings/sequence/src/compute/min_max.rs index 2ff8b2abd58..17441bbeb4b 100644 --- a/encodings/sequence/src/compute/min_max.rs +++ b/encodings/sequence/src/compute/min_max.rs @@ -16,7 +16,6 @@ use vortex_array::scalar::ScalarValue; use vortex_error::VortexResult; use crate::Sequence; -use crate::SequenceData; /// Sequence-specific min/max kernel. /// @@ -48,7 +47,7 @@ impl DynAggregateKernel for SequenceMinMaxKernel { } let base = seq.base(); - let last = SequenceData::try_last(base, seq.multiplier(), seq.ptype(), seq.len())?; + let last = seq.last(); // Determine min and max based on multiplier direction. // For unsigned types, multiplier is always >= 0. diff --git a/encodings/sequence/src/compute/slice.rs b/encodings/sequence/src/compute/slice.rs index c2b64b68ef4..3eb79f0b23b 100644 --- a/encodings/sequence/src/compute/slice.rs +++ b/encodings/sequence/src/compute/slice.rs @@ -10,13 +10,14 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::Sequence; +use crate::SequenceData; impl SliceReduce for Sequence { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // SAFETY: this is a slice of an already-validated `SequenceArray`, so this is still valid. Ok(Some( unsafe { - Sequence::new_unchecked( + SequenceData::new_unchecked( array.index_value(range.start), array.multiplier(), array.ptype(), diff --git a/encodings/sequence/src/lib.rs b/encodings/sequence/src/lib.rs index aec425e0bae..4881e261a1b 100644 --- a/encodings/sequence/src/lib.rs +++ b/encodings/sequence/src/lib.rs @@ -13,8 +13,8 @@ pub use array::Sequence; /// Represents the equation A\[i\] = a * i + b. /// This can be used for compression, fast comparisons and also for row ids. pub use array::SequenceArray; +pub use array::SequenceArrayParts; pub use array::SequenceData; -pub use array::SequenceDataParts; pub use compress::sequence_encode; use vortex_array::aggregate_fn::AggregateFnVTable; use vortex_array::aggregate_fn::fns::is_sorted::IsSorted; diff --git a/encodings/sparse/public-api.lock b/encodings/sparse/public-api.lock index b937d3df263..71b42ab8dbd 100644 --- a/encodings/sparse/public-api.lock +++ b/encodings/sparse/public-api.lock @@ -1,5 +1,25 @@ pub mod vortex_sparse +#[repr(C)] pub struct vortex_sparse::ProstPatchesMetadata + +impl core::clone::Clone for vortex_sparse::ProstPatchesMetadata + +pub fn vortex_sparse::ProstPatchesMetadata::clone(&self) -> vortex_sparse::ProstPatchesMetadata + +impl core::default::Default for vortex_sparse::ProstPatchesMetadata + +pub fn vortex_sparse::ProstPatchesMetadata::default() -> Self + +impl core::fmt::Debug for vortex_sparse::ProstPatchesMetadata + +pub fn vortex_sparse::ProstPatchesMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl prost::message::Message for vortex_sparse::ProstPatchesMetadata + +pub fn vortex_sparse::ProstPatchesMetadata::clear(&mut self) + +pub fn vortex_sparse::ProstPatchesMetadata::encoded_len(&self) -> usize + pub struct vortex_sparse::Sparse impl vortex_sparse::Sparse @@ -10,8 +30,6 @@ pub fn vortex_sparse::Sparse::encode(array: &vortex_array::array::erased::ArrayR pub fn vortex_sparse::Sparse::try_new(indices: vortex_array::array::erased::ArrayRef, values: vortex_array::array::erased::ArrayRef, len: usize, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult -pub fn vortex_sparse::Sparse::try_new_from_patches(patches: vortex_array::patches::Patches, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult - impl core::clone::Clone for vortex_sparse::Sparse pub fn vortex_sparse::Sparse::clone(&self) -> vortex_sparse::Sparse @@ -24,6 +42,8 @@ impl vortex_array::array::vtable::VTable for vortex_sparse::Sparse pub type vortex_sparse::Sparse::ArrayData = vortex_sparse::SparseData +pub type vortex_sparse::Sparse::Metadata = vortex_sparse::SparseMetadata + pub type vortex_sparse::Sparse::OperationsVTable = vortex_sparse::Sparse pub type vortex_sparse::Sparse::ValidityVTable = vortex_sparse::Sparse @@ -36,7 +56,11 @@ pub fn vortex_sparse::Sparse::buffer(array: vortex_array::array::view::ArrayView pub fn vortex_sparse::Sparse::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_sparse::Sparse::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_sparse::Sparse::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_sparse::Sparse::deserialize(bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_sparse::Sparse::dtype(array: &vortex_sparse::SparseData) -> &vortex_array::dtype::DType pub fn vortex_sparse::Sparse::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -44,17 +68,23 @@ pub fn vortex_sparse::Sparse::execute_parent(array: vortex_array::array::view::A pub fn vortex_sparse::Sparse::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_sparse::Sparse::len(array: &vortex_sparse::SparseData) -> usize + +pub fn vortex_sparse::Sparse::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_sparse::Sparse::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_sparse::Sparse::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_sparse::Sparse::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_sparse::Sparse::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_sparse::Sparse::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_sparse::Sparse::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_sparse::Sparse::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_sparse::Sparse::stats(array: &vortex_sparse::SparseData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_sparse::Sparse::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_sparse::Sparse::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -108,34 +138,26 @@ pub fn vortex_sparse::SparseData::try_new(indices: vortex_array::array::erased:: pub fn vortex_sparse::SparseData::try_new_from_patches(patches: vortex_array::patches::Patches, fill_value: vortex_array::scalar::Scalar) -> vortex_error::VortexResult -pub fn vortex_sparse::SparseData::validate(patches: &vortex_array::patches::Patches, fill_value: &vortex_array::scalar::Scalar, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> - impl core::clone::Clone for vortex_sparse::SparseData pub fn vortex_sparse::SparseData::clone(&self) -> vortex_sparse::SparseData -impl core::fmt::Debug for vortex_sparse::SparseData +impl core::convert::From for vortex_array::array::erased::ArrayRef -pub fn vortex_sparse::SparseData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_sparse::SparseData) -> vortex_array::array::erased::ArrayRef -#[repr(C)] pub struct vortex_sparse::SparseMetadata +impl core::fmt::Debug for vortex_sparse::SparseData -impl core::clone::Clone for vortex_sparse::SparseMetadata +pub fn vortex_sparse::SparseData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub fn vortex_sparse::SparseMetadata::clone(&self) -> vortex_sparse::SparseMetadata +impl vortex_array::array::IntoArray for vortex_sparse::SparseData -impl core::default::Default for vortex_sparse::SparseMetadata +pub fn vortex_sparse::SparseData::into_array(self) -> vortex_array::array::erased::ArrayRef -pub fn vortex_sparse::SparseMetadata::default() -> Self +pub struct vortex_sparse::SparseMetadata impl core::fmt::Debug for vortex_sparse::SparseMetadata pub fn vortex_sparse::SparseMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl prost::message::Message for vortex_sparse::SparseMetadata - -pub fn vortex_sparse::SparseMetadata::clear(&mut self) - -pub fn vortex_sparse::SparseMetadata::encoded_len(&self) -> usize - pub type vortex_sparse::SparseArray = vortex_array::array::typed::Array diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index c0da00a14d7..a3cb566dddb 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -5,13 +5,10 @@ use std::sync::Arc; use itertools::Itertools; use num_traits::NumCast; -use vortex_array::Array; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::BoolArray; -use vortex_array::arrays::Decimal; use vortex_array::arrays::FixedSizeListArray; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::NullArray; @@ -55,8 +52,8 @@ use vortex_error::vortex_bail; use vortex_error::vortex_panic; use crate::ConstantArray; -use crate::Sparse; use crate::SparseArray; +use crate::SparseData; pub(super) fn execute_sparse( array: &SparseArray, ctx: &mut ExecutionCtx, @@ -440,7 +437,7 @@ fn execute_sparse_struct( .cloned() .zip_eq(fill_values) .map(|(patch_values, fill_value)| unsafe { - Sparse::new_unchecked( + SparseData::new_unchecked( unresolved_patches .clone() .map_values(|_| Ok(patch_values)) @@ -477,10 +474,8 @@ fn execute_sparse_decimal( } } let filled_array = builder.finish_into_decimal(); - let dtype = filled_array.dtype().clone(); - let len = filled_array.len(); - let data = filled_array.into_data().patch(patches, ctx)?; - Ok(Array::try_from_parts(ArrayParts::new(Decimal, dtype, len, data))?.into_array()) + let array = filled_array.into_data().patch(patches, ctx)?; + Ok(array.into_array()) } fn execute_varbin( diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs index a9fe0046646..9806974979a 100644 --- a/encodings/sparse/src/compute/cast.rs +++ b/encodings/sparse/src/compute/cast.rs @@ -11,6 +11,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::Sparse; +use crate::SparseData; impl CastReduce for Sparse { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { let casted_patches = array @@ -27,7 +28,7 @@ impl CastReduce for Sparse { }; Ok(Some( - Sparse::try_new_from_patches(casted_patches, casted_fill)?.into_array(), + SparseData::try_new_from_patches(casted_patches, casted_fill)?.into_array(), )) } } diff --git a/encodings/sparse/src/compute/filter.rs b/encodings/sparse/src/compute/filter.rs index 2cc8064a6cd..a3d3d5a1946 100644 --- a/encodings/sparse/src/compute/filter.rs +++ b/encodings/sparse/src/compute/filter.rs @@ -11,6 +11,7 @@ use vortex_mask::Mask; use crate::ConstantArray; use crate::Sparse; +use crate::SparseData; impl FilterKernel for Sparse { fn filter( array: ArrayView<'_, Self>, @@ -26,7 +27,8 @@ impl FilterKernel for Sparse { }; Ok(Some( - Sparse::try_new_from_patches(new_patches, array.fill_scalar().clone())?.into_array(), + SparseData::try_new_from_patches(new_patches, array.fill_scalar().clone())? + .into_array(), )) } } diff --git a/encodings/sparse/src/compute/take.rs b/encodings/sparse/src/compute/take.rs index b0b5f83cad0..ebcc2570fc1 100644 --- a/encodings/sparse/src/compute/take.rs +++ b/encodings/sparse/src/compute/take.rs @@ -10,6 +10,7 @@ use vortex_error::VortexResult; use crate::ConstantArray; use crate::Sparse; +use crate::SparseData; impl TakeExecute for Sparse { fn take( array: ArrayView<'_, Self>, @@ -39,7 +40,7 @@ impl TakeExecute for Sparse { } Ok(Some( - Sparse::try_new_from_patches( + SparseData::try_new_from_patches( new_patches, array.fill_scalar().cast( &array diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index fc9613a369d..2d79238230b 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -10,7 +10,6 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; @@ -29,6 +28,7 @@ use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_array::scalar_fn::fns::operators::Operator; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::VTable; @@ -57,9 +57,15 @@ mod slice; vtable!(Sparse, Sparse, SparseData); +#[derive(Debug)] +pub struct SparseMetadata { + patches: PatchesMetadata, + fill_value: Scalar, +} + #[derive(Clone, prost::Message)] #[repr(C)] -pub struct SparseMetadata { +pub struct ProstPatchesMetadata { #[prost(message, required, tag = "1")] patches: PatchesMetadata, } @@ -67,15 +73,28 @@ pub struct SparseMetadata { impl VTable for Sparse { type ArrayData = SparseData; + type Metadata = SparseMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Sparse + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - SparseData::validate(data.patches(), data.fill_scalar(), dtype, len) + fn len(array: &SparseData) -> usize { + array.patches.array_len() + } + + fn dtype(array: &SparseData) -> &DType { + array.fill_scalar().dtype() + } + + fn stats(array: &SparseData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &SparseData, state: &mut H, precision: Precision) { @@ -109,24 +128,32 @@ impl VTable for Sparse { } } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { let patches = array.patches().to_metadata(array.len(), array.dtype())?; - let metadata = SparseMetadata { patches }; + + Ok(SparseMetadata { + patches, + fill_value: array.fill_value.clone(), + }) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + let prost_patches = ProstPatchesMetadata { + patches: metadata.patches, + }; // Note that we DO NOT serialize the fill value since that is stored in the buffers. - Ok(Some(metadata.encode_to_vec())) + Ok(Some(prost_patches.encode_to_vec())) } fn deserialize( - &self, + bytes: &[u8], dtype: &DType, - len: usize, - metadata: &[u8], + _len: usize, buffers: &[BufferHandle], - children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { - let metadata = SparseMetadata::decode(metadata)?; + ) -> VortexResult { + let prost_patches = ProstPatchesMetadata::decode(bytes)?; // Once we have the patches metadata, we need to get the fill value from the buffers. @@ -138,6 +165,19 @@ impl VTable for Sparse { let scalar_value = ScalarValue::from_proto_bytes(scalar_bytes, dtype, session)?; let fill_value = Scalar::try_new(dtype.clone(), scalar_value)?; + Ok(SparseMetadata { + patches: prost_patches.patches, + fill_value, + }) + } + + fn build( + dtype: &DType, + len: usize, + metadata: &Self::Metadata, + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + ) -> VortexResult { vortex_ensure_eq!( children.len(), 2, @@ -160,7 +200,7 @@ impl VTable for Sparse { patch_values, None, )?, - fill_value, + metadata.fill_value.clone(), ) } @@ -236,6 +276,7 @@ pub struct SparseData { pub(crate) slots: Vec>, patches: Patches, fill_value: Scalar, + stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -251,23 +292,7 @@ impl Sparse { len: usize, fill_value: Scalar, ) -> VortexResult { - let dtype = fill_value.dtype().clone(); - let data = SparseData::try_new(indices, values, len, fill_value)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) }) - } - - pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { - let dtype = fill_value.dtype().clone(); - let len = patches.array_len(); - let data = SparseData::try_new_from_patches(patches, fill_value)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) }) - } - - pub(crate) unsafe fn new_unchecked(patches: Patches, fill_value: Scalar) -> SparseArray { - let dtype = fill_value.dtype().clone(); - let len = patches.array_len(); - let data = unsafe { SparseData::new_unchecked(patches, fill_value) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Sparse, dtype, len, data)) } + Array::try_from_data(SparseData::try_new(indices, values, len, fill_value)?) } /// Encode the given array as a [`SparseArray`]. @@ -277,71 +302,6 @@ impl Sparse { } impl SparseData { - fn normalize_values_dtype(values: ArrayRef, fill_value: &Scalar) -> VortexResult { - let fill_dtype = fill_value.dtype(); - let values_dtype = values.dtype(); - - vortex_ensure!( - values_dtype.eq_ignore_nullability(fill_dtype), - "fill value, {:?}, should be instance of values dtype, {} but was {}.", - fill_value, - values_dtype, - fill_dtype, - ); - - if values_dtype == fill_dtype { - Ok(values) - } else { - values.cast(fill_dtype.clone()) - } - } - - fn normalize_patches_dtype(patches: Patches, fill_value: &Scalar) -> VortexResult { - let fill_dtype = fill_value.dtype(); - let values_dtype = patches.values().dtype(); - - vortex_ensure!( - values_dtype.eq_ignore_nullability(fill_dtype), - "fill value, {:?}, should be instance of values dtype, {} but was {}.", - fill_value, - values_dtype, - fill_dtype, - ); - - if values_dtype == fill_dtype { - Ok(patches) - } else { - patches.cast_values(fill_dtype) - } - } - - pub fn validate( - patches: &Patches, - fill_value: &Scalar, - dtype: &DType, - len: usize, - ) -> VortexResult<()> { - vortex_ensure!( - fill_value.dtype() == dtype, - "fill value dtype {} does not match array dtype {}", - fill_value.dtype(), - dtype, - ); - vortex_ensure!( - patches.array_len() == len, - "patches length {} does not match array length {}", - patches.array_len(), - len - ); - vortex_ensure!( - patches.values().dtype() == dtype, - "patch values dtype {} does not match array dtype {}", - patches.values().dtype(), - dtype, - ); - Ok(()) - } - fn make_slots(patches: &Patches) -> Vec> { vec![ Some(patches.indices().clone()), @@ -381,16 +341,27 @@ impl SparseData { } } - let values = Self::normalize_values_dtype(values, &fill_value)?; - // TODO(0ax1): handle chunk offsets let patches = Patches::new(len, 0, indices, values, None)?; - Self::try_new_from_patches(patches, fill_value) + let slots = Self::make_slots(&patches); + + Ok(Self { + slots, + patches, + fill_value, + stats_set: Default::default(), + }) } /// Build a new SparseArray from an existing set of patches. pub fn try_new_from_patches(patches: Patches, fill_value: Scalar) -> VortexResult { - let patches = Self::normalize_patches_dtype(patches, &fill_value)?; + vortex_ensure!( + fill_value.dtype() == patches.values().dtype(), + "fill value, {:?}, should be instance of values dtype, {} but was {}.", + fill_value, + patches.values().dtype(), + fill_value.dtype(), + ); let slots = Self::make_slots(&patches); @@ -398,6 +369,7 @@ impl SparseData { slots, patches, fill_value, + stats_set: Default::default(), }) } @@ -408,6 +380,7 @@ impl SparseData { slots, patches, fill_value, + stats_set: Default::default(), } } @@ -463,10 +436,10 @@ impl SparseData { /// Optionally provided fill value will be respected if the array is less than 90% null. pub fn encode(array: &ArrayRef, fill_value: Option) -> VortexResult { if let Some(fill_value) = fill_value.as_ref() - && !array.dtype().eq_ignore_nullability(fill_value.dtype()) + && array.dtype() != fill_value.dtype() { vortex_bail!( - "Array and fill value types must have the same base type. got {} and {}", + "Array and fill value types must match. got {} and {}", array.dtype(), fill_value.dtype() ) @@ -501,17 +474,17 @@ impl SparseData { } }; - return Sparse::try_new( + return Ok(SparseData::try_new( non_null_indices, non_null_values, array.len(), Scalar::null(array.dtype().clone()), - ) - .map(IntoArray::into_array); + )? + .into_array()); } let fill = if let Some(fill) = fill_value { - fill.cast(array.dtype())? + fill } else { // TODO(robert): Support other dtypes, only thing missing is getting most common value out of the array let (top_pvalue, _) = array @@ -548,8 +521,8 @@ impl SparseData { Mask::Values(values) => values.indices().iter().map(|v| *v as u64).collect(), }; - Sparse::try_new(indices.into_array(), non_top_values, array.len(), fill) - .map(IntoArray::into_array) + SparseData::try_new(indices.into_array(), non_top_values, array.len(), fill) + .map(|a| a.into_array()) } } @@ -571,7 +544,7 @@ impl ValidityVTable for Sparse { }; Ok(Validity::Array( - unsafe { Sparse::new_unchecked(patches, array.fill_value.is_valid().into()) } + unsafe { SparseData::new_unchecked(patches, array.fill_value.is_valid().into()) } .into_array(), )) } diff --git a/encodings/sparse/src/rules.rs b/encodings/sparse/src/rules.rs index c53e7e4c6a5..91b4edf80b0 100644 --- a/encodings/sparse/src/rules.rs +++ b/encodings/sparse/src/rules.rs @@ -12,6 +12,7 @@ use vortex_array::scalar_fn::fns::not::NotReduceAdaptor; use vortex_error::VortexResult; use crate::Sparse; +use crate::SparseData; pub(crate) static RULES: ParentRuleSet = ParentRuleSet::new(&[ ParentRuleSet::lift(&CastReduceAdaptor(Sparse)), @@ -23,7 +24,7 @@ impl NotReduce for Sparse { let inverted_fill = array.fill_scalar().as_bool().invert().into_scalar(); let inverted_patches = array.patches().clone().map_values(|values| values.not())?; Ok(Some( - Sparse::try_new_from_patches(inverted_patches, inverted_fill)?.into_array(), + SparseData::try_new_from_patches(inverted_patches, inverted_fill)?.into_array(), )) } } diff --git a/encodings/sparse/src/slice.rs b/encodings/sparse/src/slice.rs index 7a36d58e209..6d06da58029 100644 --- a/encodings/sparse/src/slice.rs +++ b/encodings/sparse/src/slice.rs @@ -12,6 +12,7 @@ use vortex_error::VortexResult; use crate::ConstantArray; use crate::Sparse; +use crate::SparseData; impl SliceKernel for Sparse { fn slice( @@ -35,7 +36,8 @@ impl SliceKernel for Sparse { // patches slice will ensure that dtype of patches is unchanged and the indices and // values match Ok(Some( - unsafe { Sparse::new_unchecked(new_patches, array.fill_scalar().clone()) }.into_array(), + unsafe { SparseData::new_unchecked(new_patches, array.fill_scalar().clone()) } + .into_array(), )) } } diff --git a/encodings/zigzag/public-api.lock b/encodings/zigzag/public-api.lock index ec8ed62f664..ab49a0acca7 100644 --- a/encodings/zigzag/public-api.lock +++ b/encodings/zigzag/public-api.lock @@ -20,6 +20,8 @@ impl vortex_array::array::vtable::VTable for vortex_zigzag::ZigZag pub type vortex_zigzag::ZigZag::ArrayData = vortex_zigzag::ZigZagData +pub type vortex_zigzag::ZigZag::Metadata = vortex_array::metadata::EmptyMetadata + pub type vortex_zigzag::ZigZag::OperationsVTable = vortex_zigzag::ZigZag pub type vortex_zigzag::ZigZag::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromChild @@ -32,7 +34,11 @@ pub fn vortex_zigzag::ZigZag::buffer(_array: vortex_array::array::view::ArrayVie pub fn vortex_zigzag::ZigZag::buffer_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_zigzag::ZigZag::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_zigzag::ZigZag::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_zigzag::ZigZag::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_zigzag::ZigZag::dtype(array: &vortex_zigzag::ZigZagData) -> &vortex_array::dtype::DType pub fn vortex_zigzag::ZigZag::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult @@ -40,17 +46,23 @@ pub fn vortex_zigzag::ZigZag::execute_parent(array: vortex_array::array::view::A pub fn vortex_zigzag::ZigZag::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_zigzag::ZigZag::len(array: &vortex_zigzag::ZigZagData) -> usize + +pub fn vortex_zigzag::ZigZag::metadata(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_zigzag::ZigZag::nbuffers(_array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_zigzag::ZigZag::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_zigzag::ZigZag::serialize(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_zigzag::ZigZag::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_zigzag::ZigZag::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_zigzag::ZigZag::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_zigzag::ZigZag::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_zigzag::ZigZag::stats(array: &vortex_zigzag::ZigZagData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_zigzag::ZigZag::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_zigzag::ZigZag::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -86,6 +98,8 @@ pub struct vortex_zigzag::ZigZagData impl vortex_zigzag::ZigZagData +pub fn vortex_zigzag::ZigZagData::dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_zigzag::ZigZagData::encoded(&self) -> &vortex_array::array::erased::ArrayRef pub fn vortex_zigzag::ZigZagData::is_empty(&self) -> bool @@ -102,10 +116,18 @@ impl core::clone::Clone for vortex_zigzag::ZigZagData pub fn vortex_zigzag::ZigZagData::clone(&self) -> vortex_zigzag::ZigZagData +impl core::convert::From for vortex_array::array::erased::ArrayRef + +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_zigzag::ZigZagData) -> vortex_array::array::erased::ArrayRef + impl core::fmt::Debug for vortex_zigzag::ZigZagData pub fn vortex_zigzag::ZigZagData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::array::IntoArray for vortex_zigzag::ZigZagData + +pub fn vortex_zigzag::ZigZagData::into_array(self) -> vortex_array::array::erased::ArrayRef + pub fn vortex_zigzag::zigzag_decode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_array::arrays::primitive::vtable::PrimitiveArray pub fn vortex_zigzag::zigzag_encode(parray: vortex_array::arrays::primitive::vtable::PrimitiveArray) -> vortex_error::VortexResult diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index 896afd67fb3..0cb2102c8c6 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -5,9 +5,9 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::EmptyMetadata; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; @@ -18,6 +18,7 @@ use vortex_array::dtype::PType; use vortex_array::match_each_unsigned_integer_ptype; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; @@ -41,25 +42,28 @@ vtable!(ZigZag, ZigZag, ZigZagData); impl VTable for ZigZag { type ArrayData = ZigZagData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; + fn vtable(_array: &Self::ArrayData) -> &Self { + &ZigZag + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - let expected_dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; - vortex_ensure!( - dtype == &expected_dtype, - "expected dtype {expected_dtype}, got {dtype}" - ); - vortex_ensure!( - data.encoded().len() == len, - "expected len {len}, got {}", - data.encoded().len() - ); - Ok(()) + fn len(array: &ZigZagData) -> usize { + array.encoded().len() + } + + fn dtype(array: &ZigZagData) -> &DType { + &array.dtype + } + + fn stats(array: &ZigZagData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &ZigZagData, state: &mut H, precision: Precision) { @@ -82,25 +86,31 @@ impl VTable for ZigZag { vortex_panic!("ZigZagArray buffer_name index {idx} out of bounds") } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + _metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - if !metadata.is_empty() { - vortex_bail!( - "ZigZagArray expects empty metadata, got {} bytes", - metadata.len() - ); - } if children.len() != 1 { vortex_bail!("Expected 1 child, got {}", children.len()); } @@ -162,7 +172,9 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["encoded"]; #[derive(Clone, Debug)] pub struct ZigZagData { + dtype: DType, pub(super) slots: Vec>, + stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -173,19 +185,11 @@ impl ZigZag { /// Construct a new [`ZigZagArray`] from an encoded unsigned integer array. pub fn try_new(encoded: ArrayRef) -> VortexResult { - let data = ZigZagData::try_new(encoded)?; - let dtype = ZigZagData::dtype_from_encoded_dtype(data.encoded().dtype())?; - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ZigZag, dtype, len, data)) }) + Array::try_from_data(ZigZagData::try_new(encoded)?) } } impl ZigZagData { - fn dtype_from_encoded_dtype(encoded_dtype: &DType) -> VortexResult { - Ok(DType::from(PType::try_from(encoded_dtype)?.to_signed()) - .with_nullability(encoded_dtype.nullability())) - } - pub fn new(encoded: ArrayRef) -> Self { Self::try_new(encoded).vortex_expect("ZigZagArray new") } @@ -196,10 +200,13 @@ impl ZigZagData { vortex_bail!(MismatchedTypes: "unsigned int", encoded_dtype); } - Self::dtype_from_encoded_dtype(&encoded_dtype)?; + let dtype = DType::from(PType::try_from(&encoded_dtype)?.to_signed()) + .with_nullability(encoded_dtype.nullability()); Ok(Self { + dtype, slots: vec![Some(encoded)], + stats_set: Default::default(), }) } @@ -215,10 +222,14 @@ impl ZigZagData { self.encoded().is_empty() } + /// Returns the logical data type of the array. + #[inline] + pub fn dtype(&self) -> &DType { + &self.dtype + } + pub fn ptype(&self) -> PType { - PType::try_from(self.encoded().dtype()) - .vortex_expect("ZigZagArray encoded dtype") - .to_signed() + self.dtype().as_ptype() } pub fn encoded(&self) -> &ArrayRef { diff --git a/encodings/zigzag/src/compress.rs b/encodings/zigzag/src/compress.rs index b2d142c755c..690cd52deac 100644 --- a/encodings/zigzag/src/compress.rs +++ b/encodings/zigzag/src/compress.rs @@ -12,8 +12,8 @@ use vortex_error::vortex_bail; use vortex_error::vortex_panic; use zigzag::ZigZag as ExternalZigZag; -use crate::ZigZag; use crate::ZigZagArray; +use crate::ZigZagData; pub fn zigzag_encode(parray: PrimitiveArray) -> VortexResult { let validity = parray.validity(); let encoded = match parray.ptype() { @@ -26,7 +26,7 @@ pub fn zigzag_encode(parray: PrimitiveArray) -> VortexResult { parray.ptype() ), }; - ZigZag::try_new(encoded.into_array()) + ZigZagArray::try_from_data(ZigZagData::try_new(encoded.into_array())?) } fn zigzag_encode_primitive( diff --git a/encodings/zigzag/src/compute/cast.rs b/encodings/zigzag/src/compute/cast.rs index 138c6aae736..c69d91f5347 100644 --- a/encodings/zigzag/src/compute/cast.rs +++ b/encodings/zigzag/src/compute/cast.rs @@ -10,6 +10,7 @@ use vortex_array::scalar_fn::fns::cast::CastReduce; use vortex_error::VortexResult; use crate::ZigZag; +use crate::ZigZagData; impl CastReduce for ZigZag { fn cast(array: ArrayView<'_, Self>, dtype: &DType) -> VortexResult> { if !dtype.is_signed_int() { @@ -19,7 +20,7 @@ impl CastReduce for ZigZag { let new_encoded_dtype = DType::Primitive(dtype.as_ptype().to_unsigned(), dtype.nullability()); let new_encoded = array.encoded().cast(new_encoded_dtype)?; - Ok(Some(ZigZag::try_new(new_encoded)?.into_array())) + Ok(Some(ZigZagData::try_new(new_encoded)?.into_array())) } } diff --git a/encodings/zigzag/src/compute/mod.rs b/encodings/zigzag/src/compute/mod.rs index 11fe580e6bc..9967ba5bb95 100644 --- a/encodings/zigzag/src/compute/mod.rs +++ b/encodings/zigzag/src/compute/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use crate::ZigZagData; mod cast; use vortex_array::ArrayRef; @@ -21,7 +22,7 @@ use crate::ZigZag; impl FilterReduce for ZigZag { fn filter(array: ArrayView<'_, Self>, mask: &Mask) -> VortexResult> { let encoded = array.encoded().filter(mask.clone())?; - Ok(Some(ZigZag::try_new(encoded)?.into_array())) + Ok(Some(ZigZagData::try_new(encoded)?.into_array())) } } @@ -32,7 +33,7 @@ impl TakeExecute for ZigZag { _ctx: &mut ExecutionCtx, ) -> VortexResult> { let encoded = array.encoded().take(indices.clone())?; - Ok(Some(ZigZag::try_new(encoded)?.into_array())) + Ok(Some(ZigZagData::try_new(encoded)?.into_array())) } } @@ -43,7 +44,7 @@ impl MaskReduce for ZigZag { EmptyOptions, [array.encoded().clone(), mask.clone()], )?; - Ok(Some(ZigZag::try_new(masked_encoded)?.into_array())) + Ok(Some(ZigZagData::try_new(masked_encoded)?.into_array())) } } diff --git a/encodings/zigzag/src/slice.rs b/encodings/zigzag/src/slice.rs index e027cda04cb..599d365809f 100644 --- a/encodings/zigzag/src/slice.rs +++ b/encodings/zigzag/src/slice.rs @@ -10,11 +10,12 @@ use vortex_array::arrays::slice::SliceReduce; use vortex_error::VortexResult; use crate::ZigZag; +use crate::ZigZagData; impl SliceReduce for ZigZag { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { Ok(Some( - ZigZag::try_new(array.encoded().slice(range)?)?.into_array(), + ZigZagData::new(array.encoded().slice(range)?).into_array(), )) } } diff --git a/encodings/zstd/benches/listview_rebuild.rs b/encodings/zstd/benches/listview_rebuild.rs index 55690cdccae..0e91342c8ac 100644 --- a/encodings/zstd/benches/listview_rebuild.rs +++ b/encodings/zstd/benches/listview_rebuild.rs @@ -10,17 +10,13 @@ use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::listview::ListViewRebuildMode; use vortex_array::validity::Validity; use vortex_buffer::Buffer; -use vortex_zstd::Zstd; use vortex_zstd::ZstdData; #[divan::bench(sample_size = 1000)] fn rebuild_naive(bencher: Bencher) { let dudes = VarBinViewArray::from_iter_str(["Washington", "Adams", "Jefferson", "Madison"]) .into_array(); - let dtype = dudes.dtype().clone(); - let dudes = Zstd::try_new(dtype, ZstdData::from_array(dudes, 9, 1024).unwrap()) - .unwrap() - .into_array(); + let dudes = ZstdData::from_array(dudes, 9, 1024).unwrap().into_array(); let offsets = std::iter::repeat_n(0u32, 1024) .collect::>() diff --git a/encodings/zstd/public-api.lock b/encodings/zstd/public-api.lock index dc7e1e63809..29554686839 100644 --- a/encodings/zstd/public-api.lock +++ b/encodings/zstd/public-api.lock @@ -6,16 +6,12 @@ impl vortex_zstd::Zstd pub const vortex_zstd::Zstd::ID: vortex_array::array::ArrayId -pub fn vortex_zstd::Zstd::decompress(array: &vortex_zstd::ZstdArray, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult - pub fn vortex_zstd::Zstd::from_primitive(parray: &vortex_array::arrays::primitive::vtable::PrimitiveArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::from_var_bin_view(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult -pub fn vortex_zstd::Zstd::try_new(dtype: vortex_array::dtype::DType, data: vortex_zstd::ZstdData) -> vortex_error::VortexResult - impl core::clone::Clone for vortex_zstd::Zstd pub fn vortex_zstd::Zstd::clone(&self) -> vortex_zstd::Zstd @@ -28,6 +24,8 @@ impl vortex_array::array::vtable::VTable for vortex_zstd::Zstd pub type vortex_zstd::Zstd::ArrayData = vortex_zstd::ZstdData +pub type vortex_zstd::Zstd::Metadata = vortex_array::metadata::ProstMetadata + pub type vortex_zstd::Zstd::OperationsVTable = vortex_zstd::Zstd pub type vortex_zstd::Zstd::ValidityVTable = vortex_array::array::vtable::validity::ValidityVTableFromValiditySliceHelper @@ -40,23 +38,33 @@ pub fn vortex_zstd::Zstd::buffer(array: vortex_array::array::view::ArrayView<'_, pub fn vortex_zstd::Zstd::buffer_name(array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> core::option::Option -pub fn vortex_zstd::Zstd::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_zstd::Zstd::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_zstd::Zstd::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_zstd::Zstd::dtype(array: &vortex_zstd::ZstdData) -> &vortex_array::dtype::DType pub fn vortex_zstd::Zstd::execute(array: vortex_array::array::typed::Array, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_zstd::Zstd::id(&self) -> vortex_array::array::ArrayId +pub fn vortex_zstd::Zstd::len(array: &vortex_zstd::ZstdData) -> usize + +pub fn vortex_zstd::Zstd::metadata(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_zstd::Zstd::nbuffers(array: vortex_array::array::view::ArrayView<'_, Self>) -> usize pub fn vortex_zstd::Zstd::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_zstd::Zstd::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_zstd::Zstd::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_zstd::Zstd::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_zstd::Zstd::slots(array: vortex_array::array::view::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_zstd::Zstd::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_zstd::Zstd::stats(array: &vortex_zstd::ZstdData) -> &vortex_array::stats::array::ArrayStats + +pub fn vortex_zstd::Zstd::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_zstd::Zstd::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -72,6 +80,28 @@ impl vortex_array::scalar_fn::fns::cast::kernel::CastReduce for vortex_zstd::Zst pub fn vortex_zstd::Zstd::cast(array: vortex_array::array::view::ArrayView<'_, Self>, dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult> +pub struct vortex_zstd::ZstdArrayParts + +pub vortex_zstd::ZstdArrayParts::dictionary: core::option::Option + +pub vortex_zstd::ZstdArrayParts::dtype: vortex_array::dtype::DType + +pub vortex_zstd::ZstdArrayParts::frames: alloc::vec::Vec + +pub vortex_zstd::ZstdArrayParts::metadata: vortex_zstd::ZstdMetadata + +pub vortex_zstd::ZstdArrayParts::n_rows: usize + +pub vortex_zstd::ZstdArrayParts::slice_start: usize + +pub vortex_zstd::ZstdArrayParts::slice_stop: usize + +pub vortex_zstd::ZstdArrayParts::validity: vortex_array::validity::Validity + +impl core::fmt::Debug for vortex_zstd::ZstdArrayParts + +pub fn vortex_zstd::ZstdArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + pub struct vortex_zstd::ZstdBuffersMetadata pub vortex_zstd::ZstdBuffersMetadata::buffer_alignments: alloc::vec::Vec @@ -104,6 +134,10 @@ pub struct vortex_zstd::ZstdData impl vortex_zstd::ZstdData +pub fn vortex_zstd::ZstdData::decompress(&self, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult + +pub fn vortex_zstd::ZstdData::dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_zstd::ZstdData::from_array(array: vortex_array::array::erased::ArrayRef, level: i32, values_per_frame: usize) -> vortex_error::VortexResult pub fn vortex_zstd::ZstdData::from_canonical(canonical: &vortex_array::canonical::Canonical, level: i32, values_per_frame: usize) -> vortex_error::VortexResult> @@ -116,43 +150,33 @@ pub fn vortex_zstd::ZstdData::from_var_bin_view(vbv: &vortex_array::arrays::varb pub fn vortex_zstd::ZstdData::from_var_bin_view_without_dict(vbv: &vortex_array::arrays::varbinview::vtable::VarBinViewArray, level: i32, values_per_frame: usize) -> vortex_error::VortexResult -pub fn vortex_zstd::ZstdData::into_parts(self) -> vortex_zstd::ZstdDataParts +pub fn vortex_zstd::ZstdData::into_parts(self) -> vortex_zstd::ZstdArrayParts pub fn vortex_zstd::ZstdData::is_empty(&self) -> bool pub fn vortex_zstd::ZstdData::len(&self) -> usize -pub fn vortex_zstd::ZstdData::new(dictionary: core::option::Option, frames: alloc::vec::Vec, metadata: vortex_zstd::ZstdMetadata, n_rows: usize, validity: vortex_array::validity::Validity) -> Self - -pub fn vortex_zstd::ZstdData::validate(&self, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_zstd::ZstdData::new(dictionary: core::option::Option, frames: alloc::vec::Vec, dtype: vortex_array::dtype::DType, metadata: vortex_zstd::ZstdMetadata, n_rows: usize, validity: vortex_array::validity::Validity) -> Self impl core::clone::Clone for vortex_zstd::ZstdData pub fn vortex_zstd::ZstdData::clone(&self) -> vortex_zstd::ZstdData -impl core::fmt::Debug for vortex_zstd::ZstdData - -pub fn vortex_zstd::ZstdData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::convert::From for vortex_array::array::erased::ArrayRef -impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_zstd::ZstdData +pub fn vortex_array::array::erased::ArrayRef::from(value: vortex_zstd::ZstdData) -> vortex_array::array::erased::ArrayRef -pub fn vortex_zstd::ZstdData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) - -pub struct vortex_zstd::ZstdDataParts - -pub vortex_zstd::ZstdDataParts::dictionary: core::option::Option - -pub vortex_zstd::ZstdDataParts::frames: alloc::vec::Vec +impl core::fmt::Debug for vortex_zstd::ZstdData -pub vortex_zstd::ZstdDataParts::metadata: vortex_zstd::ZstdMetadata +pub fn vortex_zstd::ZstdData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_zstd::ZstdDataParts::n_rows: usize +impl vortex_array::array::IntoArray for vortex_zstd::ZstdData -pub vortex_zstd::ZstdDataParts::slice_start: usize +pub fn vortex_zstd::ZstdData::into_array(self) -> vortex_array::array::erased::ArrayRef -pub vortex_zstd::ZstdDataParts::slice_stop: usize +impl vortex_array::array::vtable::validity::ValiditySliceHelper for vortex_zstd::ZstdData -pub vortex_zstd::ZstdDataParts::validity: vortex_array::validity::Validity +pub fn vortex_zstd::ZstdData::unsliced_validity_and_slice(&self) -> (&vortex_array::validity::Validity, usize, usize) pub struct vortex_zstd::ZstdFrameMetadata diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index a494cfc572f..d8c6875fa39 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -11,7 +11,6 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::Canonical; @@ -20,6 +19,7 @@ use vortex_array::ExecutionResult; use vortex_array::IntoArray; use vortex_array::LEGACY_SESSION; use vortex_array::Precision; +use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; @@ -32,6 +32,7 @@ use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; +use vortex_array::stats::ArrayStats; use vortex_array::validity::Validity; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; @@ -84,15 +85,28 @@ vtable!(Zstd, Zstd, ZstdData); impl VTable for Zstd { type ArrayData = ZstdData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromValiditySliceHelper; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Zstd + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - data.validate(dtype, len) + fn len(array: &ZstdData) -> usize { + array.slice_stop - array.slice_start + } + + fn dtype(array: &ZstdData) -> &DType { + &array.dtype + } + + fn stats(array: &ZstdData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &ZstdData, state: &mut H, precision: Precision) { @@ -165,20 +179,31 @@ impl VTable for Zstd { } } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(array.metadata.clone().encode_to_vec())) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(array.metadata.clone())) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.0.encode_to_vec())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata(ZstdMetadata::decode(bytes)?)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = ZstdMetadata::decode(metadata)?; let validity = if children.is_empty() { Validity::from(dtype.nullability()) } else if children.len() == 1 { @@ -188,7 +213,7 @@ impl VTable for Zstd { vortex_bail!("ZstdArray expected 0 or 1 child, got {}", children.len()); }; - let (dictionary_buffer, compressed_buffers) = if metadata.dictionary_size == 0 { + let (dictionary_buffer, compressed_buffers) = if metadata.0.dictionary_size == 0 { // no dictionary ( None, @@ -211,7 +236,8 @@ impl VTable for Zstd { Ok(ZstdData::new( dictionary_buffer, compressed_buffers, - metadata, + dtype.clone(), + metadata.0.clone(), len, validity, )) @@ -235,7 +261,7 @@ impl VTable for Zstd { array.unsliced_validity = match &slots[VALIDITY_SLOT] { Some(arr) => Validity::Array(arr.clone()), - None => Validity::from(array.unsliced_validity.nullability()), + None => Validity::from(array.dtype.nullability()), }; array.slots = slots; @@ -243,7 +269,8 @@ impl VTable for Zstd { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - Zstd::decompress(&array, ctx)? + array + .decompress(ctx)? .execute::(ctx) .map(ExecutionResult::done) } @@ -263,22 +290,17 @@ pub struct Zstd; impl Zstd { pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd"); - pub fn try_new(dtype: DType, data: ZstdData) -> VortexResult { - let len = data.len(); - data.validate(&dtype, len)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Zstd, dtype, len, data)) }) - } - /// Compress a [`VarBinViewArray`] using Zstd without a dictionary. pub fn from_var_bin_view_without_dict( vbv: &VarBinViewArray, level: i32, values_per_frame: usize, ) -> VortexResult { - Self::try_new( - vbv.dtype().clone(), - ZstdData::from_var_bin_view_without_dict(vbv, level, values_per_frame)?, - ) + Array::try_from_data(ZstdData::from_var_bin_view_without_dict( + vbv, + level, + values_per_frame, + )?) } /// Compress a [`PrimitiveArray`] using Zstd. @@ -287,10 +309,7 @@ impl Zstd { level: i32, values_per_frame: usize, ) -> VortexResult { - Self::try_new( - parray.dtype().clone(), - ZstdData::from_primitive(parray, level, values_per_frame)?, - ) + Array::try_from_data(ZstdData::from_primitive(parray, level, values_per_frame)?) } /// Compress a [`VarBinViewArray`] using Zstd. @@ -299,14 +318,7 @@ impl Zstd { level: i32, values_per_frame: usize, ) -> VortexResult { - Self::try_new( - vbv.dtype().clone(), - ZstdData::from_var_bin_view(vbv, level, values_per_frame)?, - ) - } - - pub fn decompress(array: &ZstdArray, ctx: &mut ExecutionCtx) -> VortexResult { - array.data().decompress(array.dtype(), ctx) + Array::try_from_data(ZstdData::from_var_bin_view(vbv, level, values_per_frame)?) } } @@ -320,25 +332,36 @@ pub struct ZstdData { pub(crate) dictionary: Option, pub(crate) frames: Vec, pub(crate) metadata: ZstdMetadata, + dtype: DType, pub(crate) unsliced_validity: Validity, unsliced_n_rows: usize, pub(super) slots: Vec>, + stats_set: ArrayStats, slice_start: usize, slice_stop: usize, } -pub struct ZstdDataParts { +/// The parts of a [`ZstdArray`] returned by [`ZstdArray::into_parts`]. +#[derive(Debug)] +pub struct ZstdArrayParts { + /// The optional dictionary used for compression. pub dictionary: Option, + /// The compressed frames. pub frames: Vec, + /// The compression metadata. pub metadata: ZstdMetadata, + /// The data type of the uncompressed array. + pub dtype: DType, + /// The validity of the uncompressed array. pub validity: Validity, + /// The number of rows in the uncompressed array. pub n_rows: usize, + /// Slice start offset. pub slice_start: usize, + /// Slice stop offset. pub slice_stop: usize, } -/// The parts of a [`ZstdArray`] returned by [`ZstdArray::into_parts`]. -#[derive(Debug)] struct Frames { dictionary: Option, frames: Vec, @@ -439,6 +462,7 @@ impl ZstdData { pub fn new( dictionary: Option, frames: Vec, + dtype: DType, metadata: ZstdMetadata, n_rows: usize, validity: Validity, @@ -448,94 +472,16 @@ impl ZstdData { dictionary, frames, metadata, + dtype, unsliced_validity: validity, unsliced_n_rows: n_rows, slots: vec![validity_slot], + stats_set: Default::default(), slice_start: 0, slice_stop: n_rows, } } - pub fn validate(&self, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - matches!( - dtype, - DType::Primitive(..) | DType::Binary(_) | DType::Utf8(_) - ), - "Unsupported dtype for Zstd array: {dtype}" - ); - vortex_ensure!( - self.slice_start <= self.slice_stop, - "Invalid slice range {}..{}", - self.slice_start, - self.slice_stop - ); - vortex_ensure!( - self.slice_stop <= self.unsliced_n_rows, - "Slice stop {} exceeds unsliced row count {}", - self.slice_stop, - self.unsliced_n_rows - ); - vortex_ensure!( - self.slice_stop - self.slice_start == len, - "Slice length {} does not match array length {}", - self.slice_stop - self.slice_start, - len - ); - if let Some(validity_len) = self.unsliced_validity.maybe_len() { - vortex_ensure!( - validity_len == self.unsliced_n_rows, - "Validity length {} does not match unsliced row count {}", - validity_len, - self.unsliced_n_rows - ); - } - - match &self.dictionary { - Some(dictionary) => vortex_ensure!( - usize::try_from(self.metadata.dictionary_size)? == dictionary.len(), - "Dictionary size metadata {} does not match buffer size {}", - self.metadata.dictionary_size, - dictionary.len() - ), - None => vortex_ensure!( - self.metadata.dictionary_size == 0, - "Dictionary metadata present without dictionary buffer" - ), - } - vortex_ensure!( - self.frames.len() == self.metadata.frames.len(), - "Frame count {} does not match metadata frame count {}", - self.frames.len(), - self.metadata.frames.len() - ); - - Ok(()) - } - - pub(crate) fn with_slice(&self, start: usize, stop: usize) -> Self { - let new_start = self.slice_start + start; - let new_stop = self.slice_start + stop; - - assert!( - new_start <= self.slice_stop, - "new slice start {new_start} exceeds end {}", - self.slice_stop - ); - - assert!( - new_stop <= self.slice_stop, - "new slice stop {new_stop} exceeds end {}", - self.slice_stop - ); - - Self { - slice_start: new_start, - slice_stop: new_stop, - ..self.clone() - } - } - fn compress_values( value_bytes: &ByteBuffer, frame_byte_starts: &[usize], @@ -639,6 +585,7 @@ impl ZstdData { values_per_frame: usize, use_dictionary: bool, ) -> VortexResult { + let dtype = parray.dtype().clone(); let byte_width = parray.ptype().byte_width(); // We compress only the valid elements. @@ -682,6 +629,7 @@ impl ZstdData { Ok(ZstdData::new( dictionary, frames, + dtype, metadata, parray.len(), parray.validity(), @@ -734,6 +682,8 @@ impl ZstdData { // to separate the binary data and lengths as two separate streams, but // this approach is simpler and can be best in cases when there is // mutual information between strings and their lengths. + let dtype = vbv.dtype().clone(); + // We compress only the valid elements. let (value_bytes, value_byte_indices) = collect_valid_vbv(vbv)?; let n_values = value_byte_indices.len(); @@ -770,6 +720,7 @@ impl ZstdData { Ok(ZstdData::new( dictionary, frames, + dtype, metadata, vbv.len(), vbv.validity(), @@ -801,18 +752,18 @@ impl ZstdData { .ok_or_else(|| vortex_err!("Zstd can only encode Primitive and VarBinView arrays")) } - fn byte_width(dtype: &DType) -> usize { - if dtype.is_primitive() { - dtype.as_ptype().byte_width() + fn byte_width(&self) -> usize { + if self.dtype.is_primitive() { + self.dtype.as_ptype().byte_width() } else { 1 } } - fn decompress(&self, dtype: &DType, ctx: &mut ExecutionCtx) -> VortexResult { + pub fn decompress(&self, ctx: &mut ExecutionCtx) -> VortexResult { // To start, we figure out which frames we need to decompress, and with // what row offset into the first such frame. - let byte_width = Self::byte_width(dtype); + let byte_width = self.byte_width(); let slice_n_rows = self.slice_stop - self.slice_start; let slice_value_indices = self .unsliced_validity @@ -894,21 +845,21 @@ impl ZstdData { // // We ensure that the validity of the decompressed array ALWAYS matches the validity // implied by the DType. - if !dtype.is_nullable() && !matches!(slice_validity, Validity::NonNullable) { + if !self.dtype().is_nullable() && !matches!(slice_validity, Validity::NonNullable) { assert!( matches!(slice_validity, Validity::AllValid), "ZSTD array expects to be non-nullable but there are nulls after decompression" ); slice_validity = Validity::NonNullable; - } else if dtype.is_nullable() && matches!(slice_validity, Validity::NonNullable) { + } else if self.dtype.is_nullable() && matches!(slice_validity, Validity::NonNullable) { slice_validity = Validity::AllValid; } // // END OF IMPORTANT BLOCK // - match dtype { + match &self.dtype { DType::Primitive(..) => { let slice_values_buffer = decompressed.slice( (slice_value_idx_start - n_skipped_values) * byte_width @@ -916,7 +867,7 @@ impl ZstdData { ); let primitive = PrimitiveArray::from_values_byte_buffer( slice_values_buffer, - dtype.as_ptype(), + self.dtype.as_ptype(), slice_validity, slice_n_rows, ); @@ -937,14 +888,14 @@ impl ZstdData { VarBinViewArray::new_unchecked( valid_views, Arc::from(buffers), - dtype.clone(), + self.dtype.clone(), slice_validity, ) } .into_array()) } AllOr::None => Ok(ConstantArray::new( - Scalar::null(dtype.clone()), + Scalar::null(self.dtype.clone()), slice_n_rows, ) .into_array()), @@ -965,7 +916,7 @@ impl ZstdData { VarBinViewArray::new_unchecked( views.freeze(), Arc::from(buffers), - dtype.clone(), + self.dtype.clone(), slice_validity, ) } @@ -973,7 +924,46 @@ impl ZstdData { } } } - _ => vortex_panic!("Unsupported dtype for Zstd array: {}", dtype), + _ => vortex_panic!("Unsupported dtype for Zstd array: {}", self.dtype), + } + } + + pub(crate) fn _slice(&self, start: usize, stop: usize) -> ZstdArray { + let new_start = self.slice_start + start; + let new_stop = self.slice_start + stop; + + assert!( + new_start <= self.slice_stop, + "new slice start {new_start} exceeds end {}", + self.slice_stop + ); + + assert!( + new_stop <= self.slice_stop, + "new slice stop {new_stop} exceeds end {}", + self.slice_stop + ); + + Array::try_from_data(ZstdData { + slice_start: self.slice_start + start, + slice_stop: self.slice_start + stop, + stats_set: Default::default(), + ..self.clone() + }) + .vortex_expect("ZstdData is always valid") + } + + /// Consumes the array and returns its parts. + pub fn into_parts(self) -> ZstdArrayParts { + ZstdArrayParts { + dictionary: self.dictionary, + frames: self.frames, + metadata: self.metadata, + dtype: self.dtype, + validity: self.unsliced_validity, + n_rows: self.unsliced_n_rows, + slice_start: self.slice_start, + slice_stop: self.slice_stop, } } @@ -989,16 +979,10 @@ impl ZstdData { self.slice_stop == self.slice_start } - pub fn into_parts(self) -> ZstdDataParts { - ZstdDataParts { - dictionary: self.dictionary, - frames: self.frames, - metadata: self.metadata, - validity: self.unsliced_validity, - n_rows: self.unsliced_n_rows, - slice_start: self.slice_start, - slice_stop: self.slice_stop, - } + /// Returns the logical data type of the array. + #[inline] + pub fn dtype(&self) -> &DType { + &self.dtype } pub(crate) fn slice_start(&self) -> usize { @@ -1027,11 +1011,10 @@ impl OperationsVTable for Zstd { _ctx: &mut ExecutionCtx, ) -> VortexResult { let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let sliced = Zstd::try_new( - array.dtype().clone(), - array.data().with_slice(index, index + 1), - )?; - Zstd::decompress(&sliced, &mut ctx)?.scalar_at(0) + array + ._slice(index, index + 1) + .decompress(&mut ctx)? + .scalar_at(0) } } diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs index 4ad636fe88a..feadd131e95 100644 --- a/encodings/zstd/src/compute/cast.rs +++ b/encodings/zstd/src/compute/cast.rs @@ -34,16 +34,14 @@ impl CastReduce for Zstd { (Nullability::NonNullable, Nullability::Nullable) => { // nonnull => null, trivial cast by altering the validity Ok(Some( - Zstd::try_new( + ZstdData::new( + array.dictionary.clone(), + array.frames.clone(), dtype.clone(), - ZstdData::new( - array.dictionary.clone(), - array.frames.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - array.unsliced_validity.clone(), - ), - )? + array.metadata.clone(), + array.unsliced_n_rows(), + array.unsliced_validity.clone(), + ) .into_array() .slice(array.slice_start()..array.slice_stop())?, )) @@ -62,16 +60,14 @@ impl CastReduce for Zstd { // If there are no nulls, the cast is trivial Ok(Some( - Zstd::try_new( + ZstdData::new( + array.dictionary.clone(), + array.frames.clone(), dtype.clone(), - ZstdData::new( - array.dictionary.clone(), - array.frames.clone(), - array.metadata.clone(), - array.unsliced_n_rows(), - array.unsliced_validity.clone(), - ), - )? + array.metadata.clone(), + array.unsliced_n_rows(), + array.unsliced_validity.clone(), + ) .into_array() .slice(array.slice_start()..array.slice_stop())?, )) diff --git a/encodings/zstd/src/slice.rs b/encodings/zstd/src/slice.rs index 2f74d40072c..fad72ae3279 100644 --- a/encodings/zstd/src/slice.rs +++ b/encodings/zstd/src/slice.rs @@ -13,12 +13,6 @@ use crate::Zstd; impl SliceReduce for Zstd { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { - Ok(Some( - Zstd::try_new( - array.dtype().clone(), - array.data().with_slice(range.start, range.end), - )? - .into_array(), - )) + Ok(Some(array._slice(range.start, range.end).into_array())) } } diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index 7548f32cb80..e95ce025667 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -32,7 +32,7 @@ fn test_zstd_compress_decompress() { // check full decompression works let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = Zstd::decompress(&compressed, &mut ctx).unwrap(); + let decompressed = compressed.decompress(&mut ctx).unwrap(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data)); // check slicing works @@ -78,9 +78,7 @@ fn test_zstd_with_validity_and_multi_frame() { assert_nth_scalar!(compressed, 177, 177); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = Zstd::decompress(&compressed, &mut ctx) - .unwrap() - .to_primitive(); + let decompressed = compressed.decompress(&mut ctx).unwrap().to_primitive(); let decompressed_values = decompressed.as_slice::(); assert_eq!(decompressed_values[3], 3); assert_eq!(decompressed_values[177], 177); @@ -123,9 +121,7 @@ fn test_zstd_with_dict() { assert_nth_scalar!(compressed, 199, 199); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = Zstd::decompress(&compressed, &mut ctx) - .unwrap() - .to_primitive(); + let decompressed = compressed.decompress(&mut ctx).unwrap().to_primitive(); assert_arrays_eq!(decompressed, PrimitiveArray::from_iter(data)); // check slicing works @@ -197,9 +193,7 @@ fn test_zstd_decompress_var_bin_view() { assert_nth_scalar!(compressed, 4, "baz"); let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let decompressed = Zstd::decompress(&compressed, &mut ctx) - .unwrap() - .to_varbinview(); + let decompressed = compressed.decompress(&mut ctx).unwrap().to_varbinview(); assert_nth_scalar!(decompressed, 0, "foo"); assert_nth_scalar!(decompressed, 1, "bar"); assert_nth_scalar!(decompressed, 2, None::); diff --git a/encodings/zstd/src/zstd_buffers.rs b/encodings/zstd/src/zstd_buffers.rs index 1e74a69a9a0..4f5b8a9b918 100644 --- a/encodings/zstd/src/zstd_buffers.rs +++ b/encodings/zstd/src/zstd_buffers.rs @@ -10,17 +10,19 @@ use vortex_array::Array; use vortex_array::ArrayEq; use vortex_array::ArrayHash; use vortex_array::ArrayId; -use vortex_array::ArrayParts; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; +use vortex_array::IntoArray; use vortex_array::Precision; +use vortex_array::ProstMetadata; use vortex_array::buffer::BufferHandle; use vortex_array::dtype::DType; use vortex_array::scalar::Scalar; use vortex_array::serde::ArrayChildren; use vortex_array::session::ArraySessionExt; +use vortex_array::stats::ArrayStats; use vortex_array::vtable; use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; @@ -42,67 +44,6 @@ pub struct ZstdBuffers; impl ZstdBuffers { pub const ID: ArrayId = ArrayId::new_ref("vortex.zstd_buffers"); - - pub fn try_new( - dtype: DType, - len: usize, - data: ZstdBuffersData, - ) -> VortexResult { - Array::try_from_parts(ArrayParts::new(ZstdBuffers, dtype, len, data)) - } - - pub fn compress(array: &ArrayRef, level: i32) -> VortexResult { - let encoding_id = array.encoding_id(); - let metadata = array - .metadata()? - .ok_or_else(|| vortex_err!("Array does not support serialization"))?; - let buffer_handles = array.buffer_handles(); - let children = array.children(); - - let mut compressed_buffers = Vec::with_capacity(buffer_handles.len()); - let mut uncompressed_sizes = Vec::with_capacity(buffer_handles.len()); - let mut buffer_alignments = Vec::with_capacity(buffer_handles.len()); - - let mut compressor = zstd::bulk::Compressor::new(level)?; - // Compression is currently CPU-only, so we gather all buffers on the host. - for handle in &buffer_handles { - buffer_alignments.push(u32::from(handle.alignment())); - let host_buf = handle.clone().try_to_host_sync()?; - uncompressed_sizes.push(host_buf.len() as u64); - let compressed = compressor.compress(&host_buf)?; - compressed_buffers.push(BufferHandle::new_host(ByteBuffer::from(compressed))); - } - - let data = ZstdBuffersData { - inner_encoding_id: encoding_id, - inner_metadata: metadata, - compressed_buffers, - uncompressed_sizes, - buffer_alignments, - slots: children.into_iter().map(Some).collect(), - }; - let compressed = Self::try_new(array.dtype().clone(), array.len(), data)?; - compressed.statistics().inherit_from(array.statistics()); - Ok(compressed) - } - - pub fn build_inner( - array: &ZstdBuffersArray, - buffer_handles: &[BufferHandle], - session: &VortexSession, - ) -> VortexResult { - array - .data() - .build_inner(array.dtype(), array.len(), buffer_handles, session) - } - - fn decompress_and_build_inner( - array: &ZstdBuffersArray, - session: &VortexSession, - ) -> VortexResult { - let decompressed_buffers = array.data().decompress_buffers()?; - Self::build_inner(array, &decompressed_buffers, session) - } } /// An encoding that ZSTD-compresses the buffers of any wrapped array. @@ -118,6 +59,9 @@ pub struct ZstdBuffersData { uncompressed_sizes: Vec, buffer_alignments: Vec, pub(crate) slots: Vec>, + dtype: DType, + len: usize, + stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -180,6 +124,24 @@ impl ZstdBuffersDecodePlan { } impl ZstdBuffersData { + /// Returns the length of the array. + #[inline] + pub fn len(&self) -> usize { + self.len + } + + /// Returns whether the array is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Returns the logical data type of the array. + #[inline] + pub fn dtype(&self) -> &DType { + &self.dtype + } + fn validate(&self) -> VortexResult<()> { vortex_ensure_eq!( self.compressed_buffers.len(), @@ -198,6 +160,51 @@ impl ZstdBuffersData { Ok(()) } + /// Compresses the buffers of the given array using ZSTD. + /// + /// Each buffer of the input array is independently ZSTD-compressed. The children + /// and metadata of the input array are preserved as-is. + pub fn compress(array: &ArrayRef, level: i32) -> VortexResult { + let encoding_id = array.encoding_id(); + let metadata = array + .metadata()? + .ok_or_else(|| vortex_err!("Array does not support serialization"))?; + let buffer_handles = array.buffer_handles(); + let children = array.children(); + + let mut compressed_buffers = Vec::with_capacity(buffer_handles.len()); + let mut uncompressed_sizes = Vec::with_capacity(buffer_handles.len()); + let mut buffer_alignments = Vec::with_capacity(buffer_handles.len()); + + let mut compressor = zstd::bulk::Compressor::new(level)?; + // Compression is currently CPU-only, so we gather all buffers on the host. + for handle in &buffer_handles { + buffer_alignments.push(u32::from(handle.alignment())); + let host_buf = handle.clone().try_to_host_sync()?; + uncompressed_sizes.push(host_buf.len() as u64); + let compressed = compressor.compress(&host_buf)?; + compressed_buffers.push(BufferHandle::new_host(ByteBuffer::from(compressed))); + } + + let compressed = Self { + inner_encoding_id: encoding_id, + inner_metadata: metadata, + compressed_buffers, + uncompressed_sizes, + buffer_alignments, + slots: children.into_iter().map(Some).collect(), + dtype: array.dtype().clone(), + len: array.len(), + stats_set: Default::default(), + }; + let compressed_ref = compressed.clone().into_array(); + compressed + .stats_set + .to_ref(&compressed_ref) + .inherit_from(array.statistics()); + Ok(compressed) + } + fn decompress_buffers(&self) -> VortexResult> { // CPU decode path: zstd::bulk works on host bytes, so compressed buffers are // materialized on the host via `try_to_host_sync`. @@ -245,22 +252,15 @@ impl ZstdBuffersData { Ok(result) } - fn decompress_and_build_inner( - &self, - dtype: &DType, - len: usize, - session: &VortexSession, - ) -> VortexResult { + fn decompress_and_build_inner(&self, session: &VortexSession) -> VortexResult { let decompressed_buffers = self.decompress_buffers()?; - self.build_inner(dtype, len, &decompressed_buffers, session) + self.build_inner(&decompressed_buffers, session) } // This is exposed to help non-CPU executors pass uncompressed buffer handles // to build the inner array. pub fn build_inner( &self, - dtype: &DType, - len: usize, buffer_handles: &[BufferHandle], session: &VortexSession, ) -> VortexResult { @@ -272,8 +272,8 @@ impl ZstdBuffersData { let children: Vec = self.slots.iter().flatten().cloned().collect(); inner_vtable.build( self.inner_encoding_id.clone(), - dtype, - len, + &self.dtype, + self.len, &self.inner_metadata, buffer_handles, &children.as_slice(), @@ -346,15 +346,29 @@ fn array_id_from_string(s: &str) -> ArrayId { impl VTable for ZstdBuffers { type ArrayData = ZstdBuffersData; + + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &ZstdBuffers + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, _dtype: &DType, _len: usize) -> VortexResult<()> { - data.validate() + fn len(array: &ZstdBuffersData) -> usize { + array.len + } + + fn dtype(array: &ZstdBuffersData) -> &DType { + &array.dtype + } + + fn stats(array: &ZstdBuffersData) -> &ArrayStats { + &array.stats_set } fn array_hash( @@ -419,28 +433,36 @@ impl VTable for ZstdBuffers { Ok(()) } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - ZstdBuffersMetadata { - inner_encoding_id: array.inner_encoding_id.to_string(), - inner_metadata: array.inner_metadata.clone(), - uncompressed_sizes: array.uncompressed_sizes.clone(), - buffer_alignments: array.buffer_alignments.clone(), - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(ZstdBuffersMetadata { + inner_encoding_id: array.inner_encoding_id.to_string(), + inner_metadata: array.inner_metadata.clone(), + uncompressed_sizes: array.uncompressed_sizes.clone(), + buffer_alignments: array.buffer_alignments.clone(), + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.0.encode_to_vec())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata(ZstdBuffersMetadata::decode(bytes)?)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], + metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = ZstdBuffersMetadata::decode(metadata)?; let compressed_buffers: Vec = buffers.to_vec(); let child_arrays: Vec> = (0..children.len()) @@ -448,12 +470,15 @@ impl VTable for ZstdBuffers { .collect::>>()?; let data = ZstdBuffersData { - inner_encoding_id: array_id_from_string(&metadata.inner_encoding_id), - inner_metadata: metadata.inner_metadata.clone(), + inner_encoding_id: array_id_from_string(&metadata.0.inner_encoding_id), + inner_metadata: metadata.0.inner_metadata.clone(), compressed_buffers, - uncompressed_sizes: metadata.uncompressed_sizes.clone(), - buffer_alignments: metadata.buffer_alignments.clone(), + uncompressed_sizes: metadata.0.uncompressed_sizes.clone(), + buffer_alignments: metadata.0.buffer_alignments.clone(), slots: child_arrays, + dtype: dtype.clone(), + len, + stats_set: Default::default(), }; data.validate()?; @@ -464,7 +489,7 @@ impl VTable for ZstdBuffers { fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { let session = ctx.session(); - let inner_array = ZstdBuffers::decompress_and_build_inner(&array, session)?; + let inner_array = array.decompress_and_build_inner(session)?; inner_array .execute::(ctx) .map(ExecutionResult::done) @@ -480,11 +505,7 @@ impl OperationsVTable for ZstdBuffers { // TODO(os): maybe we should not support scalar_at, it is really slow, and adding a cache // layer here is weird. Valid use of zstd buffers array would be by executing it first into // canonical - let inner_array = array.data().decompress_and_build_inner( - array.dtype(), - array.len(), - &vortex_array::LEGACY_SESSION, - )?; + let inner_array = array.decompress_and_build_inner(&vortex_array::LEGACY_SESSION)?; inner_array.scalar_at(index) } } @@ -493,15 +514,11 @@ impl ValidityVTable for ZstdBuffers { fn validity( array: ArrayView<'_, ZstdBuffers>, ) -> VortexResult { - if !array.dtype().is_nullable() { + if !array.dtype.is_nullable() { return Ok(vortex_array::validity::Validity::NonNullable); } - let inner_array = array.data().decompress_and_build_inner( - array.dtype(), - array.len(), - &vortex_array::LEGACY_SESSION, - )?; + let inner_array = array.decompress_and_build_inner(&vortex_array::LEGACY_SESSION)?; inner_array.validity() } } @@ -563,10 +580,10 @@ mod tests { #[case::empty_primitive(make_empty_primitive_array())] #[case::inlined_varbinview(make_inlined_varbinview_array())] fn test_roundtrip(#[case] input: ArrayRef) -> VortexResult<()> { - let compressed = ZstdBuffers::compress(&input, 3)?; + let compressed = ZstdBuffersData::compress(&input, 3)?; - assert_eq!(compressed.len(), input.len()); - assert_eq!(compressed.dtype(), input.dtype()); + assert_eq!(compressed.len, input.len()); + assert_eq!(&compressed.dtype, input.dtype()); let mut ctx = LEGACY_SESSION.create_execution_ctx(); let decompressed = compressed.into_array().execute::(&mut ctx)?; @@ -580,7 +597,8 @@ mod tests { let input = make_primitive_array(); input.statistics().set(Stat::Min, Precision::exact(0i32)); - let compressed = ZstdBuffers::compress(&input, 3)?; + let compressed = ZstdBuffersData::compress(&input, 3)?; + let compressed = ZstdBuffersArray::try_from_data(compressed)?; assert!(compressed.statistics().get(Stat::Min).is_some()); Ok(()) @@ -589,7 +607,7 @@ mod tests { #[test] fn test_validity_delegates_for_nullable_input() -> VortexResult<()> { let input = make_nullable_primitive_array(); - let compressed = ZstdBuffers::compress(&input, 3)?.into_array(); + let compressed = ZstdBuffersData::compress(&input, 3)?.into_array(); assert_eq!(compressed.all_valid()?, input.all_valid()?); assert_eq!(compressed.all_invalid()?, input.all_invalid()?); diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index 9431cac68f6..7db3e1f8f74 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -98,7 +98,7 @@ pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult StructArray::try_new_with_dtype( filtered_children, - struct_array.struct_fields(), + struct_array.struct_fields().clone(), filter.iter().filter(|b| **b).map(|b| *b as usize).sum(), validity, ) diff --git a/fuzz/src/array/mask.rs b/fuzz/src/array/mask.rs index 89192b3e360..02974d72df7 100644 --- a/fuzz/src/array/mask.rs +++ b/fuzz/src/array/mask.rs @@ -117,7 +117,7 @@ pub fn mask_canonical_array(canonical: Canonical, mask: &Mask) -> VortexResult>>()?; StructArray::try_new_with_dtype( sliced_children, - struct_array.struct_fields(), + struct_array.struct_fields().clone(), stop - start, validity, ) diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 7e6487c34d1..f1daebb26c6 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -864,6 +864,8 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData +pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -878,20 +880,26 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize + +pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -900,13 +908,15 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -944,15 +954,25 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Bo pub fn vortex_array::arrays::Bool::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Bool>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::bool::BoolArrayParts + +pub vortex_array::arrays::bool::BoolArrayParts::bits: vortex_array::buffer::BufferHandle + +pub vortex_array::arrays::bool::BoolArrayParts::len: usize + +pub vortex_array::arrays::bool::BoolArrayParts::offset: usize + +pub vortex_array::arrays::bool::BoolArrayParts::validity: vortex_array::validity::Validity + pub struct vortex_array::arrays::bool::BoolData impl vortex_array::arrays::bool::BoolData -pub fn vortex_array::arrays::bool::BoolData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::bool::BoolData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::bool::BoolData::into_bit_buffer(self) -> vortex_buffer::bit::buf::BitBuffer -pub fn vortex_array::arrays::bool::BoolData::into_parts(self) -> vortex_array::arrays::bool::BoolDataParts +pub fn vortex_array::arrays::bool::BoolData::into_parts(self) -> vortex_array::arrays::bool::BoolArrayParts pub fn vortex_array::arrays::bool::BoolData::is_empty(&self) -> bool @@ -974,19 +994,17 @@ impl core::clone::Clone for vortex_array::arrays::bool::BoolData pub fn vortex_array::arrays::bool::BoolData::clone(&self) -> vortex_array::arrays::bool::BoolData -impl core::fmt::Debug for vortex_array::arrays::bool::BoolData - -pub fn vortex_array::arrays::bool::BoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::convert::From for vortex_array::ArrayRef -pub struct vortex_array::arrays::bool::BoolDataParts +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::bool::BoolData) -> vortex_array::ArrayRef -pub vortex_array::arrays::bool::BoolDataParts::bits: vortex_array::buffer::BufferHandle +impl core::fmt::Debug for vortex_array::arrays::bool::BoolData -pub vortex_array::arrays::bool::BoolDataParts::len: usize +pub fn vortex_array::arrays::bool::BoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_array::arrays::bool::BoolDataParts::offset: usize +impl vortex_array::IntoArray for vortex_array::arrays::bool::BoolData -pub vortex_array::arrays::bool::BoolDataParts::validity: vortex_array::validity::Validity +pub fn vortex_array::arrays::bool::BoolData::into_array(self) -> vortex_array::ArrayRef pub struct vortex_array::arrays::bool::BoolMaskedValidityRule @@ -1030,6 +1048,8 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData +pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -1044,20 +1064,26 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize + +pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1066,13 +1092,15 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1146,10 +1174,18 @@ impl core::clone::Clone for vortex_array::arrays::chunked::ChunkedData pub fn vortex_array::arrays::chunked::ChunkedData::clone(&self) -> vortex_array::arrays::chunked::ChunkedData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::chunked::ChunkedData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::chunked::ChunkedData pub fn vortex_array::arrays::chunked::ChunkedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::IntoArray for vortex_array::arrays::chunked::ChunkedData + +pub fn vortex_array::arrays::chunked::ChunkedData::into_array(self) -> vortex_array::ArrayRef + pub type vortex_array::arrays::chunked::ChunkedArray = vortex_array::Array pub mod vortex_array::arrays::constant @@ -1180,6 +1216,8 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData +pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar + pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -1194,20 +1232,26 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize + +pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1216,13 +1260,15 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1262,9 +1308,15 @@ pub struct vortex_array::arrays::constant::ConstantData impl vortex_array::arrays::constant::ConstantData +pub fn vortex_array::arrays::constant::ConstantData::dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_array::arrays::constant::ConstantData::into_parts(self) -> vortex_array::scalar::Scalar -pub fn vortex_array::arrays::constant::ConstantData::new(scalar: S) -> Self where S: core::convert::Into +pub fn vortex_array::arrays::constant::ConstantData::is_empty(&self) -> bool + +pub fn vortex_array::arrays::constant::ConstantData::len(&self) -> usize + +pub fn vortex_array::arrays::constant::ConstantData::new(scalar: S, len: usize) -> Self where S: core::convert::Into pub fn vortex_array::arrays::constant::ConstantData::scalar(&self) -> &vortex_array::scalar::Scalar @@ -1272,10 +1324,18 @@ impl core::clone::Clone for vortex_array::arrays::constant::ConstantData pub fn vortex_array::arrays::constant::ConstantData::clone(&self) -> vortex_array::arrays::constant::ConstantData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::constant::ConstantData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::constant::ConstantData pub fn vortex_array::arrays::constant::ConstantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::IntoArray for vortex_array::arrays::constant::ConstantData + +pub fn vortex_array::arrays::constant::ConstantData::into_array(self) -> vortex_array::ArrayRef + pub type vortex_array::arrays::constant::ConstantArray = vortex_array::Array pub mod vortex_array::arrays::datetime @@ -1362,6 +1422,8 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData +pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -1376,20 +1438,26 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize + +pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1398,13 +1466,15 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1442,6 +1512,16 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::De pub fn vortex_array::arrays::Decimal::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Decimal>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::decimal::DecimalArrayParts + +pub vortex_array::arrays::decimal::DecimalArrayParts::decimal_dtype: vortex_array::dtype::DecimalDType + +pub vortex_array::arrays::decimal::DecimalArrayParts::validity: vortex_array::validity::Validity + +pub vortex_array::arrays::decimal::DecimalArrayParts::values: vortex_array::buffer::BufferHandle + +pub vortex_array::arrays::decimal::DecimalArrayParts::values_type: vortex_array::dtype::DecimalType + pub struct vortex_array::arrays::decimal::DecimalData impl vortex_array::arrays::decimal::DecimalData @@ -1452,13 +1532,13 @@ pub fn vortex_array::arrays::decimal::DecimalData::buffer_handle(&self) -> &vort pub fn vortex_array::arrays::decimal::DecimalData::decimal_dtype(&self) -> vortex_array::dtype::DecimalDType -pub fn vortex_array::arrays::decimal::DecimalData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::decimal::DecimalData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::decimal::DecimalData::from_iter>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self pub fn vortex_array::arrays::decimal::DecimalData::from_option_iter>>(iter: I, decimal_dtype: vortex_array::dtype::DecimalDType) -> Self -pub fn vortex_array::arrays::decimal::DecimalData::into_parts(self) -> vortex_array::arrays::decimal::DecimalDataParts +pub fn vortex_array::arrays::decimal::DecimalData::into_parts(self) -> vortex_array::arrays::decimal::DecimalArrayParts pub fn vortex_array::arrays::decimal::DecimalData::is_empty(&self) -> bool @@ -1492,19 +1572,17 @@ impl core::clone::Clone for vortex_array::arrays::decimal::DecimalData pub fn vortex_array::arrays::decimal::DecimalData::clone(&self) -> vortex_array::arrays::decimal::DecimalData -impl core::fmt::Debug for vortex_array::arrays::decimal::DecimalData - -pub fn vortex_array::arrays::decimal::DecimalData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::convert::From for vortex_array::ArrayRef -pub struct vortex_array::arrays::decimal::DecimalDataParts +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::decimal::DecimalData) -> vortex_array::ArrayRef -pub vortex_array::arrays::decimal::DecimalDataParts::decimal_dtype: vortex_array::dtype::DecimalDType +impl core::fmt::Debug for vortex_array::arrays::decimal::DecimalData -pub vortex_array::arrays::decimal::DecimalDataParts::validity: vortex_array::validity::Validity +pub fn vortex_array::arrays::decimal::DecimalData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_array::arrays::decimal::DecimalDataParts::values: vortex_array::buffer::BufferHandle +impl vortex_array::IntoArray for vortex_array::arrays::decimal::DecimalData -pub vortex_array::arrays::decimal::DecimalDataParts::values_type: vortex_array::dtype::DecimalType +pub fn vortex_array::arrays::decimal::DecimalData::into_array(self) -> vortex_array::ArrayRef pub struct vortex_array::arrays::decimal::DecimalMaskedValidityRule @@ -1552,6 +1630,8 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData +pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -1566,20 +1646,26 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize + +pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1588,13 +1674,15 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1658,6 +1746,8 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData +pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -1672,20 +1762,26 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize + +pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1694,13 +1790,15 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -1740,17 +1838,25 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::di pub fn vortex_array::arrays::dict::Dict::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::dict::DictArrayParts + +pub vortex_array::arrays::dict::DictArrayParts::codes: vortex_array::ArrayRef + +pub vortex_array::arrays::dict::DictArrayParts::dtype: vortex_array::dtype::DType + +pub vortex_array::arrays::dict::DictArrayParts::values: vortex_array::ArrayRef + pub struct vortex_array::arrays::dict::DictData impl vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::codes(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::dict::DictData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::dict::DictData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::dict::DictData::has_all_values_referenced(&self) -> bool -pub fn vortex_array::arrays::dict::DictData::into_parts(self) -> vortex_array::arrays::dict::DictDataParts +pub fn vortex_array::arrays::dict::DictData::into_parts(self) -> vortex_array::arrays::dict::DictArrayParts pub fn vortex_array::arrays::dict::DictData::is_empty(&self) -> bool @@ -1762,6 +1868,8 @@ pub unsafe fn vortex_array::arrays::dict::DictData::new_unchecked(codes: vortex_ pub unsafe fn vortex_array::arrays::dict::DictData::set_all_values_referenced(self, all_values_referenced: bool) -> Self +pub fn vortex_array::arrays::dict::DictData::try_new(codes: vortex_array::ArrayRef, values: vortex_array::ArrayRef) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::DictData::validate_all_values_referenced(&self) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::dict::DictData::values(&self) -> &vortex_array::ArrayRef @@ -1774,15 +1882,17 @@ impl core::clone::Clone for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::clone(&self) -> vortex_array::arrays::dict::DictData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::dict::DictData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::dict::DictData pub fn vortex_array::arrays::dict::DictData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct vortex_array::arrays::dict::DictDataParts +impl vortex_array::IntoArray for vortex_array::arrays::dict::DictData -pub vortex_array::arrays::dict::DictDataParts::codes: vortex_array::ArrayRef - -pub vortex_array::arrays::dict::DictDataParts::values: vortex_array::ArrayRef +pub fn vortex_array::arrays::dict::DictData::into_array(self) -> vortex_array::ArrayRef pub struct vortex_array::arrays::dict::DictMetadata @@ -1946,6 +2056,8 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -1960,20 +2072,26 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize + +pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -1982,13 +2100,15 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2024,7 +2144,7 @@ pub struct vortex_array::arrays::extension::ExtensionData impl vortex_array::arrays::extension::ExtensionData -pub fn vortex_array::arrays::extension::ExtensionData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::extension::ExtensionData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::extension::ExtensionData::ext_dtype(&self) -> &vortex_array::dtype::extension::ExtDTypeRef @@ -2044,10 +2164,18 @@ impl core::clone::Clone for vortex_array::arrays::extension::ExtensionData pub fn vortex_array::arrays::extension::ExtensionData::clone(&self) -> vortex_array::arrays::extension::ExtensionData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::extension::ExtensionData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::extension::ExtensionData pub fn vortex_array::arrays::extension::ExtensionData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::IntoArray for vortex_array::arrays::extension::ExtensionData + +pub fn vortex_array::arrays::extension::ExtensionData::into_array(self) -> vortex_array::ArrayRef + pub type vortex_array::arrays::extension::ExtensionArray = vortex_array::Array pub mod vortex_array::arrays::filter @@ -2074,6 +2202,8 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData +pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata + pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -2088,20 +2218,26 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize + +pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2110,13 +2246,15 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2124,6 +2262,12 @@ impl vortex_array::ValidityVTable for vortex_array pub fn vortex_array::arrays::Filter::validity(array: vortex_array::ArrayView<'_, vortex_array::arrays::Filter>) -> vortex_error::VortexResult +pub struct vortex_array::arrays::filter::FilterArrayParts + +pub vortex_array::arrays::filter::FilterArrayParts::child: vortex_array::ArrayRef + +pub vortex_array::arrays::filter::FilterArrayParts::mask: vortex_mask::Mask + pub struct vortex_array::arrays::filter::FilterData impl vortex_array::arrays::filter::FilterData @@ -2134,27 +2278,33 @@ pub fn vortex_array::arrays::filter::FilterData::dtype(&self) -> &vortex_array:: pub fn vortex_array::arrays::filter::FilterData::filter_mask(&self) -> &vortex_mask::Mask -pub fn vortex_array::arrays::filter::FilterData::into_parts(self) -> vortex_array::arrays::filter::FilterDataParts - pub fn vortex_array::arrays::filter::FilterData::is_empty(&self) -> bool pub fn vortex_array::arrays::filter::FilterData::len(&self) -> usize pub fn vortex_array::arrays::filter::FilterData::new(array: vortex_array::ArrayRef, mask: vortex_mask::Mask) -> Self +pub fn vortex_array::arrays::filter::FilterData::try_new(array: vortex_array::ArrayRef, mask: vortex_mask::Mask) -> vortex_error::VortexResult + +impl vortex_array::arrays::filter::FilterData + +pub fn vortex_array::arrays::filter::FilterData::into_parts(self) -> vortex_array::arrays::filter::FilterArrayParts + impl core::clone::Clone for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::clone(&self) -> vortex_array::arrays::filter::FilterData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::filter::FilterData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::filter::FilterData pub fn vortex_array::arrays::filter::FilterData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct vortex_array::arrays::filter::FilterDataParts - -pub vortex_array::arrays::filter::FilterDataParts::child: vortex_array::ArrayRef +impl vortex_array::IntoArray for vortex_array::arrays::filter::FilterData -pub vortex_array::arrays::filter::FilterDataParts::mask: vortex_mask::Mask +pub fn vortex_array::arrays::filter::FilterData::into_array(self) -> vortex_array::ArrayRef pub struct vortex_array::arrays::filter::FilterExecuteAdaptor(pub V) @@ -2262,6 +2412,8 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData +pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -2276,20 +2428,26 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize + +pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2298,13 +2456,15 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2332,7 +2492,7 @@ pub struct vortex_array::arrays::fixed_size_list::FixedSizeListData impl vortex_array::arrays::fixed_size_list::FixedSizeListData -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::elements(&self) -> &vortex_array::ArrayRef @@ -2348,6 +2508,8 @@ pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new(elements: v pub unsafe fn vortex_array::arrays::fixed_size_list::FixedSizeListData::new_unchecked(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> Self +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::try_new(elements: vortex_array::ArrayRef, list_size: u32, validity: vortex_array::validity::Validity, len: usize) -> vortex_error::VortexResult + pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validate(elements: &vortex_array::ArrayRef, len: usize, list_size: u32, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::validity(&self) -> vortex_array::validity::Validity @@ -2362,12 +2524,148 @@ impl core::clone::Clone for vortex_array::arrays::fixed_size_list::FixedSizeList pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::clone(&self) -> vortex_array::arrays::fixed_size_list::FixedSizeListData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::fixed_size_list::FixedSizeListData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::fixed_size_list::FixedSizeListData pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::IntoArray for vortex_array::arrays::fixed_size_list::FixedSizeListData + +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::into_array(self) -> vortex_array::ArrayRef + pub type vortex_array::arrays::fixed_size_list::FixedSizeListArray = vortex_array::Array +pub mod vortex_array::arrays::lazy_patched + +pub struct vortex_array::arrays::lazy_patched::LazyPatched + +impl vortex_array::arrays::lazy_patched::LazyPatched + +pub const vortex_array::arrays::lazy_patched::LazyPatched::ID: vortex_array::ArrayId + +impl core::clone::Clone for vortex_array::arrays::lazy_patched::LazyPatched + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::clone(&self) -> vortex_array::arrays::lazy_patched::LazyPatched + +impl core::fmt::Debug for vortex_array::arrays::lazy_patched::LazyPatched + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl vortex_array::OperationsVTable for vortex_array::arrays::lazy_patched::LazyPatched + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::lazy_patched::LazyPatched>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + +impl vortex_array::VTable for vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ArrayData = vortex_array::arrays::lazy_patched::LazyPatchedData + +pub type vortex_array::arrays::lazy_patched::LazyPatched::Metadata = vortex_array::ProstMetadata + +pub type vortex_array::arrays::lazy_patched::LazyPatched::OperationsVTable = vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ValidityVTable = vortex_array::ValidityVTableFromChild + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: vortex_array::Precision) -> bool + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_hash(array: &Self::ArrayData, state: &mut H, precision: vortex_array::Precision) + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::id(&self) -> vortex_array::ArrayId + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::vtable(_array: &Self::ArrayData) -> &Self + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> + +impl vortex_array::ValidityChild for vortex_array::arrays::lazy_patched::LazyPatched + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::validity_child(array: &vortex_array::arrays::lazy_patched::LazyPatchedData) -> &vortex_array::ArrayRef + +pub struct vortex_array::arrays::lazy_patched::LazyPatchedData + +impl vortex_array::arrays::lazy_patched::LazyPatchedData + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedData::try_new(inner: vortex_array::ArrayRef, patches: vortex_array::patches::Patches) -> vortex_error::VortexResult + +impl core::clone::Clone for vortex_array::arrays::lazy_patched::LazyPatchedData + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedData::clone(&self) -> vortex_array::arrays::lazy_patched::LazyPatchedData + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::lazy_patched::LazyPatchedData) -> vortex_array::ArrayRef + +impl core::fmt::Debug for vortex_array::arrays::lazy_patched::LazyPatchedData + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl vortex_array::IntoArray for vortex_array::arrays::lazy_patched::LazyPatchedData + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedData::into_array(self) -> vortex_array::ArrayRef + +pub struct vortex_array::arrays::lazy_patched::LazyPatchedMetadata + +impl core::clone::Clone for vortex_array::arrays::lazy_patched::LazyPatchedMetadata + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedMetadata::clone(&self) -> vortex_array::arrays::lazy_patched::LazyPatchedMetadata + +impl core::default::Default for vortex_array::arrays::lazy_patched::LazyPatchedMetadata + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedMetadata::default() -> Self + +impl core::fmt::Debug for vortex_array::arrays::lazy_patched::LazyPatchedMetadata + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedMetadata::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result + +impl prost::message::Message for vortex_array::arrays::lazy_patched::LazyPatchedMetadata + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedMetadata::clear(&mut self) + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedMetadata::encoded_len(&self) -> usize + +pub type vortex_array::arrays::lazy_patched::LazyPatchedArray = vortex_array::Array + pub mod vortex_array::arrays::list pub struct vortex_array::arrays::list::List @@ -2392,6 +2690,8 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData +pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -2406,20 +2706,26 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize + +pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2428,13 +2734,15 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2462,17 +2770,27 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Li pub fn vortex_array::arrays::List::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::List>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::list::ListArrayParts + +pub vortex_array::arrays::list::ListArrayParts::dtype: vortex_array::dtype::DType + +pub vortex_array::arrays::list::ListArrayParts::elements: vortex_array::ArrayRef + +pub vortex_array::arrays::list::ListArrayParts::offsets: vortex_array::ArrayRef + +pub vortex_array::arrays::list::ListArrayParts::validity: vortex_array::validity::Validity + pub struct vortex_array::arrays::list::ListData impl vortex_array::arrays::list::ListData -pub fn vortex_array::arrays::list::ListData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::list::ListData::dtype(&self) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::list::ListData::element_dtype(&self) -> &vortex_array::dtype::DType +pub fn vortex_array::arrays::list::ListData::element_dtype(&self) -> &alloc::sync::Arc pub fn vortex_array::arrays::list::ListData::elements(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::list::ListData::into_parts(self) -> vortex_array::arrays::list::ListDataParts +pub fn vortex_array::arrays::list::ListData::into_parts(self) -> vortex_array::arrays::list::ListArrayParts pub fn vortex_array::arrays::list::ListData::is_empty(&self) -> bool @@ -2490,6 +2808,8 @@ pub fn vortex_array::arrays::list::ListData::offsets(&self) -> &vortex_array::Ar pub fn vortex_array::arrays::list::ListData::sliced_elements(&self) -> vortex_error::VortexResult +pub fn vortex_array::arrays::list::ListData::try_new(elements: vortex_array::ArrayRef, offsets: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult + pub fn vortex_array::arrays::list::ListData::validate(elements: &vortex_array::ArrayRef, offsets: &vortex_array::ArrayRef, validity: &vortex_array::validity::Validity) -> vortex_error::VortexResult<()> pub fn vortex_array::arrays::list::ListData::validity(&self) -> vortex_array::validity::Validity @@ -2504,19 +2824,17 @@ impl core::clone::Clone for vortex_array::arrays::list::ListData pub fn vortex_array::arrays::list::ListData::clone(&self) -> vortex_array::arrays::list::ListData -impl core::fmt::Debug for vortex_array::arrays::list::ListData - -pub fn vortex_array::arrays::list::ListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::convert::From for vortex_array::ArrayRef -pub struct vortex_array::arrays::list::ListDataParts +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::list::ListData) -> vortex_array::ArrayRef -pub vortex_array::arrays::list::ListDataParts::dtype: vortex_array::dtype::DType +impl core::fmt::Debug for vortex_array::arrays::list::ListData -pub vortex_array::arrays::list::ListDataParts::elements: vortex_array::ArrayRef +pub fn vortex_array::arrays::list::ListData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_array::arrays::list::ListDataParts::offsets: vortex_array::ArrayRef +impl vortex_array::IntoArray for vortex_array::arrays::list::ListData -pub vortex_array::arrays::list::ListDataParts::validity: vortex_array::validity::Validity +pub fn vortex_array::arrays::list::ListData::into_array(self) -> vortex_array::ArrayRef pub type vortex_array::arrays::list::ListArray = vortex_array::Array @@ -2554,6 +2872,8 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData +pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -2568,20 +2888,26 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize + +pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2590,13 +2916,15 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2620,15 +2948,27 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Li pub fn vortex_array::arrays::ListView::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::ListView>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::listview::ListViewArrayParts + +pub vortex_array::arrays::listview::ListViewArrayParts::elements: vortex_array::ArrayRef + +pub vortex_array::arrays::listview::ListViewArrayParts::elements_dtype: alloc::sync::Arc + +pub vortex_array::arrays::listview::ListViewArrayParts::offsets: vortex_array::ArrayRef + +pub vortex_array::arrays::listview::ListViewArrayParts::sizes: vortex_array::ArrayRef + +pub vortex_array::arrays::listview::ListViewArrayParts::validity: vortex_array::validity::Validity + pub struct vortex_array::arrays::listview::ListViewData impl vortex_array::arrays::listview::ListViewData -pub fn vortex_array::arrays::listview::ListViewData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::listview::ListViewData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::listview::ListViewData::elements(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::listview::ListViewData::into_parts(self) -> vortex_array::arrays::listview::ListViewDataParts +pub fn vortex_array::arrays::listview::ListViewData::into_parts(self) -> vortex_array::arrays::listview::ListViewArrayParts pub fn vortex_array::arrays::listview::ListViewData::is_empty(&self) -> bool @@ -2666,21 +3006,17 @@ impl core::clone::Clone for vortex_array::arrays::listview::ListViewData pub fn vortex_array::arrays::listview::ListViewData::clone(&self) -> vortex_array::arrays::listview::ListViewData -impl core::fmt::Debug for vortex_array::arrays::listview::ListViewData - -pub fn vortex_array::arrays::listview::ListViewData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -pub struct vortex_array::arrays::listview::ListViewDataParts +impl core::convert::From for vortex_array::ArrayRef -pub vortex_array::arrays::listview::ListViewDataParts::elements: vortex_array::ArrayRef +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::listview::ListViewData) -> vortex_array::ArrayRef -pub vortex_array::arrays::listview::ListViewDataParts::elements_dtype: alloc::sync::Arc +impl core::fmt::Debug for vortex_array::arrays::listview::ListViewData -pub vortex_array::arrays::listview::ListViewDataParts::offsets: vortex_array::ArrayRef +pub fn vortex_array::arrays::listview::ListViewData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_array::arrays::listview::ListViewDataParts::sizes: vortex_array::ArrayRef +impl vortex_array::IntoArray for vortex_array::arrays::listview::ListViewData -pub vortex_array::arrays::listview::ListViewDataParts::validity: vortex_array::validity::Validity +pub fn vortex_array::arrays::listview::ListViewData::into_array(self) -> vortex_array::ArrayRef pub fn vortex_array::arrays::listview::list_from_list_view(list_view: vortex_array::arrays::ListViewArray) -> vortex_error::VortexResult @@ -2714,6 +3050,8 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData +pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -2728,20 +3066,26 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize + +pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2750,13 +3094,15 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2786,12 +3132,14 @@ impl vortex_array::arrays::masked::MaskedData pub fn vortex_array::arrays::masked::MaskedData::child(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::masked::MaskedData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::masked::MaskedData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::masked::MaskedData::is_empty(&self) -> bool pub fn vortex_array::arrays::masked::MaskedData::len(&self) -> usize +pub fn vortex_array::arrays::masked::MaskedData::try_new(child: vortex_array::ArrayRef, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult + pub fn vortex_array::arrays::masked::MaskedData::validity(&self) -> vortex_array::validity::Validity pub fn vortex_array::arrays::masked::MaskedData::validity_mask(&self) -> vortex_mask::Mask @@ -2800,10 +3148,18 @@ impl core::clone::Clone for vortex_array::arrays::masked::MaskedData pub fn vortex_array::arrays::masked::MaskedData::clone(&self) -> vortex_array::arrays::masked::MaskedData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::masked::MaskedData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::masked::MaskedData pub fn vortex_array::arrays::masked::MaskedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::IntoArray for vortex_array::arrays::masked::MaskedData + +pub fn vortex_array::arrays::masked::MaskedData::into_array(self) -> vortex_array::ArrayRef + pub fn vortex_array::arrays::masked::mask_validity_canonical(canonical: vortex_array::Canonical, validity_mask: &vortex_mask::Mask, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub type vortex_array::arrays::masked::MaskedArray = vortex_array::Array @@ -2836,6 +3192,8 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData +pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -2850,20 +3208,26 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize + +pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2872,13 +3236,15 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -2910,20 +3276,30 @@ pub struct vortex_array::arrays::null::NullData impl vortex_array::arrays::null::NullData -pub fn vortex_array::arrays::null::NullData::new() -> Self +pub fn vortex_array::arrays::null::NullData::dtype(&self) -> &vortex_array::dtype::DType + +pub fn vortex_array::arrays::null::NullData::is_empty(&self) -> bool + +pub fn vortex_array::arrays::null::NullData::len(&self) -> usize + +pub fn vortex_array::arrays::null::NullData::new(len: usize) -> Self impl core::clone::Clone for vortex_array::arrays::null::NullData pub fn vortex_array::arrays::null::NullData::clone(&self) -> vortex_array::arrays::null::NullData -impl core::default::Default for vortex_array::arrays::null::NullData +impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::arrays::null::NullData::default() -> Self +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::null::NullData) -> vortex_array::ArrayRef impl core::fmt::Debug for vortex_array::arrays::null::NullData pub fn vortex_array::arrays::null::NullData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::IntoArray for vortex_array::arrays::null::NullData + +pub fn vortex_array::arrays::null::NullData::into_array(self) -> vortex_array::ArrayRef + pub type vortex_array::arrays::null::NullArray = vortex_array::Array pub mod vortex_array::arrays::patched @@ -2946,6 +3322,8 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -2960,20 +3338,26 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -2982,13 +3366,15 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3018,12 +3404,8 @@ impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::patched::PatchedArray::is_empty(&self) -> bool - pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize - pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef @@ -3034,7 +3416,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner impl vortex_array::arrays::patched::PatchedArray -pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef where Self: core::clone::Clone + vortex_array::IntoArray +pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray @@ -3042,7 +3424,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::clone(&self) -> vortex_array impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> vortex_array::ArrayRef impl core::fmt::Debug for vortex_array::arrays::patched::PatchedArray @@ -3162,6 +3544,8 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData +pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -3176,20 +3560,26 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize + +pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3198,13 +3588,15 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3242,6 +3634,14 @@ impl vortex_array::scalar_fn::fns::mask::MaskReduce for vortex_array::arrays::Pr pub fn vortex_array::arrays::Primitive::mask(array: vortex_array::ArrayView<'_, vortex_array::arrays::Primitive>, mask: &vortex_array::ArrayRef) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::primitive::PrimitiveArrayParts + +pub vortex_array::arrays::primitive::PrimitiveArrayParts::buffer: vortex_array::buffer::BufferHandle + +pub vortex_array::arrays::primitive::PrimitiveArrayParts::ptype: vortex_array::dtype::PType + +pub vortex_array::arrays::primitive::PrimitiveArrayParts::validity: vortex_array::validity::Validity + pub struct vortex_array::arrays::primitive::PrimitiveData impl vortex_array::arrays::primitive::PrimitiveData @@ -3254,7 +3654,7 @@ impl vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::primitive::PrimitiveData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::primitive::PrimitiveData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::primitive::PrimitiveData::from_buffer_handle(handle: vortex_array::buffer::BufferHandle, ptype: vortex_array::dtype::PType, validity: vortex_array::validity::Validity) -> Self @@ -3300,23 +3700,23 @@ pub fn vortex_array::arrays::primitive::PrimitiveData::validate vortex_array::arrays::primitive::PrimitiveDataParts +pub fn vortex_array::arrays::primitive::PrimitiveData::into_parts(self) -> vortex_array::arrays::primitive::PrimitiveArrayParts impl core::clone::Clone for vortex_array::arrays::primitive::PrimitiveData pub fn vortex_array::arrays::primitive::PrimitiveData::clone(&self) -> vortex_array::arrays::primitive::PrimitiveData -impl core::fmt::Debug for vortex_array::arrays::primitive::PrimitiveData +impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::arrays::primitive::PrimitiveData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::primitive::PrimitiveData) -> vortex_array::ArrayRef -pub struct vortex_array::arrays::primitive::PrimitiveDataParts +impl core::fmt::Debug for vortex_array::arrays::primitive::PrimitiveData -pub vortex_array::arrays::primitive::PrimitiveDataParts::buffer: vortex_array::buffer::BufferHandle +pub fn vortex_array::arrays::primitive::PrimitiveData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_array::arrays::primitive::PrimitiveDataParts::ptype: vortex_array::dtype::PType +impl vortex_array::IntoArray for vortex_array::arrays::primitive::PrimitiveData -pub vortex_array::arrays::primitive::PrimitiveDataParts::validity: vortex_array::validity::Validity +pub fn vortex_array::arrays::primitive::PrimitiveData::into_array(self) -> vortex_array::ArrayRef pub struct vortex_array::arrays::primitive::PrimitiveMaskedValidityRule @@ -3392,10 +3792,16 @@ impl vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::children(&self) -> alloc::vec::Vec +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::dtype(&self) -> &vortex_array::dtype::DType + pub fn vortex_array::arrays::scalar_fn::ScalarFnData::get_child(&self, idx: usize) -> &vortex_array::ArrayRef +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::is_empty(&self) -> bool + pub fn vortex_array::arrays::scalar_fn::ScalarFnData::iter_children(&self) -> impl core::iter::traits::iterator::Iterator + '_ +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::len(&self) -> usize + pub fn vortex_array::arrays::scalar_fn::ScalarFnData::nchildren(&self) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnData::scalar_fn(&self) -> &vortex_array::scalar_fn::ScalarFnRef @@ -3406,10 +3812,18 @@ impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::clone(&self) -> vortex_array::arrays::scalar_fn::ScalarFnData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::scalar_fn::ScalarFnData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::scalar_fn::ScalarFnData pub fn vortex_array::arrays::scalar_fn::ScalarFnData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::IntoArray for vortex_array::arrays::scalar_fn::ScalarFnData + +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::into_array(self) -> vortex_array::ArrayRef + pub struct vortex_array::arrays::scalar_fn::ScalarFnVTable impl core::clone::Clone for vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3428,6 +3842,8 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata + pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -3442,20 +3858,26 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3464,13 +3886,15 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3512,6 +3936,8 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData +pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -3526,20 +3952,26 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize + +pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3548,13 +3980,15 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3582,10 +4016,18 @@ impl core::clone::Clone for vortex_array::arrays::shared::SharedData pub fn vortex_array::arrays::shared::SharedData::clone(&self) -> vortex_array::arrays::shared::SharedData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::shared::SharedData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::shared::SharedData pub fn vortex_array::arrays::shared::SharedData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::IntoArray for vortex_array::arrays::shared::SharedData + +pub fn vortex_array::arrays::shared::SharedData::into_array(self) -> vortex_array::ArrayRef + pub type vortex_array::arrays::shared::SharedArray = vortex_array::Array pub mod vortex_array::arrays::slice @@ -3612,6 +4054,8 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData +pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata + pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -3626,20 +4070,26 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize + +pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3648,13 +4098,15 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3666,6 +4118,12 @@ impl vortex_array::arrays::slice::SliceReduce for vortex_array::arrays::slice::S pub fn vortex_array::arrays::slice::Slice::slice(array: vortex_array::ArrayView<'_, Self>, range: core::ops::range::Range) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::slice::SliceArrayParts + +pub vortex_array::arrays::slice::SliceArrayParts::child: vortex_array::ArrayRef + +pub vortex_array::arrays::slice::SliceArrayParts::range: core::ops::range::Range + pub struct vortex_array::arrays::slice::SliceData impl vortex_array::arrays::slice::SliceData @@ -3674,8 +4132,6 @@ pub fn vortex_array::arrays::slice::SliceData::child(&self) -> &vortex_array::Ar pub fn vortex_array::arrays::slice::SliceData::dtype(&self) -> &vortex_array::dtype::DType -pub fn vortex_array::arrays::slice::SliceData::into_parts(self) -> vortex_array::arrays::slice::SliceDataParts - pub fn vortex_array::arrays::slice::SliceData::is_empty(&self) -> bool pub fn vortex_array::arrays::slice::SliceData::len(&self) -> usize @@ -3684,19 +4140,27 @@ pub fn vortex_array::arrays::slice::SliceData::new(child: vortex_array::ArrayRef pub fn vortex_array::arrays::slice::SliceData::slice_range(&self) -> &core::ops::range::Range +pub fn vortex_array::arrays::slice::SliceData::try_new(child: vortex_array::ArrayRef, range: core::ops::range::Range) -> vortex_error::VortexResult + +impl vortex_array::arrays::slice::SliceData + +pub fn vortex_array::arrays::slice::SliceData::into_parts(self) -> vortex_array::arrays::slice::SliceArrayParts + impl core::clone::Clone for vortex_array::arrays::slice::SliceData pub fn vortex_array::arrays::slice::SliceData::clone(&self) -> vortex_array::arrays::slice::SliceData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::slice::SliceData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::slice::SliceData pub fn vortex_array::arrays::slice::SliceData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct vortex_array::arrays::slice::SliceDataParts - -pub vortex_array::arrays::slice::SliceDataParts::child: vortex_array::ArrayRef +impl vortex_array::IntoArray for vortex_array::arrays::slice::SliceData -pub vortex_array::arrays::slice::SliceDataParts::range: core::ops::range::Range +pub fn vortex_array::arrays::slice::SliceData::into_array(self) -> vortex_array::ArrayRef pub struct vortex_array::arrays::slice::SliceExecuteAdaptor(pub V) @@ -3838,6 +4302,8 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData +pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -3852,20 +4318,26 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize + +pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -3874,13 +4346,15 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -3908,17 +4382,25 @@ impl vortex_array::scalar_fn::fns::zip::ZipKernel for vortex_array::arrays::Stru pub fn vortex_array::arrays::Struct::zip(if_true: vortex_array::ArrayView<'_, vortex_array::arrays::Struct>, if_false: &vortex_array::ArrayRef, mask: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::struct_::StructArrayParts + +pub vortex_array::arrays::struct_::StructArrayParts::fields: alloc::sync::Arc<[vortex_array::ArrayRef]> + +pub vortex_array::arrays::struct_::StructArrayParts::struct_fields: vortex_array::dtype::StructFields + +pub vortex_array::arrays::struct_::StructArrayParts::validity: vortex_array::validity::Validity + pub struct vortex_array::arrays::struct_::StructData impl vortex_array::arrays::struct_::StructData -pub fn vortex_array::arrays::struct_::StructData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::struct_::StructData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::struct_::StructData::from_fields>(items: &[(N, vortex_array::ArrayRef)]) -> vortex_error::VortexResult pub fn vortex_array::arrays::struct_::StructData::into_fields(self) -> alloc::vec::Vec -pub fn vortex_array::arrays::struct_::StructData::into_parts(self) -> vortex_array::arrays::struct_::StructDataParts +pub fn vortex_array::arrays::struct_::StructData::into_parts(self) -> vortex_array::arrays::struct_::StructArrayParts pub fn vortex_array::arrays::struct_::StructData::is_empty(&self) -> bool @@ -3934,14 +4416,20 @@ pub fn vortex_array::arrays::struct_::StructData::new_fieldless_with_len(len: us pub unsafe fn vortex_array::arrays::struct_::StructData::new_unchecked(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> Self +pub fn vortex_array::arrays::struct_::StructData::project(&self, projection: &[vortex_array::dtype::FieldName]) -> vortex_error::VortexResult + pub fn vortex_array::arrays::struct_::StructData::remove_column(&mut self, name: impl core::convert::Into) -> core::option::Option -pub fn vortex_array::arrays::struct_::StructData::struct_fields(&self) -> vortex_array::dtype::StructFields +pub fn vortex_array::arrays::struct_::StructData::struct_fields(&self) -> &vortex_array::dtype::StructFields pub fn vortex_array::arrays::struct_::StructData::try_from_iter, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T) -> vortex_error::VortexResult pub fn vortex_array::arrays::struct_::StructData::try_from_iter_with_validity, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult +pub fn vortex_array::arrays::struct_::StructData::try_new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::struct_::StructData::try_new_with_dtype(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult + pub fn vortex_array::arrays::struct_::StructData::unmasked_field(&self, idx: usize) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::struct_::StructData::unmasked_field_by_name(&self, name: impl core::convert::AsRef) -> vortex_error::VortexResult<&vortex_array::ArrayRef> @@ -3962,17 +4450,17 @@ impl core::clone::Clone for vortex_array::arrays::struct_::StructData pub fn vortex_array::arrays::struct_::StructData::clone(&self) -> vortex_array::arrays::struct_::StructData -impl core::fmt::Debug for vortex_array::arrays::struct_::StructData +impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::arrays::struct_::StructData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::struct_::StructData) -> vortex_array::ArrayRef -pub struct vortex_array::arrays::struct_::StructDataParts +impl core::fmt::Debug for vortex_array::arrays::struct_::StructData -pub vortex_array::arrays::struct_::StructDataParts::fields: alloc::sync::Arc<[vortex_array::ArrayRef]> +pub fn vortex_array::arrays::struct_::StructData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub vortex_array::arrays::struct_::StructDataParts::struct_fields: vortex_array::dtype::StructFields +impl vortex_array::IntoArray for vortex_array::arrays::struct_::StructData -pub vortex_array::arrays::struct_::StructDataParts::validity: vortex_array::validity::Validity +pub fn vortex_array::arrays::struct_::StructData::into_array(self) -> vortex_array::ArrayRef pub type vortex_array::arrays::struct_::StructArray = vortex_array::Array @@ -4030,6 +4518,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData +pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -4044,20 +4534,26 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize + +pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4066,13 +4562,15 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4112,7 +4610,7 @@ pub fn vortex_array::arrays::varbin::VarBinData::bytes(&self) -> &vortex_buffer: pub fn vortex_array::arrays::varbin::VarBinData::bytes_handle(&self) -> &vortex_array::buffer::BufferHandle -pub fn vortex_array::arrays::varbin::VarBinData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::varbin::VarBinData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::varbin::VarBinData::from_iter, I: core::iter::traits::collect::IntoIterator>>(iter: I, dtype: vortex_array::dtype::DType) -> Self @@ -4190,6 +4688,10 @@ impl core::convert::From>>) -> Self +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbin::VarBinData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::varbin::VarBinData pub fn vortex_array::arrays::varbin::VarBinData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -4202,6 +4704,10 @@ impl core::iter::traits::collect::FromIterator>>>(iter: T) -> Self +impl vortex_array::IntoArray for vortex_array::arrays::varbin::VarBinData + +pub fn vortex_array::arrays::varbin::VarBinData::into_array(self) -> vortex_array::ArrayRef + impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbin::VarBinData pub fn vortex_array::arrays::varbin::VarBinData::from_iter>>(iter: T) -> Self @@ -4420,6 +4926,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData +pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -4434,20 +4942,26 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize + +pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4456,13 +4970,15 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4490,6 +5006,16 @@ impl vortex_array::scalar_fn::fns::zip::ZipKernel for vortex_array::arrays::VarB pub fn vortex_array::arrays::VarBinView::zip(if_true: vortex_array::ArrayView<'_, vortex_array::arrays::VarBinView>, if_false: &vortex_array::ArrayRef, mask: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> +pub struct vortex_array::arrays::varbinview::VarBinViewArrayParts + +pub vortex_array::arrays::varbinview::VarBinViewArrayParts::buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> + +pub vortex_array::arrays::varbinview::VarBinViewArrayParts::dtype: vortex_array::dtype::DType + +pub vortex_array::arrays::varbinview::VarBinViewArrayParts::validity: vortex_array::validity::Validity + +pub vortex_array::arrays::varbinview::VarBinViewArrayParts::views: vortex_array::buffer::BufferHandle + pub struct vortex_array::arrays::varbinview::VarBinViewData impl vortex_array::arrays::varbinview::VarBinViewData @@ -4500,7 +5026,7 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::bytes_at(&self, index: pub fn vortex_array::arrays::varbinview::VarBinViewData::data_buffers(&self) -> &alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> -pub fn vortex_array::arrays::varbinview::VarBinViewData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::varbinview::VarBinViewData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter, I: core::iter::traits::collect::IntoIterator>>(iter: I, dtype: vortex_array::dtype::DType) -> Self @@ -4512,7 +5038,7 @@ pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter_nullable_str< pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter_str, I: core::iter::traits::collect::IntoIterator>(iter: I) -> Self -pub fn vortex_array::arrays::varbinview::VarBinViewData::into_parts(self) -> vortex_array::arrays::varbinview::VarBinViewDataParts +pub fn vortex_array::arrays::varbinview::VarBinViewData::into_parts(self) -> vortex_array::arrays::varbinview::VarBinViewArrayParts pub fn vortex_array::arrays::varbinview::VarBinViewData::is_empty(&self) -> bool @@ -4544,6 +5070,10 @@ impl core::clone::Clone for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::clone(&self) -> vortex_array::arrays::varbinview::VarBinViewData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbinview::VarBinViewData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result @@ -4556,6 +5086,10 @@ impl core::iter::traits::collect::FromIterator>>>(iter: T) -> Self +impl vortex_array::IntoArray for vortex_array::arrays::varbinview::VarBinViewData + +pub fn vortex_array::arrays::varbinview::VarBinViewData::into_array(self) -> vortex_array::ArrayRef + impl<'a> core::iter::traits::collect::FromIterator> for vortex_array::arrays::varbinview::VarBinViewData pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter>>(iter: T) -> Self @@ -4564,16 +5098,6 @@ impl<'a> core::iter::traits::collect::FromIterator pub fn vortex_array::arrays::varbinview::VarBinViewData::from_iter>>(iter: T) -> Self -pub struct vortex_array::arrays::varbinview::VarBinViewDataParts - -pub vortex_array::arrays::varbinview::VarBinViewDataParts::buffers: alloc::sync::Arc<[vortex_array::buffer::BufferHandle]> - -pub vortex_array::arrays::varbinview::VarBinViewDataParts::dtype: vortex_array::dtype::DType - -pub vortex_array::arrays::varbinview::VarBinViewDataParts::validity: vortex_array::validity::Validity - -pub vortex_array::arrays::varbinview::VarBinViewDataParts::views: vortex_array::buffer::BufferHandle - pub type vortex_array::arrays::varbinview::VarBinViewArray = vortex_array::Array pub mod vortex_array::arrays::variant @@ -4600,6 +5124,8 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData +pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -4614,20 +5140,26 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4636,13 +5168,15 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4656,7 +5190,7 @@ impl vortex_array::arrays::variant::VariantData pub fn vortex_array::arrays::variant::VariantData::child(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::variant::VariantData::dtype(&self) -> vortex_array::dtype::DType +pub fn vortex_array::arrays::variant::VariantData::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::variant::VariantData::is_empty(&self) -> bool @@ -4668,10 +5202,18 @@ impl core::clone::Clone for vortex_array::arrays::variant::VariantData pub fn vortex_array::arrays::variant::VariantData::clone(&self) -> vortex_array::arrays::variant::VariantData +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::variant::VariantData) -> vortex_array::ArrayRef + impl core::fmt::Debug for vortex_array::arrays::variant::VariantData pub fn vortex_array::arrays::variant::VariantData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl vortex_array::IntoArray for vortex_array::arrays::variant::VariantData + +pub fn vortex_array::arrays::variant::VariantData::into_array(self) -> vortex_array::ArrayRef + pub type vortex_array::arrays::variant::VariantArray = vortex_array::Array pub struct vortex_array::arrays::Bool @@ -4696,6 +5238,8 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData +pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -4710,20 +5254,26 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize + +pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4732,13 +5282,15 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4798,6 +5350,8 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData +pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -4812,20 +5366,26 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize + +pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4834,13 +5394,15 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -4902,6 +5464,8 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData +pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar + pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -4916,20 +5480,26 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize + +pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -4938,13 +5508,15 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5002,6 +5574,8 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData +pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -5016,20 +5590,26 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize + +pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5038,13 +5618,15 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5104,6 +5686,8 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData +pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -5118,20 +5702,26 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize + +pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5140,13 +5730,15 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5208,6 +5800,8 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -5222,20 +5816,26 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize + +pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5244,13 +5844,15 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5304,6 +5906,8 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData +pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata + pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -5318,20 +5922,26 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize + +pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5340,13 +5950,15 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5376,6 +5988,8 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData +pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -5390,20 +6004,26 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize + +pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5412,13 +6032,15 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5464,6 +6086,8 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData +pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -5478,20 +6102,26 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize + +pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5500,13 +6130,15 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5556,6 +6188,8 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData +pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -5570,20 +6204,26 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize + +pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5592,13 +6232,15 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5644,6 +6286,8 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData +pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -5658,20 +6302,26 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize + +pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5680,13 +6330,15 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5736,6 +6388,8 @@ impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData +pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -5750,20 +6404,26 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize + +pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5772,13 +6432,15 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5824,6 +6486,8 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -5838,20 +6502,26 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5860,13 +6530,15 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -5896,12 +6568,8 @@ impl vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::base_array(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::patched::PatchedArray::is_empty(&self) -> bool - pub fn vortex_array::arrays::patched::PatchedArray::lane_offsets(&self) -> &vortex_array::ArrayRef -pub fn vortex_array::arrays::patched::PatchedArray::len(&self) -> usize - pub fn vortex_array::arrays::patched::PatchedArray::patch_indices(&self) -> &vortex_array::ArrayRef pub fn vortex_array::arrays::patched::PatchedArray::patch_values(&self) -> &vortex_array::ArrayRef @@ -5912,7 +6580,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::from_array_and_patches(inner impl vortex_array::arrays::patched::PatchedArray -pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef where Self: core::clone::Clone + vortex_array::IntoArray +pub fn vortex_array::arrays::patched::PatchedArray::to_array(&self) -> vortex_array::ArrayRef impl core::clone::Clone for vortex_array::arrays::patched::PatchedArray @@ -5920,7 +6588,7 @@ pub fn vortex_array::arrays::patched::PatchedArray::clone(&self) -> vortex_array impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> vortex_array::ArrayRef impl core::fmt::Debug for vortex_array::arrays::patched::PatchedArray @@ -5952,6 +6620,8 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData +pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -5966,20 +6636,26 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize + +pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -5988,13 +6664,15 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6050,6 +6728,8 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata + pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -6064,20 +6744,26 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6086,13 +6772,15 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6122,6 +6810,8 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData +pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -6136,20 +6826,26 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize + +pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6158,13 +6854,15 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6194,6 +6892,8 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData +pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata + pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -6208,20 +6908,26 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize + +pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6230,13 +6936,15 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6270,6 +6978,8 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData +pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -6284,20 +6994,26 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize + +pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6306,13 +7022,15 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6366,6 +7084,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData +pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -6380,20 +7100,26 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize + +pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6402,13 +7128,15 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6462,6 +7190,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData +pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -6476,20 +7206,26 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize + +pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6498,13 +7234,15 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6554,6 +7292,8 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData +pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -6568,20 +7308,26 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -6590,13 +7336,15 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -6950,11 +7698,11 @@ impl core::clone::Clone for vortex_array::buffer::BufferHandle pub fn vortex_array::buffer::BufferHandle::clone(&self) -> vortex_array::buffer::BufferHandle -impl core::convert::TryFrom for vortex_array::serde::SerializedArray +impl core::convert::TryFrom for vortex_array::serde::ArrayParts -pub type vortex_array::serde::SerializedArray::Error = vortex_error::VortexError +pub type vortex_array::serde::ArrayParts::Error = vortex_error::VortexError -pub fn vortex_array::serde::SerializedArray::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result +pub fn vortex_array::serde::ArrayParts::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result impl core::fmt::Debug for vortex_array::buffer::BufferHandle @@ -18184,67 +18932,67 @@ pub fn vortex_array::serde::ArrayNodeFlatBuffer<'a>::try_new(ctx: &'a vortex_arr pub fn vortex_array::serde::ArrayNodeFlatBuffer<'a>::try_write_flatbuffer<'fb>(&self, fbb: &mut flatbuffers::builder::FlatBufferBuilder<'fb>) -> vortex_error::VortexResult>> -pub struct vortex_array::serde::SerializeOptions +pub struct vortex_array::serde::ArrayParts -pub vortex_array::serde::SerializeOptions::include_padding: bool +impl vortex_array::serde::ArrayParts -pub vortex_array::serde::SerializeOptions::offset: usize +pub fn vortex_array::serde::ArrayParts::buffer(&self, idx: usize) -> vortex_error::VortexResult -impl core::default::Default for vortex_array::serde::SerializeOptions +pub fn vortex_array::serde::ArrayParts::buffer_lengths(&self) -> alloc::vec::Vec -pub fn vortex_array::serde::SerializeOptions::default() -> vortex_array::serde::SerializeOptions +pub fn vortex_array::serde::ArrayParts::child(&self, idx: usize) -> vortex_array::serde::ArrayParts -impl core::fmt::Debug for vortex_array::serde::SerializeOptions +pub fn vortex_array::serde::ArrayParts::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_session::registry::ReadContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult -pub fn vortex_array::serde::SerializeOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::serde::ArrayParts::encoding_id(&self) -> u16 -pub struct vortex_array::serde::SerializedArray +pub fn vortex_array::serde::ArrayParts::from_array_tree(array_tree: impl core::convert::Into) -> vortex_error::VortexResult -impl vortex_array::serde::SerializedArray +pub fn vortex_array::serde::ArrayParts::from_flatbuffer_and_segment(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle) -> vortex_error::VortexResult -pub fn vortex_array::serde::SerializedArray::buffer(&self, idx: usize) -> vortex_error::VortexResult +pub fn vortex_array::serde::ArrayParts::from_flatbuffer_and_segment_with_overrides(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle, buffer_overrides: &vortex_utils::aliases::hash_map::HashMap) -> vortex_error::VortexResult -pub fn vortex_array::serde::SerializedArray::buffer_lengths(&self) -> alloc::vec::Vec +pub fn vortex_array::serde::ArrayParts::from_flatbuffer_with_buffers(array_tree: impl core::convert::Into, buffers: alloc::vec::Vec) -> vortex_error::VortexResult -pub fn vortex_array::serde::SerializedArray::child(&self, idx: usize) -> vortex_array::serde::SerializedArray +pub fn vortex_array::serde::ArrayParts::metadata(&self) -> &[u8] -pub fn vortex_array::serde::SerializedArray::decode(&self, dtype: &vortex_array::dtype::DType, len: usize, ctx: &vortex_session::registry::ReadContext, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::serde::ArrayParts::nbuffers(&self) -> usize -pub fn vortex_array::serde::SerializedArray::encoding_id(&self) -> u16 +pub fn vortex_array::serde::ArrayParts::nchildren(&self) -> usize -pub fn vortex_array::serde::SerializedArray::from_array_tree(array_tree: impl core::convert::Into) -> vortex_error::VortexResult +impl core::clone::Clone for vortex_array::serde::ArrayParts -pub fn vortex_array::serde::SerializedArray::from_flatbuffer_and_segment(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle) -> vortex_error::VortexResult +pub fn vortex_array::serde::ArrayParts::clone(&self) -> vortex_array::serde::ArrayParts -pub fn vortex_array::serde::SerializedArray::from_flatbuffer_and_segment_with_overrides(array_tree: vortex_buffer::ByteBuffer, segment: vortex_array::buffer::BufferHandle, buffer_overrides: &vortex_utils::aliases::hash_map::HashMap) -> vortex_error::VortexResult +impl core::convert::TryFrom for vortex_array::serde::ArrayParts -pub fn vortex_array::serde::SerializedArray::from_flatbuffer_with_buffers(array_tree: impl core::convert::Into, buffers: alloc::vec::Vec) -> vortex_error::VortexResult +pub type vortex_array::serde::ArrayParts::Error = vortex_error::VortexError -pub fn vortex_array::serde::SerializedArray::metadata(&self) -> &[u8] +pub fn vortex_array::serde::ArrayParts::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result -pub fn vortex_array::serde::SerializedArray::nbuffers(&self) -> usize +impl core::convert::TryFrom> for vortex_array::serde::ArrayParts -pub fn vortex_array::serde::SerializedArray::nchildren(&self) -> usize +pub type vortex_array::serde::ArrayParts::Error = vortex_error::VortexError -impl core::clone::Clone for vortex_array::serde::SerializedArray +pub fn vortex_array::serde::ArrayParts::try_from(value: vortex_buffer::ByteBuffer) -> core::result::Result -pub fn vortex_array::serde::SerializedArray::clone(&self) -> vortex_array::serde::SerializedArray +impl core::fmt::Debug for vortex_array::serde::ArrayParts -impl core::convert::TryFrom for vortex_array::serde::SerializedArray +pub fn vortex_array::serde::ArrayParts::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub type vortex_array::serde::SerializedArray::Error = vortex_error::VortexError +pub struct vortex_array::serde::SerializeOptions -pub fn vortex_array::serde::SerializedArray::try_from(value: vortex_array::buffer::BufferHandle) -> core::result::Result +pub vortex_array::serde::SerializeOptions::include_padding: bool -impl core::convert::TryFrom> for vortex_array::serde::SerializedArray +pub vortex_array::serde::SerializeOptions::offset: usize -pub type vortex_array::serde::SerializedArray::Error = vortex_error::VortexError +impl core::default::Default for vortex_array::serde::SerializeOptions -pub fn vortex_array::serde::SerializedArray::try_from(value: vortex_buffer::ByteBuffer) -> core::result::Result +pub fn vortex_array::serde::SerializeOptions::default() -> vortex_array::serde::SerializeOptions -impl core::fmt::Debug for vortex_array::serde::SerializedArray +impl core::fmt::Debug for vortex_array::serde::SerializeOptions -pub fn vortex_array::serde::SerializedArray::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub fn vortex_array::serde::SerializeOptions::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub trait vortex_array::serde::ArrayChildren @@ -18788,7 +19536,9 @@ pub fn vortex_array::ValidityVTableFromValiditySliceHelper::validity(array: vort pub trait vortex_array::vtable::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::vtable::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug +pub type vortex_array::vtable::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray + +pub type vortex_array::vtable::ArrayVTable::Metadata: core::fmt::Debug pub type vortex_array::vtable::ArrayVTable::OperationsVTable: vortex_array::OperationsVTable @@ -18804,20 +19554,26 @@ pub fn vortex_array::vtable::ArrayVTable::buffer(array: vortex_array::ArrayView< pub fn vortex_array::vtable::ArrayVTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::vtable::ArrayVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::vtable::ArrayVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::vtable::ArrayVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::vtable::ArrayVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::vtable::ArrayVTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::vtable::ArrayVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::vtable::ArrayVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::ArrayVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::vtable::ArrayVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::vtable::ArrayVTable::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::vtable::ArrayVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::vtable::ArrayVTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::vtable::ArrayVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -18826,13 +19582,15 @@ pub fn vortex_array::vtable::ArrayVTable::reduce(array: vortex_array::ArrayView< pub fn vortex_array::vtable::ArrayVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::ArrayVTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::vtable::ArrayVTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::vtable::ArrayVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::vtable::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::vtable::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::vtable::ArrayVTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::vtable::ArrayVTable::vtable(array: &Self::ArrayData) -> &Self pub fn vortex_array::vtable::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -18840,6 +19598,8 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData +pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -18854,20 +19614,26 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize + +pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -18876,13 +19642,15 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -18890,6 +19658,8 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData +pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -18904,20 +19674,26 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize + +pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -18926,13 +19702,15 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -18940,6 +19718,8 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData +pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar + pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -18954,20 +19734,26 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize + +pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -18976,13 +19762,15 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -18990,6 +19778,8 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData +pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -19004,20 +19794,26 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize + +pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19026,13 +19822,15 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19040,6 +19838,8 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -19054,20 +19854,26 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize + +pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19076,13 +19882,15 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19090,6 +19898,8 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData +pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata + pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -19104,20 +19914,26 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize + +pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19126,13 +19942,15 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19140,6 +19958,8 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData +pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -19154,20 +19974,26 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize + +pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19176,13 +20002,15 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19190,6 +20018,8 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData +pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -19204,20 +20034,26 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize + +pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19226,13 +20062,15 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19240,6 +20078,8 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData +pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -19254,20 +20094,26 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize + +pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19276,13 +20122,15 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19290,6 +20138,8 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData +pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -19304,20 +20154,26 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize + +pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19326,13 +20182,15 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19340,6 +20198,8 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData +pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -19354,20 +20214,26 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize + +pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19376,13 +20242,15 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19390,6 +20258,8 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData +pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -19404,20 +20274,26 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize + +pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19426,13 +20302,15 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19440,6 +20318,8 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData +pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -19454,20 +20334,26 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize + +pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19476,13 +20362,15 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19490,6 +20378,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData +pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -19504,20 +20394,26 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize + +pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19526,13 +20422,15 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19540,6 +20438,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData +pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -19554,20 +20454,26 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize + +pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19576,13 +20482,15 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19590,6 +20498,8 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData +pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -19604,20 +20514,26 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19626,13 +20542,15 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19640,6 +20558,8 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData +pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -19654,20 +20574,26 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize + +pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19676,20 +20602,84 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::VTable for vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ArrayData = vortex_array::arrays::lazy_patched::LazyPatchedData + +pub type vortex_array::arrays::lazy_patched::LazyPatched::Metadata = vortex_array::ProstMetadata + +pub type vortex_array::arrays::lazy_patched::LazyPatched::OperationsVTable = vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ValidityVTable = vortex_array::ValidityVTableFromChild + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: vortex_array::Precision) -> bool + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_hash(array: &Self::ArrayData, state: &mut H, precision: vortex_array::Precision) + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::id(&self) -> vortex_array::ArrayId + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::vtable(_array: &Self::ArrayData) -> &Self + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> + impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData +pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -19704,20 +20694,26 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize + +pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19726,13 +20722,15 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19740,6 +20738,8 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -19754,20 +20754,26 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19776,13 +20782,15 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19790,6 +20798,8 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata + pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -19804,20 +20814,26 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19826,13 +20842,15 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19840,6 +20858,8 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData +pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata + pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -19854,20 +20874,26 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize + +pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -19876,13 +20902,15 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -19990,6 +21018,10 @@ impl vortex_array::OperationsVTable for vortex pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +impl vortex_array::OperationsVTable for vortex_array::arrays::lazy_patched::LazyPatched + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::lazy_patched::LazyPatched>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + impl vortex_array::OperationsVTable for vortex_array::arrays::null::Null pub fn vortex_array::arrays::null::Null::scalar_at(_array: vortex_array::ArrayView<'_, vortex_array::arrays::null::Null>, _index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -20012,7 +21044,9 @@ pub fn vortex_array::NotSupported::scalar_at(array: vortex_array::ArrayView<'_, pub trait vortex_array::vtable::VTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::vtable::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug +pub type vortex_array::vtable::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray + +pub type vortex_array::vtable::VTable::Metadata: core::fmt::Debug pub type vortex_array::vtable::VTable::OperationsVTable: vortex_array::OperationsVTable @@ -20028,20 +21062,26 @@ pub fn vortex_array::vtable::VTable::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::vtable::VTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::vtable::VTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::vtable::VTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::vtable::VTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::vtable::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::vtable::VTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::vtable::VTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::vtable::VTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::vtable::VTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::VTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::vtable::VTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::vtable::VTable::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::vtable::VTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::vtable::VTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::vtable::VTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20050,13 +21090,15 @@ pub fn vortex_array::vtable::VTable::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::vtable::VTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::vtable::VTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::vtable::VTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::vtable::VTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::vtable::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::vtable::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::vtable::VTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::vtable::VTable::vtable(array: &Self::ArrayData) -> &Self pub fn vortex_array::vtable::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20064,6 +21106,8 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData +pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -20078,20 +21122,26 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize + +pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20100,13 +21150,15 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20114,6 +21166,8 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData +pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -20128,20 +21182,26 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize + +pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20150,13 +21210,15 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20164,6 +21226,8 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData +pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar + pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -20178,20 +21242,26 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize + +pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20200,13 +21270,15 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20214,6 +21286,8 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData +pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -20228,20 +21302,26 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize + +pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20250,13 +21330,15 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20264,6 +21346,8 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -20278,20 +21362,26 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize + +pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20300,13 +21390,15 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20314,6 +21406,8 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData +pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata + pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -20328,20 +21422,26 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize + +pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20350,13 +21450,15 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20364,6 +21466,8 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData +pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -20378,20 +21482,26 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize + +pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20400,13 +21510,15 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20414,6 +21526,8 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData +pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -20428,20 +21542,26 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize + +pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20450,13 +21570,15 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20464,6 +21586,8 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData +pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -20478,20 +21602,26 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize + +pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20500,13 +21630,15 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20514,6 +21646,8 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData +pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -20528,20 +21662,26 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize + +pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20550,13 +21690,15 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20564,6 +21706,8 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData +pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -20578,20 +21722,26 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize + +pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20600,13 +21750,15 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20614,6 +21766,8 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData +pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -20628,20 +21782,26 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize + +pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20650,13 +21810,15 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20664,6 +21826,8 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData +pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -20678,20 +21842,26 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize + +pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20700,13 +21870,15 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20714,6 +21886,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData +pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -20728,20 +21902,26 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize + +pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20750,13 +21930,15 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20764,6 +21946,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData +pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -20778,20 +21962,26 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize + +pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20800,13 +21990,15 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20814,6 +22006,8 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData +pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -20828,20 +22022,26 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20850,13 +22050,15 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20864,6 +22066,8 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData +pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -20878,20 +22082,26 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize + +pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20900,20 +22110,84 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::VTable for vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ArrayData = vortex_array::arrays::lazy_patched::LazyPatchedData + +pub type vortex_array::arrays::lazy_patched::LazyPatched::Metadata = vortex_array::ProstMetadata + +pub type vortex_array::arrays::lazy_patched::LazyPatched::OperationsVTable = vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ValidityVTable = vortex_array::ValidityVTableFromChild + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: vortex_array::Precision) -> bool + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_hash(array: &Self::ArrayData, state: &mut H, precision: vortex_array::Precision) + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::id(&self) -> vortex_array::ArrayId + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::vtable(_array: &Self::ArrayData) -> &Self + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> + impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData +pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -20928,20 +22202,26 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize + +pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -20950,13 +22230,15 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -20964,6 +22246,8 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -20978,20 +22262,26 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21000,13 +22290,15 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21014,6 +22306,8 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata + pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -21028,20 +22322,26 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21050,13 +22350,15 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21064,6 +22366,8 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData +pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata + pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -21078,20 +22382,26 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize + +pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -21100,13 +22410,15 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -21118,6 +22430,10 @@ impl vortex_array::ValidityChild for vortex_arr pub fn vortex_array::arrays::Extension::validity_child(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::ArrayRef +impl vortex_array::ValidityChild for vortex_array::arrays::lazy_patched::LazyPatched + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::validity_child(array: &vortex_array::arrays::lazy_patched::LazyPatchedData) -> &vortex_array::ArrayRef + impl vortex_array::ValidityChild for vortex_array::arrays::patched::Patched pub fn vortex_array::arrays::patched::Patched::validity_child(array: &vortex_array::arrays::patched::PatchedArray) -> &vortex_array::ArrayRef @@ -21502,6 +22818,8 @@ pub fn vortex_array::Array::from_indices::into_bit_buffer(self) -> vortex_buffer::bit::buf::BitBuffer +pub fn vortex_array::Array::into_parts(self) -> vortex_array::arrays::bool::BoolArrayParts + pub fn vortex_array::Array::new(bits: vortex_buffer::bit::buf::BitBuffer, validity: vortex_array::validity::Validity) -> Self pub fn vortex_array::Array::new_handle(handle: vortex_array::buffer::BufferHandle, offset: usize, len: usize, validity: vortex_array::validity::Validity) -> Self @@ -21522,8 +22840,6 @@ impl vortex_array::Array pub unsafe fn vortex_array::Array::new_unchecked(chunks: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> Self -pub fn vortex_array::Array::rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> vortex_error::VortexResult - pub fn vortex_array::Array::try_new(chunks: alloc::vec::Vec, dtype: vortex_array::dtype::DType) -> vortex_error::VortexResult impl vortex_array::Array @@ -21628,8 +22944,6 @@ impl vortex_array::Array pub fn vortex_array::Array::narrow(&self) -> vortex_error::VortexResult -pub fn vortex_array::Array::reinterpret_cast(&self, ptype: vortex_array::dtype::PType) -> Self - impl vortex_array::Array pub fn vortex_array::Array::patch(self, patches: &vortex_array::patches::Patches, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -21646,14 +22960,14 @@ impl vortex_array::Array pub fn vortex_array::Array::from_fields>(items: &[(N, vortex_array::ArrayRef)]) -> vortex_error::VortexResult +pub fn vortex_array::Array::into_parts(self) -> vortex_array::arrays::struct_::StructArrayParts + pub fn vortex_array::Array::new(names: vortex_array::dtype::FieldNames, fields: impl core::convert::Into>, length: usize, validity: vortex_array::validity::Validity) -> Self pub fn vortex_array::Array::new_fieldless_with_len(len: usize) -> Self pub unsafe fn vortex_array::Array::new_unchecked(fields: impl core::convert::Into>, dtype: vortex_array::dtype::StructFields, length: usize, validity: vortex_array::validity::Validity) -> Self -pub fn vortex_array::Array::project(&self, projection: &[vortex_array::dtype::FieldName]) -> vortex_error::VortexResult - pub fn vortex_array::Array::try_from_iter, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T) -> vortex_error::VortexResult pub fn vortex_array::Array::try_from_iter_with_validity, A: vortex_array::IntoArray, T: core::iter::traits::collect::IntoIterator>(iter: T, validity: vortex_array::validity::Validity) -> vortex_error::VortexResult @@ -21800,9 +23114,9 @@ pub fn vortex_array::Array::dtype(&self) -> &vortex_array::dtype::DType pub fn vortex_array::Array::encoding_id(&self) -> vortex_array::ArrayId -pub fn vortex_array::Array::into_data(self) -> ::ArrayData +pub unsafe fn vortex_array::Array::from_parts_unchecked(new: vortex_array::ArrayParts) -> Self -pub fn vortex_array::Array::into_parts(self) -> vortex_array::ArrayParts +pub fn vortex_array::Array::into_data(self) -> ::ArrayData pub fn vortex_array::Array::is_empty(&self) -> bool @@ -21812,9 +23126,9 @@ pub fn vortex_array::Array::statistics(&self) -> vortex_array::stats::StatsSe pub fn vortex_array::Array::try_from_array_ref(array: vortex_array::ArrayRef) -> core::result::Result -pub fn vortex_array::Array::try_from_parts(new: vortex_array::ArrayParts) -> vortex_error::VortexResult +pub fn vortex_array::Array::try_from_data(data: ::ArrayData) -> vortex_error::VortexResult -pub fn vortex_array::Array::with_stats_set(self, stats: vortex_array::stats::StatsSet) -> Self +pub fn vortex_array::Array::try_from_parts(new: vortex_array::ArrayParts) -> vortex_error::VortexResult impl core::convert::From> for vortex_array::Array @@ -22106,13 +23420,97 @@ impl core::convert::From for vortex_array::ArrayRef pub fn vortex_array::ArrayRef::from(value: vortex_array::Canonical) -> Self +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::bool::BoolData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::chunked::ChunkedData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::constant::ConstantData) -> vortex_array::ArrayRef + impl core::convert::From for vortex_array::ArrayRef pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::datetime::TemporalData) -> Self +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::decimal::DecimalData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::dict::DictData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::extension::ExtensionData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::filter::FilterData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::fixed_size_list::FixedSizeListData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::lazy_patched::LazyPatchedData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::list::ListData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::listview::ListViewData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::masked::MaskedData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::null::NullData) -> vortex_array::ArrayRef + impl core::convert::From for vortex_array::ArrayRef -pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> Self +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::patched::PatchedArray) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::primitive::PrimitiveData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::scalar_fn::ScalarFnData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::shared::SharedData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::slice::SliceData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::struct_::StructData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbin::VarBinData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::varbinview::VarBinViewData) -> vortex_array::ArrayRef + +impl core::convert::From for vortex_array::ArrayRef + +pub fn vortex_array::ArrayRef::from(value: vortex_array::arrays::variant::VariantData) -> vortex_array::ArrayRef impl core::convert::TryFrom<&vortex_array::ArrayRef> for arrow_array::record_batch::RecordBatch @@ -22694,7 +24092,9 @@ pub fn core::option::Option::array_hash(&self, state: pub trait vortex_array::ArrayVTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug +pub type vortex_array::ArrayVTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray + +pub type vortex_array::ArrayVTable::Metadata: core::fmt::Debug pub type vortex_array::ArrayVTable::OperationsVTable: vortex_array::OperationsVTable @@ -22710,20 +24110,26 @@ pub fn vortex_array::ArrayVTable::buffer(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::ArrayVTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::ArrayVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::ArrayVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::ArrayVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::ArrayVTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::ArrayVTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::ArrayVTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::ArrayVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::ArrayVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::ArrayVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::ArrayVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::ArrayVTable::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::ArrayVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::ArrayVTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::ArrayVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22732,13 +24138,15 @@ pub fn vortex_array::ArrayVTable::reduce(array: vortex_array::ArrayView<'_, Self pub fn vortex_array::ArrayVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::ArrayVTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::ArrayVTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::ArrayVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::ArrayVTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::ArrayVTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::ArrayVTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::ArrayVTable::vtable(array: &Self::ArrayData) -> &Self pub fn vortex_array::ArrayVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22746,6 +24154,8 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData +pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -22760,20 +24170,26 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize + +pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22782,13 +24198,15 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22796,6 +24214,8 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData +pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -22810,20 +24230,26 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize + +pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22832,13 +24258,15 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22846,6 +24274,8 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData +pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar + pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -22860,20 +24290,26 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize + +pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22882,13 +24318,15 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22896,6 +24334,8 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData +pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -22910,20 +24350,26 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize + +pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22932,13 +24378,15 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22946,6 +24394,8 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -22960,20 +24410,26 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize + +pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -22982,13 +24438,15 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -22996,6 +24454,8 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData +pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata + pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -23010,20 +24470,26 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize + +pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23032,13 +24498,15 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23046,6 +24514,8 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData +pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -23060,20 +24530,26 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize + +pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23082,13 +24558,15 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23096,6 +24574,8 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData +pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -23110,20 +24590,26 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize + +pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23132,13 +24618,15 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23146,6 +24634,8 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData +pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -23160,20 +24650,26 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize + +pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23182,13 +24678,15 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23196,6 +24694,8 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData +pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -23210,20 +24710,26 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize + +pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23232,13 +24738,15 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23246,6 +24754,8 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData +pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -23260,20 +24770,26 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize + +pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23282,13 +24798,15 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23296,6 +24814,8 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData +pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -23310,20 +24830,26 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize + +pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23332,13 +24858,15 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23346,6 +24874,8 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData +pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -23360,20 +24890,26 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize + +pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23382,13 +24918,15 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23396,6 +24934,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData +pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -23410,20 +24950,26 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize + +pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23432,13 +24978,15 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23446,6 +24994,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData +pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -23460,20 +25010,26 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize + +pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23482,13 +25038,15 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23496,6 +25054,8 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData +pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -23510,20 +25070,26 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23532,13 +25098,15 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23546,6 +25114,8 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData +pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -23560,20 +25130,26 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize + +pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23582,20 +25158,84 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::VTable for vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ArrayData = vortex_array::arrays::lazy_patched::LazyPatchedData + +pub type vortex_array::arrays::lazy_patched::LazyPatched::Metadata = vortex_array::ProstMetadata + +pub type vortex_array::arrays::lazy_patched::LazyPatched::OperationsVTable = vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ValidityVTable = vortex_array::ValidityVTableFromChild + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: vortex_array::Precision) -> bool + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_hash(array: &Self::ArrayData, state: &mut H, precision: vortex_array::Precision) + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::id(&self) -> vortex_array::ArrayId + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::vtable(_array: &Self::ArrayData) -> &Self + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> + impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData +pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -23610,20 +25250,26 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize + +pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23632,13 +25278,15 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23646,6 +25294,8 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -23660,20 +25310,26 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23682,13 +25338,15 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23696,6 +25354,8 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata + pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -23710,20 +25370,26 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23732,13 +25398,15 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23746,6 +25414,8 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData +pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata + pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -23760,20 +25430,26 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize + +pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -23782,13 +25458,15 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -23964,14 +25642,98 @@ impl vortex_array::IntoArray for vortex_array::Columnar pub fn vortex_array::Columnar::into_array(self) -> vortex_array::ArrayRef +impl vortex_array::IntoArray for vortex_array::arrays::bool::BoolData + +pub fn vortex_array::arrays::bool::BoolData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::chunked::ChunkedData + +pub fn vortex_array::arrays::chunked::ChunkedData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::constant::ConstantData + +pub fn vortex_array::arrays::constant::ConstantData::into_array(self) -> vortex_array::ArrayRef + impl vortex_array::IntoArray for vortex_array::arrays::datetime::TemporalData pub fn vortex_array::arrays::datetime::TemporalData::into_array(self) -> vortex_array::ArrayRef +impl vortex_array::IntoArray for vortex_array::arrays::decimal::DecimalData + +pub fn vortex_array::arrays::decimal::DecimalData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::dict::DictData + +pub fn vortex_array::arrays::dict::DictData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::extension::ExtensionData + +pub fn vortex_array::arrays::extension::ExtensionData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::filter::FilterData + +pub fn vortex_array::arrays::filter::FilterData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::fixed_size_list::FixedSizeListData + +pub fn vortex_array::arrays::fixed_size_list::FixedSizeListData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::lazy_patched::LazyPatchedData + +pub fn vortex_array::arrays::lazy_patched::LazyPatchedData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::list::ListData + +pub fn vortex_array::arrays::list::ListData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::listview::ListViewData + +pub fn vortex_array::arrays::listview::ListViewData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::masked::MaskedData + +pub fn vortex_array::arrays::masked::MaskedData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::null::NullData + +pub fn vortex_array::arrays::null::NullData::into_array(self) -> vortex_array::ArrayRef + impl vortex_array::IntoArray for vortex_array::arrays::patched::PatchedArray pub fn vortex_array::arrays::patched::PatchedArray::into_array(self) -> vortex_array::ArrayRef +impl vortex_array::IntoArray for vortex_array::arrays::primitive::PrimitiveData + +pub fn vortex_array::arrays::primitive::PrimitiveData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::scalar_fn::ScalarFnData + +pub fn vortex_array::arrays::scalar_fn::ScalarFnData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::shared::SharedData + +pub fn vortex_array::arrays::shared::SharedData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::slice::SliceData + +pub fn vortex_array::arrays::slice::SliceData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::struct_::StructData + +pub fn vortex_array::arrays::struct_::StructData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::varbin::VarBinData + +pub fn vortex_array::arrays::varbin::VarBinData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::varbinview::VarBinViewData + +pub fn vortex_array::arrays::varbinview::VarBinViewData::into_array(self) -> vortex_array::ArrayRef + +impl vortex_array::IntoArray for vortex_array::arrays::variant::VariantData + +pub fn vortex_array::arrays::variant::VariantData::into_array(self) -> vortex_array::ArrayRef + impl vortex_array::IntoArray for vortex_buffer::bit::buf::BitBuffer pub fn vortex_buffer::bit::buf::BitBuffer::into_array(self) -> vortex_array::ArrayRef @@ -24080,6 +25842,10 @@ impl vortex_array::OperationsVTable for vortex pub fn vortex_array::arrays::dict::Dict::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::dict::Dict>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +impl vortex_array::OperationsVTable for vortex_array::arrays::lazy_patched::LazyPatched + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::scalar_at(array: vortex_array::ArrayView<'_, vortex_array::arrays::lazy_patched::LazyPatched>, index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + impl vortex_array::OperationsVTable for vortex_array::arrays::null::Null pub fn vortex_array::arrays::null::Null::scalar_at(_array: vortex_array::ArrayView<'_, vortex_array::arrays::null::Null>, _index: usize, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -24158,7 +25924,9 @@ pub fn vortex_array::ArrayRef::to_varbinview(&self) -> vortex_array::arrays::Var pub trait vortex_array::VTable: 'static + core::clone::Clone + core::marker::Sized + core::marker::Send + core::marker::Sync + core::fmt::Debug -pub type vortex_array::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug +pub type vortex_array::VTable::ArrayData: 'static + core::marker::Send + core::marker::Sync + core::clone::Clone + core::fmt::Debug + vortex_array::IntoArray + +pub type vortex_array::VTable::Metadata: core::fmt::Debug pub type vortex_array::VTable::OperationsVTable: vortex_array::OperationsVTable @@ -24174,20 +25942,26 @@ pub fn vortex_array::VTable::buffer(array: vortex_array::ArrayView<'_, Self>, id pub fn vortex_array::VTable::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::VTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::VTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::VTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::VTable::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::VTable::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::VTable::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::VTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::VTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::VTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::VTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::VTable::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::VTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::VTable::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::VTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24196,13 +25970,15 @@ pub fn vortex_array::VTable::reduce(array: vortex_array::ArrayView<'_, Self>) -> pub fn vortex_array::VTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::VTable::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::VTable::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::VTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::VTable::slots<'a>(array: vortex_array::ArrayView<'a, Self>) -> &'a [core::option::Option] -pub fn vortex_array::VTable::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::VTable::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::VTable::vtable(array: &Self::ArrayData) -> &Self pub fn vortex_array::VTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24210,6 +25986,8 @@ impl vortex_array::VTable for vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ArrayData = vortex_array::arrays::bool::BoolData +pub type vortex_array::arrays::Bool::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Bool::OperationsVTable = vortex_array::arrays::Bool pub type vortex_array::arrays::Bool::ValidityVTable = vortex_array::arrays::Bool @@ -24224,20 +26002,26 @@ pub fn vortex_array::arrays::Bool::buffer(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Bool::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Bool::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Bool::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Bool::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Bool::dtype(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Bool::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Bool::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Bool::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Bool::len(array: &vortex_array::arrays::bool::BoolData) -> usize + +pub fn vortex_array::arrays::Bool::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Bool::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Bool::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24246,13 +26030,15 @@ pub fn vortex_array::arrays::Bool::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::Bool::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Bool::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Bool::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Bool::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Bool::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Bool::validate(&self, data: &vortex_array::arrays::bool::BoolData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Bool::stats(array: &vortex_array::arrays::bool::BoolData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Bool::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Bool::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24260,6 +26046,8 @@ impl vortex_array::VTable for vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ArrayData = vortex_array::arrays::chunked::ChunkedData +pub type vortex_array::arrays::Chunked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Chunked::OperationsVTable = vortex_array::arrays::Chunked pub type vortex_array::arrays::Chunked::ValidityVTable = vortex_array::arrays::Chunked @@ -24274,20 +26062,26 @@ pub fn vortex_array::arrays::Chunked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Chunked::build(dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Chunked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Chunked::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Chunked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Chunked::dtype(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Chunked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Chunked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Chunked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Chunked::len(array: &vortex_array::arrays::chunked::ChunkedData) -> usize + +pub fn vortex_array::arrays::Chunked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Chunked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Chunked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24296,13 +26090,15 @@ pub fn vortex_array::arrays::Chunked::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Chunked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Chunked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Chunked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Chunked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Chunked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Chunked::validate(&self, data: &vortex_array::arrays::chunked::ChunkedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Chunked::stats(array: &vortex_array::arrays::chunked::ChunkedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Chunked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Chunked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24310,6 +26106,8 @@ impl vortex_array::VTable for vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ArrayData = vortex_array::arrays::constant::ConstantData +pub type vortex_array::arrays::Constant::Metadata = vortex_array::scalar::Scalar + pub type vortex_array::arrays::Constant::OperationsVTable = vortex_array::arrays::Constant pub type vortex_array::arrays::Constant::ValidityVTable = vortex_array::arrays::Constant @@ -24324,20 +26122,26 @@ pub fn vortex_array::arrays::Constant::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Constant::build(_dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Constant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Constant::deserialize(&self, dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Constant::deserialize(_bytes: &[u8], dtype: &vortex_array::dtype::DType, _len: usize, buffers: &[vortex_array::buffer::BufferHandle], session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Constant::dtype(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Constant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Constant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Constant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Constant::len(array: &vortex_array::arrays::constant::ConstantData) -> usize + +pub fn vortex_array::arrays::Constant::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Constant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Constant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24346,13 +26150,15 @@ pub fn vortex_array::arrays::Constant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Constant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Constant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Constant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Constant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Constant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Constant::validate(&self, data: &vortex_array::arrays::constant::ConstantData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Constant::stats(array: &vortex_array::arrays::constant::ConstantData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Constant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Constant::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24360,6 +26166,8 @@ impl vortex_array::VTable for vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ArrayData = vortex_array::arrays::decimal::DecimalData +pub type vortex_array::arrays::Decimal::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::Decimal::OperationsVTable = vortex_array::arrays::Decimal pub type vortex_array::arrays::Decimal::ValidityVTable = vortex_array::arrays::Decimal @@ -24374,20 +26182,26 @@ pub fn vortex_array::arrays::Decimal::buffer(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Decimal::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Decimal::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Decimal::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Decimal::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Decimal::dtype(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Decimal::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Decimal::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Decimal::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Decimal::len(array: &vortex_array::arrays::decimal::DecimalData) -> usize + +pub fn vortex_array::arrays::Decimal::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Decimal::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Decimal::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24396,13 +26210,15 @@ pub fn vortex_array::arrays::Decimal::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Decimal::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Decimal::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Decimal::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Decimal::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Decimal::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Decimal::validate(&self, data: &vortex_array::arrays::decimal::DecimalData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Decimal::stats(array: &vortex_array::arrays::decimal::DecimalData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Decimal::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Decimal::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24410,6 +26226,8 @@ impl vortex_array::VTable for vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ArrayData = vortex_array::arrays::extension::ExtensionData +pub type vortex_array::arrays::Extension::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Extension::OperationsVTable = vortex_array::arrays::Extension pub type vortex_array::arrays::Extension::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -24424,20 +26242,26 @@ pub fn vortex_array::arrays::Extension::buffer(_array: vortex_array::ArrayView<' pub fn vortex_array::arrays::Extension::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Extension::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Extension::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Extension::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Extension::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Extension::dtype(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Extension::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Extension::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Extension::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Extension::len(array: &vortex_array::arrays::extension::ExtensionData) -> usize + +pub fn vortex_array::arrays::Extension::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Extension::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Extension::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24446,13 +26270,15 @@ pub fn vortex_array::arrays::Extension::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Extension::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Extension::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Extension::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Extension::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Extension::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Extension::validate(&self, data: &vortex_array::arrays::extension::ExtensionData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Extension::stats(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Extension::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Extension::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24460,6 +26286,8 @@ impl vortex_array::VTable for vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ArrayData = vortex_array::arrays::filter::FilterData +pub type vortex_array::arrays::Filter::Metadata = vortex_array::arrays::filter::vtable::FilterMetadata + pub type vortex_array::arrays::Filter::OperationsVTable = vortex_array::arrays::Filter pub type vortex_array::arrays::Filter::ValidityVTable = vortex_array::arrays::Filter @@ -24474,20 +26302,26 @@ pub fn vortex_array::arrays::Filter::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Filter::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Filter::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::filter::vtable::FilterMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Filter::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Filter::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Filter::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Filter::dtype(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Filter::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Filter::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Filter::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Filter::len(array: &vortex_array::arrays::filter::FilterData) -> usize + +pub fn vortex_array::arrays::Filter::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Filter::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Filter::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24496,13 +26330,15 @@ pub fn vortex_array::arrays::Filter::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Filter::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Filter::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Filter::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Filter::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Filter::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Filter::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Filter::stats(array: &vortex_array::arrays::filter::FilterData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Filter::vtable(_array: &vortex_array::arrays::filter::FilterData) -> &Self pub fn vortex_array::arrays::Filter::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24510,6 +26346,8 @@ impl vortex_array::VTable for vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ArrayData = vortex_array::arrays::fixed_size_list::FixedSizeListData +pub type vortex_array::arrays::FixedSizeList::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::FixedSizeList::OperationsVTable = vortex_array::arrays::FixedSizeList pub type vortex_array::arrays::FixedSizeList::ValidityVTable = vortex_array::arrays::FixedSizeList @@ -24524,20 +26362,26 @@ pub fn vortex_array::arrays::FixedSizeList::buffer(_array: vortex_array::ArrayVi pub fn vortex_array::arrays::FixedSizeList::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::FixedSizeList::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::FixedSizeList::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::FixedSizeList::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::FixedSizeList::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::FixedSizeList::dtype(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::FixedSizeList::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::FixedSizeList::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::FixedSizeList::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::FixedSizeList::len(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> usize + +pub fn vortex_array::arrays::FixedSizeList::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::FixedSizeList::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::FixedSizeList::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24546,13 +26390,15 @@ pub fn vortex_array::arrays::FixedSizeList::reduce(array: vortex_array::ArrayVie pub fn vortex_array::arrays::FixedSizeList::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::FixedSizeList::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::FixedSizeList::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::FixedSizeList::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::FixedSizeList::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::FixedSizeList::validate(&self, data: &vortex_array::arrays::fixed_size_list::FixedSizeListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::FixedSizeList::stats(array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::FixedSizeList::vtable(_array: &vortex_array::arrays::fixed_size_list::FixedSizeListData) -> &Self pub fn vortex_array::arrays::FixedSizeList::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24560,6 +26406,8 @@ impl vortex_array::VTable for vortex_array::arrays::List pub type vortex_array::arrays::List::ArrayData = vortex_array::arrays::list::ListData +pub type vortex_array::arrays::List::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::List::OperationsVTable = vortex_array::arrays::List pub type vortex_array::arrays::List::ValidityVTable = vortex_array::arrays::List @@ -24574,20 +26422,26 @@ pub fn vortex_array::arrays::List::buffer(_array: vortex_array::ArrayView<'_, Se pub fn vortex_array::arrays::List::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::List::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::List::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::List::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::List::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::List::dtype(array: &vortex_array::arrays::list::ListData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::List::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::List::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::List::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::List::len(array: &vortex_array::arrays::list::ListData) -> usize + +pub fn vortex_array::arrays::List::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::List::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::List::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24596,13 +26450,15 @@ pub fn vortex_array::arrays::List::reduce(array: vortex_array::ArrayView<'_, Sel pub fn vortex_array::arrays::List::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::List::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::List::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::List::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::List::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::List::validate(&self, data: &vortex_array::arrays::list::ListData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::List::stats(array: &vortex_array::arrays::list::ListData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::List::vtable(_array: &vortex_array::arrays::list::ListData) -> &Self pub fn vortex_array::arrays::List::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24610,6 +26466,8 @@ impl vortex_array::VTable for vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ArrayData = vortex_array::arrays::listview::ListViewData +pub type vortex_array::arrays::ListView::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::ListView::OperationsVTable = vortex_array::arrays::ListView pub type vortex_array::arrays::ListView::ValidityVTable = vortex_array::arrays::ListView @@ -24624,20 +26482,26 @@ pub fn vortex_array::arrays::ListView::buffer(_array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::ListView::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::ListView::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::ListView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::ListView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::ListView::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::ListView::dtype(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::ListView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::ListView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::ListView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::ListView::len(array: &vortex_array::arrays::listview::ListViewData) -> usize + +pub fn vortex_array::arrays::ListView::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::ListView::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::ListView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24646,13 +26510,15 @@ pub fn vortex_array::arrays::ListView::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::ListView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::ListView::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::ListView::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::ListView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::ListView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::ListView::validate(&self, data: &vortex_array::arrays::listview::ListViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::ListView::stats(array: &vortex_array::arrays::listview::ListViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::ListView::vtable(_array: &vortex_array::arrays::listview::ListViewData) -> &Self pub fn vortex_array::arrays::ListView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24660,6 +26526,8 @@ impl vortex_array::VTable for vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ArrayData = vortex_array::arrays::masked::MaskedData +pub type vortex_array::arrays::Masked::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Masked::OperationsVTable = vortex_array::arrays::Masked pub type vortex_array::arrays::Masked::ValidityVTable = vortex_array::arrays::Masked @@ -24674,20 +26542,26 @@ pub fn vortex_array::arrays::Masked::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Masked::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Masked::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Masked::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Masked::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Masked::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Masked::dtype(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Masked::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Masked::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Masked::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Masked::len(array: &vortex_array::arrays::masked::MaskedData) -> usize + +pub fn vortex_array::arrays::Masked::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Masked::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Masked::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24696,13 +26570,15 @@ pub fn vortex_array::arrays::Masked::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Masked::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Masked::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Masked::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Masked::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Masked::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Masked::validate(&self, data: &vortex_array::arrays::masked::MaskedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Masked::stats(array: &vortex_array::arrays::masked::MaskedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Masked::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Masked::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24710,6 +26586,8 @@ impl vortex_array::VTable for vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ArrayData = vortex_array::arrays::primitive::PrimitiveData +pub type vortex_array::arrays::Primitive::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Primitive::OperationsVTable = vortex_array::arrays::Primitive pub type vortex_array::arrays::Primitive::ValidityVTable = vortex_array::arrays::Primitive @@ -24724,20 +26602,26 @@ pub fn vortex_array::arrays::Primitive::buffer(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Primitive::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Primitive::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Primitive::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Primitive::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Primitive::dtype(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Primitive::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Primitive::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Primitive::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Primitive::len(array: &vortex_array::arrays::primitive::PrimitiveData) -> usize + +pub fn vortex_array::arrays::Primitive::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Primitive::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Primitive::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24746,13 +26630,15 @@ pub fn vortex_array::arrays::Primitive::reduce(array: vortex_array::ArrayView<'_ pub fn vortex_array::arrays::Primitive::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Primitive::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Primitive::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Primitive::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Primitive::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Primitive::validate(&self, data: &vortex_array::arrays::primitive::PrimitiveData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Primitive::stats(array: &vortex_array::arrays::primitive::PrimitiveData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Primitive::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Primitive::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24760,6 +26646,8 @@ impl vortex_array::VTable for vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ArrayData = vortex_array::arrays::shared::SharedData +pub type vortex_array::arrays::Shared::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Shared::OperationsVTable = vortex_array::arrays::Shared pub type vortex_array::arrays::Shared::ValidityVTable = vortex_array::arrays::Shared @@ -24774,20 +26662,26 @@ pub fn vortex_array::arrays::Shared::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Shared::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Shared::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Shared::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Shared::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Shared::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Shared::dtype(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Shared::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Shared::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Shared::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Shared::len(array: &vortex_array::arrays::shared::SharedData) -> usize + +pub fn vortex_array::arrays::Shared::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Shared::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Shared::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24796,13 +26690,15 @@ pub fn vortex_array::arrays::Shared::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Shared::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Shared::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Shared::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Shared::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Shared::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Shared::validate(&self, data: &vortex_array::arrays::shared::SharedData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Shared::stats(array: &vortex_array::arrays::shared::SharedData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Shared::vtable(_array: &vortex_array::arrays::shared::SharedData) -> &Self pub fn vortex_array::arrays::Shared::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24810,6 +26706,8 @@ impl vortex_array::VTable for vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ArrayData = vortex_array::arrays::struct_::StructData +pub type vortex_array::arrays::Struct::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Struct::OperationsVTable = vortex_array::arrays::Struct pub type vortex_array::arrays::Struct::ValidityVTable = vortex_array::arrays::Struct @@ -24824,20 +26722,26 @@ pub fn vortex_array::arrays::Struct::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Struct::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Struct::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Struct::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Struct::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Struct::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Struct::dtype(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Struct::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Struct::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Struct::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Struct::len(array: &vortex_array::arrays::struct_::StructData) -> usize + +pub fn vortex_array::arrays::Struct::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Struct::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Struct::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24846,13 +26750,15 @@ pub fn vortex_array::arrays::Struct::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::Struct::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Struct::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Struct::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Struct::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Struct::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Struct::validate(&self, data: &vortex_array::arrays::struct_::StructData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Struct::stats(array: &vortex_array::arrays::struct_::StructData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Struct::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Struct::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24860,6 +26766,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ArrayData = vortex_array::arrays::varbin::VarBinData +pub type vortex_array::arrays::VarBin::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::VarBin::OperationsVTable = vortex_array::arrays::VarBin pub type vortex_array::arrays::VarBin::ValidityVTable = vortex_array::arrays::VarBin @@ -24874,20 +26782,26 @@ pub fn vortex_array::arrays::VarBin::buffer(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBin::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBin::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBin::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBin::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBin::dtype(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBin::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBin::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBin::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBin::len(array: &vortex_array::arrays::varbin::VarBinData) -> usize + +pub fn vortex_array::arrays::VarBin::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBin::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBin::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24896,13 +26810,15 @@ pub fn vortex_array::arrays::VarBin::reduce(array: vortex_array::ArrayView<'_, S pub fn vortex_array::arrays::VarBin::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBin::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBin::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBin::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBin::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBin::validate(&self, data: &vortex_array::arrays::varbin::VarBinData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBin::stats(array: &vortex_array::arrays::varbin::VarBinData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBin::vtable(_array: &vortex_array::arrays::varbin::VarBinData) -> &Self pub fn vortex_array::arrays::VarBin::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24910,6 +26826,8 @@ impl vortex_array::VTable for vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ArrayData = vortex_array::arrays::varbinview::VarBinViewData +pub type vortex_array::arrays::VarBinView::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::VarBinView::OperationsVTable = vortex_array::arrays::VarBinView pub type vortex_array::arrays::VarBinView::ValidityVTable = vortex_array::arrays::VarBinView @@ -24924,20 +26842,26 @@ pub fn vortex_array::arrays::VarBinView::buffer(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::buffer_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::VarBinView::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::VarBinView::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::VarBinView::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::VarBinView::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::VarBinView::dtype(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::VarBinView::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::VarBinView::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::VarBinView::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::VarBinView::len(array: &vortex_array::arrays::varbinview::VarBinViewData) -> usize + +pub fn vortex_array::arrays::VarBinView::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::VarBinView::nbuffers(array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::VarBinView::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24946,13 +26870,15 @@ pub fn vortex_array::arrays::VarBinView::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::VarBinView::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::VarBinView::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::VarBinView::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::VarBinView::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::VarBinView::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::VarBinView::validate(&self, data: &vortex_array::arrays::varbinview::VarBinViewData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::VarBinView::stats(array: &vortex_array::arrays::varbinview::VarBinViewData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::VarBinView::vtable(_array: &vortex_array::arrays::varbinview::VarBinViewData) -> &Self pub fn vortex_array::arrays::VarBinView::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -24960,6 +26886,8 @@ impl vortex_array::VTable for vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ArrayData = vortex_array::arrays::variant::VariantData +pub type vortex_array::arrays::Variant::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::Variant::OperationsVTable = vortex_array::arrays::Variant pub type vortex_array::arrays::Variant::ValidityVTable = vortex_array::arrays::Variant @@ -24974,20 +26902,26 @@ pub fn vortex_array::arrays::Variant::buffer(_array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::Variant::build(dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::Variant::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::Variant::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::Variant::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::Variant::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::Variant::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::Variant::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::Variant::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::Variant::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::Variant::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::Variant::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::Variant::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -24996,13 +26930,15 @@ pub fn vortex_array::arrays::Variant::reduce(array: vortex_array::ArrayView<'_, pub fn vortex_array::arrays::Variant::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::Variant::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::Variant::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::Variant::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::Variant::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::Variant::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::Variant::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::Variant::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::Variant::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25010,6 +26946,8 @@ impl vortex_array::VTable for vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ArrayData = vortex_array::arrays::dict::DictData +pub type vortex_array::arrays::dict::Dict::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::dict::Dict::OperationsVTable = vortex_array::arrays::dict::Dict pub type vortex_array::arrays::dict::Dict::ValidityVTable = vortex_array::arrays::dict::Dict @@ -25024,20 +26962,26 @@ pub fn vortex_array::arrays::dict::Dict::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::dict::Dict::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::dict::Dict::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::dict::Dict::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::dict::Dict::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::dict::Dict::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::dict::Dict::dtype(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::dict::Dict::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::dict::Dict::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::dict::Dict::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::dict::Dict::len(array: &vortex_array::arrays::dict::DictData) -> usize + +pub fn vortex_array::arrays::dict::Dict::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::dict::Dict::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::dict::Dict::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25046,20 +26990,84 @@ pub fn vortex_array::arrays::dict::Dict::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::dict::Dict::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::dict::Dict::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::dict::Dict::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::dict::Dict::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::dict::Dict::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::dict::Dict::validate(&self, data: &vortex_array::arrays::dict::DictData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::dict::Dict::stats(array: &vortex_array::arrays::dict::DictData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::dict::Dict::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::dict::Dict::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> +impl vortex_array::VTable for vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ArrayData = vortex_array::arrays::lazy_patched::LazyPatchedData + +pub type vortex_array::arrays::lazy_patched::LazyPatched::Metadata = vortex_array::ProstMetadata + +pub type vortex_array::arrays::lazy_patched::LazyPatched::OperationsVTable = vortex_array::arrays::lazy_patched::LazyPatched + +pub type vortex_array::arrays::lazy_patched::LazyPatched::ValidityVTable = vortex_array::ValidityVTableFromChild + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::append_to_builder(array: vortex_array::ArrayView<'_, Self>, builder: &mut dyn vortex_array::builders::ArrayBuilder, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: vortex_array::Precision) -> bool + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::array_hash(array: &Self::ArrayData, state: &mut H, precision: vortex_array::Precision) + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> vortex_array::buffer::BufferHandle + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::id(&self) -> vortex_array::ArrayId + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::vtable(_array: &Self::ArrayData) -> &Self + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> + impl vortex_array::VTable for vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ArrayData = vortex_array::arrays::null::NullData +pub type vortex_array::arrays::null::Null::Metadata = vortex_array::EmptyMetadata + pub type vortex_array::arrays::null::Null::OperationsVTable = vortex_array::arrays::null::Null pub type vortex_array::arrays::null::Null::ValidityVTable = vortex_array::arrays::null::Null @@ -25074,20 +27082,26 @@ pub fn vortex_array::arrays::null::Null::buffer(_array: vortex_array::ArrayView< pub fn vortex_array::arrays::null::Null::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::null::Null::build(_dtype: &vortex_array::dtype::DType, len: usize, _metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::null::Null::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::null::Null::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::null::Null::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::null::Null::dtype(_array: &vortex_array::arrays::null::NullData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::null::Null::execute(array: vortex_array::Array, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::null::Null::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::null::Null::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::null::Null::len(array: &vortex_array::arrays::null::NullData) -> usize + +pub fn vortex_array::arrays::null::Null::metadata(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::null::Null::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::null::Null::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25096,13 +27110,15 @@ pub fn vortex_array::arrays::null::Null::reduce(array: vortex_array::ArrayView<' pub fn vortex_array::arrays::null::Null::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::null::Null::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::null::Null::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::null::Null::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::null::Null::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::null::Null::validate(&self, _data: &vortex_array::arrays::null::NullData, dtype: &vortex_array::dtype::DType, _len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::null::Null::stats(array: &vortex_array::arrays::null::NullData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::null::Null::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::null::Null::with_slots(_array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25110,6 +27126,8 @@ impl vortex_array::VTable for vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ArrayData = vortex_array::arrays::patched::PatchedArray +pub type vortex_array::arrays::patched::Patched::Metadata = vortex_array::ProstMetadata + pub type vortex_array::arrays::patched::Patched::OperationsVTable = vortex_array::arrays::patched::Patched pub type vortex_array::arrays::patched::Patched::ValidityVTable = vortex_array::ValidityVTableFromChild @@ -25124,20 +27142,26 @@ pub fn vortex_array::arrays::patched::Patched::buffer(_array: vortex_array::Arra pub fn vortex_array::arrays::patched::Patched::buffer_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> core::option::Option +pub fn vortex_array::arrays::patched::Patched::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &Self::Metadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::patched::Patched::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::patched::Patched::deserialize(&self, dtype: &vortex_array::dtype::DType, len: usize, metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::patched::Patched::deserialize(bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::patched::Patched::dtype(array: &Self::ArrayData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::patched::Patched::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::patched::Patched::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::patched::Patched::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::patched::Patched::len(array: &Self::ArrayData) -> usize + +pub fn vortex_array::arrays::patched::Patched::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::patched::Patched::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::patched::Patched::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25146,13 +27170,15 @@ pub fn vortex_array::arrays::patched::Patched::reduce(array: vortex_array::Array pub fn vortex_array::arrays::patched::Patched::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::patched::Patched::serialize(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::patched::Patched::serialize(metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::patched::Patched::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::patched::Patched::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::patched::Patched::validate(&self, data: &vortex_array::arrays::patched::PatchedArray, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::patched::Patched::stats(array: &Self::ArrayData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::patched::Patched::vtable(_array: &Self::ArrayData) -> &Self pub fn vortex_array::arrays::patched::Patched::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25160,6 +27186,8 @@ impl vortex_array::VTable for vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ArrayData = vortex_array::arrays::scalar_fn::ScalarFnData +pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::Metadata = vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata + pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::OperationsVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable pub type vortex_array::arrays::scalar_fn::ScalarFnVTable::ValidityVTable = vortex_array::arrays::scalar_fn::ScalarFnVTable @@ -25174,20 +27202,26 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer(_array: vortex_ar pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::scalar_fn::metadata::ScalarFnMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::dtype(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::len(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> usize + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25196,13 +27230,15 @@ pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce(array: vortex_arr pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slot_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::validate(&self, data: &vortex_array::arrays::scalar_fn::ScalarFnData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::stats(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::vtable(array: &vortex_array::arrays::scalar_fn::ScalarFnData) -> &Self pub fn vortex_array::arrays::scalar_fn::ScalarFnVTable::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25210,6 +27246,8 @@ impl vortex_array::VTable for vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ArrayData = vortex_array::arrays::slice::SliceData +pub type vortex_array::arrays::slice::Slice::Metadata = vortex_array::arrays::slice::SliceMetadata + pub type vortex_array::arrays::slice::Slice::OperationsVTable = vortex_array::arrays::slice::Slice pub type vortex_array::arrays::slice::Slice::ValidityVTable = vortex_array::arrays::slice::Slice @@ -25224,20 +27262,26 @@ pub fn vortex_array::arrays::slice::Slice::buffer(_array: vortex_array::ArrayVie pub fn vortex_array::arrays::slice::Slice::buffer_name(_array: vortex_array::ArrayView<'_, Self>, _idx: usize) -> core::option::Option +pub fn vortex_array::arrays::slice::Slice::build(dtype: &vortex_array::dtype::DType, len: usize, metadata: &vortex_array::arrays::slice::SliceMetadata, _buffers: &[vortex_array::buffer::BufferHandle], children: &dyn vortex_array::serde::ArrayChildren) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::child(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> vortex_array::ArrayRef pub fn vortex_array::arrays::slice::Slice::child_name(array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String -pub fn vortex_array::arrays::slice::Slice::deserialize(&self, _dtype: &vortex_array::dtype::DType, _len: usize, _metadata: &[u8], _buffers: &[vortex_array::buffer::BufferHandle], _children: &dyn vortex_array::serde::ArrayChildren, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult +pub fn vortex_array::arrays::slice::Slice::deserialize(_bytes: &[u8], _dtype: &vortex_array::dtype::DType, _len: usize, _buffers: &[vortex_array::buffer::BufferHandle], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult + +pub fn vortex_array::arrays::slice::Slice::dtype(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::dtype::DType pub fn vortex_array::arrays::slice::Slice::execute(array: vortex_array::Array, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrays::slice::Slice::execute_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::fmt_metadata(array: vortex_array::ArrayView<'_, Self>, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result - pub fn vortex_array::arrays::slice::Slice::id(&self) -> vortex_array::ArrayId +pub fn vortex_array::arrays::slice::Slice::len(array: &vortex_array::arrays::slice::SliceData) -> usize + +pub fn vortex_array::arrays::slice::Slice::metadata(array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult + pub fn vortex_array::arrays::slice::Slice::nbuffers(_array: vortex_array::ArrayView<'_, Self>) -> usize pub fn vortex_array::arrays::slice::Slice::nchildren(array: vortex_array::ArrayView<'_, Self>) -> usize @@ -25246,13 +27290,15 @@ pub fn vortex_array::arrays::slice::Slice::reduce(array: vortex_array::ArrayView pub fn vortex_array::arrays::slice::Slice::reduce_parent(array: vortex_array::ArrayView<'_, Self>, parent: &vortex_array::ArrayRef, child_idx: usize) -> vortex_error::VortexResult> -pub fn vortex_array::arrays::slice::Slice::serialize(_array: vortex_array::ArrayView<'_, Self>) -> vortex_error::VortexResult>> +pub fn vortex_array::arrays::slice::Slice::serialize(_metadata: Self::Metadata) -> vortex_error::VortexResult>> pub fn vortex_array::arrays::slice::Slice::slot_name(_array: vortex_array::ArrayView<'_, Self>, idx: usize) -> alloc::string::String pub fn vortex_array::arrays::slice::Slice::slots(array: vortex_array::ArrayView<'_, Self>) -> &[core::option::Option] -pub fn vortex_array::arrays::slice::Slice::validate(&self, data: &Self::ArrayData, dtype: &vortex_array::dtype::DType, len: usize) -> vortex_error::VortexResult<()> +pub fn vortex_array::arrays::slice::Slice::stats(array: &vortex_array::arrays::slice::SliceData) -> &vortex_array::stats::ArrayStats + +pub fn vortex_array::arrays::slice::Slice::vtable(_array: &vortex_array::arrays::slice::SliceData) -> &Self pub fn vortex_array::arrays::slice::Slice::with_slots(array: &mut Self::ArrayData, slots: alloc::vec::Vec>) -> vortex_error::VortexResult<()> @@ -25264,6 +27310,10 @@ impl vortex_array::ValidityChild for vortex_arr pub fn vortex_array::arrays::Extension::validity_child(array: &vortex_array::arrays::extension::ExtensionData) -> &vortex_array::ArrayRef +impl vortex_array::ValidityChild for vortex_array::arrays::lazy_patched::LazyPatched + +pub fn vortex_array::arrays::lazy_patched::LazyPatched::validity_child(array: &vortex_array::arrays::lazy_patched::LazyPatchedData) -> &vortex_array::ArrayRef + impl vortex_array::ValidityChild for vortex_array::arrays::patched::Patched pub fn vortex_array::arrays::patched::Patched::validity_child(array: &vortex_array::arrays::patched::PatchedArray) -> &vortex_array::ArrayRef diff --git a/vortex-array/src/array/mod.rs b/vortex-array/src/array/mod.rs index 0dbf065e6a1..842f759ace7 100644 --- a/vortex-array/src/array/mod.rs +++ b/vortex-array/src/array/mod.rs @@ -304,12 +304,15 @@ impl DynArray for ArrayInner { fn metadata(&self, this: &ArrayRef) -> VortexResult>> { let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - V::serialize(view) + V::serialize(V::metadata(view)?) } fn metadata_fmt(&self, this: &ArrayRef, f: &mut Formatter<'_>) -> std::fmt::Result { let view = unsafe { ArrayView::new_unchecked(this, &self.data) }; - V::fmt_metadata(view, f) + match V::metadata(view) { + Err(e) => write!(f, ""), + Ok(metadata) => Debug::fmt(&metadata, f), + } } fn dyn_array_hash(&self, state: &mut dyn Hasher, precision: crate::Precision) { diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 7918c6b9956..ac79eb7451d 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -22,7 +22,6 @@ use crate::array::ArrayView; use crate::array::VTable; use crate::dtype::DType; use crate::stats::ArrayStats; -use crate::stats::StatsSet; use crate::stats::StatsSetRef; use crate::validity::Validity; @@ -44,6 +43,7 @@ impl ArrayParts { } } } + // ============================================================================= // ArrayInner — the concrete type stored inside Arc // ============================================================================= @@ -63,19 +63,14 @@ pub(crate) struct ArrayInner { } impl ArrayInner { - /// Create a new inner array from explicit construction parameters. + /// Create a new inner array from encoding-specific data. #[doc(hidden)] - pub fn try_new(new: ArrayParts) -> VortexResult { - new.vtable.validate(&new.data, &new.dtype, new.len)?; - Ok(unsafe { - Self::from_data_unchecked( - new.vtable, - new.dtype, - new.len, - new.data, - ArrayStats::default(), - ) - }) + pub fn try_from_data(data: V::ArrayData) -> VortexResult { + let vtable = V::vtable(&data).clone(); + let dtype = V::dtype(&data).clone(); + let len = V::len(&data); + let stats = V::stats(&data).clone(); + Ok(unsafe { Self::from_data_unchecked(vtable, dtype, len, data, stats) }) } /// Create without validation. @@ -165,32 +160,49 @@ pub struct Array { #[allow(clippy::same_name_method)] impl Array { - /// Create a typed array from explicit construction parameters. - pub fn try_from_parts(new: ArrayParts) -> VortexResult { - let inner = ArrayRef::from_inner(Arc::new(ArrayInner::::try_new(new)?)); + /// Create a typed array from encoding-specific data. + pub fn try_from_data(data: V::ArrayData) -> VortexResult { + let inner = ArrayRef::from_inner(Arc::new(ArrayInner::::try_from_data(data)?)); Ok(Self { inner, _phantom: PhantomData, }) } + /// Create a typed array from explicit construction parameters, validating that the + /// length and dtype are consistent with the data. + pub fn try_from_parts(new: ArrayParts) -> VortexResult { + let inner = unsafe { + ArrayInner::from_data_unchecked( + new.vtable, + new.dtype, + new.len, + new.data, + ArrayStats::default(), + ) + }; + Ok(Self { + inner: ArrayRef::from_inner(Arc::new(inner)), + _phantom: PhantomData, + }) + } + /// Create a typed array from explicit construction parameters without validation. /// /// # Safety - /// Caller must ensure the provided parts are logically consistent. - #[doc(hidden)] + /// Caller must ensure dtype and len match the data. pub unsafe fn from_parts_unchecked(new: ArrayParts) -> Self { - let inner = ArrayRef::from_inner(Arc::new(unsafe { - ArrayInner::::from_data_unchecked( + let inner = unsafe { + ArrayInner::from_data_unchecked( new.vtable, new.dtype, new.len, new.data, ArrayStats::default(), ) - })); + }; Self { - inner, + inner: ArrayRef::from_inner(Arc::new(inner)), _phantom: PhantomData, } } @@ -249,22 +261,6 @@ impl Array { &self.downcast_inner().data } - /// Returns the full typed array construction parts. - pub fn into_parts(self) -> ArrayParts { - let inner = self.downcast_inner(); - ArrayParts { - vtable: inner.vtable.clone(), - dtype: inner.dtype.clone(), - len: inner.len, - data: inner.data.clone(), - } - } - - pub fn with_stats_set(self, stats: StatsSet) -> Self { - self.statistics().replace(stats); - self - } - /// Returns a clone of the inner encoding-specific data. pub fn into_data(self) -> V::ArrayData { self.downcast_inner().data.clone() @@ -444,25 +440,3 @@ impl From> for ArrayRef { value.inner } } - -#[cfg(test)] -mod tests { - use vortex_buffer::buffer; - - use super::Array; - use crate::arrays::Primitive; - use crate::arrays::PrimitiveArray; - use crate::assert_arrays_eq; - use crate::validity::Validity; - - #[test] - fn typed_array_into_parts_roundtrips() { - let array = PrimitiveArray::new(buffer![1i32, 2, 3], Validity::NonNullable); - let expected = array.clone(); - - let parts = array.into_parts(); - let rebuilt = Array::::try_from_parts(parts).unwrap(); - - assert_arrays_eq!(rebuilt, expected); - } -} diff --git a/vortex-array/src/array/vtable/dyn_.rs b/vortex-array/src/array/vtable/dyn_.rs index 681267403e2..ca6cbaf0cb4 100644 --- a/vortex-array/src/array/vtable/dyn_.rs +++ b/vortex-array/src/array/vtable/dyn_.rs @@ -16,12 +16,14 @@ use crate::ExecutionStep; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayId; -use crate::array::ArrayParts; +use crate::array::ArrayInner; use crate::array::VTable; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; + /// Reference-counted DynVTable pub type DynVTableRef = Arc; @@ -83,11 +85,27 @@ impl DynVTable for V { children: &dyn ArrayChildren, session: &VortexSession, ) -> VortexResult { - let inner = self.deserialize(dtype, len, metadata, buffers, children, session)?; - Ok( - Array::::try_from_parts(ArrayParts::new(self.clone(), dtype.clone(), len, inner))? - .into_array(), - ) + let metadata = V::deserialize(metadata, dtype, len, buffers, session)?; + let inner = V::build(dtype, len, &metadata, buffers, children)?; + // Validate the inner array's properties before wrapping. + assert_eq!(V::len(&inner), len, "Array length mismatch after building"); + assert_eq!( + V::dtype(&inner), + dtype, + "Array dtype mismatch after building" + ); + // Wrap in ArrayInner for safe downcasting. + // SAFETY: We just validated that V::len(&inner) == len and V::dtype(&inner) == dtype. + let array = unsafe { + ArrayInner::from_data_unchecked( + self.clone(), + dtype.clone(), + len, + inner, + ArrayStats::default(), + ) + }; + Ok(array.into_array()) } fn with_slots(&self, array: ArrayRef, slots: Vec>) -> VortexResult { @@ -96,17 +114,7 @@ impl DynVTable for V { .vortex_expect("Failed to downcast array"); let mut data = typed.data().clone(); V::with_slots(&mut data, slots)?; - let stats = array.statistics().to_owned(); - Ok(unsafe { - Array::::from_parts_unchecked(ArrayParts::new( - self.clone(), - array.dtype().clone(), - array.len(), - data, - )) - .with_stats_set(stats) - .into_array() - }) + Ok(Array::::try_from_data(data)?.into_array()) } fn reduce(&self, array: &ArrayRef) -> VortexResult> { diff --git a/vortex-array/src/array/vtable/mod.rs b/vortex-array/src/array/vtable/mod.rs index 305cbef6ca2..7f233b72f5d 100644 --- a/vortex-array/src/array/vtable/mod.rs +++ b/vortex-array/src/array/vtable/mod.rs @@ -8,7 +8,6 @@ mod operations; mod validity; use std::fmt::Debug; -use std::fmt::Formatter; use std::hash::Hasher; pub use dyn_::*; @@ -36,6 +35,7 @@ use crate::executor::ExecutionCtx; use crate::patches::Patches; use crate::scalar::ScalarValue; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The array [`VTable`] encapsulates logic for an Array type within Vortex. @@ -52,16 +52,28 @@ use crate::validity::Validity; /// out of bounds). Post-conditions are validated after invocation of the vtable function and will /// panic if violated. pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { - type ArrayData: 'static + Send + Sync + Clone + Debug; + type ArrayData: 'static + Send + Sync + Clone + Debug + IntoArray; + type Metadata: Debug; type OperationsVTable: OperationsVTable; type ValidityVTable: ValidityVTable; + /// Returns the VTable from the array instance. + /// + // NOTE(ngates): this function is temporary while we migrate Arrays over to the unified vtable + fn vtable(array: &Self::ArrayData) -> &Self; + /// Returns the ID of the array. fn id(&self) -> ArrayId; - /// Validates that externally supplied logical metadata matches the array data. - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()>; + /// Returns the length of the array. + fn len(array: &Self::ArrayData) -> usize; + + /// Returns the DType of the array. + fn dtype(array: &Self::ArrayData) -> &DType; + + /// Returns the stats set for the array. + fn stats(array: &Self::ArrayData) -> &ArrayStats; /// Hashes the array contents. fn array_hash(array: &Self::ArrayData, state: &mut H, precision: Precision); @@ -118,28 +130,21 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { .vortex_expect("child_name index out of bounds") } + /// Exports metadata for an array. + fn metadata(array: ArrayView<'_, Self>) -> VortexResult; + /// Serialize metadata into a byte buffer for IPC or file storage. /// Return `None` if the array cannot be serialized. - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>>; + fn serialize(metadata: Self::Metadata) -> VortexResult>>; - /// Formats a human-readable metadata description for display tooling. - fn fmt_metadata(array: ArrayView<'_, Self>, f: &mut Formatter<'_>) -> std::fmt::Result { - match Self::serialize(array) { - Ok(Some(metadata)) if metadata.is_empty() => f.write_str("EmptyMetadata"), - _ => Debug::fmt(array.data(), f), - } - } - - /// Deserialize an array from serialized components. + /// Deserialize array metadata from a byte buffer. fn deserialize( - &self, - dtype: &DType, - len: usize, - metadata: &[u8], - buffers: &[BufferHandle], - children: &dyn ArrayChildren, - session: &VortexSession, - ) -> VortexResult; + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult; /// Writes the array into a canonical builder. fn append_to_builder( @@ -156,6 +161,15 @@ pub trait VTable: 'static + Clone + Sized + Send + Sync + Debug { Ok(()) } + /// Build an array from components. + fn build( + dtype: &DType, + len: usize, + metadata: &Self::Metadata, + buffers: &[BufferHandle], + children: &dyn ArrayChildren, + ) -> VortexResult; + /// Returns the slots of the array as a slice. /// /// Slots provide fixed-position storage for child arrays. Each encoding defines named @@ -329,12 +343,23 @@ macro_rules! vtable { // Legacy form: FooArray is the inner struct name, no type alias generated. ($Base:ident, $VT:ident) => { $crate::aliases::paste::paste! { + impl $crate::IntoArray for [<$Base Array>] { + fn into_array(self) -> $crate::ArrayRef { + use $crate::aliases::vortex_error::VortexExpect; + $crate::ArrayRef::from($crate::Array::<$VT>::try_from_data(self).vortex_expect("data is always valid")) + } + } + + impl From<[<$Base Array>]> for $crate::ArrayRef { + fn from(value: [<$Base Array>]) -> $crate::ArrayRef { + use $crate::IntoArray; + value.into_array() + } + } + impl [<$Base Array>] { #[deprecated(note = "use `.into_array()` (owned) or `.clone().into_array()` (ref) to make clones explicit")] - pub fn to_array(&self) -> $crate::ArrayRef - where - Self: Clone + $crate::IntoArray, - { + pub fn to_array(&self) -> $crate::ArrayRef { use $crate::IntoArray; self.clone().into_array() } @@ -346,6 +371,20 @@ macro_rules! vtable { $crate::aliases::paste::paste! { /// Type alias: `FooArray = Array`. pub type [<$Base Array>] = $crate::Array<$VT>; + + impl $crate::IntoArray for $Data { + fn into_array(self) -> $crate::ArrayRef { + use $crate::aliases::vortex_error::VortexExpect; + $crate::Array::<$VT>::try_from_data(self).vortex_expect("data is always valid").into_array() + } + } + + impl From<$Data> for $crate::ArrayRef { + fn from(value: $Data) -> $crate::ArrayRef { + use $crate::IntoArray; + value.into_array() + } + } } }; } diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 688e5301c43..4c654a44081 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -12,14 +12,13 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Bool; use crate::arrays::BoolArray; use crate::buffer::BufferHandle; use crate::dtype::DType; -use crate::dtype::Nullability; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -63,13 +62,14 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; pub struct BoolData { /// Child arrays stored as slots. See [`VTable::slots`] for design rationale. pub(super) slots: Vec>, - pub(super) nullability: Nullability, + pub(super) dtype: DType, pub(super) bits: BufferHandle, pub(super) offset: usize, pub(super) len: usize, + pub(super) stats_set: ArrayStats, } -pub struct BoolDataParts { +pub struct BoolArrayParts { pub bits: BufferHandle, pub offset: usize, pub len: usize, @@ -84,8 +84,8 @@ impl BoolData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Bool(self.nullability) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns `true` if this array is empty. @@ -96,7 +96,7 @@ impl BoolData { /// Returns the [`Validity`] of this array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } /// Returns the validity as a [`Mask`]. @@ -118,9 +118,9 @@ impl BoolData { /// Splits into owned parts #[inline] - pub fn into_parts(self) -> BoolDataParts { + pub fn into_parts(self) -> BoolArrayParts { let validity = self.validity(); - BoolDataParts { + BoolArrayParts { bits: self.bits, offset: self.offset, len: self.len, @@ -186,10 +186,7 @@ impl Array { /// /// Returns an error if the provided components do not satisfy the invariants. pub fn try_new(bits: BitBuffer, validity: Validity) -> VortexResult { - let dtype = DType::Bool(validity.nullability()); - let len = bits.len(); - let data = BoolData::try_new(bits, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) }) + Array::try_from_data(BoolData::try_new(bits, validity)?) } /// Build a new bool array from a `BufferHandle`, returning an error if the offset is @@ -200,9 +197,7 @@ impl Array { len: usize, validity: Validity, ) -> VortexResult { - let dtype = DType::Bool(validity.nullability()); - let data = BoolData::try_new_from_handle(bits, offset, len, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) }) + Array::try_from_data(BoolData::try_new_from_handle(bits, offset, len, validity)?) } /// Creates a new [`BoolArray`] without validation. @@ -211,11 +206,9 @@ impl Array { /// /// The caller must ensure that the validity length is equal to the bit buffer length. pub unsafe fn new_unchecked(bits: BitBuffer, validity: Validity) -> Self { - let dtype = DType::Bool(validity.nullability()); - let len = bits.len(); // SAFETY: caller guarantees validity length equals bit buffer length. - let data = unsafe { BoolData::new_unchecked(bits, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Bool, dtype, len, data)) } + Array::try_from_data(unsafe { BoolData::new_unchecked(bits, validity) }) + .vortex_expect("BoolData is always valid") } /// Validates the components that would be used to create a [`BoolArray`]. @@ -240,6 +233,12 @@ impl Array { pub fn into_bit_buffer(self) -> BitBuffer { self.into_data().into_bit_buffer() } + + /// Splits into owned parts, consuming self. + #[inline] + pub fn into_parts(self) -> BoolArrayParts { + self.into_data().into_parts() + } } /// Internal constructors on BoolData (used by Array constructors and VTable::build). @@ -251,12 +250,15 @@ impl BoolData { let (offset, len, buffer) = bits.into_inner(); let slots = Self::make_slots(&validity, len); + let dtype = DType::Bool(validity.nullability()); + Ok(Self { slots, - nullability: validity.nullability(), + dtype, bits: BufferHandle::new_host(buffer), offset, len, + stats_set: ArrayStats::default(), }) } @@ -282,12 +284,15 @@ impl BoolData { ); let slots = Self::make_slots(&validity, len); + let dtype = DType::Bool(validity.nullability()); + Ok(Self { slots, - nullability: validity.nullability(), + dtype, bits, offset, len, + stats_set: ArrayStats::default(), }) } @@ -297,13 +302,15 @@ impl BoolData { } else { let (offset, len, buffer) = bits.into_inner(); let slots = Self::make_slots(&validity, len); + let dtype = DType::Bool(validity.nullability()); Self { slots, - nullability: validity.nullability(), + dtype, bits: BufferHandle::new_host(buffer), offset, len, + stats_set: ArrayStats::default(), } } } diff --git a/vortex-array/src/arrays/bool/mod.rs b/vortex-array/src/arrays/bool/mod.rs index b0f20794ca8..9378983286f 100644 --- a/vortex-array/src/arrays/bool/mod.rs +++ b/vortex-array/src/arrays/bool/mod.rs @@ -4,8 +4,8 @@ mod array; mod patch; +pub use array::BoolArrayParts; pub use array::BoolData; -pub use array::BoolDataParts; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index c2a5c25ca8c..17a87a92cbe 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -1,10 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::fmt::Formatter; - use kernel::PARENT_KERNELS; -use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -13,8 +10,11 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; +use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; +use crate::ProstMetadata; +use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayView; use crate::array::VTable; @@ -36,6 +36,7 @@ use crate::array::ArrayId; use crate::arrays::bool::compute::rules::RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; +use crate::stats::ArrayStats; vtable!(Bool, Bool, BoolData); @@ -49,13 +50,30 @@ pub struct BoolMetadata { impl VTable for Bool { type ArrayData = BoolData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Bool + } + fn id(&self) -> ArrayId { Self::ID } + fn len(array: &BoolData) -> usize { + array.len + } + + fn dtype(array: &BoolData) -> &DType { + &array.dtype + } + + fn stats(array: &BoolData) -> &ArrayStats { + &array.stats_set + } + fn array_hash(array: &BoolData, state: &mut H, precision: Precision) { array.to_bit_buffer().array_hash(state, precision); array.validity().array_hash(state, precision); @@ -86,50 +104,35 @@ impl VTable for Bool { } } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { assert!(array.offset < 8, "Offset must be <8, got {}", array.offset); - Ok(Some( - BoolMetadata { - offset: u32::try_from(array.offset).vortex_expect("checked"), - } - .encode_to_vec(), - )) + Ok(ProstMetadata(BoolMetadata { + offset: u32::try_from(array.offset).vortex_expect("checked"), + })) } - fn fmt_metadata(array: ArrayView<'_, Self>, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "BoolMetadata {{ offset: {} }}", array.offset) + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } - fn validate(&self, data: &BoolData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "BoolArray length {} does not match outer length {}", - data.len(), - len - ); - - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "BoolArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); - - Ok(()) + fn deserialize( + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + let metadata = ::deserialize(bytes)?; + Ok(ProstMetadata(metadata)) } - fn deserialize( - &self, + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = BoolMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } @@ -206,8 +209,8 @@ mod tests { use crate::LEGACY_SESSION; use crate::arrays::BoolArray; use crate::assert_arrays_eq; + use crate::serde::ArrayParts; use crate::serde::SerializeOptions; - use crate::serde::SerializedArray; #[test] fn test_nullable_bool_serde_roundtrip() { @@ -226,7 +229,7 @@ mod tests { for buf in serialized { concat.extend_from_slice(buf.as_ref()); } - let parts = SerializedArray::try_from(concat.freeze()).unwrap(); + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index 33b471e1852..5aa15ce6d57 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -17,7 +17,6 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::Chunked; use crate::arrays::primitive::PrimitiveData; use crate::dtype::DType; @@ -25,6 +24,7 @@ use crate::iter::ArrayIterator; use crate::iter::ArrayIteratorAdapter; use crate::search_sorted::SearchSorted; use crate::search_sorted::SearchSortedSide; +use crate::stats::ArrayStats; use crate::stream::ArrayStream; use crate::stream::ArrayStreamAdapter; use crate::validity::Validity; @@ -34,10 +34,12 @@ pub(super) const CHUNKS_OFFSET: usize = 1; #[derive(Clone, Debug)] pub struct ChunkedData { - pub(super) empty_dtype: Option, + pub(super) dtype: DType, + pub(super) len: usize, pub(super) chunk_offsets: PrimitiveData, pub(super) chunks: Vec, pub(super) slots: Vec>, + pub(super) stats_set: ArrayStats, } impl ChunkedData { @@ -47,17 +49,7 @@ impl ChunkedData { chunks: &[ArrayRef], ) -> Vec> { let mut slots = Vec::with_capacity(1 + chunks.len()); - slots.push(Some( - unsafe { - Array::from_parts_unchecked(ArrayParts::new( - crate::arrays::Primitive, - chunk_offsets.dtype(), - chunk_offsets.len(), - chunk_offsets.clone(), - )) - } - .into_array(), - )); + slots.push(Some(chunk_offsets.clone().into_array())); slots.extend(chunks.iter().map(|c| Some(c.clone()))); slots } @@ -106,10 +98,14 @@ impl ChunkedData { let slots = Self::make_slots(&chunk_offsets, &chunks); Self { - empty_dtype: chunks.is_empty().then_some(dtype), + dtype, + len: curr_offset + .try_into() + .vortex_expect("chunk offset must fit in usize"), chunk_offsets, chunks, slots, + stats_set: Default::default(), } } @@ -128,20 +124,12 @@ impl ChunkedData { /// Returns the length of this array. pub fn len(&self) -> usize { - self.chunk_offsets() - .last() - .copied() - .and_then(|len| usize::try_from(len).ok()) - .vortex_expect("chunk offset must fit in usize") + self.len } /// Returns the [`DType`] of this array. pub fn dtype(&self) -> &DType { - self.chunks - .first() - .map(|chunk| chunk.dtype()) - .or(self.empty_dtype.as_ref()) - .vortex_expect("ChunkedArray dtype must come from chunks or the empty dtype") + &self.dtype } /// Returns `true` if this array is empty. @@ -231,12 +219,10 @@ impl ChunkedData { new_chunks.push( // SAFETY: chunks_to_combine contains valid chunks of the same dtype as self. // All chunks are guaranteed to be valid arrays matching self.dtype(). - unsafe { - Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) - } - .into_array() - .to_canonical()? - .into_array(), + unsafe { ChunkedData::new_unchecked(chunks_to_combine, self.dtype().clone()) } + .into_array() + .to_canonical()? + .into_array(), ); new_chunk_n_bytes = 0; @@ -257,7 +243,7 @@ impl ChunkedData { new_chunks.push( // SAFETY: chunks_to_combine contains valid chunks of the same dtype as self. // All chunks are guaranteed to be valid arrays matching self.dtype(). - unsafe { Array::::new_unchecked(chunks_to_combine, self.dtype().clone()) } + unsafe { ChunkedData::new_unchecked(chunks_to_combine, self.dtype().clone()) } .into_array() .to_canonical()? .into_array(), @@ -273,22 +259,7 @@ impl ChunkedData { impl Array { /// Constructs a new `ChunkedArray`. pub fn try_new(chunks: Vec, dtype: DType) -> VortexResult { - let data = ChunkedData::try_new(chunks, dtype)?; - let dtype = data.dtype().clone(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Chunked, dtype, len, data)) }) - } - - pub fn rechunk(&self, target_bytesize: u64, target_rowsize: usize) -> VortexResult { - let data = self.data().rechunk(target_bytesize, target_rowsize)?; - Ok(unsafe { - Array::from_parts_unchecked(ArrayParts::new( - Chunked, - self.dtype().clone(), - data.len(), - data, - )) - }) + Array::try_from_data(ChunkedData::try_new(chunks, dtype)?) } /// Creates a new `ChunkedArray` without validation. @@ -297,10 +268,8 @@ impl Array { /// /// See [`ChunkedData::new_unchecked`]. pub unsafe fn new_unchecked(chunks: Vec, dtype: DType) -> Self { - let data = unsafe { ChunkedData::new_unchecked(chunks, dtype) }; - let dtype = data.dtype().clone(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Chunked, dtype, len, data)) } + Array::try_from_data(unsafe { ChunkedData::new_unchecked(chunks, dtype) }) + .vortex_expect("ChunkedData is always valid") } } diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index f1266a5373d..8eefe189a29 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -8,7 +8,6 @@ use vortex_buffer::buffer; use crate::IntoArray; use crate::accessor::ArrayAccessor; -use crate::array::ArrayParts; use crate::array::VTable; use crate::arrays::Chunked; use crate::arrays::ChunkedArray; @@ -208,12 +207,10 @@ fn with_slots_updates_nchunks_len_and_offsets() { ]; let expected_nchunks = slots.len() - 1; let expected_len = orig.len(); - let dtype = orig.dtype().clone(); let mut data = orig.into_data(); ::with_slots(&mut data, slots).unwrap(); - let array = - ChunkedArray::try_from_parts(ArrayParts::new(Chunked, dtype, expected_len, data)).unwrap(); + let array = ChunkedArray::try_from_data(data).unwrap(); assert_eq!(array.nchunks(), expected_nchunks); assert_eq!(array.len(), expected_len); diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index 6349bfa97ec..3f20f9d56ea 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -11,6 +11,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Canonical; +use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; @@ -20,7 +21,6 @@ use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::PrimitiveArray; use crate::arrays::chunked::ChunkedData; use crate::arrays::chunked::array::CHUNK_OFFSETS_SLOT; use crate::arrays::chunked::array::CHUNKS_OFFSET; @@ -36,6 +36,7 @@ use crate::dtype::PType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod canonical; @@ -53,39 +54,47 @@ impl Chunked { impl VTable for Chunked { type ArrayData = ChunkedData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Chunked + } fn id(&self) -> ArrayId { Self::ID } + fn len(array: &ChunkedData) -> usize { + array.len + } + + fn dtype(array: &ChunkedData) -> &DType { + &array.dtype + } + + fn stats(array: &ChunkedData) -> &ArrayStats { + &array.stats_set + } + fn array_hash(array: &ChunkedData, state: &mut H, precision: Precision) { - PrimitiveArray::new( - array.chunk_offsets.to_buffer::(), - Validity::NonNullable, - ) - .into_array() - .array_hash(state, precision); + array + .chunk_offsets + .clone() + .into_array() + .array_hash(state, precision); for chunk in &array.chunks { chunk.array_hash(state, precision); } } fn array_eq(array: &ChunkedData, other: &ChunkedData, precision: Precision) -> bool { - PrimitiveArray::new( - array.chunk_offsets.to_buffer::(), - Validity::NonNullable, - ) - .into_array() - .array_eq( - &PrimitiveArray::new( - other.chunk_offsets.to_buffer::(), - Validity::NonNullable, - ) - .into_array(), - precision, - ) && array.chunks.len() == other.chunks.len() + array + .chunk_offsets + .clone() + .into_array() + .array_eq(&other.chunk_offsets.clone().into_array(), precision) + && array.chunks.len() == other.chunks.len() && array .iter_chunks() .zip(other.iter_chunks()) @@ -104,42 +113,31 @@ impl VTable for Chunked { vortex_panic!("ChunkedArray buffer_name index {idx} out of bounds") } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(vec![])) + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) } - fn validate(&self, data: &ChunkedData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "ChunkedArray length {} does not match outer length {}", - data.len(), - len - ); - vortex_ensure!( - data.dtype() == dtype, - "ChunkedArray dtype {} does not match outer dtype {}", - data.dtype(), - dtype - ); - Ok(()) + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(vec![])) } fn deserialize( - &self, - dtype: &DType, + _bytes: &[u8], + _dtype: &DType, _len: usize, - metadata: &[u8], + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + fn build( + dtype: &DType, + _len: usize, + _metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - if !metadata.is_empty() { - vortex_bail!( - "ChunkedArray expects empty metadata, got {} bytes", - metadata.len() - ); - } if children.is_empty() { vortex_bail!("Chunked array needs at least one child"); } @@ -170,15 +168,23 @@ impl VTable for Chunked { }) .try_collect()?; - let chunk_offsets = PrimitiveData::new(chunk_offsets_buf, Validity::NonNullable); + let chunk_offsets = PrimitiveData::new(chunk_offsets_buf.clone(), Validity::NonNullable); + + let total_len = chunk_offsets_buf + .last() + .ok_or_else(|| vortex_err!("chunk_offsets must not be empty"))?; + let len = usize::try_from(*total_len) + .map_err(|_| vortex_err!("total length {} exceeds usize range", total_len))?; let slots = ChunkedData::make_slots(&chunk_offsets, &chunks); // Construct directly using the struct fields to avoid recomputing chunk_offsets Ok(ChunkedData { - empty_dtype: chunks.is_empty().then_some(dtype.clone()), + dtype: dtype.clone(), + len, chunk_offsets, chunks, slots, + stats_set: Default::default(), }) } @@ -217,11 +223,16 @@ impl VTable for Chunked { .ok_or_else(|| vortex_err!("chunk slot must not be None")) }) .try_collect()?; - array.chunk_offsets = PrimitiveData::new(chunk_offsets_buf, Validity::NonNullable); - array.empty_dtype = chunks.is_empty().then_some(array.dtype().clone()); + array.chunk_offsets = PrimitiveData::new(chunk_offsets_buf.clone(), Validity::NonNullable); array.chunks = chunks; array.slots = slots; + let total_len = chunk_offsets_buf + .last() + .ok_or_else(|| vortex_err!("chunk_offsets must not be empty"))?; + array.len = usize::try_from(*total_len) + .map_err(|_| vortex_err!("total length {} exceeds usize range", total_len))?; + Ok(()) } diff --git a/vortex-array/src/arrays/chunked/vtable/validity.rs b/vortex-array/src/arrays/chunked/vtable/validity.rs index a250c98be89..18e429b46b0 100644 --- a/vortex-array/src/arrays/chunked/vtable/validity.rs +++ b/vortex-array/src/arrays/chunked/vtable/validity.rs @@ -8,7 +8,7 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::array::ValidityVTable; use crate::arrays::Chunked; -use crate::arrays::ChunkedArray; +use crate::arrays::chunked::ChunkedData; use crate::dtype::DType; use crate::dtype::Nullability; use crate::validity::Validity; @@ -39,7 +39,7 @@ impl ValidityVTable for Chunked { Ok(Validity::Array( unsafe { - ChunkedArray::new_unchecked( + ChunkedData::new_unchecked( validities .into_iter() .zip(array.iter_chunks()) diff --git a/vortex-array/src/arrays/constant/array.rs b/vortex-array/src/arrays/constant/array.rs index 3841a9d2b02..ff32124210a 100644 --- a/vortex-array/src/arrays/constant/array.rs +++ b/vortex-array/src/arrays/constant/array.rs @@ -1,32 +1,54 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexExpect; + use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::Constant; +use crate::dtype::DType; use crate::scalar::Scalar; +use crate::stats::ArrayStats; pub(super) const NUM_SLOTS: usize = 0; #[derive(Clone, Debug)] pub struct ConstantData { pub(super) scalar: Scalar, + pub(super) len: usize, pub(super) slots: Vec>, + pub(super) stats_set: ArrayStats, } impl ConstantData { - pub fn new(scalar: S) -> Self + pub fn new(scalar: S, len: usize) -> Self where S: Into, { let scalar = scalar.into(); Self { scalar, + len, slots: vec![], + stats_set: Default::default(), } } + /// Returns the length of this array. + pub fn len(&self) -> usize { + self.len + } + + /// Returns the [`DType`] of this array. + pub fn dtype(&self) -> &DType { + self.scalar.dtype() + } + + /// Returns `true` if this array is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Returns the [`Scalar`] value of this constant array. pub fn scalar(&self) -> &Scalar { &self.scalar @@ -42,9 +64,7 @@ impl Array { where S: Into, { - let scalar = scalar.into(); - let dtype = scalar.dtype().clone(); - let data = ConstantData::new(scalar); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Constant, dtype, len, data)) } + Array::try_from_data(ConstantData::new(scalar, len)) + .vortex_expect("ConstantData is always valid") } } diff --git a/vortex-array/src/arrays/constant/compute/take.rs b/vortex-array/src/arrays/constant/compute/take.rs index 96aa71fd9ac..4bbd44132e1 100644 --- a/vortex-array/src/arrays/constant/compute/take.rs +++ b/vortex-array/src/arrays/constant/compute/take.rs @@ -9,9 +9,9 @@ use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; -use crate::arrays::MaskedArray; use crate::arrays::dict::TakeReduce; use crate::arrays::dict::TakeReduceAdaptor; +use crate::arrays::masked::MaskedData; use crate::optimizer::rules::ParentRuleSet; use crate::scalar::Scalar; use crate::validity::Validity; @@ -45,7 +45,7 @@ impl TakeReduce for Constant { return Ok(Some(arr)); } - MaskedArray::try_new(arr, Validity::from(v.clone()))?.into_array() + MaskedData::try_new(arr, Validity::from(v.clone()))?.into_array() } }; Ok(Some(result)) diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index 55869cbd220..d6fd4407635 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -39,6 +39,7 @@ use crate::scalar::DecimalValue; use crate::scalar::Scalar; use crate::scalar::ScalarValue; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::vtable; pub(crate) mod canonical; mod operations; @@ -56,19 +57,28 @@ impl Constant { impl VTable for Constant { type ArrayData = ConstantData; + type Metadata = Scalar; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Constant + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &ConstantData, dtype: &DType, _len: usize) -> VortexResult<()> { - vortex_ensure!( - data.scalar.dtype() == dtype, - "ConstantArray scalar dtype does not match outer dtype" - ); - Ok(()) + fn len(array: &ConstantData) -> usize { + array.len + } + + fn dtype(array: &ConstantData) -> &DType { + array.scalar.dtype() + } + + fn stats(array: &ConstantData) -> &ArrayStats { + &array.stats_set } fn array_hash( @@ -121,22 +131,23 @@ impl VTable for Constant { Ok(()) } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(array.scalar().clone()) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { // HACK: Because the scalar is stored in the buffers, we do not need to serialize the // metadata at all. Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], dtype: &DType, _len: usize, - _metadata: &[u8], - buffers: &[BufferHandle], - _children: &dyn ArrayChildren, session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult { vortex_ensure!( buffers.len() == 1, "Expected 1 buffer, got {}", @@ -149,7 +160,17 @@ impl VTable for Constant { let scalar_value = ScalarValue::from_proto_bytes(bytes, dtype, session)?; let scalar = Scalar::try_new(dtype.clone(), scalar_value)?; - Ok(ConstantData::new(scalar)) + Ok(scalar) + } + + fn build( + _dtype: &DType, + len: usize, + metadata: &Self::Metadata, + _buffers: &[BufferHandle], + _children: &dyn ArrayChildren, + ) -> VortexResult { + Ok(ConstantData::new(metadata.clone(), len)) } fn reduce_parent( diff --git a/vortex-array/src/arrays/decimal/array.rs b/vortex-array/src/arrays/decimal/array.rs index a51657fd958..f8e3ce0c976 100644 --- a/vortex-array/src/arrays/decimal/array.rs +++ b/vortex-array/src/arrays/decimal/array.rs @@ -15,7 +15,6 @@ use crate::ArrayRef; use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Decimal; @@ -28,10 +27,10 @@ use crate::dtype::DecimalDType; use crate::dtype::DecimalType; use crate::dtype::IntegerPType; use crate::dtype::NativeDecimalType; -use crate::dtype::Nullability; use crate::match_each_decimal_value_type; use crate::match_each_integer_ptype; use crate::patches::Patches; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -99,13 +98,13 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct DecimalData { pub(super) slots: Vec>, - pub(super) decimal_dtype: DecimalDType, - pub(super) nullability: Nullability, + pub(super) dtype: DType, pub(super) values: BufferHandle, pub(super) values_type: DecimalType, + pub(super) stats_set: ArrayStats, } -pub struct DecimalDataParts { +pub struct DecimalArrayParts { pub decimal_dtype: DecimalDType, pub values: BufferHandle, pub values_type: DecimalType, @@ -243,8 +242,8 @@ impl DecimalData { slots: Self::make_slots(&validity, len), values, values_type, - decimal_dtype, - nullability: validity.nullability(), + dtype: DType::Decimal(decimal_dtype, validity.nullability()), + stats_set: Default::default(), } } @@ -301,8 +300,8 @@ impl DecimalData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Decimal(self.decimal_dtype, self.nullability) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns `true` if this array is empty. @@ -312,13 +311,15 @@ impl DecimalData { /// Reconstructs the validity from the slot state. pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } - pub fn into_parts(self) -> DecimalDataParts { + pub fn into_parts(self) -> DecimalArrayParts { let validity = self.validity(); - DecimalDataParts { - decimal_dtype: self.decimal_dtype, + let decimal_dtype = self.dtype.into_decimal_opt().vortex_expect("cannot fail"); + + DecimalArrayParts { + decimal_dtype, values: self.values, values_type: self.values_type, validity, @@ -343,7 +344,11 @@ impl DecimalData { /// Returns the decimal type information pub fn decimal_dtype(&self) -> DecimalDType { - self.decimal_dtype + if let DType::Decimal(decimal_dtype, _) = self.dtype { + decimal_dtype + } else { + vortex_panic!("Expected Decimal dtype, got {:?}", self.dtype) + } } /// Return the `DecimalType` used to represent the values in the array. @@ -445,10 +450,8 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - let dtype = DType::Decimal(decimal_dtype, validity.nullability()); - let len = buffer.len(); - let data = DecimalData::new(buffer, decimal_dtype, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + Array::try_from_data(DecimalData::new(buffer, decimal_dtype, validity)) + .vortex_expect("DecimalData is always valid") } /// Creates a new [`DecimalArray`] without validation. @@ -461,10 +464,8 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - let dtype = DType::Decimal(decimal_dtype, validity.nullability()); - let len = buffer.len(); - let data = unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + Array::try_from_data(unsafe { DecimalData::new_unchecked(buffer, decimal_dtype, validity) }) + .vortex_expect("DecimalData is always valid") } /// Creates a new [`DecimalArray`] from a host-native buffer with validation. @@ -473,10 +474,7 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> VortexResult { - let dtype = DType::Decimal(decimal_dtype, validity.nullability()); - let len = buffer.len(); - let data = DecimalData::try_new(buffer, decimal_dtype, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) }) + Array::try_from_data(DecimalData::try_new(buffer, decimal_dtype, validity)?) } /// Creates a new [`DecimalArray`] from an iterator of values. @@ -488,10 +486,8 @@ impl Array { iter: I, decimal_dtype: DecimalDType, ) -> Self { - let data = DecimalData::from_iter(iter, decimal_dtype); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + Array::try_from_data(DecimalData::from_iter(iter, decimal_dtype)) + .vortex_expect("DecimalData is always valid") } /// Creates a new [`DecimalArray`] from an iterator of optional values. @@ -499,10 +495,8 @@ impl Array { iter: I, decimal_dtype: DecimalDType, ) -> Self { - let data = DecimalData::from_option_iter(iter, decimal_dtype); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + Array::try_from_data(DecimalData::from_option_iter(iter, decimal_dtype)) + .vortex_expect("DecimalData is always valid") } /// Creates a new [`DecimalArray`] from a [`BufferHandle`]. @@ -512,10 +506,13 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - let dtype = DType::Decimal(decimal_dtype, validity.nullability()); - let len = values.len() / values_type.byte_width(); - let data = DecimalData::new_handle(values, values_type, decimal_dtype, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + Array::try_from_data(DecimalData::new_handle( + values, + values_type, + decimal_dtype, + validity, + )) + .vortex_expect("DecimalData is always valid") } /// Creates a new [`DecimalArray`] without validation from a [`BufferHandle`]. @@ -529,12 +526,10 @@ impl Array { decimal_dtype: DecimalDType, validity: Validity, ) -> Self { - let dtype = DType::Decimal(decimal_dtype, validity.nullability()); - let len = values.len() / values_type.byte_width(); - let data = unsafe { + Array::try_from_data(unsafe { DecimalData::new_unchecked_handle(values, values_type, decimal_dtype, validity) - }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Decimal, dtype, len, data)) } + }) + .vortex_expect("DecimalData is always valid") } } diff --git a/vortex-array/src/arrays/decimal/compute/between.rs b/vortex-array/src/arrays/decimal/compute/between.rs index b521d2148e0..fd8627c898c 100644 --- a/vortex-array/src/arrays/decimal/compute/between.rs +++ b/vortex-array/src/arrays/decimal/compute/between.rs @@ -36,7 +36,7 @@ impl BetweenKernel for Decimal { // NOTE: we know that have checked before that the lower and upper bounds are not all null. let nullability = - arr.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); + arr.dtype.nullability() | lower.dtype().nullability() | upper.dtype().nullability(); match_each_decimal_value_type!(arr.values_type(), |D| { between_unpack::(arr, lower, upper, nullability, options) diff --git a/vortex-array/src/arrays/decimal/mod.rs b/vortex-array/src/arrays/decimal/mod.rs index a0b90613f52..137331a492f 100644 --- a/vortex-array/src/arrays/decimal/mod.rs +++ b/vortex-array/src/arrays/decimal/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::DecimalArrayParts; pub use array::DecimalData; -pub use array::DecimalDataParts; pub use vtable::DecimalArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index dc7a46c779d..922b1b80fe2 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use kernel::PARENT_KERNELS; -use prost::Message; use vortex_buffer::Alignment; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -11,8 +10,11 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; +use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; +use crate::ProstMetadata; +use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayView; use crate::array::VTable; @@ -38,6 +40,7 @@ use crate::arrays::decimal::array::SLOT_NAMES; use crate::arrays::decimal::compute::rules::RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; +use crate::stats::ArrayStats; vtable!(Decimal, Decimal, DecimalData); // The type of the values can be determined by looking at the type info...right? @@ -50,13 +53,38 @@ pub struct DecimalMetadata { impl VTable for Decimal { type ArrayData = DecimalData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Decimal + } + fn id(&self) -> ArrayId { Self::ID } + fn len(array: &DecimalData) -> usize { + let divisor = match array.values_type { + DecimalType::I8 => 1, + DecimalType::I16 => 2, + DecimalType::I32 => 4, + DecimalType::I64 => 8, + DecimalType::I128 => 16, + DecimalType::I256 => 32, + }; + array.values.len() / divisor + } + + fn dtype(array: &DecimalData) -> &DType { + &array.dtype + } + + fn stats(array: &DecimalData) -> &ArrayStats { + &array.stats_set + } + fn array_hash(array: &DecimalData, state: &mut H, precision: Precision) { array.values.array_hash(state, precision); std::mem::discriminant(&array.values_type).hash(state); @@ -87,45 +115,34 @@ impl VTable for Decimal { } } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - DecimalMetadata { - values_type: array.values_type() as i32, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(DecimalMetadata { + values_type: array.values_type() as i32, + })) } - fn validate(&self, data: &DecimalData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "DecimalArray length {} does not match outer length {}", - data.len(), - len - ); - - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "DecimalArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); - - Ok(()) + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + let metadata = ProstMetadata::::deserialize(bytes)?; + Ok(ProstMetadata(metadata)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = DecimalMetadata::decode(metadata)?; if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } @@ -216,8 +233,8 @@ mod tests { use crate::arrays::DecimalArray; use crate::assert_arrays_eq; use crate::dtype::DecimalDType; + use crate::serde::ArrayParts; use crate::serde::SerializeOptions; - use crate::serde::SerializedArray; use crate::validity::Validity; #[test] @@ -242,7 +259,7 @@ mod tests { let concat = concat.freeze(); - let parts = SerializedArray::try_from(concat).unwrap(); + let parts = ArrayParts::try_from(concat).unwrap(); let decoded = parts .decode(&dtype, 5, &ReadContext::new(ctx.to_ids()), &LEGACY_SESSION) .unwrap(); @@ -270,7 +287,7 @@ mod tests { concat.extend_from_slice(buf.as_ref()); } - let parts = SerializedArray::try_from(concat.freeze()).unwrap(); + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index a791d79b798..ceff69fa242 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -11,11 +11,11 @@ use vortex_mask::AllOr; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::Dict; use crate::dtype::DType; use crate::dtype::PType; use crate::match_each_integer_ptype; +use crate::stats::ArrayStats; #[derive(Clone, prost::Message)] pub struct DictMetadata { @@ -43,6 +43,8 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["codes", "values"]; #[derive(Debug, Clone)] pub struct DictData { pub(super) slots: Vec>, + pub(super) stats_set: ArrayStats, + pub(super) dtype: DType, /// Indicates whether all dictionary values are definitely referenced by at least one code. /// `true` = all values are referenced (computed during encoding). /// `false` = unknown/might have unreferenced values. @@ -51,9 +53,10 @@ pub struct DictData { pub(super) all_values_referenced: bool, } -pub struct DictDataParts { +pub struct DictArrayParts { pub codes: ArrayRef, pub values: ArrayRef, + pub dtype: DType, } impl DictData { @@ -64,8 +67,13 @@ impl DictData { /// by the safe `DictArray::try_new` constructor are valid, for example when /// you are filtering or slicing an existing valid `DictArray`. pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { + let dtype = values + .dtype() + .union_nullability(codes.dtype().nullability()); Self { slots: vec![Some(codes), Some(values)], + stats_set: Default::default(), + dtype, all_values_referenced: false, } } @@ -111,7 +119,7 @@ impl DictData { /// of the `values` array. Otherwise, this constructor returns an error. /// /// It is an error to provide a nullable `codes` with non-nullable `values`. - pub(crate) fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { + pub fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { if !codes.dtype().is_int() { vortex_bail!(MismatchedTypes: "int", codes.dtype()); } @@ -125,10 +133,8 @@ impl DictData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - self.values() - .dtype() - .union_nullability(self.codes().dtype().nullability()) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns `true` if this array is empty. @@ -136,14 +142,15 @@ impl DictData { self.len() == 0 } - pub fn into_parts(mut self) -> DictDataParts { - DictDataParts { + pub fn into_parts(mut self) -> DictArrayParts { + DictArrayParts { codes: self.slots[CODES_SLOT] .take() .vortex_expect("DictArray codes slot"), values: self.slots[VALUES_SLOT] .take() .vortex_expect("DictArray values slot"), + dtype: self.dtype, } } @@ -197,18 +204,12 @@ impl DictData { impl Array { /// Build a new `DictArray` from its components, `codes` and `values`. pub fn new(codes: ArrayRef, values: ArrayRef) -> Self { - let data = DictData::new(codes, values); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } + Array::try_from_data(DictData::new(codes, values)).vortex_expect("DictData is always valid") } /// Build a new `DictArray` from its components, `codes` and `values`. pub fn try_new(codes: ArrayRef, values: ArrayRef) -> VortexResult { - let data = DictData::try_new(codes, values)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) }) + Array::try_from_data(DictData::try_new(codes, values)?) } /// Build a new `DictArray` without validating the codes or values. @@ -217,10 +218,8 @@ impl Array { /// /// See [`DictData::new_unchecked`]. pub unsafe fn new_unchecked(codes: ArrayRef, values: ArrayRef) -> Self { - let data = unsafe { DictData::new_unchecked(codes, values) }; - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } + Array::try_from_data(unsafe { DictData::new_unchecked(codes, values) }) + .vortex_expect("DictData is always valid") } /// Set whether all values in the dictionary are referenced by at least one code. @@ -229,13 +228,11 @@ impl Array { /// /// See [`DictData::set_all_values_referenced`]. pub unsafe fn set_all_values_referenced(self, all_values_referenced: bool) -> Self { - let dtype = self.dtype().clone(); - let len = self.len(); - let data = unsafe { + Array::try_from_data(unsafe { self.into_data() .set_all_values_referenced(all_values_referenced) - }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Dict, dtype, len, data)) } + }) + .vortex_expect("data is always valid") } } @@ -445,20 +442,18 @@ mod test { #[cfg_attr(miri, ignore)] #[test] fn test_dict_metadata() { - use prost::Message; - use super::DictMetadata; + use crate::ProstMetadata; use crate::test_harness::check_metadata; check_metadata( "dict.metadata", - &DictMetadata { + ProstMetadata(DictMetadata { codes_ptype: PType::U64 as i32, values_len: u32::MAX, is_nullable_codes: None, all_values_referenced: None, - } - .encode_to_vec(), + }), ); } } diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index 25be2053332..f65e2d43412 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use kernel::PARENT_KERNELS; -use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -11,8 +10,8 @@ use vortex_error::vortex_err; use vortex_error::vortex_panic; use vortex_session::VortexSession; +use super::DictArrayParts; use super::DictData; -use super::DictDataParts; use super::DictMetadata; use super::array::NUM_SLOTS; use super::array::SLOT_NAMES; @@ -20,13 +19,16 @@ use super::take_canonical; use crate::AnyCanonical; use crate::ArrayRef; use crate::Canonical; +use crate::DeserializeMetadata; use crate::Precision; +use crate::ProstMetadata; +use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::ConstantArray; use crate::arrays::Primitive; +use crate::arrays::constant::ConstantData; use crate::arrays::dict::compute::rules::PARENT_RULES; use crate::buffer::BufferHandle; use crate::dtype::DType; @@ -39,6 +41,7 @@ use crate::hash::ArrayHash; use crate::require_child; use crate::scalar::Scalar; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::vtable; mod kernel; mod operations; @@ -56,20 +59,28 @@ impl Dict { impl VTable for Dict { type ArrayData = DictData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Dict + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &DictData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!(data.codes().len() == len, "DictArray codes length mismatch"); - vortex_ensure!( - data.dtype() == *dtype, - "DictArray dtype does not match codes/values dtype" - ); - Ok(()) + fn len(array: &DictData) -> usize { + array.codes().len() + } + + fn dtype(array: &DictData) -> &DType { + &array.dtype + } + + fn stats(array: &DictData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &DictData, state: &mut H, precision: Precision) { @@ -94,34 +105,42 @@ impl VTable for Dict { None } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - DictMetadata { - codes_ptype: PType::try_from(array.codes().dtype())? as i32, - values_len: u32::try_from(array.values().len()).map_err(|_| { - vortex_err!( - "Dictionary values size {} overflowed u32", - array.values().len() - ) - })?, - is_nullable_codes: Some(array.codes().dtype().is_nullable()), - all_values_referenced: Some(array.all_values_referenced), - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(DictMetadata { + codes_ptype: PType::try_from(array.codes().dtype())? as i32, + values_len: u32::try_from(array.values().len()).map_err(|_| { + vortex_err!( + "Dictionary values size {} overflowed u32", + array.values().len() + ) + })?, + is_nullable_codes: Some(array.codes().dtype().is_nullable()), + all_values_referenced: Some(array.all_values_referenced), + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + let metadata = ::deserialize(bytes)?; + Ok(ProstMetadata(metadata)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = DictMetadata::decode(metadata)?; if children.len() != 2 { vortex_bail!( "Expected 2 children for dict encoding, found {}", @@ -178,7 +197,7 @@ impl VTable for Dict { // Also not the check to do here it take value validity using code validity, but this approx // is correct. if array.codes().all_invalid()? { - return Ok(ExecutionResult::done(ConstantArray::new( + return Ok(ExecutionResult::done(ConstantData::new( Scalar::null(array.dtype().as_nullable()), array.codes().len(), ))); @@ -186,7 +205,7 @@ impl VTable for Dict { let array = require_child!(array, array.values(), 1 => AnyCanonical); - let DictDataParts { codes, values, .. } = array.into_data().into_parts(); + let DictArrayParts { codes, values, .. } = array.into_data().into_parts(); let codes = codes .try_into::() diff --git a/vortex-array/src/arrays/dict/vtable/validity.rs b/vortex-array/src/arrays/dict/vtable/validity.rs index fc13dcdc84d..cdaeb4a08d5 100644 --- a/vortex-array/src/arrays/dict/vtable/validity.rs +++ b/vortex-array/src/arrays/dict/vtable/validity.rs @@ -7,7 +7,7 @@ use super::Dict; use crate::IntoArray; use crate::array::ArrayView; use crate::array::ValidityVTable; -use crate::arrays::DictArray; +use crate::arrays::dict::DictData; use crate::builtins::ArrayBuiltins; use crate::dtype::Nullability; use crate::scalar::Scalar; @@ -32,13 +32,13 @@ impl ValidityVTable for Dict { // We know codes are all valid, so the cast is free. let codes = array.codes().cast(array.codes().dtype().as_nonnullable())?; Validity::Array( - unsafe { DictArray::new_unchecked(codes, values_validity) }.into_array(), + unsafe { DictData::new_unchecked(codes, values_validity) }.into_array(), ) } (Validity::Array(_codes_validity), Validity::Array(values_validity)) => { // Create a mask representing "is the value at codes[i] valid?" let values_valid_mask = - unsafe { DictArray::new_unchecked(array.codes().clone(), values_validity) } + unsafe { DictData::new_unchecked(array.codes().clone(), values_validity) } .into_array(); let values_valid_mask = values_valid_mask .fill_null(Scalar::bool(false, Nullability::NonNullable))?; diff --git a/vortex-array/src/arrays/extension/array.rs b/vortex-array/src/arrays/extension/array.rs index 757c3042484..f641b1202d4 100644 --- a/vortex-array/src/arrays/extension/array.rs +++ b/vortex-array/src/arrays/extension/array.rs @@ -6,10 +6,10 @@ use vortex_error::VortexResult; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::Extension; use crate::dtype::DType; use crate::dtype::extension::ExtDTypeRef; +use crate::stats::ArrayStats; /// The backing storage array for this extension array. pub(super) const STORAGE_SLOT: usize = 0; @@ -57,8 +57,9 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["storage"]; #[derive(Clone, Debug)] pub struct ExtensionData { /// The storage dtype. This **must** be a [`Extension::DType`] variant. - pub(super) ext_dtype: ExtDTypeRef, + pub(super) dtype: DType, pub(super) slots: Vec>, + pub(super) stats_set: ArrayStats, } impl ExtensionData { @@ -109,8 +110,9 @@ impl ExtensionData { ); Self { - ext_dtype, + dtype: DType::Extension(ext_dtype), slots: vec![Some(storage_array)], + stats_set: ArrayStats::default(), } } @@ -120,8 +122,8 @@ impl ExtensionData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Extension(self.ext_dtype.clone()) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns `true` if this array is empty. @@ -131,7 +133,11 @@ impl ExtensionData { /// The extension dtype of this array. pub fn ext_dtype(&self) -> &ExtDTypeRef { - &self.ext_dtype + let DType::Extension(ext) = &self.dtype else { + unreachable!("ExtensionArray: dtype must be an ExtDType") + }; + + ext } pub fn storage_array(&self) -> &ArrayRef { @@ -148,17 +154,12 @@ impl Array { /// /// Panics if the storage array is not compatible with the extension dtype. pub fn new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> Self { - let dtype = DType::Extension(ext_dtype.clone()); - let len = storage_array.len(); - let data = ExtensionData::new(ext_dtype, storage_array); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Extension, dtype, len, data)) } + Array::try_from_data(ExtensionData::new(ext_dtype, storage_array)) + .vortex_expect("ExtensionData is always valid") } /// Tries to construct a new `ExtensionArray`. pub fn try_new(ext_dtype: ExtDTypeRef, storage_array: ArrayRef) -> VortexResult { - let dtype = DType::Extension(ext_dtype.clone()); - let len = storage_array.len(); - let data = ExtensionData::try_new(ext_dtype, storage_array)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Extension, dtype, len, data)) }) + Array::try_from_data(ExtensionData::try_new(ext_dtype, storage_array)?) } } diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index 31ceb8ccf8f..161592ad264 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -79,7 +79,7 @@ mod tests { let storage = Buffer::::empty().into_array(); let arr = ExtensionArray::new(ext_dtype.clone(), storage); - assert!(!arr.dtype().is_nullable()); + assert!(!arr.dtype.is_nullable()); let new_dtype = DType::Extension(ext_dtype).with_nullability(Nullability::Nullable); diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index 3ef3bc05617..7eb48c10228 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; +use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -30,6 +31,7 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::vtable; vtable!(Extension, Extension, ExtensionData); @@ -37,13 +39,30 @@ vtable!(Extension, Extension, ExtensionData); impl VTable for Extension { type ArrayData = ExtensionData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Extension + } + fn id(&self) -> ArrayId { Self::ID } + fn len(array: &ExtensionData) -> usize { + array.storage_array().len() + } + + fn dtype(array: &ExtensionData) -> &DType { + &array.dtype + } + + fn stats(array: &ExtensionData) -> &ArrayStats { + &array.stats_set + } + fn array_hash( array: &ExtensionData, state: &mut H, @@ -78,45 +97,31 @@ impl VTable for Extension { SLOT_NAMES[idx].to_string() } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(vec![])) + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) } - fn validate(&self, data: &ExtensionData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "ExtensionArray length {} does not match outer length {}", - data.len(), - len - ); - - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "ExtensionArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); - - Ok(()) + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + _metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - if !metadata.is_empty() { - vortex_bail!( - "ExtensionArray expects empty metadata, got {} bytes", - metadata.len() - ); - } let DType::Extension(ext_dtype) = dtype else { vortex_bail!("Not an extension DType"); }; diff --git a/vortex-array/src/arrays/filter/array.rs b/vortex-array/src/arrays/filter/array.rs index 7a0677d5d92..04f4312ca1f 100644 --- a/vortex-array/src/arrays/filter/array.rs +++ b/vortex-array/src/arrays/filter/array.rs @@ -8,15 +8,23 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::Filter; use crate::dtype::DType; +use crate::stats::ArrayStats; /// The source array being filtered. pub(super) const CHILD_SLOT: usize = 0; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; +/// Decomposed parts of the filter array. +pub struct FilterArrayParts { + /// Child array that is filtered by the mask + pub child: ArrayRef, + /// Mask to apply at filter time. Child elements with set indices are kept, the rest discarded. + pub mask: Mask, +} + // TODO(connor): Write docs on why we have this, and what we had in the old world so that the future // does not repeat the mistakes of the past. /// A lazy array that represents filtering a child array by a boolean [`Mask`]. @@ -28,11 +36,9 @@ pub struct FilterData { /// The boolean mask selecting which elements to keep. pub(super) mask: Mask, -} -pub struct FilterDataParts { - pub child: ArrayRef, - pub mask: Mask, + /// The stats for this array. + pub(super) stats: ArrayStats, } impl FilterData { @@ -40,7 +46,7 @@ impl FilterData { Self::try_new(array, mask).vortex_expect("FilterArray construction failed") } - fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { + pub fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { vortex_ensure_eq!( array.len(), mask.len(), @@ -52,6 +58,7 @@ impl FilterData { Ok(Self { slots: vec![Some(array)], mask, + stats: ArrayStats::default(), }) } @@ -81,31 +88,29 @@ impl FilterData { pub fn filter_mask(&self) -> &Mask { &self.mask } - - pub fn into_parts(mut self) -> FilterDataParts { - FilterDataParts { - child: self.slots[CHILD_SLOT] - .take() - .vortex_expect("FilterArray child slot"), - mask: self.mask, - } - } } impl Array { /// Creates a new `FilterArray`. pub fn new(array: ArrayRef, mask: Mask) -> Self { - let dtype = array.dtype().clone(); - let len = mask.true_count(); - let data = FilterData::new(array, mask); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Filter, dtype, len, data)) } + Array::try_from_data(FilterData::new(array, mask)) + .vortex_expect("FilterData is always valid") } /// Constructs a new `FilterArray`. pub fn try_new(array: ArrayRef, mask: Mask) -> VortexResult { - let dtype = array.dtype().clone(); - let len = mask.true_count(); - let data = FilterData::try_new(array, mask)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Filter, dtype, len, data)) }) + Array::try_from_data(FilterData::try_new(array, mask)?) + } +} + +impl FilterData { + /// Consume the array and return its individual components. + pub fn into_parts(mut self) -> FilterArrayParts { + FilterArrayParts { + child: self.slots[CHILD_SLOT] + .take() + .vortex_expect("FilterArray child slot"), + mask: self.mask, + } } } diff --git a/vortex-array/src/arrays/filter/execute/struct_.rs b/vortex-array/src/arrays/filter/execute/struct_.rs index e424dbd2d33..74540a8739d 100644 --- a/vortex-array/src/arrays/filter/execute/struct_.rs +++ b/vortex-array/src/arrays/filter/execute/struct_.rs @@ -29,8 +29,13 @@ pub fn filter_struct(array: &StructArray, mask: &Arc) -> StructArray .map(|a| a.len()) .unwrap_or_else(|| mask.true_count()); - StructArray::try_new_with_dtype(fields, array.struct_fields(), length, filtered_validity) - .vortex_expect("filtered StructArray fields have consistent lengths") + StructArray::try_new_with_dtype( + fields, + array.struct_fields().clone(), + length, + filtered_validity, + ) + .vortex_expect("filtered StructArray fields have consistent lengths") } #[cfg(test)] diff --git a/vortex-array/src/arrays/filter/mod.rs b/vortex-array/src/arrays/filter/mod.rs index 7491899230e..d10cae11e41 100644 --- a/vortex-array/src/arrays/filter/mod.rs +++ b/vortex-array/src/arrays/filter/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::FilterArrayParts; pub use array::FilterData; -pub use array::FilterDataParts; pub use vtable::FilterArray; mod execute; diff --git a/vortex-array/src/arrays/filter/rules.rs b/vortex-array/src/arrays/filter/rules.rs index 0095d47b422..402a7b14585 100644 --- a/vortex-array/src/arrays/filter/rules.rs +++ b/vortex-array/src/arrays/filter/rules.rs @@ -11,7 +11,7 @@ use crate::array::ArrayView; use crate::arrays::Filter; use crate::arrays::Struct; use crate::arrays::StructArray; -use crate::arrays::struct_::StructDataParts; +use crate::arrays::struct_::StructArrayParts; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; use crate::optimizer::rules::ParentRuleSet; @@ -69,7 +69,7 @@ impl ArrayReduceRule for FilterStructRule { }; let len = mask.true_count(); - let StructDataParts { + let StructArrayParts { fields, struct_fields, validity, diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index fb7bdbf9d0b..42448d7ddd9 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use std::fmt::Debug; +use std::fmt::Formatter; use std::hash::Hasher; use vortex_error::VortexResult; @@ -34,6 +36,7 @@ use crate::executor::ExecutionCtx; use crate::executor::ExecutionResult; use crate::scalar::Scalar; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -48,33 +51,27 @@ impl Filter { impl VTable for Filter { type ArrayData = FilterData; + type Metadata = FilterMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &FilterData) -> &Self { + &Filter + } fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.child().dtype() == dtype, - "FilterArray dtype {} does not match outer dtype {}", - data.child().dtype(), - dtype - ); - vortex_ensure!( - data.len() == len, - "FilterArray length {} does not match outer length {}", - data.len(), - len - ); - vortex_ensure!( - data.child().len() == data.mask.len(), - "FilterArray child length {} does not match mask length {}", - data.child().len(), - data.mask.len() - ); - Ok(()) + fn len(array: &FilterData) -> usize { + array.mask.true_count() + } + + fn dtype(array: &FilterData) -> &DType { + array.child().dtype() + } + + fn stats(array: &FilterData) -> &ArrayStats { + &array.stats } fn array_hash(array: &FilterData, state: &mut H, precision: Precision) { @@ -107,24 +104,37 @@ impl VTable for Filter { SLOT_NAMES[idx].to_string() } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(FilterMetadata(array.mask.clone())) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { // TODO(joe): make this configurable vortex_bail!("Filter array is not serializable") } fn deserialize( - &self, + _bytes: &[u8], _dtype: &DType, _len: usize, - _metadata: &[u8], - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult { vortex_bail!("Filter array is not serializable") } + fn build( + dtype: &DType, + len: usize, + metadata: &FilterMetadata, + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + ) -> VortexResult { + assert_eq!(len, metadata.0.true_count()); + let child = children.get(0, dtype, metadata.0.len())?; + FilterData::try_new(child, metadata.0.clone()) + } + fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { vortex_ensure!( slots.len() == NUM_SLOTS, @@ -179,3 +189,17 @@ impl ValidityVTable for Filter { array.child().validity()?.filter(&array.mask) } } + +pub struct FilterMetadata(pub(super) Mask); + +impl Debug for FilterMetadata { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} / {} => {}", + self.0.true_count(), + self.0.len(), + self.0.density() + ) + } +} diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index eb1ef6d7aff..9d5dda32f1b 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -9,12 +9,11 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::FixedSizeList; use crate::dtype::DType; -use crate::dtype::Nullability; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The `elements` data array, where each fixed-size list scalar is a _slice_ of the `elements` @@ -81,8 +80,10 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "validity"]; /// ``` #[derive(Clone, Debug)] pub struct FixedSizeListData { - /// The nullability of the fixed-size list array. - pub(super) nullability: Nullability, + /// The [`DType`] of the fixed-size list. + /// + /// This type **must** be the variant [`DType::FixedSizeList`]. + pub(super) dtype: DType, /// Slots holding [elements]. pub(super) slots: Vec>, @@ -99,7 +100,10 @@ pub struct FixedSizeListData { /// The main reason we need to store this (rather than calculate it on the fly via `list_size` /// and `elements.len()`) is because in the degenerate case where `list_size == 0`, we cannot /// use `0 / 0` to determine the length. - pub(super) degenerate_len: usize, + pub(super) len: usize, + + /// The stats for this array. + pub(super) stats_set: ArrayStats, } impl FixedSizeListData { @@ -122,7 +126,7 @@ impl FixedSizeListData { /// /// Returns an error if the provided components do not satisfy the invariants documented /// in `FixedSizeListArray::new_unchecked`. - pub(crate) fn try_new( + pub fn try_new( elements: ArrayRef, list_size: u32, validity: Validity, @@ -164,22 +168,22 @@ impl FixedSizeListData { let validity_slot = validity_to_child(&validity, len); Self { - nullability, + dtype: DType::FixedSizeList(Arc::new(elements.dtype().clone()), list_size, nullability), slots: vec![Some(elements), validity_slot], list_size, - degenerate_len: if list_size == 0 { len } else { 0 }, + len, + stats_set: Default::default(), } } pub fn into_parts(mut self) -> (ArrayRef, Validity, DType) { - let dtype = self.dtype(); let validity = self.validity(); ( self.slots[ELEMENTS_SLOT] .take() .vortex_expect("FixedSizeListArray elements slot"), validity, - dtype, + self.dtype, ) } @@ -220,21 +224,13 @@ impl FixedSizeListData { } /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - DType::FixedSizeList( - Arc::new(self.elements().dtype().clone()), - self.list_size, - self.nullability, - ) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns the length of the array. pub fn len(&self) -> usize { - if self.list_size == 0 { - self.degenerate_len - } else { - self.elements().len() / self.list_size as usize - } + self.len } /// Returns `true` if the array is empty. @@ -245,7 +241,7 @@ impl FixedSizeListData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -269,13 +265,8 @@ impl FixedSizeListData { impl Array { /// Creates a new `FixedSizeListArray`. pub fn new(elements: ArrayRef, list_size: u32, validity: Validity, len: usize) -> Self { - let dtype = DType::FixedSizeList( - Arc::new(elements.dtype().clone()), - list_size, - validity.nullability(), - ); - let data = FixedSizeListData::new(elements, list_size, validity, len); - unsafe { Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) } + Array::try_from_data(FixedSizeListData::new(elements, list_size, validity, len)) + .vortex_expect("FixedSizeListData is always valid") } /// Constructs a new `FixedSizeListArray`. @@ -285,17 +276,9 @@ impl Array { validity: Validity, len: usize, ) -> VortexResult { - let dtype = DType::FixedSizeList( - Arc::new(elements.dtype().clone()), - list_size, - validity.nullability(), - ); - let data = FixedSizeListData::try_new(elements, list_size, validity, len)?; - Ok( - unsafe { - Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) - }, - ) + Array::try_from_data(FixedSizeListData::try_new( + elements, list_size, validity, len, + )?) } /// Creates a new `FixedSizeListArray` without validation. @@ -309,23 +292,20 @@ impl Array { validity: Validity, len: usize, ) -> Self { - let dtype = DType::FixedSizeList( - Arc::new(elements.dtype().clone()), - list_size, - validity.nullability(), - ); - let data = unsafe { FixedSizeListData::new_unchecked(elements, list_size, validity, len) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(FixedSizeList, dtype, len, data)) } + Array::try_from_data(unsafe { + FixedSizeListData::new_unchecked(elements, list_size, validity, len) + }) + .vortex_expect("FixedSizeListData is always valid") } } impl FixedSizeListData { pub fn fixed_size_list_elements_at(&self, index: usize) -> VortexResult { debug_assert!( - index < self.len(), + index < self.len, "index {} out of bounds: the len is {}", index, - self.len(), + self.len, ); debug_assert!(self.validity().is_valid(index).unwrap_or(false)); diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index ab3e468c187..31fa5342185 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -10,6 +10,7 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; +use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -26,6 +27,7 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod kernel; @@ -44,13 +46,29 @@ impl FixedSizeList { impl VTable for FixedSizeList { type ArrayData = FixedSizeListData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &FixedSizeListData) -> &Self { + &FixedSizeList + } fn id(&self) -> ArrayId { Self::ID } + fn len(array: &FixedSizeListData) -> usize { + array.len + } + + fn dtype(array: &FixedSizeListData) -> &DType { + &array.dtype + } + + fn stats(array: &FixedSizeListData) -> &ArrayStats { + &array.stats_set + } + fn array_hash( array: &FixedSizeListData, state: &mut H, @@ -100,45 +118,34 @@ impl VTable for FixedSizeList { Self::PARENT_KERNELS.execute(array, parent, child_idx, ctx) } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(vec![])) + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) } - fn validate(&self, data: &FixedSizeListData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "FixedSizeListArray length {} does not match outer length {}", - data.len(), - len - ); - - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "FixedSizeListArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); - - Ok(()) + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + + /// Builds a [`FixedSizeListArray`]. + /// + /// This method expects 1 or 2 children (a second child indicates a validity array). + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + _metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - if !metadata.is_empty() { - vortex_bail!( - "FixedSizeListArray expects empty metadata, got {} bytes", - metadata.len() - ); - } vortex_ensure!( buffers.is_empty(), "`FixedSizeList::build` expects no buffers" diff --git a/vortex-array/src/arrays/lazy_patched/mod.rs b/vortex-array/src/arrays/lazy_patched/mod.rs new file mode 100644 index 00000000000..7f2d1d29cf2 --- /dev/null +++ b/vortex-array/src/arrays/lazy_patched/mod.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +mod vtable; + +pub use vtable::*; diff --git a/vortex-array/src/arrays/lazy_patched/vtable/mod.rs b/vortex-array/src/arrays/lazy_patched/vtable/mod.rs new file mode 100644 index 00000000000..cb56174fae4 --- /dev/null +++ b/vortex-array/src/arrays/lazy_patched/vtable/mod.rs @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +mod operations; +mod validity; + +use std::hash::Hasher; + +use vortex_error::VortexExpect; +use vortex_error::VortexResult; +use vortex_error::vortex_ensure; +use vortex_error::vortex_ensure_eq; +use vortex_error::vortex_err; +use vortex_error::vortex_panic; +use vortex_session::VortexSession; + +use crate::ArrayEq; +use crate::ArrayHash; +use crate::ArrayRef; +use crate::DeserializeMetadata; +use crate::ExecutionCtx; +use crate::ExecutionResult; +use crate::IntoArray; +use crate::Precision; +use crate::ProstMetadata; +use crate::SerializeMetadata; +use crate::array::Array; +use crate::array::ArrayId; +use crate::array::ArrayView; +use crate::array::VTable; +use crate::array::ValidityVTableFromChild; +use crate::arrays::PatchedArray; +use crate::buffer::BufferHandle; +use crate::dtype::DType; +use crate::patches::Patches; +use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; +use crate::vtable; + +#[derive(Clone, Debug)] +pub struct LazyPatched; + +vtable!(LazyPatched, LazyPatched, LazyPatchedData); + +#[derive(Clone, prost::Message)] +pub struct LazyPatchedMetadata { + #[prost(uint32, tag = "1")] + pub(crate) num_patches: u32, + #[prost(uint32, tag = "2")] + pub(crate) offset: u32, +} + +impl LazyPatched { + pub const ID: ArrayId = ArrayId::new_ref("vortex.patched_lazy"); +} + +const INNER_SLOT: usize = 0; +const PATCH_INDICES_SLOT: usize = 1; +const PATCH_VALUES_SLOT: usize = 2; +const NUM_SLOTS: usize = 3; +const SLOT_NAMES: [&str; NUM_SLOTS] = ["inner", "patch_indices", "patch_values"]; + +impl VTable for LazyPatched { + type ArrayData = LazyPatchedData; + type Metadata = ProstMetadata; + + type OperationsVTable = Self; + type ValidityVTable = ValidityVTableFromChild; + + fn vtable(_array: &Self::ArrayData) -> &Self { + &LazyPatched + } + + fn id(&self) -> ArrayId { + Self::ID + } + + fn len(array: &Self::ArrayData) -> usize { + array.inner().len() + } + + fn dtype(array: &Self::ArrayData) -> &DType { + array.inner().dtype() + } + + fn stats(array: &Self::ArrayData) -> &ArrayStats { + &array.stats + } + + fn array_hash(array: &Self::ArrayData, state: &mut H, precision: Precision) { + array.slots[INNER_SLOT] + .as_ref() + .vortex_expect("present") + .array_hash(state, precision); + array.slots[PATCH_INDICES_SLOT] + .as_ref() + .vortex_expect("present") + .array_hash(state, precision); + array.slots[PATCH_VALUES_SLOT] + .as_ref() + .vortex_expect("present") + .array_hash(state, precision); + } + + fn array_eq(array: &Self::ArrayData, other: &Self::ArrayData, precision: Precision) -> bool { + array.inner().array_eq(other.inner(), precision) + && array.patches().array_eq(&other.patches(), precision) + } + + fn nbuffers(_array: ArrayView<'_, Self>) -> usize { + 0 + } + + fn buffer(_array: ArrayView<'_, Self>, _idx: usize) -> BufferHandle { + vortex_panic!("LazyPatched array holds no buffers") + } + + fn buffer_name(_array: ArrayView<'_, Self>, _idx: usize) -> Option { + vortex_panic!("LazyPatched array holds no buffers") + } + + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + let num_patches = u32::try_from(array.num_patches())?; + let offset = u32::try_from(array.offset)?; + + Ok(ProstMetadata(LazyPatchedMetadata { + num_patches, + offset, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) + } + + fn deserialize( + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + let deserialized = ::deserialize(bytes)?; + Ok(ProstMetadata(deserialized)) + } + + fn build( + dtype: &DType, + len: usize, + metadata: &Self::Metadata, + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + ) -> VortexResult { + // There should be 3 children + // 1. inner + // 2. patch_indices + // 3. patch_values + vortex_ensure!( + children.len() == 3, + "expected exactly 3 children from LazyPatched, found {}", + children.len() + ); + + let inner = children.get(0, dtype, len)?; + + let num_patches = metadata.num_patches as usize; + let offset = metadata.offset as usize; + let patch_indices = children.get(1, dtype, num_patches)?; + let patch_values = children.get(2, dtype, num_patches)?; + + let slots = vec![Some(inner), Some(patch_indices), Some(patch_values)]; + + Ok(LazyPatchedData { + slots, + offset, + stats: ArrayStats::default(), + }) + } + + fn slots(array: ArrayView<'_, Self>) -> &[Option] { + &array.data().slots + } + + fn slot_name(_array: ArrayView<'_, Self>, idx: usize) -> String { + SLOT_NAMES[idx].to_string() + } + + fn with_slots( + array: &mut Self::ArrayData, + mut slots: Vec>, + ) -> VortexResult<()> { + vortex_ensure_eq!(slots.len(), NUM_SLOTS); + + array.slots[INNER_SLOT] = Some( + slots + .remove(0) + .ok_or_else(|| vortex_err!("inner slot required"))?, + ); + + array.slots[PATCH_INDICES_SLOT] = Some( + slots + .remove(0) + .ok_or_else(|| vortex_err!("patch_indices slot required"))?, + ); + array.slots[PATCH_VALUES_SLOT] = Some( + slots + .remove(0) + .ok_or_else(|| vortex_err!("patch_values slot required"))?, + ); + + Ok(()) + } + + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { + // Execution => actually transpose the patches, get back a `PatchedArray`. + let patched = + PatchedArray::from_array_and_patches(array.inner().clone(), &array.patches(), ctx)? + .into_array(); + + Ok(ExecutionResult::done(patched)) + } +} + +#[derive(Debug, Clone)] +pub struct LazyPatchedData { + /// Slots. Contains the inner, the patch_indices and patch_values. + /// All slots must be occupied. + pub(crate) slots: Vec>, + /// Offset into the patches. + pub(crate) offset: usize, + + pub(crate) stats: ArrayStats, +} + +impl LazyPatchedData { + /// Create a new `LazyPatchedData` from an inner array and an aligned set of [`Patches`]. + /// + /// # Errors + /// + /// Returns an error if the patches are not aligned to the array, i.e. the `array_len` of + /// the patches does not equal the length of the inner array. + pub fn try_new(inner: ArrayRef, patches: Patches) -> VortexResult { + vortex_ensure_eq!( + inner.len(), + patches.array_len(), + "Patches array_len does not match array len" + ); + + vortex_ensure_eq!( + inner.dtype(), + patches.dtype(), + "Array and Patches types must match" + ); + + let offset = patches.offset(); + let slots = vec![ + Some(inner), + Some(patches.indices().clone()), + Some(patches.values().clone()), + ]; + + Ok(Self { + slots, + offset, + stats: ArrayStats::default(), + }) + } + + pub(crate) fn inner(&self) -> &ArrayRef { + self.slots[INNER_SLOT] + .as_ref() + .vortex_expect("always occupied") + } + + pub(crate) fn patches(&self) -> Patches { + let patch_indices = self.slots[PATCH_INDICES_SLOT] + .clone() + .vortex_expect("must be occupied"); + let patch_values = self.slots[PATCH_VALUES_SLOT] + .clone() + .vortex_expect("must be occupied"); + + // SAFETY: the components are shredded from an original Patches at construction time, + // we are just re-assembling them without modification. + unsafe { + Patches::new_unchecked( + self.inner().len(), + self.offset, + patch_indices, + patch_values, + None, + None, + ) + } + } + + pub(crate) fn num_patches(&self) -> usize { + self.slots[PATCH_INDICES_SLOT] + .as_ref() + .vortex_expect("must be occupied") + .len() + } +} diff --git a/vortex-array/src/arrays/lazy_patched/vtable/operations.rs b/vortex-array/src/arrays/lazy_patched/vtable/operations.rs new file mode 100644 index 00000000000..2cf3dc528b0 --- /dev/null +++ b/vortex-array/src/arrays/lazy_patched/vtable/operations.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_error::VortexResult; + +use crate::ExecutionCtx; +use crate::array::ArrayView; +use crate::array::OperationsVTable; +use crate::arrays::lazy_patched::LazyPatched; +use crate::scalar::Scalar; + +impl OperationsVTable for LazyPatched { + fn scalar_at( + array: ArrayView<'_, LazyPatched>, + index: usize, + _ctx: &mut ExecutionCtx, + ) -> VortexResult { + Ok(if let Some(scalar) = array.patches().get_patched(index)? { + scalar + } else { + array.inner().scalar_at(index)? + }) + } +} diff --git a/vortex-array/src/arrays/lazy_patched/vtable/validity.rs b/vortex-array/src/arrays/lazy_patched/vtable/validity.rs new file mode 100644 index 00000000000..03fa21fbdd4 --- /dev/null +++ b/vortex-array/src/arrays/lazy_patched/vtable/validity.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use crate::ArrayRef; +use crate::array::ValidityChild; +use crate::arrays::lazy_patched::LazyPatched; +use crate::arrays::lazy_patched::LazyPatchedData; + +impl ValidityChild for LazyPatched { + fn validity_child(array: &LazyPatchedData) -> &ArrayRef { + array.inner() + } +} diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index b8f99c35804..2678ab551e2 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -16,7 +16,6 @@ use crate::LEGACY_SESSION; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::min_max::min_max; use crate::array::Array; -use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ConstantArray; @@ -25,10 +24,10 @@ use crate::arrays::Primitive; use crate::builtins::ArrayBuiltins; use crate::dtype::DType; use crate::dtype::NativePType; -use crate::dtype::Nullability; use crate::match_each_integer_ptype; use crate::match_each_native_ptype; use crate::scalar_fn::fns::operators::Operator; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The elements data array containing all list elements concatenated together. @@ -94,15 +93,16 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "validi /// ``` #[derive(Clone, Debug)] pub struct ListData { - pub(super) nullability: Nullability, + pub(super) dtype: DType, pub(super) slots: Vec>, + pub(super) stats_set: ArrayStats, } -pub struct ListDataParts { +pub struct ListArrayParts { + pub dtype: DType, pub elements: ArrayRef, pub offsets: ArrayRef, pub validity: Validity, - pub dtype: DType, } impl ListData { @@ -124,7 +124,7 @@ impl ListData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `ListArray::new_unchecked`. - pub(crate) fn try_new( + pub fn try_new( elements: ArrayRef, offsets: ArrayRef, validity: Validity, @@ -160,8 +160,9 @@ impl ListData { let validity_slot = validity_to_child(&validity, len); Self { - nullability: validity.nullability(), + dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), slots: vec![Some(elements), Some(offsets), validity_slot], + stats_set: Default::default(), } } @@ -249,9 +250,25 @@ impl ListData { Ok(()) } + + /// Splits an array into its parts + pub fn into_parts(mut self) -> ListArrayParts { + let validity = self.validity(); + ListArrayParts { + dtype: self.dtype, + elements: self.slots[ELEMENTS_SLOT] + .take() + .vortex_expect("ListArray elements slot"), + offsets: self.slots[OFFSETS_SLOT] + .take() + .vortex_expect("ListArray offsets slot"), + validity, + } + } + /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - DType::List(Arc::new(self.elements().dtype().clone()), self.nullability) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns the length of the array. @@ -267,7 +284,7 @@ impl ListData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -323,8 +340,11 @@ impl ListData { } /// Returns the element dtype of the list array. - pub fn element_dtype(&self) -> &DType { - self.elements().dtype() + pub fn element_dtype(&self) -> &Arc { + match &self.dtype { + DType::List(element_dtype, _) => element_dtype, + _ => vortex_panic!("ListArray has invalid dtype {}", self.dtype), + } } /// Returns the elements array. @@ -334,21 +354,6 @@ impl ListData { .vortex_expect("ListArray elements slot") } - pub fn into_parts(mut self) -> ListDataParts { - let dtype = self.dtype(); - let validity = self.validity(); - ListDataParts { - elements: self.slots[ELEMENTS_SLOT] - .take() - .vortex_expect("ListArray elements slot"), - offsets: self.slots[OFFSETS_SLOT] - .take() - .vortex_expect("ListArray offsets slot"), - validity, - dtype, - } - } - // TODO(connor)[ListView]: Create 2 functions `reset_offsets` and `recursive_reset_offsets`, // where `reset_offsets` is infallible. // Also, `reset_offsets` can be made more efficient by replacing `sub_scalar` with a match on @@ -358,10 +363,8 @@ impl ListData { impl Array { /// Creates a new `ListArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { - let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); - let len = offsets.len().saturating_sub(1); - let data = ListData::new(elements, offsets, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } + Array::try_from_data(ListData::new(elements, offsets, validity)) + .vortex_expect("ListData is always valid") } /// Constructs a new `ListArray`. @@ -370,10 +373,7 @@ impl Array { offsets: ArrayRef, validity: Validity, ) -> VortexResult { - let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); - let len = offsets.len().saturating_sub(1); - let data = ListData::try_new(elements, offsets, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) }) + Array::try_from_data(ListData::try_new(elements, offsets, validity)?) } /// Creates a new `ListArray` without validation. @@ -382,10 +382,8 @@ impl Array { /// /// See [`ListData::new_unchecked`]. pub unsafe fn new_unchecked(elements: ArrayRef, offsets: ArrayRef, validity: Validity) -> Self { - let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); - let len = offsets.len().saturating_sub(1); - let data = unsafe { ListData::new_unchecked(elements, offsets, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } + Array::try_from_data(unsafe { ListData::new_unchecked(elements, offsets, validity) }) + .vortex_expect("ListData is always valid") } } @@ -395,12 +393,7 @@ impl ListData { if recurse && elements.is_canonical() { elements = elements.to_canonical()?.compact()?.into_array(); } else if recurse && let Some(child_list_array) = elements.as_opt::() { - let data = child_list_array.reset_offsets(recurse)?; - let dtype = data.dtype(); - let len = data.len(); - elements = - unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) } - .into_array(); + elements = child_list_array.reset_offsets(recurse)?.into_array(); } let offsets = self.offsets(); diff --git a/vortex-array/src/arrays/list/mod.rs b/vortex-array/src/arrays/list/mod.rs index 3147097ca80..753a2a35758 100644 --- a/vortex-array/src/arrays/list/mod.rs +++ b/vortex-array/src/arrays/list/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::ListArrayParts; pub use array::ListData; -pub use array::ListDataParts; pub use vtable::ListArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/list/tests.rs b/vortex-array/src/arrays/list/tests.rs index f815acb3bb0..874be33275b 100644 --- a/vortex-array/src/arrays/list/tests.rs +++ b/vortex-array/src/arrays/list/tests.rs @@ -14,8 +14,6 @@ use crate::Canonical; use crate::IntoArray; use crate::LEGACY_SESSION; use crate::VortexSessionExecute; -use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::FilterArray; use crate::arrays::List; use crate::arrays::PrimitiveArray; @@ -915,23 +913,9 @@ fn test_recursive_compact_list_of_lists() { assert_eq!(recursive_flat_elements.len(), 7); // Verify data integrity is preserved - let non_recursive_array = Array::try_from_parts(ArrayParts::new( - List, - non_recursive.dtype(), - non_recursive.len(), - non_recursive.clone(), - )) - .unwrap(); - let recursive_array = Array::try_from_parts(ArrayParts::new( - List, - recursive.dtype(), - recursive.len(), - recursive.clone(), - )) - .unwrap(); assert_eq!( - non_recursive_array.scalar_at(0).unwrap(), - recursive_array.scalar_at(0).unwrap() + non_recursive.clone().into_array().scalar_at(0).unwrap(), + recursive.clone().into_array().scalar_at(0).unwrap() ); } diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index dd999ee3419..051ee80d2d0 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use prost::Message; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -13,6 +12,7 @@ use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; use crate::Precision; +use crate::ProstMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -29,7 +29,10 @@ use crate::dtype::Nullability; use crate::dtype::PType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; +use crate::metadata::DeserializeMetadata; +use crate::metadata::SerializeMetadata; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod operations; @@ -47,13 +50,29 @@ pub struct ListMetadata { impl VTable for List { type ArrayData = ListData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &ListData) -> &Self { + &List + } fn id(&self) -> ArrayId { Self::ID } + fn len(array: &ListData) -> usize { + array.offsets().len().saturating_sub(1) + } + + fn dtype(array: &ListData) -> &DType { + &array.dtype + } + + fn stats(array: &ListData) -> &ArrayStats { + &array.stats_set + } + fn array_hash(array: &ListData, state: &mut H, precision: Precision) { array.elements().array_hash(state, precision); array.offsets().array_hash(state, precision); @@ -86,46 +105,36 @@ impl VTable for List { PARENT_RULES.evaluate(array, parent, child_idx) } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - ListMetadata { - elements_len: array.elements().len() as u64, - offset_ptype: PType::try_from(array.offsets().dtype())? as i32, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(ListMetadata { + elements_len: array.elements().len() as u64, + offset_ptype: PType::try_from(array.offsets().dtype())? as i32, + })) } - fn validate(&self, data: &ListData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "ListArray length {} does not match outer length {}", - data.len(), - len - ); - - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "ListArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); - - Ok(()) + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(SerializeMetadata::serialize(metadata))) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata( + as DeserializeMetadata>::deserialize(bytes)?, + )) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = ListMetadata::decode(metadata)?; let validity = if children.len() == 2 { Validity::from(dtype.nullability()) } else if children.len() == 3 { @@ -141,12 +150,12 @@ impl VTable for List { let elements = children.get( 0, element_dtype.as_ref(), - usize::try_from(metadata.elements_len)?, + usize::try_from(metadata.0.elements_len)?, )?; let offsets = children.get( 1, - &DType::Primitive(metadata.offset_ptype(), Nullability::NonNullable), + &DType::Primitive(metadata.0.offset_ptype(), Nullability::NonNullable), len + 1, )?; diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index c25dce40f43..7a8c7122d4c 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -13,7 +13,6 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::ListView; @@ -22,8 +21,8 @@ use crate::arrays::PrimitiveArray; use crate::arrays::bool; use crate::dtype::DType; use crate::dtype::IntegerPType; -use crate::dtype::Nullability; use crate::match_each_integer_ptype; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The `elements` data array, where each list scalar is a _slice_ of the `elements` array, and @@ -108,8 +107,10 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["elements", "offsets", "sizes" /// [`ListArray`]: crate::arrays::ListArray #[derive(Clone, Debug)] pub struct ListViewData { - /// The nullability of the list array. - pub(super) nullability: Nullability, + /// The [`DType`] of the list array. + /// + /// This type **must** be the variant [`DType::List`]. + pub(super) dtype: DType, /// Slots holding [elements, offsets, sizes]. pub(super) slots: Vec>, @@ -123,9 +124,12 @@ pub struct ListViewData { /// `offsets[i] + sizes[i]` are in order), conversions can bypass the very expensive rebuild /// process which must rebuild the array from scratch. is_zero_copy_to_list: bool, + + /// The stats for this array. + pub(super) stats_set: ArrayStats, } -pub struct ListViewDataParts { +pub struct ListViewArrayParts { pub elements_dtype: Arc, /// See `ListViewArray::elements` @@ -171,9 +175,10 @@ impl ListViewData { let validity_slot = validity_to_child(&validity, len); Ok(Self { - nullability: validity.nullability(), + dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], is_zero_copy_to_list: false, + stats_set: Default::default(), }) } @@ -211,9 +216,10 @@ impl ListViewData { let validity_slot = validity_to_child(&validity, len); Self { - nullability: validity.nullability(), + dtype: DType::List(Arc::new(elements.dtype().clone()), validity.nullability()), slots: vec![Some(elements), Some(offsets), Some(sizes), validity_slot], is_zero_copy_to_list: false, + stats_set: Default::default(), } } @@ -337,10 +343,11 @@ impl ListViewData { .is_ok() } - pub fn into_parts(mut self) -> ListViewDataParts { + pub fn into_parts(mut self) -> ListViewArrayParts { let validity = self.validity(); - ListViewDataParts { - elements_dtype: Arc::new(self.elements().dtype().clone()), + let dtype = self.dtype.into_list_element_opt().vortex_expect("is list"); + ListViewArrayParts { + elements_dtype: dtype, elements: self.slots[ELEMENTS_SLOT] .take() .vortex_expect("ListViewArray elements slot"), @@ -355,8 +362,8 @@ impl ListViewData { } /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - DType::List(Arc::new(self.elements().dtype().clone()), self.nullability) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns the length of the array. @@ -373,7 +380,7 @@ impl ListViewData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -478,10 +485,8 @@ impl ListViewData { impl Array { /// Creates a new `ListViewArray`. pub fn new(elements: ArrayRef, offsets: ArrayRef, sizes: ArrayRef, validity: Validity) -> Self { - let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); - let len = offsets.len(); - let data = ListViewData::new(elements, offsets, sizes, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } + Array::try_from_data(ListViewData::new(elements, offsets, sizes, validity)) + .vortex_expect("ListViewData is always valid") } /// Constructs a new `ListViewArray`. @@ -491,10 +496,7 @@ impl Array { sizes: ArrayRef, validity: Validity, ) -> VortexResult { - let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); - let len = offsets.len(); - let data = ListViewData::try_new(elements, offsets, sizes, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) }) + Array::try_from_data(ListViewData::try_new(elements, offsets, sizes, validity)?) } /// Creates a new `ListViewArray` without validation. @@ -508,10 +510,10 @@ impl Array { sizes: ArrayRef, validity: Validity, ) -> Self { - let dtype = DType::List(Arc::new(elements.dtype().clone()), validity.nullability()); - let len = offsets.len(); - let data = unsafe { ListViewData::new_unchecked(elements, offsets, sizes, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } + Array::try_from_data(unsafe { + ListViewData::new_unchecked(elements, offsets, sizes, validity) + }) + .vortex_expect("ListViewData is always valid") } /// Mark whether this list view can be zero-copy converted to a list. @@ -520,10 +522,8 @@ impl Array { /// /// See [`ListViewData::with_zero_copy_to_list`]. pub unsafe fn with_zero_copy_to_list(self, is_zctl: bool) -> Self { - let dtype = self.dtype().clone(); - let len = self.len(); - let data = unsafe { self.into_data().with_zero_copy_to_list(is_zctl) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(ListView, dtype, len, data)) } + Array::try_from_data(unsafe { self.into_data().with_zero_copy_to_list(is_zctl) }) + .vortex_expect("data is always valid") } } diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index d2334c59526..be4ac64ab41 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -10,10 +10,8 @@ use crate::ExecutionCtx; use crate::IntoArray; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::ExtensionArray; use crate::arrays::FixedSizeListArray; -use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -38,11 +36,8 @@ pub fn list_view_from_list(list: ListArray, ctx: &mut ExecutionCtx) -> VortexRes // We reset the offsets here because mostly for convenience, and also because callers of this // function might not expect the output `ListViewArray` to have a bunch of leading and trailing // garbage data when they turn it back into a `ListArray`. - let data = list.reset_offsets(false).vortex_expect("This can't fail"); - let dtype = data.dtype(); - let len = data.len(); let list: ListArray = - unsafe { Array::from_parts_unchecked(ArrayParts::new(List, dtype, len, data)) }; + Array::try_from_data(list.reset_offsets(false).vortex_expect("This can't fail"))?; let list_offsets = list.offsets().clone(); diff --git a/vortex-array/src/arrays/listview/mod.rs b/vortex-array/src/arrays/listview/mod.rs index 20dd6fe15d5..4ca62ef0820 100644 --- a/vortex-array/src/arrays/listview/mod.rs +++ b/vortex-array/src/arrays/listview/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::ListViewArrayParts; pub use array::ListViewData; -pub use array::ListViewDataParts; pub use vtable::ListViewArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index 9dae3483a16..e8fab2f9d82 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use prost::Message; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; @@ -9,9 +8,12 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; +use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; +use crate::ProstMetadata; +use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -27,6 +29,7 @@ use crate::dtype::PType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod operations; @@ -53,13 +56,30 @@ pub struct ListViewMetadata { impl VTable for ListView { type ArrayData = ListViewData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &ListViewData) -> &Self { + &ListView + } fn id(&self) -> ArrayId { Self::ID } + fn len(array: &ListViewData) -> usize { + debug_assert_eq!(array.offsets().len(), array.sizes().len()); + array.offsets().len() + } + + fn dtype(array: &ListViewData) -> &DType { + &array.dtype + } + + fn stats(array: &ListViewData) -> &ArrayStats { + &array.stats_set + } + fn array_hash(array: &ListViewData, state: &mut H, precision: Precision) { array.elements().array_hash(state, precision); array.offsets().array_hash(state, precision); @@ -86,47 +106,36 @@ impl VTable for ListView { vortex_panic!("ListViewArray buffer_name index {idx} out of bounds") } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - ListViewMetadata { - elements_len: array.elements().len() as u64, - offset_ptype: PType::try_from(array.offsets().dtype())? as i32, - size_ptype: PType::try_from(array.sizes().dtype())? as i32, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(ListViewMetadata { + elements_len: array.elements().len() as u64, + offset_ptype: PType::try_from(array.offsets().dtype())? as i32, + size_ptype: PType::try_from(array.sizes().dtype())? as i32, + })) } - fn validate(&self, data: &ListViewData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "ListViewArray length {} does not match outer length {}", - data.len(), - len - ); - - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "ListViewArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); - - Ok(()) + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + let metadata = ::deserialize(bytes)?; + Ok(ProstMetadata(metadata)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = ListViewMetadata::decode(metadata)?; vortex_ensure!( buffers.is_empty(), "`ListViewArray::build` expects no buffers" @@ -152,20 +161,20 @@ impl VTable for ListView { let elements = children.get( 0, element_dtype.as_ref(), - usize::try_from(metadata.elements_len)?, + usize::try_from(metadata.0.elements_len)?, )?; // Get offsets with proper type from metadata. let offsets = children.get( 1, - &DType::Primitive(metadata.offset_ptype(), Nullability::NonNullable), + &DType::Primitive(metadata.0.offset_ptype(), Nullability::NonNullable), len, )?; // Get sizes with proper type from metadata. let sizes = children.get( 2, - &DType::Primitive(metadata.size_ptype(), Nullability::NonNullable), + &DType::Primitive(metadata.0.size_ptype(), Nullability::NonNullable), len, )?; diff --git a/vortex-array/src/arrays/listview/vtable/operations.rs b/vortex-array/src/arrays/listview/vtable/operations.rs index b32bb87de1f..4a15c3dd8d0 100644 --- a/vortex-array/src/arrays/listview/vtable/operations.rs +++ b/vortex-array/src/arrays/listview/vtable/operations.rs @@ -26,7 +26,7 @@ impl OperationsVTable for ListView { Ok(Scalar::list( Arc::new(list.dtype().clone()), children, - array.dtype().nullability(), + array.dtype.nullability(), )) } } diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index 9e011140930..0c2bc773817 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -7,12 +7,11 @@ use vortex_error::vortex_bail; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Masked; use crate::dtype::DType; -use crate::dtype::Nullability; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The underlying child array being masked. @@ -25,10 +24,12 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child", "validity"]; #[derive(Clone, Debug)] pub struct MaskedData { pub(super) slots: Vec>, + pub(super) dtype: DType, + pub(super) stats: ArrayStats, } impl MaskedData { - pub(crate) fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { + pub fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { if matches!(validity, Validity::NonNullable) { vortex_bail!("MaskedArray must have nullable validity, got {validity:?}") } @@ -45,17 +46,20 @@ impl MaskedData { // MaskedArray's nullability is determined solely by its validity, not the child's dtype. // The child can have nullable dtype but must not have any actual null values. + let dtype = child.dtype().as_nullable(); let len = child.len(); let validity_slot = validity_to_child(&validity, len); Ok(Self { slots: vec![Some(child), validity_slot], + dtype, + stats: ArrayStats::default(), }) } /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - self.child().dtype().as_nullable() + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns the length of the array. @@ -71,7 +75,7 @@ impl MaskedData { /// Returns the validity of the array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], Nullability::Nullable) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -89,9 +93,6 @@ impl MaskedData { impl Array { /// Constructs a new `MaskedArray`. pub fn try_new(child: ArrayRef, validity: Validity) -> VortexResult { - let dtype = child.dtype().as_nullable(); - let len = child.len(); - let data = MaskedData::try_new(child, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Masked, dtype, len, data)) }) + Array::try_from_data(MaskedData::try_new(child, validity)?) } } diff --git a/vortex-array/src/arrays/masked/compute/filter.rs b/vortex-array/src/arrays/masked/compute/filter.rs index 0eb8fb25011..a96c57feb1f 100644 --- a/vortex-array/src/arrays/masked/compute/filter.rs +++ b/vortex-array/src/arrays/masked/compute/filter.rs @@ -8,8 +8,8 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; -use crate::arrays::MaskedArray; use crate::arrays::filter::FilterReduce; +use crate::arrays::masked::MaskedData; impl FilterReduce for Masked { fn filter(array: ArrayView<'_, Masked>, mask: &Mask) -> VortexResult> { @@ -22,7 +22,7 @@ impl FilterReduce for Masked { // Construct new MaskedArray Ok(Some( - MaskedArray::try_new(filtered_child, filtered_validity)?.into_array(), + MaskedData::try_new(filtered_child, filtered_validity)?.into_array(), )) } } diff --git a/vortex-array/src/arrays/masked/compute/slice.rs b/vortex-array/src/arrays/masked/compute/slice.rs index 1be55ea6c30..afa9b73a6e6 100644 --- a/vortex-array/src/arrays/masked/compute/slice.rs +++ b/vortex-array/src/arrays/masked/compute/slice.rs @@ -9,7 +9,7 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; -use crate::arrays::MaskedArray; +use crate::arrays::masked::MaskedData; use crate::arrays::slice::SliceReduce; impl SliceReduce for Masked { @@ -17,6 +17,6 @@ impl SliceReduce for Masked { let child = array.child().slice(range.clone())?; let validity = array.validity().slice(range)?; - Ok(Some(MaskedArray::try_new(child, validity)?.into_array())) + Ok(Some(MaskedData::try_new(child, validity)?.into_array())) } } diff --git a/vortex-array/src/arrays/masked/compute/take.rs b/vortex-array/src/arrays/masked/compute/take.rs index 7061529b897..bd9a45b4711 100644 --- a/vortex-array/src/arrays/masked/compute/take.rs +++ b/vortex-array/src/arrays/masked/compute/take.rs @@ -7,8 +7,8 @@ use crate::ArrayRef; use crate::IntoArray; use crate::array::ArrayView; use crate::arrays::Masked; -use crate::arrays::MaskedArray; use crate::arrays::dict::TakeReduce; +use crate::arrays::masked::MaskedData; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; @@ -28,7 +28,7 @@ impl TakeReduce for Masked { // Construct new MaskedArray Ok(Some( - MaskedArray::try_new(taken_child, taken_validity)?.into_array(), + MaskedData::try_new(taken_child, taken_validity)?.into_array(), )) } } diff --git a/vortex-array/src/arrays/masked/execute.rs b/vortex-array/src/arrays/masked/execute.rs index 0f11a537ba2..bbf5b7fcb87 100644 --- a/vortex-array/src/arrays/masked/execute.rs +++ b/vortex-array/src/arrays/masked/execute.rs @@ -177,7 +177,7 @@ fn mask_validity_struct( let len = array.len(); let new_validity = combine_validity(&array.validity(), mask, len, ctx)?; let fields = array.unmasked_fields(); - let struct_fields = array.struct_fields(); + let struct_fields = array.struct_fields().clone(); // SAFETY: We're only changing validity, not the data structure Ok(unsafe { StructArray::new_unchecked(fields, struct_fields, len, new_validity) }) } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 4a79d53b78b..b7069998f99 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -12,13 +12,14 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Canonical; +use crate::EmptyMetadata; use crate::IntoArray; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; use crate::array::VTable; -use crate::arrays::ConstantArray; +use crate::arrays::constant::ConstantData; use crate::arrays::masked::MaskedData; use crate::arrays::masked::array::NUM_SLOTS; use crate::arrays::masked::array::SLOT_NAMES; @@ -32,6 +33,7 @@ use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; vtable!(Masked, Masked, MaskedData); @@ -46,23 +48,28 @@ impl Masked { impl VTable for Masked { type ArrayData = MaskedData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Masked + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &MaskedData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.child().len() == len, - "MaskedArray child length mismatch" - ); - vortex_ensure!( - data.dtype() == *dtype, - "MaskedArray dtype does not match child and validity" - ); - Ok(()) + fn len(array: &MaskedData) -> usize { + array.child().len() + } + + fn dtype(array: &MaskedData) -> &DType { + &array.dtype + } + + fn stats(array: &MaskedData) -> &ArrayStats { + &array.stats } fn array_hash(array: &MaskedData, state: &mut H, precision: Precision) { @@ -87,26 +94,31 @@ impl VTable for Masked { None } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + _metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - if !metadata.is_empty() { - vortex_bail!( - "MaskedArray expects empty metadata, got {} bytes", - metadata.len() - ); - } if !buffers.is_empty() { vortex_bail!("Expected 0 buffer, got {}", buffers.len()); } @@ -135,7 +147,7 @@ impl VTable for Masked { // Fast path: all masked means result is all nulls. if validity_mask.all_false() { return Ok(ExecutionResult::done( - ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.len()) + ConstantData::new(Scalar::null(array.dtype().as_nullable()), array.len()) .into_array(), )); } @@ -196,8 +208,8 @@ mod tests { use crate::arrays::MaskedArray; use crate::arrays::PrimitiveArray; use crate::dtype::Nullability; + use crate::serde::ArrayParts; use crate::serde::SerializeOptions; - use crate::serde::SerializedArray; use crate::validity::Validity; #[rstest] @@ -237,7 +249,7 @@ mod tests { } let concat = concat.freeze(); - let parts = SerializedArray::try_from(concat).unwrap(); + let parts = ArrayParts::try_from(concat).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/mod.rs b/vortex-array/src/arrays/mod.rs index 2597708919a..68ac8fa91cc 100644 --- a/vortex-array/src/arrays/mod.rs +++ b/vortex-array/src/arrays/mod.rs @@ -104,3 +104,4 @@ pub use variant::VariantArray; #[cfg(feature = "arbitrary")] pub mod arbitrary; +pub mod lazy_patched; diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 183a2008654..2643a9ae4cf 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -1,18 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; +use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; -use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::OperationsVTable; use crate::array::VTable; @@ -22,6 +23,7 @@ use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::scalar::Scalar; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -34,16 +36,28 @@ vtable!(Null, Null, NullData); impl VTable for Null { type ArrayData = NullData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Null + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, _data: &NullData, dtype: &DType, _len: usize) -> VortexResult<()> { - vortex_ensure!(*dtype == DType::Null, "NullArray dtype must be DType::Null"); - Ok(()) + fn len(array: &NullData) -> usize { + array.len + } + + fn dtype(_array: &NullData) -> &DType { + &DType::Null + } + + fn stats(array: &NullData) -> &ArrayStats { + &array.stats_set } fn array_hash(_array: &NullData, _state: &mut H, _precision: Precision) { @@ -85,26 +99,32 @@ impl VTable for Null { Ok(()) } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], _dtype: &DType, _len: usize, - metadata: &[u8], + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + fn build( + _dtype: &DType, + len: usize, + _metadata: &Self::Metadata, _buffers: &[BufferHandle], _children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - vortex_ensure!( - metadata.is_empty(), - "NullArray expects empty metadata, got {} bytes", - metadata.len() - ); - Ok(NullData::new()) + Ok(NullData::new(len)) } fn reduce_parent( @@ -149,7 +169,9 @@ impl VTable for Null { /// ``` #[derive(Clone, Debug)] pub struct NullData { + len: usize, slots: Vec>, + stats_set: ArrayStats, } #[derive(Clone, Debug)] @@ -161,21 +183,32 @@ impl Null { impl Array { pub fn new(len: usize) -> Self { - unsafe { - Array::from_parts_unchecked(ArrayParts::new(Null, DType::Null, len, NullData::new())) - } + Array::try_from_data(NullData::new(len)).vortex_expect("NullData is always valid") } } -impl Default for NullData { - fn default() -> Self { - Self::new() +impl NullData { + pub fn new(len: usize) -> Self { + Self { + len, + slots: vec![], + stats_set: Default::default(), + } } -} -impl NullData { - pub fn new() -> Self { - Self { slots: vec![] } + /// Returns the dtype of the array (always [`DType::Null`]). + pub fn dtype(&self) -> &DType { + &DType::Null + } + + /// Returns the length of the array. + pub fn len(&self) -> usize { + self.len + } + + /// Returns `true` if the array is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 } } impl OperationsVTable for Null { diff --git a/vortex-array/src/arrays/patched/array.rs b/vortex-array/src/arrays/patched/array.rs index 0f1d5ba4ca3..72130233b78 100644 --- a/vortex-array/src/arrays/patched/array.rs +++ b/vortex-array/src/arrays/patched/array.rs @@ -14,9 +14,6 @@ use crate::ArrayRef; use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; -use crate::array::Array; -use crate::array::ArrayParts; -use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::arrays::patched::TransposedPatches; use crate::arrays::patched::patch_lanes; @@ -27,6 +24,7 @@ use crate::dtype::PType; use crate::match_each_native_ptype; use crate::match_each_unsigned_integer_ptype; use crate::patches::Patches; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The inner array containing the base unpatched values. @@ -61,23 +59,10 @@ pub struct PatchedArray { /// should be subtracted out of the remaining offsets to get their final position in the /// executed array. pub(super) offset: usize, -} + /// Length of the array + pub(super) len: usize, -impl IntoArray for PatchedArray { - fn into_array(self) -> ArrayRef { - let dtype = self.base_array().dtype().clone(); - let len = self.len(); - unsafe { - Array::::from_parts_unchecked(ArrayParts::new(Patched, dtype, len, self)) - } - .into_array() - } -} - -impl From for ArrayRef { - fn from(value: PatchedArray) -> Self { - value.into_array() - } + pub(super) stats_set: ArrayStats, } impl PatchedArray { @@ -142,10 +127,14 @@ impl PatchedArray { ) .into_array(); + let len = inner.len(); + Ok(Self { slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], n_lanes, offset: 0, + len, + stats_set: ArrayStats::default(), }) } } @@ -182,16 +171,6 @@ impl PatchedArray { .as_ref() .vortex_expect("PatchedArray values slot") } - - #[inline] - pub fn len(&self) -> usize { - self.base_array().len() - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } } impl PatchedArray { @@ -202,7 +181,7 @@ impl PatchedArray { /// /// Note that this function will panic if the caller requests out of bounds chunk/lane ordinals. pub(crate) fn lane_range(&self, chunk: usize, lane: usize) -> VortexResult> { - assert!(chunk * 1024 <= self.len() + self.offset); + assert!(chunk * 1024 <= self.len + self.offset); assert!(lane < self.n_lanes); let start = self.lane_offsets().scalar_at(chunk * self.n_lanes + lane)?; @@ -240,12 +219,14 @@ impl PatchedArray { let begin = (chunks.start * 1024).saturating_sub(self.offset); let end = (chunks.end * 1024) .saturating_sub(self.offset) - .min(self.len()); + .min(self.len); let offset = if chunks.start == 0 { self.offset } else { 0 }; let inner = self.base_array().slice(begin..end)?; + let len = end - begin; + Ok(PatchedArray { slots: vec![ Some(inner), @@ -255,6 +236,8 @@ impl PatchedArray { ], n_lanes: self.n_lanes, offset, + len, + stats_set: ArrayStats::default(), }) } } diff --git a/vortex-array/src/arrays/patched/compute/compare.rs b/vortex-array/src/arrays/patched/compute/compare.rs index 54c89c1744d..0ec4448da37 100644 --- a/vortex-array/src/arrays/patched/compute/compare.rs +++ b/vortex-array/src/arrays/patched/compute/compare.rs @@ -14,7 +14,7 @@ use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::Patched; use crate::arrays::PrimitiveArray; -use crate::arrays::bool::BoolDataParts; +use crate::arrays::bool::BoolArrayParts; use crate::arrays::primitive::NativeValue; use crate::builtins::ArrayBuiltins; use crate::dtype::NativePType; @@ -50,13 +50,12 @@ impl CompareKernel for Patched { .execute::(ctx)? .into_bool(); - let result = result.into_parts(); - let BoolDataParts { + let BoolArrayParts { bits, offset, len, validity, - } = result.data.into_parts(); + } = result.into_parts(); let mut bits = BitBufferMut::from_buffer(bits.unwrap_host().into_mut(), offset, len); @@ -165,6 +164,7 @@ mod tests { use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; + use crate::array::Array; use crate::arrays::BoolArray; use crate::arrays::ConstantArray; use crate::arrays::Patched; @@ -191,11 +191,10 @@ mod tests { let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx) - .unwrap() - .into_array() - .try_into::() - .unwrap(); + let lhs = Array::::try_from_data( + PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx).unwrap(), + ) + .unwrap(); let rhs = ConstantArray::new(u32::MAX, 512).into_array(); @@ -260,10 +259,9 @@ mod tests { )?; let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx)? - .into_array() - .try_into::() - .unwrap(); + let lhs = Array::::try_from_data(PatchedArray::from_array_and_patches( + lhs, &patches, &mut ctx, + )?)?; let rhs = ConstantArray::new(subnormal, 512).into_array(); @@ -294,10 +292,9 @@ mod tests { )?; let mut ctx = ExecutionCtx::new(LEGACY_SESSION.clone()); - let lhs = PatchedArray::from_array_and_patches(lhs, &patches, &mut ctx)? - .into_array() - .try_into::() - .unwrap(); + let lhs = Array::::try_from_data(PatchedArray::from_array_and_patches( + lhs, &patches, &mut ctx, + )?)?; let rhs = ConstantArray::new(0.0f32, 10).into_array(); diff --git a/vortex-array/src/arrays/patched/compute/filter.rs b/vortex-array/src/arrays/patched/compute/filter.rs index 0a2060964aa..71aee0238ee 100644 --- a/vortex-array/src/arrays/patched/compute/filter.rs +++ b/vortex-array/src/arrays/patched/compute/filter.rs @@ -37,7 +37,7 @@ impl FilterReduce for Patched { } }; - let n_chunks = (array.offset + array.len()).div_ceil(1024); + let n_chunks = (array.offset + array.len).div_ceil(1024); // If all chunks already covered, there is nothing to do. if chunk_start == 0 && chunk_stop == n_chunks { diff --git a/vortex-array/src/arrays/patched/compute/take.rs b/vortex-array/src/arrays/patched/compute/take.rs index 1257eec5169..6fac5826537 100644 --- a/vortex-array/src/arrays/patched/compute/take.rs +++ b/vortex-array/src/arrays/patched/compute/take.rs @@ -12,7 +12,7 @@ use crate::array::ArrayView; use crate::arrays::Patched; use crate::arrays::PrimitiveArray; use crate::arrays::dict::TakeExecute; -use crate::arrays::primitive::PrimitiveDataParts; +use crate::arrays::primitive::PrimitiveArrayParts; use crate::dtype::IntegerPType; use crate::dtype::NativePType; use crate::match_each_native_ptype; @@ -35,7 +35,7 @@ impl TakeExecute for Patched { .take(indices.clone())? .execute::(ctx)?; - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer, validity, ptype, @@ -63,7 +63,7 @@ impl TakeExecute for Patched { output.as_mut(), indices.as_slice::(), array.offset, - array.len(), + array.len, array.n_lanes, lane_offsets.as_slice::(), patch_indices.as_slice::(), diff --git a/vortex-array/src/arrays/patched/vtable/mod.rs b/vortex-array/src/arrays/patched/vtable/mod.rs index 0e592452e27..eb5c887772a 100644 --- a/vortex-array/src/arrays/patched/vtable/mod.rs +++ b/vortex-array/src/arrays/patched/vtable/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use prost::Message; mod kernels; mod operations; mod slice; @@ -20,10 +19,13 @@ use crate::ArrayEq; use crate::ArrayHash; use crate::ArrayRef; use crate::Canonical; +use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; use crate::Precision; +use crate::ProstMetadata; +use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -36,7 +38,7 @@ use crate::arrays::patched::array::NUM_SLOTS; use crate::arrays::patched::array::SLOT_NAMES; use crate::arrays::patched::compute::rules::PARENT_RULES; use crate::arrays::patched::vtable::kernels::PARENT_KERNELS; -use crate::arrays::primitive::PrimitiveDataParts; +use crate::arrays::primitive::PrimitiveArrayParts; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::builders::PrimitiveBuilder; @@ -45,6 +47,7 @@ use crate::dtype::NativePType; use crate::dtype::PType; use crate::match_each_native_ptype; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::vtable; vtable!(Patched); @@ -77,13 +80,30 @@ pub struct PatchedMetadata { impl VTable for Patched { type ArrayData = PatchedArray; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = ValidityVTableFromChild; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Patched + } + fn id(&self) -> ArrayId { ArrayId::new_ref("vortex.patched") } + fn len(array: &Self::ArrayData) -> usize { + array.len + } + + fn dtype(array: &Self::ArrayData) -> &DType { + array.base_array().dtype() + } + + fn stats(array: &Self::ArrayData) -> &ArrayStats { + &array.stats_set + } + fn array_hash(array: &PatchedArray, state: &mut H, precision: Precision) { array.offset.hash(state); array.n_lanes.hash(state); @@ -112,28 +132,6 @@ impl VTable for Patched { 0 } - fn validate(&self, data: &PatchedArray, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.base_array().dtype() == dtype, - "PatchedArray base dtype {} does not match outer dtype {}", - data.base_array().dtype(), - dtype - ); - vortex_ensure!( - data.len() == len, - "PatchedArray base len {} does not match outer len {}", - data.len(), - len - ); - vortex_ensure!( - data.patch_indices().len() == data.patch_values().len(), - "PatchedArray patch indices len {} does not match patch values len {}", - data.patch_indices().len(), - data.patch_values().len() - ); - Ok(()) - } - fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { vortex_panic!("invalid buffer index for PatchedArray: {idx}"); } @@ -152,45 +150,27 @@ impl VTable for Patched { } } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - PatchedMetadata { - n_patches: u32::try_from(array.patch_indices().len())?, - n_lanes: u32::try_from(array.n_lanes)?, - offset: u32::try_from(array.offset)?, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(PatchedMetadata { + n_patches: u32::try_from(array.patch_indices().len())?, + n_lanes: u32::try_from(array.n_lanes)?, + offset: u32::try_from(array.offset)?, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, - dtype: &DType, - len: usize, - metadata: &[u8], + bytes: &[u8], + _dtype: &DType, + _len: usize, _buffers: &[BufferHandle], - children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { - let metadata = PatchedMetadata::decode(metadata)?; - let n_patches = metadata.n_patches as usize; - let n_lanes = metadata.n_lanes as usize; - let offset = metadata.offset as usize; - - // n_chunks should correspond to the chunk in the `inner`. - // After slicing when offset > 0, there may be additional chunks. - let n_chunks = (len + offset).div_ceil(1024); - - let inner = children.get(0, dtype, len)?; - let lane_offsets = children.get(1, PType::U32.into(), n_chunks * n_lanes + 1)?; - let indices = children.get(2, PType::U16.into(), n_patches)?; - let values = children.get(3, dtype, n_patches)?; - - Ok(PatchedArray { - slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], - n_lanes, - offset, - }) + ) -> VortexResult { + let inner = as DeserializeMetadata>::deserialize(bytes)?; + Ok(ProstMetadata(inner)) } fn append_to_builder( @@ -256,6 +236,35 @@ impl VTable for Patched { Ok(()) } + fn build( + dtype: &DType, + len: usize, + metadata: &Self::Metadata, + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + ) -> VortexResult { + let n_patches = metadata.n_patches as usize; + let n_lanes = metadata.n_lanes as usize; + let offset = metadata.offset as usize; + + // n_chunks should correspond to the chunk in the `inner`. + // After slicing when offset > 0, there may be additional chunks. + let n_chunks = (len + offset).div_ceil(1024); + + let inner = children.get(0, dtype, len)?; + let lane_offsets = children.get(1, PType::U32.into(), n_chunks * n_lanes + 1)?; + let indices = children.get(2, PType::U16.into(), n_patches)?; + let values = children.get(3, dtype, n_patches)?; + + Ok(PatchedArray { + slots: vec![Some(inner), Some(lane_offsets), Some(indices), Some(values)], + n_lanes, + offset, + len, + stats_set: ArrayStats::default(), + }) + } + fn slots(array: ArrayView<'_, Self>) -> &[Option] { &array.data().slots } @@ -282,7 +291,7 @@ impl VTable for Patched { .execute::(ctx)? .into_primitive(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer, ptype, validity, @@ -305,7 +314,7 @@ impl VTable for Patched { let patched_values = match_each_native_ptype!(values.ptype(), |V| { let offset = array.offset; - let len = array.len(); + let len = array.len; let mut output = Buffer::::from_byte_buffer(buffer.unwrap_host()).into_mut(); @@ -395,8 +404,8 @@ mod tests { use crate::assert_arrays_eq; use crate::builders::builder_with_capacity; use crate::patches::Patches; + use crate::serde::ArrayParts; use crate::serde::SerializeOptions; - use crate::serde::SerializedArray; use crate::validity::Validity; #[test] @@ -622,7 +631,7 @@ mod tests { } let concat = concat.freeze(); - let parts = SerializedArray::try_from(concat).unwrap(); + let parts = ArrayParts::try_from(concat).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/patched/vtable/slice.rs b/vortex-array/src/arrays/patched/vtable/slice.rs index bbdb4df2464..71d68585ebe 100644 --- a/vortex-array/src/arrays/patched/vtable/slice.rs +++ b/vortex-array/src/arrays/patched/vtable/slice.rs @@ -11,12 +11,15 @@ use crate::array::ArrayView; use crate::arrays::Patched; use crate::arrays::PatchedArray; use crate::arrays::slice::SliceReduce; +use crate::stats::ArrayStats; impl SliceReduce for Patched { fn slice(array: ArrayView<'_, Self>, range: Range) -> VortexResult> { // We **always** slice the patches at 1024-element chunk boundaries. We keep the offset + len // around so that when we execute we know how much to chop off. let new_offset = (range.start + array.offset) % 1024; + let new_len = range.end - range.start; + let chunk_start = (range.start + array.offset) / 1024; let chunk_stop = (range.end + array.offset).div_ceil(1024); let sliced_lane_offsets = array @@ -38,6 +41,8 @@ impl SliceReduce for Patched { ], n_lanes: array.n_lanes, offset: new_offset, + len: new_len, + stats_set: ArrayStats::default(), } .into_array(), )) diff --git a/vortex-array/src/arrays/primitive/array/cast.rs b/vortex-array/src/arrays/primitive/array/cast.rs index f213f6a60c5..c92977616ff 100644 --- a/vortex-array/src/arrays/primitive/array/cast.rs +++ b/vortex-array/src/arrays/primitive/array/cast.rs @@ -61,20 +61,6 @@ impl PrimitiveData { } impl PrimitiveArray { - pub fn reinterpret_cast(&self, ptype: PType) -> Self { - if self.ptype() == ptype { - return self.clone(); - } - - assert_eq!( - self.ptype().byte_width(), - ptype.byte_width(), - "can't reinterpret cast between integers of two different widths" - ); - - PrimitiveArray::from_buffer_handle(self.buffer_handle().clone(), ptype, self.validity()) - } - /// Narrow the array to the smallest possible integer type that can represent all values. pub fn narrow(&self) -> VortexResult { if !self.ptype().is_int() { diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index 25cbf8c755b..1a7a9a0efdd 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -15,13 +15,13 @@ use vortex_error::vortex_panic; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::Primitive; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; use crate::dtype::PType; use crate::match_each_native_ptype; +use crate::stats::ArrayStats; use crate::validity::Validity; mod accessor; @@ -76,12 +76,12 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct PrimitiveData { pub(super) slots: Vec>, - pub(super) ptype: PType, - pub(super) nullability: Nullability, + pub(super) dtype: DType, pub(super) buffer: BufferHandle, + pub(super) stats_set: ArrayStats, } -pub struct PrimitiveDataParts { +pub struct PrimitiveArrayParts { pub ptype: PType, pub buffer: BufferHandle, pub validity: Validity, @@ -107,11 +107,12 @@ impl PrimitiveData { ) -> Self { let len = handle.len() / ptype.byte_width(); let slots = Self::make_slots(&validity, len); + let dtype = DType::Primitive(ptype, validity.nullability()); Self { slots, buffer: handle, - ptype, - nullability: validity.nullability(), + dtype, + stats_set: ArrayStats::default(), } } @@ -162,11 +163,12 @@ impl PrimitiveData { let len = buffer.len(); let slots = Self::make_slots(&validity, len); + let dtype = DType::Primitive(T::PTYPE, validity.nullability()); Self { slots, - ptype: T::PTYPE, - nullability: validity.nullability(), + dtype, buffer: BufferHandle::new_host(buffer.into_byte_buffer()), + stats_set: Default::default(), } } @@ -195,10 +197,8 @@ impl PrimitiveData { impl Array { pub fn empty(nullability: Nullability) -> Self { - let dtype = DType::Primitive(T::PTYPE, nullability); - let len = 0; - let data = PrimitiveData::empty::(nullability); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + Array::try_from_data(PrimitiveData::empty::(nullability)) + .vortex_expect("PrimitiveData is always valid") } /// Creates a new `PrimitiveArray`. @@ -207,19 +207,13 @@ impl Array { /// /// Panics if the provided components do not satisfy the invariants. pub fn new(buffer: impl Into>, validity: Validity) -> Self { - let buffer = buffer.into(); - let dtype = DType::Primitive(T::PTYPE, validity.nullability()); - let len = buffer.len(); - let data = PrimitiveData::new(buffer, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + Array::try_from_data(PrimitiveData::new(buffer, validity)) + .vortex_expect("PrimitiveData is always valid") } /// Constructs a new `PrimitiveArray`. pub fn try_new(buffer: Buffer, validity: Validity) -> VortexResult { - let dtype = DType::Primitive(T::PTYPE, validity.nullability()); - let len = buffer.len(); - let data = PrimitiveData::try_new(buffer, validity)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) }) + Array::try_from_data(PrimitiveData::try_new(buffer, validity)?) } /// Creates a new `PrimitiveArray` without validation. @@ -228,10 +222,8 @@ impl Array { /// /// See [`PrimitiveData::new_unchecked`]. pub unsafe fn new_unchecked(buffer: Buffer, validity: Validity) -> Self { - let dtype = DType::Primitive(T::PTYPE, validity.nullability()); - let len = buffer.len(); - let data = unsafe { PrimitiveData::new_unchecked(buffer, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + Array::try_from_data(unsafe { PrimitiveData::new_unchecked(buffer, validity) }) + .vortex_expect("PrimitiveData is always valid") } /// Create a new array from a buffer handle. @@ -244,27 +236,22 @@ impl Array { ptype: PType, validity: Validity, ) -> Self { - let dtype = DType::Primitive(ptype, validity.nullability()); - let len = handle.len() / ptype.byte_width(); - let data = unsafe { PrimitiveData::new_unchecked_from_handle(handle, ptype, validity) }; - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + Array::try_from_data(unsafe { + PrimitiveData::new_unchecked_from_handle(handle, ptype, validity) + }) + .vortex_expect("PrimitiveData is always valid") } /// Creates a new `PrimitiveArray` from a [`BufferHandle`]. pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { - let dtype = DType::Primitive(ptype, validity.nullability()); - let len = handle.len() / ptype.byte_width(); - let data = PrimitiveData::from_buffer_handle(handle, ptype, validity); - Array::try_from_parts(ArrayParts::new(Primitive, dtype, len, data)) + Array::try_from_data(PrimitiveData::from_buffer_handle(handle, ptype, validity)) .vortex_expect("PrimitiveData is always valid") } /// Creates a new `PrimitiveArray` from a [`ByteBuffer`]. pub fn from_byte_buffer(buffer: ByteBuffer, ptype: PType, validity: Validity) -> Self { - let dtype = DType::Primitive(ptype, validity.nullability()); - let len = buffer.len() / ptype.byte_width(); - let data = PrimitiveData::from_byte_buffer(buffer, ptype, validity); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + Array::try_from_data(PrimitiveData::from_byte_buffer(buffer, ptype, validity)) + .vortex_expect("PrimitiveData is always valid") } /// Create a PrimitiveArray from a byte buffer containing only the valid elements. @@ -274,11 +261,13 @@ impl Array { validity: Validity, n_rows: usize, ) -> Self { - let dtype = DType::Primitive(ptype, validity.nullability()); - let len = n_rows; - let data = - PrimitiveData::from_values_byte_buffer(valid_elems_buffer, ptype, validity, n_rows); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Primitive, dtype, len, data)) } + Array::try_from_data(PrimitiveData::from_values_byte_buffer( + valid_elems_buffer, + ptype, + validity, + n_rows, + )) + .vortex_expect("PrimitiveData is always valid") } /// Validates the components that would be used to create a `PrimitiveArray`. @@ -289,10 +278,10 @@ impl Array { impl PrimitiveData { /// Consume the primitive array and returns its component parts. - pub fn into_parts(self) -> PrimitiveDataParts { + pub fn into_parts(self) -> PrimitiveArrayParts { let ptype = self.ptype(); let validity = self.validity(); - PrimitiveDataParts { + PrimitiveArrayParts { ptype, buffer: self.buffer, validity, @@ -302,8 +291,8 @@ impl PrimitiveData { impl PrimitiveData { /// Returns the dtype of the array. - pub fn dtype(&self) -> DType { - DType::Primitive(self.ptype, self.nullability) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns the length of the array. @@ -319,7 +308,7 @@ impl PrimitiveData { /// Reconstructs the validity from the slot state. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } /// Returns the validity as a [`Mask`](vortex_mask::Mask). @@ -328,7 +317,7 @@ impl PrimitiveData { } pub fn ptype(&self) -> PType { - self.ptype + self.dtype().as_ptype() } /// Get access to the buffer handle backing the array. @@ -337,13 +326,14 @@ impl PrimitiveData { } pub fn from_buffer_handle(handle: BufferHandle, ptype: PType, validity: Validity) -> Self { + let dtype = DType::Primitive(ptype, validity.nullability()); let len = handle.len() / ptype.byte_width(); let slots = Self::make_slots(&validity, len); Self { slots, buffer: handle, - ptype, - nullability: validity.nullability(), + dtype, + stats_set: ArrayStats::default(), } } diff --git a/vortex-array/src/arrays/primitive/compute/between.rs b/vortex-array/src/arrays/primitive/compute/between.rs index 251aa9c9e7c..09ea69e6d16 100644 --- a/vortex-array/src/arrays/primitive/compute/between.rs +++ b/vortex-array/src/arrays/primitive/compute/between.rs @@ -33,7 +33,7 @@ impl BetweenKernel for Primitive { // null values let nullability = - arr.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability(); + arr.dtype.nullability() | lower.dtype().nullability() | upper.dtype().nullability(); Ok(Some(match_each_native_ptype!(arr.ptype(), |P| { between_impl::

( diff --git a/vortex-array/src/arrays/primitive/mod.rs b/vortex-array/src/arrays/primitive/mod.rs index 7c526c2cfbb..522a03afc86 100644 --- a/vortex-array/src/arrays/primitive/mod.rs +++ b/vortex-array/src/arrays/primitive/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::PrimitiveArrayParts; pub use array::PrimitiveData; -pub use array::PrimitiveDataParts; pub use array::chunk_range; pub use array::patch_chunk; pub use vtable::PrimitiveArray; diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index ab29d6305f5..db8bd8c5258 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -8,6 +8,7 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use crate::ArrayRef; +use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::array::Array; @@ -36,19 +37,37 @@ use crate::arrays::primitive::array::SLOT_NAMES; use crate::arrays::primitive::compute::rules::RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; +use crate::stats::ArrayStats; vtable!(Primitive, Primitive, PrimitiveData); impl VTable for Primitive { type ArrayData = PrimitiveData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Primitive + } + fn id(&self) -> ArrayId { Self::ID } + fn len(array: &PrimitiveData) -> usize { + array.buffer_handle().len() / array.ptype().byte_width() + } + + fn dtype(array: &PrimitiveData) -> &DType { + &array.dtype + } + + fn stats(array: &PrimitiveData) -> &ArrayStats { + &array.stats_set + } + fn array_hash(array: &PrimitiveData, state: &mut H, precision: Precision) { array.buffer.array_hash(state, precision); array.validity().array_hash(state, precision); @@ -77,45 +96,31 @@ impl VTable for Primitive { } } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some(vec![])) + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) } - fn validate(&self, data: &PrimitiveData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "PrimitiveArray length {} does not match outer length {}", - data.len(), - len - ); - - let actual_dtype = data.dtype(); - vortex_ensure!( - &actual_dtype == dtype, - "PrimitiveArray dtype {} does not match outer dtype {}", - actual_dtype, - dtype - ); - - Ok(()) + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + _metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - if !metadata.is_empty() { - vortex_bail!( - "PrimitiveArray expects empty metadata, got {} bytes", - metadata.len() - ); - } if buffers.len() != 1 { vortex_bail!("Expected 1 buffer, got {}", buffers.len()); } @@ -222,8 +227,8 @@ mod tests { use crate::LEGACY_SESSION; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; + use crate::serde::ArrayParts; use crate::serde::SerializeOptions; - use crate::serde::SerializedArray; use crate::validity::Validity; #[test] @@ -246,7 +251,7 @@ mod tests { for buf in serialized { concat.extend_from_slice(buf.as_ref()); } - let parts = SerializedArray::try_from(concat.freeze()).unwrap(); + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/scalar_fn/array.rs b/vortex-array/src/arrays/scalar_fn/array.rs index 1fc9accc886..029ff0802fa 100644 --- a/vortex-array/src/arrays/scalar_fn/array.rs +++ b/vortex-array/src/arrays/scalar_fn/array.rs @@ -7,16 +7,20 @@ use vortex_error::vortex_ensure; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::ScalarFnVTable; +use crate::dtype::DType; use crate::scalar_fn::ScalarFnRef; +use crate::stats::ArrayStats; // ScalarFnArray has a variable number of slots (one per child) #[derive(Clone, Debug)] pub struct ScalarFnData { - pub(super) scalar_fn: ScalarFnRef, + pub(super) vtable: ScalarFnVTable, + pub(super) dtype: DType, + pub(super) len: usize, pub(super) slots: Vec>, + pub(super) stats: ArrayStats, } impl ScalarFnData { @@ -26,6 +30,9 @@ impl ScalarFnData { children: Vec, len: usize, ) -> VortexResult { + let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); + let dtype = scalar_fn.return_dtype(&arg_dtypes)?; + vortex_ensure!( children.iter().all(|c| c.len() == len), "ScalarFnArray must have children equal to the array length" @@ -33,14 +40,35 @@ impl ScalarFnData { let slots = children.into_iter().map(Some).collect(); - Ok(Self { scalar_fn, slots }) + Ok(Self { + vtable: ScalarFnVTable { scalar_fn }, + dtype, + len, + slots, + stats: Default::default(), + }) + } + + /// Returns the dtype of the array. + pub fn dtype(&self) -> &DType { + &self.dtype + } + + /// Returns the length of the array. + pub fn len(&self) -> usize { + self.len + } + + /// Returns `true` if the array is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 } /// Get the scalar function bound to this array. #[allow(clippy::same_name_method)] #[inline(always)] pub fn scalar_fn(&self) -> &ScalarFnRef { - &self.scalar_fn + &self.vtable.scalar_fn } /// Get a child array by index. @@ -88,10 +116,6 @@ impl Array { children: Vec, len: usize, ) -> VortexResult { - let arg_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); - let dtype = scalar_fn.return_dtype(&arg_dtypes)?; - let data = ScalarFnData::try_new(scalar_fn.clone(), children, len)?; - let vtable = ScalarFnVTable { scalar_fn }; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(vtable, dtype, len, data)) }) + Array::try_from_data(ScalarFnData::try_new(scalar_fn, children, len)?) } } diff --git a/vortex-array/src/arrays/scalar_fn/metadata.rs b/vortex-array/src/arrays/scalar_fn/metadata.rs new file mode 100644 index 00000000000..8a0817bf089 --- /dev/null +++ b/vortex-array/src/arrays/scalar_fn/metadata.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Debug; +use std::fmt::Formatter; + +use crate::dtype::DType; +use crate::scalar_fn::ScalarFnRef; + +#[derive(Clone)] +pub struct ScalarFnMetadata { + pub(super) scalar_fn: ScalarFnRef, + pub(super) child_dtypes: Vec, +} + +// Array tree display wrongly uses debug... +impl Debug for ScalarFnMetadata { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.scalar_fn.options()) + } +} diff --git a/vortex-array/src/arrays/scalar_fn/mod.rs b/vortex-array/src/arrays/scalar_fn/mod.rs index 778075b9fe0..d19d1157d0b 100644 --- a/vortex-array/src/arrays/scalar_fn/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/mod.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +mod metadata; mod rules; mod vtable; diff --git a/vortex-array/src/arrays/scalar_fn/rules.rs b/vortex-array/src/arrays/scalar_fn/rules.rs index 2c1aa67cdaa..40fec2594cf 100644 --- a/vortex-array/src/arrays/scalar_fn/rules.rs +++ b/vortex-array/src/arrays/scalar_fn/rules.rs @@ -19,6 +19,7 @@ use crate::arrays::ScalarFnArray; use crate::arrays::ScalarFnVTable; use crate::arrays::Slice; use crate::arrays::StructArray; +use crate::arrays::scalar_fn::ScalarFnData; use crate::dtype::DType; use crate::optimizer::rules::ArrayParentReduceRule; use crate::optimizer::rules::ArrayReduceRule; @@ -60,7 +61,7 @@ impl ArrayReduceRule for ScalarFnPackToStructRule { StructArray::try_new( pack_options.names.clone(), array.children(), - array.len(), + array.len, validity, )? .into_array(), @@ -79,7 +80,7 @@ impl ArrayReduceRule for ScalarFnConstantRule { Ok(Some(Canonical::empty(array.dtype()).into_array())) } else { let result = array.array().scalar_at(0)?; - Ok(Some(ConstantArray::new(result, array.len()).into_array())) + Ok(Some(ConstantArray::new(result, array.len).into_array())) } } } @@ -103,7 +104,7 @@ impl ArrayParentReduceRule for ScalarFnSliceReduceRule { .collect::>()?; Ok(Some( - ScalarFnArray::try_new(array.scalar_fn().clone(), children, range.len())?.into_array(), + ScalarFnData::try_new(array.scalar_fn().clone(), children, range.len())?.into_array(), )) } } @@ -114,7 +115,7 @@ impl ArrayReduceRule for ScalarFnAbstractReduceRule { fn reduce(&self, array: ArrayView<'_, ScalarFnVTable>) -> VortexResult> { if let Some(reduced) = array .scalar_fn() - .reduce(array.as_ref(), &ArrayReduceCtx { len: array.len() })? + .reduce(array.as_ref(), &ArrayReduceCtx { len: array.len })? { return Ok(Some( reduced diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index c74a53d7730..5f4d3e2835a 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -23,10 +23,10 @@ use crate::IntoArray; use crate::Precision; use crate::array::Array; use crate::array::ArrayId; -use crate::array::ArrayParts; use crate::array::ArrayView; use crate::array::VTable; use crate::arrays::scalar_fn::array::ScalarFnData; +use crate::arrays::scalar_fn::metadata::ScalarFnMetadata; use crate::arrays::scalar_fn::rules::PARENT_RULES; use crate::arrays::scalar_fn::rules::RULES; use crate::buffer::BufferHandle; @@ -44,6 +44,7 @@ use crate::scalar_fn::ScalarFnRef; use crate::scalar_fn::ScalarFnVTableExt; use crate::scalar_fn::VecExecutionArgs; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::vtable; vtable!(ScalarFn, ScalarFnVTable, ScalarFnData); @@ -55,32 +56,28 @@ pub struct ScalarFnVTable { impl VTable for ScalarFnVTable { type ArrayData = ScalarFnData; + type Metadata = ScalarFnMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(array: &ScalarFnData) -> &Self { + &array.vtable + } + fn id(&self) -> ArrayId { self.scalar_fn.id() } - fn validate(&self, data: &ScalarFnData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.scalar_fn == self.scalar_fn, - "ScalarFnArray data scalar_fn does not match vtable" - ); - vortex_ensure!( - data.iter_children().all(|c| c.len() == len), - "All child arrays must have the same length as the scalar function array" - ); + fn len(array: &ScalarFnData) -> usize { + array.len + } - let child_dtypes = data - .iter_children() - .map(|c| c.dtype().clone()) - .collect_vec(); - vortex_ensure!( - self.scalar_fn.return_dtype(&child_dtypes)? == *dtype, - "ScalarFnArray dtype does not match scalar function return dtype" - ); - Ok(()) + fn dtype(array: &ScalarFnData) -> &DType { + &array.dtype + } + + fn stats(array: &ScalarFnData) -> &ArrayStats { + &array.stats } fn array_hash(array: &ScalarFnData, state: &mut H, precision: Precision) { @@ -114,24 +111,63 @@ impl VTable for ScalarFnVTable { None } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + let child_dtypes = array.iter_children().map(|c| c.dtype().clone()).collect(); + Ok(ScalarFnMetadata { + scalar_fn: array.scalar_fn().clone(), + child_dtypes, + }) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { // Not supported Ok(None) } fn deserialize( - &self, + _bytes: &[u8], _dtype: &DType, _len: usize, - _metadata: &[u8], - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult { vortex_bail!("Deserialization of ScalarFnVTable metadata is not supported"); } + fn build( + dtype: &DType, + len: usize, + metadata: &ScalarFnMetadata, + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + ) -> VortexResult { + let children: Vec<_> = metadata + .child_dtypes + .iter() + .enumerate() + .map(|(idx, child_dtype)| children.get(idx, child_dtype, len)) + .try_collect()?; + + #[cfg(debug_assertions)] + { + let child_dtypes: Vec<_> = children.iter().map(|c| c.dtype().clone()).collect(); + vortex_error::vortex_ensure!( + &metadata.scalar_fn.return_dtype(&child_dtypes)? == dtype, + "Return dtype mismatch when building ScalarFnArray" + ); + } + + Ok(ScalarFnData { + vtable: ScalarFnVTable { + scalar_fn: metadata.scalar_fn.clone(), + }, + dtype: dtype.clone(), + len, + slots: children.into_iter().map(Some).collect(), + stats: Default::default(), + }) + } + fn slots(array: ArrayView<'_, Self>) -> &[Option] { &array.data().slots } @@ -191,15 +227,14 @@ pub trait ScalarFnArrayExt: scalar_fn::ScalarFnVTable { let child_dtypes = children.iter().map(|c| c.dtype().clone()).collect_vec(); let dtype = scalar_fn.return_dtype(&child_dtypes)?; - let data = ScalarFnData { - scalar_fn: scalar_fn.clone(), + Ok(ScalarFnData { + vtable: ScalarFnVTable { scalar_fn }, + dtype, + len, slots: children.into_iter().map(Some).collect(), - }; - let vtable = ScalarFnVTable { scalar_fn }; - Ok( - unsafe { Array::from_parts_unchecked(ArrayParts::new(vtable, dtype, len, data)) } - .into_array(), - ) + stats: Default::default(), + } + .into_array()) } } impl ScalarFnArrayExt for V {} diff --git a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs index c697d784ea8..f02f0b7f9bf 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/operations.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/operations.rs @@ -44,11 +44,11 @@ impl OperationsVTable for ScalarFnVTable { debug_assert_eq!( scalar.dtype(), - array.dtype(), + &array.dtype, "Scalar function {} returned dtype {:?} but expected {:?}", array.scalar_fn(), scalar.dtype(), - array.dtype() + array.dtype ); Ok(scalar) diff --git a/vortex-array/src/arrays/shared/array.rs b/vortex-array/src/arrays/shared/array.rs index cf4df660520..c0d4ab33f86 100644 --- a/vortex-array/src/arrays/shared/array.rs +++ b/vortex-array/src/arrays/shared/array.rs @@ -14,9 +14,9 @@ use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::Shared; use crate::dtype::DType; +use crate::stats::ArrayStats; /// The source array that is shared and lazily computed. pub(super) const SOURCE_SLOT: usize = 0; @@ -32,14 +32,18 @@ pub struct SharedData { pub(super) slots: Vec>, cached: Arc>>, async_compute_lock: Arc>, + pub(super) dtype: DType, + pub(super) stats: ArrayStats, } impl SharedData { pub fn new(source: ArrayRef) -> Self { Self { + dtype: source.dtype().clone(), slots: vec![Some(source)], cached: Arc::new(OnceLock::new()), async_compute_lock: Arc::new(AsyncMutex::new(())), + stats: ArrayStats::default(), } } @@ -109,7 +113,7 @@ impl SharedData { /// Returns the [`DType`] of this array. pub fn dtype(&self) -> &DType { - self.current_array_ref().dtype() + &self.dtype } /// Returns `true` if this array is empty. @@ -121,21 +125,15 @@ impl SharedData { impl Array { /// Creates a new `SharedArray`. pub fn new(source: ArrayRef) -> Self { - let dtype = source.dtype().clone(); - let len = source.len(); - unsafe { - Array::from_parts_unchecked(ArrayParts::new( - Shared, - dtype, - len, - SharedData::new(source), - )) - } + Array::try_from_data(SharedData::new(source)).vortex_expect("SharedData is always valid") } } impl SharedData { pub(super) fn set_source(&mut self, source: Option) { + if let Some(ref s) = source { + self.dtype = s.dtype().clone(); + } self.slots = vec![source]; self.cached = Arc::new(OnceLock::new()); self.async_compute_lock = Arc::new(AsyncMutex::new(())); diff --git a/vortex-array/src/arrays/shared/vtable.rs b/vortex-array/src/arrays/shared/vtable.rs index abd4723ca5d..8f04b2a37f9 100644 --- a/vortex-array/src/arrays/shared/vtable.rs +++ b/vortex-array/src/arrays/shared/vtable.rs @@ -8,6 +8,7 @@ use vortex_session::VortexSession; use crate::ArrayRef; use crate::Canonical; +use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -24,6 +25,7 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::scalar::Scalar; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -40,17 +42,27 @@ impl Shared { impl VTable for Shared { type ArrayData = SharedData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &SharedData) -> &Self { + &Shared + } fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &SharedData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_error::vortex_ensure!(data.source().dtype() == dtype, "SharedArray dtype mismatch"); - vortex_error::vortex_ensure!(data.source().len() == len, "SharedArray len mismatch"); - Ok(()) + fn len(array: &SharedData) -> usize { + array.current_array_ref().len() + } + + fn dtype(array: &SharedData) -> &DType { + &array.dtype + } + + fn stats(array: &SharedData) -> &ArrayStats { + &array.stats } fn array_hash(array: &SharedData, state: &mut H, precision: Precision) { @@ -98,23 +110,35 @@ impl VTable for Shared { Ok(()) } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { vortex_error::vortex_bail!("Shared array is not serializable") } fn deserialize( - &self, + _bytes: &[u8], _dtype: &DType, _len: usize, - _metadata: &[u8], - _buffers: &[BufferHandle], - _children: &dyn crate::serde::ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult { vortex_error::vortex_bail!("Shared array is not serializable") } + fn build( + dtype: &DType, + len: usize, + _metadata: &Self::Metadata, + _buffers: &[BufferHandle], + children: &dyn crate::serde::ArrayChildren, + ) -> VortexResult { + let child = children.get(0, dtype, len)?; + Ok(SharedData::new(child)) + } + fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { array .get_or_compute(|source| source.clone().execute::(ctx)) diff --git a/vortex-array/src/arrays/slice/array.rs b/vortex-array/src/arrays/slice/array.rs index 2d438c3fe9c..a8ded19335b 100644 --- a/vortex-array/src/arrays/slice/array.rs +++ b/vortex-array/src/arrays/slice/array.rs @@ -9,9 +9,9 @@ use vortex_error::vortex_panic; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayParts; use crate::arrays::Slice; use crate::dtype::DType; +use crate::stats::ArrayStats; /// The underlying child array being sliced. pub(super) const CHILD_SLOT: usize = 0; @@ -22,15 +22,16 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; pub struct SliceData { pub(super) slots: Vec>, pub(super) range: Range, + pub(super) stats: ArrayStats, } -pub struct SliceDataParts { +pub struct SliceArrayParts { pub child: ArrayRef, pub range: Range, } impl SliceData { - fn try_new(child: ArrayRef, range: Range) -> VortexResult { + pub fn try_new(child: ArrayRef, range: Range) -> VortexResult { if range.end > child.len() { vortex_panic!( "SliceArray range out of bounds: range {:?} exceeds child array length {}", @@ -41,6 +42,7 @@ impl SliceData { Ok(Self { slots: vec![Some(child)], range, + stats: ArrayStats::default(), }) } @@ -74,31 +76,29 @@ impl SliceData { .as_ref() .vortex_expect("SliceArray child slot") } - - pub fn into_parts(mut self) -> SliceDataParts { - SliceDataParts { - child: self.slots[CHILD_SLOT] - .take() - .vortex_expect("SliceArray child slot"), - range: self.range, - } - } } impl Array { /// Constructs a new `SliceArray`. pub fn try_new(child: ArrayRef, range: Range) -> VortexResult { - let len = range.len(); - let dtype = child.dtype().clone(); - let data = SliceData::try_new(child, range)?; - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Slice, dtype, len, data)) }) + Array::try_from_data(SliceData::try_new(child, range)?) } /// Constructs a new `SliceArray`. pub fn new(child: ArrayRef, range: Range) -> Self { - let len = range.len(); - let dtype = child.dtype().clone(); - let data = SliceData::new(child, range); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Slice, dtype, len, data)) } + Array::try_from_data(SliceData::new(child, range)) + .vortex_expect("SliceData is always valid") + } +} + +impl SliceData { + /// Consume the slice array and return its components. + pub fn into_parts(mut self) -> SliceArrayParts { + SliceArrayParts { + child: self.slots[CHILD_SLOT] + .take() + .vortex_expect("SliceArray child slot"), + range: self.range, + } } } diff --git a/vortex-array/src/arrays/slice/mod.rs b/vortex-array/src/arrays/slice/mod.rs index f701a274a14..928367ea213 100644 --- a/vortex-array/src/arrays/slice/mod.rs +++ b/vortex-array/src/arrays/slice/mod.rs @@ -15,8 +15,8 @@ mod vtable; use std::ops::Range; +pub use array::SliceArrayParts; pub use array::SliceData; -pub use array::SliceDataParts; use vortex_error::VortexResult; pub use vtable::*; diff --git a/vortex-array/src/arrays/slice/vtable.rs b/vortex-array/src/arrays/slice/vtable.rs index 562fcd4c117..c5fa9241001 100644 --- a/vortex-array/src/arrays/slice/vtable.rs +++ b/vortex-array/src/arrays/slice/vtable.rs @@ -36,6 +36,7 @@ use crate::executor::ExecutionCtx; use crate::executor::ExecutionResult; use crate::scalar::Scalar; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; @@ -50,33 +51,27 @@ impl Slice { impl VTable for Slice { type ArrayData = SliceData; + type Metadata = SliceMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &SliceData) -> &Self { + &Slice + } fn id(&self) -> ArrayId { Slice::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.child().dtype() == dtype, - "SliceArray dtype {} does not match outer dtype {}", - data.child().dtype(), - dtype - ); - vortex_ensure!( - data.len() == len, - "SliceArray length {} does not match outer length {}", - data.len(), - len - ); - vortex_ensure!( - data.range.end <= data.child().len(), - "SliceArray range {:?} exceeds child length {}", - data.range, - data.child().len() - ); - Ok(()) + fn len(array: &SliceData) -> usize { + array.range.len() + } + + fn dtype(array: &SliceData) -> &DType { + array.child().dtype() + } + + fn stats(array: &SliceData) -> &ArrayStats { + &array.stats } fn array_hash(array: &SliceData, state: &mut H, precision: Precision) { @@ -109,24 +104,37 @@ impl VTable for Slice { SLOT_NAMES[idx].to_string() } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(SliceMetadata(array.range.clone())) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { // TODO(joe): make this configurable vortex_bail!("Slice array is not serializable") } fn deserialize( - &self, + _bytes: &[u8], _dtype: &DType, _len: usize, - _metadata: &[u8], - _buffers: &[BufferHandle], - _children: &dyn ArrayChildren, _session: &VortexSession, - ) -> VortexResult { + ) -> VortexResult { vortex_bail!("Slice array is not serializable") } + fn build( + dtype: &DType, + len: usize, + metadata: &SliceMetadata, + _buffers: &[BufferHandle], + children: &dyn ArrayChildren, + ) -> VortexResult { + assert_eq!(len, metadata.0.len()); + let child = children.get(0, dtype, metadata.0.end)?; + SliceData::try_new(child, metadata.0.clone()) + } + fn with_slots(array: &mut Self::ArrayData, slots: Vec>) -> VortexResult<()> { vortex_ensure!( slots.len() == NUM_SLOTS, diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 3f75a48e958..f8ce8a54bae 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -13,7 +13,6 @@ use vortex_error::vortex_err; use crate::ArrayRef; use crate::IntoArray; use crate::array::Array; -use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::Struct; @@ -21,6 +20,7 @@ use crate::dtype::DType; use crate::dtype::FieldName; use crate::dtype::FieldNames; use crate::dtype::StructFields; +use crate::stats::ArrayStats; use crate::validity::Validity; // StructArray has a variable number of slots: [validity?, field_0, ..., field_N] @@ -149,13 +149,13 @@ pub(super) const FIELDS_OFFSET: usize = 1; /// ``` #[derive(Clone, Debug)] pub struct StructData { - pub(super) names: FieldNames, - pub(super) nullability: crate::dtype::Nullability, - pub(super) fieldless_len: Option, + pub(super) len: usize, + pub(super) dtype: DType, pub(super) slots: Vec>, + pub(super) stats_set: ArrayStats, } -pub struct StructDataParts { +pub struct StructArrayParts { pub struct_fields: StructFields, pub fields: Arc<[ArrayRef]>, pub validity: Validity, @@ -164,15 +164,12 @@ pub struct StructDataParts { impl StructData { /// Returns the length of this array. pub fn len(&self) -> usize { - self.slots[FIELDS_OFFSET..] - .first() - .and_then(|slot| slot.as_ref()) - .map_or_else(|| self.fieldless_len.unwrap_or(0), |field| field.len()) + self.len } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Struct(self.struct_fields(), self.nullability) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns `true` if this array is empty. @@ -182,7 +179,7 @@ impl StructData { /// Reconstructs the validity from the slots. pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } /// Return an iterator over the struct fields without the validity of the struct applied. @@ -218,7 +215,7 @@ impl StructData { /// Return the struct field without the validity of the struct applied pub fn unmasked_field_by_name_opt(&self, name: impl AsRef) -> Option<&ArrayRef> { let name = name.as_ref(); - self.names.find(name).map(|idx| { + self.struct_fields().find(name).map(|idx| { self.slots[FIELDS_OFFSET + idx] .as_ref() .vortex_expect("StructArray field slot") @@ -226,16 +223,16 @@ impl StructData { } pub fn names(&self) -> &FieldNames { - &self.names + self.struct_fields().names() } - pub fn struct_fields(&self) -> StructFields { - StructFields::new( - self.names.clone(), - self.iter_unmasked_fields() - .map(|field| field.dtype().clone()) - .collect(), - ) + pub fn struct_fields(&self) -> &StructFields { + let Some(struct_dtype) = &self.dtype.as_struct_fields_opt() else { + unreachable!( + "struct arrays must have be a DType::Struct, this is likely an internal bug." + ) + }; + struct_dtype } /// Create a new `StructArray` with the given length, but without any fields. @@ -273,7 +270,7 @@ impl StructData { /// /// Returns an error if the provided components do not satisfy the invariants documented in /// `StructArray::new_unchecked`. - pub(crate) fn try_new( + pub fn try_new( names: FieldNames, fields: impl Into>, length: usize, @@ -332,10 +329,10 @@ impl StructData { .collect(); Self { - names: dtype.names().clone(), - nullability: validity.nullability(), - fieldless_len: fields.is_empty().then_some(length), + len: length, + dtype: DType::Struct(dtype, validity.nullability()), slots, + stats_set: Default::default(), } } @@ -392,7 +389,7 @@ impl StructData { Ok(()) } - pub(crate) fn try_new_with_dtype( + pub fn try_new_with_dtype( fields: impl Into>, dtype: StructFields, length: usize, @@ -405,33 +402,16 @@ impl StructData { Ok(unsafe { Self::new_unchecked(fields, dtype, length, validity) }) } - pub fn into_parts(self) -> StructDataParts { - let StructData { - names, - slots, - nullability, - .. - } = self; - let validity = child_to_validity(&slots[VALIDITY_SLOT], nullability); - let struct_fields = StructFields::new( - names, - slots - .iter() - .skip(FIELDS_OFFSET) - .map(|slot| { - slot.as_ref() - .vortex_expect("StructArray field slot") - .dtype() - .clone() - }) - .collect(), - ); - let fields: Arc<[ArrayRef]> = slots + pub fn into_parts(self) -> StructArrayParts { + let validity = self.validity(); + let struct_fields = self.dtype.into_struct_fields(); + let fields: Arc<[ArrayRef]> = self + .slots .into_iter() .skip(FIELDS_OFFSET) .map(|s| s.vortex_expect("StructArray field slot")) .collect(); - StructDataParts { + StructArrayParts { struct_fields, fields, validity, @@ -472,13 +452,47 @@ impl StructData { Self::try_from_iter_with_validity(iter, Validity::NonNullable) } + // TODO(aduffy): Add equivalent function to support field masks for nested column access. + /// Return a new StructArray with the given projection applied. + /// + /// Projection does not copy data arrays. Projection is defined by an ordinal array slice + /// which specifies the new ordering of columns in the struct. The projection can be used to + /// perform column re-ordering, deletion, or duplication at a logical level, without any data + /// copying. + pub fn project(&self, projection: &[FieldName]) -> VortexResult { + let mut children = Vec::with_capacity(projection.len()); + let mut names = Vec::with_capacity(projection.len()); + + for f_name in projection.iter() { + let idx = self + .names() + .iter() + .position(|name| name == f_name) + .ok_or_else(|| vortex_err!("Unknown field {f_name}"))?; + + names.push(self.names()[idx].clone()); + children.push( + self.slots[FIELDS_OFFSET + idx] + .as_ref() + .vortex_expect("StructArray field slot") + .clone(), + ); + } + + StructData::try_new( + FieldNames::from(names.as_slice()), + children, + self.len(), + self.validity(), + ) + } + /// Removes and returns a column from the struct array by name. /// If the column does not exist, returns `None`. pub fn remove_column(&mut self, name: impl Into) -> Option { let name = name.into(); - let struct_dtype = self.struct_fields(); - let len = self.len(); + let struct_dtype = self.struct_fields().clone(); let position = struct_dtype .names() @@ -500,13 +514,7 @@ impl StructData { if let Ok(new_dtype) = struct_dtype.without_field(position) { self.slots = new_slots; - self.names = new_dtype.names().clone(); - self.fieldless_len = self - .slots - .get(FIELDS_OFFSET) - .and_then(|slot| slot.as_ref()) - .map(|_| None) - .unwrap_or(Some(len)); + self.dtype = DType::Struct(new_dtype, self.dtype.nullability()); return Some(field); } None @@ -521,10 +529,8 @@ impl Array { length: usize, validity: Validity, ) -> Self { - let data = StructData::new(names, fields, length, validity); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } + Array::try_from_data(StructData::new(names, fields, length, validity)) + .vortex_expect("StructData is always valid") } /// Constructs a new `StructArray`. @@ -534,10 +540,7 @@ impl Array { length: usize, validity: Validity, ) -> VortexResult { - let data = StructData::try_new(names, fields, length, validity)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + Array::try_from_data(StructData::try_new(names, fields, length, validity)?) } /// Creates a new `StructArray` without validation. @@ -551,10 +554,8 @@ impl Array { length: usize, validity: Validity, ) -> Self { - let data = unsafe { StructData::new_unchecked(fields, dtype, length, validity) }; - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } + Array::try_from_data(unsafe { StructData::new_unchecked(fields, dtype, length, validity) }) + .vortex_expect("StructData is always valid") } /// Constructs a new `StructArray` with an explicit dtype. @@ -564,18 +565,19 @@ impl Array { length: usize, validity: Validity, ) -> VortexResult { - let data = StructData::try_new_with_dtype(fields, dtype, length, validity)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + Array::try_from_data(StructData::try_new_with_dtype( + fields, dtype, length, validity, + )?) } /// Construct a `StructArray` from named fields. pub fn from_fields>(items: &[(N, ArrayRef)]) -> VortexResult { - let data = StructData::from_fields(items)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + Array::try_from_data(StructData::from_fields(items)?) + } + + /// Decompose this struct array into its constituent parts. + pub fn into_parts(self) -> StructArrayParts { + self.into_data().into_parts() } /// Create a `StructArray` from an iterator of (name, array) pairs with validity. @@ -587,65 +589,27 @@ impl Array { iter: T, validity: Validity, ) -> VortexResult { - let data = StructData::try_from_iter_with_validity(iter, validity)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) + Array::try_from_data(StructData::try_from_iter_with_validity(iter, validity)?) } /// Create a `StructArray` from an iterator of (name, array) pairs. pub fn try_from_iter, A: IntoArray, T: IntoIterator>( iter: T, ) -> VortexResult { - let data = StructData::try_from_iter(iter)?; - let dtype = data.dtype(); - let len = data.len(); - Ok(unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) }) - } - - // TODO(aduffy): Add equivalent function to support field masks for nested column access. - /// Return a new StructArray with the given projection applied. - /// - /// Projection does not copy data arrays. Projection is defined by an ordinal array slice - /// which specifies the new ordering of columns in the struct. The projection can be used to - /// perform column re-ordering, deletion, or duplication at a logical level, without any data - /// copying. - pub fn project(&self, projection: &[FieldName]) -> VortexResult { - let mut children = Vec::with_capacity(projection.len()); - let mut names = Vec::with_capacity(projection.len()); - - for f_name in projection { - let idx = self - .names() - .iter() - .position(|name| name == f_name) - .ok_or_else(|| vortex_err!("Unknown field {f_name}"))?; - - names.push(self.names()[idx].clone()); - children.push(self.unmasked_field(idx).clone()); - } - - Self::try_new( - FieldNames::from(names.as_slice()), - children, - self.len(), - self.validity(), - ) + Array::try_from_data(StructData::try_from_iter(iter)?) } /// Create a fieldless `StructArray` with the given length. pub fn new_fieldless_with_len(len: usize) -> Self { - let data = StructData::new_fieldless_with_len(len); - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(Struct, dtype, len, data)) } + Array::try_from_data(StructData::new_fieldless_with_len(len)) + .vortex_expect("StructData is always valid") } } impl StructData { pub fn with_column(&self, name: impl Into, array: ArrayRef) -> VortexResult { let name = name.into(); - let struct_dtype = self.struct_fields(); + let struct_dtype = self.struct_fields().clone(); let names = struct_dtype.names().iter().cloned().chain(once(name)); let types = struct_dtype.fields().chain(once(array.dtype().clone())); @@ -657,6 +621,6 @@ impl StructData { .chain(once(array)) .collect(); - Self::try_new_with_dtype(children, new_fields, self.len(), self.validity()) + Self::try_new_with_dtype(children, new_fields, self.len, self.validity()) } } diff --git a/vortex-array/src/arrays/struct_/compute/mask.rs b/vortex-array/src/arrays/struct_/compute/mask.rs index c17493c11bf..cdf8a772bbf 100644 --- a/vortex-array/src/arrays/struct_/compute/mask.rs +++ b/vortex-array/src/arrays/struct_/compute/mask.rs @@ -15,7 +15,7 @@ impl MaskReduce for Struct { fn mask(array: ArrayView<'_, Struct>, mask: &ArrayRef) -> VortexResult> { StructArray::try_new_with_dtype( array.unmasked_fields().iter().cloned().collect::>(), - array.struct_fields(), + array.struct_fields().clone(), array.len(), array.validity().and(Validity::Array(mask.clone()))?, ) diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs index 4d21b190151..ddb471e8930 100644 --- a/vortex-array/src/arrays/struct_/compute/rules.rs +++ b/vortex-array/src/arrays/struct_/compute/rules.rs @@ -82,7 +82,7 @@ impl ArrayParentReduceRule for StructCastPushDownRule { } else { array .validity() - .into_non_nullable(array.len()) + .into_non_nullable(array.len) .ok_or_else(|| vortex_err!("Failed to cast nullable struct to non-nullable"))? }; diff --git a/vortex-array/src/arrays/struct_/compute/slice.rs b/vortex-array/src/arrays/struct_/compute/slice.rs index 0f36d87c560..8d4a4a69737 100644 --- a/vortex-array/src/arrays/struct_/compute/slice.rs +++ b/vortex-array/src/arrays/struct_/compute/slice.rs @@ -25,7 +25,7 @@ impl SliceReduce for Struct { unsafe { StructArray::new_unchecked( fields, - array.struct_fields(), + array.struct_fields().clone(), range.len(), array.validity().slice(range)?, ) diff --git a/vortex-array/src/arrays/struct_/compute/take.rs b/vortex-array/src/arrays/struct_/compute/take.rs index 8ce8b3f3851..8836b45a2aa 100644 --- a/vortex-array/src/arrays/struct_/compute/take.rs +++ b/vortex-array/src/arrays/struct_/compute/take.rs @@ -20,7 +20,7 @@ impl TakeReduce for Struct { if array.is_empty() { return StructArray::try_new_with_dtype( array.iter_unmasked_fields().cloned().collect::>(), - array.struct_fields(), + array.struct_fields().clone(), indices.len(), Validity::AllInvalid, ) @@ -41,7 +41,7 @@ impl TakeReduce for Struct { .iter_unmasked_fields() .map(|field| field.take(inner_indices.clone())) .collect::, _>>()?, - array.struct_fields(), + array.struct_fields().clone(), indices.len(), array.validity().take(indices)?, ) diff --git a/vortex-array/src/arrays/struct_/compute/zip.rs b/vortex-array/src/arrays/struct_/compute/zip.rs index 75fe4288dd6..bf13f4f72cf 100644 --- a/vortex-array/src/arrays/struct_/compute/zip.rs +++ b/vortex-array/src/arrays/struct_/compute/zip.rs @@ -54,7 +54,7 @@ impl ZipKernel for Struct { let combined = (v1m.bitand(&mask_mask)).bitor(&v2m.bitand(&mask_mask.not())); Validity::from_mask( combined, - if_true.dtype().nullability() | if_false.dtype().nullability(), + if_true.dtype.nullability() | if_false.dtype.nullability(), ) } }; diff --git a/vortex-array/src/arrays/struct_/mod.rs b/vortex-array/src/arrays/struct_/mod.rs index 65b53e2d603..b867f903901 100644 --- a/vortex-array/src/arrays/struct_/mod.rs +++ b/vortex-array/src/arrays/struct_/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::StructArrayParts; pub use array::StructData; -pub use array::StructDataParts; pub use vtable::StructArray; pub(crate) mod compute; diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 3d32534cd3f..819db769eff 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -10,6 +10,7 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; +use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::array::Array; @@ -32,19 +33,36 @@ use crate::Precision; use crate::array::ArrayId; use crate::hash::ArrayEq; use crate::hash::ArrayHash; +use crate::stats::ArrayStats; vtable!(Struct, Struct, StructData); impl VTable for Struct { type ArrayData = StructData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Struct + } fn id(&self) -> ArrayId { Self::ID } + fn len(array: &StructData) -> usize { + array.len + } + + fn dtype(array: &StructData) -> &DType { + &array.dtype + } + + fn stats(array: &StructData) -> &ArrayStats { + &array.stats_set + } + fn array_hash(array: &StructData, state: &mut H, precision: Precision) { for field in array.iter_unmasked_fields() { field.array_hash(state, precision); @@ -65,29 +83,6 @@ impl VTable for Struct { 0 } - fn validate(&self, data: &StructData, dtype: &DType, len: usize) -> VortexResult<()> { - match dtype { - DType::Struct(..) => {} - _ => vortex_bail!("Expected struct dtype, found {:?}", dtype), - } - if data.len() != len { - vortex_bail!( - InvalidArgument: "StructArray length {} does not match outer length {}", - data.len(), - len - ); - } - let data_dtype = data.dtype(); - if &data_dtype != dtype { - vortex_bail!( - InvalidArgument: "StructArray dtype {} does not match outer dtype {}", - data_dtype, - dtype - ); - } - Ok(()) - } - fn buffer(_array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { vortex_panic!("StructArray buffer index {idx} out of bounds") } @@ -96,26 +91,31 @@ impl VTable for Struct { vortex_panic!("StructArray buffer_name index {idx} out of bounds") } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + _metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - if !metadata.is_empty() { - vortex_bail!( - "StructArray expects empty metadata, got {} bytes", - metadata.len() - ); - } let DType::Struct(struct_dtype, nullability) = dtype else { vortex_bail!("Expected struct dtype, found {:?}", dtype) }; diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 329cf3da04c..c50c06285ef 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -5,7 +5,6 @@ use num_traits::AsPrimitive; use vortex_buffer::ByteBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_mask::Mask; @@ -13,7 +12,6 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::ToCanonical; use crate::array::Array; -use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBin; @@ -23,6 +21,7 @@ use crate::dtype::DType; use crate::dtype::IntegerPType; use crate::dtype::Nullability; use crate::match_each_integer_ptype; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The offsets array defining the start/end of each variable-length binary element. @@ -34,29 +33,13 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["offsets", "validity"]; #[derive(Clone, Debug)] pub struct VarBinData { - pub(super) is_utf8: bool, - pub(super) nullability: Nullability, + pub(super) dtype: DType, pub(super) bytes: BufferHandle, pub(super) slots: Vec>, + pub(super) stats_set: ArrayStats, } impl VarBinData { - fn dtype_parts(dtype: &DType) -> VortexResult<(bool, Nullability)> { - match dtype { - DType::Utf8(nullability) => Ok((true, *nullability)), - DType::Binary(nullability) => Ok((false, *nullability)), - _ => vortex_bail!(MismatchedTypes: "utf8 or binary", dtype), - } - } - - fn make_dtype(is_utf8: bool, nullability: Nullability) -> DType { - if is_utf8 { - DType::Utf8(nullability) - } else { - DType::Binary(nullability) - } - } - /// Creates a new `VarBinArray`. /// /// # Panics @@ -183,14 +166,12 @@ impl VarBinData { let len = offsets.len().saturating_sub(1); let validity_slot = validity_to_child(&validity, len); - let (is_utf8, nullability) = - Self::dtype_parts(&dtype).vortex_expect("VarBinArray dtype must be utf8 or binary"); Self { - is_utf8, - nullability, + dtype, bytes, slots: vec![Some(offsets), validity_slot], + stats_set: Default::default(), } } @@ -294,8 +275,8 @@ impl VarBinData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - Self::make_dtype(self.is_utf8, self.nullability) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns `true` if this array is empty. @@ -306,7 +287,7 @@ impl VarBinData { /// Returns the [`Validity`] of this array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } /// Returns the validity as a [`Mask`]. @@ -400,15 +381,9 @@ impl VarBinData { /// Forwarding constructors for `VarBinArray` (= `Array`). impl Array { - #[inline] - fn from_prevalidated_data(data: VarBinData) -> Self { - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBin, dtype, len, data)) } - } - pub fn from_vec>(vec: Vec, dtype: DType) -> Self { - Self::from_prevalidated_data(VarBinData::from_vec(vec, dtype)) + Array::try_from_data(VarBinData::from_vec(vec, dtype)) + .vortex_expect("VarBinData is always valid") } #[expect( @@ -419,14 +394,16 @@ impl Array { iter: I, dtype: DType, ) -> Self { - Self::from_prevalidated_data(VarBinData::from_iter(iter, dtype)) + Array::try_from_data(VarBinData::from_iter(iter, dtype)) + .vortex_expect("VarBinData is always valid") } pub fn from_iter_nonnull, I: IntoIterator>( iter: I, dtype: DType, ) -> Self { - Self::from_prevalidated_data(VarBinData::from_iter_nonnull(iter, dtype)) + Array::try_from_data(VarBinData::from_iter_nonnull(iter, dtype)) + .vortex_expect("VarBinData is always valid") } /// Create from a vector of string slices. @@ -488,14 +465,15 @@ impl VarBinData { let offsets = self.slots[OFFSETS_SLOT] .take() .vortex_expect("VarBinArray offsets slot"); - (self.dtype(), self.bytes, offsets, validity) + (self.dtype, self.bytes, offsets, validity) } } impl Array { /// Creates a new `VarBinArray`. pub fn new(offsets: ArrayRef, bytes: ByteBuffer, dtype: DType, validity: Validity) -> Self { - Self::from_prevalidated_data(VarBinData::new(offsets, bytes, dtype, validity)) + Array::try_from_data(VarBinData::new(offsets, bytes, dtype, validity)) + .vortex_expect("VarBinData is always valid") } /// Creates a new `VarBinArray` without validation. @@ -509,9 +487,8 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(unsafe { - VarBinData::new_unchecked(offsets, bytes, dtype, validity) - }) + Array::try_from_data(unsafe { VarBinData::new_unchecked(offsets, bytes, dtype, validity) }) + .vortex_expect("VarBinData is always valid") } /// Creates a new `VarBinArray` without validation from a [`BufferHandle`]. @@ -525,9 +502,10 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(unsafe { + Array::try_from_data(unsafe { VarBinData::new_unchecked_from_handle(offsets, bytes, dtype, validity) }) + .vortex_expect("VarBinData is always valid") } /// Constructs a new `VarBinArray`. @@ -537,9 +515,7 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - Ok(Self::from_prevalidated_data(VarBinData::try_new( - offsets, bytes, dtype, validity, - )?)) + Array::try_from_data(VarBinData::try_new(offsets, bytes, dtype, validity)?) } } @@ -619,72 +595,76 @@ impl<'a> FromIterator> for VarBinData { impl From> for Array { fn from(value: Vec<&[u8]>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") } } impl From>> for Array { fn from(value: Vec>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") } } impl From> for Array { fn from(value: Vec) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") } } impl From> for Array { fn from(value: Vec<&str>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") } } impl From>> for Array { fn from(value: Vec>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") } } impl From>>> for Array { fn from(value: Vec>>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") } } impl From>> for Array { fn from(value: Vec>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") } } impl From>> for Array { fn from(value: Vec>) -> Self { - Self::from_prevalidated_data(VarBinData::from(value)) + Array::try_from_data(VarBinData::from(value)).vortex_expect("VarBinData is always valid") } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Array::try_from_data(>::from_iter(iter)) + .vortex_expect(" is always valid") } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Array::try_from_data(>::from_iter(iter)) + .vortex_expect(" is always valid") } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Array::try_from_data(>::from_iter(iter)) + .vortex_expect(" is always valid") } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Array::try_from_data(>::from_iter(iter)) + .vortex_expect(" is always valid") } } diff --git a/vortex-array/src/arrays/varbin/builder.rs b/vortex-array/src/arrays/varbin/builder.rs index 8dd5066a889..78675e337d7 100644 --- a/vortex-array/src/arrays/varbin/builder.rs +++ b/vortex-array/src/arrays/varbin/builder.rs @@ -7,6 +7,8 @@ use vortex_buffer::BufferMut; use vortex_error::vortex_panic; use crate::IntoArray; +use crate::array::VTable; +use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; use crate::arrays::VarBinArray; use crate::dtype::DType; @@ -103,9 +105,7 @@ impl VarBinBuilder { offsets.statistics().compute_is_sorted().unwrap_or(false), "VarBinBuilder offsets must be sorted" ); - offsets - .statistics() - .set(Stat::IsSorted, Precision::Exact(true.into())); + Primitive::stats(&offsets).set(Stat::IsSorted, Precision::Exact(true.into())); // SAFETY: The builder maintains all invariants: // - Offsets are monotonically increasing starting from 0 (guaranteed by builder logic). diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index 0b6813ddc99..8b8664ce506 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use prost::Message; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -9,9 +8,12 @@ use vortex_error::vortex_ensure; use vortex_error::vortex_panic; use crate::ArrayRef; +use crate::DeserializeMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::IntoArray; +use crate::ProstMetadata; +use crate::SerializeMetadata; use crate::array::Array; use crate::array::ArrayId; use crate::array::ArrayView; @@ -39,6 +41,7 @@ use crate::Precision; use crate::arrays::varbin::compute::rules::PARENT_RULES; use crate::hash::ArrayEq; use crate::hash::ArrayHash; +use crate::stats::ArrayStats; vtable!(VarBin, VarBin, VarBinData); @@ -51,13 +54,29 @@ pub struct VarBinMetadata { impl VTable for VarBin { type ArrayData = VarBinData; + type Metadata = ProstMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &VarBinData) -> &Self { + &VarBin + } fn id(&self) -> ArrayId { Self::ID } + fn len(array: &VarBinData) -> usize { + array.offsets().len().saturating_sub(1) + } + + fn dtype(array: &VarBinData) -> &DType { + &array.dtype + } + + fn stats(array: &VarBinData) -> &ArrayStats { + &array.stats_set + } + fn array_hash(array: &VarBinData, state: &mut H, precision: Precision) { array.bytes().array_hash(state, precision); array.offsets().array_hash(state, precision); @@ -74,22 +93,6 @@ impl VTable for VarBin { 1 } - fn validate(&self, data: &VarBinData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "VarBinArray length {} does not match outer length {}", - data.len(), - len - ); - vortex_ensure!( - data.dtype() == *dtype, - "VarBinArray dtype {} does not match outer dtype {}", - data.dtype(), - dtype - ); - Ok(()) - } - fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { match idx { 0 => array.bytes_handle().clone(), @@ -104,27 +107,36 @@ impl VTable for VarBin { } } - fn serialize(array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(Some( - VarBinMetadata { - offsets_ptype: PType::try_from(array.offsets().dtype()) - .vortex_expect("Must be a valid PType") as i32, - } - .encode_to_vec(), - )) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Ok(ProstMetadata(VarBinMetadata { + offsets_ptype: PType::try_from(array.offsets().dtype()) + .vortex_expect("Must be a valid PType") as i32, + })) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(ProstMetadata(ProstMetadata::::deserialize( + bytes, + )?)) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - let metadata = VarBinMetadata::decode(metadata)?; let validity = if children.len() == 1 { Validity::from(dtype.nullability()) } else if children.len() == 2 { diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index fd61c204feb..d7228c65ed0 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -17,7 +17,6 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayParts; use crate::array::child_to_validity; use crate::array::validity_to_child; use crate::arrays::VarBinView; @@ -27,6 +26,7 @@ use crate::builders::ArrayBuilder; use crate::builders::VarBinViewBuilder; use crate::dtype::DType; use crate::dtype::Nullability; +use crate::stats::ArrayStats; use crate::validity::Validity; /// The validity bitmap indicating which elements are non-null. @@ -96,13 +96,13 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["validity"]; #[derive(Clone, Debug)] pub struct VarBinViewData { pub(super) slots: Vec>, - pub(super) is_utf8: bool, - pub(super) nullability: Nullability, + pub(super) dtype: DType, pub(super) buffers: Arc<[BufferHandle]>, pub(super) views: BufferHandle, + pub(super) stats_set: ArrayStats, } -pub struct VarBinViewDataParts { +pub struct VarBinViewArrayParts { pub dtype: DType, pub buffers: Arc<[BufferHandle]>, pub views: BufferHandle, @@ -110,22 +110,6 @@ pub struct VarBinViewDataParts { } impl VarBinViewData { - fn dtype_parts(dtype: &DType) -> VortexResult<(bool, Nullability)> { - match dtype { - DType::Utf8(nullability) => Ok((true, *nullability)), - DType::Binary(nullability) => Ok((false, *nullability)), - _ => vortex_bail!(InvalidArgument: "invalid DType {dtype} for `VarBinViewArray`"), - } - } - - fn make_dtype(is_utf8: bool, nullability: Nullability) -> DType { - if is_utf8 { - DType::Utf8(nullability) - } else { - DType::Binary(nullability) - } - } - /// Build the slots vector for this array. pub(super) fn make_slots(validity: &Validity, len: usize) -> Vec> { vec![validity_to_child(validity, len)] @@ -279,14 +263,12 @@ impl VarBinViewData { ) -> Self { let len = views.len() / size_of::(); let slots = Self::make_slots(&validity, len); - let (is_utf8, nullability) = - Self::dtype_parts(&dtype).vortex_expect("VarBinViewArray dtype must be utf8 or binary"); Self { slots, - is_utf8, - nullability, - buffers, views, + buffers, + dtype, + stats_set: Default::default(), } } @@ -385,8 +367,8 @@ impl VarBinViewData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - Self::make_dtype(self.is_utf8, self.nullability) + pub fn dtype(&self) -> &DType { + &self.dtype } /// Returns `true` if this array is empty. @@ -397,7 +379,7 @@ impl VarBinViewData { /// Returns the [`Validity`] of this array. #[allow(clippy::same_name_method)] pub fn validity(&self) -> Validity { - child_to_validity(&self.slots[VALIDITY_SLOT], self.nullability) + child_to_validity(&self.slots[VALIDITY_SLOT], self.dtype.nullability()) } /// Returns the validity as a [`Mask`]. @@ -406,10 +388,10 @@ impl VarBinViewData { } /// Splits the array into owned parts - pub fn into_parts(self) -> VarBinViewDataParts { + pub fn into_parts(self) -> VarBinViewArrayParts { let validity = self.validity(); - VarBinViewDataParts { - dtype: self.dtype(), + VarBinViewArrayParts { + dtype: self.dtype, buffers: self.buffers, views: self.views, validity, @@ -570,13 +552,6 @@ impl VarBinViewData { } impl Array { - #[inline] - fn from_prevalidated_data(data: VarBinViewData) -> Self { - let dtype = data.dtype(); - let len = data.len(); - unsafe { Array::from_parts_unchecked(ArrayParts::new(VarBinView, dtype, len, data)) } - } - /// Construct a `VarBinViewArray` from an iterator of optional byte slices. #[expect( clippy::same_name_method, @@ -586,27 +561,32 @@ impl Array { iter: I, dtype: DType, ) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter(iter, dtype)) + Array::try_from_data(VarBinViewData::from_iter(iter, dtype)) + .vortex_expect("VarBinViewData is always valid") } pub fn from_iter_str, I: IntoIterator>(iter: I) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter_str(iter)) + Array::try_from_data(VarBinViewData::from_iter_str(iter)) + .vortex_expect("VarBinViewData is always valid") } pub fn from_iter_nullable_str, I: IntoIterator>>( iter: I, ) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter_nullable_str(iter)) + Array::try_from_data(VarBinViewData::from_iter_nullable_str(iter)) + .vortex_expect("VarBinViewData is always valid") } pub fn from_iter_bin, I: IntoIterator>(iter: I) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter_bin(iter)) + Array::try_from_data(VarBinViewData::from_iter_bin(iter)) + .vortex_expect("VarBinViewData is always valid") } pub fn from_iter_nullable_bin, I: IntoIterator>>( iter: I, ) -> Self { - Self::from_prevalidated_data(VarBinViewData::from_iter_nullable_bin(iter)) + Array::try_from_data(VarBinViewData::from_iter_nullable_bin(iter)) + .vortex_expect("VarBinViewData is always valid") } /// Creates a new `VarBinViewArray`. @@ -616,9 +596,7 @@ impl Array { dtype: DType, validity: Validity, ) -> VortexResult { - Ok(Self::from_prevalidated_data(VarBinViewData::try_new( - views, buffers, dtype, validity, - )?)) + Array::try_from_data(VarBinViewData::try_new(views, buffers, dtype, validity)?) } /// Creates a new `VarBinViewArray` without validation. @@ -632,9 +610,10 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(unsafe { + Array::try_from_data(unsafe { VarBinViewData::new_unchecked(views, buffers, dtype, validity) }) + .vortex_expect("VarBinViewData is always valid") } /// Creates a new `VarBinViewArray` with device or host memory. @@ -644,7 +623,8 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(VarBinViewData::new_handle(views, buffers, dtype, validity)) + Array::try_from_data(VarBinViewData::new_handle(views, buffers, dtype, validity)) + .vortex_expect("VarBinViewData is always valid") } /// Construct a new array from `BufferHandle`s without validation. @@ -658,9 +638,10 @@ impl Array { dtype: DType, validity: Validity, ) -> Self { - Self::from_prevalidated_data(unsafe { + Array::try_from_data(unsafe { VarBinViewData::new_handle_unchecked(views, buffers, dtype, validity) }) + .vortex_expect("VarBinViewData is always valid") } } @@ -692,24 +673,28 @@ impl<'a> FromIterator> for VarBinViewData { impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Array::try_from_data(>::from_iter(iter)) + .vortex_expect(" is always valid") } } impl FromIterator>> for Array { fn from_iter>>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Array::try_from_data(>::from_iter(iter)) + .vortex_expect(" is always valid") } } impl FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Array::try_from_data(>::from_iter(iter)) + .vortex_expect(" is always valid") } } impl<'a> FromIterator> for Array { fn from_iter>>(iter: T) -> Self { - Self::from_prevalidated_data(>::from_iter(iter)) + Array::try_from_data(>::from_iter(iter)) + .vortex_expect(" is always valid") } } diff --git a/vortex-array/src/arrays/varbinview/mod.rs b/vortex-array/src/arrays/varbinview/mod.rs index d2d0041424d..d29fd5bf3e5 100644 --- a/vortex-array/src/arrays/varbinview/mod.rs +++ b/vortex-array/src/arrays/varbinview/mod.rs @@ -2,8 +2,8 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod array; +pub use array::VarBinViewArrayParts; pub use array::VarBinViewData; -pub use array::VarBinViewDataParts; pub use vtable::VarBinViewArray; mod accessor; diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 7c2b674aa9e..2efb586e619 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -14,6 +14,7 @@ use vortex_error::vortex_panic; use vortex_session::VortexSession; use crate::ArrayRef; +use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -31,6 +32,7 @@ use crate::dtype::DType; use crate::hash::ArrayEq; use crate::hash::ArrayHash; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::validity::Validity; use crate::vtable; mod kernel; @@ -48,13 +50,29 @@ impl VarBinView { impl VTable for VarBinView { type ArrayData = VarBinViewData; + type Metadata = EmptyMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &VarBinViewData) -> &Self { + &VarBinView + } fn id(&self) -> ArrayId { Self::ID } + fn len(array: &VarBinViewData) -> usize { + array.views_handle().len() / size_of::() + } + + fn dtype(array: &VarBinViewData) -> &DType { + &array.dtype + } + + fn stats(array: &VarBinViewData) -> &ArrayStats { + &array.stats_set + } + fn array_hash( array: &VarBinViewData, state: &mut H, @@ -82,22 +100,6 @@ impl VTable for VarBinView { array.data_buffers().len() + 1 } - fn validate(&self, data: &VarBinViewData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - data.len() == len, - "VarBinViewArray length {} does not match outer length {}", - data.len(), - len - ); - vortex_ensure!( - data.dtype() == *dtype, - "VarBinViewArray dtype {} does not match outer dtype {}", - data.dtype(), - dtype - ); - Ok(()) - } - fn buffer(array: ArrayView<'_, Self>, idx: usize) -> BufferHandle { let ndata = array.data_buffers().len(); if idx < ndata { @@ -120,26 +122,31 @@ impl VTable for VarBinView { } } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + _metadata: &Self::Metadata, buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &VortexSession, ) -> VortexResult { - if !metadata.is_empty() { - vortex_bail!( - "VarBinViewArray expects empty metadata, got {} bytes", - metadata.len() - ); - } let Some((views_handle, data_handles)) = buffers.split_last() else { vortex_bail!("Expected at least 1 buffer, got 0"); }; @@ -235,8 +242,8 @@ mod tests { use crate::IntoArray; use crate::LEGACY_SESSION; use crate::assert_arrays_eq; + use crate::serde::ArrayParts; use crate::serde::SerializeOptions; - use crate::serde::SerializedArray; #[test] fn test_nullable_varbinview_serde_roundtrip() { @@ -261,7 +268,7 @@ mod tests { for buf in serialized { concat.extend_from_slice(buf.as_ref()); } - let parts = SerializedArray::try_from(concat.freeze()).unwrap(); + let parts = ArrayParts::try_from(concat.freeze()).unwrap(); let decoded = parts .decode( &dtype, diff --git a/vortex-array/src/arrays/variant/mod.rs b/vortex-array/src/arrays/variant/mod.rs index 02a89fae953..29fdb5edcd2 100644 --- a/vortex-array/src/arrays/variant/mod.rs +++ b/vortex-array/src/arrays/variant/mod.rs @@ -9,8 +9,8 @@ pub use self::vtable::Variant; pub use self::vtable::VariantArray; use crate::ArrayRef; use crate::array::Array; -use crate::array::ArrayParts; use crate::dtype::DType; +use crate::stats::ArrayStats; pub(super) const NUM_SLOTS: usize = 1; pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; @@ -21,18 +21,21 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["child"]; /// (e.g. a `ParquetVariantArray` or any other variant encoding). /// /// Nullability is delegated to the child array: `VariantArray`'s dtype is -/// always `DType::Variant(child.dtype().nullability())`. The child's validity -/// determines which rows are null. +/// always the child's dtype. The child's validity determines which rows are +/// null. #[derive(Clone, Debug)] pub struct VariantData { pub(super) slots: Vec>, + pub(crate) stats_set: ArrayStats, } impl VariantData { /// Creates a new VariantArray. Nullability comes from the child's dtype. pub fn new(child: ArrayRef) -> Self { + let stats_set = child.statistics().to_array_stats(); Self { slots: vec![Some(child)], + stats_set, } } @@ -42,8 +45,8 @@ impl VariantData { } /// Returns the [`DType`] of this array. - pub fn dtype(&self) -> DType { - DType::Variant(self.child().dtype().nullability()) + pub fn dtype(&self) -> &DType { + self.child().dtype() } /// Returns `true` if this array is empty. @@ -62,17 +65,6 @@ impl VariantData { impl Array { /// Creates a new `VariantArray`. pub fn new(child: ArrayRef) -> Self { - let dtype = DType::Variant(child.dtype().nullability()); - let len = child.len(); - let stats = child.statistics().to_owned(); - unsafe { - Array::from_parts_unchecked(ArrayParts::new( - Variant, - dtype, - len, - VariantData::new(child), - )) - } - .with_stats_set(stats) + Array::try_from_data(VariantData::new(child)).vortex_expect("VariantData is always valid") } } diff --git a/vortex-array/src/arrays/variant/vtable/mod.rs b/vortex-array/src/arrays/variant/vtable/mod.rs index 49981a6f955..31f52911e6d 100644 --- a/vortex-array/src/arrays/variant/vtable/mod.rs +++ b/vortex-array/src/arrays/variant/vtable/mod.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_panic; use crate::ArrayEq; use crate::ArrayHash; use crate::ArrayRef; +use crate::EmptyMetadata; use crate::ExecutionCtx; use crate::ExecutionResult; use crate::Precision; @@ -26,6 +27,7 @@ use crate::arrays::variant::VariantData; use crate::buffer::BufferHandle; use crate::dtype::DType; use crate::serde::ArrayChildren; +use crate::stats::ArrayStats; use crate::vtable; vtable!(Variant, Variant, VariantData); @@ -40,32 +42,30 @@ impl Variant { impl VTable for Variant { type ArrayData = VariantData; + type Metadata = EmptyMetadata; + type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(_array: &Self::ArrayData) -> &Self { + &Variant + } + fn id(&self) -> ArrayId { Self::ID } - fn validate(&self, data: &Self::ArrayData, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!( - matches!(dtype, DType::Variant(_)), - "Expected Variant DType, got {dtype}" - ); - vortex_ensure!( - data.child().dtype() == dtype, - "VariantArray child dtype {} does not match outer dtype {}", - data.child().dtype(), - dtype - ); - vortex_ensure!( - data.len() == len, - "VariantArray length {} does not match outer length {}", - data.len(), - len - ); - Ok(()) + fn len(array: &Self::ArrayData) -> usize { + array.child().len() + } + + fn dtype(array: &Self::ArrayData) -> &DType { + array.child().dtype() + } + + fn stats(array: &Self::ArrayData) -> &ArrayStats { + &array.stats_set } fn array_hash(array: &VariantData, state: &mut H, precision: Precision) { @@ -88,25 +88,31 @@ impl VTable for Variant { None } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { + fn metadata(_array: ArrayView<'_, Self>) -> VortexResult { + Ok(EmptyMetadata) + } + + fn serialize(_metadata: Self::Metadata) -> VortexResult>> { Ok(Some(vec![])) } fn deserialize( - &self, + _bytes: &[u8], + _dtype: &DType, + _len: usize, + _buffers: &[BufferHandle], + _session: &vortex_session::VortexSession, + ) -> VortexResult { + Ok(EmptyMetadata) + } + + fn build( dtype: &DType, len: usize, - metadata: &[u8], - + _metadata: &Self::Metadata, _buffers: &[BufferHandle], children: &dyn ArrayChildren, - _session: &vortex_session::VortexSession, ) -> VortexResult { - vortex_ensure!( - metadata.is_empty(), - "VariantArray expects empty metadata, got {} bytes", - metadata.len() - ); vortex_ensure!(matches!(dtype, DType::Variant(_)), "Expected Variant DType"); vortex_ensure!( children.len() == 1, @@ -153,16 +159,11 @@ impl VTable for Variant { mod tests { use super::*; use crate::IntoArray; - use crate::arrays::ConstantArray; - use crate::dtype::DType; - use crate::dtype::Nullability; - use crate::scalar::Scalar; + use crate::arrays::PrimitiveArray; #[test] fn with_slots_rejects_missing_child() { - let child = - ConstantArray::new(Scalar::null(DType::Variant(Nullability::Nullable)), 3).into_array(); - let array = VariantArray::new(child); + let array = VariantArray::new(PrimitiveArray::from_iter([1u8, 2, 3]).into_array()); let mut data = array.into_data(); let err = ::with_slots(&mut data, vec![None]).unwrap_err(); diff --git a/vortex-array/src/arrow/executor/dictionary.rs b/vortex-array/src/arrow/executor/dictionary.rs index 23cc71e65dd..5d9b75182b3 100644 --- a/vortex-array/src/arrow/executor/dictionary.rs +++ b/vortex-array/src/arrow/executor/dictionary.rs @@ -21,7 +21,7 @@ use crate::arrays::Constant; use crate::arrays::ConstantArray; use crate::arrays::Dict; use crate::arrays::DictArray; -use crate::arrays::dict::DictDataParts; +use crate::arrays::dict::DictArrayParts; use crate::arrow::ArrowArrayExecutor; pub(super) fn to_arrow_dictionary( @@ -78,7 +78,7 @@ fn dict_to_dict( values_type: &DataType, ctx: &mut ExecutionCtx, ) -> VortexResult { - let DictDataParts { codes, values, .. } = array.into_data().into_parts(); + let DictArrayParts { codes, values, .. } = array.into_data().into_parts(); let codes = codes.execute_arrow(Some(codes_type), ctx)?; let values = values.execute_arrow(Some(values_type), ctx)?; make_dict_array(codes_type, codes, values) diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-array/src/arrow/executor/list.rs index f17ededa54d..661e459f62e 100644 --- a/vortex-array/src/arrow/executor/list.rs +++ b/vortex-array/src/arrow/executor/list.rs @@ -21,7 +21,7 @@ use crate::arrays::List; use crate::arrays::ListArray; use crate::arrays::ListView; use crate::arrays::ListViewArray; -use crate::arrays::listview::ListViewDataParts; +use crate::arrays::listview::ListViewArrayParts; use crate::arrays::listview::ListViewRebuildMode; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; @@ -133,7 +133,7 @@ fn list_view_zctl( ))); } - let ListViewDataParts { + let ListViewArrayParts { elements, offsets, sizes, diff --git a/vortex-array/src/arrow/executor/list_view.rs b/vortex-array/src/arrow/executor/list_view.rs index 614ea6e7c93..fb536bc3c1b 100644 --- a/vortex-array/src/arrow/executor/list_view.rs +++ b/vortex-array/src/arrow/executor/list_view.rs @@ -14,7 +14,7 @@ use crate::ExecutionCtx; use crate::arrays::ListView; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; -use crate::arrays::listview::ListViewDataParts; +use crate::arrays::listview::ListViewArrayParts; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; use crate::builtins::ArrayBuiltins; @@ -43,7 +43,7 @@ fn list_view_to_list_view( elements_field: &FieldRef, ctx: &mut ExecutionCtx, ) -> VortexResult { - let ListViewDataParts { + let ListViewArrayParts { elements, offsets, sizes, diff --git a/vortex-array/src/arrow/executor/struct_.rs b/vortex-array/src/arrow/executor/struct_.rs index 7edd1b05e35..eab871de2eb 100644 --- a/vortex-array/src/arrow/executor/struct_.rs +++ b/vortex-array/src/arrow/executor/struct_.rs @@ -19,7 +19,7 @@ use crate::arrays::Chunked; use crate::arrays::ScalarFnVTable; use crate::arrays::Struct; use crate::arrays::StructArray; -use crate::arrays::struct_::StructDataParts; +use crate::arrays::struct_::StructArrayParts; use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::validity::to_arrow_null_buffer; use crate::builtins::ArrayBuiltins; @@ -49,14 +49,13 @@ pub(super) fn to_arrow_struct( // Attempt to short-circuit if the array is already a Struct: let array = match array.try_into::() { Ok(array) => { - let parts = array.into_parts(); - let len = parts.len; - let StructDataParts { + let len = array.len(); + let StructArrayParts { validity, fields, struct_fields, .. - } = parts.data.into_parts(); + } = array.into_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; return create_from_fields( target_fields.ok_or_else(|| struct_fields.names().clone()), @@ -98,14 +97,13 @@ pub(super) fn to_arrow_struct( }; let struct_array = array.execute::(ctx)?; - let parts = struct_array.into_parts(); - let len = parts.len; - let StructDataParts { + let len = struct_array.len(); + let StructArrayParts { validity, fields, struct_fields, .. - } = parts.data.into_parts(); + } = struct_array.into_parts(); let validity = to_arrow_null_buffer(validity, len, ctx)?; create_from_fields( diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index e50458bcebc..e1220bf4837 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -37,13 +37,13 @@ use crate::arrays::VarBinView; use crate::arrays::VarBinViewArray; use crate::arrays::Variant; use crate::arrays::VariantArray; -use crate::arrays::bool::BoolDataParts; -use crate::arrays::decimal::DecimalDataParts; -use crate::arrays::listview::ListViewDataParts; +use crate::arrays::bool::BoolArrayParts; +use crate::arrays::decimal::DecimalArrayParts; +use crate::arrays::listview::ListViewArrayParts; use crate::arrays::listview::ListViewRebuildMode; -use crate::arrays::primitive::PrimitiveDataParts; -use crate::arrays::struct_::StructDataParts; -use crate::arrays::varbinview::VarBinViewDataParts; +use crate::arrays::primitive::PrimitiveArrayParts; +use crate::arrays::struct_::StructArrayParts; +use crate::arrays::varbinview::VarBinViewArrayParts; use crate::dtype::DType; use crate::dtype::NativePType; use crate::dtype::Nullability; @@ -544,19 +544,18 @@ impl Executable for CanonicalValidity { match array.execute::(ctx)? { n @ Canonical::Null(_) => Ok(CanonicalValidity(n)), Canonical::Bool(b) => { - let b = b.into_parts(); - let BoolDataParts { + let BoolArrayParts { bits, offset, len, validity, - } = b.data.into_parts(); + } = b.into_parts(); Ok(CanonicalValidity(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) } Canonical::Primitive(p) => { - let PrimitiveDataParts { + let PrimitiveArrayParts { ptype, buffer, validity, @@ -566,7 +565,7 @@ impl Executable for CanonicalValidity { }))) } Canonical::Decimal(d) => { - let DecimalDataParts { + let DecimalArrayParts { decimal_dtype, values, values_type, @@ -582,7 +581,7 @@ impl Executable for CanonicalValidity { }))) } Canonical::VarBinView(vbv) => { - let VarBinViewDataParts { + let VarBinViewArrayParts { dtype, buffers, views, @@ -599,7 +598,7 @@ impl Executable for CanonicalValidity { } Canonical::List(l) => { let zctl = l.is_zero_copy_to_list(); - let ListViewDataParts { + let ListViewArrayParts { elements, offsets, sizes, @@ -620,13 +619,12 @@ impl Executable for CanonicalValidity { ))) } Canonical::Struct(st) => { - let parts = st.into_parts(); - let len = parts.len; - let StructDataParts { + let len = st.len(); + let StructArrayParts { struct_fields, fields, validity, - } = parts.data.into_parts(); + } = st.into_parts(); Ok(CanonicalValidity(Canonical::Struct(unsafe { StructArray::new_unchecked(fields, struct_fields, len, validity.execute(ctx)?) }))) @@ -666,19 +664,18 @@ impl Executable for RecursiveCanonical { match array.execute::(ctx)? { n @ Canonical::Null(_) => Ok(RecursiveCanonical(n)), Canonical::Bool(b) => { - let b = b.into_parts(); - let BoolDataParts { + let BoolArrayParts { bits, offset, len, validity, - } = b.data.into_parts(); + } = b.into_parts(); Ok(RecursiveCanonical(Canonical::Bool( BoolArray::try_new_from_handle(bits, offset, len, validity.execute(ctx)?)?, ))) } Canonical::Primitive(p) => { - let PrimitiveDataParts { + let PrimitiveArrayParts { ptype, buffer, validity, @@ -688,7 +685,7 @@ impl Executable for RecursiveCanonical { }))) } Canonical::Decimal(d) => { - let DecimalDataParts { + let DecimalArrayParts { decimal_dtype, values, values_type, @@ -704,7 +701,7 @@ impl Executable for RecursiveCanonical { }))) } Canonical::VarBinView(vbv) => { - let VarBinViewDataParts { + let VarBinViewArrayParts { dtype, buffers, views, @@ -721,7 +718,7 @@ impl Executable for RecursiveCanonical { } Canonical::List(l) => { let zctl = l.is_zero_copy_to_list(); - let ListViewDataParts { + let ListViewArrayParts { elements, offsets, sizes, @@ -752,13 +749,12 @@ impl Executable for RecursiveCanonical { ))) } Canonical::Struct(st) => { - let parts = st.into_parts(); - let len = parts.len; - let StructDataParts { + let len = st.len(); + let StructArrayParts { struct_fields, fields, validity, - } = parts.data.into_parts(); + } = st.into_parts(); let executed_fields = fields .iter() .map(|f| Ok(f.clone().execute::(ctx)?.0.into_array())) diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index 6f296ac3885..b98f1da6c38 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -281,14 +281,14 @@ impl ArrayChildren for &[ArrayRef] { } } -/// [`SerializedArray`] represents a parsed but not-yet-decoded deserialized array. +/// [`ArrayParts`] represents a parsed but not-yet-decoded deserialized array. /// It contains all the information from the serialized form, without anything extra. i.e. /// it is missing a [`DType`] and `len`, and the `encoding_id` is not yet resolved to a concrete /// vtable. /// -/// An [`SerializedArray`] can be fully decoded into an [`ArrayRef`] using the `decode` function. +/// An [`ArrayParts`] can be fully decoded into an [`ArrayRef`] using the `decode` function. #[derive(Clone)] -pub struct SerializedArray { +pub struct ArrayParts { // Typed as fb::ArrayNode flatbuffer: FlatBuffer, // The location of the current fb::ArrayNode @@ -296,9 +296,9 @@ pub struct SerializedArray { buffers: Arc<[BufferHandle]>, } -impl Debug for SerializedArray { +impl Debug for ArrayParts { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SerializedArray") + f.debug_struct("ArrayParts") .field("encoding_id", &self.encoding_id()) .field("children", &(0..self.nchildren()).map(|i| self.child(i))) .field( @@ -310,8 +310,8 @@ impl Debug for SerializedArray { } } -impl SerializedArray { - /// Decode an [`SerializedArray`] into an [`ArrayRef`]. +impl ArrayParts { + /// Decode an [`ArrayParts`] into an [`ArrayRef`]. pub fn decode( &self, dtype: &DType, @@ -329,8 +329,8 @@ impl SerializedArray { .find(&encoding_id) .ok_or_else(|| vortex_err!("Unknown encoding: {}", encoding_id))?; - let children = SerializedArrayChildren { - ser: self, + let children = ArrayPartsChildren { + parts: self, ctx, session, }; @@ -402,7 +402,7 @@ impl SerializedArray { } /// Returns the nth child of the array. - pub fn child(&self, idx: usize) -> SerializedArray { + pub fn child(&self, idx: usize) -> ArrayParts { let children = self .flatbuffer() .children() @@ -487,7 +487,7 @@ impl SerializedArray { /// without needing to access the actual buffer data. pub fn buffer_lengths(&self) -> Vec { let fb_array = root::(self.flatbuffer.as_ref()) - .vortex_expect("SerializedArray flatbuffer must be a valid Array"); + .vortex_expect("ArrayParts flatbuffer must be a valid Array"); fb_array .buffers() .map(|buffers| buffers.iter().map(|b| b.length() as usize).collect()) @@ -505,7 +505,7 @@ impl SerializedArray { Ok((fb_buffer, flatbuffer_loc)) } - /// Create an [`SerializedArray`] from a pre-existing array tree flatbuffer and pre-resolved buffer + /// Create an [`ArrayParts`] from a pre-existing array tree flatbuffer and pre-resolved buffer /// handles. /// /// The caller is responsible for resolving buffers from whatever source (device segments, host @@ -516,24 +516,24 @@ impl SerializedArray { buffers: Vec, ) -> VortexResult { let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?; - Ok(SerializedArray { + Ok(ArrayParts { flatbuffer, flatbuffer_loc, buffers: buffers.into(), }) } - /// Create an [`SerializedArray`] from a raw array tree flatbuffer (metadata only). + /// Create an [`ArrayParts`] from a raw array tree flatbuffer (metadata only). /// - /// This constructor creates a `SerializedArray` with no buffer data, useful for + /// This constructor creates an `ArrayParts` with no buffer data, useful for /// inspecting the metadata when the actual buffer data is not needed /// (e.g., displaying buffer sizes from inlined array tree metadata). /// - /// Note: Calling `buffer()` on the returned `SerializedArray` will fail since + /// Note: Calling `buffer()` on the returned `ArrayParts` will fail since /// no actual buffer data is available. pub fn from_array_tree(array_tree: impl Into) -> VortexResult { let (flatbuffer, flatbuffer_loc) = Self::validate_array_tree(array_tree)?; - Ok(SerializedArray { + Ok(ArrayParts { flatbuffer, flatbuffer_loc, buffers: Arc::new([]), @@ -545,7 +545,7 @@ impl SerializedArray { unsafe { fba::ArrayNode::follow(self.flatbuffer.as_ref(), self.flatbuffer_loc) } } - /// Returns a new [`SerializedArray`] with the given node as the root + /// Returns a new [`ArrayParts`] with the given node as the root // TODO(ngates): we may want a wrapper that avoids this clone. fn with_root(&self, root: fba::ArrayNode) -> Self { let mut this = self.clone(); @@ -553,7 +553,7 @@ impl SerializedArray { this } - /// Create an [`SerializedArray`] from a pre-existing flatbuffer (ArrayNode) and a segment containing + /// Create an [`ArrayParts`] from a pre-existing flatbuffer (ArrayNode) and a segment containing /// only the data buffers (without the flatbuffer suffix). /// /// This is used when the flatbuffer is stored separately in layout metadata (e.g., when @@ -566,7 +566,7 @@ impl SerializedArray { Self::from_flatbuffer_and_segment_with_overrides(array_tree, segment, &HashMap::new()) } - /// Create an [`SerializedArray`] from a pre-existing flatbuffer (ArrayNode) and a segment, + /// Create an [`ArrayParts`] from a pre-existing flatbuffer (ArrayNode) and a segment, /// substituting host-resident buffer overrides for specific buffer indices. /// /// Buffers whose index appears in `buffer_overrides` are resolved from the provided @@ -611,7 +611,7 @@ impl SerializedArray { }) .collect::>>()?; - Ok(SerializedArray { + Ok(ArrayParts { flatbuffer: fb_buffer, flatbuffer_loc, buffers, @@ -619,31 +619,31 @@ impl SerializedArray { } } -struct SerializedArrayChildren<'a> { - ser: &'a SerializedArray, +struct ArrayPartsChildren<'a> { + parts: &'a ArrayParts, ctx: &'a ReadContext, session: &'a VortexSession, } -impl ArrayChildren for SerializedArrayChildren<'_> { +impl ArrayChildren for ArrayPartsChildren<'_> { fn get(&self, index: usize, dtype: &DType, len: usize) -> VortexResult { - self.ser + self.parts .child(index) .decode(dtype, len, self.ctx, self.session) } fn len(&self) -> usize { - self.ser.nchildren() + self.parts.nchildren() } } -impl TryFrom for SerializedArray { +impl TryFrom for ArrayParts { type Error = VortexError; fn try_from(value: ByteBuffer) -> Result { // The final 4 bytes contain the length of the flatbuffer. if value.len() < 4 { - vortex_bail!("SerializedArray buffer is too short"); + vortex_bail!("ArrayParts buffer is too short"); } // We align each buffer individually, so we remove alignment requirements on the buffer. @@ -651,7 +651,7 @@ impl TryFrom for SerializedArray { let fb_length = u32::try_from_le_bytes(&value.as_slice()[value.len() - 4..])? as usize; if value.len() < 4 + fb_length { - vortex_bail!("SerializedArray buffer is too short for flatbuffer"); + vortex_bail!("ArrayParts buffer is too short for flatbuffer"); } let fb_offset = value.len() - 4 - fb_length; @@ -662,7 +662,7 @@ impl TryFrom for SerializedArray { } } -impl TryFrom for SerializedArray { +impl TryFrom for ArrayParts { type Error = VortexError; fn try_from(value: BufferHandle) -> Result { diff --git a/vortex-array/src/stats/array.rs b/vortex-array/src/stats/array.rs index bd2e3cf7dea..a49272359be 100644 --- a/vortex-array/src/stats/array.rs +++ b/vortex-array/src/stats/array.rs @@ -83,6 +83,7 @@ impl From for StatsSet { } impl StatsSetRef<'_> { + #[allow(dead_code)] pub(crate) fn replace(&self, stats: StatsSet) { *self.array_stats.inner.write() = stats; } diff --git a/vortex-array/src/test_harness.rs b/vortex-array/src/test_harness.rs index cee718f3920..348f79829fd 100644 --- a/vortex-array/src/test_harness.rs +++ b/vortex-array/src/test_harness.rs @@ -8,18 +8,25 @@ use goldenfile::differs::binary_diff; use itertools::Itertools; use vortex_error::VortexResult; +use crate::DeserializeMetadata; +use crate::SerializeMetadata; use crate::arrays::BoolArray; /// Check that a named metadata matches its previous versioning. /// /// Goldenfile takes care of checking for equality against a checked-in file. #[allow(clippy::unwrap_used)] -pub fn check_metadata(name: &str, metadata: &[u8]) { +pub fn check_metadata(name: &str, metadata: T) +where + T: SerializeMetadata, + T: DeserializeMetadata, +{ let mut mint = Mint::new("goldenfiles/"); + let meta = metadata.serialize(); let mut f = mint .new_goldenfile_with_differ(name, Box::new(binary_diff)) .unwrap(); - f.write_all(metadata).unwrap(); + f.write_all(&meta).unwrap(); } /// Outputs the indices of the true values in a BoolArray diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index 3ff8e872b19..08650bf6185 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -143,6 +143,7 @@ impl BtrBlocksCompressorBuilder { /// With the `unstable_encodings` feature, buffer-level Zstd compression is used which /// preserves the array buffer layout for zero-conversion GPU decompression. Without it, /// interleaved Zstd compression is used. + #[cfg(feature = "zstd")] pub fn only_cuda_compatible(self) -> Self { let builder = self.exclude_schemes([ integer::SparseScheme.id(), @@ -153,9 +154,9 @@ impl BtrBlocksCompressorBuilder { string::FSSTScheme.id(), ]); - #[cfg(all(feature = "zstd", feature = "unstable_encodings"))] + #[cfg(feature = "unstable_encodings")] let builder = builder.with_new_scheme(&string::ZstdBuffersScheme); - #[cfg(all(feature = "zstd", not(feature = "unstable_encodings")))] + #[cfg(not(feature = "unstable_encodings"))] let builder = builder.with_new_scheme(&string::ZstdScheme); builder diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index 3ed6a883c44..b995f0827c5 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -6,6 +6,7 @@ use vortex_alp::ALP; use vortex_alp::RDEncoder; use vortex_alp::alp_encode; +use vortex_array::Array; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; @@ -146,8 +147,6 @@ impl Scheme for ALPRDScheme { }; let alp_rd = encoder.encode(stats.source()); - let dtype = alp_rd.dtype().clone(); - let right_bit_width = alp_rd.right_bit_width(); let mut alp_rd_data = alp_rd.into_data(); let patches = alp_rd_data @@ -155,17 +154,8 @@ impl Scheme for ALPRDScheme { .map(compress_patches) .transpose()?; alp_rd_data.replace_left_parts_patches(patches); - let parts = alp_rd_data.into_parts(); - - Ok(vortex_alp::ALPRD::try_new( - dtype, - parts.left_parts, - parts.left_parts_dictionary, - parts.right_parts, - right_bit_width, - parts.left_parts_patches, - )? - .into_array()) + + Ok(Array::::try_from_data(alp_rd_data)?.into_array()) } } diff --git a/vortex-btrblocks/src/schemes/integer.rs b/vortex-btrblocks/src/schemes/integer.rs index 9309637c158..c6b4f3fee56 100644 --- a/vortex-btrblocks/src/schemes/integer.rs +++ b/vortex-btrblocks/src/schemes/integer.rs @@ -3,6 +3,7 @@ //! Integer compression schemes. +use vortex_array::Array; use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray; @@ -335,27 +336,12 @@ impl Scheme for BitPackingScheme { return Ok(stats.source().clone().into_array()); } let packed = bitpack_encode(stats.source(), bw, Some(&histogram))?; - let packed_stats = packed.statistics().to_owned(); - let ptype = packed.dtype().as_ptype(); - let len = packed.len(); - let nullability = packed.dtype().nullability(); let mut packed_data = packed.into_data(); - let patches = packed_data.patches(len).map(compress_patches).transpose()?; + let patches = packed_data.patches().map(compress_patches).transpose()?; packed_data.replace_patches(patches); - let parts = packed_data.into_parts(len, nullability); - - Ok(vortex_fastlanes::BitPacked::try_new( - parts.packed, - ptype, - parts.validity, - parts.patches, - parts.bit_width, - parts.len, - parts.offset, - )? - .with_stats_set(packed_stats) - .into_array()) + + Ok(Array::::try_from_data(packed_data)?.into_array()) } } @@ -871,9 +857,6 @@ mod scheme_selection_tests { use vortex_array::arrays::Constant; use vortex_array::arrays::Dict; use vortex_array::arrays::PrimitiveArray; - use vortex_array::expr::stats::Precision; - use vortex_array::expr::stats::Stat; - use vortex_array::expr::stats::StatsProviderExt; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; @@ -912,18 +895,6 @@ mod scheme_selection_tests { let btr = BtrBlocksCompressor::default(); let compressed = btr.compress(&array.into_array())?; assert!(compressed.is::()); - assert_eq!( - compressed.statistics().get_as::(Stat::NullCount), - Some(Precision::exact(0u64)) - ); - assert_eq!( - compressed.statistics().get_as::(Stat::Min), - Some(Precision::exact(0u32)) - ); - assert_eq!( - compressed.statistics().get_as::(Stat::Max), - Some(Precision::exact(15u32)) - ); Ok(()) } diff --git a/vortex-btrblocks/src/schemes/rle.rs b/vortex-btrblocks/src/schemes/rle.rs index 42fd4295bb7..4da83209f39 100644 --- a/vortex-btrblocks/src/schemes/rle.rs +++ b/vortex-btrblocks/src/schemes/rle.rs @@ -21,8 +21,6 @@ use vortex_compressor::scheme::SchemeId; use vortex_compressor::stats::FloatStats; use vortex_compressor::stats::IntegerStats; use vortex_error::VortexResult; -#[cfg(feature = "unstable_encodings")] -use vortex_fastlanes::Delta; use vortex_fastlanes::RLE; use crate::ArrayAndStats; @@ -283,6 +281,7 @@ impl Scheme for RLEScheme { compressed_values, compressed_indices, compressed_offsets, + rle_array.dtype().clone(), rle_array.offset(), rle_array.len(), ) @@ -307,5 +306,6 @@ fn try_compress_delta( let compressed_deltas = compressor.compress_child(&deltas.into_array(), parent_ctx, parent_id, child_index)?; - Delta::try_new(compressed_bases, compressed_deltas, 0, child.len()).map(IntoArray::into_array) + vortex_fastlanes::DeltaData::try_new(compressed_bases, compressed_deltas, 0, child.len()) + .map(IntoArray::into_array) } diff --git a/vortex-btrblocks/src/schemes/string.rs b/vortex-btrblocks/src/schemes/string.rs index f222ce1623b..06ba6d03314 100644 --- a/vortex-btrblocks/src/schemes/string.rs +++ b/vortex-btrblocks/src/schemes/string.rs @@ -240,7 +240,7 @@ impl Scheme for ZstdBuffersScheme { let stats = data.string_stats(); Ok( - vortex_zstd::ZstdBuffers::compress(&stats.source().clone().into_array(), 3)? + vortex_zstd::ZstdBuffersData::compress(&stats.source().clone().into_array(), 3)? .into_array(), ) } diff --git a/vortex-cuda/benches/date_time_parts_cuda.rs b/vortex-cuda/benches/date_time_parts_cuda.rs index 53f303d0b2c..a91111e3f8a 100644 --- a/vortex-cuda/benches/date_time_parts_cuda.rs +++ b/vortex-cuda/benches/date_time_parts_cuda.rs @@ -24,8 +24,8 @@ use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::dtype::DType; use vortex::dtype::Nullability; -use vortex::encodings::datetime_parts::DateTimeParts; use vortex::encodings::datetime_parts::DateTimePartsArray; +use vortex::encodings::datetime_parts::DateTimePartsData; use vortex::error::VortexExpect; use vortex::extension::datetime::TimeUnit; use vortex::extension::datetime::Timestamp; @@ -45,8 +45,11 @@ fn make_datetimeparts_array(len: usize, time_unit: TimeUnit) -> DateTimePartsArr let dtype = DType::Extension(Timestamp::new(time_unit, Nullability::NonNullable).erased()); - DateTimeParts::try_new(dtype, days_arr, seconds_arr, subseconds_arr) - .vortex_expect("Failed to create DateTimePartsArray") + DateTimePartsArray::try_from_data( + DateTimePartsData::try_new(dtype, days_arr, seconds_arr, subseconds_arr) + .vortex_expect("Failed to create DateTimePartsArray"), + ) + .vortex_expect("DateTimePartsData is always valid") } fn benchmark_datetimeparts(c: &mut Criterion) { diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 6d4bb42d619..21537b51031 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -30,7 +30,6 @@ use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::Exponents; use vortex::encodings::alp::alp_encode; use vortex::encodings::fastlanes::BitPackedData; -use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRData; use vortex::encodings::runend::RunEnd; use vortex::error::VortexExpect; @@ -188,9 +187,9 @@ fn bench_for_bitpacked(c: &mut Criterion) { .collect(); let prim = PrimitiveArray::new(Buffer::from(residuals), NonNullable); let bp = BitPackedData::encode(&prim.into_array(), bit_width).vortex_expect("bitpack"); - let array = FoR::try_new(bp.into_array(), Scalar::from(reference)) - .vortex_expect("for") - .into_array(); + let for_arr = + FoRData::try_new(bp.into_array(), Scalar::from(reference)).vortex_expect("for"); + let array = for_arr.into_array(); group.bench_with_input( BenchmarkId::new("dynamic_dispatch_u32", len_str), @@ -321,8 +320,8 @@ fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { let dict_prim = PrimitiveArray::new(Buffer::from(dict_residuals), NonNullable); let dict_bp = BitPackedData::encode(&dict_prim.into_array(), dict_bit_width) .vortex_expect("bitpack dict"); - let dict_for = - FoR::try_new(dict_bp.into_array(), Scalar::from(dict_reference)).vortex_expect("for dict"); + let dict_for = FoRData::try_new(dict_bp.into_array(), Scalar::from(dict_reference)) + .vortex_expect("for dict"); for (len, len_str) in BENCH_ARGS { group.throughput(Throughput::Bytes((len * size_of::()) as u64)); @@ -385,7 +384,7 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { BitPackedData::encode(for_arr.encoded(), bit_width).vortex_expect("bitpack encode"); let tree = ALP::new( - FoR::try_new(bp.into_array(), for_arr.reference_scalar().clone()) + FoRData::try_new(bp.into_array(), for_arr.reference_scalar().clone()) .vortex_expect("for_new") .into_array(), exponents, diff --git a/vortex-cuda/benches/for_cuda.rs b/vortex-cuda/benches/for_cuda.rs index cc5fab5e22c..eb87eb6d770 100644 --- a/vortex-cuda/benches/for_cuda.rs +++ b/vortex-cuda/benches/for_cuda.rs @@ -26,8 +26,8 @@ use vortex::buffer::Buffer; use vortex::dtype::NativePType; use vortex::dtype::PType; use vortex::encodings::fastlanes::BitPackedData; -use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArray; +use vortex::encodings::fastlanes::FoRData; use vortex::error::VortexExpect; use vortex::scalar::Scalar; use vortex::session::VortexSession; @@ -57,10 +57,17 @@ where if bp && T::PTYPE != PType::U8 { let child = BitPackedData::encode(&primitive_array, 8).vortex_expect("failed to bitpack"); - FoR::try_new(child.into_array(), reference.into()) - .vortex_expect("failed to create FoR array") + FoRArray::try_from_data( + FoRData::try_new(child.into_array(), reference.into()) + .vortex_expect("failed to create FoR array"), + ) + .vortex_expect("FoRData is always valid") } else { - FoR::try_new(primitive_array, reference.into()).vortex_expect("failed to create FoR array") + FoRArray::try_from_data( + FoRData::try_new(primitive_array, reference.into()) + .vortex_expect("failed to create FoR array"), + ) + .vortex_expect("FoRData is always valid") } } diff --git a/vortex-cuda/benches/zstd_cuda.rs b/vortex-cuda/benches/zstd_cuda.rs index d3e49d03c48..7fe68380439 100644 --- a/vortex-cuda/benches/zstd_cuda.rs +++ b/vortex-cuda/benches/zstd_cuda.rs @@ -15,7 +15,7 @@ use futures::executor::block_on; use vortex::array::arrays::VarBinViewArray; use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdArray; -use vortex::encodings::zstd::ZstdDataParts; +use vortex::encodings::zstd::ZstdArrayParts; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -138,7 +138,7 @@ fn benchmark_zstd_cuda_decompress(c: &mut Criterion) { let mut total_time = Duration::ZERO; for _ in 0..iters { - let ZstdDataParts { + let ZstdArrayParts { frames, metadata, .. } = zstd_array.clone().into_data().into_parts(); let exec = block_on(zstd_kernel_prepare(frames, &metadata, &mut cuda_ctx)) diff --git a/vortex-cuda/src/arrow/canonical.rs b/vortex-cuda/src/arrow/canonical.rs index 605d2a9e356..b3bb2824b4c 100644 --- a/vortex-cuda/src/arrow/canonical.rs +++ b/vortex-cuda/src/arrow/canonical.rs @@ -7,10 +7,10 @@ use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::ToCanonical; use vortex::array::arrays::StructArray; -use vortex::array::arrays::bool::BoolDataParts; -use vortex::array::arrays::decimal::DecimalDataParts; -use vortex::array::arrays::primitive::PrimitiveDataParts; -use vortex::array::arrays::struct_::StructDataParts; +use vortex::array::arrays::bool::BoolArrayParts; +use vortex::array::arrays::decimal::DecimalArrayParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::struct_::StructArrayParts; use vortex::array::buffer::BufferHandle; use vortex::dtype::DecimalType; use vortex::error::VortexResult; @@ -66,7 +66,7 @@ fn export_canonical( Canonical::Struct(struct_array) => export_struct(struct_array, ctx).await, Canonical::Primitive(primitive) => { let len = primitive.len(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer, validity, .. } = primitive.into_data().into_parts(); @@ -90,7 +90,7 @@ fn export_canonical( } Canonical::Decimal(decimal) => { let len = decimal.len(); - let DecimalDataParts { + let DecimalArrayParts { values, values_type, validity, @@ -118,7 +118,7 @@ fn export_canonical( let values = extension.storage_array().to_primitive(); let len = extension.len(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer, validity, .. } = values.into_data().into_parts(); @@ -128,13 +128,13 @@ fn export_canonical( export_fixed_size(buffer, len, 0, ctx) } Canonical::Bool(bool_array) => { - let BoolDataParts { + let BoolArrayParts { bits, offset, len, validity, .. - } = bool_array.into_data().into_parts(); + } = bool_array.into_parts(); check_validity_empty(&validity)?; @@ -182,9 +182,9 @@ async fn export_struct( ctx: &mut CudaExecutionCtx, ) -> VortexResult<(ArrowArray, SyncEvent)> { let len = array.len(); - let StructDataParts { + let StructArrayParts { validity, fields, .. - } = array.into_data().into_parts(); + } = array.into_parts(); check_validity_empty(&validity)?; diff --git a/vortex-cuda/src/arrow/varbinview.rs b/vortex-cuda/src/arrow/varbinview.rs index 72ea96437dd..12edc39c9bf 100644 --- a/vortex-cuda/src/arrow/varbinview.rs +++ b/vortex-cuda/src/arrow/varbinview.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use cudarc::driver::LaunchConfig; use cudarc::driver::PushKernelArg; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::varbinview::VarBinViewDataParts; +use vortex::array::arrays::varbinview::VarBinViewArrayParts; use vortex::array::buffer::BufferHandle; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -33,7 +33,7 @@ pub(crate) async fn copy_varbinview_to_varbin( ctx: &mut CudaExecutionCtx, ) -> VortexResult { let len = array.len(); - let VarBinViewDataParts { + let VarBinViewArrayParts { views, buffers, validity, diff --git a/vortex-cuda/src/canonical.rs b/vortex-cuda/src/canonical.rs index da87deb9f89..0cbfbf718cc 100644 --- a/vortex-cuda/src/canonical.rs +++ b/vortex-cuda/src/canonical.rs @@ -13,12 +13,12 @@ use vortex::array::arrays::ExtensionArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::bool::BoolDataParts; -use vortex::array::arrays::decimal::DecimalDataParts; -use vortex::array::arrays::primitive::PrimitiveDataParts; -use vortex::array::arrays::struct_::StructDataParts; +use vortex::array::arrays::bool::BoolArrayParts; +use vortex::array::arrays::decimal::DecimalArrayParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::struct_::StructArrayParts; use vortex::array::arrays::varbinview::BinaryView; -use vortex::array::arrays::varbinview::VarBinViewDataParts; +use vortex::array::arrays::varbinview::VarBinViewArrayParts; use vortex::array::buffer::BufferHandle; use vortex::buffer::BitBuffer; use vortex::buffer::Buffer; @@ -40,12 +40,12 @@ impl CanonicalCudaExt for Canonical { Canonical::Struct(struct_array) => { // Children should all be canonical now let len = struct_array.len(); - let StructDataParts { + let StructArrayParts { fields, struct_fields, validity, .. - } = struct_array.into_data().into_parts(); + } = struct_array.into_parts(); let mut host_fields = vec![]; for field in fields.iter() { @@ -63,19 +63,19 @@ impl CanonicalCudaExt for Canonical { Canonical::Bool(bool) => { // NOTE: update to copy to host when adding buffer handle. // Also update other method to copy validity to host. - let BoolDataParts { + let BoolArrayParts { bits, validity, offset, len, .. - } = bool.into_data().into_parts(); + } = bool.into_parts(); let bits = BitBuffer::new_with_offset(bits.try_into_host()?.await?, offset, len); Ok(Canonical::Bool(BoolArray::new(bits, validity))) } Canonical::Primitive(prim) => { - let PrimitiveDataParts { + let PrimitiveArrayParts { ptype, buffer, validity, @@ -88,7 +88,7 @@ impl CanonicalCudaExt for Canonical { ))) } Canonical::Decimal(decimal) => { - let DecimalDataParts { + let DecimalArrayParts { decimal_dtype, values, values_type, @@ -105,7 +105,7 @@ impl CanonicalCudaExt for Canonical { })) } Canonical::VarBinView(varbinview) => { - let VarBinViewDataParts { + let VarBinViewArrayParts { views, buffers, validity, diff --git a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs index 19e19119eb0..a59a69a173d 100644 --- a/vortex-cuda/src/dynamic_dispatch/plan_builder.rs +++ b/vortex-cuda/src/dynamic_dispatch/plan_builder.rs @@ -51,7 +51,7 @@ fn is_dyn_dispatch_compatible(array: &ArrayRef) -> bool { return arr.patches().is_none() && arr.dtype().as_ptype() == PType::F32; } if id == BitPacked::ID { - return array.as_::().patches(array.len()).is_none(); + return array.as_::().patches().is_none(); } if id == Dict::ID { let arr = array.as_::(); @@ -412,7 +412,7 @@ impl FusedPlan { fn walk_bitpacked(&mut self, array: ArrayRef) -> VortexResult { let bp = array.as_::(); - if bp.patches(array.len()).is_some() { + if bp.patches().is_some() { vortex_bail!("Dynamic dispatch does not support BitPackedArray with patches"); } diff --git a/vortex-cuda/src/executor.rs b/vortex-cuda/src/executor.rs index 4a14be98968..9ae518540bd 100644 --- a/vortex-cuda/src/executor.rs +++ b/vortex-cuda/src/executor.rs @@ -21,7 +21,7 @@ use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::arrays::Struct; use vortex::array::arrays::StructArray; -use vortex::array::arrays::struct_::StructDataParts; +use vortex::array::arrays::struct_::StructArrayParts; use vortex::array::buffer::BufferHandle; use vortex::dtype::PType; use vortex::error::VortexResult; @@ -360,12 +360,12 @@ impl CudaArrayExt for ArrayRef { async fn execute_cuda(self, ctx: &mut CudaExecutionCtx) -> VortexResult { if self.encoding_id() == Struct::ID { let len = self.len(); - let StructDataParts { + let StructArrayParts { fields, struct_fields, validity, .. - } = self.try_into::().unwrap().into_data().into_parts(); + } = self.try_into::().unwrap().into_parts(); let mut cuda_fields = Vec::with_capacity(fields.len()); for field in fields.iter() { diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs index e81309db88e..fc33d099122 100644 --- a/vortex-cuda/src/hybrid_dispatch/mod.rs +++ b/vortex-cuda/src/hybrid_dispatch/mod.rs @@ -235,6 +235,7 @@ mod tests { use vortex::array::session::ArraySessionExt; use vortex::encodings::fastlanes; use vortex::encodings::zstd::ZstdBuffers; + use vortex::encodings::zstd::ZstdBuffersData; let session = VortexSession::empty(); fastlanes::initialize(&session); @@ -255,7 +256,7 @@ mod tests { 0u32.into(), ) .vortex_expect("for"); - let vals = ZstdBuffers::compress(&vals.into_array(), 3).vortex_expect("zstd"); + let vals = ZstdBuffersData::compress(&vals.into_array(), 3).vortex_expect("zstd"); // codes = FoR(BitPacked) let codes = PrimitiveArray::new( diff --git a/vortex-cuda/src/kernel/arrays/dict.rs b/vortex-cuda/src/kernel/arrays/dict.rs index a8437e0b85b..4015bbcddbb 100644 --- a/vortex-cuda/src/kernel/arrays/dict.rs +++ b/vortex-cuda/src/kernel/arrays/dict.rs @@ -15,10 +15,10 @@ use vortex::array::arrays::Dict; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::decimal::DecimalDataParts; -use vortex::array::arrays::dict::DictDataParts; -use vortex::array::arrays::primitive::PrimitiveDataParts; -use vortex::array::arrays::varbinview::VarBinViewDataParts; +use vortex::array::arrays::decimal::DecimalArrayParts; +use vortex::array::arrays::dict::DictArrayParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; +use vortex::array::arrays::varbinview::VarBinViewArrayParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_decimal_value_type; use vortex::array::match_each_integer_ptype; @@ -65,7 +65,7 @@ impl CudaExecute for DictExecutor { #[expect(clippy::cognitive_complexity)] async fn execute_dict_prim(dict: DictArray, ctx: &mut CudaExecutionCtx) -> VortexResult { - let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); + let DictArrayParts { values, codes, .. } = dict.into_data().into_parts(); // Execute both children to get them as primitives on the device let values_canonical = values.execute_cuda(ctx).await?; @@ -93,14 +93,14 @@ async fn execute_dict_prim_typed VortexResult { - let dtype = dict.dtype().clone(); - let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); + let DictArrayParts { + values, + codes, + dtype, + .. + } = dict.into_data().into_parts(); // Execute codes to get them as primitives on the device let codes_prim = codes.execute_cuda(ctx).await?.into_primitive(); @@ -175,14 +179,14 @@ async fn execute_dict_decimal_typed< let codes_len = codes.len(); let codes_len_u64 = codes_len as u64; - let DecimalDataParts { + let DecimalArrayParts { values: values_buffer, validity: values_validity, .. } = values.into_data().into_parts(); let output_validity = values_validity.take(&codes.clone().into_array())?; - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer: codes_buffer, .. } = codes.into_data().into_parts(); @@ -230,15 +234,19 @@ async fn execute_dict_varbinview( dict: DictArray, ctx: &mut CudaExecutionCtx, ) -> VortexResult { - let dtype = dict.dtype().clone(); - let DictDataParts { values, codes, .. } = dict.into_data().into_parts(); + let DictArrayParts { + values, + codes, + dtype, + .. + } = dict.into_data().into_parts(); let codes_prim = codes.execute_cuda(ctx).await?.into_primitive(); let codes_ptype = codes_prim.ptype(); let codes_len = codes_prim.len(); let values_vbv = values.execute_cuda(ctx).await?.into_varbinview(); - let VarBinViewDataParts { + let VarBinViewArrayParts { views: values_views_handle, buffers: values_data_buffers, validity: values_validity, @@ -246,7 +254,7 @@ async fn execute_dict_varbinview( } = values_vbv.into_data().into_parts(); let output_validity = values_validity.take(&codes_prim.clone().into_array())?; - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer: codes_buffer, .. } = codes_prim.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index 1635927a42e..e33a35bbf56 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -11,7 +11,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_unsigned_integer_ptype; use vortex::dtype::NativePType; @@ -46,9 +46,7 @@ impl CudaExecute for ALPExecutor { .try_into::() .map_err(|_| vortex_err!("Expected ALPArray"))?; - match_each_alp_float_ptype!(array.dtype().as_ptype(), |A| { - decode_alp::(array, ctx).await - }) + match_each_alp_float_ptype!(array.ptype(), |A| { decode_alp::(array, ctx).await }) } } @@ -68,7 +66,7 @@ where // Execute child and copy to device let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer, validity, .. } = primitive.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 3bdff6b439d..eb7273f1ddf 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -18,7 +18,7 @@ use vortex::array::match_each_integer_ptype; use vortex::dtype::NativePType; use vortex::encodings::fastlanes::BitPacked; use vortex::encodings::fastlanes::BitPackedArray; -use vortex::encodings::fastlanes::BitPackedDataParts; +use vortex::encodings::fastlanes::BitPackedArrayParts; use vortex::encodings::fastlanes::unpack_iter::BitPacked as BitPackedUnpack; use vortex::error::VortexResult; use vortex::error::vortex_ensure; @@ -52,7 +52,7 @@ impl CudaExecute for BitPackedExecutor { let array = Self::try_specialize(array).ok_or_else(|| vortex_err!("Expected BitPackedArray"))?; - match_each_integer_ptype!(array.ptype(array.dtype()), |A| { + match_each_integer_ptype!(array.ptype(), |A| { decode_bitpacked::(array, A::default(), ctx).await }) } @@ -96,14 +96,14 @@ where A: BitPackedUnpack + NativePType + DeviceRepr + Send + Sync + 'static, A::Physical: DeviceRepr + Send + Sync + 'static, { - let BitPackedDataParts { + let BitPackedArrayParts { offset, bit_width, len, packed, patches, validity, - } = BitPacked::into_parts(array); + } = array.into_data().into_parts(); vortex_ensure!(len > 0, "Non empty array"); let offset = offset as usize; @@ -199,7 +199,7 @@ mod tests { // Last two items should be patched let bp_with_patches = BitPacked::encode(&array.into_array(), bw)?; - assert!(bp_with_patches.patches(bp_with_patches.len()).is_some()); + assert!(bp_with_patches.patches().is_some()); let cpu_result = bp_with_patches.to_canonical()?.into_array(); @@ -230,7 +230,7 @@ mod tests { // Last two items should be patched let bp_with_patches = BitPacked::encode(&array.into_array(), 9)?; - assert!(bp_with_patches.patches(bp_with_patches.len()).is_some()); + assert!(bp_with_patches.patches().is_some()); let cpu_result = bp_with_patches.to_canonical()?.into_array(); diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index a0c4a446567..da21f267d32 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -13,7 +13,7 @@ use vortex::array::IntoArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::TemporalArray; -use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_signed_integer_ptype; use vortex::array::validity::Validity; @@ -147,15 +147,15 @@ where { let output_len = days.len(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer: days_buffer, .. } = days.into_data().into_parts(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer: seconds_buffer, .. } = seconds.into_data().into_parts(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer: subseconds_buffer, .. } = subseconds.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs index b5aa921a9b6..134e5c99374 100644 --- a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs +++ b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs @@ -8,10 +8,9 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::DecimalArray; -use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::encodings::decimal_byte_parts::DecimalByteParts; -use vortex::encodings::decimal_byte_parts::DecimalBytePartsDataParts; -use vortex::error::VortexExpect; +use vortex::encodings::decimal_byte_parts::DecimalBytePartsArrayParts; use vortex::error::VortexResult; use vortex::error::vortex_bail; @@ -35,12 +34,9 @@ impl CudaExecute for DecimalBytePartsExecutor { vortex_bail!("cannot downcast to DecimalBytePartsArray") }; - let decimal_dtype = *array - .dtype() - .as_decimal_opt() - .vortex_expect("DecimalBytePartsArray dtype must be decimal"); - let DecimalBytePartsDataParts { msp, .. } = array.into_data().into_parts(); - let PrimitiveDataParts { + let decimal_dtype = *array.decimal_dtype(); + let DecimalBytePartsArrayParts { msp, .. } = array.into_data().into_parts(); + let PrimitiveArrayParts { buffer, ptype, validity, diff --git a/vortex-cuda/src/kernel/encodings/for_.rs b/vortex-cuda/src/kernel/encodings/for_.rs index 7fe9d89d64c..57361905c90 100644 --- a/vortex-cuda/src/kernel/encodings/for_.rs +++ b/vortex-cuda/src/kernel/encodings/for_.rs @@ -12,7 +12,7 @@ use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::Slice; -use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::array::match_each_integer_ptype; use vortex::array::match_each_native_simd_ptype; use vortex::dtype::NativePType; @@ -52,7 +52,7 @@ impl CudaExecute for FoRExecutor { // Fuse FOR + BP => FFOR if let Some(bitpacked) = array.encoded().as_opt::() { - match_each_integer_ptype!(bitpacked.ptype(bitpacked.dtype()), |P| { + match_each_integer_ptype!(bitpacked.ptype(), |P| { let reference: P = array.reference_scalar().try_into()?; return decode_bitpacked(bitpacked.into_owned(), reference, ctx).await; }) @@ -63,7 +63,7 @@ impl CudaExecute for FoRExecutor { && let Some(bitpacked) = slice_array.child().as_opt::() { let slice_range = slice_array.slice_range().clone(); - let unpacked = match_each_integer_ptype!(bitpacked.ptype(bitpacked.dtype()), |P| { + let unpacked = match_each_integer_ptype!(bitpacked.ptype(), |P| { let reference: P = array.reference_scalar().try_into()?; decode_bitpacked(bitpacked.into_owned(), reference, ctx).await? }); @@ -96,7 +96,7 @@ where // Execute child and copy to device let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer, validity, .. } = primitive.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index 88b64393a5e..6f14ec20c9c 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -12,7 +12,7 @@ use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::ConstantArray; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_native_ptype; use vortex::array::match_each_unsigned_integer_ptype; @@ -21,7 +21,7 @@ use vortex::dtype::NativePType; use vortex::dtype::PType; use vortex::encodings::runend::RunEnd; use vortex::encodings::runend::RunEndArray; -use vortex::encodings::runend::RunEndDataParts; +use vortex::encodings::runend::RunEndArrayParts; use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::error::vortex_ensure; @@ -61,7 +61,7 @@ impl CudaExecute for RunEndExecutor { let offset = array.offset(); let output_len = array.len(); - let RunEndDataParts { ends, values, .. } = array.into_data().into_parts(); + let RunEndArrayParts { ends, values } = array.into_data().into_parts(); let values_ptype = PType::try_from(values.dtype())?; let ends_ptype = PType::try_from(ends.dtype())?; @@ -105,14 +105,14 @@ async fn decode_runend_typed() .map_err(|_| vortex_err!("SequenceExecutor can only accept SequenceArray"))?; - let len = array.len(); - let nullability = array.dtype().nullability(); - - let SequenceDataParts { + let SequenceArrayParts { base, multiplier, + len, ptype, + nullability, } = array.into_data().into_parts(); match_each_native_ptype!(ptype, |P| { diff --git a/vortex-cuda/src/kernel/encodings/zigzag.rs b/vortex-cuda/src/kernel/encodings/zigzag.rs index e7cf89b2181..f1feac1bbf3 100644 --- a/vortex-cuda/src/kernel/encodings/zigzag.rs +++ b/vortex-cuda/src/kernel/encodings/zigzag.rs @@ -10,7 +10,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::array::match_each_unsigned_integer_ptype; use vortex::dtype::NativePType; use vortex::dtype::PType; @@ -70,7 +70,7 @@ where // Execute child and copy to device let canonical = array.encoded().clone().execute_cuda(ctx).await?; let primitive = canonical.into_primitive(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer, validity, .. } = primitive.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/encodings/zstd.rs b/vortex-cuda/src/kernel/encodings/zstd.rs index e5b7da77fec..a521fe7a27f 100644 --- a/vortex-cuda/src/kernel/encodings/zstd.rs +++ b/vortex-cuda/src/kernel/encodings/zstd.rs @@ -26,7 +26,7 @@ use vortex::buffer::ByteBuffer; use vortex::dtype::DType; use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdArray; -use vortex::encodings::zstd::ZstdDataParts; +use vortex::encodings::zstd::ZstdArrayParts; use vortex::encodings::zstd::ZstdMetadata; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -206,17 +206,17 @@ impl CudaExecute for ZstdExecutor { dtype = %_other, "Only Binary/Utf8 ZSTD arrays supported on GPU, falling back to CPU" ); - Zstd::decompress(&zstd, ctx.execution_ctx())?.to_canonical() + zstd.decompress(ctx.execution_ctx())?.to_canonical() } } } } async fn decode_zstd(array: ZstdArray, ctx: &mut CudaExecutionCtx) -> VortexResult { - let dtype = array.dtype().clone(); - let ZstdDataParts { + let ZstdArrayParts { frames, metadata, + dtype, validity, n_rows, dictionary, @@ -373,7 +373,9 @@ mod tests { let zstd_array = Zstd::from_var_bin_view(&strings, 3, 0)?; - let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())?.to_canonical()?; + let cpu_result = zstd_array + .decompress(cuda_ctx.execution_ctx())? + .to_canonical()?; let gpu_result = ZstdExecutor .execute(zstd_array.into_array(), &mut cuda_ctx) .await?; @@ -408,7 +410,9 @@ mod tests { // 14 strings and 3 values per frame = ceil(14/3) = 5 frames. let zstd_array = Zstd::from_var_bin_view(&strings, 3, 3)?; - let cpu_result = Zstd::decompress(&zstd_array, cuda_ctx.execution_ctx())?.to_canonical()?; + let cpu_result = zstd_array + .decompress(cuda_ctx.execution_ctx())? + .to_canonical()?; let gpu_result = ZstdExecutor .execute(zstd_array.into_array(), &mut cuda_ctx) .await?; diff --git a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs index d44b57615dc..1891ee2375c 100644 --- a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs +++ b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs @@ -55,7 +55,7 @@ async fn decode_zstd_buffers( let compressed_buffers = plan.compressed_buffers(); if compressed_buffers.is_empty() { - let inner_array = ZstdBuffers::build_inner(&array, &[], ctx.session())?; + let inner_array = array.build_inner(&[], ctx.session())?; return inner_array.execute_cuda(ctx).await; } @@ -158,7 +158,7 @@ async fn decode_zstd_buffers( let output_handle = BufferHandle::new_device(Arc::new(CudaDeviceBuffer::new(device_output))); let decompressed_buffers = plan.split_output_handle(&output_handle)?; - let inner_array = ZstdBuffers::build_inner(&array, &decompressed_buffers, ctx.session())?; + let inner_array = array.build_inner(&decompressed_buffers, ctx.session())?; inner_array.execute_cuda(ctx).await } @@ -223,6 +223,7 @@ mod tests { use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; use vortex::array::assert_arrays_eq; + use vortex::encodings::zstd::ZstdBuffersData; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::session::VortexSession; @@ -237,7 +238,7 @@ mod tests { .vortex_expect("failed to create execution context"); let input = PrimitiveArray::from_iter(0i64..1024).into_array(); - let compressed = ZstdBuffers::compress(&input, 3)?; + let compressed = ZstdBuffersData::compress(&input, 3)?; let cpu_result = compressed.clone().into_array().to_canonical()?; let gpu_result = ZstdBuffersExecutor @@ -264,7 +265,7 @@ mod tests { "baz", ]) .into_array(); - let compressed = ZstdBuffers::compress(&input, 3)?; + let compressed = ZstdBuffersData::compress(&input, 3)?; let cpu_result = compressed.clone().into_array().to_canonical()?; let gpu_result = ZstdBuffersExecutor diff --git a/vortex-cuda/src/kernel/filter/decimal.rs b/vortex-cuda/src/kernel/filter/decimal.rs index 5e468dadee3..88e07465968 100644 --- a/vortex-cuda/src/kernel/filter/decimal.rs +++ b/vortex-cuda/src/kernel/filter/decimal.rs @@ -4,7 +4,7 @@ use cudarc::driver::DeviceRepr; use vortex::array::Canonical; use vortex::array::arrays::DecimalArray; -use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::decimal::DecimalArrayParts; use vortex::dtype::NativeDecimalType; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -18,7 +18,7 @@ pub(super) async fn filter_decimal VortexResult { - let DecimalDataParts { + let DecimalArrayParts { values, validity, decimal_dtype, diff --git a/vortex-cuda/src/kernel/filter/mod.rs b/vortex-cuda/src/kernel/filter/mod.rs index 4cf8efbc203..588c3e07925 100644 --- a/vortex-cuda/src/kernel/filter/mod.rs +++ b/vortex-cuda/src/kernel/filter/mod.rs @@ -19,7 +19,7 @@ use tracing::instrument; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::arrays::Filter; -use vortex::array::arrays::filter::FilterDataParts; +use vortex::array::arrays::filter::FilterArrayParts; use vortex::array::buffer::BufferHandle; use vortex::array::match_each_decimal_value_type; use vortex::array::match_each_native_simd_ptype; @@ -53,7 +53,7 @@ impl CudaExecute for FilterExecutor { .try_into::() .map_err(|_| vortex_err!("Expected FilterArray"))?; - let FilterDataParts { child, mask } = filter_array.into_data().into_parts(); + let FilterArrayParts { child, mask } = filter_array.into_data().into_parts(); // Early return for trivial cases. match mask { diff --git a/vortex-cuda/src/kernel/filter/primitive.rs b/vortex-cuda/src/kernel/filter/primitive.rs index f45fe3f2994..68e39edd60f 100644 --- a/vortex-cuda/src/kernel/filter/primitive.rs +++ b/vortex-cuda/src/kernel/filter/primitive.rs @@ -4,7 +4,7 @@ use cudarc::driver::DeviceRepr; use vortex::array::Canonical; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::dtype::NativePType; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -22,7 +22,7 @@ pub(super) async fn filter_primitive( where T: NativePType + DeviceRepr + CubFilterable + Send + Sync + 'static, { - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer, validity, .. } = array.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/filter/varbinview.rs b/vortex-cuda/src/kernel/filter/varbinview.rs index 8d673180d58..143e3f26ce7 100644 --- a/vortex-cuda/src/kernel/filter/varbinview.rs +++ b/vortex-cuda/src/kernel/filter/varbinview.rs @@ -3,7 +3,7 @@ use vortex::array::Canonical; use vortex::array::arrays::VarBinViewArray; -use vortex::array::arrays::varbinview::VarBinViewDataParts; +use vortex::array::arrays::varbinview::VarBinViewArrayParts; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -15,7 +15,7 @@ pub(super) async fn filter_varbinview( mask: Mask, ctx: &mut CudaExecutionCtx, ) -> VortexResult { - let VarBinViewDataParts { + let VarBinViewArrayParts { views, buffers, validity, diff --git a/vortex-cuda/src/kernel/patched/mod.rs b/vortex-cuda/src/kernel/patched/mod.rs new file mode 100644 index 00000000000..0d735177e5d --- /dev/null +++ b/vortex-cuda/src/kernel/patched/mod.rs @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 5d833c1c417..9da4ddf8f51 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -12,7 +12,7 @@ pub mod gpu { use cudarc::driver::DeviceRepr; use cudarc::driver::PushKernelArg; use tracing::instrument; -use vortex::array::arrays::primitive::PrimitiveDataParts; +use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::array::patches::Patches; use vortex::array::validity::Validity; use vortex::dtype::NativePType; @@ -67,12 +67,12 @@ pub(crate) async fn execute_patches< let patches_len = indices.len(); let patches_len_u64 = patches_len as u64; - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer: indices_buffer, .. } = indices.into_data().into_parts(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer: values_buffer, .. } = values.into_data().into_parts(); @@ -105,7 +105,7 @@ mod tests { use vortex::array::ToCanonical; use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; - use vortex::array::arrays::primitive::PrimitiveDataParts; + use vortex::array::arrays::primitive::PrimitiveArrayParts; use vortex::array::assert_arrays_eq; use vortex::array::buffer::BufferHandle; use vortex::array::builtins::ArrayBuiltins; @@ -167,7 +167,7 @@ mod tests { ) .unwrap(); - let PrimitiveDataParts { + let PrimitiveArrayParts { buffer: cuda_buffer, .. } = values.into_data().into_parts(); diff --git a/vortex-cuda/src/kernel/slice/mod.rs b/vortex-cuda/src/kernel/slice/mod.rs index 5326780b5f9..b418e0675f7 100644 --- a/vortex-cuda/src/kernel/slice/mod.rs +++ b/vortex-cuda/src/kernel/slice/mod.rs @@ -7,7 +7,7 @@ use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; use vortex::array::arrays::Slice; -use vortex::array::arrays::slice::SliceDataParts; +use vortex::array::arrays::slice::SliceArrayParts; use vortex::error::VortexResult; use vortex::error::vortex_err; @@ -33,7 +33,7 @@ impl CudaExecute for SliceExecutor { ) })?; - let SliceDataParts { child, range } = slice_array.into_data().into_parts(); + let SliceArrayParts { child, range } = slice_array.into_data().into_parts(); let child = child.execute_cuda(ctx).await?; match child { diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index 0a8efcaac5f..bd9fd51e276 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -26,8 +26,8 @@ use vortex::array::expr::stats::Stat; use vortex::array::expr::stats::StatsProvider; use vortex::array::normalize::NormalizeOptions; use vortex::array::normalize::Operation; +use vortex::array::serde::ArrayParts; use vortex::array::serde::SerializeOptions; -use vortex::array::serde::SerializedArray; use vortex::array::session::ArrayRegistry; use vortex::array::stats::StatsSetRef; use vortex::buffer::BufferString; @@ -249,7 +249,7 @@ impl CudaFlatReader { async move { let segment = segment_fut.await?; - let parts = SerializedArray::from_flatbuffer_and_segment_with_overrides( + let parts = ArrayParts::from_flatbuffer_and_segment_with_overrides( array_tree, segment, &host_buffers, diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index 765e76272ca..00026eb9f90 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -6,7 +6,7 @@ use std::marker::PhantomData; use num_traits::ToPrimitive; use vortex::array::ExecutionCtx; use vortex::array::arrays::DecimalArray; -use vortex::array::arrays::decimal::DecimalDataParts; +use vortex::array::arrays::decimal::DecimalArrayParts; use vortex::array::match_each_decimal_value_type; use vortex::buffer::Buffer; use vortex::dtype::BigCast; @@ -42,7 +42,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let DecimalDataParts { + let DecimalArrayParts { validity, decimal_dtype, values_type, diff --git a/vortex-duckdb/src/exporter/list.rs b/vortex-duckdb/src/exporter/list.rs index f2945123a13..fab62dcc8cb 100644 --- a/vortex-duckdb/src/exporter/list.rs +++ b/vortex-duckdb/src/exporter/list.rs @@ -8,7 +8,7 @@ use parking_lot::Mutex; use vortex::array::ExecutionCtx; use vortex::array::arrays::ListArray; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::list::ListDataParts; +use vortex::array::arrays::list::ListArrayParts; use vortex::array::match_each_integer_ptype; use vortex::dtype::IntegerPType; use vortex::error::VortexResult; @@ -45,7 +45,7 @@ pub(crate) fn new_exporter( ) -> VortexResult> { let array_len = array.len(); // Cache an `elements` vector up front so that future exports can reference it. - let ListDataParts { + let ListArrayParts { elements, offsets, validity, diff --git a/vortex-duckdb/src/exporter/list_view.rs b/vortex-duckdb/src/exporter/list_view.rs index 226f399c92f..4b76f534ff2 100644 --- a/vortex-duckdb/src/exporter/list_view.rs +++ b/vortex-duckdb/src/exporter/list_view.rs @@ -8,7 +8,7 @@ use parking_lot::Mutex; use vortex::array::ExecutionCtx; use vortex::array::arrays::ListViewArray; use vortex::array::arrays::PrimitiveArray; -use vortex::array::arrays::listview::ListViewDataParts; +use vortex::array::arrays::listview::ListViewArrayParts; use vortex::array::match_each_integer_ptype; use vortex::dtype::DType; use vortex::dtype::IntegerPType; @@ -47,7 +47,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let ListViewDataParts { + let ListViewArrayParts { elements_dtype, elements, offsets, diff --git a/vortex-duckdb/src/exporter/run_end.rs b/vortex-duckdb/src/exporter/run_end.rs index af21438130d..0d05cf67a30 100644 --- a/vortex-duckdb/src/exporter/run_end.rs +++ b/vortex-duckdb/src/exporter/run_end.rs @@ -11,7 +11,7 @@ use vortex::array::search_sorted::SearchSorted; use vortex::array::search_sorted::SearchSortedSide; use vortex::dtype::IntegerPType; use vortex::encodings::runend::RunEndArray; -use vortex::encodings::runend::RunEndDataParts; +use vortex::encodings::runend::RunEndArrayParts; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -38,7 +38,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let offset = array.offset(); - let RunEndDataParts { ends, values, .. } = array.into_data().into_parts(); + let RunEndArrayParts { ends, values } = array.into_data().into_parts(); let ends = ends.execute::(ctx)?; let values_exporter = new_array_exporter(values.clone(), cache, ctx)?; diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index 19ac59c69d9..8c07ada0533 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -5,7 +5,7 @@ use vortex::array::ExecutionCtx; use vortex::array::IntoArray; use vortex::array::arrays::BoolArray; use vortex::array::arrays::StructArray; -use vortex::array::arrays::struct_::StructDataParts; +use vortex::array::arrays::struct_::StructArrayParts; use vortex::array::builtins::ArrayBuiltins; use vortex::error::VortexResult; @@ -27,12 +27,12 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let StructDataParts { + let StructArrayParts { validity, struct_fields, fields, .. - } = array.into_data().into_parts(); + } = array.into_parts(); let validity = validity.to_array(len).execute::(ctx)?; if validity.to_bit_buffer().true_count() == 0 { diff --git a/vortex-duckdb/src/exporter/varbinview.rs b/vortex-duckdb/src/exporter/varbinview.rs index 23142b5c4cb..f0667ee6144 100644 --- a/vortex-duckdb/src/exporter/varbinview.rs +++ b/vortex-duckdb/src/exporter/varbinview.rs @@ -8,7 +8,7 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::varbinview::BinaryView; use vortex::array::arrays::varbinview::Inlined; -use vortex::array::arrays::varbinview::VarBinViewDataParts; +use vortex::array::arrays::varbinview::VarBinViewArrayParts; use vortex::buffer::Buffer; use vortex::buffer::ByteBuffer; use vortex::error::VortexResult; @@ -32,7 +32,7 @@ pub(crate) fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let len = array.len(); - let VarBinViewDataParts { + let VarBinViewArrayParts { validity, dtype, views, diff --git a/vortex-ipc/public-api.lock b/vortex-ipc/public-api.lock index 2c79b3ef0cb..101ac7940ef 100644 --- a/vortex-ipc/public-api.lock +++ b/vortex-ipc/public-api.lock @@ -46,7 +46,7 @@ pub mod vortex_ipc::messages pub enum vortex_ipc::messages::DecoderMessage -pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::SerializedArray, vortex_session::registry::ReadContext, usize)) +pub vortex_ipc::messages::DecoderMessage::Array((vortex_array::serde::ArrayParts, vortex_session::registry::ReadContext, usize)) pub vortex_ipc::messages::DecoderMessage::Buffer(vortex_buffer::ByteBuffer) diff --git a/vortex-ipc/src/messages/decoder.rs b/vortex-ipc/src/messages/decoder.rs index d0108830aa3..dabbc9e47a8 100644 --- a/vortex-ipc/src/messages/decoder.rs +++ b/vortex-ipc/src/messages/decoder.rs @@ -8,7 +8,7 @@ use bytes::Buf; use flatbuffers::root; use flatbuffers::root_unchecked; use vortex_array::ArrayId; -use vortex_array::serde::SerializedArray; +use vortex_array::serde::ArrayParts; use vortex_buffer::AlignedBuf; use vortex_buffer::Alignment; use vortex_buffer::ByteBuffer; @@ -25,7 +25,7 @@ use vortex_session::registry::ReadContext; /// A message decoded from an IPC stream. #[derive(Debug)] pub enum DecoderMessage { - Array((SerializedArray, ReadContext, usize)), + Array((ArrayParts, ReadContext, usize)), Buffer(ByteBuffer), DType(FlatBuffer), } @@ -115,7 +115,7 @@ impl MessageDecoder { MessageHeader::ArrayMessage => { // We don't care about alignment here since ArrayParts will handle it. let body = bytes.copy_to_aligned(body_length, Alignment::new(1)); - let parts = SerializedArray::try_from(body)?; + let parts = ArrayParts::try_from(body)?; let header = msg .header_as_array_message() diff --git a/vortex-layout/src/display.rs b/vortex-layout/src/display.rs index 30e029ec8a3..c3c36a06de1 100644 --- a/vortex-layout/src/display.rs +++ b/vortex-layout/src/display.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use futures::future::try_join_all; use termtree::Tree; -use vortex_array::serde::SerializedArray; +use vortex_array::serde::ArrayParts; use vortex_error::VortexResult; use vortex_utils::aliases::hash_map::HashMap; @@ -35,7 +35,7 @@ pub(super) async fn display_tree_with_segment_sizes( let segment_source = segment_source.clone(); async move { let buffer = segment_source.request(segment_id).await?; - let parts = SerializedArray::try_from(buffer)?; + let parts = ArrayParts::try_from(buffer)?; VortexResult::Ok((segment_id, parts.buffer_lengths())) } }); @@ -80,7 +80,7 @@ fn format_flat_layout_buffers( // First, try to get buffer info from inline array_tree if let Some(array_tree) = flat_layout.array_tree() - && let Ok(parts) = SerializedArray::from_array_tree(array_tree.as_ref().to_vec()) + && let Ok(parts) = ArrayParts::from_array_tree(array_tree.as_ref().to_vec()) { return format_buffer_sizes(&parts.buffer_lengths(), *segment_id); } @@ -225,7 +225,7 @@ mod tests { use vortex_array::dtype::Nullability::NonNullable; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; - use vortex_array::serde::SerializedArray; + use vortex_array::serde::ArrayParts; use vortex_array::validity::Validity; use vortex_buffer::BitBufferMut; use vortex_buffer::buffer; @@ -425,7 +425,7 @@ vortex.chunked, dtype: i32, children: 2, rows: 10 .array_tree() .expect("array_tree should be populated when FLAT_LAYOUT_INLINE_ARRAY_NODE is set"); - let parts = SerializedArray::from_array_tree(array_tree.as_ref().to_vec()) + let parts = ArrayParts::from_array_tree(array_tree.as_ref().to_vec()) .expect("should parse array_tree"); assert_eq!(parts.buffer_lengths(), vec![20]); // 5 i32 values = 20 bytes diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index 7922e1c1560..779cf12b3eb 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -14,7 +14,7 @@ use vortex_array::VortexSessionExecute; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::Expression; -use vortex_array::serde::SerializedArray; +use vortex_array::serde::ArrayParts; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_mask::Mask; @@ -72,10 +72,10 @@ impl FlatReader { let segment = segment_fut.await?; let parts = if let Some(array_tree) = array_tree { // Use the pre-stored flatbuffer from layout metadata combined with segment buffers. - SerializedArray::from_flatbuffer_and_segment(array_tree, segment)? + ArrayParts::from_flatbuffer_and_segment(array_tree, segment)? } else { // Parse the flatbuffer from the segment itself. - SerializedArray::try_from(segment)? + ArrayParts::try_from(segment)? }; parts .decode(&dtype, row_count, &ctx, &session) diff --git a/vortex-layout/src/layouts/table.rs b/vortex-layout/src/layouts/table.rs index 9df9ed66809..db92059c58b 100644 --- a/vortex-layout/src/layouts/table.rs +++ b/vortex-layout/src/layouts/table.rs @@ -86,14 +86,13 @@ impl TableStrategy { /// ```ignore /// # use std::sync::Arc; /// # use vortex_array::dtype::{field_path, Field, FieldPath}; - /// # use vortex_btrblocks::BtrBlocksCompressor; /// # use vortex_layout::layouts::compressed::CompressingStrategy; /// # use vortex_layout::layouts::flat::writer::FlatLayoutStrategy; /// # use vortex_layout::layouts::table::TableStrategy; /// + /// # use vortex_btrblocks::BtrBlocksCompressor; /// // A strategy for compressing data using the balanced BtrBlocks compressor. - /// let compress = - /// CompressingStrategy::new(FlatLayoutStrategy::default(), BtrBlocksCompressor::default()); + /// let compress = CompressingStrategy::new(FlatLayoutStrategy::default(), BtrBlocksCompressor::default()); /// /// // Our combined strategy uses no compression for validity buffers, BtrBlocks compression /// // for most columns, and stores a nested binary column uncompressed (flat) because it diff --git a/vortex-python/python/vortex/__init__.py b/vortex-python/python/vortex/__init__.py index 5396f50b7a0..40c3e7c4a91 100644 --- a/vortex-python/python/vortex/__init__.py +++ b/vortex-python/python/vortex/__init__.py @@ -74,7 +74,7 @@ Utf8Scalar, scalar, ) -from ._lib.serde import ArrayContext, SerializedArray # pyright: ignore[reportMissingModuleSource] +from ._lib.serde import ArrayContext, ArrayParts # pyright: ignore[reportMissingModuleSource] from .arrays import ( Array, PyArray, @@ -177,7 +177,7 @@ "ExtensionScalar", # Serde "ArrayContext", - "SerializedArray", + "ArrayParts", # Pickle "_unpickle_array", # File diff --git a/vortex-python/python/vortex/_lib/serde.pyi b/vortex-python/python/vortex/_lib/serde.pyi index b4d052b3e47..626946a7e6c 100644 --- a/vortex-python/python/vortex/_lib/serde.pyi +++ b/vortex-python/python/vortex/_lib/serde.pyi @@ -10,9 +10,9 @@ from .arrays import Array from .dtype import DType @final -class SerializedArray: +class ArrayParts: @staticmethod - def parse(data: bytes) -> SerializedArray: ... + def parse(data: bytes) -> ArrayParts: ... @property def metadata(self) -> bytes | None: ... @property @@ -22,7 +22,7 @@ class SerializedArray: @property def nchildren(self) -> int: ... @property - def children(self) -> list[SerializedArray]: ... + def children(self) -> list[ArrayParts]: ... def decode(self, ctx: ArrayContext, dtype: DType, len: int) -> pa.Array[pa.Scalar[pa.DataType]]: ... @final diff --git a/vortex-python/python/vortex/arrays.py b/vortex-python/python/vortex/arrays.py index 5181400cb32..4180510d501 100644 --- a/vortex-python/python/vortex/arrays.py +++ b/vortex-python/python/vortex/arrays.py @@ -13,7 +13,7 @@ from vortex._lib.dtype import DType # pyright: ignore[reportMissingModuleSource] from vortex._lib.serde import ( # pyright: ignore[reportMissingModuleSource] ArrayContext, - SerializedArray, + ArrayParts, decode_ipc_array_buffers, ) @@ -463,12 +463,12 @@ def dtype(self) -> DType: @classmethod @abc.abstractmethod - def decode(cls, parts: SerializedArray, ctx: ArrayContext, dtype: DType, len: int) -> Array: + def decode(cls, parts: ArrayParts, ctx: ArrayContext, dtype: DType, len: int) -> Array: """Decode an array from its component parts. - :class:`SerializedArray` contains the metadata, buffers and child :class:`SerializedArray` - that represent the current array. Implementations of this function should validate this - information, and then construct a new array. + :class:`ArrayParts` contains the metadata, buffers and child :class:`ArrayParts` that represent the + current array. Implementations of this function should validate this information, and then construct + a new array. """ diff --git a/vortex-python/python/vortex/serde.py b/vortex-python/python/vortex/serde.py index c30ceef4058..cb1f7f6c9a0 100644 --- a/vortex-python/python/vortex/serde.py +++ b/vortex-python/python/vortex/serde.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -from vortex._lib.serde import ArrayContext, SerializedArray # pyright: ignore[reportMissingModuleSource] +from vortex._lib.serde import ArrayContext, ArrayParts # pyright: ignore[reportMissingModuleSource] -__all__ = ["SerializedArray", "ArrayContext"] +__all__ = ["ArrayParts", "ArrayContext"] diff --git a/vortex-python/src/arrays/py/array.rs b/vortex-python/src/arrays/py/array.rs index 1653ed4b2aa..9163ae0c37d 100644 --- a/vortex-python/src/arrays/py/array.rs +++ b/vortex-python/src/arrays/py/array.rs @@ -8,12 +8,7 @@ use pyo3::FromPyObject; use pyo3::Py; use pyo3::PyAny; use pyo3::prelude::*; -use vortex::array::Array; -use vortex::array::ArrayParts; -use vortex::array::ArrayRef; -use vortex::array::IntoArray; use vortex::array::stats::ArrayStats; -use vortex::array::stats::StatsSet; use vortex::dtype::DType; use crate::arrays::py::PyPythonArray; @@ -60,18 +55,3 @@ impl<'py> IntoPyObject<'py> for PythonArray { Ok(self.object.bind(py).to_owned()) } } - -impl IntoArray for PythonArray { - fn into_array(self) -> ArrayRef { - let vtable = self.vtable.clone(); - let dtype = self.dtype.clone(); - let len = self.len; - let stats = StatsSet::from(self.stats.clone()); - match Array::try_from_parts(ArrayParts::new(vtable, dtype, len, self)) { - Ok(array) => array.with_stats_set(stats).into_array(), - Err(err) => unreachable!( - "PythonArray metadata extracted from PyPythonArray must be valid: {err}" - ), - } - } -} diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index dbbc767f253..3f323295020 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -4,6 +4,9 @@ use std::hash::Hash; use std::sync::Arc; +use pyo3::intern; +use pyo3::prelude::*; +use pyo3::types::PyBytes; use vortex::array::Array; use vortex::array::ArrayId; use vortex::array::ArrayRef; @@ -12,15 +15,19 @@ use vortex::array::ExecutionCtx; use vortex::array::ExecutionResult; use vortex::array::OperationsVTable; use vortex::array::Precision; +use vortex::array::RawMetadata; +use vortex::array::SerializeMetadata; use vortex::array::VTable; use vortex::array::ValidityVTable; use vortex::array::buffer::BufferHandle; +use vortex::array::serde::ArrayChildren; +use vortex::array::stats::ArrayStats; use vortex::array::validity::Validity; use vortex::array::vtable; use vortex::dtype::DType; use vortex::error::VortexResult; -use vortex::error::vortex_bail; use vortex::error::vortex_ensure; +use vortex::error::vortex_err; use vortex::error::vortex_panic; use vortex::scalar::Scalar; use vortex::session::VortexSession; @@ -38,18 +45,28 @@ pub struct PythonVTable { impl VTable for PythonVTable { type ArrayData = PythonArray; + type Metadata = RawMetadata; type OperationsVTable = Self; type ValidityVTable = Self; + fn vtable(array: &Self::ArrayData) -> &Self { + &array.vtable + } + fn id(&self) -> ArrayId { self.id.clone() } - fn validate(&self, data: &PythonArray, dtype: &DType, len: usize) -> VortexResult<()> { - vortex_ensure!(data.vtable.id == self.id, "PythonArray vtable id mismatch"); - vortex_ensure!(&data.dtype == dtype, "PythonArray dtype mismatch"); - vortex_ensure!(data.len == len, "PythonArray len mismatch"); - Ok(()) + fn len(array: &PythonArray) -> usize { + array.len + } + + fn dtype(array: &PythonArray) -> &DType { + &array.dtype + } + + fn stats(array: &PythonArray) -> &ArrayStats { + &array.stats } fn array_hash(array: &PythonArray, state: &mut H, _precision: Precision) { @@ -84,21 +101,51 @@ impl VTable for PythonVTable { vortex_panic!("PythonArray child_name index {idx} out of bounds") } - fn serialize(_array: ArrayView<'_, Self>) -> VortexResult>> { - Ok(None) + fn metadata(array: ArrayView<'_, Self>) -> VortexResult { + Python::attach(|py| { + let obj = array.object.bind(py); + if !obj + .hasattr(intern!(py, "metadata")) + .map_err(|e| vortex_err!("{}", e))? + { + // The class does not have a metadata attribute so does not support serialization. + return Ok(RawMetadata(vec![])); + } + + let bytes = obj + .call_method(intern!(py, "__vx_metadata__"), (), None) + .map_err(|e| vortex_err!("{}", e))? + .cast::() + .map_err(|_| vortex_err!("Expected array metadata to be Python bytes"))? + .as_bytes() + .to_vec(); + + Ok(RawMetadata(bytes)) + }) + } + + fn serialize(metadata: Self::Metadata) -> VortexResult>> { + Ok(Some(metadata.serialize())) } fn deserialize( - &self, + bytes: &[u8], _dtype: &DType, _len: usize, - bytes: &[u8], _buffers: &[BufferHandle], - _children: &dyn vortex::array::serde::ArrayChildren, _session: &VortexSession, + ) -> VortexResult { + Ok(RawMetadata(bytes.to_vec())) + } + + fn build( + _dtype: &DType, + _len: usize, + _metadata: &Self::Metadata, + _buffers: &[BufferHandle], + _children: &dyn ArrayChildren, ) -> VortexResult { - _ = bytes; - vortex_bail!("PythonArray deserialization is not supported"); + todo!() } fn slots(_array: ArrayView<'_, Self>) -> &[Option] { diff --git a/vortex-python/src/serde/mod.rs b/vortex-python/src/serde/mod.rs index c4e4cc95948..e3290821194 100644 --- a/vortex-python/src/serde/mod.rs +++ b/vortex-python/src/serde/mod.rs @@ -21,7 +21,7 @@ use crate::error::PyVortexResult; use crate::install_module; use crate::serde::context::PyArrayContext; use crate::serde::context::PyReadContext; -use crate::serde::parts::PySerializedArray; +use crate::serde::parts::PyArrayParts; /// Register serde functions and classes. pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { @@ -29,7 +29,7 @@ pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { parent.add_submodule(&m)?; install_module("vortex._lib.serde", &m)?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_function(wrap_pyfunction!(decode_ipc_array, &m)?)?; diff --git a/vortex-python/src/serde/parts.rs b/vortex-python/src/serde/parts.rs index e05a6b6d1ac..2decc0a926a 100644 --- a/vortex-python/src/serde/parts.rs +++ b/vortex-python/src/serde/parts.rs @@ -11,7 +11,7 @@ use pyo3::intern; use pyo3::prelude::PyAnyMethods; use pyo3::pyclass; use pyo3::pymethods; -use vortex::array::serde::SerializedArray; +use vortex::array::serde::ArrayParts; use vortex::buffer::ByteBuffer; use crate::SESSION; @@ -20,34 +20,34 @@ use crate::dtype::PyDType; use crate::error::PyVortexResult; use crate::serde::context::PyReadContext; -/// SerializedArray is a parsed representation of a serialized array. +/// ArrayParts is a parsed representation of a serialized array. /// /// It can be decoded into a full array using the `decode` method. -#[pyclass(name = "SerializedArray", module = "vortex", frozen)] -pub(crate) struct PySerializedArray(SerializedArray); +#[pyclass(name = "ArrayParts", module = "vortex", frozen)] +pub(crate) struct PyArrayParts(ArrayParts); -impl Deref for PySerializedArray { - type Target = SerializedArray; +impl Deref for PyArrayParts { + type Target = ArrayParts; fn deref(&self) -> &Self::Target { &self.0 } } -impl From for PySerializedArray { - fn from(parts: SerializedArray) -> Self { +impl From for PyArrayParts { + fn from(parts: ArrayParts) -> Self { Self(parts) } } #[pymethods] -impl PySerializedArray { +impl PyArrayParts { /// Parse a serialized array into its parts. #[staticmethod] - fn parse(data: &[u8]) -> PyVortexResult { + fn parse(data: &[u8]) -> PyVortexResult { // TODO(ngates): create a buffer from a slice of bytes? let buffer = ByteBuffer::copy_from(data); - Ok(PySerializedArray(SerializedArray::try_from(buffer)?)) + Ok(PyArrayParts(ArrayParts::try_from(buffer)?)) } /// Decode the array parts into a full array. @@ -115,12 +115,12 @@ impl PySerializedArray { self.0.nchildren() } - /// Return the child :class:`~vortex.SerializedArray` of the array. + /// Return the child :class:`~vortex.ArrayParts` of the array. #[getter] - fn children(&self) -> Vec { + fn children(&self) -> Vec { (0..self.0.nchildren()) .map(|idx| self.0.child(idx)) - .map(PySerializedArray) + .map(PyArrayParts) .collect() } } diff --git a/vortex-python/test/test_pyarray.py b/vortex-python/test/test_pyarray.py index f29ca41408f..5713ff669f0 100644 --- a/vortex-python/test/test_pyarray.py +++ b/vortex-python/test/test_pyarray.py @@ -91,7 +91,7 @@ def encode(cls, array: pa.Array[pa.Scalar[pa.DataType]], config: ChunkConfig | N @override @classmethod - def decode(cls, parts: vx.SerializedArray, ctx: vx.ArrayContext, dtype: vx.DType, len: int) -> vx.Array: + def decode(cls, parts: vx.ArrayParts, ctx: vx.ArrayContext, dtype: vx.DType, len: int) -> vx.Array: """Decode the serialized array parts into an array.""" assert pco raise NotImplementedError diff --git a/vortex-tui/src/browse/app.rs b/vortex-tui/src/browse/app.rs index 340f19fbed6..a6292d071a8 100644 --- a/vortex-tui/src/browse/app.rs +++ b/vortex-tui/src/browse/app.rs @@ -352,7 +352,7 @@ impl AppState { #[cfg(feature = "native")] pub(crate) async fn load_flat_data(&mut self) { use vortex::array::MaskFuture; - use vortex::array::serde::SerializedArray; + use vortex::array::serde::ArrayParts; use vortex::expr::root; let layout = &self.cursor.layout().clone(); @@ -384,7 +384,7 @@ impl AppState { .await .vortex_expect("Failed to read segment"); self.cached_flatbuffer_size = Some( - SerializedArray::try_from(segment) + ArrayParts::try_from(segment) .vortex_expect("Failed to parse segment") .metadata() .len(), diff --git a/vortex-tui/src/wasm.rs b/vortex-tui/src/wasm.rs index 8b643161fe2..f8acae67c61 100644 --- a/vortex-tui/src/wasm.rs +++ b/vortex-tui/src/wasm.rs @@ -14,7 +14,7 @@ use ratzilla::CanvasBackend; use ratzilla::WebRenderer; use ratzilla::ratatui::Terminal; use vortex::array::MaskFuture; -use vortex::array::serde::SerializedArray; +use vortex::array::serde::ArrayParts; use vortex::error::VortexExpect; use vortex::expr::root; use vortex::layout::layouts::flat::Flat; @@ -102,7 +102,7 @@ async fn load_flatbuffer_size( .request(segment_id) .await .vortex_expect("Failed to read segment"); - SerializedArray::try_from(segment) + ArrayParts::try_from(segment) .vortex_expect("Failed to parse segment") .metadata() .len() diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 8a4cfa53f59..d2928a81a8f 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -27,7 +27,6 @@ use vortex::array::session::ArraySession; use vortex::dtype::PType; use vortex::encodings::alp::RDEncoder; use vortex::encodings::alp::alp_encode; -use vortex::encodings::fastlanes::Delta; use vortex::encodings::fastlanes::DeltaData; use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::delta_compress; @@ -37,7 +36,6 @@ use vortex::encodings::pco::Pco; use vortex::encodings::runend::RunEnd; use vortex::encodings::sequence::sequence_encode; use vortex::encodings::zigzag::zigzag_encode; -use vortex::encodings::zstd::Zstd; use vortex::encodings::zstd::ZstdData; use vortex_sequence::Sequence; use vortex_session::VortexSession; @@ -164,9 +162,10 @@ fn bench_delta_compress_u32(bencher: Bencher) { fn bench_delta_decompress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); let (bases, deltas) = delta_compress(&uint_array, &mut SESSION.create_execution_ctx()).unwrap(); - let compressed = Delta::try_new(bases.into_array(), deltas.into_array(), 0, uint_array.len()) - .unwrap() - .into_array(); + let compressed = + DeltaData::try_new(bases.into_array(), deltas.into_array(), 0, uint_array.len()) + .unwrap() + .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| &compressed) @@ -328,13 +327,9 @@ fn bench_zstd_compress_u32(bencher: Bencher) { #[divan::bench(name = "zstd_decompress_u32")] fn bench_zstd_decompress_u32(bencher: Bencher) { let (uint_array, ..) = setup_primitive_arrays(); - let dtype = uint_array.dtype().clone(); - let compressed = Zstd::try_new( - dtype, - ZstdData::from_array(uint_array.into_array(), 3, 8192).unwrap(), - ) - .unwrap() - .into_array(); + let compressed = ZstdData::from_array(uint_array.into_array(), 3, 8192) + .unwrap() + .into_array(); with_byte_counter(bencher, NUM_VALUES * 4) .with_inputs(|| &compressed) @@ -413,13 +408,9 @@ fn bench_zstd_compress_string(bencher: Bencher) { fn bench_zstd_decompress_string(bencher: Bencher) { let varbinview_arr = VarBinViewArray::from_iter_str(gen_varbin_words(NUM_VALUES as usize, 0.00005)); - let dtype = varbinview_arr.dtype().clone(); - let compressed = Zstd::try_new( - dtype, - ZstdData::from_array(varbinview_arr.clone().into_array(), 3, 8192).unwrap(), - ) - .unwrap() - .into_array(); + let compressed = ZstdData::from_array(varbinview_arr.clone().into_array(), 3, 8192) + .unwrap() + .into_array(); let nbytes = varbinview_arr.into_array().nbytes() as u64; with_byte_counter(bencher, nbytes)