diff --git a/crates/re_log_types/src/data_cell.rs b/crates/re_log_types/src/data_cell.rs index b5302b53dea4..a74457025421 100644 --- a/crates/re_log_types/src/data_cell.rs +++ b/crates/re_log_types/src/data_cell.rs @@ -362,17 +362,8 @@ impl DataCell { pub fn try_to_native<'a, C: Component + Default + 'a>( &'a self, ) -> DataCellResult + '_> { - Ok(C::try_iter_from_arrow(self.inner.values.as_ref())? - .map(C::convert_item_to_self) - .map(|v| { - // TODO(#2523): This unwrap and the `Default` bounds should go away once we move to fallible iterators - v.unwrap_or_else(|| { - re_log::warn_once!( - "Unexpected missing data when iterating non-optional data-cell. Falling back on Default value." - ); - C::default() - }) - })) + re_tracing::profile_function!(C::name().as_str()); + Ok(C::try_iter_from_arrow(self.inner.values.as_ref())?.map(C::convert_item_to_self)) } /// Returns the contents of an expected mono-component as an `Option`. @@ -381,7 +372,7 @@ impl DataCell { #[inline] pub fn try_to_native_mono<'a, C: Component + 'a>(&'a self) -> DataCellResult> { let mut iter = - C::try_iter_from_arrow(self.inner.values.as_ref())?.map(C::convert_item_to_self); + C::try_iter_from_arrow(self.inner.values.as_ref())?.map(C::convert_item_to_opt_self); let result = match iter.next() { // It's ok to have no result from the iteration: this is what we @@ -420,7 +411,7 @@ impl DataCell { pub fn try_to_native_opt<'a, C: Component + 'a>( &'a self, ) -> DataCellResult> + '_> { - Ok(C::try_iter_from_arrow(self.inner.values.as_ref())?.map(C::convert_item_to_self)) + Ok(C::try_iter_from_arrow(self.inner.values.as_ref())?.map(C::convert_item_to_opt_self)) } /// Returns the contents of the cell as an iterator of native optional components. diff --git a/crates/re_log_types/src/lib.rs b/crates/re_log_types/src/lib.rs index da81376ede3a..6aac7b959c67 100644 --- a/crates/re_log_types/src/lib.rs +++ b/crates/re_log_types/src/lib.rs @@ -485,7 +485,7 @@ macro_rules! component_legacy_shim { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { ::arrow_deserialize(item) } } diff --git a/crates/re_types/source_hash.txt b/crates/re_types/source_hash.txt index f79721131660..41a01d89fb28 100644 --- a/crates/re_types/source_hash.txt +++ b/crates/re_types/source_hash.txt @@ -1,4 +1,4 @@ # This is a sha256 hash for all direct and indirect dependencies of this crate's build script. # It can be safely removed at anytime to force the build script to run again. # Check out build.rs to see how it's computed. -4f4bc40656d9770837ed871649f2a9b1ff6ecf6bc635196a064d315bba26e8a8 +2a2dc1d494b0aa772a299db49fbad8272e691c3307dc10ca5f942e387e31e45d diff --git a/crates/re_types/src/arrow_buffer.rs b/crates/re_types/src/arrow_buffer.rs new file mode 100644 index 000000000000..ae7c25ed904c --- /dev/null +++ b/crates/re_types/src/arrow_buffer.rs @@ -0,0 +1,39 @@ +use arrow2::buffer::Buffer; + +/// Convenience-wrapper around an arrow [`Buffer`] that is known to contain a +/// a primitive type. +/// +/// The arrow2 [`Buffer`] object is internally reference-counted and can be +/// easily converted back to a `&[T]` referencing the underlying storage. +/// This avoids some of the lifetime complexities that would otherwise +/// arise from returning a `&[T]` directly, but is significantly more +/// performant than doing the full allocation necessary to return a `Vec`. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ArrowBuffer(pub Buffer); + +impl ArrowBuffer { + #[inline] + /// The number of instances of T stored in this buffer. + pub fn num_instances(&self) -> usize { + // WARNING: If you are touching this code, make sure you know what len() actually does. + // + // There is ambiguity in how arrow2 and arrow-rs talk about buffer lengths, including + // some incorrect documentation: https://github.com/jorgecarleitao/arrow2/issues/1430 + // + // Arrow2 `Buffer` is typed and `len()` is the number of units of `T`, but the documentation + // is currently incorrect. + // Arrow-rs `Buffer` is untyped and len() is in bytes, but `ScalarBuffer`s are in units of T. + self.0.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl From> for ArrowBuffer { + fn from(value: Vec) -> Self { + Self(value.into()) + } +} diff --git a/crates/re_types/src/components/annotation_context.rs b/crates/re_types/src/components/annotation_context.rs index 4e48442662e0..a29cd246b898 100644 --- a/crates/re_types/src/components/annotation_context.rs +++ b/crates/re_types/src/components/annotation_context.rs @@ -153,7 +153,7 @@ impl crate::Loggable for AnnotationContext { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -231,7 +231,7 @@ impl crate::Loggable for AnnotationContext { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/components/class_id.rs b/crates/re_types/src/components/class_id.rs index 594742d8c484..59ee233f6201 100644 --- a/crates/re_types/src/components/class_id.rs +++ b/crates/re_types/src/components/class_id.rs @@ -42,7 +42,7 @@ impl<'a> From<&'a ClassId> for ::std::borrow::Cow<'a, ClassId> { impl crate::Loggable for ClassId { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -119,7 +119,7 @@ impl crate::Loggable for ClassId { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -140,6 +140,38 @@ impl crate::Loggable for ClassId { .with_context("rerun.components.ClassId")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok(data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::UInt16, + data.data_type().clone(), + ) + }) + .with_context("rerun.components.ClassId#id")? + .values() + .as_slice() + .iter() + .copied() + .map(|v| crate::datatypes::ClassId(v)) + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -147,13 +179,18 @@ impl crate::Loggable for ClassId { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for ClassId {} diff --git a/crates/re_types/src/components/color.rs b/crates/re_types/src/components/color.rs index 1d5459b9055b..f9436dfdfc4f 100644 --- a/crates/re_types/src/components/color.rs +++ b/crates/re_types/src/components/color.rs @@ -50,7 +50,7 @@ impl<'a> From<&'a Color> for ::std::borrow::Cow<'a, Color> { impl crate::Loggable for Color { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -127,7 +127,7 @@ impl crate::Loggable for Color { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -148,6 +148,38 @@ impl crate::Loggable for Color { .with_context("rerun.components.Color")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok(data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::UInt32, + data.data_type().clone(), + ) + }) + .with_context("rerun.components.Color#rgba")? + .values() + .as_slice() + .iter() + .copied() + .map(|v| crate::datatypes::Color(v)) + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -155,13 +187,18 @@ impl crate::Loggable for Color { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for Color {} diff --git a/crates/re_types/src/components/disconnected_space.rs b/crates/re_types/src/components/disconnected_space.rs index 6a076918a94e..aa11a5f1bcd3 100644 --- a/crates/re_types/src/components/disconnected_space.rs +++ b/crates/re_types/src/components/disconnected_space.rs @@ -103,7 +103,7 @@ impl crate::Loggable for DisconnectedSpace { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -133,7 +133,7 @@ impl crate::Loggable for DisconnectedSpace { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/components/draw_order.rs b/crates/re_types/src/components/draw_order.rs index b12eec66d4fc..d84cc544a39b 100644 --- a/crates/re_types/src/components/draw_order.rs +++ b/crates/re_types/src/components/draw_order.rs @@ -39,7 +39,7 @@ impl<'a> From<&'a DrawOrder> for ::std::borrow::Cow<'a, DrawOrder> { impl crate::Loggable for DrawOrder { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -106,7 +106,7 @@ impl crate::Loggable for DrawOrder { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -126,6 +126,37 @@ impl crate::Loggable for DrawOrder { .with_context("rerun.components.DrawOrder")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok(data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::Float32, + data.data_type().clone(), + ) + }) + .with_context("rerun.components.DrawOrder#value")? + .values() + .as_slice() + .iter() + .copied() + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -133,13 +164,18 @@ impl crate::Loggable for DrawOrder { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for DrawOrder {} diff --git a/crates/re_types/src/components/instance_key.rs b/crates/re_types/src/components/instance_key.rs index cceaf310a199..d7b0eff1dfad 100644 --- a/crates/re_types/src/components/instance_key.rs +++ b/crates/re_types/src/components/instance_key.rs @@ -33,7 +33,7 @@ impl<'a> From<&'a InstanceKey> for ::std::borrow::Cow<'a, InstanceKey> { impl crate::Loggable for InstanceKey { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -100,7 +100,7 @@ impl crate::Loggable for InstanceKey { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -120,6 +120,37 @@ impl crate::Loggable for InstanceKey { .with_context("rerun.components.InstanceKey")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok(data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::UInt64, + data.data_type().clone(), + ) + }) + .with_context("rerun.components.InstanceKey#value")? + .values() + .as_slice() + .iter() + .copied() + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -127,13 +158,18 @@ impl crate::Loggable for InstanceKey { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for InstanceKey {} diff --git a/crates/re_types/src/components/keypoint_id.rs b/crates/re_types/src/components/keypoint_id.rs index d617816b52c4..304dbdc0ac0a 100644 --- a/crates/re_types/src/components/keypoint_id.rs +++ b/crates/re_types/src/components/keypoint_id.rs @@ -43,7 +43,7 @@ impl<'a> From<&'a KeypointId> for ::std::borrow::Cow<'a, KeypointId> { impl crate::Loggable for KeypointId { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -120,7 +120,7 @@ impl crate::Loggable for KeypointId { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -141,6 +141,38 @@ impl crate::Loggable for KeypointId { .with_context("rerun.components.KeypointId")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok(data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::UInt16, + data.data_type().clone(), + ) + }) + .with_context("rerun.components.KeypointId#id")? + .values() + .as_slice() + .iter() + .copied() + .map(|v| crate::datatypes::KeypointId(v)) + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -148,13 +180,18 @@ impl crate::Loggable for KeypointId { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for KeypointId {} diff --git a/crates/re_types/src/components/label.rs b/crates/re_types/src/components/label.rs index a688ac38d4d0..734633166849 100644 --- a/crates/re_types/src/components/label.rs +++ b/crates/re_types/src/components/label.rs @@ -132,7 +132,7 @@ impl crate::Loggable for Label { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -194,7 +194,7 @@ impl crate::Loggable for Label { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/components/line_strip2d.rs b/crates/re_types/src/components/line_strip2d.rs index cd08d0531778..962b4101b6c3 100644 --- a/crates/re_types/src/components/line_strip2d.rs +++ b/crates/re_types/src/components/line_strip2d.rs @@ -201,7 +201,7 @@ impl crate::Loggable for LineStrip2D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -349,7 +349,7 @@ impl crate::Loggable for LineStrip2D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/components/line_strip3d.rs b/crates/re_types/src/components/line_strip3d.rs index b8a7e09b84fe..e3f0687adc9b 100644 --- a/crates/re_types/src/components/line_strip3d.rs +++ b/crates/re_types/src/components/line_strip3d.rs @@ -201,7 +201,7 @@ impl crate::Loggable for LineStrip3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -349,7 +349,7 @@ impl crate::Loggable for LineStrip3D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/components/origin3d.rs b/crates/re_types/src/components/origin3d.rs index 9c3606cb6a1b..be6a653b64f2 100644 --- a/crates/re_types/src/components/origin3d.rs +++ b/crates/re_types/src/components/origin3d.rs @@ -38,7 +38,7 @@ impl<'a> From<&'a Origin3D> for ::std::borrow::Cow<'a, Origin3D> { impl crate::Loggable for Origin3D { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -155,7 +155,7 @@ impl crate::Loggable for Origin3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -231,6 +231,61 @@ impl crate::Loggable for Origin3D { .with_context("rerun.components.Origin3D")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::FixedSizeListArray>() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 3usize, + ), + data.data_type().clone(), + ) + }) + .with_context("rerun.components.Origin3D#origin")?; + let data_inner = &**data.values(); + bytemuck::cast_slice::<_, [_; 3usize]>( + data_inner + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::Float32, + data_inner.data_type().clone(), + ) + }) + .with_context("rerun.components.Origin3D#origin")? + .values() + .as_slice(), + ) + .iter() + .copied() + .map(|v| crate::datatypes::Vec3D(v)) + } + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -238,13 +293,18 @@ impl crate::Loggable for Origin3D { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for Origin3D {} diff --git a/crates/re_types/src/components/point2d.rs b/crates/re_types/src/components/point2d.rs index abec0a42a6d1..85198775ead3 100644 --- a/crates/re_types/src/components/point2d.rs +++ b/crates/re_types/src/components/point2d.rs @@ -38,7 +38,7 @@ impl<'a> From<&'a Point2D> for ::std::borrow::Cow<'a, Point2D> { impl crate::Loggable for Point2D { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -155,7 +155,7 @@ impl crate::Loggable for Point2D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -231,6 +231,61 @@ impl crate::Loggable for Point2D { .with_context("rerun.components.Point2D")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::FixedSizeListArray>() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 2usize, + ), + data.data_type().clone(), + ) + }) + .with_context("rerun.components.Point2D#xy")?; + let data_inner = &**data.values(); + bytemuck::cast_slice::<_, [_; 2usize]>( + data_inner + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::Float32, + data_inner.data_type().clone(), + ) + }) + .with_context("rerun.components.Point2D#xy")? + .values() + .as_slice(), + ) + .iter() + .copied() + .map(|v| crate::datatypes::Vec2D(v)) + } + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -238,13 +293,18 @@ impl crate::Loggable for Point2D { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for Point2D {} diff --git a/crates/re_types/src/components/point3d.rs b/crates/re_types/src/components/point3d.rs index 62fa80d6af1d..21e25419d1d5 100644 --- a/crates/re_types/src/components/point3d.rs +++ b/crates/re_types/src/components/point3d.rs @@ -38,7 +38,7 @@ impl<'a> From<&'a Point3D> for ::std::borrow::Cow<'a, Point3D> { impl crate::Loggable for Point3D { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -155,7 +155,7 @@ impl crate::Loggable for Point3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -231,6 +231,61 @@ impl crate::Loggable for Point3D { .with_context("rerun.components.Point3D")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::FixedSizeListArray>() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 3usize, + ), + data.data_type().clone(), + ) + }) + .with_context("rerun.components.Point3D#xyz")?; + let data_inner = &**data.values(); + bytemuck::cast_slice::<_, [_; 3usize]>( + data_inner + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::Float32, + data_inner.data_type().clone(), + ) + }) + .with_context("rerun.components.Point3D#xyz")? + .values() + .as_slice(), + ) + .iter() + .copied() + .map(|v| crate::datatypes::Vec3D(v)) + } + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -238,13 +293,18 @@ impl crate::Loggable for Point3D { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for Point3D {} diff --git a/crates/re_types/src/components/radius.rs b/crates/re_types/src/components/radius.rs index 1ff615064c96..1cda6203f6e5 100644 --- a/crates/re_types/src/components/radius.rs +++ b/crates/re_types/src/components/radius.rs @@ -32,7 +32,7 @@ impl<'a> From<&'a Radius> for ::std::borrow::Cow<'a, Radius> { impl crate::Loggable for Radius { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -99,7 +99,7 @@ impl crate::Loggable for Radius { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -119,6 +119,37 @@ impl crate::Loggable for Radius { .with_context("rerun.components.Radius")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok(data + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::Float32, + data.data_type().clone(), + ) + }) + .with_context("rerun.components.Radius#value")? + .values() + .as_slice() + .iter() + .copied() + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -126,13 +157,18 @@ impl crate::Loggable for Radius { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for Radius {} diff --git a/crates/re_types/src/components/transform3d.rs b/crates/re_types/src/components/transform3d.rs index 5d3f78204f05..206fb35e5b94 100644 --- a/crates/re_types/src/components/transform3d.rs +++ b/crates/re_types/src/components/transform3d.rs @@ -124,7 +124,7 @@ impl crate::Loggable for Transform3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(crate::datatypes::Transform3D::try_from_arrow_opt(data) .with_context("rerun.components.Transform3D#repr")? .into_iter() @@ -146,7 +146,7 @@ impl crate::Loggable for Transform3D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/components/vector3d.rs b/crates/re_types/src/components/vector3d.rs index 94ee5f28b5ca..a2a4f42804e0 100644 --- a/crates/re_types/src/components/vector3d.rs +++ b/crates/re_types/src/components/vector3d.rs @@ -38,7 +38,7 @@ impl<'a> From<&'a Vector3D> for ::std::borrow::Cow<'a, Vector3D> { impl crate::Loggable for Vector3D { type Name = crate::ComponentName; - type Item<'a> = Option; + type Item<'a> = Self; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -155,7 +155,7 @@ impl crate::Loggable for Vector3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -231,6 +231,61 @@ impl crate::Loggable for Vector3D { .with_context("rerun.components.Vector3D")?) } + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized, + { + use crate::{Loggable as _, ResultExt as _}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + Ok({ + let data = data + .as_any() + .downcast_ref::<::arrow2::array::FixedSizeListArray>() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::FixedSizeList( + Box::new(Field { + name: "item".to_owned(), + data_type: DataType::Float32, + is_nullable: false, + metadata: [].into(), + }), + 3usize, + ), + data.data_type().clone(), + ) + }) + .with_context("rerun.components.Vector3D#vector")?; + let data_inner = &**data.values(); + bytemuck::cast_slice::<_, [_; 3usize]>( + data_inner + .as_any() + .downcast_ref::() + .ok_or_else(|| { + crate::DeserializationError::datatype_mismatch( + DataType::Float32, + data_inner.data_type().clone(), + ) + }) + .with_context("rerun.components.Vector3D#vector")? + .values() + .as_slice(), + ) + .iter() + .copied() + .map(|v| crate::datatypes::Vec3D(v)) + } + .map(|v| Self(v)) + .collect::>()) + } + #[inline] fn try_iter_from_arrow( data: &dyn ::arrow2::array::Array, @@ -238,13 +293,18 @@ impl crate::Loggable for Vector3D { where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self::try_from_arrow(data)?.into_iter()) } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_self(item: Self::Item<'_>) -> Self { item } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } } impl crate::Component for Vector3D {} diff --git a/crates/re_types/src/datatypes/angle.rs b/crates/re_types/src/datatypes/angle.rs index 4189a7d7c4d2..462cd608d221 100644 --- a/crates/re_types/src/datatypes/angle.rs +++ b/crates/re_types/src/datatypes/angle.rs @@ -203,7 +203,7 @@ impl crate::Loggable for Angle { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -407,7 +407,7 @@ impl crate::Loggable for Angle { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/annotation_info.rs b/crates/re_types/src/datatypes/annotation_info.rs index 0b060501891f..d91fce59006c 100644 --- a/crates/re_types/src/datatypes/annotation_info.rs +++ b/crates/re_types/src/datatypes/annotation_info.rs @@ -246,7 +246,7 @@ impl crate::Loggable for AnnotationInfo { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -472,7 +472,7 @@ impl crate::Loggable for AnnotationInfo { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/class_description.rs b/crates/re_types/src/datatypes/class_description.rs index 76d63bfca916..e0b67e95846c 100644 --- a/crates/re_types/src/datatypes/class_description.rs +++ b/crates/re_types/src/datatypes/class_description.rs @@ -297,7 +297,7 @@ impl crate::Loggable for ClassDescription { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -647,7 +647,7 @@ impl crate::Loggable for ClassDescription { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/class_description_map_elem.rs b/crates/re_types/src/datatypes/class_description_map_elem.rs index 5cf522542ef9..86695a9ffa8d 100644 --- a/crates/re_types/src/datatypes/class_description_map_elem.rs +++ b/crates/re_types/src/datatypes/class_description_map_elem.rs @@ -176,7 +176,7 @@ impl crate::Loggable for ClassDescriptionMapElem { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -310,7 +310,7 @@ impl crate::Loggable for ClassDescriptionMapElem { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/class_id.rs b/crates/re_types/src/datatypes/class_id.rs index a42aa6ab52eb..39c7ce2c7720 100644 --- a/crates/re_types/src/datatypes/class_id.rs +++ b/crates/re_types/src/datatypes/class_id.rs @@ -103,7 +103,7 @@ impl crate::Loggable for ClassId { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -134,7 +134,7 @@ impl crate::Loggable for ClassId { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/color.rs b/crates/re_types/src/datatypes/color.rs index 5d912ee0112a..010abd221ecf 100644 --- a/crates/re_types/src/datatypes/color.rs +++ b/crates/re_types/src/datatypes/color.rs @@ -111,7 +111,7 @@ impl crate::Loggable for Color { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -142,7 +142,7 @@ impl crate::Loggable for Color { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/keypoint_id.rs b/crates/re_types/src/datatypes/keypoint_id.rs index 6fdbae7471df..a2da0b77bc9e 100644 --- a/crates/re_types/src/datatypes/keypoint_id.rs +++ b/crates/re_types/src/datatypes/keypoint_id.rs @@ -105,7 +105,7 @@ impl crate::Loggable for KeypointId { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -136,7 +136,7 @@ impl crate::Loggable for KeypointId { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/keypoint_pair.rs b/crates/re_types/src/datatypes/keypoint_pair.rs index 0b53998bdebe..376390077ad3 100644 --- a/crates/re_types/src/datatypes/keypoint_pair.rs +++ b/crates/re_types/src/datatypes/keypoint_pair.rs @@ -183,7 +183,7 @@ impl crate::Loggable for KeypointPair { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -321,7 +321,7 @@ impl crate::Loggable for KeypointPair { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/label.rs b/crates/re_types/src/datatypes/label.rs index a5396450bd71..ede2b6b8ef72 100644 --- a/crates/re_types/src/datatypes/label.rs +++ b/crates/re_types/src/datatypes/label.rs @@ -116,7 +116,7 @@ impl crate::Loggable for Label { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -176,7 +176,7 @@ impl crate::Loggable for Label { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/mat3x3.rs b/crates/re_types/src/datatypes/mat3x3.rs index 681743fca96d..d88bc263d044 100644 --- a/crates/re_types/src/datatypes/mat3x3.rs +++ b/crates/re_types/src/datatypes/mat3x3.rs @@ -143,7 +143,7 @@ impl crate::Loggable for Mat3x3 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -227,7 +227,7 @@ impl crate::Loggable for Mat3x3 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/mat4x4.rs b/crates/re_types/src/datatypes/mat4x4.rs index b978eb7da00a..4fb9a70292c7 100644 --- a/crates/re_types/src/datatypes/mat4x4.rs +++ b/crates/re_types/src/datatypes/mat4x4.rs @@ -143,7 +143,7 @@ impl crate::Loggable for Mat4x4 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -227,7 +227,7 @@ impl crate::Loggable for Mat4x4 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/quaternion.rs b/crates/re_types/src/datatypes/quaternion.rs index f1b4d0966206..8ca557a265c5 100644 --- a/crates/re_types/src/datatypes/quaternion.rs +++ b/crates/re_types/src/datatypes/quaternion.rs @@ -143,7 +143,7 @@ impl crate::Loggable for Quaternion { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -227,7 +227,7 @@ impl crate::Loggable for Quaternion { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/rotation3d.rs b/crates/re_types/src/datatypes/rotation3d.rs index 3a0a8cd71955..b0e3da72ba80 100644 --- a/crates/re_types/src/datatypes/rotation3d.rs +++ b/crates/re_types/src/datatypes/rotation3d.rs @@ -247,7 +247,7 @@ impl crate::Loggable for Rotation3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -501,7 +501,7 @@ impl crate::Loggable for Rotation3D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/rotation_axis_angle.rs b/crates/re_types/src/datatypes/rotation_axis_angle.rs index 8b75fe314b06..219ba0fb9bbd 100644 --- a/crates/re_types/src/datatypes/rotation_axis_angle.rs +++ b/crates/re_types/src/datatypes/rotation_axis_angle.rs @@ -204,7 +204,7 @@ impl crate::Loggable for RotationAxisAngle { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -395,7 +395,7 @@ impl crate::Loggable for RotationAxisAngle { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/scale3d.rs b/crates/re_types/src/datatypes/scale3d.rs index 6e8b3b86cac4..67f79de3553b 100644 --- a/crates/re_types/src/datatypes/scale3d.rs +++ b/crates/re_types/src/datatypes/scale3d.rs @@ -244,7 +244,7 @@ impl crate::Loggable for Scale3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -514,7 +514,7 @@ impl crate::Loggable for Scale3D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/transform3d.rs b/crates/re_types/src/datatypes/transform3d.rs index db7e38cfa9e5..839f5d4113f9 100644 --- a/crates/re_types/src/datatypes/transform3d.rs +++ b/crates/re_types/src/datatypes/transform3d.rs @@ -216,7 +216,7 @@ impl crate::Loggable for Transform3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -406,7 +406,7 @@ impl crate::Loggable for Transform3D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/translation_and_mat3x3.rs b/crates/re_types/src/datatypes/translation_and_mat3x3.rs index 26334a9480fe..788b2c072163 100644 --- a/crates/re_types/src/datatypes/translation_and_mat3x3.rs +++ b/crates/re_types/src/datatypes/translation_and_mat3x3.rs @@ -288,7 +288,7 @@ impl crate::Loggable for TranslationAndMat3x3 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -608,7 +608,7 @@ impl crate::Loggable for TranslationAndMat3x3 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/translation_rotation_scale3d.rs b/crates/re_types/src/datatypes/translation_rotation_scale3d.rs index 76560065b72a..a201bab9bec5 100644 --- a/crates/re_types/src/datatypes/translation_rotation_scale3d.rs +++ b/crates/re_types/src/datatypes/translation_rotation_scale3d.rs @@ -281,7 +281,7 @@ impl crate::Loggable for TranslationRotationScale3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -596,7 +596,7 @@ impl crate::Loggable for TranslationRotationScale3D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/vec2d.rs b/crates/re_types/src/datatypes/vec2d.rs index fc8e11514d2e..38cceb62e7d4 100644 --- a/crates/re_types/src/datatypes/vec2d.rs +++ b/crates/re_types/src/datatypes/vec2d.rs @@ -143,7 +143,7 @@ impl crate::Loggable for Vec2D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -227,7 +227,7 @@ impl crate::Loggable for Vec2D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/vec3d.rs b/crates/re_types/src/datatypes/vec3d.rs index d4f1f279acad..f6793dd38689 100644 --- a/crates/re_types/src/datatypes/vec3d.rs +++ b/crates/re_types/src/datatypes/vec3d.rs @@ -143,7 +143,7 @@ impl crate::Loggable for Vec3D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -227,7 +227,7 @@ impl crate::Loggable for Vec3D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/datatypes/vec4d.rs b/crates/re_types/src/datatypes/vec4d.rs index 0f785e83b607..dd7ecdc4e090 100644 --- a/crates/re_types/src/datatypes/vec4d.rs +++ b/crates/re_types/src/datatypes/vec4d.rs @@ -143,7 +143,7 @@ impl crate::Loggable for Vec4D { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -227,7 +227,7 @@ impl crate::Loggable for Vec4D { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/lib.rs b/crates/re_types/src/lib.rs index 86889c8f4f46..4fd2d93286da 100644 --- a/crates/re_types/src/lib.rs +++ b/crates/re_types/src/lib.rs @@ -181,16 +181,7 @@ pub trait Loggable: Sized { /// For the non-fallible version, see [`Loggable::try_from_arrow`]. #[inline] fn from_arrow(data: &dyn ::arrow2::array::Array) -> Vec { - Self::try_iter_from_arrow(data) - .detailed_unwrap() - .map(Self::convert_item_to_self) - .map(|v| { - v.ok_or_else(|| DeserializationError::MissingData { - backtrace: ::backtrace::Backtrace::new_unresolved(), - }) - .detailed_unwrap() - }) - .collect() + Self::try_from_arrow(data).detailed_unwrap() } /// Given an Arrow array, deserializes it into a collection of [`Loggable`]s. @@ -200,14 +191,9 @@ pub trait Loggable: Sized { /// For the non-fallible version, see [`Loggable::from_arrow_opt`]. #[inline] fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> DeserializationResult> { - Self::try_iter_from_arrow(data)? + Ok(Self::try_iter_from_arrow(data)? .map(Self::convert_item_to_self) - .map(|v| { - v.ok_or_else(|| DeserializationError::MissingData { - backtrace: ::backtrace::Backtrace::new_unresolved(), - }) - }) - .collect() + .collect()) } /// Given an Arrow array, deserializes it into a collection of optional [`Loggable`]s. @@ -230,7 +216,7 @@ pub trait Loggable: Sized { data: &dyn ::arrow2::array::Array, ) -> DeserializationResult>> { Ok(Self::try_iter_from_arrow(data)? - .map(Self::convert_item_to_self) + .map(Self::convert_item_to_opt_self) .collect()) } @@ -251,10 +237,20 @@ pub trait Loggable: Sized { data: &dyn ::arrow2::array::Array, ) -> DeserializationResult>; + /// Convert a [`Loggable::Item`] into a [`Loggable`] + /// + /// This is intended to be used with [`Loggable::try_iter_from_arrow`] when the type + /// is known to be non-nullible. + #[inline] + fn convert_item_to_self(item: Self::Item<'_>) -> Self { + // TODO(jleibs): This unwrap goes away when we remove the iterator abstraction + Self::convert_item_to_opt_self(item).unwrap() + } + /// Convert a [`Loggable::Item`] into an optional [`Loggable`] /// /// This is intended to be used with [`Loggable::try_iter_from_arrow`] - fn convert_item_to_self(item: Self::Item<'_>) -> Option; + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option; } /// The fully-qualified name of a [`Datatype`], e.g. `rerun.datatypes.Vec2D`. @@ -604,7 +600,9 @@ mod size_bytes; pub use component_name::ComponentName; pub use size_bytes::SizeBytes; +mod arrow_buffer; mod arrow_string; +pub use arrow_buffer::ArrowBuffer; pub use arrow_string::ArrowString; #[cfg(feature = "testing")] diff --git a/crates/re_types/src/testing/components/fuzzy.rs b/crates/re_types/src/testing/components/fuzzy.rs index 8c8d4f36c010..589189dec9bf 100644 --- a/crates/re_types/src/testing/components/fuzzy.rs +++ b/crates/re_types/src/testing/components/fuzzy.rs @@ -167,7 +167,7 @@ impl crate::Loggable for AffixFuzzer1 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer1::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer1#single_required")? @@ -191,7 +191,7 @@ impl crate::Loggable for AffixFuzzer1 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -353,7 +353,7 @@ impl crate::Loggable for AffixFuzzer2 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer1::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer2#single_required")? @@ -377,7 +377,7 @@ impl crate::Loggable for AffixFuzzer2 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -539,7 +539,7 @@ impl crate::Loggable for AffixFuzzer3 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer1::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer3#single_required")? @@ -563,7 +563,7 @@ impl crate::Loggable for AffixFuzzer3 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -727,7 +727,7 @@ impl crate::Loggable for AffixFuzzer4 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer1::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer4#single_optional")? @@ -751,7 +751,7 @@ impl crate::Loggable for AffixFuzzer4 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -915,7 +915,7 @@ impl crate::Loggable for AffixFuzzer5 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer1::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer5#single_optional")? @@ -939,7 +939,7 @@ impl crate::Loggable for AffixFuzzer5 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -1103,7 +1103,7 @@ impl crate::Loggable for AffixFuzzer6 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer1::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer6#single_optional")? @@ -1127,7 +1127,7 @@ impl crate::Loggable for AffixFuzzer6 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -1267,7 +1267,7 @@ impl crate::Loggable for AffixFuzzer7 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -1345,7 +1345,7 @@ impl crate::Loggable for AffixFuzzer7 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -1440,7 +1440,7 @@ impl crate::Loggable for AffixFuzzer8 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -1471,7 +1471,7 @@ impl crate::Loggable for AffixFuzzer8 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -1580,7 +1580,7 @@ impl crate::Loggable for AffixFuzzer9 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -1640,7 +1640,7 @@ impl crate::Loggable for AffixFuzzer9 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -1751,7 +1751,7 @@ impl crate::Loggable for AffixFuzzer10 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -1811,7 +1811,7 @@ impl crate::Loggable for AffixFuzzer10 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -1819,7 +1819,7 @@ impl crate::Loggable for AffixFuzzer10 { impl crate::Component for AffixFuzzer10 {} #[derive(Clone, Debug, Default, PartialEq)] -pub struct AffixFuzzer11(pub Option>); +pub struct AffixFuzzer11(pub Option>); impl<'a> From for ::std::borrow::Cow<'a, AffixFuzzer11> { #[inline] @@ -1887,21 +1887,22 @@ impl crate::Loggable for AffixFuzzer11 { }; { use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let data0_inner_data: Vec<_> = data0 + let data0_inner_data: Buffer<_> = data0 .iter() .flatten() - .flatten() - .cloned() - .map(Some) - .collect(); + .map(|b| b.0.as_slice()) + .collect::>() + .concat() + .into(); let data0_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = None; - let offsets = ::arrow2::offset::Offsets::::try_from_lengths( - data0 - .iter() - .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - ) - .unwrap() - .into(); + let offsets = + ::arrow2::offset::Offsets::::try_from_lengths(data0.iter().map(|opt| { + opt.as_ref() + .map(|datum| datum.num_instances()) + .unwrap_or_default() + })) + .unwrap() + .into(); ListArray::new( { _ = extension_wrapper; @@ -1930,10 +1931,7 @@ impl crate::Loggable for AffixFuzzer11 { .to_logical_type() .clone() }, - data0_inner_data - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), + data0_inner_data, data0_inner_bitmap, ) .boxed(), @@ -1952,7 +1950,7 @@ impl crate::Loggable for AffixFuzzer11 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -1986,9 +1984,7 @@ impl crate::Loggable for AffixFuzzer11 { .with_context( "rerun.testing.components.AffixFuzzer11#many_floats_optional", )? - .into_iter() - .map(|opt| opt.copied()) - .collect::>() + .values() }; let offsets = data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( @@ -2007,13 +2003,12 @@ impl crate::Loggable for AffixFuzzer11 { } #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - let data = - unsafe { data_inner.get_unchecked(start as usize..end as usize) }; - let data = data - .iter() - .cloned() - .map(Option::unwrap_or_default) - .collect(); + let data = unsafe { + data_inner + .clone() + .sliced_unchecked(start as usize, end - start as usize) + }; + let data = crate::ArrowBuffer(data); Ok(data) }) .transpose() @@ -2040,7 +2035,7 @@ impl crate::Loggable for AffixFuzzer11 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -2195,7 +2190,7 @@ impl crate::Loggable for AffixFuzzer12 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -2315,7 +2310,7 @@ impl crate::Loggable for AffixFuzzer12 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -2472,7 +2467,7 @@ impl crate::Loggable for AffixFuzzer13 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -2592,7 +2587,7 @@ impl crate::Loggable for AffixFuzzer13 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -2732,7 +2727,7 @@ impl crate::Loggable for AffixFuzzer14 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer3::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer14#single_required_union")? @@ -2756,7 +2751,7 @@ impl crate::Loggable for AffixFuzzer14 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -2898,7 +2893,7 @@ impl crate::Loggable for AffixFuzzer15 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer3::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer15#single_optional_union")? @@ -2922,7 +2917,7 @@ impl crate::Loggable for AffixFuzzer15 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -3060,7 +3055,7 @@ impl crate::Loggable for AffixFuzzer16 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -3140,7 +3135,7 @@ impl crate::Loggable for AffixFuzzer16 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -3280,7 +3275,7 @@ impl crate::Loggable for AffixFuzzer17 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -3360,7 +3355,7 @@ impl crate::Loggable for AffixFuzzer17 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -3500,7 +3495,7 @@ impl crate::Loggable for AffixFuzzer18 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -3580,7 +3575,7 @@ impl crate::Loggable for AffixFuzzer18 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -3677,7 +3672,7 @@ impl crate::Loggable for AffixFuzzer19 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer5::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer19#just_a_table_nothing_shady")? @@ -3701,7 +3696,7 @@ impl crate::Loggable for AffixFuzzer19 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -3806,7 +3801,7 @@ impl crate::Loggable for AffixFuzzer20 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok( crate::testing::datatypes::AffixFuzzer20::try_from_arrow_opt(data) .with_context("rerun.testing.components.AffixFuzzer20#nested_transparent")? @@ -3830,7 +3825,7 @@ impl crate::Loggable for AffixFuzzer20 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/testing/components/fuzzy_deps.rs b/crates/re_types/src/testing/components/fuzzy_deps.rs index 589edb50c1ba..bbe6aeded06d 100644 --- a/crates/re_types/src/testing/components/fuzzy_deps.rs +++ b/crates/re_types/src/testing/components/fuzzy_deps.rs @@ -130,7 +130,7 @@ impl crate::Loggable for PrimitiveComponent { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -300,7 +300,7 @@ impl crate::Loggable for StringComponent { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/testing/datatypes/fuzzy.rs b/crates/re_types/src/testing/datatypes/fuzzy.rs index 0a8a2f1b5062..f9f5fb6d378e 100644 --- a/crates/re_types/src/testing/datatypes/fuzzy.rs +++ b/crates/re_types/src/testing/datatypes/fuzzy.rs @@ -126,7 +126,7 @@ impl crate::Loggable for FlattenedScalar { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -209,7 +209,7 @@ impl crate::Loggable for FlattenedScalar { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -221,7 +221,7 @@ pub struct AffixFuzzer1 { pub single_float_optional: Option, pub single_string_required: crate::ArrowString, pub single_string_optional: Option, - pub many_floats_optional: Option>, + pub many_floats_optional: Option>, pub many_strings_required: Vec, pub many_strings_optional: Option>, pub flattened_scalar: f32, @@ -517,19 +517,21 @@ impl crate::Loggable for AffixFuzzer1 { }; { use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let many_floats_optional_inner_data: Vec<_> = many_floats_optional + let many_floats_optional_inner_data: Buffer<_> = many_floats_optional .iter() .flatten() - .flatten() - .cloned() - .map(Some) - .collect(); + .map(|b| b.0.as_slice()) + .collect::>() + .concat() + .into(); let many_floats_optional_inner_bitmap: Option< ::arrow2::bitmap::Bitmap, > = None; let offsets = ::arrow2::offset::Offsets::::try_from_lengths( many_floats_optional.iter().map(|opt| { - opt.as_ref().map(|datum| datum.len()).unwrap_or_default() + opt.as_ref() + .map(|datum| datum.num_instances()) + .unwrap_or_default() }), ) .unwrap() @@ -552,10 +554,7 @@ impl crate::Loggable for AffixFuzzer1 { _ = extension_wrapper; DataType::Float32.to_logical_type().clone() }, - many_floats_optional_inner_data - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), + many_floats_optional_inner_data, many_floats_optional_inner_bitmap, ) .boxed(), @@ -842,7 +841,7 @@ impl crate::Loggable for AffixFuzzer1 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data.as_any() @@ -1201,9 +1200,7 @@ impl crate::Loggable for AffixFuzzer1 { .with_context( "rerun.testing.datatypes.AffixFuzzer1#many_floats_optional", )? - .into_iter() - .map(|opt| opt.copied()) - .collect::>() + .values() }; let offsets = data.offsets(); arrow2::bitmap::utils::ZipValidity::new_with_validity( @@ -1223,13 +1220,11 @@ impl crate::Loggable for AffixFuzzer1 { #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] let data = unsafe { - data_inner.get_unchecked(start as usize..end as usize) + data_inner + .clone() + .sliced_unchecked(start as usize, end - start as usize) }; - let data = data - .iter() - .cloned() - .map(Option::unwrap_or_default) - .collect(); + let data = crate::ArrowBuffer(data); Ok(data) }) .transpose() @@ -1774,7 +1769,7 @@ impl crate::Loggable for AffixFuzzer1 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -1869,7 +1864,7 @@ impl crate::Loggable for AffixFuzzer2 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -1900,7 +1895,7 @@ impl crate::Loggable for AffixFuzzer2 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -2265,7 +2260,7 @@ impl crate::Loggable for AffixFuzzer3 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -2685,7 +2680,7 @@ impl crate::Loggable for AffixFuzzer3 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -2996,7 +2991,7 @@ impl crate::Loggable for AffixFuzzer4 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -3359,7 +3354,7 @@ impl crate::Loggable for AffixFuzzer4 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -3493,7 +3488,7 @@ impl crate::Loggable for AffixFuzzer5 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -3568,7 +3563,7 @@ impl crate::Loggable for AffixFuzzer5 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -3764,7 +3759,7 @@ impl crate::Loggable for AffixFuzzer20 { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -3930,7 +3925,7 @@ impl crate::Loggable for AffixFuzzer20 { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/src/testing/datatypes/fuzzy_deps.rs b/crates/re_types/src/testing/datatypes/fuzzy_deps.rs index e69bcb71cb22..8b4beefec5e2 100644 --- a/crates/re_types/src/testing/datatypes/fuzzy_deps.rs +++ b/crates/re_types/src/testing/datatypes/fuzzy_deps.rs @@ -99,7 +99,7 @@ impl crate::Loggable for PrimitiveComponent { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok(data .as_any() .downcast_ref::() @@ -130,7 +130,7 @@ impl crate::Loggable for PrimitiveComponent { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } @@ -240,7 +240,7 @@ impl crate::Loggable for StringComponent { Self: Sized, { use crate::{Loggable as _, ResultExt as _}; - use ::arrow2::{array::*, datatypes::*}; + use ::arrow2::{array::*, buffer::*, datatypes::*}; Ok({ let data = data .as_any() @@ -300,7 +300,7 @@ impl crate::Loggable for StringComponent { } #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { item } } diff --git a/crates/re_types/tests/fuzzy.rs b/crates/re_types/tests/fuzzy.rs index a59bc27adec2..07a5c68fdeae 100644 --- a/crates/re_types/tests/fuzzy.rs +++ b/crates/re_types/tests/fuzzy.rs @@ -13,7 +13,7 @@ fn roundtrip() { single_float_optional: Some(1.0), single_string_required: "a".into(), single_string_optional: Some("a".into()), - many_floats_optional: Some(vec![1.0, 10.0, 100.0]), + many_floats_optional: Some(vec![1.0, 10.0, 100.0].into()), many_strings_required: vec!["1".into(), "2".into()], many_strings_optional: Some(vec!["10".into(), "20".into()]), flattened_scalar: 42.0, @@ -25,7 +25,7 @@ fn roundtrip() { single_float_optional: None, single_string_required: "b".into(), single_string_optional: None, - many_floats_optional: Some(vec![2.0, 20.0, 200.0]), + many_floats_optional: Some(vec![2.0, 20.0, 200.0].into()), many_strings_required: vec!["3".into(), "4".into()], many_strings_optional: None, flattened_scalar: 43.0, @@ -37,7 +37,7 @@ fn roundtrip() { single_float_optional: Some(3.0), single_string_required: "c".into(), single_string_optional: Some("c".into()), - many_floats_optional: Some(vec![3.0, 30.0, 300.0]), + many_floats_optional: Some(vec![3.0, 30.0, 300.0].into()), many_strings_required: vec!["5".into(), "6".into()], many_strings_optional: Some(vec!["50".into(), "60".into()]), flattened_scalar: 44.0, @@ -49,7 +49,7 @@ fn roundtrip() { single_float_optional: None, single_string_required: "d".into(), single_string_optional: None, - many_floats_optional: Some(vec![4.0, 40.0, 400.0]), + many_floats_optional: Some(vec![4.0, 40.0, 400.0].into()), many_strings_required: vec!["7".into(), "8".into()], many_strings_optional: None, flattened_scalar: 45.0, @@ -84,7 +84,7 @@ fn roundtrip() { single_float_optional: None, single_string_required: "d".into(), single_string_optional: None, - many_floats_optional: Some(vec![4.0, 40.0, 400.0]), + many_floats_optional: Some(vec![4.0, 40.0, 400.0].into()), many_strings_required: vec!["7".into(), "8".into()], many_strings_optional: None, flattened_scalar: 46.0, @@ -101,8 +101,8 @@ fn roundtrip() { let fuzzy10_1 = components::AffixFuzzer10(None); let fuzzy10_2 = components::AffixFuzzer10(Some("a".into())); - let fuzzy11_1 = components::AffixFuzzer11(Some(vec![1.0, 10.0])); - let fuzzy11_2 = components::AffixFuzzer11(Some(vec![2.0, 20.0, 200.0])); + let fuzzy11_1 = components::AffixFuzzer11(Some(vec![1.0, 10.0].into())); + let fuzzy11_2 = components::AffixFuzzer11(Some(vec![2.0, 20.0, 200.0].into())); let fuzzy12_1 = components::AffixFuzzer12(vec!["1".into(), "10".into()]); let fuzzy12_2 = components::AffixFuzzer12(vec!["20".into(), "200".into(), "2000".into()]); @@ -148,7 +148,7 @@ fn roundtrip() { single_float_optional: None, single_string_required: "d".into(), single_string_optional: None, - many_floats_optional: Some(vec![4.0, 40.0, 400.0]), + many_floats_optional: Some(vec![4.0, 40.0, 400.0].into()), many_strings_required: vec!["7".into(), "8".into()], many_strings_optional: None, flattened_scalar: 46.0, @@ -168,7 +168,7 @@ fn roundtrip() { single_float_optional: Some(3.0), single_string_required: "c".into(), single_string_optional: Some("c".into()), - many_floats_optional: Some(vec![3.0, 30.0, 300.0]), + many_floats_optional: Some(vec![3.0, 30.0, 300.0].into()), many_strings_required: vec!["5".into(), "6".into()], many_strings_optional: Some(vec!["50".into(), "60".into()]), flattened_scalar: 44.0, @@ -189,7 +189,7 @@ fn roundtrip() { single_float_optional: Some(3.0), single_string_required: "c".into(), single_string_optional: Some("c".into()), - many_floats_optional: Some(vec![3.0, 30.0, 300.0]), + many_floats_optional: Some(vec![3.0, 30.0, 300.0].into()), many_strings_required: vec!["5".into(), "6".into()], many_strings_optional: Some(vec!["50".into(), "60".into()]), flattened_scalar: 44.0, diff --git a/crates/re_types/tests/util.rs b/crates/re_types/tests/util.rs index 6c5a0a795572..42d24e0389d7 100644 --- a/crates/re_types/tests/util.rs +++ b/crates/re_types/tests/util.rs @@ -1,9 +1,11 @@ +#[allow(dead_code)] pub fn assert_extensions(array: &dyn ::arrow2::array::Array, expected: &[&str]) { let mut extracted = vec![]; extract_extensions(array.data_type(), &mut extracted); similar_asserts::assert_eq!(expected, extracted); } +#[allow(dead_code)] pub fn extract_extensions(datatype: &::arrow2::datatypes::DataType, acc: &mut Vec) { match datatype { arrow2::datatypes::DataType::List(field) diff --git a/crates/re_types/tests/validity.rs b/crates/re_types/tests/validity.rs new file mode 100644 index 000000000000..0110549a9eba --- /dev/null +++ b/crates/re_types/tests/validity.rs @@ -0,0 +1,38 @@ +use re_types::{ + components::{self, Point2D}, + DeserializationError, Loggable, +}; + +#[test] +fn validity_checks() { + let good_non_nullable = vec![ + components::Point2D::new(1.0, 2.0), // + components::Point2D::new(3.0, 4.0), // + ]; + + let serialized = Point2D::try_to_arrow(good_non_nullable, None).unwrap(); + let deserialized = Point2D::try_from_arrow(serialized.as_ref()); + assert!(deserialized.is_ok()); + + let good_nullable = vec![ + Some(components::Point2D::new(1.0, 2.0)), // + Some(components::Point2D::new(3.0, 4.0)), // + ]; + + let serialized = Point2D::try_to_arrow_opt(good_nullable, None).unwrap(); + let deserialized = Point2D::try_from_arrow(serialized.as_ref()); + assert!(deserialized.is_ok()); + + let bad = vec![ + Some(components::Point2D::new(1.0, 2.0)), // + None, + ]; + + let serialized = Point2D::try_to_arrow_opt(bad, None).unwrap(); + let deserialized = Point2D::try_from_arrow(serialized.as_ref()); + assert!(deserialized.is_err()); + assert!(matches!( + deserialized.err().unwrap(), + DeserializationError::MissingData { .. } + )); +} diff --git a/crates/re_types_builder/src/codegen/rust/api.rs b/crates/re_types_builder/src/codegen/rust/api.rs index 6a9a342f7bd7..784d540faea7 100644 --- a/crates/re_types_builder/src/codegen/rust/api.rs +++ b/crates/re_types_builder/src/codegen/rust/api.rs @@ -11,7 +11,10 @@ use crate::{ codegen::{ rust::{ arrow::ArrowDataTypeTokenizer, - deserializer::quote_arrow_deserializer, + deserializer::{ + quote_arrow_deserializer, quote_arrow_deserializer_buffer_slice, + should_optimize_buffer_slice_deserialize, + }, serializer::quote_arrow_serializer, util::{is_tuple_struct_from_obj, iter_archetype_components}, }, @@ -619,6 +622,8 @@ impl quote::ToTokens for TypeTokenizer<'_> { Type::Vector { elem_type } => { if *unwrap { quote!(#elem_type) + } else if elem_type.backed_by_arrow_buffer() { + quote!(crate::ArrowBuffer<#elem_type>) } else { quote!(Vec<#elem_type>) } @@ -697,14 +702,18 @@ fn quote_trait_impls_from_obj( match kind { ObjectKind::Datatype | ObjectKind::Component => { - let kind = if *kind == ObjectKind::Datatype { + let quoted_kind = if *kind == ObjectKind::Datatype { quote!(Datatype) } else { quote!(Component) }; - let kind_name = format_ident!("{kind}Name"); + let kind_name = format_ident!("{quoted_kind}Name"); let datatype = arrow_registry.get(fqname); + + let optimize_for_buffer_slice = + should_optimize_buffer_slice_deserialize(obj, arrow_registry); + let datatype = ArrowDataTypeTokenizer(&datatype, false); let legacy_fqname = obj @@ -735,12 +744,72 @@ fn quote_trait_impls_from_obj( } }; + let quoted_item = if optimize_for_buffer_slice { + quote!(Self) + } else { + quote!(Option) + }; + + let quoted_item_converters = if optimize_for_buffer_slice { + quote! { + #[inline] + fn convert_item_to_self(item: Self::Item<'_>) -> Self { + item + } + + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + Some(item) + } + } + } else { + quote! { + #[inline] + fn convert_item_to_opt_self(item: Self::Item<'_>) -> Option { + item + } + } + }; + + let quoted_iter_from_arrow_impl = if optimize_for_buffer_slice { + quote!(try_from_arrow) + } else { + quote!(try_from_arrow_opt) + }; + + let quoted_try_from_arrow = if optimize_for_buffer_slice { + let quoted_deserializer = + quote_arrow_deserializer_buffer_slice(arrow_registry, objects, obj); + quote! { + #[allow(unused_imports, clippy::wildcard_imports)] + #[inline] + fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> + where + Self: Sized { + use ::arrow2::{datatypes::*, array::*, buffer::*}; + use crate::{Loggable as _, ResultExt as _}; + + // This code-path cannot have null fields. If it does have a validity mask + // all bits must indicate valid data. + if let Some(validity) = data.validity() { + if validity.unset_bits() != 0 { + return Err(crate::DeserializationError::missing_data()); + } + } + + Ok(#quoted_deserializer) + } + } + } else { + quote!() + }; + quote! { #into_cow impl crate::Loggable for #name { type Name = crate::#kind_name; - type Item<'a> = Option; + type Item<'a> = #quoted_item; type Iter<'a> = > as IntoIterator>::IntoIter; #[inline] @@ -774,11 +843,12 @@ fn quote_trait_impls_from_obj( fn try_from_arrow_opt(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult>> where Self: Sized { - use ::arrow2::{datatypes::*, array::*}; + use ::arrow2::{datatypes::*, array::*, buffer::*}; use crate::{Loggable as _, ResultExt as _}; Ok(#quoted_deserializer) } + #quoted_try_from_arrow #[inline] fn try_iter_from_arrow( @@ -787,16 +857,14 @@ fn quote_trait_impls_from_obj( where Self: Sized, { - Ok(Self::try_from_arrow_opt(data)?.into_iter()) + Ok(Self:: #quoted_iter_from_arrow_impl (data)?.into_iter()) } - #[inline] - fn convert_item_to_self(item: Self::Item<'_>) -> Option { - item - } + + #quoted_item_converters } - impl crate::#kind for #name {} + impl crate::#quoted_kind for #name {} } } diff --git a/crates/re_types_builder/src/codegen/rust/arrow.rs b/crates/re_types_builder/src/codegen/rust/arrow.rs index 08e20ba19039..0570d6a7085a 100644 --- a/crates/re_types_builder/src/codegen/rust/arrow.rs +++ b/crates/re_types_builder/src/codegen/rust/arrow.rs @@ -133,3 +133,20 @@ pub fn quote_fqname_as_type_path(fqname: impl AsRef) -> TokenStream { let expr: syn::TypePath = syn::parse_str(&fqname).unwrap(); quote!(#expr) } + +pub fn is_backed_by_arrow_buffer(typ: &DataType) -> bool { + matches!( + typ, + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + ) +} diff --git a/crates/re_types_builder/src/codegen/rust/deserializer.rs b/crates/re_types_builder/src/codegen/rust/deserializer.rs index e2f1c7ab303f..35a01cd9f9e3 100644 --- a/crates/re_types_builder/src/codegen/rust/deserializer.rs +++ b/crates/re_types_builder/src/codegen/rust/deserializer.rs @@ -5,10 +5,10 @@ use quote::{format_ident, quote}; use crate::{ codegen::rust::{ - arrow::{quote_fqname_as_type_path, ArrowDataTypeTokenizer}, + arrow::{is_backed_by_arrow_buffer, quote_fqname_as_type_path, ArrowDataTypeTokenizer}, util::is_tuple_struct_from_obj, }, - ArrowRegistry, Object, Objects, + ArrowRegistry, Object, ObjectKind, Objects, }; // --- @@ -81,6 +81,7 @@ pub fn quote_arrow_deserializer( obj_field.is_nullable, obj_field_fqname, &data_src, + InnerRepr::NativeIterable, ); let quoted_unwrapping = if obj_field.is_nullable { @@ -124,6 +125,7 @@ pub fn quote_arrow_deserializer( obj_field.is_nullable, obj_field.fqname.as_str(), &data_src, + InnerRepr::NativeIterable, ); quote! { @@ -216,6 +218,7 @@ pub fn quote_arrow_deserializer( obj_field.is_nullable, obj_field.fqname.as_str(), &data_src, + InnerRepr::NativeIterable, ); let i = i + 1; // NOTE: +1 to account for `nulls` virtual arm @@ -348,6 +351,16 @@ pub fn quote_arrow_deserializer( } } +#[derive(Copy, Clone)] +enum InnerRepr { + /// The inner elements of the field should be exposed as `Buffer` + /// This is only applicable when T is an arrow primitive + BufferT, + + /// The inner elements of the field should be exposed as an iterable of T + NativeIterable, +} + /// This generates code that deserializes a runtime Arrow payload according to the specified `datatype`. /// /// The `datatype` comes from our compile-time Arrow registry, not from the runtime payload! @@ -365,6 +378,7 @@ fn quote_arrow_field_deserializer( is_nullable: bool, obj_field_fqname: &str, data_src: &proc_macro2::Ident, // &dyn ::arrow2::array::Array + inner_repr: InnerRepr, ) -> TokenStream { _ = is_nullable; // not yet used, will be needed very soon @@ -395,10 +409,16 @@ fn quote_arrow_field_deserializer( quote_array_downcast(obj_field_fqname, data_src, cast_as, datatype) }; - quote! { - #quoted_downcast? - .into_iter() // NOTE: automatically checks the bitmap on our behalf - #quoted_iter_transparency + match inner_repr { + InnerRepr::BufferT => quote! { + #quoted_downcast? + .values() + }, + InnerRepr::NativeIterable => quote! { + #quoted_downcast? + .into_iter() // NOTE: automatically checks the bitmap on our behalf + #quoted_iter_transparency + }, } } @@ -465,6 +485,7 @@ fn quote_arrow_field_deserializer( inner.is_nullable, obj_field_fqname, &data_src_inner, + InnerRepr::NativeIterable, ); let quoted_downcast = { @@ -562,12 +583,20 @@ fn quote_arrow_field_deserializer( DataType::List(inner) => { let data_src_inner = format_ident!("{data_src}_inner"); + + let inner_repr = if is_backed_by_arrow_buffer(inner.data_type()) { + InnerRepr::BufferT + } else { + InnerRepr::NativeIterable + }; + let quoted_inner = quote_arrow_field_deserializer( objects, inner.data_type(), inner.is_nullable, obj_field_fqname, &data_src_inner, + inner_repr, ); let quoted_downcast = { @@ -575,6 +604,53 @@ fn quote_arrow_field_deserializer( quote_array_downcast(obj_field_fqname, data_src, cast_as, datatype) }; + let quoted_collect_inner = match inner_repr { + InnerRepr::BufferT => quote!(), + InnerRepr::NativeIterable => quote!(.collect::>()), + }; + + let quoted_inner_data_range = match inner_repr { + InnerRepr::BufferT => quote! { + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + let data = unsafe { #data_src_inner.clone().sliced_unchecked(start as usize, end - start as usize) }; + let data = crate::ArrowBuffer(data); + }, + InnerRepr::NativeIterable => quote! { + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + let data = unsafe { #data_src_inner.get_unchecked(start as usize .. end as usize) }; + + // NOTE: The call to `Option::unwrap_or_default` is very important here. + // + // Since we can only get here if the outer oob is marked as + // non-null, the only possible reason for the default() path + // to be taken is because the inner field itself is nullable and + // happens to have one or more nullable values in the referenced + // slice. + // This is perfectly fine, and when it happens, we need to fill the + // resulting vec with some data, hence default(). + // + // This does have a subtle implication though! + // Since we never even look at the inner field's data when the outer + // entry is null, it means we won't notice it if illegal/malformed/corrupt + // in any way. + // It is important that we turn a blind eye here, because most SDKs in + // the ecosystem will put illegal data (e.g. null entries in an array of + // non-null floats) in the inner buffer if the outer entry itself + // is null. + // + // TODO(#2875): use MaybeUninit rather than requiring a default impl + let data = data.iter().cloned().map(Option::unwrap_or_default).collect(); + // The following would be the correct thing to do, but costs us way + // too much performance-wise for something that only applies to + // malformed inputs. + // + // // NOTE: We don't support nullable inner elements in our IDL, so + // // this can only be a case of malformed data. + // .map(|opt| opt.ok_or_else(crate::DeserializationError::missing_data)) + // .collect::>>()?; + }, + }; + quote! {{ let #data_src = #quoted_downcast?; if #data_src.is_empty() { @@ -586,7 +662,7 @@ fn quote_arrow_field_deserializer( } else { let #data_src_inner = { let #data_src_inner = &**#data_src.values(); - #quoted_inner.collect::>() + #quoted_inner #quoted_collect_inner }; let offsets = #data_src.offsets(); @@ -609,39 +685,8 @@ fn quote_arrow_field_deserializer( (start, end), #data_src_inner.len(), )); } - // Safety: all checked above. - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - let data = unsafe { #data_src_inner.get_unchecked(start as usize .. end as usize) }; - - // NOTE: The call to `Option::unwrap_or_default` is very important here. - // - // Since we can only get here if the outer oob is marked as - // non-null, the only possible reason for the default() path - // to be taken is because the inner field itself is nullable and - // happens to have one or more nullable values in the referenced - // slice. - // This is perfectly fine, and when it happens, we need to fill the - // resulting vec with some data, hence default(). - // - // This does have a subtle implication though! - // Since we never even look at the inner field's data when the outer - // entry is null, it means we won't notice it if illegal/malformed/corrupt - // in any way. - // It is important that we turn a blind eye here, because most SDKs in - // the ecosystem will put illegal data (e.g. null entries in an array of - // non-null floats) in the inner buffer if the outer entry itself - // is null. - // - // TODO(#2875): use MaybeUninit rather than requiring a default impl - let data = data.iter().cloned().map(Option::unwrap_or_default).collect(); - // The following would be the correct thing to do, but costs us way - // too much performance-wise for something that only applies to - // malformed inputs. - // - // // NOTE: We don't support nullable inner elements in our IDL, so - // // this can only be a case of malformed data. - // .map(|opt| opt.ok_or_else(crate::DeserializationError::missing_data)) - // .collect::>>()?; + + #quoted_inner_data_range Ok(data) }).transpose() @@ -780,3 +825,213 @@ fn quote_iterator_transparency( } } } + +/// This generates code that deserializes a runtime Arrow payload into the specified `obj`, taking +/// Arrow-transparency into account. +/// +/// It contains additional performance optimizations based on the inner-type being a non-nullable primitive +/// allowing us to map directly to slices rather than iterating. The ability to use this optimization is +/// determined by [`should_optimize_buffer_slice_deserialize`]. +/// +/// There is a 1:1 relationship between `quote_arrow_deserializer_buffer_slice` and `Loggable::try_from_arrow`: +/// ```ignore +/// fn try_from_arrow(data: &dyn ::arrow2::array::Array) -> crate::DeserializationResult> { +/// Ok(#quoted_deserializer_) +/// } +/// ``` +/// +/// See [`quote_arrow_deserializer_buffer_slice`] for additional information. +pub fn quote_arrow_deserializer_buffer_slice( + arrow_registry: &ArrowRegistry, + objects: &Objects, + obj: &Object, +) -> TokenStream { + // Runtime identifier of the variable holding the Arrow payload (`&dyn ::arrow2::array::Array`). + let data_src = format_ident!("data"); + + let datatype = &arrow_registry.get(&obj.fqname); + + let is_arrow_transparent = obj.datatype.is_none(); + let is_tuple_struct = is_tuple_struct_from_obj(obj); + + if is_arrow_transparent { + // NOTE: Arrow transparent objects must have a single field, no more no less. + // The semantic pass would have failed already if this wasn't the case. + debug_assert!(obj.fields.len() == 1); + let obj_field = &obj.fields[0]; + let obj_field_fqname = obj_field.fqname.as_str(); + + let data_dst = format_ident!( + "{}", + if is_tuple_struct { + "data0" + } else { + obj_field.name.as_str() + } + ); + + let quoted_deserializer = quote_arrow_field_deserializer_buffer_slice( + objects, + &arrow_registry.get(&obj_field.fqname), + obj_field.is_nullable, + obj_field_fqname, + &data_src, + InnerRepr::NativeIterable, + ); + + let quoted_remapping = if is_tuple_struct { + quote!(.map(|v| Self(v))) + } else { + quote!(.map(|#data_dst| Self { #data_dst })) + }; + + quote! { + #quoted_deserializer + #quoted_remapping + // NOTE: implicit Vec to Result + .collect::>() + } + } else { + unimplemented!("{datatype:#?}") + } +} + +/// This generates code that deserializes a runtime Arrow payload according to the specified `datatype`. +/// +/// It contains additional performance optimizations based on the inner-type being a non-nullable primitive +/// allowing us to map directly to slices rather than iterating. The ability to use this optimization is +/// determined by [`should_optimize_buffer_slice_deserialize`]. +/// +/// See [`quote_arrow_field_deserializer`] for additional information. +fn quote_arrow_field_deserializer_buffer_slice( + objects: &Objects, + datatype: &DataType, + is_nullable: bool, + obj_field_fqname: &str, + data_src: &proc_macro2::Ident, // &dyn ::arrow2::array::Array + inner_repr: InnerRepr, +) -> TokenStream { + _ = is_nullable; // not yet used, will be needed very soon + + match datatype.to_logical_type() { + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 => { + let quoted_iter_transparency = + quote_iterator_transparency(objects, datatype, IteratorKind::Value, None); + + let quoted_iter_transparency = quote!(.copied() #quoted_iter_transparency); + + let quoted_downcast = { + let cast_as = format!("{:?}", datatype.to_logical_type()).replace("DataType::", ""); + let cast_as = format_ident!("{cast_as}Array"); // e.g. `Uint32Array` + quote_array_downcast(obj_field_fqname, data_src, cast_as, datatype) + }; + + match inner_repr { + InnerRepr::BufferT => quote! { + #quoted_downcast? + .values() + .as_slice() + }, + InnerRepr::NativeIterable => quote! { + #quoted_downcast? + .values() + .as_slice() + .iter() + #quoted_iter_transparency + }, + } + } + + DataType::FixedSizeList(inner, length) => { + if matches!(inner_repr, InnerRepr::BufferT) { + // This implies a nested fixed-sized-array + unimplemented!("{datatype:#?}") + } + let data_src_inner = format_ident!("{data_src}_inner"); + let quoted_inner = quote_arrow_field_deserializer_buffer_slice( + objects, + inner.data_type(), + inner.is_nullable, + obj_field_fqname, + &data_src_inner, + InnerRepr::BufferT, + ); + + let quoted_downcast = { + let cast_as = quote!(::arrow2::array::FixedSizeListArray); + quote_array_downcast(obj_field_fqname, data_src, cast_as, datatype) + }; + + let quoted_iter_transparency = + quote_iterator_transparency(objects, datatype, IteratorKind::Value, None); + let quoted_iter_transparency = quote!(.copied() #quoted_iter_transparency); + + quote! {{ + let #data_src = #quoted_downcast?; + + let #data_src_inner = &**#data_src.values(); + bytemuck::cast_slice::<_, [_; #length]>(#quoted_inner) + .iter() + #quoted_iter_transparency + }} + } + + _ => unimplemented!("{datatype:#?}"), + } +} + +/// Whether or not this object allows for the buffer-slice optimizations. +/// +/// These optimizations require the outer type to be non-nullable and made up exclusively +/// of primitive types. +/// +/// Nullabillity is kind of weird since it's technically a property of the field +/// rather than the datatype. However, we know that Components can only be used +/// by archetypes and as such should never be nullible. +/// +/// This should always be checked before using [`quote_arrow_deserializer_buffer_slice`]. +pub fn should_optimize_buffer_slice_deserialize( + obj: &Object, + arrow_registry: &ArrowRegistry, +) -> bool { + let is_arrow_transparent = obj.datatype.is_none(); + if obj.kind == ObjectKind::Component && is_arrow_transparent { + let typ = arrow_registry.get(&obj.fqname); + let obj_field = &obj.fields[0]; + !obj_field.is_nullable && should_optimize_buffer_slice_deserialize_datatype(&typ) + } else { + false + } +} + +/// Whether or not this datatype allows for the buffer slice optimizations. +fn should_optimize_buffer_slice_deserialize_datatype(typ: &DataType) -> bool { + match typ { + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 => true, + DataType::Extension(_, typ, _) => should_optimize_buffer_slice_deserialize_datatype(typ), + DataType::FixedSizeList(field, _) => { + should_optimize_buffer_slice_deserialize_datatype(field.data_type()) + } + _ => false, + } +} diff --git a/crates/re_types_builder/src/codegen/rust/serializer.rs b/crates/re_types_builder/src/codegen/rust/serializer.rs index 809fb34425ed..fdfc4afd0b86 100644 --- a/crates/re_types_builder/src/codegen/rust/serializer.rs +++ b/crates/re_types_builder/src/codegen/rust/serializer.rs @@ -6,7 +6,7 @@ use quote::{format_ident, quote}; use crate::{ArrowRegistry, Object, Objects}; use super::{ - arrow::{quote_fqname_as_type_path, ArrowDataTypeTokenizer}, + arrow::{is_backed_by_arrow_buffer, quote_fqname_as_type_path, ArrowDataTypeTokenizer}, util::is_tuple_struct_from_obj, }; @@ -84,6 +84,7 @@ pub fn quote_arrow_serializer( obj_field.is_nullable, &bitmap_dst, "ed_data_dst, + InnerRepr::NativeIterable, ); let quoted_bitmap = quoted_bitmap(bitmap_dst); @@ -129,6 +130,7 @@ pub fn quote_arrow_serializer( obj_field.is_nullable, &bitmap_dst, &data_dst, + InnerRepr::NativeIterable, ); let quoted_flatten = quoted_flatten(obj_field.is_nullable); @@ -191,6 +193,7 @@ pub fn quote_arrow_serializer( obj_field.is_nullable, &bitmap_dst, &data_dst, + InnerRepr::NativeIterable ); let quoted_flatten = quoted_flatten(obj_field.is_nullable); @@ -310,6 +313,16 @@ pub fn quote_arrow_serializer( } } +#[derive(Copy, Clone)] +enum InnerRepr { + /// The inner elements of the field will come from an `ArrowBuffer` + /// This is only applicable when T is an arrow primitive + ArrowBuffer, + + /// The inner elements of the field will come from an iterable of T + NativeIterable, +} + fn quote_arrow_field_serializer( objects: &Objects, extension_wrapper: Option<&str>, @@ -317,6 +330,7 @@ fn quote_arrow_field_serializer( is_nullable: bool, bitmap_src: &proc_macro2::Ident, data_src: &proc_macro2::Ident, + inner_repr: InnerRepr, ) -> TokenStream { let quoted_datatype = ArrowDataTypeTokenizer(datatype, false); let quoted_datatype = if let Some(ext) = extension_wrapper { @@ -387,12 +401,23 @@ fn quote_arrow_field_serializer( } }; - quote! { - PrimitiveArray::new( - #quoted_datatype, - #data_src.into_iter() #quoted_transparent_mapping .collect(), - #bitmap_src, - ).boxed() + match inner_repr { + // A primitive that's an inner element of a list will already have been mapped + // to a buffer type. + InnerRepr::ArrowBuffer => quote! { + PrimitiveArray::new( + #quoted_datatype, + #data_src, + #bitmap_src, + ).boxed() + }, + InnerRepr::NativeIterable => quote! { + PrimitiveArray::new( + #quoted_datatype, + #data_src.into_iter() #quoted_transparent_mapping .collect(), + #bitmap_src, + ).boxed() + }, } } @@ -476,6 +501,20 @@ fn quote_arrow_field_serializer( DataType::List(inner) | DataType::FixedSizeList(inner, _) => { let inner_datatype = inner.data_type(); + // Note: We only use the ArrowBuffer optimization for `Lists` but not `FixedSizeList`. + // This is because the `ArrowBuffer` has a dynamic length, which would add more overhead + // to simple fixed-sized types like `Point2D`. + // + // TODO(jleibs): If we need to support large FixedSizeList types where the `ArrowBuffer` + // optimization would be significant, we can introduce a new attribute to force this. + let inner_repr = if is_backed_by_arrow_buffer(inner.data_type()) + && matches!(datatype, DataType::List(_)) + { + InnerRepr::ArrowBuffer + } else { + InnerRepr::NativeIterable + }; + let quoted_inner_data = format_ident!("{data_src}_inner_data"); let quoted_inner_bitmap = format_ident!("{data_src}_inner_bitmap"); @@ -486,6 +525,7 @@ fn quote_arrow_field_serializer( inner.is_nullable, "ed_inner_bitmap, "ed_inner_data, + inner_repr, ); let quoted_transparent_mapping = if inner_is_arrow_transparent { @@ -519,18 +559,32 @@ fn quote_arrow_field_serializer( .flatten() } } else { - quote! { - .flatten() - // NOTE: Flattening yet again since we have to deconstruct the inner list. - .flatten() - .cloned() + match inner_repr { + InnerRepr::ArrowBuffer => quote! { + .flatten() + .map(|b| b.0.as_slice()) + .collect::>() + .concat() + .into(); + }, + InnerRepr::NativeIterable => quote! { + .flatten() + // NOTE: Flattening yet again since we have to deconstruct the inner list. + .flatten() + .cloned() + }, } }; + let quoted_num_instances = match inner_repr { + InnerRepr::ArrowBuffer => quote!(num_instances()), + InnerRepr::NativeIterable => quote!(len()), + }; + let quoted_create = if let DataType::List(_) = datatype { quote! { let offsets = ::arrow2::offset::Offsets::::try_from_lengths( - #data_src.iter().map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()) + #data_src.iter().map(|opt| opt.as_ref().map(|datum| datum. #quoted_num_instances).unwrap_or_default()) ).unwrap().into(); ListArray::new( @@ -553,22 +607,36 @@ fn quote_arrow_field_serializer( // TODO(cmc): We should be checking this, but right now we don't because we don't // support intra-list nullability. _ = is_nullable; - quote! {{ - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - - let #quoted_inner_data: Vec<_> = #data_src - .iter() - #quoted_transparent_mapping - // NOTE: Wrapping back into an option as the recursive call will expect the - // guaranteed nullability layer to be present! - .map(Some) - .collect(); + match inner_repr { + InnerRepr::ArrowBuffer => quote! {{ + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + + let #quoted_inner_data: Buffer<_> = #data_src + .iter() + #quoted_transparent_mapping + + // TODO(cmc): We don't support intra-list nullability in our IDL at the moment. + let #quoted_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = None; + + #quoted_create + }}, + InnerRepr::NativeIterable => quote! {{ + use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; + + let #quoted_inner_data: Vec<_> = #data_src + .iter() + #quoted_transparent_mapping + // NOTE: Wrapping back into an option as the recursive call will expect the + // guaranteed nullability layer to be present! + .map(Some) + .collect(); - // TODO(cmc): We don't support intra-list nullability in our IDL at the moment. - let #quoted_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = None; + // TODO(cmc): We don't support intra-list nullability in our IDL at the moment. + let #quoted_inner_bitmap: Option<::arrow2::bitmap::Bitmap> = None; - #quoted_create - }} + #quoted_create + }}, + } } DataType::Struct(_) | DataType::Union(_, _, _) => { diff --git a/crates/re_types_builder/src/objects.rs b/crates/re_types_builder/src/objects.rs index db3feb5ebf43..1c16d39e2751 100644 --- a/crates/re_types_builder/src/objects.rs +++ b/crates/re_types_builder/src/objects.rs @@ -1086,6 +1086,26 @@ impl ElementType { Self::Object(fqname) => objects[fqname].has_default_destructor(objects), } } + + /// Is this type directly backed by a native arrow `Buffer`. This means the data can + /// be returned using a `re_types::ArrowBuffer` which facilitates direct zero-copy access to + /// a slice representation. + pub fn backed_by_arrow_buffer(&self) -> bool { + match self { + Self::UInt8 + | Self::UInt16 + | Self::UInt32 + | Self::UInt64 + | Self::Int8 + | Self::Int16 + | Self::Int32 + | Self::Int64 + | Self::Float16 + | Self::Float32 + | Self::Float64 => true, + Self::Bool | Self::Object(_) | Self::String => false, + } + } } // --- Common ---