Check DataType

arrow-data/src/data/boolean.rs

@@ -112,6 +112,7 @@ impl BooleanArrayData {
 
 impl From<ArrayData> for BooleanArrayData {
     fn from(data: ArrayData) -> Self {
+        assert_eq!(PhysicalType::from(&data.data_type), PhysicalType::Boolean);
         let values = data.buffers.into_iter().next().unwrap();
         let values = BooleanBuffer::new(values, data.offset, data.len);
         Self {

arrow-data/src/data/bytes.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::data::types::{BytesType, OffsetType};
+use crate::data::types::{BytesType, OffsetType, PhysicalType};
 use crate::{ArrayData, ArrayDataBuilder, Buffers};
 use arrow_buffer::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
 use arrow_buffer::{ArrowNativeType, Buffer};
@@ -178,6 +178,10 @@ impl<O: BytesOffset, B: Bytes + ?Sized> BytesArrayData<O, B> {
 
 impl<O: BytesOffset, B: Bytes + ?Sized> From<ArrayData> for BytesArrayData<O, B> {
     fn from(data: ArrayData) -> Self {
+        assert_eq!(
+            PhysicalType::from(&data.data_type),
+            PhysicalType::Bytes(O::TYPE, B::TYPE)
+        );
         let mut iter = data.buffers.into_iter();
         let offsets = iter.next().unwrap();
         let values = iter.next().unwrap();

arrow-data/src/data/dictionary.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::data::types::DictionaryKeyType;
+use crate::data::types::{DictionaryKeyType, PhysicalType};
 use crate::{ArrayData, ArrayDataBuilder, Buffers};
 use arrow_buffer::buffer::{NullBuffer, ScalarBuffer};
 use arrow_buffer::ArrowNativeType;
@@ -136,6 +136,7 @@ impl<K: DictionaryKey> DictionaryArrayData<K> {
 
 impl<K: DictionaryKey> From<ArrayData> for DictionaryArrayData<K> {
     fn from(data: ArrayData) -> Self {
+        assert_eq!(PhysicalType::from(&data.data_type), PhysicalType::Dictionary(K::TYPE));
         let keys = data.buffers.into_iter().next().unwrap();
         let keys = ScalarBuffer::new(keys, data.offset, data.len);
         let values = data.child_data.into_iter().next().unwrap();

arrow-data/src/data/list.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::data::types::OffsetType;
+use crate::data::types::{OffsetType, PhysicalType};
 use crate::{ArrayData, ArrayDataBuilder, Buffers};
 use arrow_buffer::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
 use arrow_buffer::ArrowNativeType;
@@ -129,6 +129,7 @@ impl<O: ListOffset> ListArrayData<O> {
 
 impl<O: ListOffset> From<ArrayData> for ListArrayData<O> {
     fn from(data: ArrayData) -> Self {
+        assert_eq!(PhysicalType::from(&data.data_type), PhysicalType::List(O::TYPE));
         let offsets = data.buffers.into_iter().next().unwrap();
         let values = data.child_data.into_iter().next().unwrap();
 

arrow-data/src/data/null.rs

@@ -79,10 +79,11 @@ impl NullArrayData {
 }
 
 impl From<ArrayData> for NullArrayData {
-    fn from(value: ArrayData) -> Self {
+    fn from(data: ArrayData) -> Self {
+        assert_eq!(PhysicalType::from(&data.data_type), PhysicalType::Null);
         Self {
-            data_type: value.data_type,
-            len: value.len + value.offset,
+            data_type: data.data_type,
+            len: data.len + data.offset,
         }
     }
 }

arrow-data/src/data/primitive.rs

@@ -144,6 +144,11 @@ impl<T: Primitive> PrimitiveArrayData<T> {
 
 impl<T: Primitive> From<ArrayData> for PrimitiveArrayData<T> {
     fn from(data: ArrayData) -> Self {
+        assert_eq!(
+            PhysicalType::from(&data.data_type),
+            PhysicalType::Primitive(T::TYPE)
+        );
+
         let values = data.buffers.into_iter().next().unwrap();
         let values = ScalarBuffer::new(values, data.offset, data.len);
         Self {

arrow-data/src/data/run.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use crate::data::primitive::{Primitive, PrimitiveArrayData};
-use crate::data::types::RunEndType;
+use crate::data::types::{PhysicalType, RunEndType};
 use crate::{ArrayData, ArrayDataBuilder, Buffers};
 use arrow_buffer::buffer::{RunEndBuffer, ScalarBuffer};
 use arrow_buffer::ArrowNativeType;
@@ -83,12 +83,6 @@ impl<E: RunEnd> RunArrayData<E> {
         self.run_ends.len()
     }
 
-    /// Returns the offset
-    #[inline]
-    pub fn offset(&self) -> usize {
-        self.run_ends.offset()
-    }
-
     /// Returns true if this array is empty
     #[inline]
     pub fn is_empty(&self) -> bool {
@@ -130,6 +124,10 @@ impl<E: RunEnd> RunArrayData<E> {
 
 impl<E: RunEnd> From<ArrayData> for RunArrayData<E> {
     fn from(data: ArrayData) -> Self {
+        assert_eq!(
+            PhysicalType::from(&data.data_type),
+            PhysicalType::Run(E::TYPE)
+        );
         let mut iter = data.child_data.into_iter();
         let child1 = iter.next().unwrap();
         let child2 = iter.next().unwrap();

arrow-data/src/data/struct.rs

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use crate::data::types::PhysicalType;
 use crate::{ArrayData, ArrayDataBuilder, Buffers};
 use arrow_buffer::buffer::NullBuffer;
 use arrow_schema::DataType;
@@ -96,18 +97,14 @@ impl StructArrayData {
 }
 
 impl From<ArrayData> for StructArrayData {
-    fn from(data: ArrayData) -> Self {
-        let children = data
-            .child_data
-            .into_iter()
-            .map(|x| x.slice(data.offset, data.len))
-            .collect();
-
+    fn from(mut data: ArrayData) -> Self {
+        assert_eq!(PhysicalType::from(&data.data_type), PhysicalType::Struct);
         Self {
             data_type: data.data_type,
             len: data.len,
             nulls: data.nulls,
-            children,
+            // Don't slice children as assume offset already applied (#1750)
+            children: data.child_data,
         }
     }
 }

arrow-data/src/data/types.rs

@@ -76,12 +76,12 @@ pub enum PhysicalType {
     Null,
     Boolean,
     Primitive(PrimitiveType),
-    FixedSizeBinary(usize),
+    FixedSizeBinary,
     Bytes(OffsetType, BytesType),
-    FixedSizeList(usize),
+    FixedSizeList,
     List(OffsetType),
     Struct,
-    Union(UnionMode),
+    Union,
     Dictionary(DictionaryKeyType),
     Run(RunEndType),
 }
@@ -119,16 +119,16 @@ impl From<&DataType> for PhysicalType {
             DataType::Interval(IntervalUnit::MonthDayNano) => {
                 Self::Primitive(PrimitiveType::Int128)
             }
-            DataType::FixedSizeBinary(size) => Self::FixedSizeBinary(*size as usize),
+            DataType::FixedSizeBinary(_) => Self::FixedSizeBinary,
             DataType::Binary => Self::Bytes(OffsetType::Int32, BytesType::Binary),
             DataType::LargeBinary => Self::Bytes(OffsetType::Int64, BytesType::Binary),
             DataType::Utf8 => Self::Bytes(OffsetType::Int32, BytesType::Utf8),
             DataType::LargeUtf8 => Self::Bytes(OffsetType::Int64, BytesType::Utf8),
             DataType::List(_) => Self::List(OffsetType::Int32),
-            DataType::FixedSizeList(_, size) => Self::FixedSizeList(*size as usize),
+            DataType::FixedSizeList(_, _) => Self::FixedSizeList,
             DataType::LargeList(_) => Self::List(OffsetType::Int64),
             DataType::Struct(_) => Self::Struct,
-            DataType::Union(_, _, mode) => Self::Union(*mode),
+            DataType::Union(_, _, _) => Self::Union,
             DataType::Dictionary(k, _) => match k.as_ref() {
                 DataType::Int8 => Self::Dictionary(DictionaryKeyType::Int8),
                 DataType::Int16 => Self::Dictionary(DictionaryKeyType::Int16),