diff --git a/arrow/src/array/array_decimal.rs b/arrow/src/array/array_decimal.rs index d051c4b9dd98..9cf802f74c2f 100644 --- a/arrow/src/array/array_decimal.rs +++ b/arrow/src/array/array_decimal.rs @@ -16,8 +16,6 @@ // under the License. use crate::array::ArrayAccessor; - -use std::borrow::Borrow; use std::convert::From; use std::fmt; use std::{any::Any, iter::FromIterator}; @@ -45,9 +43,9 @@ use crate::util::decimal::{BasicDecimal, Decimal256}; /// /// // Create a DecimalArray with the default precision and scale /// let decimal_array: Decimal128Array = vec![ -/// Some(8_887_000_000), +/// Some(8_887_000_000_i128), /// None, -/// Some(-8_887_000_000), +/// Some(-8_887_000_000_i128), /// ] /// .into_iter().collect(); /// @@ -475,8 +473,8 @@ impl> FromIterator> for Decimal256Array { } } -impl>> FromIterator for Decimal128Array { - fn from_iter>(iter: I) -> Self { +impl> FromIterator> for Decimal128Array { + fn from_iter>>(iter: I) -> Self { let iter = iter.into_iter(); let (lower, upper) = iter.size_hint(); let size_hint = upper.unwrap_or(lower); @@ -485,9 +483,9 @@ impl>> FromIterator for Decimal128Array { let buffer: Buffer = iter .map(|item| { - if let Some(a) = item.borrow() { + if let Some(a) = item { null_buf.append(true); - *a + a.into() } else { null_buf.append(false); // arbitrary value for NULL @@ -570,6 +568,7 @@ impl<'a, const BYTE_WIDTH: usize> BasicDecimalArray { mod tests { use crate::array::Decimal256Builder; use crate::datatypes::{DECIMAL256_MAX_PRECISION, DECIMAL_DEFAULT_SCALE}; + use crate::util::decimal::Decimal128; use crate::{array::Decimal128Builder, datatypes::Field}; use num::{BigInt, Num}; @@ -772,8 +771,8 @@ mod tests { #[test] fn test_decimal_array_fmt_debug() { - let arr = [Some(8887000000), Some(-8887000000), None] - .iter() + let arr = [Some(8887000000_i128), Some(-8887000000_i128), None] + .into_iter() .collect::() .with_precision_and_scale(23, 6) .unwrap(); @@ -960,4 +959,22 @@ mod tests { ); assert!(!array.is_null(2)); } + + #[test] + fn test_from_iter_decimal128array() { + let array: Decimal128Array = vec![ + Some(Decimal128::new_from_i128(38, 10, -100)), + None, + Some(Decimal128::new_from_i128(38, 10, 101)), + ] + .into_iter() + .collect(); + assert_eq!(array.len(), 3); + assert_eq!(array.data_type(), &DataType::Decimal128(38, 10)); + assert_eq!(-100_i128, array.value(0).into()); + assert!(!array.is_null(0)); + assert!(array.is_null(1)); + assert_eq!(101_i128, array.value(2).into()); + assert!(!array.is_null(2)); + } } diff --git a/arrow/src/array/equal/mod.rs b/arrow/src/array/equal/mod.rs index 6fdc06f837c0..975fbd2b3605 100644 --- a/arrow/src/array/equal/mod.rs +++ b/arrow/src/array/equal/mod.rs @@ -841,8 +841,8 @@ mod tests { test_equal(&a_slice, &b_slice, false); } - fn create_decimal_array(data: &[Option]) -> ArrayData { - data.iter() + fn create_decimal_array(data: Vec>) -> ArrayData { + data.into_iter() .collect::() .with_precision_and_scale(23, 6) .unwrap() @@ -851,32 +851,36 @@ mod tests { #[test] fn test_decimal_equal() { - let a = create_decimal_array(&[Some(8_887_000_000), Some(-8_887_000_000)]); - let b = create_decimal_array(&[Some(8_887_000_000), Some(-8_887_000_000)]); + let a = create_decimal_array(vec![Some(8_887_000_000), Some(-8_887_000_000)]); + let b = create_decimal_array(vec![Some(8_887_000_000), Some(-8_887_000_000)]); test_equal(&a, &b, true); - let b = create_decimal_array(&[Some(15_887_000_000), Some(-8_887_000_000)]); + let b = create_decimal_array(vec![Some(15_887_000_000), Some(-8_887_000_000)]); test_equal(&a, &b, false); } // Test the case where null_count > 0 #[test] fn test_decimal_null() { - let a = create_decimal_array(&[Some(8_887_000_000), None, Some(-8_887_000_000)]); - let b = create_decimal_array(&[Some(8_887_000_000), None, Some(-8_887_000_000)]); + let a = + create_decimal_array(vec![Some(8_887_000_000), None, Some(-8_887_000_000)]); + let b = + create_decimal_array(vec![Some(8_887_000_000), None, Some(-8_887_000_000)]); test_equal(&a, &b, true); - let b = create_decimal_array(&[Some(8_887_000_000), Some(-8_887_000_000), None]); + let b = + create_decimal_array(vec![Some(8_887_000_000), Some(-8_887_000_000), None]); test_equal(&a, &b, false); - let b = create_decimal_array(&[Some(15_887_000_000), None, Some(-8_887_000_000)]); + let b = + create_decimal_array(vec![Some(15_887_000_000), None, Some(-8_887_000_000)]); test_equal(&a, &b, false); } #[test] fn test_decimal_offsets() { // Test the case where offset != 0 - let a = create_decimal_array(&[ + let a = create_decimal_array(vec![ Some(8_887_000_000), None, None, @@ -884,7 +888,7 @@ mod tests { None, None, ]); - let b = create_decimal_array(&[ + let b = create_decimal_array(vec![ None, Some(8_887_000_000), None, @@ -914,7 +918,7 @@ mod tests { let b_slice = b.slice(2, 3); test_equal(&a_slice, &b_slice, false); - let b = create_decimal_array(&[ + let b = create_decimal_array(vec![ None, None, None, diff --git a/arrow/src/array/ord.rs b/arrow/src/array/ord.rs index 47173aa7d927..dd6539589c13 100644 --- a/arrow/src/array/ord.rs +++ b/arrow/src/array/ord.rs @@ -300,8 +300,8 @@ pub mod tests { #[test] fn test_decimal() -> Result<()> { - let array = vec![Some(5), Some(2), Some(3)] - .iter() + let array = vec![Some(5_i128), Some(2_i128), Some(3_i128)] + .into_iter() .collect::() .with_precision_and_scale(23, 6) .unwrap(); diff --git a/arrow/src/array/transform/mod.rs b/arrow/src/array/transform/mod.rs index f0fccef14fd7..dafbc05c67ae 100644 --- a/arrow/src/array/transform/mod.rs +++ b/arrow/src/array/transform/mod.rs @@ -687,12 +687,12 @@ mod tests { }; fn create_decimal_array( - array: &[Option], + array: Vec>, precision: usize, scale: usize, ) -> Decimal128Array { array - .iter() + .into_iter() .collect::() .with_precision_and_scale(precision, scale) .unwrap() @@ -702,28 +702,28 @@ mod tests { #[cfg(not(feature = "force_validate"))] fn test_decimal() { let decimal_array = - create_decimal_array(&[Some(1), Some(2), None, Some(3)], 10, 3); + create_decimal_array(vec![Some(1), Some(2), None, Some(3)], 10, 3); let arrays = vec![Array::data(&decimal_array)]; let mut a = MutableArrayData::new(arrays, true, 3); a.extend(0, 0, 3); a.extend(0, 2, 3); let result = a.freeze(); let array = Decimal128Array::from(result); - let expected = create_decimal_array(&[Some(1), Some(2), None, None], 10, 3); + let expected = create_decimal_array(vec![Some(1), Some(2), None, None], 10, 3); assert_eq!(array, expected); } #[test] #[cfg(not(feature = "force_validate"))] fn test_decimal_offset() { let decimal_array = - create_decimal_array(&[Some(1), Some(2), None, Some(3)], 10, 3); + create_decimal_array(vec![Some(1), Some(2), None, Some(3)], 10, 3); let decimal_array = decimal_array.slice(1, 3); // 2, null, 3 let arrays = vec![decimal_array.data()]; let mut a = MutableArrayData::new(arrays, true, 2); a.extend(0, 0, 2); // 2, null let result = a.freeze(); let array = Decimal128Array::from(result); - let expected = create_decimal_array(&[Some(2), None], 10, 3); + let expected = create_decimal_array(vec![Some(2), None], 10, 3); assert_eq!(array, expected); } @@ -731,7 +731,7 @@ mod tests { #[cfg(not(feature = "force_validate"))] fn test_decimal_null_offset_nulls() { let decimal_array = - create_decimal_array(&[Some(1), Some(2), None, Some(3)], 10, 3); + create_decimal_array(vec![Some(1), Some(2), None, Some(3)], 10, 3); let decimal_array = decimal_array.slice(1, 3); // 2, null, 3 let arrays = vec![decimal_array.data()]; let mut a = MutableArrayData::new(arrays, true, 2); @@ -741,7 +741,7 @@ mod tests { let result = a.freeze(); let array = Decimal128Array::from(result); let expected = create_decimal_array( - &[Some(2), None, None, None, None, None, Some(3)], + vec![Some(2), None, None, None, None, None, Some(3)], 10, 3, ); diff --git a/arrow/src/compute/kernels/cast.rs b/arrow/src/compute/kernels/cast.rs index ebbf4a65b19e..4f59c00fcb5a 100644 --- a/arrow/src/compute/kernels/cast.rs +++ b/arrow/src/compute/kernels/cast.rs @@ -2591,12 +2591,12 @@ mod tests { } fn create_decimal_array( - array: &[Option], + array: Vec>, precision: usize, scale: usize, ) -> Result { array - .iter() + .into_iter() .collect::() .with_precision_and_scale(precision, scale) } @@ -2618,7 +2618,7 @@ mod tests { let output_type = DataType::Decimal128(20, 4); assert!(can_cast_types(&input_type, &output_type)); let array = vec![Some(1123456), Some(2123456), Some(3123456), None]; - let input_decimal_array = create_decimal_array(&array, 20, 3).unwrap(); + let input_decimal_array = create_decimal_array(array, 20, 3).unwrap(); let array = Arc::new(input_decimal_array) as ArrayRef; generate_cast_test_case!( &array, @@ -2633,7 +2633,7 @@ mod tests { ); // negative test let array = vec![Some(123456), None]; - let input_decimal_array = create_decimal_array(&array, 10, 0).unwrap(); + let input_decimal_array = create_decimal_array(array, 10, 0).unwrap(); let array = Arc::new(input_decimal_array) as ArrayRef; let result = cast(&array, &DataType::Decimal128(2, 2)); assert!(result.is_err()); @@ -2647,7 +2647,7 @@ mod tests { let output_type = DataType::Decimal256(20, 4); assert!(can_cast_types(&input_type, &output_type)); let array = vec![Some(1123456), Some(2123456), Some(3123456), None]; - let input_decimal_array = create_decimal_array(&array, 20, 3).unwrap(); + let input_decimal_array = create_decimal_array(array, 20, 3).unwrap(); let array = Arc::new(input_decimal_array) as ArrayRef; generate_cast_test_case!( &array, @@ -2739,7 +2739,7 @@ mod tests { assert!(!can_cast_types(&decimal_type, &DataType::UInt8)); let value_array: Vec> = vec![Some(125), Some(225), Some(325), None, Some(525)]; - let decimal_array = create_decimal_array(&value_array, 38, 2).unwrap(); + let decimal_array = create_decimal_array(value_array, 38, 2).unwrap(); let array = Arc::new(decimal_array) as ArrayRef; // i8 generate_cast_test_case!( @@ -2786,7 +2786,7 @@ mod tests { // overflow test: out of range of max i8 let value_array: Vec> = vec![Some(24400)]; - let decimal_array = create_decimal_array(&value_array, 38, 2).unwrap(); + let decimal_array = create_decimal_array(value_array, 38, 2).unwrap(); let array = Arc::new(decimal_array) as ArrayRef; let casted_array = cast(&array, &DataType::Int8); assert_eq!( @@ -2806,7 +2806,7 @@ mod tests { Some(112345678), Some(112345679), ]; - let decimal_array = create_decimal_array(&value_array, 38, 2).unwrap(); + let decimal_array = create_decimal_array(value_array, 38, 2).unwrap(); let array = Arc::new(decimal_array) as ArrayRef; generate_cast_test_case!( &array, @@ -2834,7 +2834,7 @@ mod tests { Some(112345678901234568), Some(112345678901234560), ]; - let decimal_array = create_decimal_array(&value_array, 38, 2).unwrap(); + let decimal_array = create_decimal_array(value_array, 38, 2).unwrap(); let array = Arc::new(decimal_array) as ArrayRef; generate_cast_test_case!( &array, @@ -5280,7 +5280,8 @@ mod tests { Arc::new(DurationMicrosecondArray::from(vec![1000, 2000])), Arc::new(DurationNanosecondArray::from(vec![1000, 2000])), Arc::new( - create_decimal_array(&[Some(1), Some(2), Some(3), None], 38, 0).unwrap(), + create_decimal_array(vec![Some(1), Some(2), Some(3), None], 38, 0) + .unwrap(), ), ] } diff --git a/arrow/src/compute/kernels/sort.rs b/arrow/src/compute/kernels/sort.rs index dca09a66a8cf..0e2273e92525 100644 --- a/arrow/src/compute/kernels/sort.rs +++ b/arrow/src/compute/kernels/sort.rs @@ -1062,8 +1062,8 @@ mod tests { use std::convert::TryFrom; use std::sync::Arc; - fn create_decimal_array(data: &[Option]) -> Decimal128Array { - data.iter() + fn create_decimal_array(data: Vec>) -> Decimal128Array { + data.into_iter() .collect::() .with_precision_and_scale(23, 6) .unwrap() @@ -1075,7 +1075,7 @@ mod tests { limit: Option, expected_data: Vec, ) { - let output = create_decimal_array(&data); + let output = create_decimal_array(data); let expected = UInt32Array::from(expected_data); let output = sort_to_indices(&(Arc::new(output) as ArrayRef), options, limit).unwrap(); @@ -1088,8 +1088,8 @@ mod tests { limit: Option, expected_data: Vec>, ) { - let output = create_decimal_array(&data); - let expected = Arc::new(create_decimal_array(&expected_data)) as ArrayRef; + let output = create_decimal_array(data); + let expected = Arc::new(create_decimal_array(expected_data)) as ArrayRef; let output = match limit { Some(_) => { sort_limit(&(Arc::new(output) as ArrayRef), options, limit).unwrap() diff --git a/parquet/src/arrow/array_reader/primitive_array.rs b/parquet/src/arrow/array_reader/primitive_array.rs index abe99d4d2eb5..d3f71dbab854 100644 --- a/parquet/src/arrow/array_reader/primitive_array.rs +++ b/parquet/src/arrow/array_reader/primitive_array.rs @@ -24,7 +24,10 @@ use crate::column::page::PageIterator; use crate::data_type::DataType; use crate::errors::{ParquetError, Result}; use crate::schema::types::ColumnDescPtr; -use arrow::array::{Array, ArrayDataBuilder, ArrayRef, BooleanArray, BooleanBufferBuilder, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array}; +use arrow::array::{ + ArrayDataBuilder, ArrayRef, BooleanArray, BooleanBufferBuilder, Decimal128Array, + Float32Array, Float64Array, Int32Array, Int64Array, +}; use arrow::buffer::Buffer; use arrow::datatypes::DataType as ArrowType; use std::any::Any; @@ -33,9 +36,9 @@ use std::sync::Arc; /// Primitive array readers are leaves of array reader tree. They accept page iterator /// and read them into primitive arrays. pub struct PrimitiveArrayReader - where - T: DataType, - T::T: ScalarValue, +where + T: DataType, + T::T: ScalarValue, { data_type: ArrowType, pages: Box, @@ -45,9 +48,9 @@ pub struct PrimitiveArrayReader } impl PrimitiveArrayReader - where - T: DataType, - T::T: ScalarValue, +where + T: DataType, + T::T: ScalarValue, { /// Construct primitive array reader. pub fn new( @@ -77,9 +80,9 @@ impl PrimitiveArrayReader /// Implementation of primitive array reader. impl ArrayReader for PrimitiveArrayReader - where - T: DataType, - T::T: ScalarValue, +where + T: DataType, + T::T: ScalarValue, { fn as_any(&self) -> &dyn Any { self @@ -187,7 +190,7 @@ impl ArrayReader for PrimitiveArrayReader .downcast_ref::() .unwrap() .iter() - .map(|v| v.map(|v| v.into())) + .map(|v| v.map(|v| v as i128)) .collect::(), ArrowType::Int64 => array @@ -195,7 +198,7 @@ impl ArrayReader for PrimitiveArrayReader .downcast_ref::() .unwrap() .iter() - .map(|v| v.map(|v| v.into())) + .map(|v| v.map(|v| v as i128)) .collect::(), _ => { return Err(arrow_err!( @@ -204,7 +207,7 @@ impl ArrayReader for PrimitiveArrayReader )); } } - .with_precision_and_scale(p, s)?; + .with_precision_and_scale(p, s)?; Arc::new(array) as ArrayRef } @@ -243,11 +246,11 @@ mod tests { use crate::util::test_common::rand_gen::make_pages; use crate::util::InMemoryPageIterator; use arrow::array::{Array, PrimitiveArray}; - use arrow::datatypes::{ArrowPrimitiveType}; + use arrow::datatypes::ArrowPrimitiveType; + use arrow::datatypes::DataType::Decimal128; use rand::distributions::uniform::SampleUniform; use std::collections::VecDeque; - use arrow::datatypes::DataType::Decimal128; #[allow(clippy::too_many_arguments)] fn make_column_chunks( @@ -313,7 +316,7 @@ mod tests { column_desc, None, ) - .unwrap(); + .unwrap(); // expect no values to be read let array = array_reader.next_batch(50).unwrap(); @@ -360,7 +363,7 @@ mod tests { column_desc, None, ) - .unwrap(); + .unwrap(); // Read first 50 values, which are all from the first column chunk let array = array_reader.next_batch(50).unwrap(); @@ -560,7 +563,7 @@ mod tests { column_desc, None, ) - .unwrap(); + .unwrap(); let mut accu_len: usize = 0; @@ -602,7 +605,6 @@ mod tests { } } - #[test] fn test_primitive_array_reader_decimal_types() { // parquet `INT32` to decimal @@ -641,18 +643,30 @@ mod tests { column_desc, None, ) - .unwrap(); + .unwrap(); // read data from the reader // the data type is decimal(8,2) let array = array_reader.next_batch(50).unwrap(); assert_eq!(array.data_type(), &Decimal128(8, 2)); let array = array.as_any().downcast_ref::().unwrap(); - let data_decimal_array = data[0..50].iter().copied().map(|v| Some(v as i128)).collect::().with_precision_and_scale(8, 2).unwrap(); + let data_decimal_array = data[0..50] + .iter() + .copied() + .map(|v| Some(v as i128)) + .collect::() + .with_precision_and_scale(8, 2) + .unwrap(); assert_eq!(array, &data_decimal_array); // not equal with different data type(precision and scale) - let data_decimal_array = data[0..50].iter().copied().map(|v| Some(v as i128)).collect::().with_precision_and_scale(9, 0).unwrap(); + let data_decimal_array = data[0..50] + .iter() + .copied() + .map(|v| Some(v as i128)) + .collect::() + .with_precision_and_scale(9, 0) + .unwrap(); assert_ne!(array, &data_decimal_array) } @@ -692,18 +706,30 @@ mod tests { column_desc, None, ) - .unwrap(); + .unwrap(); // read data from the reader // the data type is decimal(18,4) let array = array_reader.next_batch(50).unwrap(); assert_eq!(array.data_type(), &Decimal128(18, 4)); let array = array.as_any().downcast_ref::().unwrap(); - let data_decimal_array = data[0..50].iter().copied().map(|v| Some(v as i128)).collect::().with_precision_and_scale(18, 4).unwrap(); + let data_decimal_array = data[0..50] + .iter() + .copied() + .map(|v| Some(v as i128)) + .collect::() + .with_precision_and_scale(18, 4) + .unwrap(); assert_eq!(array, &data_decimal_array); // not equal with different data type(precision and scale) - let data_decimal_array = data[0..50].iter().copied().map(|v| Some(v as i128)).collect::().with_precision_and_scale(34, 0).unwrap(); + let data_decimal_array = data[0..50] + .iter() + .copied() + .map(|v| Some(v as i128)) + .collect::() + .with_precision_and_scale(34, 0) + .unwrap(); assert_ne!(array, &data_decimal_array) } }