diff --git a/guide/src/high_level.md b/guide/src/high_level.md index b460e44f5c2..0c86aa2618f 100644 --- a/guide/src/high_level.md +++ b/guide/src/high_level.md @@ -80,7 +80,7 @@ The following arrays are supported: * `NullArray` (just holds nulls) * `BooleanArray` (booleans) -* `PrimitiveArray` (for ints, floats) +* `PrimitiveArray` (for ints, floats, decimal) * `Utf8Array` and `Utf8Array` (for strings) * `BinaryArray` and `BinaryArray` (for opaque binaries) * `FixedSizeBinaryArray` (like `BinaryArray`, but fixed size) @@ -124,7 +124,7 @@ There is a one to one relationship between each variant of `PhysicalType` (an en an each implementation of `Array` (a struct): | `PhysicalType` | `Array` | -|-------------------|------------------------| +| ----------------- | ---------------------- | | `Primitive(_)` | `PrimitiveArray<_>` | | `Binary` | `BinaryArray` | | `LargeBinary` | `BinaryArray` | diff --git a/src/array/ord.rs b/src/array/ord.rs index 639317165ab..44103403500 100644 --- a/src/array/ord.rs +++ b/src/array/ord.rs @@ -212,7 +212,9 @@ pub fn build_compare(left: &dyn Array, right: &dyn Array) -> Result compare_primitives::(left, right), (Float32, Float32) => compare_f32(left, right), (Float64, Float64) => compare_f64(left, right), - (Decimal(_, _), Decimal(_, _)) => compare_primitives::(left, right), + (Decimal(DecimalType::Int128, _, _), Decimal(DecimalType::Int128, _, _)) => { + compare_primitives::(left, right) + } (Utf8, Utf8) => compare_string::(left, right), (LargeUtf8, LargeUtf8) => compare_string::(left, right), (Binary, Binary) => compare_binary::(left, right), diff --git a/src/array/primitive/fmt.rs b/src/array/primitive/fmt.rs index 86ad35b127f..840de2a05f3 100644 --- a/src/array/primitive/fmt.rs +++ b/src/array/primitive/fmt.rs @@ -1,7 +1,7 @@ use std::fmt::{Debug, Formatter, Result, Write}; use crate::array::Array; -use crate::datatypes::{IntervalUnit, TimeUnit}; +use crate::datatypes::{DecimalType, IntervalUnit, TimeUnit}; use crate::types::{days_ms, months_days_ns}; use super::PrimitiveArray; @@ -115,7 +115,27 @@ pub fn get_write_value<'a, T: NativeType, F: Write>( Duration(TimeUnit::Millisecond) => dyn_primitive!(array, i64, |x| format!("{}ms", x)), Duration(TimeUnit::Microsecond) => dyn_primitive!(array, i64, |x| format!("{}us", x)), Duration(TimeUnit::Nanosecond) => dyn_primitive!(array, i64, |x| format!("{}ns", x)), - Decimal(_, scale) => { + Decimal(DecimalType::Int32, _, scale) => { + // The number 999.99 has a precision of 5 and scale of 2 + let scale = *scale as u32; + let display = move |x| { + let base = x / 10i32.pow(scale); + let decimals = x - base * 10i32.pow(scale); + format!("{}.{}", base, decimals) + }; + dyn_primitive!(array, i32, display) + } + Decimal(DecimalType::Int64, _, scale) => { + // The number 999.99 has a precision of 5 and scale of 2 + let scale = *scale as u32; + let display = move |x| { + let base = x / 10i64.pow(scale); + let decimals = x - base * 10i64.pow(scale); + format!("{}.{}", base, decimals) + }; + dyn_primitive!(array, i64, display) + } + Decimal(DecimalType::Int128, _, scale) => { // The number 999.99 has a precision of 5 and scale of 2 let scale = *scale as u32; let display = move |x| { @@ -125,6 +145,7 @@ pub fn get_write_value<'a, T: NativeType, F: Write>( }; dyn_primitive!(array, i128, display) } + _ => unreachable!(), } } diff --git a/src/compute/arithmetics/decimal/add.rs b/src/compute/arithmetics/decimal/add.rs index b8775bd1a2e..fd12732147e 100644 --- a/src/compute/arithmetics/decimal/add.rs +++ b/src/compute/arithmetics/decimal/add.rs @@ -6,6 +6,7 @@ use crate::{ arity::{binary, binary_checked}, utils::{check_same_len, combine_validities}, }, + datatypes::DecimalType, }; use crate::{ datatypes::DataType, @@ -25,13 +26,13 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits}; /// ``` /// use arrow2::compute::arithmetics::decimal::add; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(1i128), Some(1i128), None, Some(2i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(1i128), Some(2i128), None, Some(2i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(1i128), Some(1i128), None, Some(2i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(1i128), Some(2i128), None, Some(2i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = add(&a, &b); -/// let expected = PrimitiveArray::from([Some(2i128), Some(3i128), None, Some(4i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([Some(2i128), Some(3i128), None, Some(4i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -64,13 +65,13 @@ pub fn add(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> PrimitiveA /// ``` /// use arrow2::compute::arithmetics::decimal::saturating_add; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = saturating_add(&a, &b); -/// let expected = PrimitiveArray::from([Some(99999i128), Some(33300i128), None, Some(33300i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([Some(99999i128), Some(33300i128), None, Some(33300i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -108,13 +109,13 @@ pub fn saturating_add( /// ``` /// use arrow2::compute::arithmetics::decimal::checked_add; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = checked_add(&a, &b); -/// let expected = PrimitiveArray::from([None, Some(33300i128), None, Some(33300i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([None, Some(33300i128), None, Some(33300i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -172,12 +173,12 @@ impl ArraySaturatingAdd> for PrimitiveArray { /// ``` /// use arrow2::compute::arithmetics::decimal::adaptive_add; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(7, 2)); -/// let b = PrimitiveArray::from([Some(11111_111i128)]).to(DataType::Decimal(8, 3)); +/// let a = PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); +/// let b = PrimitiveArray::from([Some(11111_111i128)]).to(DataType::Decimal(DecimalType::Int128, 8, 3)); /// let result = adaptive_add(&a, &b).unwrap(); -/// let expected = PrimitiveArray::from([Some(22222_221i128)]).to(DataType::Decimal(8, 3)); +/// let expected = PrimitiveArray::from([Some(22222_221i128)]).to(DataType::Decimal(DecimalType::Int128, 8, 3)); /// /// assert_eq!(result, expected); /// ``` @@ -188,7 +189,7 @@ pub fn adaptive_add( check_same_len(lhs, rhs)?; let (lhs_p, lhs_s, rhs_p, rhs_s) = - if let (DataType::Decimal(lhs_p, lhs_s), DataType::Decimal(rhs_p, rhs_s)) = + if let (DataType::Decimal(_, lhs_p, lhs_s), DataType::Decimal(_, rhs_p, rhs_s)) = (lhs.data_type(), rhs.data_type()) { (*lhs_p, *lhs_s, *rhs_p, *rhs_s) @@ -237,7 +238,7 @@ pub fn adaptive_add( let validity = combine_validities(lhs.validity(), rhs.validity()); Ok(PrimitiveArray::::new( - DataType::Decimal(res_p, res_s), + DataType::Decimal(DecimalType::Int128, res_p, res_s), values.into(), validity, )) diff --git a/src/compute/arithmetics/decimal/div.rs b/src/compute/arithmetics/decimal/div.rs index 01f590b9927..1cda279fff7 100644 --- a/src/compute/arithmetics/decimal/div.rs +++ b/src/compute/arithmetics/decimal/div.rs @@ -8,7 +8,7 @@ use crate::{ arity::{binary, binary_checked, unary}, utils::{check_same_len, combine_validities}, }, - datatypes::DataType, + datatypes::{DataType, DecimalType}, error::{Error, Result}, scalar::{PrimitiveScalar, Scalar}, }; @@ -25,13 +25,13 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits}; /// ``` /// use arrow2::compute::arithmetics::decimal::div; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = div(&a, &b); -/// let expected = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), Some(3_00i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), Some(3_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -121,13 +121,13 @@ pub fn div_scalar(lhs: &PrimitiveArray, rhs: &PrimitiveScalar) -> Pr /// ``` /// use arrow2::compute::arithmetics::decimal::saturating_div; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(999_99i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(000_01i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(999_99i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(000_01i128), Some(2_00i128), Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = saturating_div(&a, &b); -/// let expected = PrimitiveArray::from([Some(999_99i128), Some(2_00i128), Some(3_00i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([Some(999_99i128), Some(2_00i128), Some(3_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -170,13 +170,13 @@ pub fn saturating_div( /// ``` /// use arrow2::compute::arithmetics::decimal::checked_div; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(000_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(000_00i128), None, Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = checked_div(&a, &b); -/// let expected = PrimitiveArray::from([None, None, Some(3_00i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([None, None, Some(3_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -232,12 +232,12 @@ impl ArrayCheckedDiv> for PrimitiveArray { /// ``` /// use arrow2::compute::arithmetics::decimal::adaptive_div; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(1000_00i128)]).to(DataType::Decimal(7, 2)); -/// let b = PrimitiveArray::from([Some(10_0000i128)]).to(DataType::Decimal(6, 4)); +/// let a = PrimitiveArray::from([Some(1000_00i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); +/// let b = PrimitiveArray::from([Some(10_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); /// let result = adaptive_div(&a, &b).unwrap(); -/// let expected = PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(9, 4)); +/// let expected = PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 9, 4)); /// /// assert_eq!(result, expected); /// ``` @@ -248,7 +248,7 @@ pub fn adaptive_div( check_same_len(lhs, rhs)?; let (lhs_p, lhs_s, rhs_p, rhs_s) = - if let (DataType::Decimal(lhs_p, lhs_s), DataType::Decimal(rhs_p, rhs_s)) = + if let (DataType::Decimal(_, lhs_p, lhs_s), DataType::Decimal(_, rhs_p, rhs_s)) = (lhs.data_type(), rhs.data_type()) { (*lhs_p, *lhs_s, *rhs_p, *rhs_s) @@ -302,7 +302,7 @@ pub fn adaptive_div( let validity = combine_validities(lhs.validity(), rhs.validity()); Ok(PrimitiveArray::::new( - DataType::Decimal(res_p, res_s), + DataType::Decimal(DecimalType::Int128, res_p, res_s), values.into(), validity, )) diff --git a/src/compute/arithmetics/decimal/mod.rs b/src/compute/arithmetics/decimal/mod.rs index 4b412ef13c6..0998221d286 100644 --- a/src/compute/arithmetics/decimal/mod.rs +++ b/src/compute/arithmetics/decimal/mod.rs @@ -36,7 +36,7 @@ fn number_digits(num: i128) -> usize { } fn get_parameters(lhs: &DataType, rhs: &DataType) -> Result<(usize, usize)> { - if let (DataType::Decimal(lhs_p, lhs_s), DataType::Decimal(rhs_p, rhs_s)) = + if let (DataType::Decimal(_, lhs_p, lhs_s), DataType::Decimal(_, rhs_p, rhs_s)) = (lhs.to_logical_type(), rhs.to_logical_type()) { if lhs_p == rhs_p && lhs_s == rhs_s { diff --git a/src/compute/arithmetics/decimal/mul.rs b/src/compute/arithmetics/decimal/mul.rs index b279673a5de..34876a1aed6 100644 --- a/src/compute/arithmetics/decimal/mul.rs +++ b/src/compute/arithmetics/decimal/mul.rs @@ -8,7 +8,7 @@ use crate::{ arity::{binary, binary_checked, unary}, utils::{check_same_len, combine_validities}, }, - datatypes::DataType, + datatypes::{DataType, DecimalType}, error::{Error, Result}, scalar::{PrimitiveScalar, Scalar}, }; @@ -24,13 +24,13 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits}; /// ``` /// use arrow2::compute::arithmetics::decimal::mul; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(1_00i128), Some(1_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(1_00i128), Some(1_00i128), None, Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), None, Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = mul(&a, &b); -/// let expected = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), None, Some(4_00i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([Some(1_00i128), Some(2_00i128), None, Some(4_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -125,13 +125,13 @@ pub fn mul_scalar(lhs: &PrimitiveArray, rhs: &PrimitiveScalar) -> Pr /// ``` /// use arrow2::compute::arithmetics::decimal::saturating_mul; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(999_99i128), Some(1_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(10_00i128), Some(2_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(999_99i128), Some(1_00i128), None, Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(10_00i128), Some(2_00i128), None, Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = saturating_mul(&a, &b); -/// let expected = PrimitiveArray::from([Some(999_99i128), Some(2_00i128), None, Some(4_00i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([Some(999_99i128), Some(2_00i128), None, Some(4_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -175,13 +175,13 @@ pub fn saturating_mul( /// ``` /// use arrow2::compute::arithmetics::decimal::checked_mul; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(999_99i128), Some(1_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(10_00i128), Some(2_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(999_99i128), Some(1_00i128), None, Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(10_00i128), Some(2_00i128), None, Some(2_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = checked_mul(&a, &b); -/// let expected = PrimitiveArray::from([None, Some(2_00i128), None, Some(4_00i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([None, Some(2_00i128), None, Some(4_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -244,12 +244,12 @@ impl ArraySaturatingMul> for PrimitiveArray { /// ``` /// use arrow2::compute::arithmetics::decimal::adaptive_mul; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(11111_0i128), Some(1_0i128)]).to(DataType::Decimal(6, 1)); -/// let b = PrimitiveArray::from([Some(10_002i128), Some(2_000i128)]).to(DataType::Decimal(5, 3)); +/// let a = PrimitiveArray::from([Some(11111_0i128), Some(1_0i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 1)); +/// let b = PrimitiveArray::from([Some(10_002i128), Some(2_000i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 3)); /// let result = adaptive_mul(&a, &b).unwrap(); -/// let expected = PrimitiveArray::from([Some(111132_222i128), Some(2_000i128)]).to(DataType::Decimal(9, 3)); +/// let expected = PrimitiveArray::from([Some(111132_222i128), Some(2_000i128)]).to(DataType::Decimal(DecimalType::Int128, 9, 3)); /// /// assert_eq!(result, expected); /// ``` @@ -260,7 +260,7 @@ pub fn adaptive_mul( check_same_len(lhs, rhs)?; let (lhs_p, lhs_s, rhs_p, rhs_s) = - if let (DataType::Decimal(lhs_p, lhs_s), DataType::Decimal(rhs_p, rhs_s)) = + if let (DataType::Decimal(_, lhs_p, lhs_s), DataType::Decimal(_, rhs_p, rhs_s)) = (lhs.data_type(), rhs.data_type()) { (*lhs_p, *lhs_s, *rhs_p, *rhs_s) @@ -314,7 +314,7 @@ pub fn adaptive_mul( let validity = combine_validities(lhs.validity(), rhs.validity()); Ok(PrimitiveArray::::new( - DataType::Decimal(res_p, res_s), + DataType::Decimal(DecimalType::Int128, res_p, res_s), values.into(), validity, )) diff --git a/src/compute/arithmetics/decimal/sub.rs b/src/compute/arithmetics/decimal/sub.rs index 1f6adca58f9..6bd887ba81d 100644 --- a/src/compute/arithmetics/decimal/sub.rs +++ b/src/compute/arithmetics/decimal/sub.rs @@ -7,7 +7,7 @@ use crate::{ arity::{binary, binary_checked}, utils::{check_same_len, combine_validities}, }, - datatypes::DataType, + datatypes::{DataType, DecimalType}, error::{Error, Result}, }; @@ -22,13 +22,13 @@ use super::{adjusted_precision_scale, get_parameters, max_value, number_digits}; /// ``` /// use arrow2::compute::arithmetics::decimal::sub; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(1i128), Some(1i128), None, Some(2i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(1i128), Some(2i128), None, Some(2i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(1i128), Some(1i128), None, Some(2i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(1i128), Some(2i128), None, Some(2i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = sub(&a, &b); -/// let expected = PrimitiveArray::from([Some(0i128), Some(-1i128), None, Some(0i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([Some(0i128), Some(-1i128), None, Some(0i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -62,13 +62,13 @@ pub fn sub(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> PrimitiveA /// ``` /// use arrow2::compute::arithmetics::decimal::saturating_sub; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(-99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(-99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = saturating_sub(&a, &b); -/// let expected = PrimitiveArray::from([Some(-99999i128), Some(-11100i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([Some(-99999i128), Some(-11100i128), None, Some(11100i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -128,13 +128,13 @@ impl ArraySaturatingSub> for PrimitiveArray { /// ``` /// use arrow2::compute::arithmetics::decimal::checked_sub; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(-99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(5, 2)); -/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); +/// let a = PrimitiveArray::from([Some(-99000i128), Some(11100i128), None, Some(22200i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); +/// let b = PrimitiveArray::from([Some(01000i128), Some(22200i128), None, Some(11100i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// let result = checked_sub(&a, &b); -/// let expected = PrimitiveArray::from([None, Some(-11100i128), None, Some(11100i128)]).to(DataType::Decimal(5, 2)); +/// let expected = PrimitiveArray::from([None, Some(-11100i128), None, Some(11100i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); /// /// assert_eq!(result, expected); /// ``` @@ -171,12 +171,12 @@ pub fn checked_sub(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Pr /// ``` /// use arrow2::compute::arithmetics::decimal::adaptive_sub; /// use arrow2::array::PrimitiveArray; -/// use arrow2::datatypes::DataType; +/// use arrow2::datatypes::{DataType, DecimalType}; /// -/// let a = PrimitiveArray::from([Some(99_9999i128)]).to(DataType::Decimal(6, 4)); -/// let b = PrimitiveArray::from([Some(-00_0001i128)]).to(DataType::Decimal(6, 4)); +/// let a = PrimitiveArray::from([Some(99_9999i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); +/// let b = PrimitiveArray::from([Some(-00_0001i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); /// let result = adaptive_sub(&a, &b).unwrap(); -/// let expected = PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(7, 4)); +/// let expected = PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 4)); /// /// assert_eq!(result, expected); /// ``` @@ -187,7 +187,7 @@ pub fn adaptive_sub( check_same_len(lhs, rhs)?; let (lhs_p, lhs_s, rhs_p, rhs_s) = - if let (DataType::Decimal(lhs_p, lhs_s), DataType::Decimal(rhs_p, rhs_s)) = + if let (DataType::Decimal(_, lhs_p, lhs_s), DataType::Decimal(_, rhs_p, rhs_s)) = (lhs.data_type(), rhs.data_type()) { (*lhs_p, *lhs_s, *rhs_p, *rhs_s) @@ -237,7 +237,7 @@ pub fn adaptive_sub( let validity = combine_validities(lhs.validity(), rhs.validity()); Ok(PrimitiveArray::::new( - DataType::Decimal(res_p, res_s), + DataType::Decimal(DecimalType::Int128, res_p, res_s), values.into(), validity, )) diff --git a/src/compute/arithmetics/mod.rs b/src/compute/arithmetics/mod.rs index b3a52cd7e19..c86ebf57d0a 100644 --- a/src/compute/arithmetics/mod.rs +++ b/src/compute/arithmetics/mod.rs @@ -19,7 +19,7 @@ pub mod time; use crate::{ array::{Array, DictionaryArray, PrimitiveArray}, bitmap::Bitmap, - datatypes::{DataType, IntervalUnit, TimeUnit}, + datatypes::{DataType, DecimalType, IntervalUnit, TimeUnit}, scalar::{PrimitiveScalar, Scalar}, types::NativeType, }; @@ -55,7 +55,7 @@ macro_rules! arith { (Float32, Float32) => binary_dyn::(lhs, rhs, basic::$op), (Float64, Float64) => binary_dyn::(lhs, rhs, basic::$op), $ ( - (Decimal(_, _), Decimal(_, _)) => { + (Decimal(DecimalType::Int128, _, _), Decimal(DecimalType::Int128, _, _)) => { let lhs = lhs.as_any().downcast_ref().unwrap(); let rhs = rhs.as_any().downcast_ref().unwrap(); Box::new(decimal::$op_decimal(lhs, rhs)) as Box @@ -145,7 +145,7 @@ macro_rules! arith_scalar { (Float32, Float32) => binary_scalar_dyn::(lhs, rhs, basic::$op), (Float64, Float64) => binary_scalar_dyn::(lhs, rhs, basic::$op), $ ( - (Decimal(_, _), Decimal(_, _)) => { + (Decimal(DecimalType::Int128, _, _), Decimal(DecimalType::Int128, _, _)) => { let lhs = lhs.as_any().downcast_ref().unwrap(); let rhs = rhs.as_any().downcast_ref().unwrap(); decimal::$op_decimal(lhs, rhs).boxed() @@ -239,7 +239,10 @@ pub fn can_add(lhs: &DataType, rhs: &DataType) -> bool { | (Float64, Float64) | (Float32, Float32) | (Duration(_), Duration(_)) - | (Decimal(_, _), Decimal(_, _)) + | ( + Decimal(DecimalType::Int128, _, _), + Decimal(DecimalType::Int128, _, _) + ) | (Date32, Duration(_)) | (Date64, Duration(_)) | (Time32(TimeUnit::Millisecond), Duration(_)) @@ -300,7 +303,10 @@ pub fn can_sub(lhs: &DataType, rhs: &DataType) -> bool { | (Float64, Float64) | (Float32, Float32) | (Duration(_), Duration(_)) - | (Decimal(_, _), Decimal(_, _)) + | ( + Decimal(DecimalType::Int128, _, _), + Decimal(DecimalType::Int128, _, _) + ) | (Date32, Duration(_)) | (Date64, Duration(_)) | (Time32(TimeUnit::Millisecond), Duration(_)) @@ -344,7 +350,10 @@ pub fn can_mul(lhs: &DataType, rhs: &DataType) -> bool { | (UInt64, UInt64) | (Float64, Float64) | (Float32, Float32) - | (Decimal(_, _), Decimal(_, _)) + | ( + Decimal(DecimalType::Int128, _, _), + Decimal(DecimalType::Int128, _, _) + ) ) } diff --git a/src/compute/cast/decimal_to.rs b/src/compute/cast/decimal_to.rs index 219d99cff1c..58a7aa8d892 100644 --- a/src/compute/cast/decimal_to.rs +++ b/src/compute/cast/decimal_to.rs @@ -2,7 +2,10 @@ use num_traits::{AsPrimitive, Float, NumCast}; use crate::error::Result; use crate::types::NativeType; -use crate::{array::*, datatypes::DataType}; +use crate::{ + array::*, + datatypes::{DataType, DecimalType}, +}; #[inline] fn decimal_to_decimal_impl Option>( @@ -27,8 +30,11 @@ fn decimal_to_decimal_impl Option>( }) }) }); - PrimitiveArray::::from_trusted_len_iter(values) - .to(DataType::Decimal(to_precision, to_scale)) + PrimitiveArray::::from_trusted_len_iter(values).to(DataType::Decimal( + DecimalType::Int128, + to_precision, + to_scale, + )) } /// Returns a [`PrimitiveArray`] with the casted values. Values are `None` on overflow @@ -38,7 +44,7 @@ pub fn decimal_to_decimal( to_scale: usize, ) -> PrimitiveArray { let (from_precision, from_scale) = - if let DataType::Decimal(p, s) = from.data_type().to_logical_type() { + if let DataType::Decimal(DecimalType::Int128, p, s) = from.data_type().to_logical_type() { (*p, *s) } else { panic!("internal error: i128 is always a decimal") @@ -46,7 +52,11 @@ pub fn decimal_to_decimal( if to_scale == from_scale && to_precision >= from_precision { // fast path - return from.clone().to(DataType::Decimal(to_precision, to_scale)); + return from.clone().to(DataType::Decimal( + DecimalType::Int128, + to_precision, + to_scale, + )); } // todo: other fast paths include increasing scale and precision by so that // a number will never overflow (validity is preserved) @@ -85,11 +95,12 @@ where T: NativeType + Float, f64: AsPrimitive, { - let (_, from_scale) = if let DataType::Decimal(p, s) = from.data_type().to_logical_type() { - (*p, *s) - } else { - panic!("internal error: i128 is always a decimal") - }; + let (_, from_scale) = + if let DataType::Decimal(DecimalType::Int128, p, s) = from.data_type().to_logical_type() { + (*p, *s) + } else { + panic!("internal error: i128 is always a decimal") + }; let div = 10_f64.powi(from_scale as i32); let values = from @@ -115,11 +126,12 @@ pub fn decimal_to_integer(from: &PrimitiveArray) -> PrimitiveArray where T: NativeType + NumCast, { - let (_, from_scale) = if let DataType::Decimal(p, s) = from.data_type().to_logical_type() { - (*p, *s) - } else { - panic!("internal error: i128 is always a decimal") - }; + let (_, from_scale) = + if let DataType::Decimal(DecimalType::Int128, p, s) = from.data_type().to_logical_type() { + (*p, *s) + } else { + panic!("internal error: i128 is always a decimal") + }; let factor = 10_i128.pow(from_scale as u32); let values = from.iter().map(|x| x.and_then(|x| T::from(*x / factor))); diff --git a/src/compute/cast/mod.rs b/src/compute/cast/mod.rs index d3c37cd73fb..0e01696bb83 100644 --- a/src/compute/cast/mod.rs +++ b/src/compute/cast/mod.rs @@ -136,7 +136,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (UInt8, Int64) => true, (UInt8, Float32) => true, (UInt8, Float64) => true, - (UInt8, Decimal(_, _)) => true, + (UInt8, Decimal(DecimalType::Int128, _, _)) => true, (UInt16, UInt8) => true, (UInt16, UInt32) => true, @@ -147,7 +147,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (UInt16, Int64) => true, (UInt16, Float32) => true, (UInt16, Float64) => true, - (UInt16, Decimal(_, _)) => true, + (UInt16, Decimal(DecimalType::Int128, _, _)) => true, (UInt32, UInt8) => true, (UInt32, UInt16) => true, @@ -158,7 +158,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (UInt32, Int64) => true, (UInt32, Float32) => true, (UInt32, Float64) => true, - (UInt32, Decimal(_, _)) => true, + (UInt32, Decimal(DecimalType::Int128, _, _)) => true, (UInt64, UInt8) => true, (UInt64, UInt16) => true, @@ -169,7 +169,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (UInt64, Int64) => true, (UInt64, Float32) => true, (UInt64, Float64) => true, - (UInt64, Decimal(_, _)) => true, + (UInt64, Decimal(DecimalType::Int128, _, _)) => true, (Int8, UInt8) => true, (Int8, UInt16) => true, @@ -180,7 +180,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (Int8, Int64) => true, (Int8, Float32) => true, (Int8, Float64) => true, - (Int8, Decimal(_, _)) => true, + (Int8, Decimal(DecimalType::Int128, _, _)) => true, (Int16, UInt8) => true, (Int16, UInt16) => true, @@ -191,7 +191,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (Int16, Int64) => true, (Int16, Float32) => true, (Int16, Float64) => true, - (Int16, Decimal(_, _)) => true, + (Int16, Decimal(DecimalType::Int128, _, _)) => true, (Int32, UInt8) => true, (Int32, UInt16) => true, @@ -202,7 +202,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (Int32, Int64) => true, (Int32, Float32) => true, (Int32, Float64) => true, - (Int32, Decimal(_, _)) => true, + (Int32, Decimal(DecimalType::Int128, _, _)) => true, (Int64, UInt8) => true, (Int64, UInt16) => true, @@ -213,7 +213,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (Int64, Int32) => true, (Int64, Float32) => true, (Int64, Float64) => true, - (Int64, Decimal(_, _)) => true, + (Int64, Decimal(DecimalType::Int128, _, _)) => true, (Float16, Float32) => true, @@ -226,7 +226,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (Float32, Int32) => true, (Float32, Int64) => true, (Float32, Float64) => true, - (Float32, Decimal(_, _)) => true, + (Float32, Decimal(DecimalType::Int128, _, _)) => true, (Float64, UInt8) => true, (Float64, UInt16) => true, @@ -237,10 +237,10 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { (Float64, Int32) => true, (Float64, Int64) => true, (Float64, Float32) => true, - (Float64, Decimal(_, _)) => true, + (Float64, Decimal(DecimalType::Int128, _, _)) => true, ( - Decimal(_, _), + Decimal(DecimalType::Int128, _, _), UInt8 | UInt16 | UInt32 @@ -251,7 +251,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool { | Int64 | Float32 | Float64 - | Decimal(_, _), + | Decimal(DecimalType::Int128, _, _), ) => true, // end numeric casts @@ -659,7 +659,7 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (UInt8, Int64) => primitive_to_primitive_dyn::(array, to_type, options), (UInt8, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), (UInt8, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt8, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), + (UInt8, Decimal(DecimalType::Int128, p, s)) => integer_to_decimal_dyn::(array, *p, *s), (UInt16, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), (UInt16, UInt32) => primitive_to_primitive_dyn::(array, to_type, as_options), @@ -670,7 +670,9 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (UInt16, Int64) => primitive_to_primitive_dyn::(array, to_type, options), (UInt16, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), (UInt16, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt16, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), + (UInt16, Decimal(DecimalType::Int128, p, s)) => { + integer_to_decimal_dyn::(array, *p, *s) + } (UInt32, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), (UInt32, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), @@ -681,7 +683,9 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (UInt32, Int64) => primitive_to_primitive_dyn::(array, to_type, options), (UInt32, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), (UInt32, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt32, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), + (UInt32, Decimal(DecimalType::Int128, p, s)) => { + integer_to_decimal_dyn::(array, *p, *s) + } (UInt64, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), (UInt64, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), @@ -692,7 +696,9 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (UInt64, Int64) => primitive_to_primitive_dyn::(array, to_type, options), (UInt64, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), (UInt64, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (UInt64, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), + (UInt64, Decimal(DecimalType::Int128, p, s)) => { + integer_to_decimal_dyn::(array, *p, *s) + } (Int8, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), (Int8, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), @@ -703,7 +709,7 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (Int8, Int64) => primitive_to_primitive_dyn::(array, to_type, as_options), (Int8, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), (Int8, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int8, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), + (Int8, Decimal(DecimalType::Int128, p, s)) => integer_to_decimal_dyn::(array, *p, *s), (Int16, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), (Int16, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), @@ -714,7 +720,7 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (Int16, Int64) => primitive_to_primitive_dyn::(array, to_type, as_options), (Int16, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), (Int16, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int16, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), + (Int16, Decimal(DecimalType::Int128, p, s)) => integer_to_decimal_dyn::(array, *p, *s), (Int32, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), (Int32, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), @@ -725,7 +731,7 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (Int32, Int64) => primitive_to_primitive_dyn::(array, to_type, as_options), (Int32, Float32) => primitive_to_primitive_dyn::(array, to_type, as_options), (Int32, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int32, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), + (Int32, Decimal(DecimalType::Int128, p, s)) => integer_to_decimal_dyn::(array, *p, *s), (Int64, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), (Int64, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), @@ -736,7 +742,7 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (Int64, Int32) => primitive_to_primitive_dyn::(array, to_type, options), (Int64, Float32) => primitive_to_primitive_dyn::(array, to_type, options), (Int64, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Int64, Decimal(p, s)) => integer_to_decimal_dyn::(array, *p, *s), + (Int64, Decimal(DecimalType::Int128, p, s)) => integer_to_decimal_dyn::(array, *p, *s), (Float16, Float32) => { let from = array.as_any().downcast_ref().unwrap(); @@ -752,7 +758,7 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (Float32, Int32) => primitive_to_primitive_dyn::(array, to_type, options), (Float32, Int64) => primitive_to_primitive_dyn::(array, to_type, options), (Float32, Float64) => primitive_to_primitive_dyn::(array, to_type, as_options), - (Float32, Decimal(p, s)) => float_to_decimal_dyn::(array, *p, *s), + (Float32, Decimal(DecimalType::Int128, p, s)) => float_to_decimal_dyn::(array, *p, *s), (Float64, UInt8) => primitive_to_primitive_dyn::(array, to_type, options), (Float64, UInt16) => primitive_to_primitive_dyn::(array, to_type, options), @@ -763,19 +769,21 @@ pub fn cast(array: &dyn Array, to_type: &DataType, options: CastOptions) -> Resu (Float64, Int32) => primitive_to_primitive_dyn::(array, to_type, options), (Float64, Int64) => primitive_to_primitive_dyn::(array, to_type, options), (Float64, Float32) => primitive_to_primitive_dyn::(array, to_type, options), - (Float64, Decimal(p, s)) => float_to_decimal_dyn::(array, *p, *s), - - (Decimal(_, _), UInt8) => decimal_to_integer_dyn::(array), - (Decimal(_, _), UInt16) => decimal_to_integer_dyn::(array), - (Decimal(_, _), UInt32) => decimal_to_integer_dyn::(array), - (Decimal(_, _), UInt64) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Int8) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Int16) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Int32) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Int64) => decimal_to_integer_dyn::(array), - (Decimal(_, _), Float32) => decimal_to_float_dyn::(array), - (Decimal(_, _), Float64) => decimal_to_float_dyn::(array), - (Decimal(_, _), Decimal(to_p, to_s)) => decimal_to_decimal_dyn(array, *to_p, *to_s), + (Float64, Decimal(DecimalType::Int128, p, s)) => float_to_decimal_dyn::(array, *p, *s), + + (Decimal(DecimalType::Int128, _, _), UInt8) => decimal_to_integer_dyn::(array), + (Decimal(DecimalType::Int128, _, _), UInt16) => decimal_to_integer_dyn::(array), + (Decimal(DecimalType::Int128, _, _), UInt32) => decimal_to_integer_dyn::(array), + (Decimal(DecimalType::Int128, _, _), UInt64) => decimal_to_integer_dyn::(array), + (Decimal(DecimalType::Int128, _, _), Int8) => decimal_to_integer_dyn::(array), + (Decimal(DecimalType::Int128, _, _), Int16) => decimal_to_integer_dyn::(array), + (Decimal(DecimalType::Int128, _, _), Int32) => decimal_to_integer_dyn::(array), + (Decimal(DecimalType::Int128, _, _), Int64) => decimal_to_integer_dyn::(array), + (Decimal(DecimalType::Int128, _, _), Float32) => decimal_to_float_dyn::(array), + (Decimal(DecimalType::Int128, _, _), Float64) => decimal_to_float_dyn::(array), + (Decimal(DecimalType::Int128, _, _), Decimal(DecimalType::Int128, to_p, to_s)) => { + decimal_to_decimal_dyn(array, *to_p, *to_s) + } // end numeric casts // temporal casts diff --git a/src/compute/cast/primitive_to.rs b/src/compute/cast/primitive_to.rs index a36f16e3deb..1028fb53a0d 100644 --- a/src/compute/cast/primitive_to.rs +++ b/src/compute/cast/primitive_to.rs @@ -9,7 +9,7 @@ use crate::{ array::*, bitmap::Bitmap, compute::arity::unary, - datatypes::{DataType, TimeUnit}, + datatypes::{DataType, DecimalType, TimeUnit}, temporal_conversions::*, types::NativeType, }; @@ -182,8 +182,11 @@ pub fn integer_to_decimal>( }) }); - PrimitiveArray::::from_trusted_len_iter(values) - .to(DataType::Decimal(to_precision, to_scale)) + PrimitiveArray::::from_trusted_len_iter(values).to(DataType::Decimal( + DecimalType::Int128, + to_precision, + to_scale, + )) } pub(super) fn integer_to_decimal_dyn( @@ -227,8 +230,11 @@ where }) }); - PrimitiveArray::::from_trusted_len_iter(values) - .to(DataType::Decimal(to_precision, to_scale)) + PrimitiveArray::::from_trusted_len_iter(values).to(DataType::Decimal( + DecimalType::Int128, + to_precision, + to_scale, + )) } pub(super) fn float_to_decimal_dyn( diff --git a/src/compute/comparison/mod.rs b/src/compute/comparison/mod.rs index 8f67856b90e..e2c4af4bb59 100644 --- a/src/compute/comparison/mod.rs +++ b/src/compute/comparison/mod.rs @@ -478,7 +478,7 @@ fn can_partial_eq_and_ord(data_type: &DataType) -> bool { | DataType::Float64 | DataType::Utf8 | DataType::LargeUtf8 - | DataType::Decimal(_, _) + | DataType::Decimal(_, _, _) | DataType::Binary | DataType::LargeBinary ) diff --git a/src/compute/take/mod.rs b/src/compute/take/mod.rs index b9ae790d0fd..b739438057c 100644 --- a/src/compute/take/mod.rs +++ b/src/compute/take/mod.rs @@ -127,7 +127,7 @@ pub fn can_take(data_type: &DataType) -> bool { | DataType::Float16 | DataType::Float32 | DataType::Float64 - | DataType::Decimal(_, _) + | DataType::Decimal(_, _, _) | DataType::Utf8 | DataType::LargeUtf8 | DataType::Binary diff --git a/src/datatypes/mod.rs b/src/datatypes/mod.rs index 4be369da0a2..6304e318c34 100644 --- a/src/datatypes/mod.rs +++ b/src/datatypes/mod.rs @@ -149,11 +149,11 @@ pub enum DataType { /// /// The `bool` value indicates the `Dictionary` is sorted if set to `true`. Dictionary(IntegerType, Box, bool), - /// Decimal value with precision and scale + /// Decimal value with its physical presentation, precision and scale /// precision is the number of digits in the number and /// scale is the number of decimal places. /// The number 999.99 has a precision of 5 and scale of 2. - Decimal(usize, usize), + Decimal(DecimalType, usize, usize), /// Extension type. Extension(String, Box, Option), } @@ -217,6 +217,28 @@ pub enum IntervalUnit { MonthDayNano, } +/// The decimal representations supported by this crate +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde_types", derive(Serialize, Deserialize))] +pub enum DecimalType { + /// 32 bit integer + Int32, + /// 64 bit integer + Int64, + /// 128 bit integer + Int128, +} + +impl From for PhysicalType { + fn from(width: DecimalType) -> Self { + match width { + DecimalType::Int32 => PhysicalType::Primitive(PrimitiveType::Int32), + DecimalType::Int64 => PhysicalType::Primitive(PrimitiveType::Int64), + DecimalType::Int128 => PhysicalType::Primitive(PrimitiveType::Int128), + } + } +} + impl DataType { /// the [`PhysicalType`] of this [`DataType`]. pub fn to_physical_type(&self) -> PhysicalType { @@ -232,7 +254,7 @@ impl DataType { Int64 | Date64 | Timestamp(_, _) | Time64(_) | Duration(_) => { PhysicalType::Primitive(PrimitiveType::Int64) } - Decimal(_, _) => PhysicalType::Primitive(PrimitiveType::Int128), + Decimal(type_, _, _) => (*type_).into(), UInt8 => PhysicalType::Primitive(PrimitiveType::UInt8), UInt16 => PhysicalType::Primitive(PrimitiveType::UInt16), UInt32 => PhysicalType::Primitive(PrimitiveType::UInt32), @@ -298,7 +320,7 @@ impl From for DataType { PrimitiveType::UInt16 => DataType::UInt16, PrimitiveType::UInt32 => DataType::UInt32, PrimitiveType::UInt64 => DataType::UInt64, - PrimitiveType::Int128 => DataType::Decimal(32, 32), + PrimitiveType::Int128 => DataType::Decimal(DecimalType::Int128, 32, 32), PrimitiveType::Float16 => DataType::Float16, PrimitiveType::Float32 => DataType::Float32, PrimitiveType::Float64 => DataType::Float64, diff --git a/src/ffi/schema.rs b/src/ffi/schema.rs index 4f4fc5b8be3..ce1097a1913 100644 --- a/src/ffi/schema.rs +++ b/src/ffi/schema.rs @@ -2,7 +2,8 @@ use std::{collections::BTreeMap, convert::TryInto, ffi::CStr, ffi::CString, ptr} use crate::{ datatypes::{ - DataType, Extension, Field, IntegerType, IntervalUnit, Metadata, TimeUnit, UnionMode, + DataType, DecimalType, Extension, Field, IntegerType, IntervalUnit, Metadata, TimeUnit, + UnionMode, }, error::{Error, Result}, }; @@ -318,10 +319,12 @@ unsafe fn to_data_type(schema: &ArrowSchema) -> Result { } ["d", raw] => { // Decimal - let (precision, scale) = match raw.split(',').collect::>()[..] { + let (decimal_type, precision, scale) = match raw.split(',').collect::>() + [..] + { [precision_raw, scale_raw] => { // Example: "d:19,10" decimal128 [precision 19, scale 10] - (precision_raw, scale_raw) + (DecimalType::Int128, precision_raw, scale_raw) } [precision_raw, scale_raw, width_raw] => { // Example: "d:19,10,NNN" decimal bitwidth = NNN [precision 19, scale 10] @@ -331,12 +334,18 @@ unsafe fn to_data_type(schema: &ArrowSchema) -> Result { "Decimal bit width is not a valid integer".to_string(), ) })?; - if bit_width != 128 { - return Err(Error::OutOfSpec( - "Decimal256 is not supported".to_string(), - )); - } - (precision_raw, scale_raw) + let decimal_type = match bit_width { + 32 => DecimalType::Int32, + 64 => DecimalType::Int64, + 128 => DecimalType::Int128, + _ => { + return Err(Error::OutOfSpec( + "Decimal256 is not supported".to_string(), + )) + } + }; + + (decimal_type, precision_raw, scale_raw) } _ => { return Err(Error::OutOfSpec( @@ -346,6 +355,7 @@ unsafe fn to_data_type(schema: &ArrowSchema) -> Result { }; DataType::Decimal( + decimal_type, precision.parse::().map_err(|_| { Error::OutOfSpec("Decimal precision is not a valid integer".to_string()) })?, @@ -437,7 +447,11 @@ fn to_format(data_type: &DataType) -> String { tz.as_ref().map(|x| x.as_ref()).unwrap_or("") ) } - DataType::Decimal(precision, scale) => format!("d:{},{}", precision, scale), + DataType::Decimal(type_, precision, scale) => match type_ { + DecimalType::Int32 => format!("d:{},{},{}", precision, scale, 32), + DecimalType::Int64 => format!("d:{},{},{}", precision, scale, 64), + DecimalType::Int128 => format!("d:{},{}", precision, scale), + }, DataType::List(_) => "+l".to_string(), DataType::LargeList(_) => "+L".to_string(), DataType::Struct(_) => "+s".to_string(), @@ -559,7 +573,7 @@ mod tests { DataType::Time32(TimeUnit::Millisecond), DataType::Time64(TimeUnit::Microsecond), DataType::Time64(TimeUnit::Nanosecond), - DataType::Decimal(5, 5), + DataType::Decimal(DecimalType::Int128, 5, 5), DataType::Utf8, DataType::LargeUtf8, DataType::Binary, diff --git a/src/io/avro/read/schema.rs b/src/io/avro/read/schema.rs index 13408efd9c2..1d976b7bd04 100644 --- a/src/io/avro/read/schema.rs +++ b/src/io/avro/read/schema.rs @@ -76,7 +76,7 @@ fn schema_to_field(schema: &AvroSchema, name: Option<&str>, props: Metadata) -> AvroSchema::Bytes(logical) => match logical { Some(logical) => match logical { avro_schema::BytesLogical::Decimal(precision, scale) => { - DataType::Decimal(*precision, *scale) + DataType::Decimal(DecimalType::Int128, *precision, *scale) } }, None => DataType::Binary, @@ -135,7 +135,7 @@ fn schema_to_field(schema: &AvroSchema, name: Option<&str>, props: Metadata) -> AvroSchema::Fixed(Fixed { size, logical, .. }) => match logical { Some(logical) => match logical { avro_schema::FixedLogical::Decimal(precision, scale) => { - DataType::Decimal(*precision, *scale) + DataType::Decimal(DecimalType::Int128, *precision, *scale) } avro_schema::FixedLogical::Duration => { DataType::Interval(IntervalUnit::MonthDayNano) diff --git a/src/io/avro/write/schema.rs b/src/io/avro/write/schema.rs index 26a14db8b3e..23b1cfd2187 100644 --- a/src/io/avro/write/schema.rs +++ b/src/io/avro/write/schema.rs @@ -61,7 +61,9 @@ fn _type_to_schema(data_type: &DataType) -> Result { AvroSchema::Fixed(fixed) } DataType::FixedSizeBinary(size) => AvroSchema::Fixed(Fixed::new("", *size)), - DataType::Decimal(p, s) => AvroSchema::Bytes(Some(BytesLogical::Decimal(*p, *s))), + DataType::Decimal(DecimalType::Int128, p, s) => { + AvroSchema::Bytes(Some(BytesLogical::Decimal(*p, *s))) + } other => { return Err(Error::NotYetImplemented(format!( "write {:?} to avro", diff --git a/src/io/avro/write/serialize.rs b/src/io/avro/write/serialize.rs index 443848ab615..bcc6e7bbcc4 100644 --- a/src/io/avro/write/serialize.rs +++ b/src/io/avro/write/serialize.rs @@ -512,7 +512,7 @@ pub fn can_serialize(data_type: &DataType) -> bool { | Int64 | Float32 | Float64 - | Decimal(_, _) + | Decimal(_, _, _) | Utf8 | Binary | FixedSizeBinary(_) diff --git a/src/io/csv/read_utils.rs b/src/io/csv/read_utils.rs index 27bdcff163d..ceaaf55d6bb 100644 --- a/src/io/csv/read_utils.rs +++ b/src/io/csv/read_utils.rs @@ -1,4 +1,5 @@ use chrono::Datelike; +use lexical_core::FromLexical; // Ideally this trait should not be needed and both `csv` and `csv_async` crates would share // the same `ByteRecord` struct. Unfortunately, they do not and thus we must use generics @@ -13,7 +14,7 @@ use crate::{ datatypes::*, error::{Error, Result}, temporal_conversions, - types::NativeType, + types::{Decimal, NativeType}, }; use super::utils::RFC3339; @@ -24,7 +25,7 @@ fn to_utf8(bytes: &[u8]) -> Option<&str> { } #[inline] -fn deserialize_primitive( +fn deserialize_primitive( rows: &[B], column: usize, datatype: DataType, @@ -32,6 +33,7 @@ fn deserialize_primitive( ) -> Box where T: NativeType + lexical_core::FromLexical, + B: ByteRecordGeneric, F: Fn(&[u8]) -> Option, { let iter = rows.iter().map(|row| match row.get(column) { @@ -54,13 +56,28 @@ fn significant_bytes(bytes: &[u8]) -> usize { /// Deserializes bytes to a single i128 representing a decimal /// The decimal precision and scale are not checked. #[inline] -fn deserialize_decimal(bytes: &[u8], precision: usize, scale: usize) -> Option { +fn deserialize_decimal( + bytes: &[u8], + precision: usize, + scale: usize, +) -> Option { + let ten = T::one() + + T::one() + + T::one() + + T::one() + + T::one() + + T::one() + + T::one() + + T::one() + + T::one() + + T::one(); + let mut a = bytes.split(|x| *x == b'.'); let lhs = a.next(); let rhs = a.next(); match (lhs, rhs) { - (Some(lhs), Some(rhs)) => lexical_core::parse::(lhs).ok().and_then(|x| { - lexical_core::parse::(rhs) + (Some(lhs), Some(rhs)) => lexical_core::parse::(lhs).ok().and_then(|x| { + lexical_core::parse::(rhs) .ok() .map(|y| (x, lhs, y, rhs)) .and_then(|(lhs, lhs_b, rhs, rhs_b)| { @@ -72,19 +89,19 @@ fn deserialize_decimal(bytes: &[u8], precision: usize, scale: usize) -> Option { if rhs.len() != precision || rhs.len() != scale { return None; } - lexical_core::parse::(rhs).ok() + lexical_core::parse::(rhs).ok() } (Some(lhs), None) => { if lhs.len() != precision || scale != 0 { return None; } - lexical_core::parse::(lhs).ok() + lexical_core::parse::(lhs).ok() } (None, None) => None, } @@ -225,9 +242,22 @@ pub(crate) fn deserialize_column( }) }) } - Decimal(precision, scale) => deserialize_primitive(rows, column, datatype, |x| { - deserialize_decimal(x, precision, scale) - }), + Decimal(DecimalType::Int32, precision, scale) => { + deserialize_primitive::(rows, column, datatype, |x| { + deserialize_decimal(x, precision, scale) + }) + } + Decimal(DecimalType::Int64, precision, scale) => { + deserialize_primitive::(rows, column, datatype, |x| { + deserialize_decimal(x, precision, scale) + }) + } + Decimal(DecimalType::Int128, precision, scale) => { + deserialize_primitive::(rows, column, datatype, |x| { + deserialize_decimal(x, precision, scale) + }) + } + Utf8 => deserialize_utf8::(rows, column), LargeUtf8 => deserialize_utf8::(rows, column), Binary => deserialize_binary::(rows, column), diff --git a/src/io/ipc/read/schema.rs b/src/io/ipc/read/schema.rs index b54292baab8..e31b9f2dbb6 100644 --- a/src/io/ipc/read/schema.rs +++ b/src/io/ipc/read/schema.rs @@ -2,8 +2,8 @@ use arrow_format::ipc::planus::ReadAsRoot; use crate::{ datatypes::{ - get_extension, DataType, Extension, Field, IntegerType, IntervalUnit, Metadata, Schema, - TimeUnit, UnionMode, + get_extension, DataType, DecimalType, Extension, Field, IntegerType, IntervalUnit, + Metadata, Schema, TimeUnit, UnionMode, }, error::{Error, Result}, }; @@ -198,7 +198,15 @@ fn get_data_type( (DataType::Duration(time_unit), IpcField::default()) } Decimal(decimal) => { + let bit_width = decimal.bit_width()?; + let type_ = match bit_width { + 32 => DecimalType::Int32, + 64 => DecimalType::Int64, + 128 => DecimalType::Int128, + _ => return Err(Error::nyi("Decimal 256 not supported")), + }; let data_type = DataType::Decimal( + type_, decimal .precision()? .try_into() diff --git a/src/io/ipc/write/schema.rs b/src/io/ipc/write/schema.rs index 0636cb9e6c0..d6c296c581b 100644 --- a/src/io/ipc/write/schema.rs +++ b/src/io/ipc/write/schema.rs @@ -1,7 +1,7 @@ use arrow_format::ipc::planus::Builder; use crate::datatypes::{ - DataType, Field, IntegerType, IntervalUnit, Metadata, Schema, TimeUnit, UnionMode, + DataType, DecimalType, Field, IntegerType, IntervalUnit, Metadata, Schema, TimeUnit, UnionMode, }; use crate::io::ipc::endianess::is_native_little_endian; @@ -192,11 +192,18 @@ fn serialize_type(data_type: &DataType) -> arrow_format::ipc::Type { Float64 => ipc::Type::FloatingPoint(Box::new(ipc::FloatingPoint { precision: ipc::Precision::Double, })), - Decimal(precision, scale) => ipc::Type::Decimal(Box::new(ipc::Decimal { - precision: *precision as i32, - scale: *scale as i32, - bit_width: 128, - })), + Decimal(type_, precision, scale) => { + let bit_width = match type_ { + DecimalType::Int32 => 32, + DecimalType::Int64 => 64, + DecimalType::Int128 => 128, + }; + ipc::Type::Decimal(Box::new(ipc::Decimal { + precision: *precision as i32, + scale: *scale as i32, + bit_width, + })) + } Binary => ipc::Type::Binary(Box::new(ipc::Binary {})), LargeBinary => ipc::Type::LargeBinary(Box::new(ipc::LargeBinary {})), Utf8 => ipc::Type::Utf8(Box::new(ipc::Utf8 {})), @@ -281,7 +288,7 @@ fn serialize_children(data_type: &DataType, ipc_field: &IpcField) -> Vec vec![], + | Decimal(_, _, _) => vec![], FixedSizeList(inner, _) | LargeList(inner) | List(inner) | Map(inner, _) => { vec![serialize_field(inner, &ipc_field.fields[0])] } diff --git a/src/io/json_integration/read/schema.rs b/src/io/json_integration/read/schema.rs index aaf6912af05..709de43cb74 100644 --- a/src/io/json_integration/read/schema.rs +++ b/src/io/json_integration/read/schema.rs @@ -2,6 +2,7 @@ use serde_derive::Deserialize; use serde_json::Value; use crate::{ + datatypes::DecimalType, error::{Error, Result}, io::ipc::IpcField, }; @@ -170,20 +171,30 @@ fn to_data_type(item: &Value, mut children: Vec) -> Result { "largeutf8" => LargeUtf8, "decimal" => { // return a list with any type as its child isn't defined in the map - let precision = match item.get("precision") { - Some(p) => Ok(p.as_u64().unwrap() as usize), - None => Err(Error::OutOfSpec( - "Expecting a precision for decimal".to_string(), - )), - }; - let scale = match item.get("scale") { - Some(s) => Ok(s.as_u64().unwrap() as usize), - _ => Err(Error::OutOfSpec( - "Expecting a scale for decimal".to_string(), - )), + let precision = item + .get("precision") + .map(|p| p.as_u64().unwrap() as usize) + .ok_or_else(|| Error::OutOfSpec("Expecting a precision for decimal".to_string()))?; + let scale = item + .get("scale") + .map(|p| p.as_u64().unwrap() as usize) + .ok_or_else(|| Error::OutOfSpec("Expecting a scale for decimal".to_string()))?; + let bitwidth = item + .get("bitWidth") + .map(|p| p.as_u64().unwrap() as usize) + .unwrap_or(128); + let type_ = match bitwidth { + 32 => DecimalType::Int32, + 64 => DecimalType::Int64, + 128 => DecimalType::Int128, + _ => { + return Err(Error::OutOfSpec( + "Expecting a bitwidth for decimal".to_string(), + )) + } }; - DataType::Decimal(precision?, scale?) + DataType::Decimal(type_, precision, scale) } "floatingpoint" => match item.get("precision") { Some(p) if p == "HALF" => DataType::Float16, diff --git a/src/io/json_integration/write/schema.rs b/src/io/json_integration/write/schema.rs index f4297c3fb0f..3ff15200083 100644 --- a/src/io/json_integration/write/schema.rs +++ b/src/io/json_integration/write/schema.rs @@ -1,6 +1,6 @@ use serde_json::{json, Map, Value}; -use crate::datatypes::{DataType, Field, IntervalUnit, Metadata, Schema, TimeUnit}; +use crate::datatypes::{DataType, DecimalType, Field, IntervalUnit, Metadata, Schema, TimeUnit}; use crate::io::ipc::IpcField; use crate::io::json_integration::ArrowJsonSchema; @@ -86,8 +86,13 @@ fn serialize_data_type(data_type: &DataType) -> Value { TimeUnit::Nanosecond => "NANOSECOND", }}), DataType::Dictionary(_, _, _) => json!({ "name": "dictionary"}), - DataType::Decimal(precision, scale) => { - json!({"name": "decimal", "precision": precision, "scale": scale}) + DataType::Decimal(decimal, precision, scale) => { + let bitwidth = match decimal { + DecimalType::Int32 => 32, + DecimalType::Int64 => 64, + DecimalType::Int128 => 128, + }; + json!({"name": "decimal", "precision": precision, "scale": scale, "bitWidth": bitwidth}) } DataType::Extension(_, inner_data_type, _) => serialize_data_type(inner_data_type), } diff --git a/src/io/odbc/read/schema.rs b/src/io/odbc/read/schema.rs index dba4c233738..2b9aa9b94c0 100644 --- a/src/io/odbc/read/schema.rs +++ b/src/io/odbc/read/schema.rs @@ -1,4 +1,4 @@ -use crate::datatypes::{DataType, Field, TimeUnit}; +use crate::datatypes::{DataType, DecimalType, Field, TimeUnit}; use crate::error::Result; use super::super::api; @@ -41,7 +41,7 @@ fn column_to_data_type(data_type: &api::DataType) -> DataType { | OdbcDataType::Decimal { precision: p @ 0..=38, scale, - } => DataType::Decimal(*p, (*scale) as usize), + } => DataType::Decimal(DecimalType::Int128, *p, (*scale) as usize), OdbcDataType::Integer => DataType::Int32, OdbcDataType::SmallInt => DataType::Int16, OdbcDataType::Real | OdbcDataType::Float { precision: 0..=24 } => DataType::Float32, diff --git a/src/io/parquet/read/deserialize/simple.rs b/src/io/parquet/read/deserialize/simple.rs index e9257d045ee..da40074b3af 100644 --- a/src/io/parquet/read/deserialize/simple.rs +++ b/src/io/parquet/read/deserialize/simple.rs @@ -7,7 +7,7 @@ use parquet2::{ use crate::{ array::{Array, BinaryArray, DictionaryKey, MutablePrimitiveArray, PrimitiveArray, Utf8Array}, - datatypes::{DataType, IntervalUnit, TimeUnit}, + datatypes::{DataType, DecimalType, IntervalUnit, TimeUnit}, error::{Error, Result}, types::{days_ms, NativeType}, }; @@ -180,7 +180,7 @@ pub fn page_iter_to_arrays<'a, I: 'a + DataPages>( Box::new(arrays) as _ } - Decimal(_, _) => match physical_type { + Decimal(DecimalType::Int128, _, _) => match physical_type { PhysicalType::Int32 => dyn_iter(iden(primitive::Iter::new( pages, data_type, diff --git a/src/io/parquet/read/indexes/primitive.rs b/src/io/parquet/read/indexes/primitive.rs index 103d67bbcf1..8d5c12693e9 100644 --- a/src/io/parquet/read/indexes/primitive.rs +++ b/src/io/parquet/read/indexes/primitive.rs @@ -3,7 +3,7 @@ use parquet2::schema::types::{PrimitiveLogicalType, PrimitiveType, TimeUnit as P use parquet2::types::int96_to_i64_ns; use crate::array::{Array, MutablePrimitiveArray, PrimitiveArray}; -use crate::datatypes::{DataType, TimeUnit}; +use crate::datatypes::{DataType, DecimalType, TimeUnit}; use crate::trusted_len::TrustedLen; use crate::types::NativeType; @@ -28,7 +28,7 @@ fn deserialize_int32>>( PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as u32))) .to(data_type), ), - Decimal(_, _) => Box::new( + Decimal(DecimalType::Int128, _, _) => Box::new( PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as i128))) .to(data_type), ), @@ -106,7 +106,7 @@ fn deserialize_int64>>( PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as u64))) .to(data_type), ) as _, - Decimal(_, _) => Box::new( + Decimal(DecimalType::Int128, _, _) => Box::new( PrimitiveArray::::from_trusted_len_iter(iter.map(|x| x.map(|x| x as i128))) .to(data_type), ) as _, diff --git a/src/io/parquet/read/schema/convert.rs b/src/io/parquet/read/schema/convert.rs index 5f23dc71dd6..cb285c90adf 100644 --- a/src/io/parquet/read/schema/convert.rs +++ b/src/io/parquet/read/schema/convert.rs @@ -7,7 +7,7 @@ use parquet2::schema::{ Repetition, }; -use crate::datatypes::{DataType, Field, IntervalUnit, TimeUnit}; +use crate::datatypes::{DataType, DecimalType, Field, IntervalUnit, TimeUnit}; /// Converts [`ParquetType`]s to a [`Field`], ignoring parquet fields that do not contain /// any physical column. @@ -33,7 +33,9 @@ fn from_int32( // is a deviation to the parquet specification and we ignore _ => DataType::Int32, }, - (Some(Decimal(precision, scale)), _) => DataType::Decimal(precision, scale), + (Some(Decimal(precision, scale)), _) => { + DataType::Decimal(DecimalType::Int128, precision, scale) + } (Some(Date), _) => DataType::Date32, (Some(Time { unit, .. }), _) => match unit { ParquetTimeUnit::Milliseconds => DataType::Time32(TimeUnit::Millisecond), @@ -51,7 +53,7 @@ fn from_int32( (_, Some(PrimitiveConvertedType::Date)) => DataType::Date32, (_, Some(PrimitiveConvertedType::TimeMillis)) => DataType::Time32(TimeUnit::Millisecond), (_, Some(PrimitiveConvertedType::Decimal(precision, scale))) => { - DataType::Decimal(precision, scale) + DataType::Decimal(DecimalType::Int128, precision, scale) } (_, _) => DataType::Int32, } @@ -110,7 +112,9 @@ fn from_int64( // to parquet's spec, which we ignore _ => DataType::Int64, }, - (Some(Decimal(precision, scale)), _) => DataType::Decimal(precision, scale), + (Some(Decimal(precision, scale)), _) => { + DataType::Decimal(DecimalType::Int128, precision, scale) + } // handle converted types: (_, Some(PrimitiveConvertedType::TimeMicros)) => DataType::Time64(TimeUnit::Microsecond), (_, Some(PrimitiveConvertedType::TimestampMillis)) => { @@ -122,7 +126,7 @@ fn from_int64( (_, Some(PrimitiveConvertedType::Int64)) => DataType::Int64, (_, Some(PrimitiveConvertedType::Uint64)) => DataType::UInt64, (_, Some(PrimitiveConvertedType::Decimal(precision, scale))) => { - DataType::Decimal(precision, scale) + DataType::Decimal(DecimalType::Int128, precision, scale) } (_, _) => DataType::Int64, @@ -153,10 +157,10 @@ fn from_fixed_len_byte_array( ) -> DataType { match (logical_type, converted_type) { (Some(PrimitiveLogicalType::Decimal(precision, scale)), _) => { - DataType::Decimal(precision, scale) + DataType::Decimal(DecimalType::Int128, precision, scale) } (None, Some(PrimitiveConvertedType::Decimal(precision, scale))) => { - DataType::Decimal(precision, scale) + DataType::Decimal(DecimalType::Int128, precision, scale) } (None, Some(PrimitiveConvertedType::Interval)) => { // There is currently no reliable way of determining which IntervalUnit diff --git a/src/io/parquet/read/statistics/mod.rs b/src/io/parquet/read/statistics/mod.rs index d27123fc51e..2b82c18678a 100644 --- a/src/io/parquet/read/statistics/mod.rs +++ b/src/io/parquet/read/statistics/mod.rs @@ -13,7 +13,7 @@ use parquet2::statistics::{ use crate::array::*; use crate::datatypes::IntervalUnit; -use crate::datatypes::{DataType, Field, PhysicalType}; +use crate::datatypes::{DataType, DecimalType, Field, PhysicalType}; use crate::error::Error; use crate::error::Result; @@ -497,7 +497,7 @@ fn push( } Float32 => primitive::push(from, min, max, |x: f32| Ok(x as f32)), Float64 => primitive::push(from, min, max, |x: f64| Ok(x as f64)), - Decimal(_, _) => match physical_type { + Decimal(DecimalType::Int128, _, _) => match physical_type { ParquetPhysicalType::Int32 => primitive::push(from, min, max, |x: i32| Ok(x as i128)), ParquetPhysicalType::Int64 => primitive::push(from, min, max, |x: i64| Ok(x as i128)), ParquetPhysicalType::FixedLenByteArray(n) if *n > 16 => { diff --git a/src/io/parquet/write/mod.rs b/src/io/parquet/write/mod.rs index 4fea3d9c4aa..d79424e2451 100644 --- a/src/io/parquet/write/mod.rs +++ b/src/io/parquet/write/mod.rs @@ -309,8 +309,10 @@ pub fn array_to_page_simple( fixed_len_bytes::array_to_page(array, options, type_, statistics) } - DataType::Decimal(precision, _) => { - let type_ = type_; + DataType::Decimal(decimal_type_, precision, _) => { + if *decimal_type_ != DecimalType::Int128 { + return Err(Error::nyi("Only decimal 128 supported to write to parquet")); + } let precision = *precision; let array = array .as_any() diff --git a/src/io/parquet/write/schema.rs b/src/io/parquet/write/schema.rs index bf3100bdcf0..25f31e96275 100644 --- a/src/io/parquet/write/schema.rs +++ b/src/io/parquet/write/schema.rs @@ -10,7 +10,7 @@ use parquet2::{ }; use crate::{ - datatypes::{DataType, Field, Schema, TimeUnit}, + datatypes::{DataType, DecimalType, Field, Schema, TimeUnit}, error::{Error, Result}, io::ipc::write::default_ipc_fields, io::ipc::write::schema_to_bytes, @@ -272,7 +272,12 @@ pub fn to_parquet_type(field: &Field) -> Result { None, None, )?), - DataType::Decimal(precision, scale) => { + DataType::Decimal(type_, precision, scale) => { + if *type_ != DecimalType::Int128 { + return Err(Error::nyi( + "Only decimal 128 implemented to write to parquet", + )); + } let precision = *precision; let scale = *scale; let logical_type = Some(PrimitiveLogicalType::Decimal(precision, scale)); diff --git a/src/types/mod.rs b/src/types/mod.rs index 3ba86ad59ec..b4fd255dbee 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -85,3 +85,30 @@ mod private { impl Sealed for super::days_ms {} impl Sealed for super::months_days_ns {} } + +/// Trait describing the three [`NativeType`]s that can be used as decimal representations +pub trait Decimal: NativeType + num_traits::Signed + num_traits::Pow { + /// The 10 + fn ten() -> Self; +} + +impl Decimal for i32 { + #[inline] + fn ten() -> Self { + 10 + } +} + +impl Decimal for i64 { + #[inline] + fn ten() -> Self { + 10 + } +} + +impl Decimal for i128 { + #[inline] + fn ten() -> Self { + 10 + } +} diff --git a/tests/it/array/ord.rs b/tests/it/array/ord.rs index 9f5a07ed83b..5be868c3ff8 100644 --- a/tests/it/array/ord.rs +++ b/tests/it/array/ord.rs @@ -2,7 +2,7 @@ use std::cmp::Ordering; use arrow2::array::ord::build_compare; use arrow2::array::*; -use arrow2::datatypes::DataType; +use arrow2::datatypes::{DataType, DecimalType}; use arrow2::error::Result; #[test] @@ -70,7 +70,7 @@ fn f64_zeros() -> Result<()> { #[test] fn decimal() -> Result<()> { - let array = Int128Array::from_slice(&[1, 2]).to(DataType::Decimal(38, 0)); + let array = Int128Array::from_slice(&[1, 2]).to(DataType::Decimal(DecimalType::Int128, 38, 0)); let cmp = build_compare(&array, &array)?; diff --git a/tests/it/array/primitive/fmt.rs b/tests/it/array/primitive/fmt.rs index 4f62e0bf22c..dece3f724c8 100644 --- a/tests/it/array/primitive/fmt.rs +++ b/tests/it/array/primitive/fmt.rs @@ -194,24 +194,6 @@ fn debug_duration_ns() { ); } -#[test] -fn debug_decimal() { - let array = Int128Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal(5, 2)); - assert_eq!( - format!("{:?}", array), - "Decimal(5, 2)[123.45, None, 234.56]" - ); -} - -#[test] -fn debug_decimal1() { - let array = Int128Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal(5, 1)); - assert_eq!( - format!("{:?}", array), - "Decimal(5, 1)[1234.5, None, 2345.6]" - ); -} - #[test] fn debug_interval_days_ms() { let array = DaysMsArray::from(&[Some(days_ms::new(1, 1)), None, Some(days_ms::new(2, 2))]); diff --git a/tests/it/array/primitive/mod.rs b/tests/it/array/primitive/mod.rs index 5a618cba256..c481195710f 100644 --- a/tests/it/array/primitive/mod.rs +++ b/tests/it/array/primitive/mod.rs @@ -1,6 +1,12 @@ use std::iter::FromIterator; -use arrow2::{array::*, bitmap::Bitmap, buffer::Buffer, datatypes::*, types::months_days_ns}; +use arrow2::{ + array::*, + bitmap::Bitmap, + buffer::Buffer, + datatypes::*, + types::{days_ms, months_days_ns}, +}; mod fmt; mod mutable; @@ -72,17 +78,219 @@ fn from() { } #[test] -fn months_days_ns_from_slice() { +fn debug_int32() { + let array = Int32Array::from(&[Some(1), None, Some(2)]); + assert_eq!(format!("{:?}", array), "Int32[1, None, 2]"); +} + +#[test] +fn debug_date32() { + let array = Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Date32); + assert_eq!( + format!("{:?}", array), + "Date32[1970-01-02, None, 1970-01-03]" + ); +} + +#[test] +fn debug_time32s() { + let array = Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Time32(TimeUnit::Second)); + assert_eq!( + format!("{:?}", array), + "Time32(Second)[00:00:01, None, 00:00:02]" + ); +} + +#[test] +fn debug_time32ms() { + let array = + Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Time32(TimeUnit::Millisecond)); + assert_eq!( + format!("{:?}", array), + "Time32(Millisecond)[00:00:00.001, None, 00:00:00.002]" + ); +} + +#[test] +fn debug_interval_d() { + let array = + Int32Array::from(&[Some(1), None, Some(2)]).to(DataType::Interval(IntervalUnit::YearMonth)); + assert_eq!(format!("{:?}", array), "Interval(YearMonth)[1m, None, 2m]"); +} + +#[test] +fn debug_int64() { + let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Int64); + assert_eq!(format!("{:?}", array), "Int64[1, None, 2]"); +} + +#[test] +fn debug_date64() { + let array = Int64Array::from(&[Some(1), None, Some(86400000)]).to(DataType::Date64); + assert_eq!( + format!("{:?}", array), + "Date64[1970-01-01, None, 1970-01-02]" + ); +} + +#[test] +fn debug_time64us() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Time64(TimeUnit::Microsecond)); + assert_eq!( + format!("{:?}", array), + "Time64(Microsecond)[00:00:00.000001, None, 00:00:00.000002]" + ); +} + +#[test] +fn debug_time64ns() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Time64(TimeUnit::Nanosecond)); + assert_eq!( + format!("{:?}", array), + "Time64(Nanosecond)[00:00:00.000000001, None, 00:00:00.000000002]" + ); +} + +#[test] +fn debug_timestamp_s() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp(TimeUnit::Second, None)); + assert_eq!( + format!("{:?}", array), + "Timestamp(Second, None)[1970-01-01 00:00:01, None, 1970-01-01 00:00:02]" + ); +} + +#[test] +fn debug_timestamp_ms() { + let array = Int64Array::from(&[Some(1), None, Some(2)]) + .to(DataType::Timestamp(TimeUnit::Millisecond, None)); + assert_eq!( + format!("{:?}", array), + "Timestamp(Millisecond, None)[1970-01-01 00:00:00.001, None, 1970-01-01 00:00:00.002]" + ); +} + +#[test] +fn debug_timestamp_us() { + let array = Int64Array::from(&[Some(1), None, Some(2)]) + .to(DataType::Timestamp(TimeUnit::Microsecond, None)); + assert_eq!( + format!("{:?}", array), + "Timestamp(Microsecond, None)[1970-01-01 00:00:00.000001, None, 1970-01-01 00:00:00.000002]" + ); +} + +#[test] +fn debug_timestamp_ns() { + let array = Int64Array::from(&[Some(1), None, Some(2)]) + .to(DataType::Timestamp(TimeUnit::Nanosecond, None)); + assert_eq!( + format!("{:?}", array), + "Timestamp(Nanosecond, None)[1970-01-01 00:00:00.000000001, None, 1970-01-01 00:00:00.000000002]" + ); +} + +#[test] +fn debug_timestamp_tz_ns() { + let array = Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Timestamp( + TimeUnit::Nanosecond, + Some("+02:00".to_string()), + )); + assert_eq!( + format!("{:?}", array), + "Timestamp(Nanosecond, Some(\"+02:00\"))[1970-01-01 02:00:00.000000001 +02:00, None, 1970-01-01 02:00:00.000000002 +02:00]" + ); +} + +#[test] +fn debug_duration_ms() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Millisecond)); + assert_eq!( + format!("{:?}", array), + "Duration(Millisecond)[1ms, None, 2ms]" + ); +} + +#[test] +fn debug_duration_s() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Second)); + assert_eq!(format!("{:?}", array), "Duration(Second)[1s, None, 2s]"); +} + +#[test] +fn debug_duration_us() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Microsecond)); + assert_eq!( + format!("{:?}", array), + "Duration(Microsecond)[1us, None, 2us]" + ); +} + +#[test] +fn debug_duration_ns() { + let array = + Int64Array::from(&[Some(1), None, Some(2)]).to(DataType::Duration(TimeUnit::Nanosecond)); + assert_eq!( + format!("{:?}", array), + "Duration(Nanosecond)[1ns, None, 2ns]" + ); +} + +#[test] +fn debug_decimal128() { + let array = Int128Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); + assert_eq!( + format!("{:?}", array), + "Decimal(Int128, 5, 2)[123.45, None, 234.56]" + ); +} + +#[test] +fn debug_decimal32() { + let array = Int32Array::from(&[Some(12345), None, Some(23456)]).to(DataType::Decimal( + DecimalType::Int32, + 5, + 1, + )); + assert_eq!( + format!("{:?}", array), + "Decimal(Int32, 5, 1)[1234.5, None, 2345.6]" + ); +} + +#[test] +fn debug_interval_days_ms() { + let array = DaysMsArray::from(&[Some(days_ms::new(1, 1)), None, Some(days_ms::new(2, 2))]); + assert_eq!( + format!("{:?}", array), + "Interval(DayTime)[1d1ms, None, 2d2ms]" + ); +} + +#[test] +fn debug_months_days_ns() { let data = &[ - months_days_ns::new(1, 1, 2), - months_days_ns::new(1, 1, 3), - months_days_ns::new(2, 3, 3), + Some(months_days_ns::new(1, 1, 2)), + None, + Some(months_days_ns::new(2, 3, 3)), ]; - let array = MonthsDaysNsArray::from_slice(&data); + let array = MonthsDaysNsArray::from(&data); - let a = array.values().as_slice(); - assert_eq!(a, data.as_ref()); + assert_eq!( + format!("{:?}", array), + "Interval(MonthDayNano)[1m1d2ns, None, 2m3d3ns]" + ); } #[test] diff --git a/tests/it/compute/arithmetics/decimal/add.rs b/tests/it/compute/arithmetics/decimal/add.rs index 45af77b1519..e3022b25cd8 100644 --- a/tests/it/compute/arithmetics/decimal/add.rs +++ b/tests/it/compute/arithmetics/decimal/add.rs @@ -3,19 +3,19 @@ use arrow2::array::*; use arrow2::compute::arithmetics::decimal::{adaptive_add, add, checked_add, saturating_add}; use arrow2::compute::arithmetics::{ArrayAdd, ArrayCheckedAdd, ArraySaturatingAdd}; -use arrow2::datatypes::DataType; +use arrow2::datatypes::{DataType, DecimalType}; #[test] fn test_add_normal() { let a = PrimitiveArray::from([Some(11111i128), Some(11100i128), None, Some(22200i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([Some(22222i128), Some(22200i128), None, Some(11100i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = add(&a, &b); let expected = PrimitiveArray::from([Some(33333i128), Some(33300i128), None, Some(33300i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); @@ -27,30 +27,31 @@ fn test_add_normal() { #[test] #[should_panic] fn test_add_decimal_wrong_precision() { - let a = PrimitiveArray::from([None]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([None]).to(DataType::Decimal(6, 2)); + let a = PrimitiveArray::from([None]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); + let b = PrimitiveArray::from([None]).to(DataType::Decimal(DecimalType::Int128, 6, 2)); add(&a, &b); } #[test] #[should_panic(expected = "Overflow in addition presented for precision 5")] fn test_add_panic() { - let a = PrimitiveArray::from([Some(99999i128)]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([Some(1i128)]).to(DataType::Decimal(5, 2)); + let a = + PrimitiveArray::from([Some(99999i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); + let b = PrimitiveArray::from([Some(1i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); let _ = add(&a, &b); } #[test] fn test_add_saturating() { let a = PrimitiveArray::from([Some(11111i128), Some(11100i128), None, Some(22200i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([Some(22222i128), Some(22200i128), None, Some(11100i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = saturating_add(&a, &b); let expected = PrimitiveArray::from([Some(33333i128), Some(33300i128), None, Some(33300i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); @@ -67,14 +68,14 @@ fn test_add_saturating_overflow() { Some(99999i128), Some(-99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([ Some(00001i128), Some(00100i128), Some(10000i128), Some(-99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = saturating_add(&a, &b); @@ -84,7 +85,7 @@ fn test_add_saturating_overflow() { Some(99999i128), Some(-99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); @@ -96,14 +97,14 @@ fn test_add_saturating_overflow() { #[test] fn test_add_checked() { let a = PrimitiveArray::from([Some(11111i128), Some(11100i128), None, Some(22200i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([Some(22222i128), Some(22200i128), None, Some(11100i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = checked_add(&a, &b); let expected = PrimitiveArray::from([Some(33333i128), Some(33300i128), None, Some(33300i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); @@ -114,10 +115,19 @@ fn test_add_checked() { #[test] fn test_add_checked_overflow() { - let a = PrimitiveArray::from([Some(1i128), Some(99999i128)]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([Some(1i128), Some(1i128)]).to(DataType::Decimal(5, 2)); + let a = PrimitiveArray::from([Some(1i128), Some(99999i128)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); + let b = PrimitiveArray::from([Some(1i128), Some(1i128)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); let result = checked_add(&a, &b); - let expected = PrimitiveArray::from([Some(2i128), None]).to(DataType::Decimal(5, 2)); + let expected = + PrimitiveArray::from([Some(2i128), None]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); // Testing trait @@ -131,51 +141,86 @@ fn test_add_adaptive() { // 11111.11 -> 7, 2 // ----------------- // 11122.2211 -> 9, 4 - let a = PrimitiveArray::from([Some(11_1111i128)]).to(DataType::Decimal(6, 4)); - let b = PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(7, 2)); + let a = + PrimitiveArray::from([Some(11_1111i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); + let b = + PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); let result = adaptive_add(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(11122_2211i128)]).to(DataType::Decimal(9, 4)); + let expected = PrimitiveArray::from([Some(11122_2211i128)]).to(DataType::Decimal( + DecimalType::Int128, + 9, + 4, + )); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(9, 4)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 9, 4) + ); // 0.1111 -> 5, 4 // 11111.0 -> 6, 1 // ----------------- // 11111.1111 -> 9, 4 - let a = PrimitiveArray::from([Some(1111i128)]).to(DataType::Decimal(5, 4)); - let b = PrimitiveArray::from([Some(11111_0i128)]).to(DataType::Decimal(6, 1)); + let a = PrimitiveArray::from([Some(1111i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 4)); + let b = + PrimitiveArray::from([Some(11111_0i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 1)); let result = adaptive_add(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(11111_1111i128)]).to(DataType::Decimal(9, 4)); + let expected = PrimitiveArray::from([Some(11111_1111i128)]).to(DataType::Decimal( + DecimalType::Int128, + 9, + 4, + )); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(9, 4)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 9, 4) + ); // 11111.11 -> 7, 2 // 11111.111 -> 8, 3 // ----------------- // 22222.221 -> 8, 3 - let a = PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(7, 2)); - let b = PrimitiveArray::from([Some(11111_111i128)]).to(DataType::Decimal(8, 3)); + let a = + PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); + let b = PrimitiveArray::from([Some(11111_111i128)]).to(DataType::Decimal( + DecimalType::Int128, + 8, + 3, + )); let result = adaptive_add(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(22222_221i128)]).to(DataType::Decimal(8, 3)); + let expected = PrimitiveArray::from([Some(22222_221i128)]).to(DataType::Decimal( + DecimalType::Int128, + 8, + 3, + )); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(8, 3)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 8, 3) + ); // 99.9999 -> 6, 4 // 00.0001 -> 6, 4 // ----------------- // 100.0000 -> 7, 4 - let a = PrimitiveArray::from([Some(99_9999i128)]).to(DataType::Decimal(6, 4)); - let b = PrimitiveArray::from([Some(00_0001i128)]).to(DataType::Decimal(6, 4)); + let a = + PrimitiveArray::from([Some(99_9999i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); + let b = + PrimitiveArray::from([Some(00_0001i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); let result = adaptive_add(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(7, 4)); + let expected = + PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 4)); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(7, 4)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 7, 4) + ); } diff --git a/tests/it/compute/arithmetics/decimal/div.rs b/tests/it/compute/arithmetics/decimal/div.rs index 39138d05dda..e6252210f31 100644 --- a/tests/it/compute/arithmetics/decimal/div.rs +++ b/tests/it/compute/arithmetics/decimal/div.rs @@ -5,8 +5,12 @@ use arrow2::compute::arithmetics::decimal::{ adaptive_div, checked_div, div, div_scalar, saturating_div, }; use arrow2::compute::arithmetics::{ArrayCheckedDiv, ArrayDiv}; +<<<<<<< HEAD use arrow2::datatypes::DataType; use arrow2::scalar::PrimitiveScalar; +======= +use arrow2::datatypes::{DataType, DecimalType}; +>>>>>>> cfb7ff549a (Added support for decimal 32 and 64) #[test] fn test_divide_normal() { @@ -22,7 +26,7 @@ fn test_divide_normal() { Some(30_000i128), Some(123_456i128), ]) - .to(DataType::Decimal(7, 3)); + .to(DataType::Decimal(DecimalType::Int128, 7, 3)); let b = PrimitiveArray::from([ Some(123_456i128), @@ -32,7 +36,7 @@ fn test_divide_normal() { Some(4_000i128), Some(654_321i128), ]) - .to(DataType::Decimal(7, 3)); + .to(DataType::Decimal(DecimalType::Int128, 7, 3)); let result = div(&a, &b); let expected = PrimitiveArray::from([ @@ -43,7 +47,7 @@ fn test_divide_normal() { Some(7_500i128), Some(0_188i128), ]) - .to(DataType::Decimal(7, 3)); + .to(DataType::Decimal(DecimalType::Int128, 7, 3)); assert_eq!(result, expected); @@ -55,16 +59,18 @@ fn test_divide_normal() { #[test] #[should_panic] fn test_divide_decimal_wrong_precision() { - let a = PrimitiveArray::from([None]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([None]).to(DataType::Decimal(6, 2)); + let a = PrimitiveArray::from([None]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); + let b = PrimitiveArray::from([None]).to(DataType::Decimal(DecimalType::Int128, 6, 2)); div(&a, &b); } #[test] #[should_panic(expected = "Overflow in multiplication presented for precision 5")] fn test_divide_panic() { - let a = PrimitiveArray::from([Some(99999i128)]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([Some(000_01i128)]).to(DataType::Decimal(5, 2)); + let a = + PrimitiveArray::from([Some(99999i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); + let b = + PrimitiveArray::from([Some(000_01i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); div(&a, &b); } @@ -100,7 +106,7 @@ fn test_divide_saturating() { Some(30_000i128), Some(123_456i128), ]) - .to(DataType::Decimal(7, 3)); + .to(DataType::Decimal(DecimalType::Int128, 7, 3)); let b = PrimitiveArray::from([ Some(123_456i128), @@ -110,7 +116,7 @@ fn test_divide_saturating() { Some(4_000i128), Some(654_321i128), ]) - .to(DataType::Decimal(7, 3)); + .to(DataType::Decimal(DecimalType::Int128, 7, 3)); let result = saturating_div(&a, &b); let expected = PrimitiveArray::from([ @@ -121,7 +127,7 @@ fn test_divide_saturating() { Some(7_500i128), Some(0_188i128), ]) - .to(DataType::Decimal(7, 3)); + .to(DataType::Decimal(DecimalType::Int128, 7, 3)); assert_eq!(result, expected); } @@ -135,7 +141,7 @@ fn test_divide_saturating_overflow() { Some(99999i128), Some(99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([ Some(-00001i128), Some(00001i128), @@ -143,7 +149,7 @@ fn test_divide_saturating_overflow() { Some(-00020i128), Some(00000i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = saturating_div(&a, &b); @@ -154,7 +160,7 @@ fn test_divide_saturating_overflow() { Some(-99999i128), Some(00000i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); } @@ -169,7 +175,7 @@ fn test_divide_checked() { Some(30_000i128), Some(123_456i128), ]) - .to(DataType::Decimal(7, 3)); + .to(DataType::Decimal(DecimalType::Int128, 7, 3)); let b = PrimitiveArray::from([ Some(123_456i128), @@ -179,7 +185,7 @@ fn test_divide_checked() { Some(4_000i128), Some(654_321i128), ]) - .to(DataType::Decimal(7, 3)); + .to(DataType::Decimal(DecimalType::Int128, 7, 3)); let result = div(&a, &b); let expected = PrimitiveArray::from([ @@ -190,7 +196,7 @@ fn test_divide_checked() { Some(7_500i128), Some(0_188i128), ]) - .to(DataType::Decimal(7, 3)); + .to(DataType::Decimal(DecimalType::Int128, 7, 3)); assert_eq!(result, expected); } @@ -198,12 +204,19 @@ fn test_divide_checked() { #[test] fn test_divide_checked_overflow() { let a = PrimitiveArray::from([Some(1_00i128), Some(4_00i128), Some(6_00i128)]) - .to(DataType::Decimal(5, 2)); - let b = - PrimitiveArray::from([Some(000_00i128), None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); + let b = PrimitiveArray::from([Some(000_00i128), None, Some(2_00i128)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); let result = checked_div(&a, &b); - let expected = PrimitiveArray::from([None, None, Some(3_00i128)]).to(DataType::Decimal(5, 2)); + let expected = PrimitiveArray::from([None, None, Some(3_00i128)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); assert_eq!(result, expected); @@ -218,38 +231,59 @@ fn test_divide_adaptive() { // 10.0000 -> 6, 4 // ----------------- // 100.0000 -> 9, 4 - let a = PrimitiveArray::from([Some(1000_00i128)]).to(DataType::Decimal(7, 2)); - let b = PrimitiveArray::from([Some(10_0000i128)]).to(DataType::Decimal(6, 4)); + let a = + PrimitiveArray::from([Some(1000_00i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); + let b = + PrimitiveArray::from([Some(10_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); let result = adaptive_div(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(9, 4)); + let expected = + PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 9, 4)); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(9, 4)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 9, 4) + ); // 11111.0 -> 6, 1 // 10.002 -> 5, 3 // ----------------- // 1110.877 -> 8, 3 - let a = PrimitiveArray::from([Some(11111_0i128)]).to(DataType::Decimal(6, 1)); - let b = PrimitiveArray::from([Some(10_002i128)]).to(DataType::Decimal(5, 3)); + let a = + PrimitiveArray::from([Some(11111_0i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 1)); + let b = + PrimitiveArray::from([Some(10_002i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 3)); let result = adaptive_div(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(1110_877i128)]).to(DataType::Decimal(8, 3)); + let expected = + PrimitiveArray::from([Some(1110_877i128)]).to(DataType::Decimal(DecimalType::Int128, 8, 3)); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(8, 3)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 8, 3) + ); // 12345.67 -> 7, 2 // 12345.678 -> 8, 3 // ----------------- // 0.999 -> 8, 3 - let a = PrimitiveArray::from([Some(12345_67i128)]).to(DataType::Decimal(7, 2)); - let b = PrimitiveArray::from([Some(12345_678i128)]).to(DataType::Decimal(8, 3)); + let a = + PrimitiveArray::from([Some(12345_67i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); + let b = PrimitiveArray::from([Some(12345_678i128)]).to(DataType::Decimal( + DecimalType::Int128, + 8, + 3, + )); let result = adaptive_div(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(0_999i128)]).to(DataType::Decimal(8, 3)); + let expected = + PrimitiveArray::from([Some(0_999i128)]).to(DataType::Decimal(DecimalType::Int128, 8, 3)); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(8, 3)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 8, 3) + ); } diff --git a/tests/it/compute/arithmetics/decimal/mul.rs b/tests/it/compute/arithmetics/decimal/mul.rs index a4b4a71b257..3b3a087182f 100644 --- a/tests/it/compute/arithmetics/decimal/mul.rs +++ b/tests/it/compute/arithmetics/decimal/mul.rs @@ -3,7 +3,7 @@ use arrow2::array::*; use arrow2::compute::arithmetics::decimal::{adaptive_mul, checked_mul, mul, saturating_mul}; use arrow2::compute::arithmetics::{ArrayCheckedMul, ArrayMul, ArraySaturatingMul}; -use arrow2::datatypes::DataType; +use arrow2::datatypes::{DataType, DecimalType}; #[test] fn test_multiply_normal() { @@ -19,7 +19,7 @@ fn test_multiply_normal() { Some(30_00i128), Some(123_45i128), ]) - .to(DataType::Decimal(7, 2)); + .to(DataType::Decimal(DecimalType::Int128, 7, 2)); let b = PrimitiveArray::from([ Some(222_22i128), @@ -29,7 +29,7 @@ fn test_multiply_normal() { Some(4_00i128), Some(543_21i128), ]) - .to(DataType::Decimal(7, 2)); + .to(DataType::Decimal(DecimalType::Int128, 7, 2)); let result = mul(&a, &b); let expected = PrimitiveArray::from([ @@ -40,7 +40,7 @@ fn test_multiply_normal() { Some(120_00i128), Some(67059_27i128), ]) - .to(DataType::Decimal(7, 2)); + .to(DataType::Decimal(DecimalType::Int128, 7, 2)); assert_eq!(result, expected); @@ -52,16 +52,18 @@ fn test_multiply_normal() { #[test] #[should_panic] fn test_multiply_decimal_wrong_precision() { - let a = PrimitiveArray::from([None]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([None]).to(DataType::Decimal(6, 2)); + let a = PrimitiveArray::from([None]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); + let b = PrimitiveArray::from([None]).to(DataType::Decimal(DecimalType::Int128, 6, 2)); mul(&a, &b); } #[test] #[should_panic(expected = "Overflow in multiplication presented for precision 5")] fn test_multiply_panic() { - let a = PrimitiveArray::from([Some(99999i128)]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([Some(100_00i128)]).to(DataType::Decimal(5, 2)); + let a = + PrimitiveArray::from([Some(99999i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); + let b = + PrimitiveArray::from([Some(100_00i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); let _ = mul(&a, &b); } @@ -75,7 +77,7 @@ fn test_multiply_saturating() { Some(30_00i128), Some(123_45i128), ]) - .to(DataType::Decimal(7, 2)); + .to(DataType::Decimal(DecimalType::Int128, 7, 2)); let b = PrimitiveArray::from([ Some(222_22i128), @@ -85,7 +87,7 @@ fn test_multiply_saturating() { Some(4_00i128), Some(543_21i128), ]) - .to(DataType::Decimal(7, 2)); + .to(DataType::Decimal(DecimalType::Int128, 7, 2)); let result = saturating_mul(&a, &b); let expected = PrimitiveArray::from([ @@ -96,7 +98,7 @@ fn test_multiply_saturating() { Some(120_00i128), Some(67059_27i128), ]) - .to(DataType::Decimal(7, 2)); + .to(DataType::Decimal(DecimalType::Int128, 7, 2)); assert_eq!(result, expected); @@ -113,14 +115,14 @@ fn test_multiply_saturating_overflow() { Some(99999i128), Some(99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([ Some(-00100i128), Some(01000i128), Some(10000i128), Some(-99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = saturating_mul(&a, &b); @@ -130,7 +132,7 @@ fn test_multiply_saturating_overflow() { Some(99999i128), Some(-99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); @@ -149,7 +151,7 @@ fn test_multiply_checked() { Some(30_00i128), Some(123_45i128), ]) - .to(DataType::Decimal(7, 2)); + .to(DataType::Decimal(DecimalType::Int128, 7, 2)); let b = PrimitiveArray::from([ Some(222_22i128), @@ -159,7 +161,7 @@ fn test_multiply_checked() { Some(4_00i128), Some(543_21i128), ]) - .to(DataType::Decimal(7, 2)); + .to(DataType::Decimal(DecimalType::Int128, 7, 2)); let result = checked_mul(&a, &b); let expected = PrimitiveArray::from([ @@ -170,7 +172,7 @@ fn test_multiply_checked() { Some(120_00i128), Some(67059_27i128), ]) - .to(DataType::Decimal(7, 2)); + .to(DataType::Decimal(DecimalType::Int128, 7, 2)); assert_eq!(result, expected); @@ -181,10 +183,22 @@ fn test_multiply_checked() { #[test] fn test_multiply_checked_overflow() { - let a = PrimitiveArray::from([Some(99999i128), Some(1_00i128)]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([Some(10000i128), Some(2_00i128)]).to(DataType::Decimal(5, 2)); + let a = PrimitiveArray::from([Some(99999i128), Some(1_00i128)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); + let b = PrimitiveArray::from([Some(10000i128), Some(2_00i128)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); let result = checked_mul(&a, &b); - let expected = PrimitiveArray::from([None, Some(2_00i128)]).to(DataType::Decimal(5, 2)); + let expected = PrimitiveArray::from([None, Some(2_00i128)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); assert_eq!(result, expected); } @@ -195,38 +209,68 @@ fn test_multiply_adaptive() { // 10.0000 -> 6, 4 // ----------------- // 10000.0000 -> 9, 4 - let a = PrimitiveArray::from([Some(1000_00i128)]).to(DataType::Decimal(7, 2)); - let b = PrimitiveArray::from([Some(10_0000i128)]).to(DataType::Decimal(6, 4)); + let a = + PrimitiveArray::from([Some(1000_00i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); + let b = + PrimitiveArray::from([Some(10_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); let result = adaptive_mul(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(10000_0000i128)]).to(DataType::Decimal(9, 4)); + let expected = PrimitiveArray::from([Some(10000_0000i128)]).to(DataType::Decimal( + DecimalType::Int128, + 9, + 4, + )); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(9, 4)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 9, 4) + ); // 11111.0 -> 6, 1 // 10.002 -> 5, 3 // ----------------- // 111132.222 -> 9, 3 - let a = PrimitiveArray::from([Some(11111_0i128)]).to(DataType::Decimal(6, 1)); - let b = PrimitiveArray::from([Some(10_002i128)]).to(DataType::Decimal(5, 3)); + let a = + PrimitiveArray::from([Some(11111_0i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 1)); + let b = + PrimitiveArray::from([Some(10_002i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 3)); let result = adaptive_mul(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(111132_222i128)]).to(DataType::Decimal(9, 3)); + let expected = PrimitiveArray::from([Some(111132_222i128)]).to(DataType::Decimal( + DecimalType::Int128, + 9, + 3, + )); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(9, 3)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 9, 3) + ); // 12345.67 -> 7, 2 // 12345.678 -> 8, 3 // ----------------- // 152415666.514 -> 11, 3 - let a = PrimitiveArray::from([Some(12345_67i128)]).to(DataType::Decimal(7, 2)); - let b = PrimitiveArray::from([Some(12345_678i128)]).to(DataType::Decimal(8, 3)); + let a = + PrimitiveArray::from([Some(12345_67i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); + let b = PrimitiveArray::from([Some(12345_678i128)]).to(DataType::Decimal( + DecimalType::Int128, + 8, + 3, + )); let result = adaptive_mul(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(152415666_514i128)]).to(DataType::Decimal(12, 3)); + let expected = PrimitiveArray::from([Some(152415666_514i128)]).to(DataType::Decimal( + DecimalType::Int128, + 12, + 3, + )); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(12, 3)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 12, 3) + ); } diff --git a/tests/it/compute/arithmetics/decimal/sub.rs b/tests/it/compute/arithmetics/decimal/sub.rs index 343149a5646..38a234bf59a 100644 --- a/tests/it/compute/arithmetics/decimal/sub.rs +++ b/tests/it/compute/arithmetics/decimal/sub.rs @@ -3,19 +3,19 @@ use arrow2::array::*; use arrow2::compute::arithmetics::decimal::{adaptive_sub, checked_sub, saturating_sub, sub}; use arrow2::compute::arithmetics::{ArrayCheckedSub, ArraySaturatingSub, ArraySub}; -use arrow2::datatypes::DataType; +use arrow2::datatypes::{DataType, DecimalType}; #[test] fn test_subtract_normal() { let a = PrimitiveArray::from([Some(11111i128), Some(22200i128), None, Some(40000i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([Some(22222i128), Some(11100i128), None, Some(11100i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = sub(&a, &b); let expected = PrimitiveArray::from([Some(-11111i128), Some(11100i128), None, Some(28900i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); @@ -27,30 +27,31 @@ fn test_subtract_normal() { #[test] #[should_panic] fn test_subtract_decimal_wrong_precision() { - let a = PrimitiveArray::from([None]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([None]).to(DataType::Decimal(6, 2)); + let a = PrimitiveArray::from([None]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); + let b = PrimitiveArray::from([None]).to(DataType::Decimal(DecimalType::Int128, 6, 2)); sub(&a, &b); } #[test] #[should_panic(expected = "Overflow in subtract presented for precision 5")] fn test_subtract_panic() { - let a = PrimitiveArray::from([Some(-99999i128)]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([Some(1i128)]).to(DataType::Decimal(5, 2)); + let a = + PrimitiveArray::from([Some(-99999i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); + let b = PrimitiveArray::from([Some(1i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); let _ = sub(&a, &b); } #[test] fn test_subtract_saturating() { let a = PrimitiveArray::from([Some(11111i128), Some(22200i128), None, Some(40000i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([Some(22222i128), Some(11100i128), None, Some(11100i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = saturating_sub(&a, &b); let expected = PrimitiveArray::from([Some(-11111i128), Some(11100i128), None, Some(28900i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); @@ -67,14 +68,14 @@ fn test_subtract_saturating_overflow() { Some(-99999i128), Some(99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([ Some(00001i128), Some(00100i128), Some(10000i128), Some(-99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = saturating_sub(&a, &b); @@ -84,7 +85,7 @@ fn test_subtract_saturating_overflow() { Some(-99999i128), Some(99999i128), ]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); @@ -96,14 +97,14 @@ fn test_subtract_saturating_overflow() { #[test] fn test_subtract_checked() { let a = PrimitiveArray::from([Some(11111i128), Some(22200i128), None, Some(40000i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let b = PrimitiveArray::from([Some(22222i128), Some(11100i128), None, Some(11100i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); let result = checked_sub(&a, &b); let expected = PrimitiveArray::from([Some(-11111i128), Some(11100i128), None, Some(28900i128)]) - .to(DataType::Decimal(5, 2)); + .to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); @@ -114,10 +115,19 @@ fn test_subtract_checked() { #[test] fn test_subtract_checked_overflow() { - let a = PrimitiveArray::from([Some(4i128), Some(-99999i128)]).to(DataType::Decimal(5, 2)); - let b = PrimitiveArray::from([Some(2i128), Some(1i128)]).to(DataType::Decimal(5, 2)); + let a = PrimitiveArray::from([Some(4i128), Some(-99999i128)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); + let b = PrimitiveArray::from([Some(2i128), Some(1i128)]).to(DataType::Decimal( + DecimalType::Int128, + 5, + 2, + )); let result = checked_sub(&a, &b); - let expected = PrimitiveArray::from([Some(2i128), None]).to(DataType::Decimal(5, 2)); + let expected = + PrimitiveArray::from([Some(2i128), None]).to(DataType::Decimal(DecimalType::Int128, 5, 2)); assert_eq!(result, expected); } @@ -127,51 +137,86 @@ fn test_subtract_adaptive() { // 11111.11 -> 7, 2 // ------------------ // -11099.9989 -> 9, 4 - let a = PrimitiveArray::from([Some(11_1111i128)]).to(DataType::Decimal(6, 4)); - let b = PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(7, 2)); + let a = + PrimitiveArray::from([Some(11_1111i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); + let b = + PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); let result = adaptive_sub(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(-11099_9989i128)]).to(DataType::Decimal(9, 4)); + let expected = PrimitiveArray::from([Some(-11099_9989i128)]).to(DataType::Decimal( + DecimalType::Int128, + 9, + 4, + )); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(9, 4)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 9, 4) + ); // 11111.0 -> 6, 1 // 0.1111 -> 5, 4 // ----------------- // 11110.8889 -> 9, 4 - let a = PrimitiveArray::from([Some(11111_0i128)]).to(DataType::Decimal(6, 1)); - let b = PrimitiveArray::from([Some(1111i128)]).to(DataType::Decimal(5, 4)); + let a = + PrimitiveArray::from([Some(11111_0i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 1)); + let b = PrimitiveArray::from([Some(1111i128)]).to(DataType::Decimal(DecimalType::Int128, 5, 4)); let result = adaptive_sub(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(11110_8889i128)]).to(DataType::Decimal(9, 4)); + let expected = PrimitiveArray::from([Some(11110_8889i128)]).to(DataType::Decimal( + DecimalType::Int128, + 9, + 4, + )); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(9, 4)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 9, 4) + ); // 11111.11 -> 7, 2 // 11111.111 -> 8, 3 // ----------------- // -00000.001 -> 8, 3 - let a = PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(7, 2)); - let b = PrimitiveArray::from([Some(11111_111i128)]).to(DataType::Decimal(8, 3)); + let a = + PrimitiveArray::from([Some(11111_11i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 2)); + let b = PrimitiveArray::from([Some(11111_111i128)]).to(DataType::Decimal( + DecimalType::Int128, + 8, + 3, + )); let result = adaptive_sub(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(-00000_001i128)]).to(DataType::Decimal(8, 3)); + let expected = PrimitiveArray::from([Some(-00000_001i128)]).to(DataType::Decimal( + DecimalType::Int128, + 8, + 3, + )); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(8, 3)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 8, 3) + ); // 99.9999 -> 6, 4 // -00.0001 -> 6, 4 // ----------------- // 100.0000 -> 7, 4 - let a = PrimitiveArray::from([Some(99_9999i128)]).to(DataType::Decimal(6, 4)); - let b = PrimitiveArray::from([Some(-00_0001i128)]).to(DataType::Decimal(6, 4)); + let a = + PrimitiveArray::from([Some(99_9999i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); + let b = + PrimitiveArray::from([Some(-00_0001i128)]).to(DataType::Decimal(DecimalType::Int128, 6, 4)); let result = adaptive_sub(&a, &b).unwrap(); - let expected = PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(7, 4)); + let expected = + PrimitiveArray::from([Some(100_0000i128)]).to(DataType::Decimal(DecimalType::Int128, 7, 4)); assert_eq!(result, expected); - assert_eq!(result.data_type(), &DataType::Decimal(7, 4)); + assert_eq!( + result.data_type(), + &DataType::Decimal(DecimalType::Int128, 7, 4) + ); } diff --git a/tests/it/compute/cast.rs b/tests/it/compute/cast.rs index 9e9888926d3..c1463c3a8a2 100644 --- a/tests/it/compute/cast.rs +++ b/tests/it/compute/cast.rs @@ -243,11 +243,16 @@ fn int32_to_decimal() { // 10 and -10 can be represented with precision 1 and scale 0 let array = Int32Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]); - let b = cast(&array, &DataType::Decimal(1, 0), CastOptions::default()).unwrap(); + let b = cast( + &array, + &DataType::Decimal(DecimalType::Int128, 1, 0), + CastOptions::default(), + ) + .unwrap(); let c = b.as_any().downcast_ref::>().unwrap(); let expected = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(1, 0)); + .to(DataType::Decimal(DecimalType::Int128, 1, 0)); assert_eq!(c, &expected) } @@ -263,7 +268,12 @@ fn float32_to_decimal() { None, ]); - let b = cast(&array, &DataType::Decimal(10, 2), CastOptions::default()).unwrap(); + let b = cast( + &array, + &DataType::Decimal(DecimalType::Int128, 10, 2), + CastOptions::default(), + ) + .unwrap(); let c = b.as_any().downcast_ref::>().unwrap(); let expected = Int128Array::from(&[ @@ -275,7 +285,7 @@ fn float32_to_decimal() { Some(-10001), None, ]) - .to(DataType::Decimal(10, 2)); + .to(DataType::Decimal(DecimalType::Int128, 10, 2)); assert_eq!(c, &expected) } @@ -284,11 +294,16 @@ fn int32_to_decimal_scaled() { // 10 and -10 can't be represented with precision 1 and scale 1 let array = Int32Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]); - let b = cast(&array, &DataType::Decimal(1, 1), CastOptions::default()).unwrap(); + let b = cast( + &array, + &DataType::Decimal(DecimalType::Int128, 1, 1), + CastOptions::default(), + ) + .unwrap(); let c = b.as_any().downcast_ref::>().unwrap(); - let expected = - Int128Array::from(&[Some(20), None, Some(-20), None, None]).to(DataType::Decimal(1, 1)); + let expected = Int128Array::from(&[Some(20), None, Some(-20), None, None]) + .to(DataType::Decimal(DecimalType::Int128, 1, 1)); assert_eq!(c, &expected) } @@ -296,13 +311,18 @@ fn int32_to_decimal_scaled() { fn decimal_to_decimal() { // increase scale and precision let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(1, 0)); + .to(DataType::Decimal(DecimalType::Int128, 1, 0)); - let b = cast(&array, &DataType::Decimal(2, 1), CastOptions::default()).unwrap(); + let b = cast( + &array, + &DataType::Decimal(DecimalType::Int128, 2, 1), + CastOptions::default(), + ) + .unwrap(); let c = b.as_any().downcast_ref::>().unwrap(); let expected = Int128Array::from(&[Some(20), Some(100), Some(-20), Some(-100), None]) - .to(DataType::Decimal(2, 1)); + .to(DataType::Decimal(DecimalType::Int128, 2, 1)); assert_eq!(c, &expected) } @@ -311,13 +331,18 @@ fn decimal_to_decimal_scaled() { // decrease precision // 10 and -10 can't be represented with precision 1 and scale 1 let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(1, 0)); + .to(DataType::Decimal(DecimalType::Int128, 1, 0)); - let b = cast(&array, &DataType::Decimal(1, 1), CastOptions::default()).unwrap(); + let b = cast( + &array, + &DataType::Decimal(DecimalType::Int128, 1, 1), + CastOptions::default(), + ) + .unwrap(); let c = b.as_any().downcast_ref::>().unwrap(); - let expected = - Int128Array::from(&[Some(20), None, Some(-20), None, None]).to(DataType::Decimal(1, 1)); + let expected = Int128Array::from(&[Some(20), None, Some(-20), None, None]) + .to(DataType::Decimal(DecimalType::Int128, 1, 1)); assert_eq!(c, &expected) } @@ -326,20 +351,25 @@ fn decimal_to_decimal_fast() { // increase precision // 10 and -10 can't be represented with precision 1 and scale 1 let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(1, 1)); + .to(DataType::Decimal(DecimalType::Int128, 1, 1)); - let b = cast(&array, &DataType::Decimal(2, 1), CastOptions::default()).unwrap(); + let b = cast( + &array, + &DataType::Decimal(DecimalType::Int128, 2, 1), + CastOptions::default(), + ) + .unwrap(); let c = b.as_any().downcast_ref::>().unwrap(); let expected = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(2, 1)); + .to(DataType::Decimal(DecimalType::Int128, 2, 1)); assert_eq!(c, &expected) } #[test] fn decimal_to_float() { let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None]) - .to(DataType::Decimal(2, 1)); + .to(DataType::Decimal(DecimalType::Int128, 2, 1)); let b = cast(&array, &DataType::Float32, CastOptions::default()).unwrap(); let c = b.as_any().downcast_ref::>().unwrap(); @@ -351,7 +381,7 @@ fn decimal_to_float() { #[test] fn decimal_to_integer() { let array = Int128Array::from(&[Some(2), Some(10), Some(-2), Some(-10), None, Some(2560)]) - .to(DataType::Decimal(2, 1)); + .to(DataType::Decimal(DecimalType::Int128, 2, 1)); let b = cast(&array, &DataType::Int8, CastOptions::default()).unwrap(); let c = b.as_any().downcast_ref::>().unwrap(); @@ -459,8 +489,8 @@ fn consistency() { Date32, Time32(TimeUnit::Second), Time32(TimeUnit::Millisecond), - Decimal(1, 2), - Decimal(2, 2), + Decimal(DecimalType::Int128, 1, 2), + Decimal(DecimalType::Int128, 2, 2), Date64, Utf8, LargeUtf8, diff --git a/tests/it/ffi/data.rs b/tests/it/ffi/data.rs index 337326b6319..e08dda944aa 100644 --- a/tests/it/ffi/data.rs +++ b/tests/it/ffi/data.rs @@ -1,6 +1,6 @@ use arrow2::array::*; use arrow2::bitmap::Bitmap; -use arrow2::datatypes::{DataType, Field, TimeUnit}; +use arrow2::datatypes::{DataType, DecimalType, Field, TimeUnit}; use arrow2::{error::Result, ffi}; use std::collections::BTreeMap; @@ -84,8 +84,28 @@ fn decimal() -> Result<()> { } #[test] -fn decimal_nullable() -> Result<()> { - let data = Int128Array::from(&[Some(1), None, Some(2), None]); +fn decimal128() -> Result<()> { + let data = Int128Array::from(&[Some(2), None, Some(1), None]); + test_round_trip(data) +} + +#[test] +fn decimal64() -> Result<()> { + let data = Int64Array::from(&[Some(2), None, Some(1), None]).to(DataType::Decimal( + DecimalType::Int64, + 2, + 2, + )); + test_round_trip(data) +} + +#[test] +fn decimal32() -> Result<()> { + let data = Int32Array::from(&[Some(2), None, Some(1), None]).to(DataType::Decimal( + DecimalType::Int32, + 2, + 2, + )); test_round_trip(data) } diff --git a/tests/it/io/avro/read.rs b/tests/it/io/avro/read.rs index deb8ffd802f..40f11230481 100644 --- a/tests/it/io/avro/read.rs +++ b/tests/it/io/avro/read.rs @@ -75,7 +75,11 @@ pub(super) fn schema() -> (AvroSchema, Schema) { DataType::Dictionary(i32::KEY_TYPE, Box::new(DataType::Utf8), false), false, ), - Field::new("decimal", DataType::Decimal(18, 5), false), + Field::new( + "decimal", + DataType::Decimal(DecimalType::Int128, 18, 5), + false, + ), ]); (AvroSchema::parse_str(raw_schema).unwrap(), schema) @@ -108,6 +112,7 @@ pub(super) fn data() -> Chunk> { .boxed(), DictionaryArray::try_from_keys( Int32Array::from_slice([1, 0]), +<<<<<<< HEAD Box::new(Utf8Array::::from_slice(["SPADES", "HEARTS"])), ) .unwrap() @@ -115,6 +120,14 @@ pub(super) fn data() -> Chunk> { PrimitiveArray::::from_slice([12345678i128, -12345678i128]) .to(DataType::Decimal(18, 5)) .boxed(), +======= + Arc::new(Utf8Array::::from_slice(["SPADES", "HEARTS"])), + )), + Arc::new( + PrimitiveArray::::from_slice([12345678i128, -12345678i128]) + .to(DataType::Decimal(DecimalType::Int128, 18, 5)), + ), +>>>>>>> cfb7ff549a (Added support for decimal 32 and 64) ]; Chunk::try_new(columns).unwrap() diff --git a/tests/it/io/csv/read.rs b/tests/it/io/csv/read.rs index e7e732a506c..b67e986c68b 100644 --- a/tests/it/io/csv/read.rs +++ b/tests/it/io/csv/read.rs @@ -288,27 +288,45 @@ fn date64() -> Result<()> { #[test] fn decimal() -> Result<()> { - let result = test_deserialize("1.1,\n1.2,\n1.22,\n1.3,\n", DataType::Decimal(2, 1))?; - let expected = - Int128Array::from(&[Some(11), Some(12), None, Some(13)]).to(DataType::Decimal(2, 1)); + let result = test_deserialize( + "1.1,\n1.2,\n1.22,\n1.3,\n", + DataType::Decimal(DecimalType::Int128, 2, 1), + )?; + let expected = Int128Array::from(&[Some(11), Some(12), None, Some(13)]).to(DataType::Decimal( + DecimalType::Int128, + 2, + 1, + )); assert_eq!(expected, result.as_ref()); Ok(()) } #[test] fn decimal_only_scale() -> Result<()> { - let result = test_deserialize("0.01,\n0.12,\n0.222,\n0.13,\n", DataType::Decimal(2, 2))?; - let expected = - Int128Array::from(&[Some(1), Some(12), None, Some(13)]).to(DataType::Decimal(2, 2)); + let result = test_deserialize( + "0.01,\n0.12,\n0.222,\n0.13,\n", + DataType::Decimal(DecimalType::Int128, 2, 2), + )?; + let expected = Int128Array::from(&[Some(1), Some(12), None, Some(13)]).to(DataType::Decimal( + DecimalType::Int128, + 2, + 2, + )); assert_eq!(expected, result.as_ref()); Ok(()) } #[test] fn decimal_only_integer() -> Result<()> { - let result = test_deserialize("1,\n1.0,\n1.1,\n10.0,\n", DataType::Decimal(1, 0))?; - let expected = - Int128Array::from(&[Some(1), Some(1), None, Some(10)]).to(DataType::Decimal(1, 0)); + let result = test_deserialize( + "1,\n1.0,\n1.1,\n10.0,\n", + DataType::Decimal(DecimalType::Int128, 1, 0), + )?; + let expected = Int128Array::from(&[Some(1), Some(1), None, Some(10)]).to(DataType::Decimal( + DecimalType::Int128, + 1, + 0, + )); assert_eq!(expected, result.as_ref()); Ok(()) } diff --git a/tests/it/io/ipc/write/file.rs b/tests/it/io/ipc/write/file.rs index a899aa459d0..2baafd7e538 100644 --- a/tests/it/io/ipc/write/file.rs +++ b/tests/it/io/ipc/write/file.rs @@ -2,7 +2,7 @@ use std::io::Cursor; use arrow2::array::*; use arrow2::chunk::Chunk; -use arrow2::datatypes::{Field, Schema}; +use arrow2::datatypes::{DataType, DecimalType, Field, Schema}; use arrow2::error::Result; use arrow2::io::ipc::read::{read_file_metadata, FileReader}; use arrow2::io::ipc::{write::*, IpcField}; @@ -363,12 +363,22 @@ fn write_sliced_list() -> Result<()> { #[test] fn write_months_days_ns() -> Result<()> { - let array = Box::new(MonthsDaysNsArray::from([ + let array = MonthsDaysNsArray::from([ Some(months_days_ns::new(1, 1, 0)), Some(months_days_ns::new(1, 1, 1)), None, Some(months_days_ns::new(1, 1, 2)), - ])) as Box; + ]) + .boxed(); + let schema = Schema::from(vec![Field::new("a", array.data_type().clone(), true)]); + let columns = Chunk::try_new(vec![array])?; + round_trip(columns, schema, None, None) +} + +fn write_decimali32() -> Result<()> { + let array = Int32Array::from([Some(1), Some(2), None, Some(4)]) + .to(DataType::Decimal(DecimalType::Int32, 2, 2)) + .boxed(); let schema = Schema::from(vec![Field::new("a", array.data_type().clone(), true)]); let columns = Chunk::try_new(vec![array])?; round_trip(columns, schema, None, None) diff --git a/tests/it/io/parquet/mod.rs b/tests/it/io/parquet/mod.rs index 6710bf57696..edafcd2ffb4 100644 --- a/tests/it/io/parquet/mod.rs +++ b/tests/it/io/parquet/mod.rs @@ -381,21 +381,33 @@ pub fn pyarrow_nullable(column: &str) -> Box { .iter() .map(|x| x.map(|x| x as i128)) .collect::>(); - Box::new(PrimitiveArray::::from(values).to(DataType::Decimal(9, 0))) + Box::new(PrimitiveArray::::from(values).to(DataType::Decimal( + DecimalType::Int128, + 9, + 0, + ))) } "decimal_18" => { let values = i64_values .iter() .map(|x| x.map(|x| x as i128)) .collect::>(); - Box::new(PrimitiveArray::::from(values).to(DataType::Decimal(18, 0))) + Box::new(PrimitiveArray::::from(values).to(DataType::Decimal( + DecimalType::Int128, + 18, + 0, + ))) } "decimal_26" => { let values = i64_values .iter() .map(|x| x.map(|x| x as i128)) .collect::>(); - Box::new(PrimitiveArray::::from(values).to(DataType::Decimal(26, 0))) + Box::new(PrimitiveArray::::from(values).to(DataType::Decimal( + DecimalType::Int128, + 26, + 0, + ))) } "timestamp_us" => Box::new( PrimitiveArray::::from(i64_values) @@ -468,20 +480,44 @@ pub fn pyarrow_nullable_statistics(column: &str) -> Statistics { "decimal_9" => Statistics { distinct_count: Count::Single(UInt64Array::from([None])), null_count: Count::Single(UInt64Array::from([Some(3)])), - min_value: Box::new(Int128Array::from_slice([-256]).to(DataType::Decimal(9, 0))), - max_value: Box::new(Int128Array::from_slice([9]).to(DataType::Decimal(9, 0))), + min_value: Box::new(Int128Array::from_slice([-256]).to(DataType::Decimal( + DecimalType::Int128, + 9, + 0, + ))), + max_value: Box::new(Int128Array::from_slice([9]).to(DataType::Decimal( + DecimalType::Int128, + 9, + 0, + ))), }, "decimal_18" => Statistics { distinct_count: Count::Single(UInt64Array::from([None])), null_count: Count::Single(UInt64Array::from([Some(3)])), - min_value: Box::new(Int128Array::from_slice([-256]).to(DataType::Decimal(18, 0))), - max_value: Box::new(Int128Array::from_slice([9]).to(DataType::Decimal(18, 0))), + min_value: Box::new(Int128Array::from_slice([-256]).to(DataType::Decimal( + DecimalType::Int128, + 18, + 0, + ))), + max_value: Box::new(Int128Array::from_slice([9]).to(DataType::Decimal( + DecimalType::Int128, + 18, + 0, + ))), }, "decimal_26" => Statistics { distinct_count: Count::Single(UInt64Array::from([None])), null_count: Count::Single(UInt64Array::from([Some(3)])), - min_value: Box::new(Int128Array::from_slice([-256]).to(DataType::Decimal(26, 0))), - max_value: Box::new(Int128Array::from_slice([9]).to(DataType::Decimal(26, 0))), + min_value: Box::new(Int128Array::from_slice([-256]).to(DataType::Decimal( + DecimalType::Int128, + 26, + 0, + ))), + max_value: Box::new(Int128Array::from_slice([9]).to(DataType::Decimal( + DecimalType::Int128, + 26, + 0, + ))), }, "timestamp_us" => Statistics { distinct_count: Count::Single(UInt64Array::from([None])), @@ -547,21 +583,33 @@ pub fn pyarrow_required(column: &str) -> Box { .iter() .map(|x| x.map(|x| x as i128)) .collect::>(); - Box::new(PrimitiveArray::::from(values).to(DataType::Decimal(9, 0))) + Box::new(PrimitiveArray::::from(values).to(DataType::Decimal( + DecimalType::Int128, + 9, + 0, + ))) } "decimal_18" => { let values = i64_values .iter() .map(|x| x.map(|x| x as i128)) .collect::>(); - Box::new(PrimitiveArray::::from(values).to(DataType::Decimal(18, 0))) + Box::new(PrimitiveArray::::from(values).to(DataType::Decimal( + DecimalType::Int128, + 18, + 0, + ))) } "decimal_26" => { let values = i64_values .iter() .map(|x| x.map(|x| x as i128)) .collect::>(); - Box::new(PrimitiveArray::::from(values).to(DataType::Decimal(26, 0))) + Box::new(PrimitiveArray::::from(values).to(DataType::Decimal( + DecimalType::Int128, + 26, + 0, + ))) } _ => unreachable!(), }