Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use new DecimalArray creation API in arrow crate #1249

Merged
merged 9 commits into from
Feb 15, 2022
Merged

Use new DecimalArray creation API in arrow crate #1249

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
c1b25ca
Use new API in ffi.rs
alamb Jan 30, 2022
5756722
Use new API in sort.rs
alamb Jan 30, 2022
df712ec
Use new API in pretty.rs
alamb Jan 30, 2022
8dbfb25
Use new API in array_binary.rs
alamb Jan 30, 2022
55d7513
Use new API in equal_json.rs
alamb Jan 30, 2022
1aff394
Use new API in take.rs
alamb Jan 30, 2022
1b72619
Use new API in cast.rs
alamb Jan 30, 2022
d0082f8
Use new API in equal.rs
alamb Jan 30, 2022
11f1832
clippy
alamb Jan 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 18 additions & 13 deletions arrow/src/array/array_binary.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,7 +702,7 @@ impl Array for FixedSizeBinaryArray {
/// # Examples
///
/// ```
/// use arrow::array::{Array, DecimalArray, DecimalBuilder};
/// use arrow::array::{Array, DecimalArray};
/// use arrow::datatypes::DataType;
///
/// // Create a DecimalArray with the default precision and scale
Expand Down Expand Up @@ -943,6 +943,12 @@ impl From<ArrayData> for DecimalArray {
}
}

impl From<DecimalArray> for ArrayData {
fn from(array: DecimalArray) -> Self {
array.data
}
}

/// an iterator that returns Some(i128) or None, that can be used on a
/// DecimalArray
#[derive(Debug)]
Expand Down Expand Up @@ -1573,11 +1579,12 @@ mod tests {

#[test]
fn test_decimal_array_value_as_string() {
let mut decimal_builder = DecimalBuilder::new(7, 6, 3);
for value in [123450, -123450, 100, -100, 10, -10, 0] {
decimal_builder.append_value(value).unwrap();
}
let arr = decimal_builder.finish();
let arr = [123450, -123450, 100, -100, 10, -10, 0]
.into_iter()
.map(Some)
.collect::<DecimalArray>()
.with_precision_and_scale(6, 3)
.unwrap();

assert_eq!("123.450", arr.value_as_string(0));
assert_eq!("-123.450", arr.value_as_string(1));
Expand Down Expand Up @@ -1641,14 +1648,12 @@ mod tests {

#[test]
fn test_decimal_array_fmt_debug() {
let values: Vec<i128> = vec![8887000000, -8887000000];
let mut decimal_builder = DecimalBuilder::new(3, 23, 6);
let arr = [Some(8887000000), Some(-8887000000), None]
.iter()
.collect::<DecimalArray>()
.with_precision_and_scale(23, 6)
.unwrap();

values.iter().for_each(|&value| {
decimal_builder.append_value(value).unwrap();
});
decimal_builder.append_null().unwrap();
let arr = decimal_builder.finish();
assert_eq!(
"DecimalArray<23, 6>\n[\n 8887.000000,\n -8887.000000,\n null,\n]",
format!("{:?}", arr)
Expand Down
21 changes: 8 additions & 13 deletions arrow/src/array/equal/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,9 +305,9 @@ mod tests {

use crate::array::{
array::Array, ArrayData, ArrayDataBuilder, ArrayRef, BinaryOffsetSizeTrait,
BooleanArray, DecimalBuilder, FixedSizeBinaryBuilder, FixedSizeListBuilder,
GenericBinaryArray, Int32Builder, ListBuilder, NullArray, PrimitiveBuilder,
StringArray, StringDictionaryBuilder, StringOffsetSizeTrait, StructArray,
BooleanArray, FixedSizeBinaryBuilder, FixedSizeListBuilder, GenericBinaryArray,
Int32Builder, ListBuilder, NullArray, PrimitiveBuilder, StringArray,
StringDictionaryBuilder, StringOffsetSizeTrait, StructArray,
};
use crate::array::{GenericStringArray, Int32Array};
use crate::buffer::Buffer;
Expand Down Expand Up @@ -810,16 +810,11 @@ mod tests {
}

fn create_decimal_array(data: &[Option<i128>]) -> ArrayData {
let mut builder = DecimalBuilder::new(20, 23, 6);

for d in data {
if let Some(v) = d {
builder.append_value(*v).unwrap();
} else {
builder.append_null().unwrap();
}
}
builder.finish().data().clone()
data.iter()
.collect::<DecimalArray>()
.with_precision_and_scale(23, 6)
.unwrap()
.into()
}

#[test]
Expand Down
19 changes: 10 additions & 9 deletions arrow/src/array/equal_json.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -937,11 +937,11 @@ mod tests {
#[test]
fn test_decimal_json_equal() {
// Test the equal case
let mut builder = DecimalBuilder::new(30, 23, 6);
builder.append_value(1_000).unwrap();
builder.append_null().unwrap();
builder.append_value(-250).unwrap();
let arrow_array: DecimalArray = builder.finish();
let arrow_array = [Some(1_000), None, Some(-250)]
.iter()
.collect::<DecimalArray>()
.with_precision_and_scale(23, 6)
.unwrap();
let json_array: Value = serde_json::from_str(
r#"
[
Expand All @@ -956,10 +956,11 @@ mod tests {
assert!(json_array.eq(&arrow_array));

// Test unequal case
builder.append_value(1_000).unwrap();
builder.append_null().unwrap();
builder.append_value(55).unwrap();
let arrow_array: DecimalArray = builder.finish();
let arrow_array = [Some(1_000), None, Some(55)]
.iter()
.collect::<DecimalArray>()
.with_precision_and_scale(23, 6)
.unwrap();
let json_array: Value = serde_json::from_str(
r#"
[
Expand Down
19 changes: 6 additions & 13 deletions arrow/src/array/transform/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -670,7 +670,7 @@ mod tests {

use super::*;

use crate::array::{DecimalArray, DecimalBuilder};
use crate::array::DecimalArray;
use crate::{
array::{
Array, ArrayData, ArrayRef, BooleanArray, DictionaryArray,
Expand All @@ -691,18 +691,11 @@ mod tests {
precision: usize,
scale: usize,
) -> DecimalArray {
let mut decimal_builder = DecimalBuilder::new(array.len(), precision, scale);
for value in array {
match value {
None => {
decimal_builder.append_null().unwrap();
}
Some(v) => {
decimal_builder.append_value(*v).unwrap();
}
}
}
decimal_builder.finish()
array
.iter()
.collect::<DecimalArray>()
.with_precision_and_scale(precision, scale)
.unwrap()
}

#[test]
Expand Down
96 changes: 42 additions & 54 deletions arrow/src/compute/kernels/cast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,39 +260,41 @@ pub fn cast(array: &ArrayRef, to_type: &DataType) -> Result<ArrayRef> {
// cast the integer array to defined decimal data type array
macro_rules! cast_integer_to_decimal {
($ARRAY: expr, $ARRAY_TYPE: ident, $PRECISION : ident, $SCALE : ident) => {{
let mut decimal_builder = DecimalBuilder::new($ARRAY.len(), *$PRECISION, *$SCALE);
let array = $ARRAY.as_any().downcast_ref::<$ARRAY_TYPE>().unwrap();
let mul: i128 = 10_i128.pow(*$SCALE as u32);
for i in 0..array.len() {
if array.is_null(i) {
decimal_builder.append_null()?;
} else {
// convert i128 first
let v = array.value(i) as i128;
// if the input value is overflow, it will throw an error.
decimal_builder.append_value(mul * v)?;
}
}
Ok(Arc::new(decimal_builder.finish()))
let decimal_array = array
.iter()
.map(|v| {
v.map(|v| {
let v = v as i128;
// with_precision_and_scale validates the
// value is within range for the output precision
mul * v
})
})
.collect::<DecimalArray>()
.with_precision_and_scale(*$PRECISION, *$SCALE)?;
Ok(Arc::new(decimal_array))
}};
}

// cast the floating-point array to defined decimal data type array
macro_rules! cast_floating_point_to_decimal {
($ARRAY: expr, $ARRAY_TYPE: ident, $PRECISION : ident, $SCALE : ident) => {{
let mut decimal_builder = DecimalBuilder::new($ARRAY.len(), *$PRECISION, *$SCALE);
let array = $ARRAY.as_any().downcast_ref::<$ARRAY_TYPE>().unwrap();
let mul = 10_f64.powi(*$SCALE as i32);
for i in 0..array.len() {
if array.is_null(i) {
decimal_builder.append_null()?;
} else {
let v = ((array.value(i) as f64) * mul) as i128;
// if the input value is overflow, it will throw an error.
decimal_builder.append_value(v)?;
}
}
Ok(Arc::new(decimal_builder.finish()))
let decimal_array = array
.iter()
.map(|v| {
v.map(|v| {
// with_precision_and_scale validates the
// value is within range for the output precision
((v as f64) * mul) as i128
})
})
.collect::<DecimalArray>()
.with_precision_and_scale(*$PRECISION, *$SCALE)?;
Ok(Arc::new(decimal_array))
}};
}

Expand Down Expand Up @@ -1166,33 +1168,28 @@ fn cast_decimal_to_decimal(
output_precision: &usize,
output_scale: &usize,
) -> Result<ArrayRef> {
let mut decimal_builder =
DecimalBuilder::new(array.len(), *output_precision, *output_scale);
let array = array.as_any().downcast_ref::<DecimalArray>().unwrap();
if input_scale > output_scale {

let output_array = if input_scale > output_scale {
// For example, input_scale is 4 and output_scale is 3;
// Original value is 11234_i128, and will be cast to 1123_i128.
let div = 10_i128.pow((input_scale - output_scale) as u32);
for i in 0..array.len() {
if array.is_null(i) {
decimal_builder.append_null()?;
} else {
decimal_builder.append_value(array.value(i) / div)?;
}
}
array
.iter()
.map(|v| v.map(|v| v / div))
.collect::<DecimalArray>()
} else {
// For example, input_scale is 3 and output_scale is 4;
// Original value is 1123_i128, and will be cast to 11230_i128.
let mul = 10_i128.pow((output_scale - input_scale) as u32);
for i in 0..array.len() {
if array.is_null(i) {
decimal_builder.append_null()?;
} else {
decimal_builder.append_value(array.value(i) * mul)?;
}
}
array
.iter()
.map(|v| v.map(|v| v * mul))
.collect::<DecimalArray>()
}
Ok(Arc::new(decimal_builder.finish()))
.with_precision_and_scale(*output_precision, *output_scale)?;

Ok(Arc::new(output_array))
}

/// Cast an array by changing its array_data type to the desired type
Expand Down Expand Up @@ -2071,24 +2068,15 @@ mod tests {
};
}

// TODO remove this function if the decimal array has the creator function
fn create_decimal_array(
array: &[Option<i128>],
precision: usize,
scale: usize,
) -> Result<DecimalArray> {
let mut decimal_builder = DecimalBuilder::new(array.len(), precision, scale);
for value in array {
match value {
None => {
decimal_builder.append_null()?;
}
Some(v) => {
decimal_builder.append_value(*v)?;
}
}
}
Ok(decimal_builder.finish())
array
.iter()
.collect::<DecimalArray>()
.with_precision_and_scale(precision, scale)
}

#[test]
Expand Down
14 changes: 4 additions & 10 deletions arrow/src/compute/kernels/sort.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1107,16 +1107,10 @@ mod tests {
use std::sync::Arc;

fn create_decimal_array(data: &[Option<i128>]) -> DecimalArray {
let mut builder = DecimalBuilder::new(20, 23, 6);

for d in data {
if let Some(v) = d {
builder.append_value(*v).unwrap();
} else {
builder.append_null().unwrap();
}
}
builder.finish()
data.iter()
.collect::<DecimalArray>()
.with_precision_and_scale(23, 6)
.unwrap()
}

fn test_sort_to_indices_decimal_array(
Expand Down
Loading