/Users/andrewlamb/Software/datafusion/datafusion/physical-expr/src/expressions/try_cast.rs
Line | Count | Source (jump to first uncovered line) |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | use std::any::Any; |
19 | | use std::fmt; |
20 | | use std::hash::{Hash, Hasher}; |
21 | | use std::sync::Arc; |
22 | | |
23 | | use crate::physical_expr::down_cast_any_ref; |
24 | | use crate::PhysicalExpr; |
25 | | use arrow::compute; |
26 | | use arrow::compute::{cast_with_options, CastOptions}; |
27 | | use arrow::datatypes::{DataType, Schema}; |
28 | | use arrow::record_batch::RecordBatch; |
29 | | use compute::can_cast_types; |
30 | | use datafusion_common::format::DEFAULT_FORMAT_OPTIONS; |
31 | | use datafusion_common::{not_impl_err, Result, ScalarValue}; |
32 | | use datafusion_expr::ColumnarValue; |
33 | | |
34 | | /// TRY_CAST expression casts an expression to a specific data type and returns NULL on invalid cast |
35 | | #[derive(Debug, Hash)] |
36 | | pub struct TryCastExpr { |
37 | | /// The expression to cast |
38 | | expr: Arc<dyn PhysicalExpr>, |
39 | | /// The data type to cast to |
40 | | cast_type: DataType, |
41 | | } |
42 | | |
43 | | impl TryCastExpr { |
44 | | /// Create a new CastExpr |
45 | 0 | pub fn new(expr: Arc<dyn PhysicalExpr>, cast_type: DataType) -> Self { |
46 | 0 | Self { expr, cast_type } |
47 | 0 | } |
48 | | |
49 | | /// The expression to cast |
50 | 0 | pub fn expr(&self) -> &Arc<dyn PhysicalExpr> { |
51 | 0 | &self.expr |
52 | 0 | } |
53 | | |
54 | | /// The data type to cast to |
55 | 0 | pub fn cast_type(&self) -> &DataType { |
56 | 0 | &self.cast_type |
57 | 0 | } |
58 | | } |
59 | | |
60 | | impl fmt::Display for TryCastExpr { |
61 | 0 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
62 | 0 | write!(f, "TRY_CAST({} AS {:?})", self.expr, self.cast_type) |
63 | 0 | } |
64 | | } |
65 | | |
66 | | impl PhysicalExpr for TryCastExpr { |
67 | | /// Return a reference to Any that can be used for downcasting |
68 | 0 | fn as_any(&self) -> &dyn Any { |
69 | 0 | self |
70 | 0 | } |
71 | | |
72 | 0 | fn data_type(&self, _input_schema: &Schema) -> Result<DataType> { |
73 | 0 | Ok(self.cast_type.clone()) |
74 | 0 | } |
75 | | |
76 | 0 | fn nullable(&self, _input_schema: &Schema) -> Result<bool> { |
77 | 0 | Ok(true) |
78 | 0 | } |
79 | | |
80 | 0 | fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> { |
81 | 0 | let value = self.expr.evaluate(batch)?; |
82 | 0 | let options = CastOptions { |
83 | 0 | safe: true, |
84 | 0 | format_options: DEFAULT_FORMAT_OPTIONS, |
85 | 0 | }; |
86 | 0 | match value { |
87 | 0 | ColumnarValue::Array(array) => { |
88 | 0 | let cast = cast_with_options(&array, &self.cast_type, &options)?; |
89 | 0 | Ok(ColumnarValue::Array(cast)) |
90 | | } |
91 | 0 | ColumnarValue::Scalar(scalar) => { |
92 | 0 | let array = scalar.to_array()?; |
93 | 0 | let cast_array = cast_with_options(&array, &self.cast_type, &options)?; |
94 | 0 | let cast_scalar = ScalarValue::try_from_array(&cast_array, 0)?; |
95 | 0 | Ok(ColumnarValue::Scalar(cast_scalar)) |
96 | | } |
97 | | } |
98 | 0 | } |
99 | | |
100 | 0 | fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> { |
101 | 0 | vec![&self.expr] |
102 | 0 | } |
103 | | |
104 | 0 | fn with_new_children( |
105 | 0 | self: Arc<Self>, |
106 | 0 | children: Vec<Arc<dyn PhysicalExpr>>, |
107 | 0 | ) -> Result<Arc<dyn PhysicalExpr>> { |
108 | 0 | Ok(Arc::new(TryCastExpr::new( |
109 | 0 | Arc::clone(&children[0]), |
110 | 0 | self.cast_type.clone(), |
111 | 0 | ))) |
112 | 0 | } |
113 | | |
114 | 0 | fn dyn_hash(&self, state: &mut dyn Hasher) { |
115 | 0 | let mut s = state; |
116 | 0 | self.hash(&mut s); |
117 | 0 | } |
118 | | } |
119 | | |
120 | | impl PartialEq<dyn Any> for TryCastExpr { |
121 | 0 | fn eq(&self, other: &dyn Any) -> bool { |
122 | 0 | down_cast_any_ref(other) |
123 | 0 | .downcast_ref::<Self>() |
124 | 0 | .map(|x| self.expr.eq(&x.expr) && self.cast_type == x.cast_type) |
125 | 0 | .unwrap_or(false) |
126 | 0 | } |
127 | | } |
128 | | |
129 | | /// Return a PhysicalExpression representing `expr` casted to |
130 | | /// `cast_type`, if any casting is needed. |
131 | | /// |
132 | | /// Note that such casts may lose type information |
133 | 0 | pub fn try_cast( |
134 | 0 | expr: Arc<dyn PhysicalExpr>, |
135 | 0 | input_schema: &Schema, |
136 | 0 | cast_type: DataType, |
137 | 0 | ) -> Result<Arc<dyn PhysicalExpr>> { |
138 | 0 | let expr_type = expr.data_type(input_schema)?; |
139 | 0 | if expr_type == cast_type { |
140 | 0 | Ok(Arc::clone(&expr)) |
141 | 0 | } else if can_cast_types(&expr_type, &cast_type) { |
142 | 0 | Ok(Arc::new(TryCastExpr::new(expr, cast_type))) |
143 | | } else { |
144 | 0 | not_impl_err!("Unsupported TRY_CAST from {expr_type:?} to {cast_type:?}") |
145 | | } |
146 | 0 | } |
147 | | |
148 | | #[cfg(test)] |
149 | | mod tests { |
150 | | use super::*; |
151 | | use crate::expressions::col; |
152 | | use arrow::array::{ |
153 | | Decimal128Array, Decimal128Builder, StringArray, Time64NanosecondArray, |
154 | | }; |
155 | | use arrow::{ |
156 | | array::{ |
157 | | Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, |
158 | | Int8Array, TimestampNanosecondArray, UInt32Array, |
159 | | }, |
160 | | datatypes::*, |
161 | | }; |
162 | | |
163 | | // runs an end-to-end test of physical type cast |
164 | | // 1. construct a record batch with a column "a" of type A |
165 | | // 2. construct a physical expression of TRY_CAST(a AS B) |
166 | | // 3. evaluate the expression |
167 | | // 4. verify that the resulting expression is of type B |
168 | | // 5. verify that the resulting values are downcastable and correct |
169 | | macro_rules! generic_decimal_to_other_test_cast { |
170 | | ($DECIMAL_ARRAY:ident, $A_TYPE:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{ |
171 | | let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]); |
172 | | let batch = RecordBatch::try_new( |
173 | | Arc::new(schema.clone()), |
174 | | vec![Arc::new($DECIMAL_ARRAY)], |
175 | | )?; |
176 | | // verify that we can construct the expression |
177 | | let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?; |
178 | | |
179 | | // verify that its display is correct |
180 | | assert_eq!( |
181 | | format!("TRY_CAST(a@0 AS {:?})", $TYPE), |
182 | | format!("{}", expression) |
183 | | ); |
184 | | |
185 | | // verify that the expression's type is correct |
186 | | assert_eq!(expression.data_type(&schema)?, $TYPE); |
187 | | |
188 | | // compute |
189 | | let result = expression |
190 | | .evaluate(&batch)? |
191 | | .into_array(batch.num_rows()) |
192 | | .expect("Failed to convert to array"); |
193 | | |
194 | | // verify that the array's data_type is correct |
195 | | assert_eq!(*result.data_type(), $TYPE); |
196 | | |
197 | | // verify that the data itself is downcastable |
198 | | let result = result |
199 | | .as_any() |
200 | | .downcast_ref::<$TYPEARRAY>() |
201 | | .expect("failed to downcast"); |
202 | | |
203 | | // verify that the result itself is correct |
204 | | for (i, x) in $VEC.iter().enumerate() { |
205 | | match x { |
206 | | Some(x) => assert_eq!(result.value(i), *x), |
207 | | None => assert!(!result.is_valid(i)), |
208 | | } |
209 | | } |
210 | | }}; |
211 | | } |
212 | | |
213 | | // runs an end-to-end test of physical type cast |
214 | | // 1. construct a record batch with a column "a" of type A |
215 | | // 2. construct a physical expression of TRY_CAST(a AS B) |
216 | | // 3. evaluate the expression |
217 | | // 4. verify that the resulting expression is of type B |
218 | | // 5. verify that the resulting values are downcastable and correct |
219 | | macro_rules! generic_test_cast { |
220 | | ($A_ARRAY:ident, $A_TYPE:expr, $A_VEC:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{ |
221 | | let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]); |
222 | | let a_vec_len = $A_VEC.len(); |
223 | | let a = $A_ARRAY::from($A_VEC); |
224 | | let batch = |
225 | | RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?; |
226 | | |
227 | | // verify that we can construct the expression |
228 | | let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?; |
229 | | |
230 | | // verify that its display is correct |
231 | | assert_eq!( |
232 | | format!("TRY_CAST(a@0 AS {:?})", $TYPE), |
233 | | format!("{}", expression) |
234 | | ); |
235 | | |
236 | | // verify that the expression's type is correct |
237 | | assert_eq!(expression.data_type(&schema)?, $TYPE); |
238 | | |
239 | | // compute |
240 | | let result = expression |
241 | | .evaluate(&batch)? |
242 | | .into_array(batch.num_rows()) |
243 | | .expect("Failed to convert to array"); |
244 | | |
245 | | // verify that the array's data_type is correct |
246 | | assert_eq!(*result.data_type(), $TYPE); |
247 | | |
248 | | // verify that the len is correct |
249 | | assert_eq!(result.len(), a_vec_len); |
250 | | |
251 | | // verify that the data itself is downcastable |
252 | | let result = result |
253 | | .as_any() |
254 | | .downcast_ref::<$TYPEARRAY>() |
255 | | .expect("failed to downcast"); |
256 | | |
257 | | // verify that the result itself is correct |
258 | | for (i, x) in $VEC.iter().enumerate() { |
259 | | match x { |
260 | | Some(x) => assert_eq!(result.value(i), *x), |
261 | | None => assert!(!result.is_valid(i)), |
262 | | } |
263 | | } |
264 | | }}; |
265 | | } |
266 | | |
267 | | #[test] |
268 | | fn test_try_cast_decimal_to_decimal() -> Result<()> { |
269 | | // try cast one decimal data type to another decimal data type |
270 | | let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000]; |
271 | | let decimal_array = create_decimal_array(&array, 10, 3); |
272 | | generic_decimal_to_other_test_cast!( |
273 | | decimal_array, |
274 | | DataType::Decimal128(10, 3), |
275 | | Decimal128Array, |
276 | | DataType::Decimal128(20, 6), |
277 | | [ |
278 | | Some(1_234_000), |
279 | | Some(2_222_000), |
280 | | Some(3_000), |
281 | | Some(4_000_000), |
282 | | Some(5_000_000), |
283 | | None |
284 | | ] |
285 | | ); |
286 | | |
287 | | let decimal_array = create_decimal_array(&array, 10, 3); |
288 | | generic_decimal_to_other_test_cast!( |
289 | | decimal_array, |
290 | | DataType::Decimal128(10, 3), |
291 | | Decimal128Array, |
292 | | DataType::Decimal128(10, 2), |
293 | | [Some(123), Some(222), Some(0), Some(400), Some(500), None] |
294 | | ); |
295 | | |
296 | | Ok(()) |
297 | | } |
298 | | |
299 | | #[test] |
300 | | fn test_try_cast_decimal_to_numeric() -> Result<()> { |
301 | | // TODO we should add function to create Decimal128Array with value and metadata |
302 | | // https://github.com/apache/arrow-rs/issues/1009 |
303 | | let array: Vec<i128> = vec![1, 2, 3, 4, 5]; |
304 | | let decimal_array = create_decimal_array(&array, 10, 0); |
305 | | // decimal to i8 |
306 | | generic_decimal_to_other_test_cast!( |
307 | | decimal_array, |
308 | | DataType::Decimal128(10, 0), |
309 | | Int8Array, |
310 | | DataType::Int8, |
311 | | [ |
312 | | Some(1_i8), |
313 | | Some(2_i8), |
314 | | Some(3_i8), |
315 | | Some(4_i8), |
316 | | Some(5_i8), |
317 | | None |
318 | | ] |
319 | | ); |
320 | | |
321 | | // decimal to i16 |
322 | | let decimal_array = create_decimal_array(&array, 10, 0); |
323 | | generic_decimal_to_other_test_cast!( |
324 | | decimal_array, |
325 | | DataType::Decimal128(10, 0), |
326 | | Int16Array, |
327 | | DataType::Int16, |
328 | | [ |
329 | | Some(1_i16), |
330 | | Some(2_i16), |
331 | | Some(3_i16), |
332 | | Some(4_i16), |
333 | | Some(5_i16), |
334 | | None |
335 | | ] |
336 | | ); |
337 | | |
338 | | // decimal to i32 |
339 | | let decimal_array = create_decimal_array(&array, 10, 0); |
340 | | generic_decimal_to_other_test_cast!( |
341 | | decimal_array, |
342 | | DataType::Decimal128(10, 0), |
343 | | Int32Array, |
344 | | DataType::Int32, |
345 | | [ |
346 | | Some(1_i32), |
347 | | Some(2_i32), |
348 | | Some(3_i32), |
349 | | Some(4_i32), |
350 | | Some(5_i32), |
351 | | None |
352 | | ] |
353 | | ); |
354 | | |
355 | | // decimal to i64 |
356 | | let decimal_array = create_decimal_array(&array, 10, 0); |
357 | | generic_decimal_to_other_test_cast!( |
358 | | decimal_array, |
359 | | DataType::Decimal128(10, 0), |
360 | | Int64Array, |
361 | | DataType::Int64, |
362 | | [ |
363 | | Some(1_i64), |
364 | | Some(2_i64), |
365 | | Some(3_i64), |
366 | | Some(4_i64), |
367 | | Some(5_i64), |
368 | | None |
369 | | ] |
370 | | ); |
371 | | |
372 | | // decimal to float32 |
373 | | let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000]; |
374 | | let decimal_array = create_decimal_array(&array, 10, 3); |
375 | | generic_decimal_to_other_test_cast!( |
376 | | decimal_array, |
377 | | DataType::Decimal128(10, 3), |
378 | | Float32Array, |
379 | | DataType::Float32, |
380 | | [ |
381 | | Some(1.234_f32), |
382 | | Some(2.222_f32), |
383 | | Some(0.003_f32), |
384 | | Some(4.0_f32), |
385 | | Some(5.0_f32), |
386 | | None |
387 | | ] |
388 | | ); |
389 | | // decimal to float64 |
390 | | let decimal_array = create_decimal_array(&array, 20, 6); |
391 | | generic_decimal_to_other_test_cast!( |
392 | | decimal_array, |
393 | | DataType::Decimal128(20, 6), |
394 | | Float64Array, |
395 | | DataType::Float64, |
396 | | [ |
397 | | Some(0.001234_f64), |
398 | | Some(0.002222_f64), |
399 | | Some(0.000003_f64), |
400 | | Some(0.004_f64), |
401 | | Some(0.005_f64), |
402 | | None |
403 | | ] |
404 | | ); |
405 | | |
406 | | Ok(()) |
407 | | } |
408 | | |
409 | | #[test] |
410 | | fn test_try_cast_numeric_to_decimal() -> Result<()> { |
411 | | // int8 |
412 | | generic_test_cast!( |
413 | | Int8Array, |
414 | | DataType::Int8, |
415 | | vec![1, 2, 3, 4, 5], |
416 | | Decimal128Array, |
417 | | DataType::Decimal128(3, 0), |
418 | | [Some(1), Some(2), Some(3), Some(4), Some(5)] |
419 | | ); |
420 | | |
421 | | // int16 |
422 | | generic_test_cast!( |
423 | | Int16Array, |
424 | | DataType::Int16, |
425 | | vec![1, 2, 3, 4, 5], |
426 | | Decimal128Array, |
427 | | DataType::Decimal128(5, 0), |
428 | | [Some(1), Some(2), Some(3), Some(4), Some(5)] |
429 | | ); |
430 | | |
431 | | // int32 |
432 | | generic_test_cast!( |
433 | | Int32Array, |
434 | | DataType::Int32, |
435 | | vec![1, 2, 3, 4, 5], |
436 | | Decimal128Array, |
437 | | DataType::Decimal128(10, 0), |
438 | | [Some(1), Some(2), Some(3), Some(4), Some(5)] |
439 | | ); |
440 | | |
441 | | // int64 |
442 | | generic_test_cast!( |
443 | | Int64Array, |
444 | | DataType::Int64, |
445 | | vec![1, 2, 3, 4, 5], |
446 | | Decimal128Array, |
447 | | DataType::Decimal128(20, 0), |
448 | | [Some(1), Some(2), Some(3), Some(4), Some(5)] |
449 | | ); |
450 | | |
451 | | // int64 to different scale |
452 | | generic_test_cast!( |
453 | | Int64Array, |
454 | | DataType::Int64, |
455 | | vec![1, 2, 3, 4, 5], |
456 | | Decimal128Array, |
457 | | DataType::Decimal128(20, 2), |
458 | | [Some(100), Some(200), Some(300), Some(400), Some(500)] |
459 | | ); |
460 | | |
461 | | // float32 |
462 | | generic_test_cast!( |
463 | | Float32Array, |
464 | | DataType::Float32, |
465 | | vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], |
466 | | Decimal128Array, |
467 | | DataType::Decimal128(10, 2), |
468 | | [Some(150), Some(250), Some(300), Some(112), Some(550)] |
469 | | ); |
470 | | |
471 | | // float64 |
472 | | generic_test_cast!( |
473 | | Float64Array, |
474 | | DataType::Float64, |
475 | | vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50], |
476 | | Decimal128Array, |
477 | | DataType::Decimal128(20, 4), |
478 | | [ |
479 | | Some(15000), |
480 | | Some(25000), |
481 | | Some(30000), |
482 | | Some(11235), |
483 | | Some(55000) |
484 | | ] |
485 | | ); |
486 | | Ok(()) |
487 | | } |
488 | | |
489 | | #[test] |
490 | | fn test_cast_i32_u32() -> Result<()> { |
491 | | generic_test_cast!( |
492 | | Int32Array, |
493 | | DataType::Int32, |
494 | | vec![1, 2, 3, 4, 5], |
495 | | UInt32Array, |
496 | | DataType::UInt32, |
497 | | [ |
498 | | Some(1_u32), |
499 | | Some(2_u32), |
500 | | Some(3_u32), |
501 | | Some(4_u32), |
502 | | Some(5_u32) |
503 | | ] |
504 | | ); |
505 | | Ok(()) |
506 | | } |
507 | | |
508 | | #[test] |
509 | | fn test_cast_i32_utf8() -> Result<()> { |
510 | | generic_test_cast!( |
511 | | Int32Array, |
512 | | DataType::Int32, |
513 | | vec![1, 2, 3, 4, 5], |
514 | | StringArray, |
515 | | DataType::Utf8, |
516 | | [Some("1"), Some("2"), Some("3"), Some("4"), Some("5")] |
517 | | ); |
518 | | Ok(()) |
519 | | } |
520 | | |
521 | | #[test] |
522 | | fn test_try_cast_utf8_i32() -> Result<()> { |
523 | | generic_test_cast!( |
524 | | StringArray, |
525 | | DataType::Utf8, |
526 | | vec!["a", "2", "3", "b", "5"], |
527 | | Int32Array, |
528 | | DataType::Int32, |
529 | | [None, Some(2), Some(3), None, Some(5)] |
530 | | ); |
531 | | Ok(()) |
532 | | } |
533 | | |
534 | | #[test] |
535 | | fn test_cast_i64_t64() -> Result<()> { |
536 | | let original = vec![1, 2, 3, 4, 5]; |
537 | | let expected: Vec<Option<i64>> = original |
538 | | .iter() |
539 | | .map(|i| Some(Time64NanosecondArray::from(vec![*i]).value(0))) |
540 | | .collect(); |
541 | | generic_test_cast!( |
542 | | Int64Array, |
543 | | DataType::Int64, |
544 | | original, |
545 | | TimestampNanosecondArray, |
546 | | DataType::Timestamp(TimeUnit::Nanosecond, None), |
547 | | expected |
548 | | ); |
549 | | Ok(()) |
550 | | } |
551 | | |
552 | | #[test] |
553 | | fn invalid_cast() { |
554 | | // Ensure a useful error happens at plan time if invalid casts are used |
555 | | let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); |
556 | | |
557 | | let result = try_cast( |
558 | | col("a", &schema).unwrap(), |
559 | | &schema, |
560 | | DataType::Interval(IntervalUnit::MonthDayNano), |
561 | | ); |
562 | | result.expect_err("expected Invalid TRY_CAST"); |
563 | | } |
564 | | |
565 | | // create decimal array with the specified precision and scale |
566 | | fn create_decimal_array(array: &[i128], precision: u8, scale: i8) -> Decimal128Array { |
567 | | let mut decimal_builder = Decimal128Builder::with_capacity(array.len()); |
568 | | for value in array { |
569 | | decimal_builder.append_value(*value); |
570 | | } |
571 | | decimal_builder.append_null(); |
572 | | decimal_builder |
573 | | .finish() |
574 | | .with_precision_and_scale(precision, scale) |
575 | | .unwrap() |
576 | | } |
577 | | } |