From fa8508e72fe7a9cbbbdd3f641205195e202366c8 Mon Sep 17 00:00:00 2001 From: SteveLauC Date: Wed, 28 Feb 2024 09:02:14 +0800 Subject: [PATCH 1/6] docs: put flatten in top fn list (#9376) --- docs/source/user-guide/sql/scalar_functions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index d4eb5944ad09..41b5a354abc1 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -1956,6 +1956,7 @@ from_unixtime(expression) - [array_to_string](#array_to_string) - [cardinality](#cardinality) - [empty](#empty) +- [flatten](#flatten) - [generate_series](#generate_series) - [list_append](#list_append) - [list_sort](#list_sort) From 32d906fc9622af3a67b3828700272092fe0982a0 Mon Sep 17 00:00:00 2001 From: Clide S <109172241+Monkwire3@users.noreply.github.com> Date: Tue, 27 Feb 2024 21:10:15 -0500 Subject: [PATCH 2/6] Update list_to_string alias to point to array_to_string (#9374) --- docs/source/user-guide/sql/scalar_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index 41b5a354abc1..38da3fd74c26 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -3030,7 +3030,7 @@ _Alias of [array_slice](#array_slice)._ ### `list_to_string` -_Alias of [list_to_string](#list_to_string)._ +_Alias of [array_to_string](#array_to_string)._ ### `make_array` From e62240969135e2236d100c8c0c01546a87950a80 Mon Sep 17 00:00:00 2001 From: Lordworms <48054792+Lordworms@users.noreply.github.com> Date: Wed, 28 Feb 2024 04:55:03 -0600 Subject: [PATCH 3/6] feat: issue_9285: port builtin reg function into datafusion-function-* (1/3 regexpmatch) (#9329) * feat: issue_9285: port builtin reg function into datafusion-function-* crate (1/3: RegexpMatch part) * fix fmt * refact * modify test * fix msrv verify problem * port test and delete useless lines --- .../tests/dataframe/dataframe_functions.rs | 2 +- datafusion/expr/src/built_in_function.rs | 22 --- datafusion/expr/src/expr_fn.rs | 7 - datafusion/functions/Cargo.toml | 3 +- datafusion/functions/src/lib.rs | 7 +- datafusion/functions/src/regex/mod.rs | 29 ++++ datafusion/functions/src/regex/regexpmatch.rs | 145 ++++++++++++++++++ datafusion/physical-expr/src/functions.rs | 108 ------------- datafusion/proto/proto/datafusion.proto | 2 +- datafusion/proto/src/generated/pbjson.rs | 3 - datafusion/proto/src/generated/prost.rs | 4 +- .../proto/src/logical_plan/from_proto.rs | 17 +- datafusion/proto/src/logical_plan/to_proto.rs | 1 - datafusion/sqllogictest/test_files/regexp.slt | 8 +- 14 files changed, 196 insertions(+), 162 deletions(-) create mode 100644 datafusion/functions/src/regex/mod.rs create mode 100644 datafusion/functions/src/regex/regexpmatch.rs diff --git a/datafusion/core/tests/dataframe/dataframe_functions.rs b/datafusion/core/tests/dataframe/dataframe_functions.rs index 95c13fc17c90..ff553a48888b 100644 --- a/datafusion/core/tests/dataframe/dataframe_functions.rs +++ b/datafusion/core/tests/dataframe/dataframe_functions.rs @@ -467,7 +467,7 @@ async fn test_fn_regexp_like() -> Result<()> { #[tokio::test] #[cfg(feature = "unicode_expressions")] async fn test_fn_regexp_match() -> Result<()> { - let expr = regexp_match(vec![col("a"), lit("[a-z]")]); + let expr = regexp_match(col("a"), lit("[a-z]")); let expected = [ "+------------------------------------+", diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs index e04106595876..8df2f4e88d41 100644 --- a/datafusion/expr/src/built_in_function.rs +++ b/datafusion/expr/src/built_in_function.rs @@ -233,7 +233,6 @@ pub enum BuiltinScalarFunction { /// regexp_like RegexpLike, /// regexp_match - RegexpMatch, /// regexp_replace RegexpReplace, /// repeat @@ -449,7 +448,6 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::OctetLength => Volatility::Immutable, BuiltinScalarFunction::Radians => Volatility::Immutable, BuiltinScalarFunction::RegexpLike => Volatility::Immutable, - BuiltinScalarFunction::RegexpMatch => Volatility::Immutable, BuiltinScalarFunction::RegexpReplace => Volatility::Immutable, BuiltinScalarFunction::Repeat => Volatility::Immutable, BuiltinScalarFunction::Replace => Volatility::Immutable, @@ -814,16 +812,6 @@ impl BuiltinScalarFunction { ); } }), - BuiltinScalarFunction::RegexpMatch => Ok(match &input_expr_types[0] { - LargeUtf8 => List(Arc::new(Field::new("item", LargeUtf8, true))), - Utf8 => List(Arc::new(Field::new("item", Utf8, true))), - Null => Null, - other => { - return plan_err!( - "The regexp_match function can only accept strings. Got {other}" - ); - } - }), BuiltinScalarFunction::Factorial | BuiltinScalarFunction::Gcd @@ -1263,15 +1251,6 @@ impl BuiltinScalarFunction { ], self.volatility(), ), - BuiltinScalarFunction::RegexpMatch => Signature::one_of( - vec![ - Exact(vec![Utf8, Utf8]), - Exact(vec![LargeUtf8, Utf8]), - Exact(vec![Utf8, Utf8, Utf8]), - Exact(vec![LargeUtf8, Utf8, Utf8]), - ], - self.volatility(), - ), BuiltinScalarFunction::RegexpReplace => Signature::one_of( vec![ Exact(vec![Utf8, Utf8, Utf8]), @@ -1514,7 +1493,6 @@ impl BuiltinScalarFunction { // regex functions BuiltinScalarFunction::RegexpLike => &["regexp_like"], - BuiltinScalarFunction::RegexpMatch => &["regexp_match"], BuiltinScalarFunction::RegexpReplace => &["regexp_replace"], // time/date functions diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs index 67bf39050d58..7ffd2f76e783 100644 --- a/datafusion/expr/src/expr_fn.rs +++ b/datafusion/expr/src/expr_fn.rs @@ -854,11 +854,6 @@ nary_scalar_expr!( regexp_like, "matches a regular expression against a string and returns true or false if there was at least one match or not" ); -nary_scalar_expr!( - RegexpMatch, - regexp_match, - "matches a regular expression against a string and returns matched substrings." -); nary_scalar_expr!( RegexpReplace, regexp_replace, @@ -1380,8 +1375,6 @@ mod test { test_scalar_expr!(OctetLength, octet_length, string); test_nary_scalar_expr!(RegexpLike, regexp_like, string, pattern); test_nary_scalar_expr!(RegexpLike, regexp_like, string, pattern, flags); - test_nary_scalar_expr!(RegexpMatch, regexp_match, string, pattern); - test_nary_scalar_expr!(RegexpMatch, regexp_match, string, pattern, flags); test_nary_scalar_expr!( RegexpReplace, regexp_replace, diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml index f63f18f955de..89b7de9ee11a 100644 --- a/datafusion/functions/Cargo.toml +++ b/datafusion/functions/Cargo.toml @@ -32,11 +32,12 @@ rust-version = { workspace = true } # enable core functions core_expressions = [] # Enable encoding by default so the doctests work. In general don't automatically enable all packages. -default = ["core_expressions", "encoding_expressions", "math_expressions"] +default = ["core_expressions", "encoding_expressions", "math_expressions", "regex_expressions"] # enable encode/decode functions encoding_expressions = ["base64", "hex"] # enable math functions math_expressions = [] +regex_expressions = [] [lib] name = "datafusion_functions" diff --git a/datafusion/functions/src/lib.rs b/datafusion/functions/src/lib.rs index 981174c141d6..d2f0270959ee 100644 --- a/datafusion/functions/src/lib.rs +++ b/datafusion/functions/src/lib.rs @@ -93,7 +93,7 @@ make_package!( ); make_package!(math, "math_expressions", "Mathematical functions."); - +make_package!(regex, "regex_expressions", "Regex functions"); /// Fluent-style API for creating `Expr`s pub mod expr_fn { #[cfg(feature = "core_expressions")] @@ -102,6 +102,8 @@ pub mod expr_fn { pub use super::encoding::expr_fn::*; #[cfg(feature = "math_expressions")] pub use super::math::expr_fn::*; + #[cfg(feature = "regex_expressions")] + pub use super::regex::expr_fn::*; } /// Registers all enabled packages with a [`FunctionRegistry`] @@ -109,7 +111,8 @@ pub fn register_all(registry: &mut dyn FunctionRegistry) -> Result<()> { let mut all_functions = core::functions() .into_iter() .chain(encoding::functions()) - .chain(math::functions()); + .chain(math::functions()) + .chain(regex::functions()); all_functions.try_for_each(|udf| { let existing_udf = registry.register_udf(udf)?; diff --git a/datafusion/functions/src/regex/mod.rs b/datafusion/functions/src/regex/mod.rs new file mode 100644 index 000000000000..862e8b77a2d6 --- /dev/null +++ b/datafusion/functions/src/regex/mod.rs @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! "regx" DataFusion functions + +mod regexpmatch; +// create UDFs +make_udf_function!(regexpmatch::RegexpMatchFunc, REGEXP_MATCH, regexp_match); + +export_functions!(( + regexp_match, + input_arg1 + input_arg2, + "returns a list of regular expression matches in a string. " +)); diff --git a/datafusion/functions/src/regex/regexpmatch.rs b/datafusion/functions/src/regex/regexpmatch.rs new file mode 100644 index 000000000000..7ab99f96b142 --- /dev/null +++ b/datafusion/functions/src/regex/regexpmatch.rs @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Encoding expressions +use arrow::array::{Array, ArrayRef, OffsetSizeTrait}; +use arrow::compute::kernels::regexp; +use arrow::datatypes::DataType; +use arrow::datatypes::Field; +use datafusion_common::ScalarValue; +use datafusion_expr::TypeSignature::*; +use datafusion_expr::{ScalarUDFImpl, Signature, Volatility}; +use std::any::Any; +use datafusion_common::exec_err; +use datafusion_common::{arrow_datafusion_err, plan_err}; +use datafusion_common::{ + cast::as_generic_string_array, internal_err, DataFusionError, Result, +}; +use datafusion_expr::ColumnarValue; +use std::sync::Arc; + +#[derive(Debug)] +pub(super) struct RegexpMatchFunc { + signature: Signature, +} +impl RegexpMatchFunc { + pub fn new() -> Self { + use DataType::*; + Self { + signature: Signature::one_of( + vec![ + Exact(vec![Utf8, Utf8]), + Exact(vec![LargeUtf8, Utf8]), + Exact(vec![Utf8, Utf8, Utf8]), + Exact(vec![LargeUtf8, Utf8, Utf8]), + ], + Volatility::Immutable, + ), + } + } +} + +impl ScalarUDFImpl for RegexpMatchFunc { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "regexp_match" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + use DataType::*; + + Ok(match &arg_types[0] { + LargeUtf8 => List(Arc::new(Field::new("item", LargeUtf8, true))), + Utf8 => List(Arc::new(Field::new("item", Utf8, true))), + Null => Null, + other => { + return plan_err!( + "The regexp_match function can only accept strings. Got {other}" + ); + } + }) + } + fn invoke(&self, args: &[ColumnarValue]) -> Result { + let len = args + .iter() + .fold(Option::::None, |acc, arg| match arg { + ColumnarValue::Scalar(_) => acc, + ColumnarValue::Array(a) => Some(a.len()), + }); + + let is_scalar = len.is_none(); + let inferred_length = len.unwrap_or(1); + let args = args + .iter() + .map(|arg| arg.clone().into_array(inferred_length)) + .collect::>>()?; + + let result = regexp_match_func(&args); + if is_scalar { + // If all inputs are scalar, keeps output as scalar + let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); + result.map(ColumnarValue::Scalar) + } else { + result.map(ColumnarValue::Array) + } + } +} +fn regexp_match_func(args: &[ArrayRef]) -> Result { + match args[0].data_type() { + DataType::Utf8 => { + regexp_match::(args) + } + DataType::LargeUtf8 => { + regexp_match::(args) + } + other => { + internal_err!("Unsupported data type {other:?} for function regexp_match") + } + } +} +pub fn regexp_match(args: &[ArrayRef]) -> Result { + match args.len() { + 2 => { + let values = as_generic_string_array::(&args[0])?; + let regex = as_generic_string_array::(&args[1])?; + regexp::regexp_match(values, regex, None) + .map_err(|e| arrow_datafusion_err!(e)) + } + 3 => { + let values = as_generic_string_array::(&args[0])?; + let regex = as_generic_string_array::(&args[1])?; + let flags = as_generic_string_array::(&args[2])?; + + if flags.iter().any(|s| s == Some("g")) { + return plan_err!("regexp_match() does not support the \"global\" option") + } + + regexp::regexp_match(values, regex, Some(flags)) + .map_err(|e| arrow_datafusion_err!(e)) + } + other => exec_err!( + "regexp_match was called with {other} arguments. It requires at least 2 and at most 3." + ), + } +} diff --git a/datafusion/physical-expr/src/functions.rs b/datafusion/physical-expr/src/functions.rs index af079dbd2d12..186de0609b9a 100644 --- a/datafusion/physical-expr/src/functions.rs +++ b/datafusion/physical-expr/src/functions.rs @@ -243,7 +243,6 @@ where .collect::>>()?; let result = (inner)(&args); - if is_scalar { // If all inputs are scalar, keeps output as scalar let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); @@ -619,29 +618,6 @@ pub fn create_physical_fun( exec_err!("Unsupported data type {other:?} for function regexp_like") } }), - BuiltinScalarFunction::RegexpMatch => { - Arc::new(|args| match args[0].data_type() { - DataType::Utf8 => { - let func = invoke_on_array_if_regex_expressions_feature_flag!( - regexp_match, - i32, - "regexp_match" - ); - make_scalar_function_inner(func)(args) - } - DataType::LargeUtf8 => { - let func = invoke_on_array_if_regex_expressions_feature_flag!( - regexp_match, - i64, - "regexp_match" - ); - make_scalar_function_inner(func)(args) - } - other => { - exec_err!("Unsupported data type {other:?} for function regexp_match") - } - }) - } BuiltinScalarFunction::RegexpReplace => { Arc::new(|args| match args[0].data_type() { DataType::Utf8 => { @@ -3185,90 +3161,6 @@ mod tests { Ok(()) } - #[test] - #[cfg(feature = "regex_expressions")] - fn test_regexp_match() -> Result<()> { - use datafusion_common::cast::{as_list_array, as_string_array}; - let schema = Schema::new(vec![Field::new("a", DataType::Utf8, false)]); - let execution_props = ExecutionProps::new(); - - let col_value: ArrayRef = Arc::new(StringArray::from(vec!["aaa-555"])); - let pattern = lit(r".*-(\d*)"); - let columns: Vec = vec![col_value]; - let expr = create_physical_expr_with_type_coercion( - &BuiltinScalarFunction::RegexpMatch, - &[col("a", &schema)?, pattern], - &schema, - &execution_props, - )?; - - // type is correct - assert_eq!( - expr.data_type(&schema)?, - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))) - ); - - // evaluate works - let batch = RecordBatch::try_new(Arc::new(schema.clone()), columns)?; - let result = expr - .evaluate(&batch)? - .into_array(batch.num_rows()) - .expect("Failed to convert to array"); - - // downcast works - let result = as_list_array(&result)?; - let first_row = result.value(0); - let first_row = as_string_array(&first_row)?; - - // value is correct - let expected = "555".to_string(); - assert_eq!(first_row.value(0), expected); - - Ok(()) - } - - #[test] - #[cfg(feature = "regex_expressions")] - fn test_regexp_match_all_literals() -> Result<()> { - use datafusion_common::cast::{as_list_array, as_string_array}; - let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); - let execution_props = ExecutionProps::new(); - - let col_value = lit("aaa-555"); - let pattern = lit(r".*-(\d*)"); - let columns: Vec = vec![Arc::new(Int32Array::from(vec![1]))]; - let expr = create_physical_expr_with_type_coercion( - &BuiltinScalarFunction::RegexpMatch, - &[col_value, pattern], - &schema, - &execution_props, - )?; - - // type is correct - assert_eq!( - expr.data_type(&schema)?, - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))) - ); - - // evaluate works - let batch = RecordBatch::try_new(Arc::new(schema.clone()), columns)?; - let result = expr - .evaluate(&batch)? - .into_array(batch.num_rows()) - .expect("Failed to convert to array"); - - // downcast works - let result = as_list_array(&result)?; - let first_row = result.value(0); - let first_row = as_string_array(&first_row)?; - - // value is correct - let expected = "555".to_string(); - assert_eq!(first_row.value(0), expected); - - Ok(()) - } - // Helper function just for testing. // Returns `expressions` coerced to types compatible with // `signature`, if possible. diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index 2d729ffc5b3e..1f659469aa3a 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -568,7 +568,7 @@ enum ScalarFunction { Tan = 18; Trunc = 19; Array = 20; - RegexpMatch = 21; + // RegexpMatch = 21; BitLength = 22; Btrim = 23; CharacterLength = 24; diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index 5f05b8546f68..8959dd37cf13 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -22341,7 +22341,6 @@ impl serde::Serialize for ScalarFunction { Self::Tan => "Tan", Self::Trunc => "Trunc", Self::Array => "Array", - Self::RegexpMatch => "RegexpMatch", Self::BitLength => "BitLength", Self::Btrim => "Btrim", Self::CharacterLength => "CharacterLength", @@ -22483,7 +22482,6 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "Tan", "Trunc", "Array", - "RegexpMatch", "BitLength", "Btrim", "CharacterLength", @@ -22654,7 +22652,6 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "Tan" => Ok(ScalarFunction::Tan), "Trunc" => Ok(ScalarFunction::Trunc), "Array" => Ok(ScalarFunction::Array), - "RegexpMatch" => Ok(ScalarFunction::RegexpMatch), "BitLength" => Ok(ScalarFunction::BitLength), "Btrim" => Ok(ScalarFunction::Btrim), "CharacterLength" => Ok(ScalarFunction::CharacterLength), diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index 252089d5c14d..09152d99c12f 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -2656,7 +2656,7 @@ pub enum ScalarFunction { Tan = 18, Trunc = 19, Array = 20, - RegexpMatch = 21, + /// RegexpMatch = 21; BitLength = 22, Btrim = 23, CharacterLength = 24, @@ -2798,7 +2798,6 @@ impl ScalarFunction { ScalarFunction::Tan => "Tan", ScalarFunction::Trunc => "Trunc", ScalarFunction::Array => "Array", - ScalarFunction::RegexpMatch => "RegexpMatch", ScalarFunction::BitLength => "BitLength", ScalarFunction::Btrim => "Btrim", ScalarFunction::CharacterLength => "CharacterLength", @@ -2934,7 +2933,6 @@ impl ScalarFunction { "Tan" => Some(Self::Tan), "Trunc" => Some(Self::Trunc), "Array" => Some(Self::Array), - "RegexpMatch" => Some(Self::RegexpMatch), "BitLength" => Some(Self::BitLength), "Btrim" => Some(Self::Btrim), "CharacterLength" => Some(Self::CharacterLength), diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index acfa043b88af..e8059482b1b9 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -61,11 +61,11 @@ use datafusion_expr::{ instr, iszero, lcm, left, levenshtein, ln, log, log10, log2, logical_plan::{PlanType, StringifiedPlan}, lower, lpad, ltrim, md5, nanvl, now, octet_length, overlay, pi, power, radians, - random, regexp_like, regexp_match, regexp_replace, repeat, replace, reverse, right, - round, rpad, rtrim, sha224, sha256, sha384, sha512, signum, sin, sinh, split_part, - sqrt, starts_with, string_to_array, strpos, struct_fun, substr, substr_index, - substring, tan, tanh, to_hex, translate, trim, trunc, upper, uuid, AggregateFunction, - Between, BinaryExpr, BuiltInWindowFunction, BuiltinScalarFunction, Case, Cast, Expr, + random, regexp_like, regexp_replace, repeat, replace, reverse, right, round, rpad, + rtrim, sha224, sha256, sha384, sha512, signum, sin, sinh, split_part, sqrt, + starts_with, string_to_array, strpos, struct_fun, substr, substr_index, substring, + tan, tanh, to_hex, translate, trim, trunc, upper, uuid, AggregateFunction, Between, + BinaryExpr, BuiltInWindowFunction, BuiltinScalarFunction, Case, Cast, Expr, GetFieldAccess, GetIndexedField, GroupingSet, GroupingSet::GroupingSets, JoinConstraint, JoinType, Like, Operator, TryCast, WindowFrame, WindowFrameBound, @@ -535,7 +535,6 @@ impl From<&protobuf::ScalarFunction> for BuiltinScalarFunction { ScalarFunction::Lpad => Self::Lpad, ScalarFunction::Random => Self::Random, ScalarFunction::RegexpLike => Self::RegexpLike, - ScalarFunction::RegexpMatch => Self::RegexpMatch, ScalarFunction::RegexpReplace => Self::RegexpReplace, ScalarFunction::Repeat => Self::Repeat, ScalarFunction::Replace => Self::Replace, @@ -1638,12 +1637,6 @@ pub fn parse_expr( .map(|expr| parse_expr(expr, registry)) .collect::, _>>()?, )), - ScalarFunction::RegexpMatch => Ok(regexp_match( - args.to_owned() - .iter() - .map(|expr| parse_expr(expr, registry)) - .collect::, _>>()?, - )), ScalarFunction::RegexpReplace => Ok(regexp_replace( args.to_owned() .iter() diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index d19830db98ce..6f126729cb29 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -1518,7 +1518,6 @@ impl TryFrom<&BuiltinScalarFunction> for protobuf::ScalarFunction { BuiltinScalarFunction::Random => Self::Random, BuiltinScalarFunction::Uuid => Self::Uuid, BuiltinScalarFunction::RegexpLike => Self::RegexpLike, - BuiltinScalarFunction::RegexpMatch => Self::RegexpMatch, BuiltinScalarFunction::RegexpReplace => Self::RegexpReplace, BuiltinScalarFunction::Repeat => Self::Repeat, BuiltinScalarFunction::Replace => Self::Replace, diff --git a/datafusion/sqllogictest/test_files/regexp.slt b/datafusion/sqllogictest/test_files/regexp.slt index 1e951e2962ff..a80b08c41ee3 100644 --- a/datafusion/sqllogictest/test_files/regexp.slt +++ b/datafusion/sqllogictest/test_files/regexp.slt @@ -220,6 +220,12 @@ SELECT regexp_match('(?<=[A-Z]\w )Smith', 'John Smith', 'i'); ---- NULL +# ported test +query ? +SELECT regexp_match('aaa-555', '.*-(\d*)'); +---- +[555] + # # regexp_replace tests # @@ -300,4 +306,4 @@ SELECT regexp_replace(arrow_cast('foobar', 'Dictionary(Int32, Utf8)'), 'bar', 'x fooxx statement ok -drop table t; \ No newline at end of file +drop table t; From d896ebe4d7466e52a1e8fad4252067d69e62298a Mon Sep 17 00:00:00 2001 From: Jonah Gao Date: Wed, 28 Feb 2024 20:16:44 +0800 Subject: [PATCH 4/6] Add test to verify issue #9161 (#9265) * Add test to verify issue #9161 * fmt --- datafusion/sqllogictest/test_files/aggregate.slt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index 109c64f06055..b78c6287746c 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -3313,5 +3313,13 @@ query I SELECT 0 AS "t.a" FROM t HAVING MAX(t.a) = 0; ---- +# Test issue: https://github.com/apache/arrow-datafusion/issues/9161 +query I rowsort +SELECT CAST(a AS INT) FROM t GROUP BY t.a; +---- +1 +2 +3 + statement ok DROP TABLE t; From a1ae15826245097e7c12d4f0ed3425b25af6c431 Mon Sep 17 00:00:00 2001 From: Marco Neumann Date: Wed, 28 Feb 2024 13:25:36 +0100 Subject: [PATCH 5/6] refactor: fix error macros hygiene (#9366) --- benchmarks/src/tpch/convert.rs | 2 +- benchmarks/src/tpch/mod.rs | 2 +- datafusion-cli/src/catalog.rs | 2 +- datafusion-cli/src/functions.rs | 2 +- datafusion-examples/examples/advanced_udf.rs | 1 - datafusion-examples/examples/rewrite_expr.rs | 2 +- datafusion-examples/examples/simple_udtf.rs | 2 +- datafusion/common/src/error.rs | 16 +++++++- datafusion/common/src/hash_utils.rs | 2 +- .../common/src/scalar/struct_builder.rs | 2 +- datafusion/core/src/catalog/mod.rs | 2 +- datafusion/core/src/catalog/schema.rs | 4 +- .../src/datasource/default_table_source.rs | 2 +- .../core/src/datasource/file_format/json.rs | 2 +- .../core/src/datasource/file_format/mod.rs | 2 +- datafusion/core/src/datasource/memory.rs | 5 +-- datafusion/core/src/datasource/mod.rs | 2 +- .../physical_plan/file_scan_config.rs | 5 +-- .../datasource/physical_plan/file_stream.rs | 2 +- .../core/src/datasource/physical_plan/mod.rs | 2 +- datafusion/core/src/datasource/provider.rs | 2 +- datafusion/core/src/datasource/streaming.rs | 2 +- .../src/physical_optimizer/enforce_sorting.rs | 2 +- .../src/physical_optimizer/join_selection.rs | 2 +- .../physical_optimizer/pipeline_checker.rs | 2 +- .../src/physical_optimizer/sort_pushdown.rs | 2 +- datafusion/core/tests/core_integration.rs | 3 ++ .../provider_filter_pushdown.rs | 2 +- datafusion/core/tests/macro_hygiene/mod.rs | 39 +++++++++++++++++++ .../user_defined/user_defined_aggregates.rs | 4 +- .../user_defined_scalar_functions.rs | 2 +- datafusion/execution/src/registry.rs | 2 +- datafusion/expr/src/accumulator.rs | 2 +- datafusion/expr/src/columnar_value.rs | 2 +- .../expr/src/conditional_expressions.rs | 2 +- datafusion/expr/src/expr.rs | 2 +- datafusion/expr/src/expr_schema.rs | 3 +- datafusion/expr/src/field_util.rs | 4 +- datafusion/expr/src/interval_arithmetic.rs | 2 +- datafusion/expr/src/partition_evaluator.rs | 2 +- datafusion/expr/src/tree_node/expr.rs | 2 +- .../expr/src/type_coercion/aggregates.rs | 2 +- datafusion/expr/src/type_coercion/binary.rs | 4 +- .../expr/src/type_coercion/functions.rs | 4 +- datafusion/expr/src/udaf.rs | 2 +- datafusion/expr/src/utils.rs | 28 ++++++------- datafusion/functions-array/src/udf.rs | 2 +- datafusion/functions/src/core/nullif.rs | 2 +- datafusion/functions/src/core/nvl.rs | 2 +- datafusion/optimizer/src/analyzer/subquery.rs | 2 +- datafusion/optimizer/src/decorrelate.rs | 2 +- .../src/decorrelate_predicate_subquery.rs | 2 +- .../optimizer/src/eliminate_cross_join.rs | 2 +- datafusion/optimizer/src/optimizer.rs | 4 +- .../optimizer/src/propagate_empty_relation.rs | 2 +- datafusion/optimizer/src/push_down_filter.rs | 4 +- .../optimizer/src/scalar_subquery_to_join.rs | 2 +- .../src/simplify_expressions/utils.rs | 2 +- .../src/unwrap_cast_in_comparison.rs | 4 +- .../optimizer/tests/optimizer_integration.rs | 2 +- .../src/aggregate/array_agg_distinct.rs | 2 +- .../src/aggregate/array_agg_ordered.rs | 2 +- .../physical-expr/src/aggregate/average.rs | 2 +- .../physical-expr/src/aggregate/build_in.rs | 4 +- .../physical-expr/src/aggregate/grouping.rs | 2 +- datafusion/physical-expr/src/aggregate/mod.rs | 2 +- .../physical-expr/src/aggregate/nth_value.rs | 2 +- .../physical-expr/src/aggregate/stddev.rs | 2 +- .../physical-expr/src/aggregate/string_agg.rs | 2 +- datafusion/physical-expr/src/aggregate/sum.rs | 2 +- .../src/aggregate/sum_distinct.rs | 2 +- datafusion/physical-expr/src/analysis.rs | 4 +- .../src/conditional_expressions.rs | 2 +- .../physical-expr/src/equivalence/mod.rs | 2 +- .../physical-expr/src/expressions/binary.rs | 2 +- .../src/expressions/binary/kernels.rs | 2 +- .../physical-expr/src/expressions/cast.rs | 2 +- .../physical-expr/src/expressions/column.rs | 2 +- .../src/expressions/get_indexed_field.rs | 2 +- .../physical-expr/src/expressions/in_list.rs | 4 +- .../physical-expr/src/expressions/like.rs | 2 +- .../physical-expr/src/expressions/mod.rs | 4 +- .../physical-expr/src/expressions/negative.rs | 3 +- .../physical-expr/src/expressions/no_op.rs | 2 +- .../physical-expr/src/expressions/try_cast.rs | 2 +- datafusion/physical-expr/src/functions.rs | 4 +- .../physical-expr/src/intervals/cp_solver.rs | 2 +- .../physical-expr/src/intervals/utils.rs | 4 +- datafusion/physical-expr/src/physical_expr.rs | 2 +- datafusion/physical-expr/src/planner.rs | 3 +- .../physical-expr/src/string_expressions.rs | 2 +- .../physical-expr/src/struct_expressions.rs | 2 +- .../physical-expr/src/unicode_expressions.rs | 2 +- .../physical-expr/src/window/nth_value.rs | 2 +- datafusion/physical-expr/src/window/rank.rs | 2 +- .../physical-plan/src/aggregates/mod.rs | 2 +- .../physical-plan/src/coalesce_partitions.rs | 2 +- datafusion/physical-plan/src/empty.rs | 2 +- datafusion/physical-plan/src/explain.rs | 2 +- datafusion/physical-plan/src/insert.rs | 2 +- .../physical-plan/src/joins/cross_join.rs | 2 +- .../src/joins/nested_loop_join.rs | 2 +- .../src/joins/symmetric_hash_join.rs | 4 +- datafusion/physical-plan/src/lib.rs | 2 +- datafusion/physical-plan/src/limit.rs | 2 +- datafusion/physical-plan/src/memory.rs | 2 +- .../physical-plan/src/placeholder_row.rs | 2 +- .../src/sorts/sort_preserving_merge.rs | 2 +- .../src/sorts/streaming_merge.rs | 2 +- datafusion/physical-plan/src/stream.rs | 2 +- datafusion/physical-plan/src/streaming.rs | 2 +- datafusion/physical-plan/src/udaf.rs | 2 +- datafusion/physical-plan/src/union.rs | 2 +- datafusion/physical-plan/src/unnest.rs | 2 +- datafusion/physical-plan/src/values.rs | 2 +- .../src/windows/window_agg_exec.rs | 2 +- datafusion/physical-plan/src/work_table.rs | 2 +- datafusion/proto/src/bytes/mod.rs | 2 +- datafusion/proto/src/bytes/registry.rs | 2 +- datafusion/sql/examples/sql.rs | 2 +- datafusion/sql/src/expr/binary_op.rs | 2 +- datafusion/sql/src/expr/function.rs | 3 +- datafusion/sql/src/expr/grouping_set.rs | 2 +- datafusion/sql/src/expr/json_access.rs | 2 +- datafusion/sql/src/expr/mod.rs | 3 +- datafusion/sql/src/expr/order_by.rs | 4 +- datafusion/sql/src/expr/substring.rs | 2 +- datafusion/sql/src/expr/unary_op.rs | 2 +- datafusion/sql/src/relation/join.rs | 2 +- datafusion/sql/src/relation/mod.rs | 4 +- datafusion/sql/src/set_expr.rs | 2 +- .../tests/cases/roundtrip_logical_plan.rs | 2 +- 132 files changed, 205 insertions(+), 181 deletions(-) create mode 100644 datafusion/core/tests/macro_hygiene/mod.rs diff --git a/benchmarks/src/tpch/convert.rs b/benchmarks/src/tpch/convert.rs index 2fc74ce38888..12b562421e53 100644 --- a/benchmarks/src/tpch/convert.rs +++ b/benchmarks/src/tpch/convert.rs @@ -20,7 +20,7 @@ use std::path::{Path, PathBuf}; use std::time::Instant; use datafusion::common::not_impl_err; -use datafusion::error::DataFusionError; + use datafusion::error::Result; use datafusion::prelude::*; use parquet::basic::Compression; diff --git a/benchmarks/src/tpch/mod.rs b/benchmarks/src/tpch/mod.rs index 8965ebea7ff6..23d0681f560c 100644 --- a/benchmarks/src/tpch/mod.rs +++ b/benchmarks/src/tpch/mod.rs @@ -21,7 +21,7 @@ use arrow::datatypes::SchemaBuilder; use datafusion::{ arrow::datatypes::{DataType, Field, Schema}, common::plan_err, - error::{DataFusionError, Result}, + error::Result, }; use std::fs; mod run; diff --git a/datafusion-cli/src/catalog.rs b/datafusion-cli/src/catalog.rs index 67184b8257b8..29211edbb0a4 100644 --- a/datafusion-cli/src/catalog.rs +++ b/datafusion-cli/src/catalog.rs @@ -19,7 +19,7 @@ use crate::object_storage::get_object_store; use async_trait::async_trait; use datafusion::catalog::schema::SchemaProvider; use datafusion::catalog::{CatalogProvider, CatalogProviderList}; -use datafusion::common::{plan_datafusion_err, DataFusionError}; +use datafusion::common::plan_datafusion_err; use datafusion::datasource::listing::{ ListingTable, ListingTableConfig, ListingTableUrl, }; diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rs index 5390fa9f2271..806e2bb39cd4 100644 --- a/datafusion-cli/src/functions.rs +++ b/datafusion-cli/src/functions.rs @@ -21,7 +21,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use arrow::util::pretty::pretty_format_batches; use async_trait::async_trait; -use datafusion::common::DataFusionError; + use datafusion::common::{plan_err, Column}; use datafusion::datasource::function::TableFunctionImpl; use datafusion::datasource::TableProvider; diff --git a/datafusion-examples/examples/advanced_udf.rs b/datafusion-examples/examples/advanced_udf.rs index 3e7dd2e2af08..c8063c0eb1e3 100644 --- a/datafusion-examples/examples/advanced_udf.rs +++ b/datafusion-examples/examples/advanced_udf.rs @@ -176,7 +176,6 @@ impl ScalarUDFImpl for PowUdf { } // if the types were not float, it is a bug in DataFusion _ => { - use datafusion_common::DataFusionError; internal_err!("Invalid argument types to pow function") } } diff --git a/datafusion-examples/examples/rewrite_expr.rs b/datafusion-examples/examples/rewrite_expr.rs index 5e95562033e6..8d13d1201881 100644 --- a/datafusion-examples/examples/rewrite_expr.rs +++ b/datafusion-examples/examples/rewrite_expr.rs @@ -18,7 +18,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::config::ConfigOptions; use datafusion_common::tree_node::{Transformed, TreeNode}; -use datafusion_common::{plan_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{plan_err, Result, ScalarValue}; use datafusion_expr::{ AggregateUDF, Between, Expr, Filter, LogicalPlan, ScalarUDF, TableSource, WindowUDF, }; diff --git a/datafusion-examples/examples/simple_udtf.rs b/datafusion-examples/examples/simple_udtf.rs index f1d763ba6e41..09341fbf47fa 100644 --- a/datafusion-examples/examples/simple_udtf.rs +++ b/datafusion-examples/examples/simple_udtf.rs @@ -27,7 +27,7 @@ use datafusion::execution::context::{ExecutionProps, SessionState}; use datafusion::physical_plan::memory::MemoryExec; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionContext; -use datafusion_common::{plan_err, DataFusionError, ScalarValue}; +use datafusion_common::{plan_err, ScalarValue}; use datafusion_expr::{Expr, TableType}; use datafusion_optimizer::simplify_expressions::{ExprSimplifier, SimplifyContext}; use std::fs::File; diff --git a/datafusion/common/src/error.rs b/datafusion/common/src/error.rs index 331f5910d7e5..0f4e97905938 100644 --- a/datafusion/common/src/error.rs +++ b/datafusion/common/src/error.rs @@ -495,7 +495,13 @@ macro_rules! make_error { #[macro_export] macro_rules! $NAME_DF_ERR { ($d($d args:expr),*) => { - DataFusionError::$ERR(format!("{}{}", format!($d($d args),*), DataFusionError::get_back_trace()).into()) + $crate::DataFusionError::$ERR( + format!( + "{}{}", + format!($d($d args),*), + $crate::DataFusionError::get_back_trace(), + ).into() + ) } } @@ -503,7 +509,13 @@ macro_rules! make_error { #[macro_export] macro_rules! $NAME_ERR { ($d($d args:expr),*) => { - Err(DataFusionError::$ERR(format!("{}{}", format!($d($d args),*), DataFusionError::get_back_trace()).into())) + Err($crate::DataFusionError::$ERR( + format!( + "{}{}", + format!($d($d args),*), + $crate::DataFusionError::get_back_trace(), + ).into() + )) } } } diff --git a/datafusion/common/src/hash_utils.rs b/datafusion/common/src/hash_utils.rs index d5a1b3ee363b..d1a7a675cb45 100644 --- a/datafusion/common/src/hash_utils.rs +++ b/datafusion/common/src/hash_utils.rs @@ -31,7 +31,7 @@ use crate::cast::{ as_large_list_array, as_list_array, as_primitive_array, as_string_array, as_struct_array, }; -use crate::error::{DataFusionError, Result, _internal_err}; +use crate::error::{Result, _internal_err}; // Combines two hashes into one hash #[inline] diff --git a/datafusion/common/src/scalar/struct_builder.rs b/datafusion/common/src/scalar/struct_builder.rs index 1192757e890b..b1a34e4a61d0 100644 --- a/datafusion/common/src/scalar/struct_builder.rs +++ b/datafusion/common/src/scalar/struct_builder.rs @@ -18,7 +18,7 @@ //! [`ScalarStructBuilder`] for building [`ScalarValue::Struct`] use crate::error::_internal_err; -use crate::{DataFusionError, Result, ScalarValue}; +use crate::{Result, ScalarValue}; use arrow::array::{ArrayRef, StructArray}; use arrow::datatypes::{DataType, FieldRef, Fields}; use arrow_schema::Field; diff --git a/datafusion/core/src/catalog/mod.rs b/datafusion/core/src/catalog/mod.rs index a05a480bef44..8aeeaf9f72d8 100644 --- a/datafusion/core/src/catalog/mod.rs +++ b/datafusion/core/src/catalog/mod.rs @@ -25,7 +25,7 @@ pub use datafusion_sql::{ResolvedTableReference, TableReference}; use crate::catalog::schema::SchemaProvider; use dashmap::DashMap; -use datafusion_common::{exec_err, not_impl_err, DataFusionError, Result}; +use datafusion_common::{exec_err, not_impl_err, Result}; use std::any::Any; use std::sync::Arc; diff --git a/datafusion/core/src/catalog/schema.rs b/datafusion/core/src/catalog/schema.rs index 49f8350ecc5b..8249c3a5330f 100644 --- a/datafusion/core/src/catalog/schema.rs +++ b/datafusion/core/src/catalog/schema.rs @@ -20,12 +20,12 @@ use async_trait::async_trait; use dashmap::DashMap; -use datafusion_common::exec_err; +use datafusion_common::{exec_err, DataFusionError}; use std::any::Any; use std::sync::Arc; use crate::datasource::TableProvider; -use crate::error::{DataFusionError, Result}; +use crate::error::Result; /// Represents a schema, comprising a number of named tables. /// diff --git a/datafusion/core/src/datasource/default_table_source.rs b/datafusion/core/src/datasource/default_table_source.rs index fadf01c74c5d..977e681d6641 100644 --- a/datafusion/core/src/datasource/default_table_source.rs +++ b/datafusion/core/src/datasource/default_table_source.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use crate::datasource::TableProvider; use arrow::datatypes::SchemaRef; -use datafusion_common::{internal_err, Constraints, DataFusionError}; +use datafusion_common::{internal_err, Constraints}; use datafusion_expr::{Expr, TableProviderFilterPushDown, TableSource}; /// DataFusion default table source, wrapping TableProvider. diff --git a/datafusion/core/src/datasource/file_format/json.rs b/datafusion/core/src/datasource/file_format/json.rs index 0f6d3648d120..121fe5e8dcb1 100644 --- a/datafusion/core/src/datasource/file_format/json.rs +++ b/datafusion/core/src/datasource/file_format/json.rs @@ -42,7 +42,7 @@ use arrow::datatypes::SchemaRef; use arrow::json; use arrow::json::reader::{infer_json_schema_from_iterator, ValueIter}; use arrow_array::RecordBatch; -use datafusion_common::{not_impl_err, DataFusionError, FileType}; +use datafusion_common::{not_impl_err, FileType}; use datafusion_execution::TaskContext; use datafusion_physical_expr::{PhysicalExpr, PhysicalSortRequirement}; use datafusion_physical_plan::metrics::MetricsSet; diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index 12c9fb91adb1..72dc289d4b64 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -41,7 +41,7 @@ use crate::error::Result; use crate::execution::context::SessionState; use crate::physical_plan::{ExecutionPlan, Statistics}; -use datafusion_common::{not_impl_err, DataFusionError, FileType}; +use datafusion_common::{not_impl_err, FileType}; use datafusion_physical_expr::{PhysicalExpr, PhysicalSortRequirement}; use async_trait::async_trait; diff --git a/datafusion/core/src/datasource/memory.rs b/datafusion/core/src/datasource/memory.rs index 901e74dfc218..e087b4bcba51 100644 --- a/datafusion/core/src/datasource/memory.rs +++ b/datafusion/core/src/datasource/memory.rs @@ -28,9 +28,7 @@ use std::sync::Arc; use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; use async_trait::async_trait; -use datafusion_common::{ - not_impl_err, plan_err, Constraints, DFSchema, DataFusionError, SchemaExt, -}; +use datafusion_common::{not_impl_err, plan_err, Constraints, DFSchema, SchemaExt}; use datafusion_execution::TaskContext; use parking_lot::Mutex; use tokio::sync::RwLock; @@ -370,6 +368,7 @@ mod tests { use arrow::array::{AsArray, Int32Array}; use arrow::datatypes::{DataType, Field, Schema, UInt64Type}; use arrow::error::ArrowError; + use datafusion_common::DataFusionError; use datafusion_expr::LogicalPlanBuilder; use futures::StreamExt; use std::collections::HashMap; diff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rs index 8f20da183a93..351967d35324 100644 --- a/datafusion/core/src/datasource/mod.rs +++ b/datafusion/core/src/datasource/mod.rs @@ -48,7 +48,7 @@ pub use crate::logical_expr::TableType; pub use statistics::get_statistics_with_limit; use arrow_schema::{Schema, SortOptions}; -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_expr::Expr; use datafusion_physical_expr::{expressions, LexOrdering, PhysicalSortExpr}; diff --git a/datafusion/core/src/datasource/physical_plan/file_scan_config.rs b/datafusion/core/src/datasource/physical_plan/file_scan_config.rs index 516755e4d293..4a814c5b9b2c 100644 --- a/datafusion/core/src/datasource/physical_plan/file_scan_config.rs +++ b/datafusion/core/src/datasource/physical_plan/file_scan_config.rs @@ -24,10 +24,7 @@ use std::{ use super::{get_projected_output_ordering, FileGroupPartitioner}; use crate::datasource::{listing::PartitionedFile, object_store::ObjectStoreUrl}; -use crate::{ - error::{DataFusionError, Result}, - scalar::ScalarValue, -}; +use crate::{error::Result, scalar::ScalarValue}; use arrow::array::{ArrayData, BufferBuilder}; use arrow::buffer::Buffer; diff --git a/datafusion/core/src/datasource/physical_plan/file_stream.rs b/datafusion/core/src/datasource/physical_plan/file_stream.rs index 9cb58e7032db..0d25189a6124 100644 --- a/datafusion/core/src/datasource/physical_plan/file_stream.rs +++ b/datafusion/core/src/datasource/physical_plan/file_stream.rs @@ -531,7 +531,7 @@ mod tests { }; use arrow_schema::Schema; - use datafusion_common::{internal_err, DataFusionError, Statistics}; + use datafusion_common::{internal_err, Statistics}; use bytes::Bytes; use futures::StreamExt; diff --git a/datafusion/core/src/datasource/physical_plan/mod.rs b/datafusion/core/src/datasource/physical_plan/mod.rs index d6546539993b..2a8bb3b4fbaa 100644 --- a/datafusion/core/src/datasource/physical_plan/mod.rs +++ b/datafusion/core/src/datasource/physical_plan/mod.rs @@ -52,7 +52,7 @@ use std::{ }; use super::listing::ListingTableUrl; -use crate::error::{DataFusionError, Result}; +use crate::error::Result; use crate::physical_plan::{DisplayAs, DisplayFormatType}; use crate::{ datasource::{ diff --git a/datafusion/core/src/datasource/provider.rs b/datafusion/core/src/datasource/provider.rs index 8de2c6b3ea86..e769084df636 100644 --- a/datafusion/core/src/datasource/provider.rs +++ b/datafusion/core/src/datasource/provider.rs @@ -21,7 +21,7 @@ use std::any::Any; use std::sync::Arc; use async_trait::async_trait; -use datafusion_common::{not_impl_err, Constraints, DataFusionError, Statistics}; +use datafusion_common::{not_impl_err, Constraints, Statistics}; use datafusion_expr::{CreateExternalTable, LogicalPlan}; pub use datafusion_expr::{TableProviderFilterPushDown, TableType}; diff --git a/datafusion/core/src/datasource/streaming.rs b/datafusion/core/src/datasource/streaming.rs index 3eb120653ce3..f85db2280d8e 100644 --- a/datafusion/core/src/datasource/streaming.rs +++ b/datafusion/core/src/datasource/streaming.rs @@ -23,7 +23,7 @@ use std::sync::Arc; use arrow::datatypes::SchemaRef; use async_trait::async_trait; -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_expr::{Expr, TableType}; use log::debug; diff --git a/datafusion/core/src/physical_optimizer/enforce_sorting.rs b/datafusion/core/src/physical_optimizer/enforce_sorting.rs index 9b76af2dbb1f..e6f3e12aceaf 100644 --- a/datafusion/core/src/physical_optimizer/enforce_sorting.rs +++ b/datafusion/core/src/physical_optimizer/enforce_sorting.rs @@ -59,8 +59,8 @@ use crate::physical_plan::windows::{ }; use crate::physical_plan::{Distribution, ExecutionPlan, InputOrderMode}; +use datafusion_common::plan_err; use datafusion_common::tree_node::{Transformed, TreeNode}; -use datafusion_common::{plan_err, DataFusionError}; use datafusion_physical_expr::{PhysicalSortExpr, PhysicalSortRequirement}; use datafusion_physical_plan::repartition::RepartitionExec; use datafusion_physical_plan::sorts::partial_sort::PartialSortExec; diff --git a/datafusion/core/src/physical_optimizer/join_selection.rs b/datafusion/core/src/physical_optimizer/join_selection.rs index 02626056f6cc..f2e81fb053c0 100644 --- a/datafusion/core/src/physical_optimizer/join_selection.rs +++ b/datafusion/core/src/physical_optimizer/join_selection.rs @@ -41,8 +41,8 @@ use crate::physical_plan::ExecutionPlan; use arrow_schema::Schema; use datafusion_common::tree_node::{Transformed, TreeNode}; +use datafusion_common::JoinType; use datafusion_common::{internal_err, JoinSide}; -use datafusion_common::{DataFusionError, JoinType}; use datafusion_physical_expr::expressions::Column; use datafusion_physical_expr::sort_properties::SortProperties; use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr}; diff --git a/datafusion/core/src/physical_optimizer/pipeline_checker.rs b/datafusion/core/src/physical_optimizer/pipeline_checker.rs index bb0665c10bcc..c0f071cd3f64 100644 --- a/datafusion/core/src/physical_optimizer/pipeline_checker.rs +++ b/datafusion/core/src/physical_optimizer/pipeline_checker.rs @@ -27,8 +27,8 @@ use crate::physical_optimizer::PhysicalOptimizerRule; use crate::physical_plan::ExecutionPlan; use datafusion_common::config::OptimizerOptions; +use datafusion_common::plan_err; use datafusion_common::tree_node::{Transformed, TreeNode}; -use datafusion_common::{plan_err, DataFusionError}; use datafusion_physical_expr::intervals::utils::{check_support, is_datatype_supported}; use datafusion_physical_plan::joins::SymmetricHashJoinExec; use datafusion_physical_plan::tree_node::PlanContext; diff --git a/datafusion/core/src/physical_optimizer/sort_pushdown.rs b/datafusion/core/src/physical_optimizer/sort_pushdown.rs index 3413486c6b46..22e0d804acb1 100644 --- a/datafusion/core/src/physical_optimizer/sort_pushdown.rs +++ b/datafusion/core/src/physical_optimizer/sort_pushdown.rs @@ -31,7 +31,7 @@ use crate::physical_plan::tree_node::PlanContext; use crate::physical_plan::ExecutionPlan; use datafusion_common::tree_node::Transformed; -use datafusion_common::{plan_err, DataFusionError, JoinSide, Result}; +use datafusion_common::{plan_err, JoinSide, Result}; use datafusion_expr::JoinType; use datafusion_physical_expr::expressions::Column; use datafusion_physical_expr::{ diff --git a/datafusion/core/tests/core_integration.rs b/datafusion/core/tests/core_integration.rs index af39e1e18abc..befefb1d7ec5 100644 --- a/datafusion/core/tests/core_integration.rs +++ b/datafusion/core/tests/core_integration.rs @@ -21,6 +21,9 @@ mod sql; /// Run all tests that are found in the `dataframe` directory mod dataframe; +/// Run all tests that are found in the `macro_hygiene` directory +mod macro_hygiene; + #[cfg(test)] #[ctor::ctor] fn init() { diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs index e374abd6e891..bc7f88b39672 100644 --- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs +++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs @@ -34,7 +34,7 @@ use datafusion::physical_plan::{ use datafusion::prelude::*; use datafusion::scalar::ScalarValue; use datafusion_common::cast::as_primitive_array; -use datafusion_common::{internal_err, not_impl_err, DataFusionError}; +use datafusion_common::{internal_err, not_impl_err}; use datafusion_expr::expr::{BinaryExpr, Cast}; use async_trait::async_trait; diff --git a/datafusion/core/tests/macro_hygiene/mod.rs b/datafusion/core/tests/macro_hygiene/mod.rs new file mode 100644 index 000000000000..72ac6e64fb0c --- /dev/null +++ b/datafusion/core/tests/macro_hygiene/mod.rs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +//! Verifies [Macro Hygene] +//! +//! [Macro Hygene]: https://en.wikipedia.org/wiki/Hygienic_macro +mod plan_err { + // NO other imports! + use datafusion_common::plan_err; + + #[test] + fn test_macro() { + // need type annotation for Ok variant + let _res: Result<(), _> = plan_err!("foo"); + } +} + +mod plan_datafusion_err { + // NO other imports! + use datafusion_common::plan_datafusion_err; + + #[test] + fn test_macro() { + plan_datafusion_err!("foo"); + } +} diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rs index 8daeefd236f7..9e231d25f298 100644 --- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs +++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs @@ -42,9 +42,7 @@ use datafusion::{ prelude::SessionContext, scalar::ScalarValue, }; -use datafusion_common::{ - assert_contains, cast::as_primitive_array, exec_err, DataFusionError, -}; +use datafusion_common::{assert_contains, cast::as_primitive_array, exec_err}; use datafusion_expr::{ create_udaf, AggregateUDFImpl, GroupsAccumulator, SimpleAggregateUDF, }; diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs index a255498eb5f7..0546ef59b1d8 100644 --- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs @@ -24,7 +24,7 @@ use datafusion::{execution::registry::FunctionRegistry, test_util}; use datafusion_common::cast::as_float64_array; use datafusion_common::{ assert_batches_eq, assert_batches_sorted_eq, cast::as_int32_array, not_impl_err, - plan_err, DataFusionError, ExprSchema, Result, ScalarValue, + plan_err, ExprSchema, Result, ScalarValue, }; use datafusion_expr::{ create_udaf, create_udf, Accumulator, ColumnarValue, ExprSchemable, diff --git a/datafusion/execution/src/registry.rs b/datafusion/execution/src/registry.rs index 6e0a932f0bc5..5bc9a7a07b6f 100644 --- a/datafusion/execution/src/registry.rs +++ b/datafusion/execution/src/registry.rs @@ -17,7 +17,7 @@ //! FunctionRegistry trait -use datafusion_common::{not_impl_err, plan_datafusion_err, DataFusionError, Result}; +use datafusion_common::{not_impl_err, plan_datafusion_err, Result}; use datafusion_expr::{AggregateUDF, ScalarUDF, UserDefinedLogicalNode, WindowUDF}; use std::collections::HashMap; use std::{collections::HashSet, sync::Arc}; diff --git a/datafusion/expr/src/accumulator.rs b/datafusion/expr/src/accumulator.rs index fa2017586d21..031348269a38 100644 --- a/datafusion/expr/src/accumulator.rs +++ b/datafusion/expr/src/accumulator.rs @@ -18,7 +18,7 @@ //! Accumulator module contains the trait definition for aggregation function's accumulators. use arrow::array::ArrayRef; -use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{internal_err, Result, ScalarValue}; use std::fmt::Debug; /// Tracks an aggregate function's state. diff --git a/datafusion/expr/src/columnar_value.rs b/datafusion/expr/src/columnar_value.rs index 585bee3b9bfa..c845c81cb708 100644 --- a/datafusion/expr/src/columnar_value.rs +++ b/datafusion/expr/src/columnar_value.rs @@ -20,7 +20,7 @@ use arrow::array::ArrayRef; use arrow::array::NullArray; use arrow::datatypes::DataType; -use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{internal_err, Result, ScalarValue}; use std::sync::Arc; /// Represents the result of evaluating an expression: either a single diff --git a/datafusion/expr/src/conditional_expressions.rs b/datafusion/expr/src/conditional_expressions.rs index c31bd04eafa0..1346825f054d 100644 --- a/datafusion/expr/src/conditional_expressions.rs +++ b/datafusion/expr/src/conditional_expressions.rs @@ -19,7 +19,7 @@ use crate::expr::Case; use crate::{expr_schema::ExprSchemable, Expr}; use arrow::datatypes::DataType; -use datafusion_common::{plan_err, DFSchema, DataFusionError, Result}; +use datafusion_common::{plan_err, DFSchema, Result}; use std::collections::HashSet; /// Currently supported types by the coalesce function. diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index c3d9269d1559..43f6ba8f6309 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -29,7 +29,7 @@ use crate::{built_in_window_function, udaf}; use arrow::datatypes::DataType; use datafusion_common::tree_node::{Transformed, TreeNode}; use datafusion_common::{internal_err, DFSchema, OwnedTableReference}; -use datafusion_common::{plan_err, Column, DataFusionError, Result, ScalarValue}; +use datafusion_common::{plan_err, Column, Result, ScalarValue}; use sqlparser::ast::NullTreatment; use std::collections::HashSet; use std::fmt; diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs index 491b4a852261..a453730a0e71 100644 --- a/datafusion/expr/src/expr_schema.rs +++ b/datafusion/expr/src/expr_schema.rs @@ -28,8 +28,7 @@ use crate::{utils, LogicalPlan, Projection, Subquery}; use arrow::compute::can_cast_types; use arrow::datatypes::{DataType, Field}; use datafusion_common::{ - internal_err, plan_datafusion_err, plan_err, Column, DFField, DataFusionError, - ExprSchema, Result, + internal_err, plan_datafusion_err, plan_err, Column, DFField, ExprSchema, Result, }; use std::collections::HashMap; use std::sync::Arc; diff --git a/datafusion/expr/src/field_util.rs b/datafusion/expr/src/field_util.rs index c46ec50234dd..3195ce6f2dfc 100644 --- a/datafusion/expr/src/field_util.rs +++ b/datafusion/expr/src/field_util.rs @@ -18,9 +18,7 @@ //! Utility functions for complex field access use arrow::datatypes::{DataType, Field}; -use datafusion_common::{ - plan_datafusion_err, plan_err, DataFusionError, Result, ScalarValue, -}; +use datafusion_common::{plan_datafusion_err, plan_err, Result, ScalarValue}; /// Types of the field access expression of a nested type, such as `Field` or `List` pub enum GetFieldAccessSchema { diff --git a/datafusion/expr/src/interval_arithmetic.rs b/datafusion/expr/src/interval_arithmetic.rs index 5d34fe91c3ac..ca91a8c9da00 100644 --- a/datafusion/expr/src/interval_arithmetic.rs +++ b/datafusion/expr/src/interval_arithmetic.rs @@ -28,7 +28,7 @@ use arrow::compute::{cast_with_options, CastOptions}; use arrow::datatypes::DataType; use arrow::datatypes::{IntervalUnit, TimeUnit}; use datafusion_common::rounding::{alter_fp_rounding_mode, next_down, next_up}; -use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{internal_err, Result, ScalarValue}; macro_rules! get_extreme_value { ($extreme:ident, $value:expr) => { diff --git a/datafusion/expr/src/partition_evaluator.rs b/datafusion/expr/src/partition_evaluator.rs index 4b5357ddf8ba..04b6faf55ae1 100644 --- a/datafusion/expr/src/partition_evaluator.rs +++ b/datafusion/expr/src/partition_evaluator.rs @@ -18,7 +18,7 @@ //! Partition evaluation module use arrow::array::ArrayRef; -use datafusion_common::{exec_err, not_impl_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{exec_err, not_impl_err, Result, ScalarValue}; use std::fmt::Debug; use std::ops::Range; diff --git a/datafusion/expr/src/tree_node/expr.rs b/datafusion/expr/src/tree_node/expr.rs index def25ed9242f..81949f2178f6 100644 --- a/datafusion/expr/src/tree_node/expr.rs +++ b/datafusion/expr/src/tree_node/expr.rs @@ -25,7 +25,7 @@ use crate::expr::{ use crate::{Expr, GetFieldAccess}; use datafusion_common::tree_node::{TreeNode, VisitRecursion}; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; impl TreeNode for Expr { fn apply_children Result>( diff --git a/datafusion/expr/src/type_coercion/aggregates.rs b/datafusion/expr/src/type_coercion/aggregates.rs index ab994c143ac2..866aea06b4d4 100644 --- a/datafusion/expr/src/type_coercion/aggregates.rs +++ b/datafusion/expr/src/type_coercion/aggregates.rs @@ -24,7 +24,7 @@ use arrow::datatypes::{ DataType, TimeUnit, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE, }; -use datafusion_common::{internal_err, plan_err, DataFusionError, Result}; +use datafusion_common::{internal_err, plan_err, Result}; pub static STRINGS: &[DataType] = &[DataType::Utf8, DataType::LargeUtf8]; diff --git a/datafusion/expr/src/type_coercion/binary.rs b/datafusion/expr/src/type_coercion/binary.rs index 70015c699296..118844e4b266 100644 --- a/datafusion/expr/src/type_coercion/binary.rs +++ b/datafusion/expr/src/type_coercion/binary.rs @@ -28,9 +28,7 @@ use arrow::datatypes::{ DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE, }; -use datafusion_common::{ - exec_datafusion_err, plan_datafusion_err, plan_err, DataFusionError, Result, -}; +use datafusion_common::{exec_datafusion_err, plan_datafusion_err, plan_err, Result}; /// The type signature of an instantiation of binary operator expression such as /// `lhs + rhs` diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rs index 2022d67879f8..e9878fd17e8d 100644 --- a/datafusion/expr/src/type_coercion/functions.rs +++ b/datafusion/expr/src/type_coercion/functions.rs @@ -22,9 +22,7 @@ use arrow::{ datatypes::{DataType, TimeUnit}, }; use datafusion_common::utils::{coerced_fixed_size_list_to_list, list_ndims}; -use datafusion_common::{ - internal_datafusion_err, internal_err, plan_err, DataFusionError, Result, -}; +use datafusion_common::{internal_datafusion_err, internal_err, plan_err, Result}; use super::binary::comparison_coercion; diff --git a/datafusion/expr/src/udaf.rs b/datafusion/expr/src/udaf.rs index fb062e5830eb..6ff7730bd606 100644 --- a/datafusion/expr/src/udaf.rs +++ b/datafusion/expr/src/udaf.rs @@ -23,7 +23,7 @@ use crate::{ AccumulatorFactoryFunction, ReturnTypeFunction, Signature, StateTypeFunction, }; use arrow::datatypes::DataType; -use datafusion_common::{not_impl_err, DataFusionError, Result}; +use datafusion_common::{not_impl_err, Result}; use std::any::Any; use std::fmt::{self, Debug, Formatter}; use std::sync::Arc; diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs index 2fda81d8896f..fe9297b32a8e 100644 --- a/datafusion/expr/src/utils.rs +++ b/datafusion/expr/src/utils.rs @@ -35,7 +35,7 @@ use datafusion_common::tree_node::{TreeNode, VisitRecursion}; use datafusion_common::utils::get_at_indices; use datafusion_common::{ internal_err, plan_datafusion_err, plan_err, Column, DFField, DFSchema, DFSchemaRef, - DataFusionError, Result, ScalarValue, TableReference, + Result, ScalarValue, TableReference, }; use sqlparser::ast::{ExceptSelectItem, ExcludeSelectItem, WildcardAdditionalOptions}; @@ -129,14 +129,15 @@ fn check_grouping_sets_size_limit(size: usize) -> Result<()> { /// Merge two grouping_set /// -/// -/// Example: -/// +/// # Example +/// ```text /// (A, B), (C, D) -> (A, B, C, D) +/// ``` /// -/// Error: +/// # Error +/// - [`DataFusionError`]: The number of group_expression in grouping_set exceeds the maximum limit /// -/// [`DataFusionError`] The number of group_expression in grouping_set exceeds the maximum limit +/// [`DataFusionError`]: datafusion_common::DataFusionError fn merge_grouping_set(left: &[T], right: &[T]) -> Result> { check_grouping_set_size_limit(left.len() + right.len())?; Ok(left.iter().chain(right.iter()).cloned().collect()) @@ -144,15 +145,16 @@ fn merge_grouping_set(left: &[T], right: &[T]) -> Result> { /// Compute the cross product of two grouping_sets /// +/// # Example +/// ```text +/// [(A, B), (C, D)], [(E), (F)] -> [(A, B, E), (A, B, F), (C, D, E), (C, D, F)] +/// ``` /// -/// Example: -/// -/// \[(A, B), (C, D)], [(E), (F)\] -> \[(A, B, E), (A, B, F), (C, D, E), (C, D, F)\] -/// -/// Error: +/// # Error +/// - [`DataFusionError`]: The number of group_expression in grouping_set exceeds the maximum limit +/// - [`DataFusionError`]: The number of grouping_set in grouping_sets exceeds the maximum limit /// -/// [`DataFusionError`] The number of group_expression in grouping_set exceeds the maximum limit \ -/// [`DataFusionError`] The number of grouping_set in grouping_sets exceeds the maximum limit +/// [`DataFusionError`]: datafusion_common::DataFusionError fn cross_join_grouping_sets( left: &[Vec], right: &[Vec], diff --git a/datafusion/functions-array/src/udf.rs b/datafusion/functions-array/src/udf.rs index b7f9d2497fb7..79fb83c059a4 100644 --- a/datafusion/functions-array/src/udf.rs +++ b/datafusion/functions-array/src/udf.rs @@ -18,7 +18,7 @@ //! [`ScalarUDFImpl`] definitions for array functions. use arrow::datatypes::DataType; -use datafusion_common::{plan_err, DataFusionError}; +use datafusion_common::plan_err; use datafusion_expr::expr::ScalarFunction; use datafusion_expr::Expr; use datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; diff --git a/datafusion/functions/src/core/nullif.rs b/datafusion/functions/src/core/nullif.rs index 73bfba9b38b1..afb308e441f5 100644 --- a/datafusion/functions/src/core/nullif.rs +++ b/datafusion/functions/src/core/nullif.rs @@ -18,7 +18,7 @@ //! Encoding expressions use arrow::datatypes::DataType; -use datafusion_common::{exec_err, DataFusionError, Result}; +use datafusion_common::{exec_err, Result}; use datafusion_expr::ColumnarValue; use arrow::array::Array; diff --git a/datafusion/functions/src/core/nvl.rs b/datafusion/functions/src/core/nvl.rs index caf095ecbbbd..578aaeda2e89 100644 --- a/datafusion/functions/src/core/nvl.rs +++ b/datafusion/functions/src/core/nvl.rs @@ -16,7 +16,7 @@ // under the License. use arrow::datatypes::DataType; -use datafusion_common::{internal_err, Result, DataFusionError}; +use datafusion_common::{internal_err, Result}; use datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; use arrow::compute::kernels::zip::zip; use arrow::compute::is_not_null; diff --git a/datafusion/optimizer/src/analyzer/subquery.rs b/datafusion/optimizer/src/analyzer/subquery.rs index 7c5b70b19af0..a0e972fc703c 100644 --- a/datafusion/optimizer/src/analyzer/subquery.rs +++ b/datafusion/optimizer/src/analyzer/subquery.rs @@ -18,7 +18,7 @@ use crate::analyzer::check_plan; use crate::utils::collect_subquery_cols; use datafusion_common::tree_node::{TreeNode, VisitRecursion}; -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_expr::expr_rewriter::strip_outer_reference; use datafusion_expr::utils::split_conjunction; use datafusion_expr::{ diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs index b1000f042c98..0f4b39d9eee3 100644 --- a/datafusion/optimizer/src/decorrelate.rs +++ b/datafusion/optimizer/src/decorrelate.rs @@ -21,7 +21,7 @@ use datafusion_common::tree_node::{ RewriteRecursion, Transformed, TreeNode, TreeNodeRewriter, }; use datafusion_common::{plan_err, Result}; -use datafusion_common::{Column, DFSchemaRef, DataFusionError, ScalarValue}; +use datafusion_common::{Column, DFSchemaRef, ScalarValue}; use datafusion_expr::expr::{AggregateFunctionDefinition, Alias}; use datafusion_expr::utils::{conjunction, find_join_exprs, split_conjunction}; use datafusion_expr::{expr, EmptyRelation, Expr, LogicalPlan, LogicalPlanBuilder}; diff --git a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs index 450336376a23..a9e1f1228e5e 100644 --- a/datafusion/optimizer/src/decorrelate_predicate_subquery.rs +++ b/datafusion/optimizer/src/decorrelate_predicate_subquery.rs @@ -21,7 +21,7 @@ use crate::utils::replace_qualified_name; use crate::{OptimizerConfig, OptimizerRule}; use datafusion_common::alias::AliasGenerator; use datafusion_common::tree_node::TreeNode; -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_expr::expr::{Exists, InSubquery}; use datafusion_expr::expr_rewriter::create_col_from_scalar_expr; use datafusion_expr::logical_plan::{JoinType, Subquery}; diff --git a/datafusion/optimizer/src/eliminate_cross_join.rs b/datafusion/optimizer/src/eliminate_cross_join.rs index d9e96a9f2543..7f65690a4a7c 100644 --- a/datafusion/optimizer/src/eliminate_cross_join.rs +++ b/datafusion/optimizer/src/eliminate_cross_join.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use crate::{utils, OptimizerConfig, OptimizerRule}; -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_expr::expr::{BinaryExpr, Expr}; use datafusion_expr::logical_plan::{ CrossJoin, Filter, Join, JoinConstraint, JoinType, LogicalPlan, Projection, diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs index 633a32996d1c..93b3d6b8b9f2 100644 --- a/datafusion/optimizer/src/optimizer.rs +++ b/datafusion/optimizer/src/optimizer.rs @@ -458,9 +458,7 @@ mod tests { use crate::test::test_table_scan; use crate::{OptimizerConfig, OptimizerContext, OptimizerRule}; - use datafusion_common::{ - plan_err, DFField, DFSchema, DFSchemaRef, DataFusionError, Result, - }; + use datafusion_common::{plan_err, DFField, DFSchema, DFSchemaRef, Result}; use datafusion_expr::logical_plan::EmptyRelation; use datafusion_expr::{col, lit, LogicalPlan, LogicalPlanBuilder, Projection}; diff --git a/datafusion/optimizer/src/propagate_empty_relation.rs b/datafusion/optimizer/src/propagate_empty_relation.rs index 040b69fc8bf3..d1f9f87a32a3 100644 --- a/datafusion/optimizer/src/propagate_empty_relation.rs +++ b/datafusion/optimizer/src/propagate_empty_relation.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_expr::logical_plan::LogicalPlan; use datafusion_expr::{EmptyRelation, JoinType, Projection, Union}; use std::sync::Arc; diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index acdda6833285..40156d43c572 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -24,8 +24,8 @@ use crate::{OptimizerConfig, OptimizerRule}; use datafusion_common::tree_node::{Transformed, TreeNode, VisitRecursion}; use datafusion_common::{ - internal_err, plan_datafusion_err, Column, DFSchema, DFSchemaRef, DataFusionError, - JoinConstraint, Result, + internal_err, plan_datafusion_err, Column, DFSchema, DFSchemaRef, JoinConstraint, + Result, }; use datafusion_expr::expr::Alias; use datafusion_expr::expr_rewriter::replace_col; diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rs index 34ed4a9475cb..9aa08c37fa35 100644 --- a/datafusion/optimizer/src/scalar_subquery_to_join.rs +++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs @@ -23,7 +23,7 @@ use datafusion_common::alias::AliasGenerator; use datafusion_common::tree_node::{ RewriteRecursion, Transformed, TreeNode, TreeNodeRewriter, }; -use datafusion_common::{plan_err, Column, DataFusionError, Result, ScalarValue}; +use datafusion_common::{plan_err, Column, Result, ScalarValue}; use datafusion_expr::expr_rewriter::create_col_from_scalar_expr; use datafusion_expr::logical_plan::{JoinType, Subquery}; use datafusion_expr::utils::conjunction; diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs index fa91a3ace2a2..c9736061df90 100644 --- a/datafusion/optimizer/src/simplify_expressions/utils.rs +++ b/datafusion/optimizer/src/simplify_expressions/utils.rs @@ -18,7 +18,7 @@ //! Utility functions for expression simplification use crate::simplify_expressions::SimplifyInfo; -use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{internal_err, Result, ScalarValue}; use datafusion_expr::expr::ScalarFunction; use datafusion_expr::{ expr::{Between, BinaryExpr, InList}, diff --git a/datafusion/optimizer/src/unwrap_cast_in_comparison.rs b/datafusion/optimizer/src/unwrap_cast_in_comparison.rs index 9d3561d12671..4c22742c8635 100644 --- a/datafusion/optimizer/src/unwrap_cast_in_comparison.rs +++ b/datafusion/optimizer/src/unwrap_cast_in_comparison.rs @@ -25,9 +25,7 @@ use arrow::datatypes::{ }; use arrow::temporal_conversions::{MICROSECONDS, MILLISECONDS, NANOSECONDS}; use datafusion_common::tree_node::{RewriteRecursion, TreeNodeRewriter}; -use datafusion_common::{ - internal_err, DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue, -}; +use datafusion_common::{internal_err, DFSchema, DFSchemaRef, Result, ScalarValue}; use datafusion_expr::expr::{BinaryExpr, Cast, InList, TryCast}; use datafusion_expr::expr_rewriter::rewrite_preserving_name; use datafusion_expr::utils::merge_schema; diff --git a/datafusion/optimizer/tests/optimizer_integration.rs b/datafusion/optimizer/tests/optimizer_integration.rs index d857c6154ea9..fe1234de5ab8 100644 --- a/datafusion/optimizer/tests/optimizer_integration.rs +++ b/datafusion/optimizer/tests/optimizer_integration.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; use datafusion_common::config::ConfigOptions; -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_expr::{AggregateUDF, LogicalPlan, ScalarUDF, TableSource, WindowUDF}; use datafusion_optimizer::analyzer::Analyzer; use datafusion_optimizer::optimizer::Optimizer; diff --git a/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs b/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs index b073b00578a5..8e7b9d91ee49 100644 --- a/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs +++ b/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs @@ -188,7 +188,7 @@ mod tests { use arrow_array::Array; use arrow_array::ListArray; use arrow_buffer::OffsetBuffer; - use datafusion_common::{internal_err, DataFusionError}; + use datafusion_common::internal_err; // arrow::compute::sort can't sort nested ListArray directly, so we compare the scalar values pair-wise. fn compare_list_contents( diff --git a/datafusion/physical-expr/src/aggregate/array_agg_ordered.rs b/datafusion/physical-expr/src/aggregate/array_agg_ordered.rs index 587f40081c90..7e2c7bb27144 100644 --- a/datafusion/physical-expr/src/aggregate/array_agg_ordered.rs +++ b/datafusion/physical-expr/src/aggregate/array_agg_ordered.rs @@ -38,7 +38,7 @@ use arrow_schema::{Fields, SortOptions}; use datafusion_common::utils::array_into_list_array; use datafusion_common::utils::{compare_rows, get_row_at_idx}; -use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{exec_err, Result, ScalarValue}; use datafusion_expr::Accumulator; /// Expression for a `ARRAY_AGG(... ORDER BY ..., ...)` aggregation. In a multi diff --git a/datafusion/physical-expr/src/aggregate/average.rs b/datafusion/physical-expr/src/aggregate/average.rs index 57f8fa211e58..f06355293d7c 100644 --- a/datafusion/physical-expr/src/aggregate/average.rs +++ b/datafusion/physical-expr/src/aggregate/average.rs @@ -39,7 +39,7 @@ use arrow_array::{ Array, ArrowNativeTypeOp, ArrowNumericType, ArrowPrimitiveType, PrimitiveArray, }; use arrow_buffer::{i256, ArrowNativeType}; -use datafusion_common::{not_impl_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{not_impl_err, Result, ScalarValue}; use datafusion_expr::type_coercion::aggregates::avg_return_type; use datafusion_expr::{Accumulator, EmitTo, GroupsAccumulator}; diff --git a/datafusion/physical-expr/src/aggregate/build_in.rs b/datafusion/physical-expr/src/aggregate/build_in.rs index 2918856aa623..0aaf0dc0c8c5 100644 --- a/datafusion/physical-expr/src/aggregate/build_in.rs +++ b/datafusion/physical-expr/src/aggregate/build_in.rs @@ -30,7 +30,7 @@ use std::sync::Arc; use arrow::datatypes::Schema; -use datafusion_common::{exec_err, not_impl_err, DataFusionError, Result}; +use datafusion_common::{exec_err, not_impl_err, Result}; use datafusion_expr::AggregateFunction; use crate::aggregate::regr::RegrType; @@ -416,7 +416,7 @@ pub fn create_aggregate_expr( mod tests { use arrow::datatypes::{DataType, Field}; - use datafusion_common::{plan_err, ScalarValue}; + use datafusion_common::{plan_err, DataFusionError, ScalarValue}; use datafusion_expr::type_coercion::aggregates::NUMERICS; use datafusion_expr::{type_coercion, Signature}; diff --git a/datafusion/physical-expr/src/aggregate/grouping.rs b/datafusion/physical-expr/src/aggregate/grouping.rs index 70afda265aea..d43bcd5c7091 100644 --- a/datafusion/physical-expr/src/aggregate/grouping.rs +++ b/datafusion/physical-expr/src/aggregate/grouping.rs @@ -24,7 +24,7 @@ use crate::aggregate::utils::down_cast_any_ref; use crate::{AggregateExpr, PhysicalExpr}; use arrow::datatypes::DataType; use arrow::datatypes::Field; -use datafusion_common::{not_impl_err, DataFusionError, Result}; +use datafusion_common::{not_impl_err, Result}; use datafusion_expr::Accumulator; use crate::expressions::format_state_name; diff --git a/datafusion/physical-expr/src/aggregate/mod.rs b/datafusion/physical-expr/src/aggregate/mod.rs index 2bb205ce90dc..893178f29d08 100644 --- a/datafusion/physical-expr/src/aggregate/mod.rs +++ b/datafusion/physical-expr/src/aggregate/mod.rs @@ -23,7 +23,7 @@ use crate::expressions::{NthValueAgg, OrderSensitiveArrayAgg}; use crate::{PhysicalExpr, PhysicalSortExpr}; use arrow::datatypes::Field; -use datafusion_common::{not_impl_err, DataFusionError, Result}; +use datafusion_common::{not_impl_err, Result}; use datafusion_expr::{Accumulator, GroupsAccumulator}; mod hyperloglog; diff --git a/datafusion/physical-expr/src/aggregate/nth_value.rs b/datafusion/physical-expr/src/aggregate/nth_value.rs index 5d721e3a5e87..dba259a507fd 100644 --- a/datafusion/physical-expr/src/aggregate/nth_value.rs +++ b/datafusion/physical-expr/src/aggregate/nth_value.rs @@ -33,7 +33,7 @@ use arrow_array::cast::AsArray; use arrow_array::{new_empty_array, ArrayRef, StructArray}; use arrow_schema::{DataType, Field, Fields}; use datafusion_common::utils::{array_into_list_array, get_row_at_idx}; -use datafusion_common::{exec_err, internal_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{exec_err, internal_err, Result, ScalarValue}; use datafusion_expr::Accumulator; /// Expression for a `NTH_VALUE(... ORDER BY ..., ...)` aggregation. In a multi diff --git a/datafusion/physical-expr/src/aggregate/stddev.rs b/datafusion/physical-expr/src/aggregate/stddev.rs index dcc2b0e69c02..6033d63cbe21 100644 --- a/datafusion/physical-expr/src/aggregate/stddev.rs +++ b/datafusion/physical-expr/src/aggregate/stddev.rs @@ -27,7 +27,7 @@ use crate::expressions::format_state_name; use crate::{AggregateExpr, PhysicalExpr}; use arrow::{array::ArrayRef, datatypes::DataType, datatypes::Field}; use datafusion_common::ScalarValue; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_expr::Accumulator; /// STDDEV and STDDEV_SAMP (standard deviation) aggregate expression diff --git a/datafusion/physical-expr/src/aggregate/string_agg.rs b/datafusion/physical-expr/src/aggregate/string_agg.rs index 7a1da6d62246..8993c630aa49 100644 --- a/datafusion/physical-expr/src/aggregate/string_agg.rs +++ b/datafusion/physical-expr/src/aggregate/string_agg.rs @@ -23,7 +23,7 @@ use crate::{AggregateExpr, PhysicalExpr}; use arrow::array::ArrayRef; use arrow::datatypes::{DataType, Field}; use datafusion_common::cast::as_generic_string_array; -use datafusion_common::{not_impl_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{not_impl_err, Result, ScalarValue}; use datafusion_expr::Accumulator; use std::any::Any; use std::sync::Arc; diff --git a/datafusion/physical-expr/src/aggregate/sum.rs b/datafusion/physical-expr/src/aggregate/sum.rs index 6cf2810ce588..f19be62bbc95 100644 --- a/datafusion/physical-expr/src/aggregate/sum.rs +++ b/datafusion/physical-expr/src/aggregate/sum.rs @@ -33,7 +33,7 @@ use arrow_array::types::{ }; use arrow_array::{Array, ArrowNativeTypeOp, ArrowNumericType}; use arrow_buffer::ArrowNativeType; -use datafusion_common::{not_impl_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{not_impl_err, Result, ScalarValue}; use datafusion_expr::type_coercion::aggregates::sum_return_type; use datafusion_expr::{Accumulator, GroupsAccumulator}; diff --git a/datafusion/physical-expr/src/aggregate/sum_distinct.rs b/datafusion/physical-expr/src/aggregate/sum_distinct.rs index 4c0f94b3a2bb..a62a7b08da35 100644 --- a/datafusion/physical-expr/src/aggregate/sum_distinct.rs +++ b/datafusion/physical-expr/src/aggregate/sum_distinct.rs @@ -31,7 +31,7 @@ use std::collections::HashSet; use crate::aggregate::sum::downcast_sum; use crate::aggregate::utils::{down_cast_any_ref, Hashable}; use crate::{AggregateExpr, PhysicalExpr}; -use datafusion_common::{not_impl_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{not_impl_err, Result, ScalarValue}; use datafusion_expr::type_coercion::aggregates::sum_return_type; use datafusion_expr::Accumulator; diff --git a/datafusion/physical-expr/src/analysis.rs b/datafusion/physical-expr/src/analysis.rs index 6d36e2233cdd..ca25bfd647b6 100644 --- a/datafusion/physical-expr/src/analysis.rs +++ b/datafusion/physical-expr/src/analysis.rs @@ -27,9 +27,7 @@ use crate::PhysicalExpr; use arrow::datatypes::Schema; use datafusion_common::stats::Precision; -use datafusion_common::{ - internal_err, ColumnStatistics, DataFusionError, Result, ScalarValue, -}; +use datafusion_common::{internal_err, ColumnStatistics, Result, ScalarValue}; use datafusion_expr::interval_arithmetic::{cardinality_ratio, Interval}; /// The shared context used during the analysis of an expression. Includes diff --git a/datafusion/physical-expr/src/conditional_expressions.rs b/datafusion/physical-expr/src/conditional_expressions.rs index cc8f3c8dfaf0..87d63bfd32e2 100644 --- a/datafusion/physical-expr/src/conditional_expressions.rs +++ b/datafusion/physical-expr/src/conditional_expressions.rs @@ -19,7 +19,7 @@ use arrow::array::{new_null_array, Array, BooleanArray}; use arrow::compute::kernels::zip::zip; use arrow::compute::{and, is_not_null, is_null}; -use datafusion_common::{exec_err, DataFusionError, Result}; +use datafusion_common::{exec_err, Result}; use datafusion_expr::ColumnarValue; /// coalesce evaluates to the first value which is not NULL diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs index 387dce2cdc8b..a31be06ecf0b 100644 --- a/datafusion/physical-expr/src/equivalence/mod.rs +++ b/datafusion/physical-expr/src/equivalence/mod.rs @@ -68,7 +68,7 @@ mod tests { use arrow::datatypes::{DataType, Field, Schema}; use arrow_array::{ArrayRef, Float64Array, RecordBatch, UInt32Array}; use arrow_schema::{SchemaRef, SortOptions}; - use datafusion_common::{plan_datafusion_err, DataFusionError, Result}; + use datafusion_common::{plan_datafusion_err, Result}; use itertools::izip; use rand::rngs::StdRng; use rand::seq::SliceRandom; diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 3f13030092c1..f1842458d5c4 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -38,7 +38,7 @@ use arrow::datatypes::*; use arrow::record_batch::RecordBatch; use datafusion_common::cast::as_boolean_array; -use datafusion_common::{internal_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{internal_err, Result, ScalarValue}; use datafusion_expr::interval_arithmetic::{apply_operator, Interval}; use datafusion_expr::type_coercion::binary::get_result_type; use datafusion_expr::{ColumnarValue, Operator}; diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rs index 22cadec40940..b0736e140fec 100644 --- a/datafusion/physical-expr/src/expressions/binary/kernels.rs +++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs @@ -25,7 +25,7 @@ use arrow::compute::kernels::bitwise::{ }; use arrow::datatypes::DataType; use datafusion_common::internal_err; -use datafusion_common::{DataFusionError, Result, ScalarValue}; +use datafusion_common::{Result, ScalarValue}; use std::sync::Arc; diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index b0e175e711fe..9125f73048cb 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -28,7 +28,7 @@ use arrow::compute::{can_cast_types, kernels, CastOptions}; use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; use datafusion_common::format::DEFAULT_FORMAT_OPTIONS; -use datafusion_common::{not_impl_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{not_impl_err, Result, ScalarValue}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::ColumnarValue; diff --git a/datafusion/physical-expr/src/expressions/column.rs b/datafusion/physical-expr/src/expressions/column.rs index 62da8ff9ed44..a07f36e785e3 100644 --- a/datafusion/physical-expr/src/expressions/column.rs +++ b/datafusion/physical-expr/src/expressions/column.rs @@ -28,7 +28,7 @@ use arrow::{ datatypes::{DataType, Schema}, record_batch::RecordBatch, }; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_expr::ColumnarValue; /// Represents the column at a given index in a RecordBatch diff --git a/datafusion/physical-expr/src/expressions/get_indexed_field.rs b/datafusion/physical-expr/src/expressions/get_indexed_field.rs index 58fe4728543d..39eef61f963a 100644 --- a/datafusion/physical-expr/src/expressions/get_indexed_field.rs +++ b/datafusion/physical-expr/src/expressions/get_indexed_field.rs @@ -30,7 +30,7 @@ use arrow::{ }; use datafusion_common::{ cast::{as_map_array, as_struct_array}, - DataFusionError, Result, ScalarValue, + Result, ScalarValue, }; use datafusion_expr::{field_util::GetFieldAccessSchema, ColumnarValue}; use std::fmt::Debug; diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index 1a1634081c38..ecdb03e97ee3 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -38,9 +38,7 @@ use datafusion_common::cast::{ as_boolean_array, as_generic_binary_array, as_string_array, }; use datafusion_common::hash_utils::HashValue; -use datafusion_common::{ - exec_err, internal_err, not_impl_err, DataFusionError, Result, ScalarValue, -}; +use datafusion_common::{exec_err, internal_err, not_impl_err, Result, ScalarValue}; use datafusion_expr::ColumnarValue; use ahash::RandomState; diff --git a/datafusion/physical-expr/src/expressions/like.rs b/datafusion/physical-expr/src/expressions/like.rs index 37452e278484..6e0beeb0beea 100644 --- a/datafusion/physical-expr/src/expressions/like.rs +++ b/datafusion/physical-expr/src/expressions/like.rs @@ -23,7 +23,7 @@ use crate::{physical_expr::down_cast_any_ref, PhysicalExpr}; use crate::expressions::datum::apply_cmp; use arrow::record_batch::RecordBatch; use arrow_schema::{DataType, Schema}; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_expr::ColumnarValue; // Like expression diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs index 09e908586c5b..ec20345569c2 100644 --- a/datafusion/physical-expr/src/expressions/mod.rs +++ b/datafusion/physical-expr/src/expressions/mod.rs @@ -133,7 +133,7 @@ pub(crate) mod tests { assert_eq!(expected, actual); - Ok(()) as Result<(), DataFusionError> + Ok(()) as Result<(), ::datafusion_common::DataFusionError> }}; } @@ -166,7 +166,7 @@ pub(crate) mod tests { let actual = aggregate_new(&batch, agg)?; assert_eq!($EXPECTED, &actual); - Ok(()) as Result<(), DataFusionError> + Ok(()) as Result<(), ::datafusion_common::DataFusionError> }}; } diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rs index 6b5c208bae81..d6dd3ddbea5e 100644 --- a/datafusion/physical-expr/src/expressions/negative.rs +++ b/datafusion/physical-expr/src/expressions/negative.rs @@ -30,7 +30,7 @@ use arrow::{ datatypes::{DataType, Schema}, record_batch::RecordBatch, }; -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::{ type_coercion::{is_interval, is_null, is_signed_numeric, is_timestamp}, @@ -179,6 +179,7 @@ mod tests { use arrow::datatypes::*; use arrow_schema::DataType::{Float32, Float64, Int16, Int32, Int64, Int8}; use datafusion_common::cast::as_primitive_array; + use datafusion_common::DataFusionError; use datafusion_common::Result; use paste::paste; diff --git a/datafusion/physical-expr/src/expressions/no_op.rs b/datafusion/physical-expr/src/expressions/no_op.rs index 95e6879a6c2d..b558ccab154d 100644 --- a/datafusion/physical-expr/src/expressions/no_op.rs +++ b/datafusion/physical-expr/src/expressions/no_op.rs @@ -28,7 +28,7 @@ use arrow::{ use crate::physical_expr::down_cast_any_ref; use crate::PhysicalExpr; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_expr::ColumnarValue; /// A place holder expression, can not be evaluated. diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs index 0f7909097a10..ddfe49dda7a3 100644 --- a/datafusion/physical-expr/src/expressions/try_cast.rs +++ b/datafusion/physical-expr/src/expressions/try_cast.rs @@ -28,7 +28,7 @@ use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; use compute::can_cast_types; use datafusion_common::format::DEFAULT_FORMAT_OPTIONS; -use datafusion_common::{not_impl_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{not_impl_err, Result, ScalarValue}; use datafusion_expr::ColumnarValue; /// TRY_CAST expression casts an expression to a specific data type and retuns NULL on invalid cast diff --git a/datafusion/physical-expr/src/functions.rs b/datafusion/physical-expr/src/functions.rs index 186de0609b9a..d2b9a68ef8b9 100644 --- a/datafusion/physical-expr/src/functions.rs +++ b/datafusion/physical-expr/src/functions.rs @@ -42,7 +42,7 @@ use arrow::{ datatypes::{DataType, Int32Type, Int64Type, Schema}, }; use arrow_array::Array; -use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{exec_err, Result, ScalarValue}; pub use datafusion_expr::FuncMonotonicity; use datafusion_expr::{ type_coercion::functions::data_types, BuiltinScalarFunction, ColumnarValue, @@ -998,7 +998,7 @@ mod tests { }; use datafusion_common::cast::{as_boolean_array, as_uint64_array}; use datafusion_common::{exec_err, internal_err, plan_err}; - use datafusion_common::{Result, ScalarValue}; + use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_expr::type_coercion::functions::data_types; use datafusion_expr::Signature; diff --git a/datafusion/physical-expr/src/intervals/cp_solver.rs b/datafusion/physical-expr/src/intervals/cp_solver.rs index b2403dadf05a..3bd059afa6be 100644 --- a/datafusion/physical-expr/src/intervals/cp_solver.rs +++ b/datafusion/physical-expr/src/intervals/cp_solver.rs @@ -29,7 +29,7 @@ use crate::utils::{build_dag, ExprTreeNode}; use crate::PhysicalExpr; use arrow_schema::{DataType, Schema}; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_expr::interval_arithmetic::{apply_operator, satisfy_greater, Interval}; use datafusion_expr::Operator; diff --git a/datafusion/physical-expr/src/intervals/utils.rs b/datafusion/physical-expr/src/intervals/utils.rs index 03d13632104d..e188b2d56bae 100644 --- a/datafusion/physical-expr/src/intervals/utils.rs +++ b/datafusion/physical-expr/src/intervals/utils.rs @@ -25,9 +25,7 @@ use crate::{ }; use arrow_schema::{DataType, SchemaRef}; -use datafusion_common::{ - internal_datafusion_err, internal_err, DataFusionError, Result, ScalarValue, -}; +use datafusion_common::{internal_datafusion_err, internal_err, Result, ScalarValue}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::Operator; diff --git a/datafusion/physical-expr/src/physical_expr.rs b/datafusion/physical-expr/src/physical_expr.rs index e596cb2e6ceb..567054e2b59e 100644 --- a/datafusion/physical-expr/src/physical_expr.rs +++ b/datafusion/physical-expr/src/physical_expr.rs @@ -28,7 +28,7 @@ use arrow::compute::filter_record_batch; use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; use datafusion_common::utils::DataPtr; -use datafusion_common::{internal_err, not_impl_err, DataFusionError, Result}; +use datafusion_common::{internal_err, not_impl_err, Result}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::ColumnarValue; diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index b8491aea2d6f..bf279518d31d 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -26,8 +26,7 @@ use crate::{ }; use arrow::datatypes::Schema; use datafusion_common::{ - exec_err, internal_err, not_impl_err, plan_err, DFSchema, DataFusionError, Result, - ScalarValue, + exec_err, internal_err, not_impl_err, plan_err, DFSchema, Result, ScalarValue, }; use datafusion_expr::expr::{Alias, Cast, InList, ScalarFunction}; use datafusion_expr::{ diff --git a/datafusion/physical-expr/src/string_expressions.rs b/datafusion/physical-expr/src/string_expressions.rs index 6a4a29763e4b..ace7ef2888a3 100644 --- a/datafusion/physical-expr/src/string_expressions.rs +++ b/datafusion/physical-expr/src/string_expressions.rs @@ -37,13 +37,13 @@ use arrow::{ use uuid::Uuid; use datafusion_common::utils::datafusion_strsim; +use datafusion_common::Result; use datafusion_common::{ cast::{ as_generic_string_array, as_int64_array, as_primitive_array, as_string_array, }, exec_err, ScalarValue, }; -use datafusion_common::{DataFusionError, Result}; use datafusion_expr::ColumnarValue; /// applies a unary expression to `args[0]` that is expected to be downcastable to diff --git a/datafusion/physical-expr/src/struct_expressions.rs b/datafusion/physical-expr/src/struct_expressions.rs index b0ccb2a3ccb6..f420e062ef91 100644 --- a/datafusion/physical-expr/src/struct_expressions.rs +++ b/datafusion/physical-expr/src/struct_expressions.rs @@ -19,7 +19,7 @@ use arrow::array::*; use arrow::datatypes::Field; -use datafusion_common::{exec_err, DataFusionError, Result}; +use datafusion_common::{exec_err, Result}; use datafusion_expr::ColumnarValue; use std::sync::Arc; diff --git a/datafusion/physical-expr/src/unicode_expressions.rs b/datafusion/physical-expr/src/unicode_expressions.rs index 3209a6176fad..aa6a84119c34 100644 --- a/datafusion/physical-expr/src/unicode_expressions.rs +++ b/datafusion/physical-expr/src/unicode_expressions.rs @@ -33,7 +33,7 @@ use unicode_segmentation::UnicodeSegmentation; use datafusion_common::{ cast::{as_generic_string_array, as_int64_array}, - exec_err, DataFusionError, Result, + exec_err, Result, }; /// Returns number of characters in the string. diff --git a/datafusion/physical-expr/src/window/nth_value.rs b/datafusion/physical-expr/src/window/nth_value.rs index 05909ab25a07..a7bb31b6e109 100644 --- a/datafusion/physical-expr/src/window/nth_value.rs +++ b/datafusion/physical-expr/src/window/nth_value.rs @@ -29,8 +29,8 @@ use crate::PhysicalExpr; use arrow::array::{Array, ArrayRef}; use arrow::datatypes::{DataType, Field}; +use datafusion_common::Result; use datafusion_common::{exec_err, ScalarValue}; -use datafusion_common::{DataFusionError, Result}; use datafusion_expr::window_state::WindowAggState; use datafusion_expr::PartitionEvaluator; diff --git a/datafusion/physical-expr/src/window/rank.rs b/datafusion/physical-expr/src/window/rank.rs index 437fdbe0b982..fa3d4e487f14 100644 --- a/datafusion/physical-expr/src/window/rank.rs +++ b/datafusion/physical-expr/src/window/rank.rs @@ -28,7 +28,7 @@ use arrow::array::{Float64Array, UInt64Array}; use arrow::datatypes::{DataType, Field}; use arrow_schema::{SchemaRef, SortOptions}; use datafusion_common::utils::get_row_at_idx; -use datafusion_common::{exec_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{exec_err, Result, ScalarValue}; use datafusion_expr::PartitionEvaluator; use std::any::Any; diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 156362430558..855408c4baa8 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -36,7 +36,7 @@ use arrow::array::ArrayRef; use arrow::datatypes::{Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use datafusion_common::stats::Precision; -use datafusion_common::{internal_err, not_impl_err, plan_err, DataFusionError, Result}; +use datafusion_common::{internal_err, not_impl_err, plan_err, Result}; use datafusion_execution::TaskContext; use datafusion_expr::Accumulator; use datafusion_physical_expr::{ diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs index bfcff2853538..1521daee1334 100644 --- a/datafusion/physical-plan/src/coalesce_partitions.rs +++ b/datafusion/physical-plan/src/coalesce_partitions.rs @@ -29,7 +29,7 @@ use super::{DisplayAs, SendableRecordBatchStream, Statistics}; use crate::{DisplayFormatType, ExecutionPlan, Partitioning}; use arrow::datatypes::SchemaRef; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::EquivalenceProperties; diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs index 41c8dbed1453..398edc211de3 100644 --- a/datafusion/physical-plan/src/empty.rs +++ b/datafusion/physical-plan/src/empty.rs @@ -26,7 +26,7 @@ use crate::{memory::MemoryStream, DisplayFormatType, ExecutionPlan, Partitioning use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_execution::TaskContext; use log::trace; diff --git a/datafusion/physical-plan/src/explain.rs b/datafusion/physical-plan/src/explain.rs index e4904ddd3410..babcaf078bdc 100644 --- a/datafusion/physical-plan/src/explain.rs +++ b/datafusion/physical-plan/src/explain.rs @@ -27,7 +27,7 @@ use crate::{DisplayFormatType, ExecutionPlan, Partitioning}; use arrow::{array::StringBuilder, datatypes::SchemaRef, record_batch::RecordBatch}; use datafusion_common::display::StringifiedPlan; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_execution::TaskContext; use log::trace; diff --git a/datafusion/physical-plan/src/insert.rs b/datafusion/physical-plan/src/insert.rs index 81cdfd753fe6..e678425d3753 100644 --- a/datafusion/physical-plan/src/insert.rs +++ b/datafusion/physical-plan/src/insert.rs @@ -33,7 +33,7 @@ use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; use arrow_array::{ArrayRef, UInt64Array}; use arrow_schema::{DataType, Field, Schema}; -use datafusion_common::{exec_err, internal_err, DataFusionError, Result}; +use datafusion_common::{exec_err, internal_err, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::{Distribution, PhysicalSortRequirement}; diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs index 938c9e4d343d..99bd051da0f7 100644 --- a/datafusion/physical-plan/src/joins/cross_join.rs +++ b/datafusion/physical-plan/src/joins/cross_join.rs @@ -35,7 +35,7 @@ use arrow::datatypes::{Fields, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use arrow_array::RecordBatchOptions; use datafusion_common::stats::Precision; -use datafusion_common::{plan_err, DataFusionError, JoinType, Result, ScalarValue}; +use datafusion_common::{plan_err, JoinType, Result, ScalarValue}; use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation}; use datafusion_execution::TaskContext; use datafusion_physical_expr::equivalence::join_equivalence_properties; diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index f89a2445fd07..1618efd4d0f9 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -44,7 +44,7 @@ use arrow::array::{ use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use arrow::util::bit_util; -use datafusion_common::{exec_err, DataFusionError, JoinSide, Result, Statistics}; +use datafusion_common::{exec_err, JoinSide, Result, Statistics}; use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation}; use datafusion_execution::TaskContext; use datafusion_expr::JoinType; diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs index 42c7029edcc1..506324852b21 100644 --- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs +++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs @@ -62,9 +62,7 @@ use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use datafusion_common::hash_utils::create_hashes; use datafusion_common::utils::bisect; -use datafusion_common::{ - internal_err, plan_err, DataFusionError, JoinSide, JoinType, Result, -}; +use datafusion_common::{internal_err, plan_err, JoinSide, JoinType, Result}; use datafusion_execution::memory_pool::MemoryConsumer; use datafusion_execution::TaskContext; use datafusion_expr::interval_arithmetic::Interval; diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs index 562e42a7da3b..a15fd470a98b 100644 --- a/datafusion/physical-plan/src/lib.rs +++ b/datafusion/physical-plan/src/lib.rs @@ -31,7 +31,7 @@ use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; use datafusion_common::tree_node::Transformed; use datafusion_common::utils::DataPtr; -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::expressions::Column; use datafusion_physical_expr::{ diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs index 417bc4cf977b..680aa23214f9 100644 --- a/datafusion/physical-plan/src/limit.rs +++ b/datafusion/physical-plan/src/limit.rs @@ -32,7 +32,7 @@ use crate::{ use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; use datafusion_common::stats::Precision; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_execution::TaskContext; use futures::stream::{Stream, StreamExt}; diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rs index 7de474fda11c..86bd89e7ebac 100644 --- a/datafusion/physical-plan/src/memory.rs +++ b/datafusion/physical-plan/src/memory.rs @@ -30,7 +30,7 @@ use super::{ use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; -use datafusion_common::{internal_err, project_schema, DataFusionError, Result}; +use datafusion_common::{internal_err, project_schema, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::{EquivalenceProperties, LexOrdering}; diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs index 3ab3de62f37a..3371148587d1 100644 --- a/datafusion/physical-plan/src/placeholder_row.rs +++ b/datafusion/physical-plan/src/placeholder_row.rs @@ -28,7 +28,7 @@ use arrow::array::{ArrayRef, NullArray}; use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use arrow_array::RecordBatchOptions; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_execution::TaskContext; use log::trace; diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs index f4b57e8bfb45..81a26cd2188d 100644 --- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs +++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs @@ -30,7 +30,7 @@ use crate::{ }; use arrow::datatypes::SchemaRef; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_execution::memory_pool::MemoryConsumer; use datafusion_execution::TaskContext; use datafusion_physical_expr::{EquivalenceProperties, PhysicalSortRequirement}; diff --git a/datafusion/physical-plan/src/sorts/streaming_merge.rs b/datafusion/physical-plan/src/sorts/streaming_merge.rs index 4f8d8063853b..9e6618dd1af5 100644 --- a/datafusion/physical-plan/src/sorts/streaming_merge.rs +++ b/datafusion/physical-plan/src/sorts/streaming_merge.rs @@ -26,7 +26,7 @@ use crate::sorts::{ use crate::{PhysicalSortExpr, SendableRecordBatchStream}; use arrow::datatypes::{DataType, SchemaRef}; use arrow_array::*; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; use datafusion_execution::memory_pool::MemoryReservation; macro_rules! primitive_merge_helper { diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rs index b780a50cdc90..970194550045 100644 --- a/datafusion/physical-plan/src/stream.rs +++ b/datafusion/physical-plan/src/stream.rs @@ -25,7 +25,7 @@ use std::task::Poll; use crate::displayable; use arrow::{datatypes::SchemaRef, record_batch::RecordBatch}; use datafusion_common::internal_err; -use datafusion_common::DataFusionError; + use datafusion_common::Result; use datafusion_execution::TaskContext; use futures::stream::BoxStream; diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rs index 897682092831..bca37ed942d0 100644 --- a/datafusion/physical-plan/src/streaming.rs +++ b/datafusion/physical-plan/src/streaming.rs @@ -27,7 +27,7 @@ use crate::{ExecutionPlan, Partitioning, SendableRecordBatchStream}; use arrow::datatypes::SchemaRef; use arrow_schema::Schema; -use datafusion_common::{internal_err, plan_err, DataFusionError, Result}; +use datafusion_common::{internal_err, plan_err, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, PhysicalSortExpr}; diff --git a/datafusion/physical-plan/src/udaf.rs b/datafusion/physical-plan/src/udaf.rs index a82bbe1d0705..fd9279dfd552 100644 --- a/datafusion/physical-plan/src/udaf.rs +++ b/datafusion/physical-plan/src/udaf.rs @@ -28,7 +28,7 @@ use arrow::{ }; use super::{expressions::format_state_name, Accumulator, AggregateExpr}; -use datafusion_common::{not_impl_err, DataFusionError, Result}; +use datafusion_common::{not_impl_err, Result}; pub use datafusion_expr::AggregateUDF; use datafusion_physical_expr::PhysicalExpr; diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index d01ea5507449..62a6d5c0f877 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -39,7 +39,7 @@ use crate::stream::ObservedStream; use arrow::datatypes::{Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use datafusion_common::stats::Precision; -use datafusion_common::{exec_err, internal_err, DataFusionError, Result}; +use datafusion_common::{exec_err, internal_err, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::EquivalenceProperties; diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs index b9e732c317af..d5453f0924a3 100644 --- a/datafusion/physical-plan/src/unnest.rs +++ b/datafusion/physical-plan/src/unnest.rs @@ -35,7 +35,7 @@ use arrow::datatypes::{ }; use arrow::record_batch::RecordBatch; use arrow_array::{GenericListArray, OffsetSizeTrait}; -use datafusion_common::{exec_err, DataFusionError, Result, UnnestOptions}; +use datafusion_common::{exec_err, Result, UnnestOptions}; use datafusion_execution::TaskContext; use async_trait::async_trait; diff --git a/datafusion/physical-plan/src/values.rs b/datafusion/physical-plan/src/values.rs index f82f7ea2f869..9c1ce93b2a08 100644 --- a/datafusion/physical-plan/src/values.rs +++ b/datafusion/physical-plan/src/values.rs @@ -29,7 +29,7 @@ use crate::{ use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::{RecordBatch, RecordBatchOptions}; -use datafusion_common::{internal_err, plan_err, DataFusionError, Result, ScalarValue}; +use datafusion_common::{internal_err, plan_err, Result, ScalarValue}; use datafusion_execution::TaskContext; /// Execution plan for values list based relation (produces constant rows) diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs index 6c245f65ba4f..e80102812ebd 100644 --- a/datafusion/physical-plan/src/windows/window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs @@ -45,7 +45,7 @@ use arrow::{ }; use datafusion_common::stats::Precision; use datafusion_common::utils::evaluate_partition_ranges; -use datafusion_common::{internal_err, plan_err, DataFusionError, Result}; +use datafusion_common::{internal_err, plan_err, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::{EquivalenceProperties, PhysicalSortRequirement}; diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs index c74a596f3dae..0f934a76a60f 100644 --- a/datafusion/physical-plan/src/work_table.rs +++ b/datafusion/physical-plan/src/work_table.rs @@ -34,7 +34,7 @@ use super::{ metrics::{ExecutionPlanMetricsSet, MetricsSet}, SendableRecordBatchStream, Statistics, }; -use datafusion_common::{internal_err, DataFusionError, Result}; +use datafusion_common::{internal_err, Result}; /// The name is from PostgreSQL's terminology. /// See diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rs index 6d5a7c7f3063..d4abb9ed9c6f 100644 --- a/datafusion/proto/src/bytes/mod.rs +++ b/datafusion/proto/src/bytes/mod.rs @@ -23,7 +23,7 @@ use crate::physical_plan::{ AsExecutionPlan, DefaultPhysicalExtensionCodec, PhysicalExtensionCodec, }; use crate::protobuf; -use datafusion_common::{plan_datafusion_err, DataFusionError, Result}; +use datafusion_common::{plan_datafusion_err, Result}; use datafusion_expr::{ create_udaf, create_udf, create_udwf, AggregateUDF, Expr, LogicalPlan, Volatility, WindowUDF, diff --git a/datafusion/proto/src/bytes/registry.rs b/datafusion/proto/src/bytes/registry.rs index 7c993c639991..4bf2bb3d7b79 100644 --- a/datafusion/proto/src/bytes/registry.rs +++ b/datafusion/proto/src/bytes/registry.rs @@ -19,7 +19,7 @@ use std::{collections::HashSet, sync::Arc}; use datafusion::execution::registry::FunctionRegistry; use datafusion_common::plan_err; -use datafusion_common::{DataFusionError, Result}; +use datafusion_common::Result; use datafusion_expr::{AggregateUDF, ScalarUDF, WindowUDF}; /// A default [`FunctionRegistry`] registry that does not resolve any diff --git a/datafusion/sql/examples/sql.rs b/datafusion/sql/examples/sql.rs index 9df65b99a748..8744a905481f 100644 --- a/datafusion/sql/examples/sql.rs +++ b/datafusion/sql/examples/sql.rs @@ -17,7 +17,7 @@ use arrow_schema::{DataType, Field, Schema}; use datafusion_common::config::ConfigOptions; -use datafusion_common::{plan_err, DataFusionError, Result}; +use datafusion_common::{plan_err, Result}; use datafusion_expr::WindowUDF; use datafusion_expr::{ logical_plan::builder::LogicalTableSource, AggregateUDF, ScalarUDF, TableSource, diff --git a/datafusion/sql/src/expr/binary_op.rs b/datafusion/sql/src/expr/binary_op.rs index 78efaca09938..0d37742e5b07 100644 --- a/datafusion/sql/src/expr/binary_op.rs +++ b/datafusion/sql/src/expr/binary_op.rs @@ -16,7 +16,7 @@ // under the License. use crate::planner::{ContextProvider, SqlToRel}; -use datafusion_common::{not_impl_err, DataFusionError, Result}; +use datafusion_common::{not_impl_err, Result}; use datafusion_expr::Operator; use sqlparser::ast::BinaryOperator; diff --git a/datafusion/sql/src/expr/function.rs b/datafusion/sql/src/expr/function.rs index db572a23cf99..bc1d672522dd 100644 --- a/datafusion/sql/src/expr/function.rs +++ b/datafusion/sql/src/expr/function.rs @@ -18,8 +18,7 @@ use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; use arrow_schema::DataType; use datafusion_common::{ - not_impl_err, plan_datafusion_err, plan_err, DFSchema, DataFusionError, Dependency, - Result, + not_impl_err, plan_datafusion_err, plan_err, DFSchema, Dependency, Result, }; use datafusion_expr::expr::{ScalarFunction, Unnest}; use datafusion_expr::function::suggest_valid_function; diff --git a/datafusion/sql/src/expr/grouping_set.rs b/datafusion/sql/src/expr/grouping_set.rs index 254f5079b7b1..a8b3ef7e20ec 100644 --- a/datafusion/sql/src/expr/grouping_set.rs +++ b/datafusion/sql/src/expr/grouping_set.rs @@ -17,7 +17,7 @@ use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; use datafusion_common::plan_err; -use datafusion_common::{DFSchema, DataFusionError, Result}; +use datafusion_common::{DFSchema, Result}; use datafusion_expr::{Expr, GroupingSet}; use sqlparser::ast::Expr as SQLExpr; diff --git a/datafusion/sql/src/expr/json_access.rs b/datafusion/sql/src/expr/json_access.rs index 681b72b4e71a..b24482f88297 100644 --- a/datafusion/sql/src/expr/json_access.rs +++ b/datafusion/sql/src/expr/json_access.rs @@ -16,7 +16,7 @@ // under the License. use crate::planner::{ContextProvider, SqlToRel}; -use datafusion_common::{not_impl_err, DataFusionError, Result}; +use datafusion_common::{not_impl_err, Result}; use datafusion_expr::Operator; use sqlparser::ast::JsonOperator; diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index da6c3a6074d4..b058fb79b4a1 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -31,8 +31,7 @@ use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; use arrow_schema::DataType; use arrow_schema::TimeUnit; use datafusion_common::{ - internal_err, not_impl_err, plan_err, Column, DFSchema, DataFusionError, Result, - ScalarValue, + internal_err, not_impl_err, plan_err, Column, DFSchema, Result, ScalarValue, }; use datafusion_expr::expr::AggregateFunctionDefinition; use datafusion_expr::expr::InList; diff --git a/datafusion/sql/src/expr/order_by.rs b/datafusion/sql/src/expr/order_by.rs index 772255bd9773..46f19f436ccc 100644 --- a/datafusion/sql/src/expr/order_by.rs +++ b/datafusion/sql/src/expr/order_by.rs @@ -16,9 +16,7 @@ // under the License. use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; -use datafusion_common::{ - plan_datafusion_err, plan_err, DFSchema, DataFusionError, Result, -}; +use datafusion_common::{plan_datafusion_err, plan_err, DFSchema, Result}; use datafusion_expr::expr::Sort; use datafusion_expr::Expr; use sqlparser::ast::{Expr as SQLExpr, OrderByExpr, Value}; diff --git a/datafusion/sql/src/expr/substring.rs b/datafusion/sql/src/expr/substring.rs index 71b2a11cd414..a5d1abf0f265 100644 --- a/datafusion/sql/src/expr/substring.rs +++ b/datafusion/sql/src/expr/substring.rs @@ -17,7 +17,7 @@ use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; use datafusion_common::plan_err; -use datafusion_common::{DFSchema, DataFusionError, Result, ScalarValue}; +use datafusion_common::{DFSchema, Result, ScalarValue}; use datafusion_expr::expr::ScalarFunction; use datafusion_expr::{BuiltinScalarFunction, Expr}; use sqlparser::ast::Expr as SQLExpr; diff --git a/datafusion/sql/src/expr/unary_op.rs b/datafusion/sql/src/expr/unary_op.rs index 08ff6f2c3622..9fcee7a06124 100644 --- a/datafusion/sql/src/expr/unary_op.rs +++ b/datafusion/sql/src/expr/unary_op.rs @@ -16,7 +16,7 @@ // under the License. use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; -use datafusion_common::{not_impl_err, DFSchema, DataFusionError, Result}; +use datafusion_common::{not_impl_err, DFSchema, Result}; use datafusion_expr::Expr; use sqlparser::ast::{Expr as SQLExpr, UnaryOperator, Value}; diff --git a/datafusion/sql/src/relation/join.rs b/datafusion/sql/src/relation/join.rs index b119672eae5f..4ba089f48630 100644 --- a/datafusion/sql/src/relation/join.rs +++ b/datafusion/sql/src/relation/join.rs @@ -16,7 +16,7 @@ // under the License. use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; -use datafusion_common::{not_impl_err, Column, DataFusionError, Result}; +use datafusion_common::{not_impl_err, Column, Result}; use datafusion_expr::{JoinType, LogicalPlan, LogicalPlanBuilder}; use sqlparser::ast::{Join, JoinConstraint, JoinOperator, TableWithJoins}; use std::collections::HashSet; diff --git a/datafusion/sql/src/relation/mod.rs b/datafusion/sql/src/relation/mod.rs index b233f47a058f..1d52899160a9 100644 --- a/datafusion/sql/src/relation/mod.rs +++ b/datafusion/sql/src/relation/mod.rs @@ -16,9 +16,7 @@ // under the License. use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; -use datafusion_common::{ - not_impl_err, plan_err, DFSchema, DataFusionError, Result, TableReference, -}; +use datafusion_common::{not_impl_err, plan_err, DFSchema, Result, TableReference}; use datafusion_expr::{LogicalPlan, LogicalPlanBuilder}; use sqlparser::ast::{FunctionArg, FunctionArgExpr, TableFactor}; diff --git a/datafusion/sql/src/set_expr.rs b/datafusion/sql/src/set_expr.rs index 7300d49be0f5..2cbb68368f72 100644 --- a/datafusion/sql/src/set_expr.rs +++ b/datafusion/sql/src/set_expr.rs @@ -16,7 +16,7 @@ // under the License. use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; -use datafusion_common::{not_impl_err, DataFusionError, Result}; +use datafusion_common::{not_impl_err, Result}; use datafusion_expr::{LogicalPlan, LogicalPlanBuilder}; use sqlparser::ast::{SetExpr, SetOperator, SetQuantifier}; diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs index 331d63cc22b2..bc9cc66b7626 100644 --- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs @@ -27,7 +27,7 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use datafusion::common::{not_impl_err, plan_err, DFSchema, DFSchemaRef}; -use datafusion::error::{DataFusionError, Result}; +use datafusion::error::Result; use datafusion::execution::context::SessionState; use datafusion::execution::registry::SerializerRegistry; use datafusion::execution::runtime_env::RuntimeEnv; From b220f03fffda22c70f03fa84e244cf04f0e6644c Mon Sep 17 00:00:00 2001 From: Jonah Gao Date: Wed, 28 Feb 2024 20:48:20 +0800 Subject: [PATCH 6/6] feat: support for defining ARRAY columns in `CREATE TABLE` (#9381) --- datafusion/sql/src/planner.rs | 5 +-- datafusion/sqllogictest/test_files/array.slt | 36 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index 012b1c51a5c1..1f21299d8559 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -238,7 +238,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let mut fields = Vec::with_capacity(columns.len()); for column in columns { - let data_type = self.convert_simple_data_type(&column.data_type)?; + let data_type = self.convert_data_type(&column.data_type)?; let not_nullable = column .options .iter() @@ -358,7 +358,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { match sql_type { SQLDataType::Array(ArrayElemTypeDef::AngleBracket(inner_sql_type)) | SQLDataType::Array(ArrayElemTypeDef::SquareBracket(inner_sql_type)) => { - let data_type = self.convert_simple_data_type(inner_sql_type)?; + // Arrays may be multi-dimensional. + let data_type = self.convert_data_type(inner_sql_type)?; Ok(DataType::List(Arc::new(Field::new( "field", data_type, true, diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index da02a80a104f..e64346537150 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -6165,6 +6165,39 @@ NULL NULL [60, 59, 58, 57, 56, 55, 54, , 52, 51] [51, 52, , 54, 55, 56, 57, 58, 59, 60] [70, 69, 68, 67, 66, 65, 64, 63, 62, 61] [61, 62, 63, 64, 65, 66, 67, 68, 69, 70] + +# Test defining a table with array columns +statement ok +create table test_create_array_table( + a int[], + b text[], + -- two-dimensional array + c int[][], + d int +); + +query ???I +insert into test_create_array_table values + ([1, 2, 3], ['a', 'b', 'c'], [[4,6], [6,7,8]], 1); +---- +1 + +query ???I +select * from test_create_array_table; +---- +[1, 2, 3] [a, b, c] [[4, 6], [6, 7, 8]] 1 + +query T +select arrow_typeof(a) from test_create_array_table; +---- +List(Field { name: "field", data_type: Int32, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) + +query T +select arrow_typeof(c) from test_create_array_table; +---- +List(Field { name: "field", data_type: List(Field { name: "field", data_type: Int32, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }) + + ### Delete tables statement ok @@ -6334,3 +6367,6 @@ drop table large_arrays_values_without_nulls; statement ok drop table fixed_size_arrays_values_without_nulls; + +statement ok +drop table test_create_array_table;