Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix ci fails
Browse files Browse the repository at this point in the history
Signed-off-by: jayzhan211 <[email protected]>
jayzhan211 committed Aug 26, 2023
1 parent ff0c107 commit 7d30f8d
Showing 9 changed files with 49 additions and 41 deletions.
4 changes: 2 additions & 2 deletions datafusion/common/src/error.rs
Original file line number Diff line number Diff line change
@@ -315,7 +315,8 @@ impl Display for DataFusionError {
write!(f, "Object Store error: {desc}")
}
DataFusionError::Context(ref desc, ref err) => {
write!(f, "{}\ncaused by\n{}", desc, *err)
// Dont add newline in error description, it causes another error in the sqllogictest.
write!(f, "{desc} caused by {}", *err)
}
DataFusionError::Substrait(ref desc) => {
write!(f, "Substrait error: {desc}")
@@ -541,7 +542,6 @@ mod test {
}

#[test]
#[allow(clippy::unnecessary_literal_unwrap)]
fn test_make_error_parse_input() {
let res: Result<(), DataFusionError> = plan_err!("Err");
let res = res.unwrap_err();
26 changes: 20 additions & 6 deletions datafusion/optimizer/src/analyzer/type_coercion.rs
Original file line number Diff line number Diff line change
@@ -645,17 +645,17 @@ fn replace_nulls_with_coerced_types(
}
}

// Coerce array arguments types for array functions
// Convert type or return error for incompatible types in this step
fn coerce_array_args(
fun: &BuiltinScalarFunction,
expressions: Vec<Expr>,
schema: &DFSchema,
) -> Result<Vec<Expr>> {
// Array function with indices don't need coercion for the indices
if *fun == BuiltinScalarFunction::ArrayElement
|| *fun == BuiltinScalarFunction::ArraySlice
|| *fun == BuiltinScalarFunction::ArrayRepeat
|| *fun == BuiltinScalarFunction::ArrayPosition
|| *fun == BuiltinScalarFunction::ArrayRemoveN
if *fun != BuiltinScalarFunction::MakeArray
&& *fun != BuiltinScalarFunction::ArrayAppend
&& *fun != BuiltinScalarFunction::ArrayPrepend
&& *fun != BuiltinScalarFunction::ArrayConcat
{
return Ok(expressions);
}
@@ -665,6 +665,20 @@ fn coerce_array_args(
.map(|e| e.get_type(schema))
.collect::<Result<Vec<_>>>()?;

// Check dimensions and align dimensions
// TODO: Move align array dimensions here. Function used in concat, append, prepend.
if *fun == BuiltinScalarFunction::ArrayConcat {
for expr_type in input_types.iter() {
if let DataType::List(_) = expr_type {
continue;
} else {
return plan_err!(
"The array_concat function can only accept list as the args"
);
}
}
}

// Get base type for each input type
// e.g List[Int64] -> Int64
// List[List[Int64]] -> Int64
1 change: 0 additions & 1 deletion datafusion/physical-expr/src/window/cume_dist.rs
Original file line number Diff line number Diff line change
@@ -117,7 +117,6 @@ mod tests {
}

#[test]
#[allow(clippy::single_range_in_vec_init)]
fn test_cume_dist() -> Result<()> {
let r = cume_dist("arr".into());

2 changes: 0 additions & 2 deletions datafusion/physical-expr/src/window/rank.rs
Original file line number Diff line number Diff line change
@@ -227,7 +227,6 @@ mod tests {
test_i32_result(expr, vec![0..2, 2..3, 3..6, 6..7, 7..8], expected)
}

#[allow(clippy::single_range_in_vec_init)]
fn test_without_rank(expr: &Rank, expected: Vec<u64>) -> Result<()> {
test_i32_result(expr, vec![0..8], expected)
}
@@ -276,7 +275,6 @@ mod tests {
}

#[test]
#[allow(clippy::single_range_in_vec_init)]
fn test_percent_rank() -> Result<()> {
let r = percent_rank("arr".into());

12 changes: 6 additions & 6 deletions datafusion/sql/src/expr/value.rs
Original file line number Diff line number Diff line change
@@ -166,12 +166,6 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
let data_types = values.iter().map(|e| e.get_datatype()).collect::<Vec<_>>();
let seen_types: HashSet<DataType> =
values.iter().map(|e| e.get_datatype()).collect();
let coerced_type = data_types
.iter()
.skip(1)
.fold(data_types[0].clone(), |acc, d| {
comparison_coercion(&acc, d).unwrap_or(acc)
});

match seen_types.len() {
0 => Ok(lit(ScalarValue::new_list(None, DataType::Utf8))),
@@ -180,6 +174,12 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
Ok(lit(ScalarValue::new_list(Some(values), data_type)))
}
_ => {
let coerced_type = data_types
.iter()
.skip(1)
.fold(data_types[0].clone(), |acc, d| {
comparison_coercion(&acc, d).unwrap_or(acc)
});
let values = values
.iter()
.map(|e| {
25 changes: 11 additions & 14 deletions datafusion/sqllogictest/test_files/array.slt
Original file line number Diff line number Diff line change
@@ -1241,7 +1241,9 @@ select array_repeat([1], column3), array_repeat(column1, 3) from arrays_values_w
## array_concat (aliases: `array_cat`, `list_concat`, `list_cat`)

# array_concat error
query error DataFusion error: Error during planning: The array_concat function can only accept list as the args\.
query error DataFusion error: SQL error: ParserError\("Expected an SQL statement, found: caused"\)
caused by
Error during planning: The array_concat function can only accept list as the args\.
select array_concat(1, 2);

# array_concat scalar function #1
@@ -1293,21 +1295,16 @@ select array_concat(make_array(make_array(1, 2), make_array(3, 4)), make_array(n
----
[[1, 2], [3, 4], []]

query ?
query error DataFusion error: type_coercion caused by Error during planning: The array_concat function can only accept list as the args
select array_concat(make_array(make_array(1, 2), make_array(3, 4)), null);
----
[[1, 2], [3, 4]]

query ?
select array_concat(make_array(make_array(1, 2), make_array(3, 4)), make_array(5));
----
[[1, 2], [3, 4], [5]]

# TODO: Get error for this query
query ?
query error DataFusion error: type_coercion caused by Error during planning: The array_concat function can only accept list as the args
select array_concat(make_array(make_array(1, 2), make_array(3, 4)), 5);
----
[[1, 2], [3, 4]]

# array_concat scalar function #8 (with empty arrays)
query ?
@@ -1815,7 +1812,7 @@ select array_replace_all(make_array([1, 2, 3], [4, 5, 6], [4, 5, 6], [10, 11, 12
query TTT
select array_to_string(['h', 'e', 'l', 'l', 'o'], ','), array_to_string([1, 2, 3, 4, 5], '-'), array_to_string([1.0, 2.0, 3.0], '|');
----
h,e,l,l,o 1-2-3-4-5 1.0|2.0|3.0
h,e,l,l,o 1-2-3-4-5 1|2|3

# array_to_string scalar function #2
query TTT
@@ -1833,31 +1830,31 @@ select array_to_string(make_array(), ',')
query TTT
select list_to_string(['h', 'e', 'l', 'l', 'o'], ','), list_to_string([1, 2, 3, 4, 5], '-'), list_to_string([1.0, 2.0, 3.0], '|');
----
h,e,l,l,o 1-2-3-4-5 1.0|2.0|3.0
h,e,l,l,o 1-2-3-4-5 1|2|3

# array_join scalar function #5 (function alias `array_to_string`)
query TTT
select array_join(['h', 'e', 'l', 'l', 'o'], ','), array_join([1, 2, 3, 4, 5], '-'), array_join([1.0, 2.0, 3.0], '|');
----
h,e,l,l,o 1-2-3-4-5 1.0|2.0|3.0
h,e,l,l,o 1-2-3-4-5 1|2|3

# list_join scalar function #6 (function alias `list_join`)
query TTT
select list_join(['h', 'e', 'l', 'l', 'o'], ','), list_join([1, 2, 3, 4, 5], '-'), list_join([1.0, 2.0, 3.0], '|');
----
h,e,l,l,o 1-2-3-4-5 1.0|2.0|3.0
h,e,l,l,o 1-2-3-4-5 1|2|3

# array_to_string scalar function with nulls #1
query TTT
select array_to_string(make_array('h', NULL, 'l', NULL, 'o'), ','), array_to_string(make_array(1, NULL, 3, NULL, 5), '-'), array_to_string(make_array(NULL, 2.0, 3.0), '|');
----
h,l,o 1-3-5 2.0|3.0
h,l,o 1-3-5 2|3

# array_to_string scalar function with nulls #2
query TTT
select array_to_string(make_array('h', NULL, NULL, NULL, 'o'), ',', '-'), array_to_string(make_array(NULL, 2, NULL, 4, 5), '-', 'nil'), array_to_string(make_array(1.0, NULL, 3.0), '|', '0');
----
h,-,-,-,o nil-2-nil-4-5 1.0|0|3.0
h,-,-,-,o nil-2-nil-4-5 1|0|3

# array_to_string with columns #1

2 changes: 1 addition & 1 deletion datafusion/sqllogictest/test_files/dates.slt
Original file line number Diff line number Diff line change
@@ -85,7 +85,7 @@ g
h

## Plan error when compare Utf8 and timestamp in where clause
statement error DataFusion error: type_coercion\ncaused by\nError during planning: Cannot coerce arithmetic expression Timestamp\(Nanosecond, Some\("\+00:00"\)\) \+ Utf8 to valid types
statement error DataFusion error: type_coercion caused by Error during planning: Cannot coerce arithmetic expression Timestamp\(Nanosecond, Some\("\+00:00"\)\) \+ Utf8 to valid types
select i_item_desc from test
where d3_date > now() + '5 days';

4 changes: 2 additions & 2 deletions datafusion/sqllogictest/test_files/scalar.slt
Original file line number Diff line number Diff line change
@@ -1526,15 +1526,15 @@ SELECT not(true), not(false)
----
false true

query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nInternal error: NOT 'Literal \{ value: Int64\(1\) \}' can't be evaluated because the expression's type is Int64, not boolean or NULL\. This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug report in our issue tracker
query error DataFusion error: Optimizer rule 'simplify_expressions' failed caused by Internal error: NOT 'Literal \{ value: Int64\(1\) \}' can't be evaluated because the expression's type is Int64, not boolean or NULL\. This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug report in our issue tracker
SELECT not(1), not(0)

query ?B
SELECT null, not(null)
----
NULL NULL

query error DataFusion error: Optimizer rule 'simplify_expressions' failed\ncaused by\nInternal error: NOT 'Literal \{ value: Utf8\("hi"\) \}' can't be evaluated because the expression's type is Utf8, not boolean or NULL\. This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug report in our issue tracker
query error DataFusion error: Optimizer rule 'simplify_expressions' failed caused by Internal error: NOT 'Literal \{ value: Utf8\("hi"\) \}' can't be evaluated because the expression's type is Utf8, not boolean or NULL\. This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug report in our issue tracker
SELECT NOT('hi')

# test_negative_expressions()
14 changes: 7 additions & 7 deletions datafusion/sqllogictest/test_files/subquery.slt
Original file line number Diff line number Diff line change
@@ -403,23 +403,23 @@ LeftSemi Join: t1.t1_id = __correlated_sq_1.t2_id
----------TableScan: t1 projection=[t1_int]

#invalid_scalar_subquery
statement error DataFusion error: check_analyzed_plan\ncaused by\nError during planning: Scalar subquery should only return one column, but found 2: t2.t2_id, t2.t2_name
statement error DataFusion error: check_analyzed_plan caused by Error during planning: Scalar subquery should only return one column, but found 2: t2.t2_id, t2.t2_name
SELECT t1_id, t1_name, t1_int, (select t2_id, t2_name FROM t2 WHERE t2.t2_id = t1.t1_int) FROM t1

#subquery_not_allowed
#In/Exist Subquery is not allowed in ORDER BY clause.
statement error DataFusion error: check_analyzed_plan\ncaused by\nError during planning: In/Exist subquery can only be used in Projection, Filter, Window functions, Aggregate and Join plan nodes
statement error DataFusion error: check_analyzed_plan caused by Error during planning: In/Exist subquery can only be used in Projection, Filter, Window functions, Aggregate and Join plan nodes
SELECT t1_id, t1_name, t1_int FROM t1 order by t1_int in (SELECT t2_int FROM t2 WHERE t1.t1_id > t1.t1_int)

#non_aggregated_correlated_scalar_subquery
statement error DataFusion error: check_analyzed_plan\ncaused by\nError during planning: Correlated scalar subquery must be aggregated to return at most one row
statement error DataFusion error: check_analyzed_plan caused by Error during planning: Correlated scalar subquery must be aggregated to return at most one row
SELECT t1_id, (SELECT t2_int FROM t2 WHERE t2.t2_int = t1.t1_int) as t2_int from t1

statement error DataFusion error: check_analyzed_plan\ncaused by\nError during planning: Correlated scalar subquery must be aggregated to return at most one row
statement error DataFusion error: check_analyzed_plan caused by Error during planning: Correlated scalar subquery must be aggregated to return at most one row
SELECT t1_id, (SELECT t2_int FROM t2 WHERE t2.t2_int = t1_int group by t2_int) as t2_int from t1

#non_aggregated_correlated_scalar_subquery_with_limit
statement error DataFusion error: check_analyzed_plan\ncaused by\nError during planning: Correlated scalar subquery must be aggregated to return at most one row
statement error DataFusion error: check_analyzed_plan caused by Error during planning: Correlated scalar subquery must be aggregated to return at most one row
SELECT t1_id, (SELECT t2_int FROM t2 WHERE t2.t2_int = t1.t1_int limit 2) as t2_int from t1

#non_aggregated_correlated_scalar_subquery_with_single_row
@@ -468,11 +468,11 @@ SELECT t1_id, (SELECT a FROM (select 1 as a) WHERE a = t1.t1_int) as t2_int from
44 NULL

#non_equal_correlated_scalar_subquery
statement error DataFusion error: check_analyzed_plan\ncaused by\nError during planning: Correlated column is not allowed in predicate: t2\.t2_id < outer_ref\(t1\.t1_id\)
statement error DataFusion error: check_analyzed_plan caused by Error during planning: Correlated column is not allowed in predicate: t2\.t2_id < outer_ref\(t1\.t1_id\)
SELECT t1_id, (SELECT sum(t2_int) FROM t2 WHERE t2.t2_id < t1.t1_id) as t2_sum from t1

#aggregated_correlated_scalar_subquery_with_extra_group_by_columns
statement error DataFusion error: check_analyzed_plan\ncaused by\nError during planning: A GROUP BY clause in a scalar correlated subquery cannot contain non-correlated columns
statement error DataFusion error: check_analyzed_plan caused by Error during planning: A GROUP BY clause in a scalar correlated subquery cannot contain non-correlated columns
SELECT t1_id, (SELECT sum(t2_int) FROM t2 WHERE t2.t2_id = t1.t1_id group by t2_name) as t2_sum from t1

#support_agg_correlated_columns

0 comments on commit 7d30f8d

Please sign in to comment.