Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[minor]: remove same util functions from the code base. #13026

Merged
merged 5 commits into from
Oct 22, 2024
Merged

[minor]: remove same util functions from the code base. #13026

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bb6017a
Initial commit
akurmustafa Oct 19, 2024
f16b27c
Merge branch 'main' into simplification/deduplicate_equivalence
akurmustafa Oct 20, 2024
75d37b8
Merge branch 'main' into simplification/deduplicate_equivalence
akurmustafa Oct 21, 2024
ba1e3cd
Resolve linter errors
akurmustafa Oct 21, 2024
6016d89
Decrease diff
akurmustafa Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 170 additions & 2 deletions datafusion/core/tests/fuzz_cases/equivalence/ordering.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
// under the License.

use crate::fuzz_cases::equivalence::utils::{
create_random_schema, generate_table_for_eq_properties, is_table_same_after_sort,
TestScalarUDF,
create_random_schema, create_test_params, generate_table_for_eq_properties,
is_table_same_after_sort, TestScalarUDF,
};
use arrow_schema::SortOptions;
use datafusion_common::{DFSchema, Result};
Expand Down Expand Up @@ -158,3 +158,171 @@ fn test_ordering_satisfy_with_equivalence_complex_random() -> Result<()> {

Ok(())
}

#[test]
fn test_ordering_satisfy_with_equivalence() -> Result<()> {
// Schema satisfies following orderings:
// [a ASC], [d ASC, b ASC], [e DESC, f ASC, g ASC]
// and
// Column [a=c] (e.g they are aliases).
let (test_schema, eq_properties) = create_test_params()?;
let col_a = &col("a", &test_schema)?;
let col_b = &col("b", &test_schema)?;
let col_c = &col("c", &test_schema)?;
let col_d = &col("d", &test_schema)?;
let col_e = &col("e", &test_schema)?;
let col_f = &col("f", &test_schema)?;
let col_g = &col("g", &test_schema)?;
let option_asc = SortOptions {
descending: false,
nulls_first: false,
};
let option_desc = SortOptions {
descending: true,
nulls_first: true,
};
let table_data_with_properties =
generate_table_for_eq_properties(&eq_properties, 625, 5)?;

// First element in the tuple stores vector of requirement, second element is the expected return value for ordering_satisfy function
let requirements = vec![
// `a ASC NULLS LAST`, expects `ordering_satisfy` to be `true`, since existing ordering `a ASC NULLS LAST, b ASC NULLS LAST` satisfies it
(vec![(col_a, option_asc)], true),
(vec![(col_a, option_desc)], false),
// Test whether equivalence works as expected
(vec![(col_c, option_asc)], true),
(vec![(col_c, option_desc)], false),
// Test whether ordering equivalence works as expected
(vec![(col_d, option_asc)], true),
(vec![(col_d, option_asc), (col_b, option_asc)], true),
(vec![(col_d, option_desc), (col_b, option_asc)], false),
(
vec![
(col_e, option_desc),
(col_f, option_asc),
(col_g, option_asc),
],
true,
),
(vec![(col_e, option_desc), (col_f, option_asc)], true),
(vec![(col_e, option_asc), (col_f, option_asc)], false),
(vec![(col_e, option_desc), (col_b, option_asc)], false),
(vec![(col_e, option_asc), (col_b, option_asc)], false),
(
vec![
(col_d, option_asc),
(col_b, option_asc),
(col_d, option_asc),
(col_b, option_asc),
],
true,
),
(
vec![
(col_d, option_asc),
(col_b, option_asc),
(col_e, option_desc),
(col_f, option_asc),
],
true,
),
(
vec![
(col_d, option_asc),
(col_b, option_asc),
(col_e, option_desc),
(col_b, option_asc),
],
true,
),
(
vec![
(col_d, option_asc),
(col_b, option_asc),
(col_d, option_desc),
(col_b, option_asc),
],
true,
),
(
vec![
(col_d, option_asc),
(col_b, option_asc),
(col_e, option_asc),
(col_f, option_asc),
],
false,
),
(
vec![
(col_d, option_asc),
(col_b, option_asc),
(col_e, option_asc),
(col_b, option_asc),
],
false,
),
(vec![(col_d, option_asc), (col_e, option_desc)], true),
(
vec![
(col_d, option_asc),
(col_c, option_asc),
(col_b, option_asc),
],
true,
),
(
vec![
(col_d, option_asc),
(col_e, option_desc),
(col_f, option_asc),
(col_b, option_asc),
],
true,
),
(
vec![
(col_d, option_asc),
(col_e, option_desc),
(col_c, option_asc),
(col_b, option_asc),
],
true,
),
(
vec![
(col_d, option_asc),
(col_e, option_desc),
(col_b, option_asc),
(col_f, option_asc),
],
true,
),
];

for (cols, expected) in requirements {
let err_msg = format!("Error in test case:{cols:?}");
let required = cols
.into_iter()
.map(|(expr, options)| PhysicalSortExpr {
expr: Arc::clone(expr),
options,
})
.collect::<Vec<_>>();

// Check expected result with experimental result.
assert_eq!(
is_table_same_after_sort(
required.clone(),
table_data_with_properties.clone()
)?,
expected
);
assert_eq!(
eq_properties.ordering_satisfy(&required),
expected,
"{err_msg}"
);
}
Ok(())
}
80 changes: 80 additions & 0 deletions datafusion/core/tests/fuzz_cases/equivalence/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,86 @@ fn get_representative_arr(
None
}

// Convert each tuple to PhysicalSortExpr
pub fn convert_to_sort_exprs(
in_data: &[(&Arc<dyn PhysicalExpr>, SortOptions)],
) -> Vec<PhysicalSortExpr> {
in_data
.iter()
.map(|(expr, options)| PhysicalSortExpr {
expr: Arc::clone(*expr),
options: *options,
})
.collect()
}

// Convert each inner tuple to PhysicalSortExpr
pub fn convert_to_orderings(
orderings: &[Vec<(&Arc<dyn PhysicalExpr>, SortOptions)>],
) -> Vec<Vec<PhysicalSortExpr>> {
orderings
.iter()
.map(|sort_exprs| convert_to_sort_exprs(sort_exprs))
.collect()
}

// Generate a schema which consists of 8 columns (a, b, c, d, e, f, g, h)
pub fn create_test_schema() -> Result<SchemaRef> {
let a = Field::new("a", DataType::Int32, true);
let b = Field::new("b", DataType::Int32, true);
let c = Field::new("c", DataType::Int32, true);
let d = Field::new("d", DataType::Int32, true);
let e = Field::new("e", DataType::Int32, true);
let f = Field::new("f", DataType::Int32, true);
let g = Field::new("g", DataType::Int32, true);
let h = Field::new("h", DataType::Int32, true);
let schema = Arc::new(Schema::new(vec![a, b, c, d, e, f, g, h]));

Ok(schema)
}

/// Construct a schema with following properties
/// Schema satisfies following orderings:
/// [a ASC], [d ASC, b ASC], [e DESC, f ASC, g ASC]
/// and
/// Column [a=c] (e.g they are aliases).
pub fn create_test_params() -> Result<(SchemaRef, EquivalenceProperties)> {
let test_schema = create_test_schema()?;
let col_a = &col("a", &test_schema)?;
let col_b = &col("b", &test_schema)?;
let col_c = &col("c", &test_schema)?;
let col_d = &col("d", &test_schema)?;
let col_e = &col("e", &test_schema)?;
let col_f = &col("f", &test_schema)?;
let col_g = &col("g", &test_schema)?;
let mut eq_properties = EquivalenceProperties::new(Arc::clone(&test_schema));
eq_properties.add_equal_conditions(col_a, col_c)?;

let option_asc = SortOptions {
descending: false,
nulls_first: false,
};
let option_desc = SortOptions {
descending: true,
nulls_first: true,
};
let orderings = vec![
// [a ASC]
vec![(col_a, option_asc)],
// [d ASC, b ASC]
vec![(col_d, option_asc), (col_b, option_asc)],
// [e DESC, f ASC, g ASC]
vec![
(col_e, option_desc),
(col_f, option_asc),
(col_g, option_asc),
],
];
let orderings = convert_to_orderings(&orderings);
eq_properties.add_new_orderings(orderings);
Ok((test_schema, eq_properties))
}

// Generate a table that satisfies the given equivalence properties; i.e.
// equivalences, ordering equivalences, and constants.
pub fn generate_table_for_eq_properties(
Expand Down
Loading