Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle arbitrarily different data in null list column rows when checking for equivalency. #8666

Merged
merged 9 commits into from
Jul 20, 2021
Merged

Handle arbitrarily different data in null list column rows when checking for equivalency. #8666

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
66ad749
Rework column equivalency checks to handle list columns that have arb…
nvdbaranec Jul 6, 2021
affc15e
Add recursive property checking for struct columns. Add the ability t…
nvdbaranec Jul 8, 2021
1719caa
Merge branch 'branch-21.08' into list_equivalency
nvdbaranec Jul 12, 2021
adf178c
Cleanup some variable declarations and type usage.
nvdbaranec Jul 12, 2021
6f18fbd
PR review changes.
nvdbaranec Jul 14, 2021
0999a2c
Merge branch 'branch-21.08' into list_equivalency
nvdbaranec Jul 16, 2021
0769c92
Change equality and equivalence functions to return a bool instead of…
nvdbaranec Jul 16, 2021
0724a3d
Formatting changes from PR review.
nvdbaranec Jul 19, 2021
dbf338e
Merge branch 'branch-21.08' into list_equivalency
nvdbaranec Jul 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 39 additions & 14 deletions cpp/include/cudf_test/column_utilities.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2019-2020, NVIDIA CORPORATION.
* Copyright (c) 2019-2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -27,13 +27,29 @@

namespace cudf {
namespace test {

/**
* @brief Verbosity level of output from column and table comparison functions.
*
nvdbaranec marked this conversation as resolved.
Show resolved Hide resolved
*/
enum class debug_output_level {
FIRST_ERROR = 0, // print first error only
ALL_ERRORS, // print all errors
QUIET, // no debug output
nvdbaranec marked this conversation as resolved.
Show resolved Hide resolved
};

/**
* @brief Verifies the property equality of two columns.
*
* @param lhs The first column
* @param rhs The second column
* @param verbosity Level of debug output verbosity
*
* @returns True if the column properties are equal, false otherwise
*/
void expect_column_properties_equal(cudf::column_view const& lhs, cudf::column_view const& rhs);
bool expect_column_properties_equal(cudf::column_view const& lhs,
cudf::column_view const& rhs,
debug_output_level verbosity = debug_output_level::FIRST_ERROR);

/**
* @brief Verifies the property equivalence of two columns.
Expand All @@ -44,36 +60,45 @@ void expect_column_properties_equal(cudf::column_view const& lhs, cudf::column_v
*
* @param lhs The first column
* @param rhs The second column
* @param verbosity Level of debug output verbosity
*
* @returns True if the column properties are equivalent, false otherwise
*/
void expect_column_properties_equivalent(cudf::column_view const& lhs,
cudf::column_view const& rhs);
bool expect_column_properties_equivalent(
cudf::column_view const& lhs,
cudf::column_view const& rhs,
debug_output_level verbosity = debug_output_level::FIRST_ERROR);

/**
* @brief Verifies the element-wise equality of two columns.
*
* Treats null elements as equivalent.
*
* @param lhs The first column
* @param rhs The second column
* @param print_all_differences If true display all differences
* @param lhs The first column
* @param rhs The second column
* @param verbosity Level of debug output verbosity
*
* @returns True if the columns (and their properties) are equal, false otherwise
*/
void expect_columns_equal(cudf::column_view const& lhs,
bool expect_columns_equal(cudf::column_view const& lhs,
cudf::column_view const& rhs,
bool print_all_differences = false);
debug_output_level verbosity = debug_output_level::FIRST_ERROR);

/**
* @brief Verifies the element-wise equivalence of two columns.
*
* Uses machine epsilon to compare floating point types.
* Treats null elements as equivalent.
ttnghia marked this conversation as resolved.
Show resolved Hide resolved
*
* @param lhs The first column
* @param rhs The second column
* @param print_all_differences If true display all differences
* @param lhs The first column
* @param rhs The second column
* @param verbosity Level of debug output verbosity
*
* @returns True if the columns (and their properties) are equivalent, false otherwise
*/
void expect_columns_equivalent(cudf::column_view const& lhs,
bool expect_columns_equivalent(cudf::column_view const& lhs,
cudf::column_view const& rhs,
bool print_all_differences = false);
debug_output_level verbosity = debug_output_level::FIRST_ERROR);

/**
* @brief Verifies the bitwise equality of two device memory buffers.
Expand Down
46 changes: 24 additions & 22 deletions cpp/tests/ast/transform_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
template <typename T>
using column_wrapper = cudf::test::fixed_width_column_wrapper<T>;

constexpr cudf::test::debug_output_level verbosity{cudf::test::debug_output_level::ALL_ERRORS};

struct TransformTest : public cudf::test::BaseFixture {
};

Expand All @@ -58,7 +60,7 @@ TEST_F(TransformTest, BasicAddition)
auto expected = column_wrapper<int32_t>{13, 27, 21, 50};
auto result = cudf::ast::compute_column(table, expression);

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, BasicAdditionLarge)
Expand All @@ -74,7 +76,7 @@ TEST_F(TransformTest, BasicAdditionLarge)
auto expected = column_wrapper<int32_t>(b, b + 2000);
auto result = cudf::ast::compute_column(table, expression);

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, LessComparator)
Expand All @@ -90,7 +92,7 @@ TEST_F(TransformTest, LessComparator)
auto expected = column_wrapper<bool>{true, false, true, false};
auto result = cudf::ast::compute_column(table, expression);

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, LessComparatorLarge)
Expand All @@ -109,7 +111,7 @@ TEST_F(TransformTest, LessComparatorLarge)
auto expected = column_wrapper<bool>(c, c + 2000);
auto result = cudf::ast::compute_column(table, expression);

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, MultiLevelTreeArithmetic)
Expand All @@ -135,7 +137,7 @@ TEST_F(TransformTest, MultiLevelTreeArithmetic)
auto result = cudf::ast::compute_column(table, expression_tree);
auto expected = column_wrapper<int32_t>{7, 73, 22, -99};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, MultiLevelTreeArithmeticLarge)
Expand Down Expand Up @@ -163,7 +165,7 @@ TEST_F(TransformTest, MultiLevelTreeArithmeticLarge)
auto d = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { return calc(i); });
auto expected = column_wrapper<int32_t>(d, d + 2000);

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, ImbalancedTreeArithmetic)
Expand All @@ -187,7 +189,7 @@ TEST_F(TransformTest, ImbalancedTreeArithmetic)
auto expected =
column_wrapper<double>{0.6, std::numeric_limits<double>::infinity(), -3.201, -2099.18};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, MultiLevelTreeComparator)
Expand All @@ -213,7 +215,7 @@ TEST_F(TransformTest, MultiLevelTreeComparator)
auto result = cudf::ast::compute_column(table, expression_tree);
auto expected = column_wrapper<bool>{false, true, false, false};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, MultiTypeOperationFailure)
Expand Down Expand Up @@ -249,7 +251,7 @@ TEST_F(TransformTest, LiteralComparison)
auto result = cudf::ast::compute_column(table, expression);
auto expected = column_wrapper<bool>{false, false, false, true};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, UnaryNot)
Expand All @@ -264,7 +266,7 @@ TEST_F(TransformTest, UnaryNot)
auto result = cudf::ast::compute_column(table, expression);
auto expected = column_wrapper<bool>{false, true, false, false};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, UnaryTrigonometry)
Expand All @@ -277,17 +279,17 @@ TEST_F(TransformTest, UnaryTrigonometry)
auto expected_sin = column_wrapper<double>{0.0, std::sqrt(2) / 2, std::sqrt(3.0) / 2.0};
auto expression_sin = cudf::ast::expression(cudf::ast::ast_operator::SIN, col_ref_0);
auto result_sin = cudf::ast::compute_column(table, expression_sin);
cudf::test::expect_columns_equivalent(expected_sin, result_sin->view(), true);
cudf::test::expect_columns_equivalent(expected_sin, result_sin->view(), verbosity);

auto expected_cos = column_wrapper<double>{1.0, std::sqrt(2) / 2, 0.5};
auto expression_cos = cudf::ast::expression(cudf::ast::ast_operator::COS, col_ref_0);
auto result_cos = cudf::ast::compute_column(table, expression_cos);
cudf::test::expect_columns_equivalent(expected_cos, result_cos->view(), true);
cudf::test::expect_columns_equivalent(expected_cos, result_cos->view(), verbosity);

auto expected_tan = column_wrapper<double>{0.0, 1.0, std::sqrt(3.0)};
auto expression_tan = cudf::ast::expression(cudf::ast::ast_operator::TAN, col_ref_0);
auto result_tan = cudf::ast::compute_column(table, expression_tan);
cudf::test::expect_columns_equivalent(expected_tan, result_tan->view(), true);
cudf::test::expect_columns_equivalent(expected_tan, result_tan->view(), verbosity);
}

TEST_F(TransformTest, ArityCheckFailure)
Expand All @@ -311,7 +313,7 @@ TEST_F(TransformTest, StringComparison)
auto expected = column_wrapper<bool>{true, false, true, false};
auto result = cudf::ast::compute_column(table, expression);

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, CopyColumn)
Expand All @@ -325,7 +327,7 @@ TEST_F(TransformTest, CopyColumn)
auto result = cudf::ast::compute_column(table, expression);
auto expected = column_wrapper<int32_t>{3, 0, 1, 50};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, CopyLiteral)
Expand All @@ -341,7 +343,7 @@ TEST_F(TransformTest, CopyLiteral)
auto result = cudf::ast::compute_column(table, expression);
auto expected = column_wrapper<int32_t>{-123, -123, -123, -123};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, TrueDiv)
Expand All @@ -358,7 +360,7 @@ TEST_F(TransformTest, TrueDiv)
auto result = cudf::ast::compute_column(table, expression);
auto expected = column_wrapper<double>{1.5, 0.0, 0.5, 25.0};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, FloorDiv)
Expand All @@ -375,7 +377,7 @@ TEST_F(TransformTest, FloorDiv)
auto result = cudf::ast::compute_column(table, expression);
auto expected = column_wrapper<double>{1.0, 0.0, 0.0, 25.0};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, Mod)
Expand All @@ -392,7 +394,7 @@ TEST_F(TransformTest, Mod)
auto result = cudf::ast::compute_column(table, expression);
auto expected = column_wrapper<double>{1.0, 0.0, -1.0, 0.0};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, PyMod)
Expand All @@ -409,7 +411,7 @@ TEST_F(TransformTest, PyMod)
auto result = cudf::ast::compute_column(table, expression);
auto expected = column_wrapper<double>{1.0, 0.0, 1.0, 0.0};

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, BasicAdditionNulls)
Expand All @@ -425,7 +427,7 @@ TEST_F(TransformTest, BasicAdditionNulls)
auto expected = column_wrapper<int32_t>{{0, 0, 0, 50}, {0, 0, 0, 1}};
auto result = cudf::ast::compute_column(table, expression);

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

TEST_F(TransformTest, BasicAdditionLargeNulls)
Expand All @@ -451,7 +453,7 @@ TEST_F(TransformTest, BasicAdditionLargeNulls)
auto expected = column_wrapper<int32_t>(b, b + N, validities.begin());
auto result = cudf::ast::compute_column(table, expression);

cudf::test::expect_columns_equal(expected, result->view(), true);
cudf::test::expect_columns_equal(expected, result->view(), verbosity);
}

CUDF_TEST_PROGRAM_MAIN()
Loading