From 1fa71986166c1ab2baefa403e50f743f05f4c5ac Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 2 Sep 2022 15:02:29 +0800 Subject: [PATCH 1/9] tmp save --- .../tests/gtest_aggregation_executor.cpp | 6 ++- dbms/src/TestUtils/AggregationTestUtils.h | 51 +++++++++++++++++-- .../WindowFunctions/tests/gtest_lead_lag.cpp | 3 +- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp index 3f1766769f5..95d2af01570 100644 --- a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp +++ b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include namespace DB @@ -30,7 +30,7 @@ namespace tests types_col_name[a], types_col_name[b] \ } -class ExecutorAggTestRunner : public DB::tests::ExecutorTest +class ExecutorAggTestRunner : public DB::tests::AggregationTest { public: using ColStringNullableType = std::optional::FieldType>; @@ -199,6 +199,8 @@ try request = buildDAGRequest(std::make_pair(db_name, table_types), {}, group_by_exprs[i], projections[i]); executeWithConcurrency(request, expect_cols[i]); } + // ywq todo + executeGroupByAndAssert(expect_cols[0], {toNullableVec(types_col_name[2], col_tinyint)}); } { diff --git a/dbms/src/TestUtils/AggregationTestUtils.h b/dbms/src/TestUtils/AggregationTestUtils.h index f69395794cb..e7b46b4f486 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.h +++ b/dbms/src/TestUtils/AggregationTestUtils.h @@ -15,18 +15,23 @@ #pragma once #include -#include -#include +#include #include #include +#include + +#include "Core/ColumnsWithTypeAndName.h" +#include "Debug/MockStorage.h" +#include "TestUtils/FunctionTestUtils.h" + namespace DB::tests { -class AggregationTest : public ::testing::Test +class AggregationTest : public ExecutorTest { public: - ::testing::AssertionResult checkAggReturnType(const String & agg_name, const DataTypes & data_types, const DataTypePtr & expect_type) + static ::testing::AssertionResult checkAggReturnType(const String & agg_name, const DataTypes & data_types, const DataTypePtr & expect_type) { AggregateFunctionPtr agg_ptr = DB::AggregateFunctionFactory::instance().get(agg_name, data_types, {}); const DataTypePtr & ret_type = agg_ptr->getReturnType(); @@ -35,6 +40,44 @@ class AggregationTest : public ::testing::Test return ::testing::AssertionFailure() << "Expect type: " << expect_type->getName() << " Actual type: " << ret_type->getName(); } + void executeAggFunctionAndAssert(ASTPtr func, String func_name, const ColumnsWithTypeAndName & column, const ColumnsWithTypeAndName & expected_cols) + { + context.addMockTableColumnData("test_db", "test_table", column); + auto request = context.scan("test_db", "test_table") + .aggregation(func, {}) + .project({func_name}) + .build(context); + + for (size_t i = 1; i <= 10; ++i) + ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)); + } + + void executeGroupByAndAssert(const ColumnsWithTypeAndName & columns, const ColumnsWithTypeAndName & expected_cols) + { + String db_name = "test_group"; + String table_name = "test_tablexx"; + MockAstVec group_by_cols; + MockColumnNameVec proj_names; + MockColumnInfoVec column_infos; + for (const auto & column : columns) + { + group_by_cols.push_back(col(column.name)); + proj_names.push_back(column.name); + column_infos.push_back({column.name, dataTypeToTP(column.type)}); + } + + context.addMockTable(db_name, table_name, column_infos, columns); + + std::cout << "ywq test here..." << std::endl; + auto request = context.scan(db_name, table_name) + .aggregation({}, group_by_cols) + .project(proj_names) + .build(context); + + for (size_t i = 1; i <= 10; ++i) + ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)) << "actual_cols: " << getColumnsContent(expected_cols) << ", expected_cols: " << getColumnsContent(executeStreams(request, i)); + } + static void SetUpTestCase(); }; diff --git a/dbms/src/WindowFunctions/tests/gtest_lead_lag.cpp b/dbms/src/WindowFunctions/tests/gtest_lead_lag.cpp index 71206df8aab..f2274726e96 100644 --- a/dbms/src/WindowFunctions/tests/gtest_lead_lag.cpp +++ b/dbms/src/WindowFunctions/tests/gtest_lead_lag.cpp @@ -39,7 +39,8 @@ class LeadLag : public DB::tests::ExecutorTest std::vector block_sizes{1, 2, 3, DEFAULT_BLOCK_SIZE}; for (auto block_size : block_sizes) { - context.context.setSetting("max_block_size", block_size); + Field f(static_cast(block_size)); + context.context.setSetting("max_block_size", f); ASSERT_COLUMNS_EQ_R(expect_columns, executeStreams(request)); for (size_t i = 2; i <= max_concurrency_level; ++i) { From 7f02acdde1b0928c4d3dd74a15d50467e81975fd Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 2 Sep 2022 17:29:03 +0800 Subject: [PATCH 2/9] tmp save. --- .../Flash/tests/gtest_aggregation_executor.cpp | 4 ++-- dbms/src/TestUtils/AggregationTestUtils.h | 17 ++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp index 95d2af01570..ef41e7815ce 100644 --- a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp +++ b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp @@ -199,8 +199,8 @@ try request = buildDAGRequest(std::make_pair(db_name, table_types), {}, group_by_exprs[i], projections[i]); executeWithConcurrency(request, expect_cols[i]); } - // ywq todo - executeGroupByAndAssert(expect_cols[0], {toNullableVec(types_col_name[2], col_tinyint)}); + + executeGroupByAndAssert({toNullableVec("tinyint_", col_tinyint)}, expect_cols[0]); } { diff --git a/dbms/src/TestUtils/AggregationTestUtils.h b/dbms/src/TestUtils/AggregationTestUtils.h index e7b46b4f486..e379462c3d5 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.h +++ b/dbms/src/TestUtils/AggregationTestUtils.h @@ -52,30 +52,29 @@ class AggregationTest : public ExecutorTest ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)); } - void executeGroupByAndAssert(const ColumnsWithTypeAndName & columns, const ColumnsWithTypeAndName & expected_cols) + void executeGroupByAndAssert(const ColumnsWithTypeAndName & cols, const ColumnsWithTypeAndName & expected_cols) { String db_name = "test_group"; - String table_name = "test_tablexx"; + String table_name = "test_tablexx"; // todo refine. MockAstVec group_by_cols; MockColumnNameVec proj_names; MockColumnInfoVec column_infos; - for (const auto & column : columns) + for (const auto & col : cols) { - group_by_cols.push_back(col(column.name)); - proj_names.push_back(column.name); - column_infos.push_back({column.name, dataTypeToTP(column.type)}); + group_by_cols.push_back(col(col.name)); + proj_names.push_back(col.name); + column_infos.push_back({col.name, dataTypeToTP(col.type)}); } - context.addMockTable(db_name, table_name, column_infos, columns); + context.addMockTable(db_name, table_name, column_infos, cols); - std::cout << "ywq test here..." << std::endl; auto request = context.scan(db_name, table_name) .aggregation({}, group_by_cols) .project(proj_names) .build(context); for (size_t i = 1; i <= 10; ++i) - ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)) << "actual_cols: " << getColumnsContent(expected_cols) << ", expected_cols: " << getColumnsContent(executeStreams(request, i)); + ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)) << "expected_cols: " << getColumnsContent(expected_cols) << ", actual_cols: " << getColumnsContent(executeStreams(request, i)); } static void SetUpTestCase(); From ad2055fa36e706a47d342492986c6da9e60cc52b Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 2 Sep 2022 17:30:52 +0800 Subject: [PATCH 3/9] refine. --- dbms/src/TestUtils/AggregationTestUtils.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dbms/src/TestUtils/AggregationTestUtils.h b/dbms/src/TestUtils/AggregationTestUtils.h index e379462c3d5..cd52f84a5c8 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.h +++ b/dbms/src/TestUtils/AggregationTestUtils.h @@ -19,12 +19,6 @@ #include #include -#include - -#include "Core/ColumnsWithTypeAndName.h" -#include "Debug/MockStorage.h" -#include "TestUtils/FunctionTestUtils.h" - namespace DB::tests { From f4bd4cfee8a0a54fb9f7ea0221fa2940652de1ce Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Fri, 2 Sep 2022 17:46:40 +0800 Subject: [PATCH 4/9] more api. --- .../tests/gtest_aggregation_executor.cpp | 10 +++++++-- dbms/src/TestUtils/AggregationTestUtils.h | 22 ++++++++++++++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp index ef41e7815ce..e08c612005f 100644 --- a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp +++ b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp @@ -199,8 +199,6 @@ try request = buildDAGRequest(std::make_pair(db_name, table_types), {}, group_by_exprs[i], projections[i]); executeWithConcurrency(request, expect_cols[i]); } - - executeGroupByAndAssert({toNullableVec("tinyint_", col_tinyint)}, expect_cols[0]); } { @@ -357,6 +355,14 @@ try } CATCH +TEST_F(ExecutorAggTestRunner, TestFramwork) +try +{ + executeGroupByAndAssert({toNullableVec("tinyint_", col_tinyint)}, {toNullableVec(ColumnWithNullableInt8{-1, 2, {}, 0, 1, 3, -2})}); + executeGroupByAndAssertWithTable(db_name, table_types, {toNullableVec(types_col_name[2], ColumnWithNullableInt8{-1, 2, {}, 0, 1, 3, -2})}); +} +CATCH + // TODO support more type of min, max, count. // support more aggregation functions: sum, forst_row, group_concat diff --git a/dbms/src/TestUtils/AggregationTestUtils.h b/dbms/src/TestUtils/AggregationTestUtils.h index cd52f84a5c8..2e897f5e116 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.h +++ b/dbms/src/TestUtils/AggregationTestUtils.h @@ -49,7 +49,7 @@ class AggregationTest : public ExecutorTest void executeGroupByAndAssert(const ColumnsWithTypeAndName & cols, const ColumnsWithTypeAndName & expected_cols) { String db_name = "test_group"; - String table_name = "test_tablexx"; // todo refine. + String table_name = "test_table_group"; MockAstVec group_by_cols; MockColumnNameVec proj_names; MockColumnInfoVec column_infos; @@ -70,6 +70,26 @@ class AggregationTest : public ExecutorTest for (size_t i = 1; i <= 10; ++i) ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)) << "expected_cols: " << getColumnsContent(expected_cols) << ", actual_cols: " << getColumnsContent(executeStreams(request, i)); } + + // todo refine.. + void executeGroupByAndAssertWithTable(const String & db_name, const String & table_name, const ColumnsWithTypeAndName & expected_cols) + { + MockAstVec group_by_cols; + MockColumnNameVec proj_names; + for (const auto & col : expected_cols) + { + group_by_cols.push_back(col(col.name)); + proj_names.push_back(col.name); + } + + auto request = context.scan(db_name, table_name) + .aggregation({}, group_by_cols) + .project(proj_names) + .build(context); + + for (size_t i = 1; i <= 10; ++i) + ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)) << "expected_cols: " << getColumnsContent(expected_cols) << ", actual_cols: " << getColumnsContent(executeStreams(request, i)); + } static void SetUpTestCase(); }; From beea5d5a581a820f608ff5fa4d661bd6c97e785f Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Mon, 5 Sep 2022 13:39:19 +0800 Subject: [PATCH 5/9] init. --- .../tests/gtest_aggregation_executor.cpp | 5 +- dbms/src/TestUtils/AggregationTestUtils.cpp | 103 ++++++++++++++++++ dbms/src/TestUtils/AggregationTestUtils.h | 78 +++---------- 3 files changed, 120 insertions(+), 66 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp index e08c612005f..c1815a538ba 100644 --- a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp +++ b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp @@ -359,7 +359,10 @@ TEST_F(ExecutorAggTestRunner, TestFramwork) try { executeGroupByAndAssert({toNullableVec("tinyint_", col_tinyint)}, {toNullableVec(ColumnWithNullableInt8{-1, 2, {}, 0, 1, 3, -2})}); - executeGroupByAndAssertWithTable(db_name, table_types, {toNullableVec(types_col_name[2], ColumnWithNullableInt8{-1, 2, {}, 0, 1, 3, -2})}); + executeGroupByAndAssertWithTable(db_name, table_types, {types_col_name[2]}, {toNullableVec(ColumnWithNullableInt8{-1, 2, {}, 0, 1, 3, -2})}); + + executeAggFunctionAndAssert("Max", toNullableVec("tinyint_", col_tinyint), toNullableVec(ColumnWithNullableInt8{3})); + executeAggFunctionAndAssertWithTable("test_db", "types", "Max", types_col_name[2], toNullableVec(ColumnWithNullableInt8{3})); } CATCH diff --git a/dbms/src/TestUtils/AggregationTestUtils.cpp b/dbms/src/TestUtils/AggregationTestUtils.cpp index 114ad0a11e0..d6772155aaf 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.cpp +++ b/dbms/src/TestUtils/AggregationTestUtils.cpp @@ -31,4 +31,107 @@ void AggregationTest::SetUpTestCase() register_func(DB::registerAggregateFunctions); } + +::testing::AssertionResult AggregationTest::checkAggReturnType(const String & agg_name, const DataTypes & data_types, const DataTypePtr & expect_type) +{ + AggregateFunctionPtr agg_ptr = DB::AggregateFunctionFactory::instance().get(agg_name, data_types, {}); + const DataTypePtr & ret_type = agg_ptr->getReturnType(); + if (ret_type->equals(*expect_type)) + return ::testing::AssertionSuccess(); + return ::testing::AssertionFailure() << "Expect type: " << expect_type->getName() << " Actual type: " << ret_type->getName(); +} + +void AggregationTest::executeAggFunctionAndAssert(const String & func_name, const ColumnWithTypeAndName & column, const ColumnWithTypeAndName & expected_col) +{ + String db_name = "test_agg_function"; + String table_name = "test_table_agg"; + ASTPtr func = aggFunctionBuilder(func_name, column.name); + + MockColumnInfoVec column_infos; + column_infos.push_back({column.name, dataTypeToTP(column.type)}); + context.addMockTable(db_name, table_name, column_infos, {column}); + + auto request = context.scan(db_name, table_name) + .aggregation(func, {}) + .build(context); + + checkResult(request, {expected_col}); +} + +void AggregationTest::executeAggFunctionAndAssertWithTable(const String & db_name, const String & table_name, const String & func_name, const String & col_name, const ColumnWithTypeAndName & expected_col) +{ + ASTPtr func = aggFunctionBuilder(func_name, col_name); + + auto request = context.scan(db_name, table_name) + .aggregation(func, {}) + .build(context); + + checkResult(request, {expected_col}); +} + +void AggregationTest::executeGroupByAndAssert(const ColumnsWithTypeAndName & cols, const ColumnsWithTypeAndName & expected_cols) +{ + String db_name = "test_group"; + String table_name = "test_table_group"; + MockAstVec group_by_cols; + MockColumnNameVec proj_names; + MockColumnInfoVec column_infos; + for (const auto & col : cols) + { + group_by_cols.push_back(col(col.name)); + proj_names.push_back(col.name); + column_infos.push_back({col.name, dataTypeToTP(col.type)}); + } + + context.addMockTable(db_name, table_name, column_infos, cols); + + auto request = context.scan(db_name, table_name) + .aggregation({}, group_by_cols) + .project(proj_names) + .build(context); + + checkResult(request, expected_cols); +} + +void AggregationTest::executeGroupByAndAssertWithTable(const String & db_name, const String & table_name, const std::vector & group_by_cols, const ColumnsWithTypeAndName & expected_cols) +{ + MockAstVec group_by_col_asts; + MockColumnNameVec proj_names; + for (const auto & col : group_by_cols) + { + group_by_col_asts.push_back(col(col)); + proj_names.push_back(col); + } + + auto request = context.scan(db_name, table_name) + .aggregation({}, group_by_col_asts) + .project(proj_names) + .build(context); + + checkResult(request, expected_cols); +} + +void AggregationTest::checkResult(std::shared_ptr request, const ColumnsWithTypeAndName & expected_cols) +{ + for (size_t i = 1; i <= 10; ++i) + ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)) << "expected_cols: " << getColumnsContent(expected_cols) << ", actual_cols: " << getColumnsContent(executeStreams(request, i)); +} +ASTPtr AggregationTest::aggFunctionBuilder(const String & func_name, const String & col_name) +{ + ASTPtr func; + String func_name_lowercase = Poco::toLower(func_name); + + // TODO support more agg functions. + if (func_name_lowercase == "max") + func = Max(col(col_name)); + else if (func_name_lowercase == "min") + func = Min(col(col_name)); + else if (func_name_lowercase == "count") + func = Count(col(col_name)); + else if (func_name_lowercase == "sum") + func = Sum(col(col_name)); + else + throw Exception(fmt::format("Unsupported agg function {}", func_name), ErrorCodes::LOGICAL_ERROR); + return func; +} } // namespace DB::tests diff --git a/dbms/src/TestUtils/AggregationTestUtils.h b/dbms/src/TestUtils/AggregationTestUtils.h index 2e897f5e116..d712a48fca0 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.h +++ b/dbms/src/TestUtils/AggregationTestUtils.h @@ -16,8 +16,6 @@ #include #include -#include -#include namespace DB::tests { @@ -25,73 +23,23 @@ namespace DB::tests class AggregationTest : public ExecutorTest { public: - static ::testing::AssertionResult checkAggReturnType(const String & agg_name, const DataTypes & data_types, const DataTypePtr & expect_type) - { - AggregateFunctionPtr agg_ptr = DB::AggregateFunctionFactory::instance().get(agg_name, data_types, {}); - const DataTypePtr & ret_type = agg_ptr->getReturnType(); - if (ret_type->equals(*expect_type)) - return ::testing::AssertionSuccess(); - return ::testing::AssertionFailure() << "Expect type: " << expect_type->getName() << " Actual type: " << ret_type->getName(); - } + static ::testing::AssertionResult checkAggReturnType(const String & agg_name, const DataTypes & data_types, const DataTypePtr & expect_type); - void executeAggFunctionAndAssert(ASTPtr func, String func_name, const ColumnsWithTypeAndName & column, const ColumnsWithTypeAndName & expected_cols) - { - context.addMockTableColumnData("test_db", "test_table", column); - auto request = context.scan("test_db", "test_table") - .aggregation(func, {}) - .project({func_name}) - .build(context); + // Test one aggregation functions without group by. + void executeAggFunctionAndAssert(const String & func_name, const ColumnWithTypeAndName & column, const ColumnWithTypeAndName & expected_col); + // Prequisite: add a mock table in MockDAGRequestContext. + void executeAggFunctionAndAssertWithTable(const String & db_name, const String & table_name, const String & func_name, const String & col_name, const ColumnWithTypeAndName & expected_col); - for (size_t i = 1; i <= 10; ++i) - ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)); - } - - void executeGroupByAndAssert(const ColumnsWithTypeAndName & cols, const ColumnsWithTypeAndName & expected_cols) - { - String db_name = "test_group"; - String table_name = "test_table_group"; - MockAstVec group_by_cols; - MockColumnNameVec proj_names; - MockColumnInfoVec column_infos; - for (const auto & col : cols) - { - group_by_cols.push_back(col(col.name)); - proj_names.push_back(col.name); - column_infos.push_back({col.name, dataTypeToTP(col.type)}); - } - - context.addMockTable(db_name, table_name, column_infos, cols); - - auto request = context.scan(db_name, table_name) - .aggregation({}, group_by_cols) - .project(proj_names) - .build(context); - - for (size_t i = 1; i <= 10; ++i) - ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)) << "expected_cols: " << getColumnsContent(expected_cols) << ", actual_cols: " << getColumnsContent(executeStreams(request, i)); - } - - // todo refine.. - void executeGroupByAndAssertWithTable(const String & db_name, const String & table_name, const ColumnsWithTypeAndName & expected_cols) - { - MockAstVec group_by_cols; - MockColumnNameVec proj_names; - for (const auto & col : expected_cols) - { - group_by_cols.push_back(col(col.name)); - proj_names.push_back(col.name); - } - - auto request = context.scan(db_name, table_name) - .aggregation({}, group_by_cols) - .project(proj_names) - .build(context); - - for (size_t i = 1; i <= 10; ++i) - ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)) << "expected_cols: " << getColumnsContent(expected_cols) << ", actual_cols: " << getColumnsContent(executeStreams(request, i)); - } + // Test group by columns + void executeGroupByAndAssert(const ColumnsWithTypeAndName & cols, const ColumnsWithTypeAndName & expected_cols); + // Prequisite: add a mock table in MockDAGRequestContext. + void executeGroupByAndAssertWithTable(const String & db_name, const String & table_name, const std::vector & group_by_cols, const ColumnsWithTypeAndName & expected_cols); static void SetUpTestCase(); + +private: + void checkResult(std::shared_ptr request, const ColumnsWithTypeAndName & expected_cols); + ASTPtr aggFunctionBuilder(const String & func_name, const String & col_name); }; } // namespace DB::tests From 49fcdbb2c7d69c4ef6cd2a580c50f39c0be6b5ba Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Mon, 5 Sep 2022 14:27:23 +0800 Subject: [PATCH 6/9] u --- .../tests/gtest_aggregation_executor.cpp | 6 +++-- dbms/src/TestUtils/AggregationTestUtils.cpp | 25 +++++++++++++------ dbms/src/TestUtils/AggregationTestUtils.h | 7 ++++-- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp index c1815a538ba..47559c40175 100644 --- a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp +++ b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp @@ -361,8 +361,10 @@ try executeGroupByAndAssert({toNullableVec("tinyint_", col_tinyint)}, {toNullableVec(ColumnWithNullableInt8{-1, 2, {}, 0, 1, 3, -2})}); executeGroupByAndAssertWithTable(db_name, table_types, {types_col_name[2]}, {toNullableVec(ColumnWithNullableInt8{-1, 2, {}, 0, 1, 3, -2})}); - executeAggFunctionAndAssert("Max", toNullableVec("tinyint_", col_tinyint), toNullableVec(ColumnWithNullableInt8{3})); - executeAggFunctionAndAssertWithTable("test_db", "types", "Max", types_col_name[2], toNullableVec(ColumnWithNullableInt8{3})); + executeAggFunctionAndAssert({"Max"}, toNullableVec("tinyint_", col_tinyint), {toNullableVec(ColumnWithNullableInt8{3})}); + executeAggFunctionAndAssert({"Max", "Min"}, toNullableVec("tinyint_", col_tinyint), {toNullableVec(ColumnWithNullableInt8{3}), toNullableVec(ColumnWithNullableInt8{-2})}); + executeAggFunctionAndAssertWithTable("test_db", "types", {"Max"}, {types_col_name[2]}, {toNullableVec(ColumnWithNullableInt8{3})}); + executeAggFunctionAndAssertWithTable("test_db", "types", {"Max", "Min"}, {types_col_name[2], types_col_name[2]}, {toNullableVec(ColumnWithNullableInt8{3}), toNullableVec(ColumnWithNullableInt8{-2})}); } CATCH diff --git a/dbms/src/TestUtils/AggregationTestUtils.cpp b/dbms/src/TestUtils/AggregationTestUtils.cpp index d6772155aaf..1bfb475512f 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.cpp +++ b/dbms/src/TestUtils/AggregationTestUtils.cpp @@ -14,6 +14,10 @@ #include +#include + +#include "Parsers/IAST.h" + namespace DB::tests { void AggregationTest::SetUpTestCase() @@ -41,32 +45,37 @@ ::testing::AssertionResult AggregationTest::checkAggReturnType(const String & ag return ::testing::AssertionFailure() << "Expect type: " << expect_type->getName() << " Actual type: " << ret_type->getName(); } -void AggregationTest::executeAggFunctionAndAssert(const String & func_name, const ColumnWithTypeAndName & column, const ColumnWithTypeAndName & expected_col) +void AggregationTest::executeAggFunctionAndAssert(const std::vector & func_names, const ColumnWithTypeAndName & column, const ColumnsWithTypeAndName & expected_cols) { String db_name = "test_agg_function"; String table_name = "test_table_agg"; - ASTPtr func = aggFunctionBuilder(func_name, column.name); + std::vector agg_funcs; + for (const auto & func_name : func_names) + agg_funcs.push_back(aggFunctionBuilder(func_name, column.name)); MockColumnInfoVec column_infos; column_infos.push_back({column.name, dataTypeToTP(column.type)}); context.addMockTable(db_name, table_name, column_infos, {column}); auto request = context.scan(db_name, table_name) - .aggregation(func, {}) + .aggregation(agg_funcs, {}) .build(context); - checkResult(request, {expected_col}); + checkResult(request, expected_cols); } -void AggregationTest::executeAggFunctionAndAssertWithTable(const String & db_name, const String & table_name, const String & func_name, const String & col_name, const ColumnWithTypeAndName & expected_col) +void AggregationTest::executeAggFunctionAndAssertWithTable(const String & db_name, const String & table_name, const std::vector & func_names, const std::vector & col_names, const ColumnsWithTypeAndName & expected_cols) { - ASTPtr func = aggFunctionBuilder(func_name, col_name); + std::vector agg_funcs; + + for (size_t i = 0; i < func_names.size(); ++i) + agg_funcs.push_back(aggFunctionBuilder(func_names[i], col_names[i])); auto request = context.scan(db_name, table_name) - .aggregation(func, {}) + .aggregation(agg_funcs, {}) .build(context); - checkResult(request, {expected_col}); + checkResult(request, expected_cols); } void AggregationTest::executeGroupByAndAssert(const ColumnsWithTypeAndName & cols, const ColumnsWithTypeAndName & expected_cols) diff --git a/dbms/src/TestUtils/AggregationTestUtils.h b/dbms/src/TestUtils/AggregationTestUtils.h index d712a48fca0..06dcc7261b6 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.h +++ b/dbms/src/TestUtils/AggregationTestUtils.h @@ -26,9 +26,12 @@ class AggregationTest : public ExecutorTest static ::testing::AssertionResult checkAggReturnType(const String & agg_name, const DataTypes & data_types, const DataTypePtr & expect_type); // Test one aggregation functions without group by. - void executeAggFunctionAndAssert(const String & func_name, const ColumnWithTypeAndName & column, const ColumnWithTypeAndName & expected_col); + void executeAggFunctionAndAssert(const std::vector & func_names, const ColumnWithTypeAndName & column, const ColumnsWithTypeAndName & expected_cols); + // Prequisite: add a mock table in MockDAGRequestContext. - void executeAggFunctionAndAssertWithTable(const String & db_name, const String & table_name, const String & func_name, const String & col_name, const ColumnWithTypeAndName & expected_col); + // func_names: agg functions which are going to execute. + // col_names: columns each agg function executed on. + void executeAggFunctionAndAssertWithTable(const String & db_name, const String & table_name, const std::vector & func_names, const std::vector & col_names, const ColumnsWithTypeAndName & expected_cols); // Test group by columns void executeGroupByAndAssert(const ColumnsWithTypeAndName & cols, const ColumnsWithTypeAndName & expected_cols); From ca8f03f19932b1f4d1cdc4f4046c6168829603c5 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Mon, 5 Sep 2022 14:49:05 +0800 Subject: [PATCH 7/9] remove useless api. --- .../tests/gtest_aggregation_executor.cpp | 7 +--- dbms/src/TestUtils/AggregationTestUtils.cpp | 39 ++----------------- dbms/src/TestUtils/AggregationTestUtils.h | 17 ++++---- 3 files changed, 13 insertions(+), 50 deletions(-) diff --git a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp index 47559c40175..30f85867a5d 100644 --- a/dbms/src/Flash/tests/gtest_aggregation_executor.cpp +++ b/dbms/src/Flash/tests/gtest_aggregation_executor.cpp @@ -358,13 +358,10 @@ CATCH TEST_F(ExecutorAggTestRunner, TestFramwork) try { - executeGroupByAndAssert({toNullableVec("tinyint_", col_tinyint)}, {toNullableVec(ColumnWithNullableInt8{-1, 2, {}, 0, 1, 3, -2})}); - executeGroupByAndAssertWithTable(db_name, table_types, {types_col_name[2]}, {toNullableVec(ColumnWithNullableInt8{-1, 2, {}, 0, 1, 3, -2})}); - + executeGroupByAndAssert({toNullableVec("tinyint_", col_tinyint)}, {toNullableVec({-1, 2, {}, 0, 1, 3, -2})}); + executeGroupByAndAssert({toNullableVec("tinyint_", col_tinyint), toNullableVec("smallint_", col_smallint)}, {toNullableVec({0, 2, 0, -1, 1, -2, 3, {}, {}}), toNullableVec({-1, 3, -2, 4, 2, 0, {}, {}, 0})}); executeAggFunctionAndAssert({"Max"}, toNullableVec("tinyint_", col_tinyint), {toNullableVec(ColumnWithNullableInt8{3})}); executeAggFunctionAndAssert({"Max", "Min"}, toNullableVec("tinyint_", col_tinyint), {toNullableVec(ColumnWithNullableInt8{3}), toNullableVec(ColumnWithNullableInt8{-2})}); - executeAggFunctionAndAssertWithTable("test_db", "types", {"Max"}, {types_col_name[2]}, {toNullableVec(ColumnWithNullableInt8{3})}); - executeAggFunctionAndAssertWithTable("test_db", "types", {"Max", "Min"}, {types_col_name[2], types_col_name[2]}, {toNullableVec(ColumnWithNullableInt8{3}), toNullableVec(ColumnWithNullableInt8{-2})}); } CATCH diff --git a/dbms/src/TestUtils/AggregationTestUtils.cpp b/dbms/src/TestUtils/AggregationTestUtils.cpp index 1bfb475512f..0c4d95b9e84 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.cpp +++ b/dbms/src/TestUtils/AggregationTestUtils.cpp @@ -14,10 +14,6 @@ #include -#include - -#include "Parsers/IAST.h" - namespace DB::tests { void AggregationTest::SetUpTestCase() @@ -64,22 +60,10 @@ void AggregationTest::executeAggFunctionAndAssert(const std::vector & fu checkResult(request, expected_cols); } -void AggregationTest::executeAggFunctionAndAssertWithTable(const String & db_name, const String & table_name, const std::vector & func_names, const std::vector & col_names, const ColumnsWithTypeAndName & expected_cols) -{ - std::vector agg_funcs; - - for (size_t i = 0; i < func_names.size(); ++i) - agg_funcs.push_back(aggFunctionBuilder(func_names[i], col_names[i])); - - auto request = context.scan(db_name, table_name) - .aggregation(agg_funcs, {}) - .build(context); - - checkResult(request, expected_cols); -} - void AggregationTest::executeGroupByAndAssert(const ColumnsWithTypeAndName & cols, const ColumnsWithTypeAndName & expected_cols) { + RUNTIME_CHECK_MSG(cols.size() == expected_cols.size(), "number of group_by columns don't match number of expected columns"); + String db_name = "test_group"; String table_name = "test_table_group"; MockAstVec group_by_cols; @@ -102,29 +86,12 @@ void AggregationTest::executeGroupByAndAssert(const ColumnsWithTypeAndName & col checkResult(request, expected_cols); } -void AggregationTest::executeGroupByAndAssertWithTable(const String & db_name, const String & table_name, const std::vector & group_by_cols, const ColumnsWithTypeAndName & expected_cols) -{ - MockAstVec group_by_col_asts; - MockColumnNameVec proj_names; - for (const auto & col : group_by_cols) - { - group_by_col_asts.push_back(col(col)); - proj_names.push_back(col); - } - - auto request = context.scan(db_name, table_name) - .aggregation({}, group_by_col_asts) - .project(proj_names) - .build(context); - - checkResult(request, expected_cols); -} - void AggregationTest::checkResult(std::shared_ptr request, const ColumnsWithTypeAndName & expected_cols) { for (size_t i = 1; i <= 10; ++i) ASSERT_COLUMNS_EQ_UR(expected_cols, executeStreams(request, i)) << "expected_cols: " << getColumnsContent(expected_cols) << ", actual_cols: " << getColumnsContent(executeStreams(request, i)); } + ASTPtr AggregationTest::aggFunctionBuilder(const String & func_name, const String & col_name) { ASTPtr func; diff --git a/dbms/src/TestUtils/AggregationTestUtils.h b/dbms/src/TestUtils/AggregationTestUtils.h index 06dcc7261b6..7cbfb68514b 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.h +++ b/dbms/src/TestUtils/AggregationTestUtils.h @@ -26,17 +26,16 @@ class AggregationTest : public ExecutorTest static ::testing::AssertionResult checkAggReturnType(const String & agg_name, const DataTypes & data_types, const DataTypePtr & expect_type); // Test one aggregation functions without group by. - void executeAggFunctionAndAssert(const std::vector & func_names, const ColumnWithTypeAndName & column, const ColumnsWithTypeAndName & expected_cols); - - // Prequisite: add a mock table in MockDAGRequestContext. - // func_names: agg functions which are going to execute. - // col_names: columns each agg function executed on. - void executeAggFunctionAndAssertWithTable(const String & db_name, const String & table_name, const std::vector & func_names, const std::vector & col_names, const ColumnsWithTypeAndName & expected_cols); + void executeAggFunctionAndAssert( + const std::vector & func_names, + const ColumnWithTypeAndName & column, + const ColumnsWithTypeAndName & expected_cols); // Test group by columns - void executeGroupByAndAssert(const ColumnsWithTypeAndName & cols, const ColumnsWithTypeAndName & expected_cols); - // Prequisite: add a mock table in MockDAGRequestContext. - void executeGroupByAndAssertWithTable(const String & db_name, const String & table_name, const std::vector & group_by_cols, const ColumnsWithTypeAndName & expected_cols); + // Note that we must give columns in cols a name. + void executeGroupByAndAssert( + const ColumnsWithTypeAndName & cols, + const ColumnsWithTypeAndName & expected_cols); static void SetUpTestCase(); From 2fc46d64bfc47d299c49787ddb32a6de6dd0f3e6 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Mon, 5 Sep 2022 14:51:42 +0800 Subject: [PATCH 8/9] update. --- dbms/src/TestUtils/AggregationTestUtils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/TestUtils/AggregationTestUtils.h b/dbms/src/TestUtils/AggregationTestUtils.h index 7cbfb68514b..638b8ed2504 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.h +++ b/dbms/src/TestUtils/AggregationTestUtils.h @@ -25,7 +25,7 @@ class AggregationTest : public ExecutorTest public: static ::testing::AssertionResult checkAggReturnType(const String & agg_name, const DataTypes & data_types, const DataTypePtr & expect_type); - // Test one aggregation functions without group by. + // Test aggregation functions without group by. void executeAggFunctionAndAssert( const std::vector & func_names, const ColumnWithTypeAndName & column, From 604a5c75490bff832b19fe002cb80841afcd71a6 Mon Sep 17 00:00:00 2001 From: ywqzzy <592838129@qq.com> Date: Mon, 5 Sep 2022 17:54:03 +0800 Subject: [PATCH 9/9] fix ut. --- dbms/src/TestUtils/AggregationTestUtils.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/TestUtils/AggregationTestUtils.cpp b/dbms/src/TestUtils/AggregationTestUtils.cpp index 0c4d95b9e84..a66149e84f0 100644 --- a/dbms/src/TestUtils/AggregationTestUtils.cpp +++ b/dbms/src/TestUtils/AggregationTestUtils.cpp @@ -30,6 +30,7 @@ void AggregationTest::SetUpTestCase() }; register_func(DB::registerAggregateFunctions); + register_func(DB::registerFunctions); } ::testing::AssertionResult AggregationTest::checkAggReturnType(const String & agg_name, const DataTypes & data_types, const DataTypePtr & expect_type)