From 9634041e2a259a6a50cba2ec5cbe420c54f5ad6c Mon Sep 17 00:00:00 2001 From: hey-kong Date: Sun, 26 Jun 2022 15:20:25 +0800 Subject: [PATCH 1/4] finish time_to_sec push down --- dbms/src/Common/MyDuration.cpp | 8 +- dbms/src/Common/MyDuration.h | 4 + dbms/src/Flash/Coprocessor/DAGUtils.cpp | 2 +- dbms/src/Functions/FunctionsDuration.cpp | 69 ++++++++++++++++ dbms/src/Functions/FunctionsDuration.h | 19 +++++ .../tests/gtest_duration_pushdown.cpp | 80 +++++++++++++++++++ .../expr/duration_pushdown.test | 8 ++ 7 files changed, 188 insertions(+), 2 deletions(-) diff --git a/dbms/src/Common/MyDuration.cpp b/dbms/src/Common/MyDuration.cpp index 8801ae0de44..5c9b7a47ef0 100644 --- a/dbms/src/Common/MyDuration.cpp +++ b/dbms/src/Common/MyDuration.cpp @@ -67,4 +67,10 @@ String MyDuration::toString() const auto frac_str = fmt::format("{:06}", microsecond); return fmt::format(fmt_str, sign > 0 ? "" : "-", hour, minute, second, frac_str); } -} // namespace DB \ No newline at end of file + +UInt64 calcSeconds(int hour, int minute, int second) +{ + return hour * 3600 + minute * 60 + second; +} + +} // namespace DB diff --git a/dbms/src/Common/MyDuration.h b/dbms/src/Common/MyDuration.h index 730d31b232b..b835a2392fa 100644 --- a/dbms/src/Common/MyDuration.h +++ b/dbms/src/Common/MyDuration.h @@ -89,4 +89,8 @@ class MyDuration String toString() const; }; + +// returns seconds since '00:00:00' +UInt64 calcSeconds(int hour, int minute, int second); + } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index 9ffa29cd14d..22758ad55cb 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -561,7 +561,7 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::Quarter, "toQuarter"}, //{tipb::ScalarFuncSig::SecToTime, "cast"}, - //{tipb::ScalarFuncSig::TimeToSec, "cast"}, + {tipb::ScalarFuncSig::TimeToSec, "tidbTimeToSec"}, //{tipb::ScalarFuncSig::TimestampAdd, "cast"}, {tipb::ScalarFuncSig::ToDays, "tidbToDays"}, {tipb::ScalarFuncSig::ToSeconds, "tidbToSeconds"}, diff --git a/dbms/src/Functions/FunctionsDuration.cpp b/dbms/src/Functions/FunctionsDuration.cpp index ea7b86ac670..0d328201fb6 100644 --- a/dbms/src/Functions/FunctionsDuration.cpp +++ b/dbms/src/Functions/FunctionsDuration.cpp @@ -97,6 +97,57 @@ void FunctionDurationSplit::executeImpl(Block & block, const ColumnNumbers ErrorCodes::ILLEGAL_COLUMN); }; +template +DataTypePtr FunctionMyDurationToSec::getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const +{ + if (!arguments[0].type->isMyTime()) + { + throw Exception( + fmt::format("Illegal type {} of first argument of function {}", arguments[0].type->getName(), getName()), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + return std::make_shared(); +} + +template +void FunctionMyDurationToSec::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const +{ + const auto * from_type = checkAndGetDataType(block.getByPosition(arguments[0]).type.get()); + if (from_type == nullptr) + { + throw Exception( + fmt::format( + "Illegal column {} of first argument of function {}", + block.getByPosition(arguments[0]).column->getName(), + name), + ErrorCodes::ILLEGAL_COLUMN); + } + + using FromFieldType = typename DataTypeMyDuration::FieldType; + const auto * col_from = checkAndGetColumn>(block.getByPosition(arguments[0]).column.get()); + if (col_from != nullptr) + { + const typename ColumnVector::Container & vec_from = col_from->getData(); + const size_t size = vec_from.size(); + auto col_to = ColumnVector::create(size); + typename ColumnVector::Container & vec_to = col_to->getData(); + + for (size_t i = 0; i < vec_from.size(); ++i) + { + MyDuration val(vec_from[i], from_type->getFsp()); + vec_to[i] = Impl::apply(val); + } + block.getByPosition(result).column = std::move(col_to); + } + else + throw Exception( + fmt::format( + "Illegal column {} of first argument of function {}", + block.getByPosition(arguments[0]).column->getName(), + name), + ErrorCodes::ILLEGAL_COLUMN); +} + struct DurationSplitHourImpl { static constexpr auto name = "hour"; @@ -133,11 +184,27 @@ struct DurationSplitMicroSecondImpl } }; +struct TiDBTimeToSecTransformerImpl +{ + static constexpr auto name = "tidbTimeToSec"; + static Int64 apply(const MyDuration & val) + { + Int64 sign = 1; + if (val.isNeg()) + { + sign = -1; + } + return sign * calcSeconds(val.hours(), val.minutes(), val.seconds()); + } +}; + using FunctionDurationHour = FunctionDurationSplit; using FunctionDurationMinute = FunctionDurationSplit; using FunctionDurationSecond = FunctionDurationSplit; using FunctionDurationMicroSecond = FunctionDurationSplit; +using FunctionToTiDBTimeToSec = FunctionMyDurationToSec; + void registerFunctionsDuration(FunctionFactory & factory) { factory.registerFunction(); @@ -146,5 +213,7 @@ void registerFunctionsDuration(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + + factory.registerFunction(); } } // namespace DB diff --git a/dbms/src/Functions/FunctionsDuration.h b/dbms/src/Functions/FunctionsDuration.h index 4247cde03ff..5bc54d425f4 100644 --- a/dbms/src/Functions/FunctionsDuration.h +++ b/dbms/src/Functions/FunctionsDuration.h @@ -69,4 +69,23 @@ class FunctionDurationSplit : public IFunction void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override; }; +template +class FunctionMyDurationToSec : public IFunction +{ +public: + static constexpr auto name = Impl::name; + + static FunctionPtr create(const Context &) { return std::make_shared(); }; + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override; + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override; +}; + } // namespace DB \ No newline at end of file diff --git a/dbms/src/Functions/tests/gtest_duration_pushdown.cpp b/dbms/src/Functions/tests/gtest_duration_pushdown.cpp index 4501a4c9fae..d05cd372e3d 100644 --- a/dbms/src/Functions/tests/gtest_duration_pushdown.cpp +++ b/dbms/src/Functions/tests/gtest_duration_pushdown.cpp @@ -166,5 +166,85 @@ try ASSERT_COLUMN_EQ(microSecond_out, executeFunction("microSecond", input4)); } CATCH + +TEST_F(DurationPushDown, timeToSecPushDownTest) +try +{ + ColumnWithTypeAndName input( + createColumn>({(838 * 3600 + 59 * 60 + 59) * 1000000000L + 999999000L, + -(838 * 3600 + 59 * 60 + 59) * 1000000000L - 123456000L, + 0, + (1 * 3600 + 2 * 60 + 3) * 1000000000L + 4000L}) + .column, + makeNullable(std::make_shared(6)), + "input"); + auto second_output = createColumn>({3020399, -3020399, 0, 3723}); + ASSERT_COLUMN_EQ(second_output, executeFunction("tidbTimeToSec", input)); + + // Test Overflow + ColumnWithTypeAndName input2( + createColumn>({(838 * 3600 + 59 * 60 + 59) * 1000000000L + 999999000L + 1000L}).column, + makeNullable(std::make_shared(6)), + "result"); + try + { + auto result = executeFunction("tidbTimeToSec", input2); + FAIL() << "Expected overflow"; + } + catch (DB::Exception & e) + { + ASSERT_EQ(e.message(), std::string("nanos must >= -3020399999999000 and <= 3020399999999000")); + } + catch (...) + { + FAIL() << "Expected overflow"; + }; + + ColumnWithTypeAndName input3( + createColumn>({-(838 * 3600 + 59 * 60 + 59) * 1000000000L - 999999000L - 1000L}).column, + makeNullable(std::make_shared(6)), + "result"); + try + { + auto result = executeFunction("tidbTimeToSec", input3); + FAIL() << "Expected overflow"; + } + catch (DB::Exception & e) + { + ASSERT_EQ(e.message(), std::string("nanos must >= -3020399999999000 and <= 3020399999999000")); + } + catch (...) + { + FAIL() << "Expected overflow"; + }; + + // Random Test + constexpr int rowNum = 1000; + auto dur_column = ColumnVector::create(); + auto & dur_data = dur_column->getData(); + auto second_column = ColumnVector::create(); + auto & second_data = second_column->getData(); + dur_data.resize(rowNum); + second_data.resize(rowNum); + + std::random_device rd; + std::default_random_engine gen = std::default_random_engine(rd()); + std::uniform_int_distribution sign_dis(0, 1), hour_dis(0, 838), minute_dis(0, 59), second_dis(0, 59), microSecond_dis(0, 999999); + for (int i = 0; i < rowNum; i++) + { + auto sign = (sign_dis(gen) == 0) ? 1 : -1; + auto hour = hour_dis(gen); + auto minute = minute_dis(gen); + auto second = second_dis(gen); + auto microSecond = microSecond_dis(gen); + dur_data[i] = sign * ((hour * 3600 + minute * 60 + second) * 1000000000L + microSecond * 1000L); + second_data[i] = sign * (hour * 3600 + minute * 60 + second); + } + + ColumnWithTypeAndName input4(std::move(dur_column), std::make_shared(6), "duration"); + ColumnWithTypeAndName second_out(std::move(second_column), std::make_shared(), "time_to_sec"); + ASSERT_COLUMN_EQ(second_out, executeFunction("tidbTimeToSec", input4)); +} +CATCH } // namespace tests } // namespace DB \ No newline at end of file diff --git a/tests/fullstack-test/expr/duration_pushdown.test b/tests/fullstack-test/expr/duration_pushdown.test index 63106fa1788..442a708a802 100644 --- a/tests/fullstack-test/expr/duration_pushdown.test +++ b/tests/fullstack-test/expr/duration_pushdown.test @@ -106,6 +106,14 @@ mysql> use test; set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflas # | 123500 | # +----------------+ +mysql> use test; set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select time_to_sec(a) from t; ++----------------+ +| time_to_sec(a) | ++----------------+ +| 2520610 | +| -2520610 | ++----------------+ + mysql> drop table if exists test.time_test; mysql> create table test.time_test(id int(11),v1 time(3) not null, v2 time(3)); From eab7c7baa332b60830683c303d375b2eba3bd9c4 Mon Sep 17 00:00:00 2001 From: hey-kong Date: Tue, 28 Jun 2022 12:24:04 +0800 Subject: [PATCH 2/4] update --- dbms/src/Functions/FunctionsDuration.cpp | 8 ++++---- dbms/src/Functions/tests/gtest_duration_pushdown.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/src/Functions/FunctionsDuration.cpp b/dbms/src/Functions/FunctionsDuration.cpp index 0d328201fb6..f1a00089343 100644 --- a/dbms/src/Functions/FunctionsDuration.cpp +++ b/dbms/src/Functions/FunctionsDuration.cpp @@ -103,7 +103,7 @@ DataTypePtr FunctionMyDurationToSec::getReturnTypeImpl(const ColumnsWithTy if (!arguments[0].type->isMyTime()) { throw Exception( - fmt::format("Illegal type {} of first argument of function {}", arguments[0].type->getName(), getName()), + fmt::format("Illegal type {} of the first argument of function {}", arguments[0].type->getName(), getName()), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } return std::make_shared(); @@ -117,7 +117,7 @@ void FunctionMyDurationToSec::executeImpl(Block & block, const ColumnNumbe { throw Exception( fmt::format( - "Illegal column {} of first argument of function {}", + "Illegal column {} of the first argument of function {}", block.getByPosition(arguments[0]).column->getName(), name), ErrorCodes::ILLEGAL_COLUMN); @@ -132,7 +132,7 @@ void FunctionMyDurationToSec::executeImpl(Block & block, const ColumnNumbe auto col_to = ColumnVector::create(size); typename ColumnVector::Container & vec_to = col_to->getData(); - for (size_t i = 0; i < vec_from.size(); ++i) + for (size_t i = 0; i < size; ++i) { MyDuration val(vec_from[i], from_type->getFsp()); vec_to[i] = Impl::apply(val); @@ -142,7 +142,7 @@ void FunctionMyDurationToSec::executeImpl(Block & block, const ColumnNumbe else throw Exception( fmt::format( - "Illegal column {} of first argument of function {}", + "Illegal column {} of the first argument of function {}", block.getByPosition(arguments[0]).column->getName(), name), ErrorCodes::ILLEGAL_COLUMN); diff --git a/dbms/src/Functions/tests/gtest_duration_pushdown.cpp b/dbms/src/Functions/tests/gtest_duration_pushdown.cpp index d05cd372e3d..106f3d84642 100644 --- a/dbms/src/Functions/tests/gtest_duration_pushdown.cpp +++ b/dbms/src/Functions/tests/gtest_duration_pushdown.cpp @@ -230,7 +230,7 @@ try std::random_device rd; std::default_random_engine gen = std::default_random_engine(rd()); std::uniform_int_distribution sign_dis(0, 1), hour_dis(0, 838), minute_dis(0, 59), second_dis(0, 59), microSecond_dis(0, 999999); - for (int i = 0; i < rowNum; i++) + for (int i = 0; i < rowNum; ++i) { auto sign = (sign_dis(gen) == 0) ? 1 : -1; auto hour = hour_dis(gen); From d49f3c459c79c0dd077a0bac8f1d2ec39711044a Mon Sep 17 00:00:00 2001 From: hey-kong Date: Tue, 28 Jun 2022 18:06:01 +0800 Subject: [PATCH 3/4] update --- dbms/src/Common/MyDuration.cpp | 2 +- dbms/src/Common/MyDuration.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Common/MyDuration.cpp b/dbms/src/Common/MyDuration.cpp index 5c9b7a47ef0..828be5551ce 100644 --- a/dbms/src/Common/MyDuration.cpp +++ b/dbms/src/Common/MyDuration.cpp @@ -68,7 +68,7 @@ String MyDuration::toString() const return fmt::format(fmt_str, sign > 0 ? "" : "-", hour, minute, second, frac_str); } -UInt64 calcSeconds(int hour, int minute, int second) +UInt64 calcSeconds(Int32 hour, Int32 minute, Int32 second) { return hour * 3600 + minute * 60 + second; } diff --git a/dbms/src/Common/MyDuration.h b/dbms/src/Common/MyDuration.h index b835a2392fa..fbed4e24656 100644 --- a/dbms/src/Common/MyDuration.h +++ b/dbms/src/Common/MyDuration.h @@ -91,6 +91,6 @@ class MyDuration }; // returns seconds since '00:00:00' -UInt64 calcSeconds(int hour, int minute, int second); +UInt64 calcSeconds(Int32 hour, Int32 minute, Int32 second); } // namespace DB From b6f70a740b43df5cacb4f73b062fbc770c50831d Mon Sep 17 00:00:00 2001 From: hey-kong Date: Tue, 28 Jun 2022 23:41:19 +0800 Subject: [PATCH 4/4] update --- dbms/src/Common/MyDuration.cpp | 6 ------ dbms/src/Common/MyDuration.h | 4 ---- dbms/src/Functions/FunctionsDuration.cpp | 2 +- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/dbms/src/Common/MyDuration.cpp b/dbms/src/Common/MyDuration.cpp index 828be5551ce..513c40b6dbc 100644 --- a/dbms/src/Common/MyDuration.cpp +++ b/dbms/src/Common/MyDuration.cpp @@ -67,10 +67,4 @@ String MyDuration::toString() const auto frac_str = fmt::format("{:06}", microsecond); return fmt::format(fmt_str, sign > 0 ? "" : "-", hour, minute, second, frac_str); } - -UInt64 calcSeconds(Int32 hour, Int32 minute, Int32 second) -{ - return hour * 3600 + minute * 60 + second; -} - } // namespace DB diff --git a/dbms/src/Common/MyDuration.h b/dbms/src/Common/MyDuration.h index fbed4e24656..730d31b232b 100644 --- a/dbms/src/Common/MyDuration.h +++ b/dbms/src/Common/MyDuration.h @@ -89,8 +89,4 @@ class MyDuration String toString() const; }; - -// returns seconds since '00:00:00' -UInt64 calcSeconds(Int32 hour, Int32 minute, Int32 second); - } // namespace DB diff --git a/dbms/src/Functions/FunctionsDuration.cpp b/dbms/src/Functions/FunctionsDuration.cpp index f1a00089343..9ccafd2794d 100644 --- a/dbms/src/Functions/FunctionsDuration.cpp +++ b/dbms/src/Functions/FunctionsDuration.cpp @@ -194,7 +194,7 @@ struct TiDBTimeToSecTransformerImpl { sign = -1; } - return sign * calcSeconds(val.hours(), val.minutes(), val.seconds()); + return sign * (val.hours() * 3600 + val.minutes() * 60 + val.seconds()); } };