diff --git a/CMakeLists.txt b/CMakeLists.txt index f1dd740b0b4..f2ec9f3316b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,6 +48,14 @@ endif () set(CMAKE_EXPORT_COMPILE_COMMANDS 1) option (USE_CCACHE "Set to OFF to disable ccache" ON) +if (USE_CCACHE) + set(NOT_USE_CCACHE 0) +else() + set(NOT_USE_CCACHE 1) +endif() + +option(ENABLE_PCH "Enable `Precompiled header`" OFF) + include (cmake/find_ccache.cmake) if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") @@ -424,9 +432,6 @@ else (ENABLE_FAILPOINTS) message (STATUS "Failpoints are disabled") endif (ENABLE_FAILPOINTS) -# Enable PageStorage V3 test. -option (ENABLE_V3_PAGESTORAGE "Enables V3 PageStorage" ON) - # Flags for test coverage option (TEST_COVERAGE "Enables flags for test coverage" OFF) option (TEST_COVERAGE_XML "Output XML report for test coverage" OFF) diff --git a/README.md b/README.md index 8a2217b9a42..02af727105b 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ TiFlash repository is based on [ClickHouse](https://github.com/ClickHouse/ClickH ### Start with TiDB Cloud -Quickly explore TiFlash with [a free trial of TiDB Cloud](https://tidbcloud.com/signup). +Quickly explore TiFlash with [a free trial of TiDB Cloud](https://tidbcloud.com/free-trial). See [TiDB Cloud Quick Start Guide](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart). diff --git a/cmake/find_ccache.cmake b/cmake/find_ccache.cmake index 910caebb046..71fa337a922 100644 --- a/cmake/find_ccache.cmake +++ b/cmake/find_ccache.cmake @@ -24,6 +24,20 @@ if (USE_CCACHE AND CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "cca message ("${CCACHE_CONFIG}") set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) + + if (ENABLE_PCH) + execute_process (COMMAND ${CCACHE_FOUND} --get-config sloppiness OUTPUT_VARIABLE _CCACHE_SLOPPINESS OUTPUT_STRIP_TRAILING_WHITESPACE) + string (FIND "${_CCACHE_SLOPPINESS}" "pch_defines" _CCACHE_SLOPPINESS_RES) + if (NOT _CCACHE_SLOPPINESS_RES STREQUAL "-1") + string (FIND "${_CCACHE_SLOPPINESS}" "time_macros" _CCACHE_SLOPPINESS_RES) + endif () + + if (_CCACHE_SLOPPINESS_RES STREQUAL "-1") + message(WARNING "`Precompiled header` won't be cached by ccache, sloppiness = `${CCACHE_SLOPPINESS}`,please execute `ccache -o sloppiness=pch_defines,time_macros`") + set (ENABLE_PCH FALSE CACHE BOOL "" FORCE) + endif () + endif () + else () message (STATUS "Not using ccache ${CCACHE_FOUND}, USE_CCACHE=${USE_CCACHE}") endif () diff --git a/contrib/tiflash-proxy b/contrib/tiflash-proxy index 7578b816399..ca2f51f94e5 160000 --- a/contrib/tiflash-proxy +++ b/contrib/tiflash-proxy @@ -1 +1 @@ -Subproject commit 7578b8163992ce933074135f8687ad447d88ea9b +Subproject commit ca2f51f94e55bdd23749dcc02ab4afb94eeb5ae5 diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 21cf06cbe31..e1e52fab73b 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -91,12 +91,10 @@ add_headers_and_sources(dbms src/Storages/Page/V2/VersionSet) add_headers_and_sources(dbms src/Storages/Page/V2/gc) add_headers_and_sources(dbms src/WindowFunctions) add_headers_and_sources(dbms src/TiDB/Schema) -if (ENABLE_V3_PAGESTORAGE) - add_headers_and_sources(dbms src/Storages/Page/V3) - add_headers_and_sources(dbms src/Storages/Page/V3/LogFile) - add_headers_and_sources(dbms src/Storages/Page/V3/WAL) - add_headers_and_sources(dbms src/Storages/Page/V3/spacemap) -endif() +add_headers_and_sources(dbms src/Storages/Page/V3) +add_headers_and_sources(dbms src/Storages/Page/V3/LogFile) +add_headers_and_sources(dbms src/Storages/Page/V3/WAL) +add_headers_and_sources(dbms src/Storages/Page/V3/spacemap) add_headers_and_sources(dbms src/Storages/Page/) add_headers_and_sources(dbms src/TiDB) add_headers_and_sources(dbms src/Client) @@ -259,6 +257,16 @@ target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE # https://cmake.org/pipermail/cmake/2016-May/063400.html target_link_libraries (clickhouse_common_io PUBLIC ${TIFLASH_XXHASH_LIBRARY}) +function(add_target_pch context target) + if (ENABLE_PCH) + message(STATUS "Add PCH `${context}` for target `${target}`") + target_precompile_headers(${target} PRIVATE ${context}) + endif () + if(${ARGC} GREATER 2) + add_target_pch(${context} ${ARGN}) + endif() +endfunction() + if (ENABLE_TESTS) include (${TiFlash_SOURCE_DIR}/cmake/find_gtest.cmake) @@ -297,6 +305,8 @@ if (ENABLE_TESTS) target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy) add_check(gtests_dbms) + add_target_pch("pch-dbms.h" gtests_dbms) + grep_bench_sources(${TiFlash_SOURCE_DIR}/dbms dbms_bench_sources) add_executable(bench_dbms EXCLUDE_FROM_ALL ${dbms_bench_sources} @@ -311,6 +321,9 @@ if (ENABLE_TESTS) if (ENABLE_TIFLASH_DTWORKLOAD) target_link_libraries(bench_dbms dt-workload-lib) endif () + if (ENABLE_TIFLASH_PAGEWORKLOAD) + target_link_libraries(bench_dbms page-workload-lib) + endif () add_check(bench_dbms) endif () @@ -342,3 +355,23 @@ if (TEST_COVERAGE AND CMAKE_BUILD_TYPE STREQUAL "Debug") ) endif () endif () + +# dbms +add_target_pch("pch-dbms.h" dbms flash_service) +add_target_pch("pch-common.h" clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions) +add_target_pch("pch-common.h" clickhouse_parsers clickhouse_storages_system dt-workload-lib clickhouse-server-lib) + +# common +add_target_pch("pch-kvpb.h" kv_client) + +add_target_pch("pch-stl.h" ${Boost_SYSTEM_LIBRARY} cctz ${RE2_LIBRARY} ${RE2_ST_LIBRARY}) + +# grpc +add_target_pch("$<$:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" grpc grpc++) + +# pb +add_target_pch("pch-stl.h" libprotobuf kvproto tipb libprotoc) + +# poco +add_target_pch("pch-stl.h" Net Crypto Util Data NetSSL) +add_target_pch("$<$:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" XML Foundation JSON) diff --git a/dbms/src/TableFunctions/TableFunctionFile.h b/dbms/pch-common.h similarity index 52% rename from dbms/src/TableFunctions/TableFunctionFile.h rename to dbms/pch-common.h index dda367c2679..878254a3529 100644 --- a/dbms/src/TableFunctions/TableFunctionFile.h +++ b/dbms/pch-common.h @@ -14,26 +14,10 @@ #pragma once -#include +#include +#include +#include +#include +#include - -namespace DB -{ -/* file(path, format, structure) - creates a temporary storage from file - * - * - * The file must be in the clickhouse data directory. - * The relative path begins with the clickhouse data directory. - */ -class TableFunctionFile : public ITableFunction -{ -public: - static constexpr auto name = "file"; - std::string getName() const override { return name; } - -private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context) const override; -}; - - -} // namespace DB +#include "pch-stl.h" diff --git a/dbms/pch-dbms.h b/dbms/pch-dbms.h new file mode 100644 index 00000000000..60687073bf8 --- /dev/null +++ b/dbms/pch-dbms.h @@ -0,0 +1,18 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "pch-common.h" +#include "pch-kvpb.h" diff --git a/dbms/pch-kvpb.h b/dbms/pch-kvpb.h new file mode 100644 index 00000000000..d74bfc6bb89 --- /dev/null +++ b/dbms/pch-kvpb.h @@ -0,0 +1,26 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +/** +There may be unexpected behaviors for `ccache` to deal with PCH which includes those header files generated by tools: + 'xxx' has been modified since the precompiled header 'xxx' was built: mtime changed + `Precompiled header includes xxx.h, which has a new mtime` +*/ +#include +#include +#include + +#include "pch-stl.h" diff --git a/dbms/pch-stl.h b/dbms/pch-stl.h new file mode 100644 index 00000000000..01b3123650d --- /dev/null +++ b/dbms/pch-stl.h @@ -0,0 +1,34 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include \ No newline at end of file diff --git a/dbms/src/Common/FailPoint.cpp b/dbms/src/Common/FailPoint.cpp index 921dd5bf748..c6c3caa44ad 100644 --- a/dbms/src/Common/FailPoint.cpp +++ b/dbms/src/Common/FailPoint.cpp @@ -65,7 +65,8 @@ std::unordered_map> FailPointHelper::f M(exception_when_read_from_log) \ M(exception_mpp_hash_build) \ M(exception_before_drop_segment) \ - M(exception_after_drop_segment) + M(exception_after_drop_segment) \ + M(exception_between_schema_change_in_the_same_diff) #define APPLY_FOR_FAILPOINTS(M) \ M(skip_check_segment_update) \ diff --git a/dbms/src/Common/FmtUtils.h b/dbms/src/Common/FmtUtils.h index 71f93a34078..ab37194ed10 100644 --- a/dbms/src/Common/FmtUtils.h +++ b/dbms/src/Common/FmtUtils.h @@ -14,8 +14,6 @@ #pragma once -#include -#include #include namespace DB @@ -32,9 +30,9 @@ class FmtBuffer return *this; } - FmtBuffer & append(StringRef s) + FmtBuffer & append(std::string_view s) { - buffer.append(s.data, s.data + s.size); + buffer.append(s.data(), s.data() + s.size()); return *this; } @@ -55,7 +53,7 @@ class FmtBuffer FmtBuffer & joinStr( Iter first, Iter end, - StringRef delimiter) + std::string_view delimiter) { auto func = [](const auto & s, FmtBuffer & fb) { fb.append(s); @@ -68,7 +66,7 @@ class FmtBuffer Iter first, Iter end, FF && toStringFunc, // void (const auto &, FmtBuffer &) - StringRef delimiter) + std::string_view delimiter) { if (first == end) return *this; diff --git a/dbms/src/Common/TiFlashException.cpp b/dbms/src/Common/TiFlashException.cpp index bd28a57c093..11ee229ded0 100644 --- a/dbms/src/Common/TiFlashException.cpp +++ b/dbms/src/Common/TiFlashException.cpp @@ -18,6 +18,32 @@ namespace DB { +struct TiFlashErrorRegistry::Errors : std::map, TiFlashError> +{ +}; + +TiFlashErrorRegistry::Errors & TiFlashErrorRegistry::errors() +{ + return *inner_data; +} + +TiFlashErrorRegistry::Errors & TiFlashErrorRegistry::errors() const +{ + return *inner_data; +} + +TiFlashErrorRegistry::TiFlashErrorRegistry() + : inner_data(new Errors{}) +{ + initialize(); +} + +TiFlashErrorRegistry::~TiFlashErrorRegistry() +{ + delete inner_data; + inner_data = nullptr; +} + void TiFlashErrorRegistry::initialize() { // Used to check uniqueness of classes @@ -46,9 +72,9 @@ void TiFlashErrorRegistry::initialize() void TiFlashErrorRegistry::registerError(const std::string & error_class, const std::string & error_code, const std::string & description, const std::string & workaround, const std::string & message_template) { TiFlashError error{error_class, error_code, description, workaround, message_template}; - if (all_errors.find({error_class, error_code}) == all_errors.end()) + if (errors().find({error_class, error_code}) == errors().end()) { - all_errors.emplace(std::make_pair(error_class, error_code), std::move(error)); + errors().emplace(std::make_pair(error_class, error_code), std::move(error)); } else { @@ -77,4 +103,51 @@ std::string TiFlashException::standardText() const return text; } +std::optional TiFlashErrorRegistry::get(const std::string & error_class, const std::string & error_code) const +{ + auto error = errors().find({error_class, error_code}); + if (error != errors().end()) + { + return error->second; + } + else + { + return {}; + } +} +std::optional TiFlashErrorRegistry::get(const std::string & error_class, int error_code) const +{ + return get(error_class, std::to_string(error_code)); +} + +std::vector TiFlashErrorRegistry::allErrors() const +{ + std::vector res; + res.reserve(errors().size()); + for (const auto & error : errors()) + { + res.push_back(error.second); + } + return res; +} + +TiFlashError TiFlashErrorRegistry::simpleGet(const std::string & error_class, const std::string & error_code) +{ + auto & m_instance = instance(); + auto error = m_instance.get(error_class, error_code); + if (error.has_value()) + { + return error.value(); + } + else + { + throw Exception("Unregistered TiFlashError: FLASH:" + error_class + ":" + error_code); + } +} +TiFlashError TiFlashErrorRegistry::simpleGet(const std::string & error_class, int error_code) +{ + return simpleGet(error_class, std::to_string(error_code)); +} + + } // namespace DB diff --git a/dbms/src/Common/TiFlashException.h b/dbms/src/Common/TiFlashException.h index 2026571859e..3b4e3d75813 100644 --- a/dbms/src/Common/TiFlashException.h +++ b/dbms/src/Common/TiFlashException.h @@ -194,56 +194,19 @@ class TiFlashErrorRegistry : public ext::Singleton public: friend ext::Singleton; - static TiFlashError simpleGet(const std::string & error_class, const std::string & error_code) - { - auto & m_instance = instance(); - auto error = m_instance.get(error_class, error_code); - if (error.has_value()) - { - return error.value(); - } - else - { - throw Exception("Unregistered TiFlashError: FLASH:" + error_class + ":" + error_code); - } - } + static TiFlashError simpleGet(const std::string & error_class, const std::string & error_code); - static TiFlashError simpleGet(const std::string & error_class, int error_code) - { - return simpleGet(error_class, std::to_string(error_code)); - } + static TiFlashError simpleGet(const std::string & error_class, int error_code); - std::optional get(const std::string & error_class, const std::string & error_code) const - { - auto error = all_errors.find({error_class, error_code}); - if (error != all_errors.end()) - { - return error->second; - } - else - { - return {}; - } - } + std::optional get(const std::string & error_class, const std::string & error_code) const; - std::optional get(const std::string & error_class, int error_code) const - { - return get(error_class, std::to_string(error_code)); - } + std::optional get(const std::string & error_class, int error_code) const; - std::vector allErrors() const - { - std::vector res; - res.reserve(all_errors.size()); - for (const auto & error : all_errors) - { - res.push_back(error.second); - } - return res; - } + std::vector allErrors() const; protected: - TiFlashErrorRegistry() { initialize(); } + TiFlashErrorRegistry(); + ~TiFlashErrorRegistry(); private: void registerError(const std::string & error_class, const std::string & error_code, const std::string & description, const std::string & workaround, const std::string & message_template = ""); @@ -252,8 +215,13 @@ class TiFlashErrorRegistry : public ext::Singleton void initialize(); + struct Errors; + + Errors & errors(); + Errors & errors() const; + private: - std::map, TiFlashError> all_errors; + Errors * inner_data; // PImpl }; /// TiFlashException implements TiDB's standardized error. diff --git a/dbms/src/Common/getNumberOfPhysicalCPUCores.h b/dbms/src/Common/getNumberOfPhysicalCPUCores.h index 6f7eaef4bb4..b3ab65a66e5 100644 --- a/dbms/src/Common/getNumberOfPhysicalCPUCores.h +++ b/dbms/src/Common/getNumberOfPhysicalCPUCores.h @@ -15,4 +15,5 @@ #pragma once /// Get number of CPU cores without hyper-threading. +/// Note: do not support environment under resource isolation mechanism like Docker, CGroup. unsigned getNumberOfPhysicalCPUCores(); diff --git a/dbms/src/Core/Defines.h b/dbms/src/Core/Defines.h index 33d116dae33..75f6f16bb25 100644 --- a/dbms/src/Core/Defines.h +++ b/dbms/src/Core/Defines.h @@ -78,7 +78,6 @@ /// too short a period can cause errors to disappear immediately after creation. #define DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_DECREASE_ERROR_PERIOD (2 * DBMS_DEFAULT_SEND_TIMEOUT_SEC) #define DEFAULT_QUERIES_QUEUE_WAIT_TIME_MS 5000 /// Maximum waiting time in the request queue. -#define DBMS_DEFAULT_BACKGROUND_POOL_SIZE 16 #define DBMS_MIN_REVISION_WITH_CLIENT_INFO 54032 #define DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE 54058 diff --git a/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp b/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp index 47e7a56aa35..22545327edd 100644 --- a/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp +++ b/dbms/src/DataStreams/CreatingSetsBlockInputStream.cpp @@ -228,7 +228,7 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) if (done_with_set && done_with_join && done_with_table) { - if (IProfilingBlockInputStream * profiling_in = dynamic_cast(&*subquery.source)) + if (auto * profiling_in = dynamic_cast(&*subquery.source)) profiling_in->cancel(false); break; @@ -248,7 +248,7 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) watch.stop(); size_t head_rows = 0; - if (IProfilingBlockInputStream * profiling_in = dynamic_cast(&*subquery.source)) + if (auto * profiling_in = dynamic_cast(&*subquery.source)) { const BlockStreamProfileInfo & profile_info = profiling_in->getProfileInfo(); @@ -272,7 +272,7 @@ void CreatingSetsBlockInputStream::createOne(SubqueryForSet & subquery) if (subquery.table) msg.fmtAppend("Table with {} rows. ", head_rows); - msg.fmtAppend("In {.3f} sec. ", watch.elapsedSeconds()); + msg.fmtAppend("In {:.3f} sec. ", watch.elapsedSeconds()); msg.fmtAppend("using {} threads.", subquery.join ? subquery.join->getBuildConcurrency() : 1); return msg.toString(); }; diff --git a/dbms/src/DataStreams/MockExchangeReceiverInputStream.cpp b/dbms/src/DataStreams/MockExchangeReceiverInputStream.cpp index b1de3e23914..3f46bb46cc8 100644 --- a/dbms/src/DataStreams/MockExchangeReceiverInputStream.cpp +++ b/dbms/src/DataStreams/MockExchangeReceiverInputStream.cpp @@ -30,6 +30,22 @@ MockExchangeReceiverInputStream::MockExchangeReceiverInputStream(const tipb::Exc } } +MockExchangeReceiverInputStream::MockExchangeReceiverInputStream(ColumnsWithTypeAndName columns, size_t max_block_size) + : columns(columns) + , output_index(0) + , max_block_size(max_block_size) +{ + rows = 0; + for (const auto & elem : columns) + { + if (elem.column) + { + assert(rows == 0 || rows == elem.column->size()); + rows = elem.column->size(); + } + } +} + ColumnPtr MockExchangeReceiverInputStream::makeColumn(ColumnWithTypeAndName elem) const { auto column = elem.type->createColumn(); diff --git a/dbms/src/DataStreams/MockExchangeReceiverInputStream.h b/dbms/src/DataStreams/MockExchangeReceiverInputStream.h index 24ae80d4f62..8c0a5b85822 100644 --- a/dbms/src/DataStreams/MockExchangeReceiverInputStream.h +++ b/dbms/src/DataStreams/MockExchangeReceiverInputStream.h @@ -26,7 +26,11 @@ class MockExchangeReceiverInputStream : public IProfilingBlockInputStream { public: MockExchangeReceiverInputStream(const tipb::ExchangeReceiver & receiver, size_t max_block_size, size_t rows_); - Block getHeader() const override { return Block(columns); } + MockExchangeReceiverInputStream(ColumnsWithTypeAndName columns, size_t max_block_size); + Block getHeader() const override + { + return Block(columns); + } String getName() const override { return "MockExchangeReceiver"; } ColumnsWithTypeAndName columns; size_t output_index; diff --git a/dbms/src/DataStreams/MultiplexInputStream.h b/dbms/src/DataStreams/MultiplexInputStream.h new file mode 100644 index 00000000000..4fa33262e66 --- /dev/null +++ b/dbms/src/DataStreams/MultiplexInputStream.h @@ -0,0 +1,246 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int LOGICAL_ERROR; +} // namespace ErrorCodes + +class MultiPartitionStreamPool +{ +public: + MultiPartitionStreamPool() = default; + + void addPartitionStreams(const BlockInputStreams & cur_streams) + { + if (cur_streams.empty()) + return; + std::unique_lock lk(mu); + streams_queue_by_partition.push_back( + std::make_shared>>()); + for (const auto & stream : cur_streams) + streams_queue_by_partition.back()->push(stream); + added_streams.insert(added_streams.end(), cur_streams.begin(), cur_streams.end()); + } + + std::shared_ptr pickOne() + { + std::unique_lock lk(mu); + if (streams_queue_by_partition.empty()) + return nullptr; + if (streams_queue_id >= static_cast(streams_queue_by_partition.size())) + streams_queue_id = 0; + + auto & q = *streams_queue_by_partition[streams_queue_id]; + std::shared_ptr ret = nullptr; + assert(!q.empty()); + ret = q.front(); + q.pop(); + if (q.empty()) + streams_queue_id = removeQueue(streams_queue_id); + else + streams_queue_id = nextQueueId(streams_queue_id); + return ret; + } + + int exportAddedStreams(BlockInputStreams & ret_streams) + { + std::unique_lock lk(mu); + for (auto & stream : added_streams) + ret_streams.push_back(stream); + return added_streams.size(); + } + + int addedStreamsCnt() + { + std::unique_lock lk(mu); + return added_streams.size(); + } + +private: + int removeQueue(int queue_id) + { + streams_queue_by_partition[queue_id] = nullptr; + if (queue_id != static_cast(streams_queue_by_partition.size()) - 1) + { + swap(streams_queue_by_partition[queue_id], streams_queue_by_partition.back()); + streams_queue_by_partition.pop_back(); + return queue_id; + } + else + { + streams_queue_by_partition.pop_back(); + return 0; + } + } + + int nextQueueId(int queue_id) const + { + if (queue_id + 1 < static_cast(streams_queue_by_partition.size())) + return queue_id + 1; + else + return 0; + } + + static void swap(std::shared_ptr>> & a, + std::shared_ptr>> & b) + { + a.swap(b); + } + + std::vector< + std::shared_ptr>>> + streams_queue_by_partition; + std::vector> added_streams; + int streams_queue_id = 0; + std::mutex mu; +}; + +class MultiplexInputStream final : public IProfilingBlockInputStream +{ +private: + static constexpr auto NAME = "Multiplex"; + +public: + MultiplexInputStream( + std::shared_ptr & shared_pool, + const String & req_id) + : log(Logger::get(NAME, req_id)) + , shared_pool(shared_pool) + { + shared_pool->exportAddedStreams(children); + size_t num_children = children.size(); + if (num_children > 1) + { + Block header = children.at(0)->getHeader(); + for (size_t i = 1; i < num_children; ++i) + assertBlocksHaveEqualStructure( + children[i]->getHeader(), + header, + "MULTIPLEX"); + } + } + + String getName() const override { return NAME; } + + ~MultiplexInputStream() override + { + try + { + if (!all_read) + cancel(false); + } + catch (...) + { + tryLogCurrentException(log, __PRETTY_FUNCTION__); + } + } + + /** Different from the default implementation by trying to stop all sources, + * skipping failed by execution. + */ + void cancel(bool kill) override + { + if (kill) + is_killed = true; + + bool old_val = false; + if (!is_cancelled.compare_exchange_strong( + old_val, + true, + std::memory_order_seq_cst, + std::memory_order_relaxed)) + return; + + if (cur_stream) + { + if (IProfilingBlockInputStream * child = dynamic_cast(&*cur_stream)) + { + child->cancel(kill); + } + } + } + + Block getHeader() const override { return children.at(0)->getHeader(); } + +protected: + /// Do nothing, to make the preparation when underlying InputStream is picked from the pool + void readPrefix() override + { + } + + /** The following options are possible: + * 1. `readImpl` function is called until it returns an empty block. + * Then `readSuffix` function is called and then destructor. + * 2. `readImpl` function is called. At some point, `cancel` function is called perhaps from another thread. + * Then `readSuffix` function is called and then destructor. + * 3. At any time, the object can be destroyed (destructor called). + */ + + Block readImpl() override + { + if (all_read) + return {}; + + Block ret; + while (!cur_stream || !(ret = cur_stream->read())) + { + if (cur_stream) + cur_stream->readSuffix(); // release old inputstream + cur_stream = shared_pool->pickOne(); + if (!cur_stream) + { // shared_pool is empty + all_read = true; + return {}; + } + cur_stream->readPrefix(); + } + return ret; + } + + /// Called either after everything is read, or after cancel. + void readSuffix() override + { + if (!all_read && !is_cancelled) + throw Exception("readSuffix called before all data is read", ErrorCodes::LOGICAL_ERROR); + + if (cur_stream) + { + cur_stream->readSuffix(); + cur_stream = nullptr; + } + } + +private: + LoggerPtr log; + + std::shared_ptr shared_pool; + std::shared_ptr cur_stream; + + bool all_read = false; +}; + +} // namespace DB diff --git a/dbms/src/DataStreams/TiRemoteBlockInputStream.h b/dbms/src/DataStreams/TiRemoteBlockInputStream.h index f8e313a25be..f249bf1a0dc 100644 --- a/dbms/src/DataStreams/TiRemoteBlockInputStream.h +++ b/dbms/src/DataStreams/TiRemoteBlockInputStream.h @@ -22,7 +22,6 @@ #include #include #include -#include #include #include diff --git a/dbms/src/DataStreams/WindowBlockInputStream.cpp b/dbms/src/DataStreams/WindowBlockInputStream.cpp index bc63db52873..2cc61df8104 100644 --- a/dbms/src/DataStreams/WindowBlockInputStream.cpp +++ b/dbms/src/DataStreams/WindowBlockInputStream.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include namespace DB @@ -574,4 +575,21 @@ void WindowBlockInputStream::tryCalculate() peer_group_number = 1; } } + +void WindowBlockInputStream::appendInfo(FmtBuffer & buffer) const +{ + buffer.append(", function: {"); + buffer.joinStr( + window_description.window_functions_descriptions.begin(), + window_description.window_functions_descriptions.end(), + [&](const auto & func, FmtBuffer & b) { + b.append(func.window_function->getName()); + }, + ", "); + buffer.fmtAppend( + "}}, frame: {{type: {}, boundary_begin: {}, boundary_end: {}}}", + frameTypeToString(window_description.frame.type), + boundaryTypeToString(window_description.frame.begin_type), + boundaryTypeToString(window_description.frame.end_type)); +} } // namespace DB diff --git a/dbms/src/DataStreams/WindowBlockInputStream.h b/dbms/src/DataStreams/WindowBlockInputStream.h index 46b18dec1ee..0ef23aa9f6f 100644 --- a/dbms/src/DataStreams/WindowBlockInputStream.h +++ b/dbms/src/DataStreams/WindowBlockInputStream.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -169,6 +170,7 @@ class WindowBlockInputStream : public IProfilingBlockInputStream protected: Block readImpl() override; + void appendInfo(FmtBuffer & buffer) const override; LoggerPtr log; diff --git a/dbms/src/Debug/astToExecutor.cpp b/dbms/src/Debug/astToExecutor.cpp index 82f894905e6..fec76d7a085 100644 --- a/dbms/src/Debug/astToExecutor.cpp +++ b/dbms/src/Debug/astToExecutor.cpp @@ -24,13 +24,13 @@ #include #include #include -#include #include #include #include namespace DB { +using ASTPartitionByElement = ASTOrderByElement; void literalFieldToTiPBExpr(const ColumnInfo & ci, const Field & val_field, tipb::Expr * expr, Int32 collator_id) { *(expr->mutable_field_type()) = columnInfoToFieldType(ci); @@ -190,6 +190,12 @@ std::unordered_map agg_func_name_to_sig({ {"group_concat", tipb::ExprType::GroupConcat}, }); +std::unordered_map window_func_name_to_sig({ + {"RowNumber", tipb::ExprType::RowNumber}, + {"Rank", tipb::ExprType::Rank}, + {"DenseRank", tipb::ExprType::DenseRank}, +}); + DAGColumnInfo toNullableDAGColumnInfo(const DAGColumnInfo & input) { DAGColumnInfo output = input; @@ -1343,6 +1349,105 @@ void Join::toMPPSubPlan(size_t & executor_index, const DAGProperties & propertie exchange_map[left_exchange_receiver->name] = std::make_pair(left_exchange_receiver, left_exchange_sender); exchange_map[right_exchange_receiver->name] = std::make_pair(right_exchange_receiver, right_exchange_sender); } + +bool Window::toTiPBExecutor(tipb::Executor * tipb_executor, uint32_t collator_id, const MPPInfo & mpp_info, const Context & context) +{ + tipb_executor->set_tp(tipb::ExecType::TypeWindow); + tipb_executor->set_executor_id(name); + tipb::Window * window = tipb_executor->mutable_window(); + auto & input_schema = children[0]->output_schema; + for (const auto & expr : func_descs) + { + tipb::Expr * window_expr = window->add_func_desc(); + const auto * window_func = typeid_cast(expr.get()); + for (const auto & arg : window_func->arguments->children) + { + tipb::Expr * func = window_expr->add_children(); + astToPB(input_schema, arg, func, collator_id, context); + } + auto window_sig_it = window_func_name_to_sig.find(window_func->name); + if (window_sig_it == window_func_name_to_sig.end()) + throw Exception(fmt::format("Unsupported window function {}", window_func->name), ErrorCodes::LOGICAL_ERROR); + auto window_sig = window_sig_it->second; + window_expr->set_tp(window_sig); + auto * ft = window_expr->mutable_field_type(); + // TODO: Maybe more window functions with different field type. + ft->set_tp(TiDB::TypeLongLong); + ft->set_flag(TiDB::ColumnFlagBinary); + ft->set_collate(collator_id); + ft->set_flen(21); + ft->set_decimal(-1); + } + + for (const auto & child : order_by_exprs) + { + auto * elem = typeid_cast(child.get()); + if (!elem) + throw Exception("Invalid order by element", ErrorCodes::LOGICAL_ERROR); + tipb::ByItem * by = window->add_order_by(); + by->set_desc(elem->direction < 0); + tipb::Expr * expr = by->mutable_expr(); + astToPB(children[0]->output_schema, elem->children[0], expr, collator_id, context); + } + + for (const auto & child : partition_by_exprs) + { + auto * elem = typeid_cast(child.get()); + if (!elem) + throw Exception("Invalid partition by element", ErrorCodes::LOGICAL_ERROR); + tipb::ByItem * by = window->add_partition_by(); + by->set_desc(elem->direction < 0); + tipb::Expr * expr = by->mutable_expr(); + astToPB(children[0]->output_schema, elem->children[0], expr, collator_id, context); + } + + if (frame.type.has_value()) + { + tipb::WindowFrame * mut_frame = window->mutable_frame(); + mut_frame->set_type(frame.type.value()); + if (frame.start.has_value()) + { + auto * start = mut_frame->mutable_start(); + start->set_offset(std::get<2>(frame.start.value())); + start->set_unbounded(std::get<1>(frame.start.value())); + start->set_type(std::get<0>(frame.start.value())); + } + + if (frame.end.has_value()) + { + auto * end = mut_frame->mutable_end(); + end->set_offset(std::get<2>(frame.end.value())); + end->set_unbounded(std::get<1>(frame.end.value())); + end->set_type(std::get<0>(frame.end.value())); + } + } + + auto * children_executor = window->mutable_child(); + return children[0]->toTiPBExecutor(children_executor, collator_id, mpp_info, context); +} + +bool Sort::toTiPBExecutor(tipb::Executor * tipb_executor, uint32_t collator_id, const MPPInfo & mpp_info, const Context & context) +{ + tipb_executor->set_tp(tipb::ExecType::TypeSort); + tipb_executor->set_executor_id(name); + tipb::Sort * sort = tipb_executor->mutable_sort(); + sort->set_ispartialsort(is_partial_sort); + + for (const auto & child : by_exprs) + { + auto * elem = typeid_cast(child.get()); + if (!elem) + throw Exception("Invalid order by element", ErrorCodes::LOGICAL_ERROR); + tipb::ByItem * by = sort->add_byitems(); + by->set_desc(elem->direction < 0); + tipb::Expr * expr = by->mutable_expr(); + astToPB(children[0]->output_schema, elem->children[0], expr, collator_id, context); + } + + auto * children_executor = sort->mutable_child(); + return children[0]->toTiPBExecutor(children_executor, collator_id, mpp_info, context); +} + } // namespace mock ExecutorPtr compileTableScan(size_t & executor_index, TableInfo & table_info, String & table_alias, bool append_pk_column) @@ -1561,11 +1666,101 @@ ExecutorPtr compileExchangeSender(ExecutorPtr input, size_t & executor_index, ti return exchange_sender; } - ExecutorPtr compileExchangeReceiver(size_t & executor_index, DAGSchema schema) { ExecutorPtr exchange_receiver = std::make_shared(executor_index, schema); return exchange_receiver; } +ExecutorPtr compileWindow(ExecutorPtr input, size_t & executor_index, ASTPtr func_desc_list, ASTPtr partition_by_expr_list, ASTPtr order_by_expr_list, mock::MockWindowFrame frame) +{ + std::vector partition_columns; + if (partition_by_expr_list != nullptr) + { + for (const auto & child : partition_by_expr_list->children) + { + auto * elem = typeid_cast(child.get()); + if (!elem) + throw Exception("Invalid partition by element", ErrorCodes::LOGICAL_ERROR); + partition_columns.push_back(child); + compileExpr(input->output_schema, elem->children[0]); + } + } + + std::vector order_columns; + if (order_by_expr_list != nullptr) + { + for (const auto & child : order_by_expr_list->children) + { + auto * elem = typeid_cast(child.get()); + if (!elem) + throw Exception("Invalid order by element", ErrorCodes::LOGICAL_ERROR); + order_columns.push_back(child); + compileExpr(input->output_schema, elem->children[0]); + } + } + + DAGSchema output_schema; + output_schema.insert(output_schema.end(), input->output_schema.begin(), input->output_schema.end()); + + std::vector window_exprs; + if (func_desc_list != nullptr) + { + for (const auto & expr : func_desc_list->children) + { + const auto * func = typeid_cast(expr.get()); + window_exprs.push_back(expr); + std::vector children_ci; + for (const auto & arg : func->arguments->children) + { + children_ci.push_back(compileExpr(input->output_schema, arg)); + } + // TODO: add more window functions + TiDB::ColumnInfo ci; + switch (window_func_name_to_sig[func->name]) + { + case tipb::ExprType::RowNumber: + case tipb::ExprType::Rank: + case tipb::ExprType::DenseRank: + { + ci.tp = TiDB::TypeLongLong; + ci.flag = TiDB::ColumnFlagBinary; + break; + } + default: + throw Exception(fmt::format("Unsupported window function {}", func->name), ErrorCodes::LOGICAL_ERROR); + } + output_schema.emplace_back(std::make_pair(func->getColumnName(), ci)); + } + } + + ExecutorPtr window = std::make_shared( + executor_index, + output_schema, + window_exprs, + std::move(partition_columns), + std::move(order_columns), + frame); + window->children.push_back(input); + return window; +} + +ExecutorPtr compileSort(ExecutorPtr input, size_t & executor_index, ASTPtr order_by_expr_list, bool is_partial_sort) +{ + std::vector order_columns; + if (order_by_expr_list != nullptr) + { + for (const auto & child : order_by_expr_list->children) + { + auto * elem = typeid_cast(child.get()); + if (!elem) + throw Exception("Invalid order by element", ErrorCodes::LOGICAL_ERROR); + order_columns.push_back(child); + compileExpr(input->output_schema, elem->children[0]); + } + } + ExecutorPtr sort = std::make_shared(executor_index, input->output_schema, std::move(order_columns), is_partial_sort); + sort->children.push_back(input); + return sort; +} } // namespace DB \ No newline at end of file diff --git a/dbms/src/Debug/astToExecutor.h b/dbms/src/Debug/astToExecutor.h index 37d3f22b6e1..cbd2e5ade3a 100644 --- a/dbms/src/Debug/astToExecutor.h +++ b/dbms/src/Debug/astToExecutor.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,8 @@ #include #include +#include + namespace DB { namespace ErrorCodes @@ -272,6 +275,54 @@ struct Join : Executor void toMPPSubPlan(size_t & executor_index, const DAGProperties & properties, std::unordered_map, std::shared_ptr>> & exchange_map) override; }; + +using MockWindowFrameBound = std::tuple; + +struct MockWindowFrame +{ + std::optional type; + std::optional start; + std::optional end; + // TODO: support calcFuncs +}; + +struct Window : Executor +{ + std::vector func_descs; + std::vector partition_by_exprs; + std::vector order_by_exprs; + MockWindowFrame frame; + + Window(size_t & index_, const DAGSchema & output_schema_, std::vector func_descs_, std::vector partition_by_exprs_, std::vector order_by_exprs_, MockWindowFrame frame_) + : Executor(index_, "window_" + std::to_string(index_), output_schema_) + , func_descs(std::move(func_descs_)) + , partition_by_exprs(std::move(partition_by_exprs_)) + , order_by_exprs(order_by_exprs_) + , frame(frame_) + { + } + // Currently only use Window Executor in Unit Test which don't call columnPrume. + // TODO: call columnPrune in unit test and further benchmark test to eliminate compute process. + void columnPrune(std::unordered_set &) override { throw Exception("Should not reach here"); } + bool toTiPBExecutor(tipb::Executor * tipb_executor, uint32_t collator_id, const MPPInfo & mpp_info, const Context & context) override; +}; + +struct Sort : Executor +{ + std::vector by_exprs; + bool is_partial_sort; + + Sort(size_t & index_, const DAGSchema & output_schema_, std::vector by_exprs_, bool is_partial_sort_) + : Executor(index_, "sort_" + std::to_string(index_), output_schema_) + , by_exprs(by_exprs_) + , is_partial_sort(is_partial_sort_) + { + } + // Currently only use Sort Executor in Unit Test which don't call columnPrume. + // TODO: call columnPrune in unit test and further benchmark test to eliminate compute process. + void columnPrune(std::unordered_set &) override { throw Exception("Should not reach here"); } + bool toTiPBExecutor(tipb::Executor * tipb_executor, uint32_t collator_id, const MPPInfo & mpp_info, const Context & context) override; +}; } // namespace mock using ExecutorPtr = std::shared_ptr; @@ -294,8 +345,9 @@ ExecutorPtr compileExchangeSender(ExecutorPtr input, size_t & executor_index, ti ExecutorPtr compileExchangeReceiver(size_t & executor_index, DAGSchema schema); -void literalFieldToTiPBExpr(const ColumnInfo & ci, const Field & field, tipb::Expr * expr, Int32 collator_id); +ExecutorPtr compileWindow(ExecutorPtr input, size_t & executor_index, ASTPtr func_desc_list, ASTPtr partition_by_expr_list, ASTPtr order_by_expr_list, mock::MockWindowFrame frame); -//TODO: add compileWindow +ExecutorPtr compileSort(ExecutorPtr input, size_t & executor_index, ASTPtr order_by_expr_list, bool is_partial_sort); +void literalFieldToTiPBExpr(const ColumnInfo & ci, const Field & field, tipb::Expr * expr, Int32 collator_id); } // namespace DB \ No newline at end of file diff --git a/dbms/src/Debug/dbgFuncRegion.cpp b/dbms/src/Debug/dbgFuncRegion.cpp index 7924c086508..b2024eac1d8 100644 --- a/dbms/src/Debug/dbgFuncRegion.cpp +++ b/dbms/src/Debug/dbgFuncRegion.cpp @@ -40,7 +40,7 @@ extern const int UNKNOWN_TABLE; // put_region(region_id, start, end, database_name, table_name[, partition-name]) void dbgFuncPutRegion(Context & context, const ASTs & args, DBGInvoker::Printer output) { - RegionID region_id = static_cast(safeGet(typeid_cast(*args[0]).value)); + auto region_id = static_cast(safeGet(typeid_cast(*args[0]).value)); bool has_partition_id = false; size_t args_size = args.size(); if (dynamic_cast(args[args_size - 1].get()) != nullptr) @@ -81,8 +81,8 @@ void dbgFuncPutRegion(Context & context, const ASTs & args, DBGInvoker::Printer } else { - HandleID start = static_cast(safeGet(typeid_cast(*args[1]).value)); - HandleID end = static_cast(safeGet(typeid_cast(*args[2]).value)); + auto start = static_cast(safeGet(typeid_cast(*args[1]).value)); + auto end = static_cast(safeGet(typeid_cast(*args[2]).value)); TMTContext & tmt = context.getTMTContext(); RegionPtr region = RegionBench::createRegion(table_id, region_id, start, end); @@ -107,7 +107,7 @@ void dbgFuncTryFlushRegion(Context & context, const ASTs & args, DBGInvoker::Pri throw Exception("Args not matched, should be: region-id", ErrorCodes::BAD_ARGUMENTS); } - RegionID region_id = static_cast(safeGet(typeid_cast(*args[0]).value)); + auto region_id = static_cast(safeGet(typeid_cast(*args[0]).value)); TMTContext & tmt = context.getTMTContext(); tmt.getRegionTable().tryFlushRegion(region_id); @@ -160,7 +160,7 @@ void dbgFuncDumpAllRegion(Context & context, const ASTs & args, DBGInvoker::Prin if (args.empty()) throw Exception("Args not matched, should be: table_id", ErrorCodes::BAD_ARGUMENTS); - TableID table_id = static_cast(safeGet(typeid_cast(*args[0]).value)); + auto table_id = static_cast(safeGet(typeid_cast(*args[0]).value)); bool ignore_none = false; if (args.size() > 1) @@ -190,7 +190,7 @@ void dbgFuncRemoveRegion(Context & context, const ASTs & args, DBGInvoker::Print if (args.empty()) throw Exception("Args not matched, should be: region_id", ErrorCodes::BAD_ARGUMENTS); - RegionID region_id = static_cast(safeGet(typeid_cast(*args[0]).value)); + auto region_id = static_cast(safeGet(typeid_cast(*args[0]).value)); TMTContext & tmt = context.getTMTContext(); KVStorePtr & kvstore = tmt.getKVStore(); diff --git a/dbms/src/Debug/dbgFuncSchema.cpp b/dbms/src/Debug/dbgFuncSchema.cpp index 8b73ddc23a3..c388015dc10 100644 --- a/dbms/src/Debug/dbgFuncSchema.cpp +++ b/dbms/src/Debug/dbgFuncSchema.cpp @@ -34,6 +34,7 @@ namespace DB { namespace ErrorCodes { +extern const int FAIL_POINT_ERROR; extern const int UNKNOWN_TABLE; } // namespace ErrorCodes @@ -62,7 +63,22 @@ void dbgFuncRefreshSchemas(Context & context, const ASTs &, DBGInvoker::Printer { TMTContext & tmt = context.getTMTContext(); auto schema_syncer = tmt.getSchemaSyncer(); - schema_syncer->syncSchemas(context); + try + { + schema_syncer->syncSchemas(context); + } + catch (Exception & e) + { + if (e.code() == ErrorCodes::FAIL_POINT_ERROR) + { + output(e.message()); + return; + } + else + { + throw; + } + } output("schemas refreshed"); } diff --git a/dbms/src/Flash/Coprocessor/CoprocessorReader.h b/dbms/src/Flash/Coprocessor/CoprocessorReader.h index 8a3eb471e54..25c07cff49c 100644 --- a/dbms/src/Flash/Coprocessor/CoprocessorReader.h +++ b/dbms/src/Flash/Coprocessor/CoprocessorReader.h @@ -20,7 +20,6 @@ #include #include #include -#include #include #include @@ -29,6 +28,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include #include #include diff --git a/dbms/src/Flash/Coprocessor/DAGContext.cpp b/dbms/src/Flash/Coprocessor/DAGContext.cpp index 17fb6553eab..1ef7338a589 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.cpp +++ b/dbms/src/Flash/Coprocessor/DAGContext.cpp @@ -205,60 +205,13 @@ void DAGContext::attachBlockIO(const BlockIO & io_) { io = io_; } -void DAGContext::initExchangeReceiverIfMPP(Context & context, size_t max_streams) -{ - if (isMPPTask()) - { - if (mpp_exchange_receiver_map_inited) - throw TiFlashException("Repeatedly initialize mpp_exchange_receiver_map", Errors::Coprocessor::Internal); - traverseExecutors(dag_request, [&](const tipb::Executor & executor) { - if (executor.tp() == tipb::ExecType::TypeExchangeReceiver) - { - assert(executor.has_executor_id()); - const auto & executor_id = executor.executor_id(); - // In order to distinguish different exchange receivers. - auto exchange_receiver = std::make_shared( - std::make_shared( - executor.exchange_receiver(), - getMPPTaskMeta(), - context.getTMTContext().getKVCluster(), - context.getTMTContext().getMPPTaskManager(), - context.getSettingsRef().enable_local_tunnel, - context.getSettingsRef().enable_async_grpc_client), - executor.exchange_receiver().encoded_task_meta_size(), - max_streams, - log->identifier(), - executor_id); - mpp_exchange_receiver_map[executor_id] = exchange_receiver; - new_thread_count_of_exchange_receiver += exchange_receiver->computeNewThreadCount(); - } - return true; - }); - mpp_exchange_receiver_map_inited = true; - } -} - const std::unordered_map> & DAGContext::getMPPExchangeReceiverMap() const { if (!isMPPTask()) throw TiFlashException("mpp_exchange_receiver_map is used in mpp only", Errors::Coprocessor::Internal); - if (!mpp_exchange_receiver_map_inited) - throw TiFlashException("mpp_exchange_receiver_map has not been initialized", Errors::Coprocessor::Internal); - return mpp_exchange_receiver_map; -} - -void DAGContext::cancelAllExchangeReceiver() -{ - for (auto & it : mpp_exchange_receiver_map) - { - it.second->cancel(); - } -} - -int DAGContext::getNewThreadCountOfExchangeReceiver() const -{ - return new_thread_count_of_exchange_receiver; + RUNTIME_ASSERT(mpp_exchange_receiver_map != nullptr, log, "MPPTask without exchange receiver map"); + return *mpp_exchange_receiver_map; } bool DAGContext::containsRegionsInfoForTable(Int64 table_id) const @@ -271,4 +224,13 @@ const SingleTableRegions & DAGContext::getTableRegionsInfoByTableID(Int64 table_ return tables_regions_info.getTableRegionInfoByTableID(table_id); } +ColumnsWithTypeAndName DAGContext::columnsForTest(String executor_id) +{ + auto it = columns_for_test_map.find(executor_id); + if (unlikely(it == columns_for_test_map.end())) + { + throw DB::Exception("Don't have columns for mock source executors"); + } + return it->second; +} } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGContext.h b/dbms/src/Flash/Coprocessor/DAGContext.h index e3e5efdcbc6..07b65b2d8fe 100644 --- a/dbms/src/Flash/Coprocessor/DAGContext.h +++ b/dbms/src/Flash/Coprocessor/DAGContext.h @@ -37,6 +37,8 @@ namespace DB class Context; class MPPTunnelSet; class ExchangeReceiver; +using ExchangeReceiverMap = std::unordered_map>; +using ExchangeReceiverMapPtr = std::shared_ptr>>; class Join; using JoinPtr = std::shared_ptr; @@ -254,7 +256,6 @@ class DAGContext return io; } - int getNewThreadCountOfExchangeReceiver() const; UInt64 getFlags() const { return flags; @@ -298,11 +299,16 @@ class DAGContext } bool isTest() const { return is_test; } + void setColumnsForTest(std::unordered_map & columns_for_test_map_) { columns_for_test_map = columns_for_test_map_; } + ColumnsWithTypeAndName columnsForTest(String executor_id); - void cancelAllExchangeReceiver(); + bool columnsForTestEmpty() { return columns_for_test_map.empty(); } - void initExchangeReceiverIfMPP(Context & context, size_t max_streams); const std::unordered_map> & getMPPExchangeReceiverMap() const; + void setMPPExchangeReceiverMap(ExchangeReceiverMapPtr & exchange_receiver_map) + { + mpp_exchange_receiver_map = exchange_receiver_map; + } void addSubquery(const String & subquery_id, SubqueryForSet && subquery); bool hasSubquery() const { return !subqueries.empty(); } @@ -317,8 +323,8 @@ class DAGContext Clock::time_point read_wait_index_end_timestamp{Clock::duration::zero()}; String table_scan_executor_id; String tidb_host = "Unknown"; - bool collect_execution_summaries; - bool return_executor_id; + bool collect_execution_summaries{}; + bool return_executor_id{}; bool is_mpp_task = false; bool is_root_mpp_task = false; bool is_batch_cop = false; @@ -363,15 +369,14 @@ class DAGContext ConcurrentBoundedQueue warnings; /// warning_count is the actual warning count during the entire execution std::atomic warning_count; - int new_thread_count_of_exchange_receiver = 0; /// key: executor_id of ExchangeReceiver nodes in dag. - std::unordered_map> mpp_exchange_receiver_map; - bool mpp_exchange_receiver_map_inited = false; + ExchangeReceiverMapPtr mpp_exchange_receiver_map; /// vector of SubqueriesForSets(such as join build subquery). /// The order of the vector is also the order of the subquery. std::vector subqueries; bool is_test = false; /// switch for test, do not use it in production. + std::unordered_map columns_for_test_map; /// , for multiple sources }; } // namespace DB diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h index 3b7112af02d..9f201006a88 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzer.h @@ -151,11 +151,9 @@ class DAGExpressionAnalyzer : private boost::noncopyable void appendCastAfterWindow( const ExpressionActionsPtr & actions, const tipb::Window & window, - const size_t window_columns_start_index); + size_t window_columns_start_index); -#ifndef DBMS_PUBLIC_GTEST private: -#endif NamesAndTypes buildOrderColumns( const ExpressionActionsPtr & actions, const ::google::protobuf::RepeatedPtrField & order_by); diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp index 5fac49faaed..86d6428c92a 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -159,13 +160,22 @@ AnalysisResult analyzeExpressions( // for tests, we need to mock tableScan blockInputStream as the source stream. void DAGQueryBlockInterpreter::handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline) { - auto names_and_types = genNamesAndTypes(table_scan); - auto columns_with_type_and_name = getColumnWithTypeAndName(names_and_types); - analyzer = std::make_unique(std::move(names_and_types), context); - for (size_t i = 0; i < max_streams; ++i) + if (context.getDAGContext()->columnsForTestEmpty() || context.getDAGContext()->columnsForTest(table_scan.getTableScanExecutorID()).empty()) + { + auto names_and_types = genNamesAndTypes(table_scan); + auto columns_with_type_and_name = getColumnWithTypeAndName(names_and_types); + analyzer = std::make_unique(std::move(names_and_types), context); + for (size_t i = 0; i < max_streams; ++i) + { + auto mock_table_scan_stream = std::make_shared(columns_with_type_and_name, context.getSettingsRef().max_block_size); + pipeline.streams.emplace_back(mock_table_scan_stream); + } + } + else { - auto mock_table_scan_stream = std::make_shared(columns_with_type_and_name, context.getSettingsRef().max_block_size); - pipeline.streams.emplace_back(mock_table_scan_stream); + auto [names_and_types, mock_table_scan_streams] = mockSourceStream(context, max_streams, log, table_scan.getTableScanExecutorID()); + analyzer = std::make_unique(std::move(names_and_types), context); + pipeline.streams.insert(pipeline.streams.end(), mock_table_scan_streams.begin(), mock_table_scan_streams.end()); } } @@ -266,7 +276,8 @@ void DAGQueryBlockInterpreter::handleJoin(const tipb::Join & join, DAGPipeline & stream->setExtraInfo( fmt::format("join build, build_side_root_executor_id = {}", dagContext().getJoinExecuteInfoMap()[query_block.source_name].build_side_root_executor_id)); }); - executeUnion(build_pipeline, max_streams, log, /*ignore_block=*/true, "for join"); + // for test, join executor need the return blocks to output. + executeUnion(build_pipeline, max_streams, log, /*ignore_block=*/!dagContext().isTest(), "for join"); right_query.source = build_pipeline.firstStream(); right_query.join = join_ptr; @@ -491,19 +502,29 @@ void DAGQueryBlockInterpreter::handleExchangeReceiver(DAGPipeline & pipeline) analyzer = std::make_unique(std::move(source_columns), context); } +// for tests, we need to mock ExchangeReceiver blockInputStream as the source stream. void DAGQueryBlockInterpreter::handleMockExchangeReceiver(DAGPipeline & pipeline) { - for (size_t i = 0; i < max_streams; ++i) + if (context.getDAGContext()->columnsForTestEmpty() || context.getDAGContext()->columnsForTest(query_block.source_name).empty()) { - // use max_block_size / 10 to determine the mock block's size - pipeline.streams.push_back(std::make_shared(query_block.source->exchange_receiver(), context.getSettingsRef().max_block_size, context.getSettingsRef().max_block_size / 10)); + for (size_t i = 0; i < max_streams; ++i) + { + // use max_block_size / 10 to determine the mock block's size + pipeline.streams.push_back(std::make_shared(query_block.source->exchange_receiver(), context.getSettingsRef().max_block_size, context.getSettingsRef().max_block_size / 10)); + } + NamesAndTypes source_columns; + for (const auto & col : pipeline.firstStream()->getHeader()) + { + source_columns.emplace_back(col.name, col.type); + } + analyzer = std::make_unique(std::move(source_columns), context); } - NamesAndTypes source_columns; - for (const auto & col : pipeline.firstStream()->getHeader()) + else { - source_columns.emplace_back(col.name, col.type); + auto [names_and_types, mock_exchange_streams] = mockSourceStream(context, max_streams, log, query_block.source_name); + analyzer = std::make_unique(std::move(names_and_types), context); + pipeline.streams.insert(pipeline.streams.end(), mock_exchange_streams.begin(), mock_exchange_streams.end()); } - analyzer = std::make_unique(std::move(source_columns), context); } void DAGQueryBlockInterpreter::handleProjection(DAGPipeline & pipeline, const tipb::Projection & projection) diff --git a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h index e68c4f91cee..0b3b2db9623 100644 --- a/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h +++ b/dbms/src/Flash/Coprocessor/DAGQueryBlockInterpreter.h @@ -54,9 +54,7 @@ class DAGQueryBlockInterpreter BlockInputStreams execute(); -#ifndef DBMS_PUBLIC_GTEST private: -#endif void executeImpl(DAGPipeline & pipeline); void handleMockTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline); void handleTableScan(const TiDBTableScan & table_scan, DAGPipeline & pipeline); diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index 11a1b7e2d3e..14cddd94730 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #pragma GCC diagnostic push @@ -633,6 +635,9 @@ void DAGStorageInterpreter::buildLocalStreams(DAGPipeline & pipeline, size_t max if (total_local_region_num == 0) return; const auto table_query_infos = generateSelectQueryInfos(); + bool has_multiple_partitions = table_query_infos.size() > 1; + // MultiPartitionStreamPool will be disabled in no partition mode or single-partition case + std::shared_ptr stream_pool = has_multiple_partitions ? std::make_shared() : nullptr; for (const auto & table_query_info : table_query_infos) { DAGPipeline current_pipeline; @@ -641,9 +646,6 @@ void DAGStorageInterpreter::buildLocalStreams(DAGPipeline & pipeline, size_t max size_t region_num = query_info.mvcc_query_info->regions_query_info.size(); if (region_num == 0) continue; - /// calculate weighted max_streams for each partition, note at least 1 stream is needed for each partition - size_t current_max_streams = table_query_infos.size() == 1 ? max_streams : (max_streams * region_num + total_local_region_num - 1) / total_local_region_num; - QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; assert(storages_with_structure_lock.find(table_id) != storages_with_structure_lock.end()); auto & storage = storages_with_structure_lock[table_id].storage; @@ -653,7 +655,7 @@ void DAGStorageInterpreter::buildLocalStreams(DAGPipeline & pipeline, size_t max { try { - current_pipeline.streams = storage->read(required_columns, query_info, context, from_stage, max_block_size, current_max_streams); + current_pipeline.streams = storage->read(required_columns, query_info, context, from_stage, max_block_size, max_streams); // After getting streams from storage, we need to validate whether Regions have changed or not after learner read. // (by calling `validateQueryInfo`). In case the key ranges of Regions have changed (Region merge/split), those `streams` @@ -777,7 +779,19 @@ void DAGStorageInterpreter::buildLocalStreams(DAGPipeline & pipeline, size_t max throw; } } - pipeline.streams.insert(pipeline.streams.end(), current_pipeline.streams.begin(), current_pipeline.streams.end()); + if (has_multiple_partitions) + stream_pool->addPartitionStreams(current_pipeline.streams); + else + pipeline.streams.insert(pipeline.streams.end(), current_pipeline.streams.begin(), current_pipeline.streams.end()); + } + if (has_multiple_partitions) + { + String req_info = dag_context.isMPPTask() ? dag_context.getMPPTaskId().toString() : ""; + int exposed_streams_cnt = std::min(static_cast(max_streams), stream_pool->addedStreamsCnt()); + for (int i = 0; i < exposed_streams_cnt; ++i) + { + pipeline.streams.push_back(std::make_shared(stream_pool, req_info)); + } } } diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.h b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.h index d86274a1e22..0425abe04db 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.h +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.h @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -33,6 +32,7 @@ namespace DB { +class TMTContext; using TablesRegionInfoMap = std::unordered_map>; /// DAGStorageInterpreter encapsulates operations around storage during interprete stage. /// It's only intended to be used by DAGQueryBlockInterpreter. diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index 87f58131c8c..9ffa29cd14d 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -29,7 +29,6 @@ #include namespace DB { - const Int8 VAR_SIZE = 0; extern const String uniq_raw_res_name; @@ -770,6 +769,10 @@ const String & getFunctionName(const tipb::Expr & expr) { return getAggFunctionName(expr); } + else if (isWindowFunctionExpr(expr)) + { + return getWindowFunctionName(expr); + } else { auto it = scalar_func_map.find(expr.sig()); @@ -1429,6 +1432,7 @@ tipb::EncodeType analyzeDAGEncodeType(DAGContext & dag_context) return tipb::EncodeType::TypeDefault; return encode_type; } + tipb::ScalarFuncSig reverseGetFuncSigByFuncName(const String & name) { static std::unordered_map func_name_sig_map = getFuncNameToSigMap(); diff --git a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp index e7964021709..be3475f714f 100644 --- a/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp +++ b/dbms/src/Flash/Coprocessor/GenSchemaAndColumn.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include #include +#include namespace DB { diff --git a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp index 741aa7b5e26..0e767d65d77 100644 --- a/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp +++ b/dbms/src/Flash/Coprocessor/InterpreterDAG.cpp @@ -24,19 +24,8 @@ namespace DB InterpreterDAG::InterpreterDAG(Context & context_, const DAGQuerySource & dag_) : context(context_) , dag(dag_) + , max_streams(context.getMaxStreams()) { - const Settings & settings = context.getSettingsRef(); - if (dagContext().isBatchCop() || (dagContext().isMPPTask() && !dagContext().isTest())) - max_streams = settings.max_threads; - else if (dagContext().isTest()) - max_streams = dagContext().initialize_concurrency; - else - max_streams = 1; - - if (max_streams > 1) - { - max_streams *= settings.max_streams_to_max_threads_ratio; - } } void setRestorePipelineConcurrency(DAGQueryBlock & query_block) @@ -75,19 +64,17 @@ BlockInputStreams InterpreterDAG::executeQueryBlock(DAGQueryBlock & query_block) BlockIO InterpreterDAG::execute() { - /// Due to learner read, DAGQueryBlockInterpreter may take a long time to build - /// the query plan, so we init mpp exchange receiver before executeQueryBlock - dagContext().initExchangeReceiverIfMPP(context, max_streams); - BlockInputStreams streams = executeQueryBlock(*dag.getRootQueryBlock()); DAGPipeline pipeline; pipeline.streams = streams; /// add union to run in parallel if needed - if (dagContext().isMPPTask()) + if (unlikely(dagContext().isTest())) + executeUnion(pipeline, max_streams, dagContext().log, /*ignore_block=*/false, "for test"); + else if (dagContext().isMPPTask()) /// MPPTask do not need the returned blocks. executeUnion(pipeline, max_streams, dagContext().log, /*ignore_block=*/true, "for mpp"); else - executeUnion(pipeline, max_streams, dagContext().log, false, "for non mpp"); + executeUnion(pipeline, max_streams, dagContext().log, /*ignore_block=*/false, "for non mpp"); if (dagContext().hasSubquery()) { const Settings & settings = context.getSettingsRef(); diff --git a/dbms/src/Flash/Coprocessor/MockSourceStream.h b/dbms/src/Flash/Coprocessor/MockSourceStream.h new file mode 100644 index 00000000000..039cba22e3d --- /dev/null +++ b/dbms/src/Flash/Coprocessor/MockSourceStream.h @@ -0,0 +1,60 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include +#include + +namespace DB +{ +template +std::pair>> mockSourceStream(Context & context, size_t max_streams, DB::LoggerPtr log, String executor_id) +{ + ColumnsWithTypeAndName columns_with_type_and_name; + NamesAndTypes names_and_types; + size_t rows = 0; + std::vector> mock_source_streams; + columns_with_type_and_name = context.getDAGContext()->columnsForTest(executor_id); + for (const auto & col : columns_with_type_and_name) + { + if (rows == 0) + rows = col.column->size(); + RUNTIME_ASSERT(rows == col.column->size(), log, "each column must has same size"); + names_and_types.push_back({col.name, col.type}); + } + size_t row_for_each_stream = rows / max_streams; + size_t rows_left = rows - row_for_each_stream * max_streams; + size_t start = 0; + for (size_t i = 0; i < max_streams; ++i) + { + ColumnsWithTypeAndName columns_for_stream; + size_t row_for_current_stream = row_for_each_stream + (i < rows_left ? 1 : 0); + for (const auto & column_with_type_and_name : columns_with_type_and_name) + { + columns_for_stream.push_back( + ColumnWithTypeAndName( + column_with_type_and_name.column->cut(start, row_for_current_stream), + column_with_type_and_name.type, + column_with_type_and_name.name)); + } + start += row_for_current_stream; + mock_source_streams.emplace_back(std::make_shared(columns_for_stream, context.getSettingsRef().max_block_size)); + } + RUNTIME_ASSERT(start == rows, log, "mock source streams' total size must same as user input"); + return {names_and_types, mock_source_streams}; +} +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp index 086cdb43d20..d3b5c202136 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp @@ -13,8 +13,10 @@ // limitations under the License. #include +#include #include #include +#include namespace DB { diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.h b/dbms/src/Flash/Coprocessor/RemoteRequest.h index 1e42e18a7bd..5af3f66298c 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.h +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.h @@ -17,11 +17,12 @@ #include #include #include -#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include +#include #include #pragma GCC diagnostic pop diff --git a/dbms/src/Flash/Coprocessor/TablesRegionsInfo.cpp b/dbms/src/Flash/Coprocessor/TablesRegionsInfo.cpp index 7c4ddedabf4..ab4a0f82e95 100644 --- a/dbms/src/Flash/Coprocessor/TablesRegionsInfo.cpp +++ b/dbms/src/Flash/Coprocessor/TablesRegionsInfo.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB { diff --git a/dbms/src/Flash/Coprocessor/TablesRegionsInfo.h b/dbms/src/Flash/Coprocessor/TablesRegionsInfo.h index dccb8b95466..f80a44b92a3 100644 --- a/dbms/src/Flash/Coprocessor/TablesRegionsInfo.h +++ b/dbms/src/Flash/Coprocessor/TablesRegionsInfo.h @@ -15,7 +15,7 @@ #pragma once #include #include -#include + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" @@ -25,6 +25,7 @@ namespace DB { +class TMTContext; struct SingleTableRegions { RegionInfoMap local_regions; diff --git a/dbms/src/Flash/Coprocessor/collectOutputFieldTypes.cpp b/dbms/src/Flash/Coprocessor/collectOutputFieldTypes.cpp index b68279faa13..87744c553e0 100644 --- a/dbms/src/Flash/Coprocessor/collectOutputFieldTypes.cpp +++ b/dbms/src/Flash/Coprocessor/collectOutputFieldTypes.cpp @@ -45,19 +45,36 @@ bool collectForAgg(std::vector & output_field_types, const tipb { for (const auto & expr : agg.agg_func()) { - if (!exprHasValidFieldType(expr)) + if (unlikely(!exprHasValidFieldType(expr))) throw TiFlashException("Agg expression without valid field type", Errors::Coprocessor::BadRequest); output_field_types.push_back(expr.field_type()); } for (const auto & expr : agg.group_by()) { - if (!exprHasValidFieldType(expr)) + if (unlikely(!exprHasValidFieldType(expr))) throw TiFlashException("Group by expression without valid field type", Errors::Coprocessor::BadRequest); output_field_types.push_back(expr.field_type()); } return false; } +bool collectForExecutor(std::vector & output_field_types, const tipb::Executor & executor); +bool collectForWindow(std::vector & output_field_types, const tipb::Executor & executor) +{ + // collect output_field_types of child + getChildren(executor).forEach([&output_field_types](const tipb::Executor & child) { + traverseExecutorTree(child, [&output_field_types](const tipb::Executor & e) { return collectForExecutor(output_field_types, e); }); + }); + + for (const auto & expr : executor.window().func_desc()) + { + if (unlikely(!exprHasValidFieldType(expr))) + throw TiFlashException("Window expression without valid field type", Errors::Coprocessor::BadRequest); + output_field_types.push_back(expr.field_type()); + } + return false; +} + bool collectForReceiver(std::vector & output_field_types, const tipb::ExchangeReceiver & receiver) { for (const auto & field_type : receiver.field_types()) @@ -82,7 +99,6 @@ bool collectForTableScan(std::vector & output_field_types, cons return false; } -bool collectForExecutor(std::vector & output_field_types, const tipb::Executor & executor); bool collectForJoin(std::vector & output_field_types, const tipb::Executor & executor) { // collect output_field_types of children @@ -147,8 +163,8 @@ bool collectForExecutor(std::vector & output_field_types, const case tipb::ExecType::TypeWindow: // Window will only be pushed down in mpp mode. // In mpp mode, ExchangeSender or Sender will return output_field_types directly. - // If not in mpp mode, window executor type is invalid. - throw TiFlashException("Window executor type is invalid in non-mpp mode, should not reach here.", Errors::Coprocessor::Internal); + // If not in mpp mode or debug mode, window executor type is invalid. + return collectForWindow(output_field_types, executor); case tipb::ExecType::TypeExchangeReceiver: return collectForReceiver(output_field_types, executor.exchange_receiver()); case tipb::ExecType::TypeTableScan: diff --git a/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp b/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp index 8ec3eb54406..1e9da93ffe3 100644 --- a/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp +++ b/dbms/src/Flash/Management/tests/gtest_manual_compact.cpp @@ -14,12 +14,12 @@ #include #include -#include #include #include +#include #include -#include #include +#include #include #include #include @@ -47,7 +47,6 @@ class BasicManualCompactTest BasicManualCompactTest() { - log = &Poco::Logger::get(DB::base::TiFlashStorageTestBasic::getCurrentFullTestName()); pk_type = GetParam(); } @@ -62,7 +61,7 @@ class BasicManualCompactTest setupStorage(); // In tests let's only compact one segment. - db_context->setSetting("manual_compact_more_until_ms", UInt64(0)); + db_context->setSetting("manual_compact_more_until_ms", Field(UInt64(0))); // Split into 4 segments, and prepare some delta data for first 3 segments. helper = std::make_unique(*db_context); @@ -115,8 +114,6 @@ class BasicManualCompactTest std::unique_ptr manager; DM::tests::DMTestEnv::PkType pk_type; - - [[maybe_unused]] Poco::Logger * log; }; @@ -314,7 +311,7 @@ CATCH TEST_P(BasicManualCompactTest, CompactMultiple) try { - db_context->setSetting("manual_compact_more_until_ms", UInt64(60 * 1000)); // Hope it's long enough! + db_context->setSetting("manual_compact_more_until_ms", Field(UInt64(60 * 1000))); // Hope it's long enough! auto request = ::kvrpcpb::CompactRequest(); request.set_physical_table_id(TABLE_ID); diff --git a/dbms/src/Flash/Mpp/MPPTask.cpp b/dbms/src/Flash/Mpp/MPPTask.cpp index 0f18ad582b4..40f03ff79ba 100644 --- a/dbms/src/Flash/Mpp/MPPTask.cpp +++ b/dbms/src/Flash/Mpp/MPPTask.cpp @@ -22,11 +22,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -56,6 +59,7 @@ MPPTask::MPPTask(const mpp::TaskMeta & meta_, const ContextPtr & context_) , id(meta.start_ts(), meta.task_id()) , log(Logger::get("MPPTask", id.toString())) , mpp_task_statistics(id, meta.address()) + , needed_threads(0) , schedule_state(ScheduleState::WAITING) {} @@ -78,18 +82,14 @@ MPPTask::~MPPTask() void MPPTask::closeAllTunnels(const String & reason) { - for (auto & it : tunnel_map) - { - it.second->close(reason); - } + if (likely(tunnel_set)) + tunnel_set->close(reason); } void MPPTask::finishWrite() { - for (const auto & it : tunnel_map) - { - it.second->writeDone(); - } + RUNTIME_ASSERT(tunnel_set != nullptr, log, "mpp task without tunnel set"); + tunnel_set->finishWrite(); } void MPPTask::run() @@ -97,15 +97,73 @@ void MPPTask::run() newThreadManager()->scheduleThenDetach(true, "MPPTask", [self = shared_from_this()] { self->runImpl(); }); } -void MPPTask::registerTunnel(const MPPTaskId & id, MPPTunnelPtr tunnel) +void MPPTask::registerTunnels(const mpp::DispatchTaskRequest & task_request) { - if (status == CANCELLED) - throw Exception("the tunnel " + tunnel->id() + " can not been registered, because the task is cancelled"); + tunnel_set = std::make_shared(log->identifier()); + std::chrono::seconds timeout(task_request.timeout()); + const auto & exchange_sender = dag_req.root_executor().exchange_sender(); + + for (int i = 0; i < exchange_sender.encoded_task_meta_size(); ++i) + { + // exchange sender will register the tunnels and wait receiver to found a connection. + mpp::TaskMeta task_meta; + if (unlikely(!task_meta.ParseFromString(exchange_sender.encoded_task_meta(i)))) + throw TiFlashException("Failed to decode task meta info in ExchangeSender", Errors::Coprocessor::BadRequest); + bool is_local = context->getSettingsRef().enable_local_tunnel && meta.address() == task_meta.address(); + bool is_async = !is_local && context->getSettingsRef().enable_async_server; + MPPTunnelPtr tunnel = std::make_shared(task_meta, task_request.meta(), timeout, context->getSettingsRef().max_threads, is_local, is_async, log->identifier()); + LOG_FMT_DEBUG(log, "begin to register the tunnel {}", tunnel->id()); + if (status != INITIALIZING) + throw Exception(fmt::format("The tunnel {} can not be registered, because the task is not in initializing state", tunnel->id())); + tunnel_set->registerTunnel(MPPTaskId{task_meta.start_ts(), task_meta.task_id()}, tunnel); + if (!dag_context->isRootMPPTask()) + { + FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::exception_during_mpp_register_tunnel_for_non_root_mpp_task); + } + } +} - if (tunnel_map.find(id) != tunnel_map.end()) - throw Exception("the tunnel " + tunnel->id() + " has been registered"); +void MPPTask::initExchangeReceivers() +{ + mpp_exchange_receiver_map = std::make_shared(); + traverseExecutors(&dag_req, [&](const tipb::Executor & executor) { + if (executor.tp() == tipb::ExecType::TypeExchangeReceiver) + { + assert(executor.has_executor_id()); + const auto & executor_id = executor.executor_id(); + // In order to distinguish different exchange receivers. + auto exchange_receiver = std::make_shared( + std::make_shared( + executor.exchange_receiver(), + dag_context->getMPPTaskMeta(), + context->getTMTContext().getKVCluster(), + context->getTMTContext().getMPPTaskManager(), + context->getSettingsRef().enable_local_tunnel, + context->getSettingsRef().enable_async_grpc_client), + executor.exchange_receiver().encoded_task_meta_size(), + context->getMaxStreams(), + log->identifier(), + executor_id); + if (status != RUNNING) + throw Exception("exchange receiver map can not be initialized, because the task is not in running state"); + + (*mpp_exchange_receiver_map)[executor_id] = exchange_receiver; + new_thread_count_of_exchange_receiver += exchange_receiver->computeNewThreadCount(); + } + return true; + }); + dag_context->setMPPExchangeReceiverMap(mpp_exchange_receiver_map); +} - tunnel_map[id] = tunnel; +void MPPTask::cancelAllExchangeReceivers() +{ + if (likely(mpp_exchange_receiver_map != nullptr)) + { + for (auto & it : *mpp_exchange_receiver_map) + { + it.second->cancel(); + } + } } std::pair MPPTask::getTunnel(const ::mpp::EstablishMPPConnectionRequest * request) @@ -120,8 +178,9 @@ std::pair MPPTask::getTunnel(const ::mpp::EstablishMPPConn } MPPTaskId receiver_id{request->receiver_meta().start_ts(), request->receiver_meta().task_id()}; - auto it = tunnel_map.find(receiver_id); - if (it == tunnel_map.end()) + RUNTIME_ASSERT(tunnel_set != nullptr, log, "mpp task without tunnel set"); + auto tunnel_ptr = tunnel_set->getTunnelByReceiverTaskId(receiver_id); + if (tunnel_ptr == nullptr) { auto err_msg = fmt::format( "can't find tunnel ({} + {})", @@ -129,7 +188,7 @@ std::pair MPPTask::getTunnel(const ::mpp::EstablishMPPConn request->receiver_meta().task_id()); return {nullptr, err_msg}; } - return {it->second, ""}; + return {tunnel_ptr, ""}; } void MPPTask::unregisterTask() @@ -211,26 +270,8 @@ void MPPTask::prepare(const mpp::DispatchTaskRequest & task_request) } // register tunnels - tunnel_set = std::make_shared(); - std::chrono::seconds timeout(task_request.timeout()); + registerTunnels(task_request); - for (int i = 0; i < exchange_sender.encoded_task_meta_size(); i++) - { - // exchange sender will register the tunnels and wait receiver to found a connection. - mpp::TaskMeta task_meta; - if (!task_meta.ParseFromString(exchange_sender.encoded_task_meta(i))) - throw TiFlashException("Failed to decode task meta info in ExchangeSender", Errors::Coprocessor::BadRequest); - bool is_local = context->getSettingsRef().enable_local_tunnel && meta.address() == task_meta.address(); - bool is_async = !is_local && context->getSettingsRef().enable_async_server; - MPPTunnelPtr tunnel = std::make_shared(task_meta, task_request.meta(), timeout, context->getSettingsRef().max_threads, is_local, is_async, log->identifier()); - LOG_FMT_DEBUG(log, "begin to register the tunnel {}", tunnel->id()); - registerTunnel(MPPTaskId{task_meta.start_ts(), task_meta.task_id()}, tunnel); - tunnel_set->addTunnel(tunnel); - if (!dag_context->isRootMPPTask()) - { - FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::exception_during_mpp_register_tunnel_for_non_root_mpp_task); - } - } dag_context->tunnel_set = tunnel_set; // register task. auto task_manager = tmt_context.getMPPTaskManager(); @@ -256,6 +297,7 @@ void MPPTask::prepare(const mpp::DispatchTaskRequest & task_request) void MPPTask::preprocess() { auto start_time = Clock::now(); + initExchangeReceivers(); DAGQuerySource dag(*context); executeQuery(dag, *context, false, QueryProcessingStage::Complete); auto end_time = Clock::now(); @@ -285,7 +327,7 @@ void MPPTask::runImpl() LOG_FMT_INFO(log, "task starts preprocessing"); preprocess(); needed_threads = estimateCountOfNewThreads(); - LOG_FMT_DEBUG(log, "Estimate new thread count of query :{} including tunnel_threads: {} , receiver_threads: {}", needed_threads, dag_context->tunnel_set->getRemoteTunnelCnt(), dag_context->getNewThreadCountOfExchangeReceiver()); + LOG_FMT_DEBUG(log, "Estimate new thread count of query :{} including tunnel_threads: {} , receiver_threads: {}", needed_threads, dag_context->tunnel_set->getRemoteTunnelCnt(), new_thread_count_of_exchange_receiver); scheduleOrWait(); @@ -351,8 +393,7 @@ void MPPTask::runImpl() else { context->getProcessList().sendCancelToQuery(context->getCurrentQueryId(), context->getClientInfo().current_user, true); - if (dag_context) - dag_context->cancelAllExchangeReceiver(); + cancelAllExchangeReceivers(); writeErrToAllTunnels(err_msg); } LOG_FMT_INFO(log, "task ends, time cost is {} ms.", stopwatch.elapsedMilliseconds()); @@ -369,19 +410,8 @@ void MPPTask::runImpl() void MPPTask::writeErrToAllTunnels(const String & e) { - for (auto & it : tunnel_map) - { - try - { - FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::exception_during_mpp_write_err_to_tunnel); - it.second->write(getPacketWithError(e), true); - } - catch (...) - { - it.second->close("Failed to write error msg to tunnel"); - tryLogCurrentException(log, "Failed to write error " + e + " to tunnel: " + it.second->id()); - } - } + RUNTIME_ASSERT(tunnel_set != nullptr, log, "mpp task without tunnel set"); + tunnel_set->writeError(e); } void MPPTask::cancel(const String & reason) diff --git a/dbms/src/Flash/Mpp/MPPTask.h b/dbms/src/Flash/Mpp/MPPTask.h index c34cae49699..c8423ac484c 100644 --- a/dbms/src/Flash/Mpp/MPPTask.h +++ b/dbms/src/Flash/Mpp/MPPTask.h @@ -62,8 +62,6 @@ class MPPTask : public std::enable_shared_from_this void run(); - void registerTunnel(const MPPTaskId & id, MPPTunnelPtr tunnel); - int getNeededThreads(); enum class ScheduleState @@ -107,6 +105,12 @@ class MPPTask : public std::enable_shared_from_this int estimateCountOfNewThreads(); + void registerTunnels(const mpp::DispatchTaskRequest & task_request); + + void initExchangeReceivers(); + + void cancelAllExchangeReceivers(); + tipb::DAGRequest dag_req; ContextPtr context; @@ -122,9 +126,10 @@ class MPPTask : public std::enable_shared_from_this MPPTaskId id; MPPTunnelSetPtr tunnel_set; + /// key: executor_id of ExchangeReceiver nodes in dag. + ExchangeReceiverMapPtr mpp_exchange_receiver_map; - // which targeted task we should send data by which tunnel. - std::unordered_map tunnel_map; + int new_thread_count_of_exchange_receiver = 0; MPPTaskManager * manager = nullptr; diff --git a/dbms/src/Flash/Mpp/MPPTunnelSet.cpp b/dbms/src/Flash/Mpp/MPPTunnelSet.cpp index 12de07d4a18..8d709bb7d38 100644 --- a/dbms/src/Flash/Mpp/MPPTunnelSet.cpp +++ b/dbms/src/Flash/Mpp/MPPTunnelSet.cpp @@ -13,11 +13,17 @@ // limitations under the License. #include +#include #include +#include #include namespace DB { +namespace FailPoints +{ +extern const char exception_during_mpp_write_err_to_tunnel[]; +} // namespace FailPoints namespace { inline mpp::MPPDataPacket serializeToPacket(const tipb::SelectResponse & response) @@ -108,6 +114,65 @@ void MPPTunnelSetBase::write(mpp::MPPDataPacket & packet, int16_t partit tunnels[partition_id]->write(packet); } +template +void MPPTunnelSetBase::writeError(const String & msg) +{ + for (auto & tunnel : tunnels) + { + try + { + FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::exception_during_mpp_write_err_to_tunnel); + tunnel->write(getPacketWithError(msg), true); + } + catch (...) + { + tunnel->close("Failed to write error msg to tunnel"); + tryLogCurrentException(log, "Failed to write error " + msg + " to tunnel: " + tunnel->id()); + } + } +} + +template +void MPPTunnelSetBase::registerTunnel(const MPPTaskId & receiver_task_id, const TunnelPtr & tunnel) +{ + if (receiver_task_id_to_index_map.find(receiver_task_id) != receiver_task_id_to_index_map.end()) + throw Exception(fmt::format("the tunnel {} has been registered", tunnel->id())); + + receiver_task_id_to_index_map[receiver_task_id] = tunnels.size(); + tunnels.push_back(tunnel); + if (!tunnel->isLocal()) + { + remote_tunnel_cnt++; + } +} + +template +void MPPTunnelSetBase::close(const String & reason) +{ + for (auto & tunnel : tunnels) + tunnel->close(reason); +} + +template +void MPPTunnelSetBase::finishWrite() +{ + for (auto & tunnel : tunnels) + { + tunnel->writeDone(); + } +} + +template +typename MPPTunnelSetBase::TunnelPtr MPPTunnelSetBase::getTunnelByReceiverTaskId(const MPPTaskId & id) +{ + auto it = receiver_task_id_to_index_map.find(id); + if (it == receiver_task_id_to_index_map.end()) + { + return nullptr; + } + return tunnels[it->second]; +} + /// Explicit template instantiations - to avoid code bloat in headers. template class MPPTunnelSetBase; diff --git a/dbms/src/Flash/Mpp/MPPTunnelSet.h b/dbms/src/Flash/Mpp/MPPTunnelSet.h index f2279b945cb..e4123db1be5 100644 --- a/dbms/src/Flash/Mpp/MPPTunnelSet.h +++ b/dbms/src/Flash/Mpp/MPPTunnelSet.h @@ -14,6 +14,7 @@ #pragma once +#include #include #ifdef __clang__ #pragma clang diagnostic push @@ -32,6 +33,9 @@ class MPPTunnelSetBase : private boost::noncopyable { public: using TunnelPtr = std::shared_ptr; + explicit MPPTunnelSetBase(const String & req_id) + : log(Logger::get("MPPTunnelSet", req_id)) + {} void clearExecutionSummaries(tipb::SelectResponse & response); @@ -50,17 +54,14 @@ class MPPTunnelSetBase : private boost::noncopyable // this is a partition writing. void write(tipb::SelectResponse & response, int16_t partition_id); void write(mpp::MPPDataPacket & packet, int16_t partition_id); + void writeError(const String & msg); + void close(const String & reason); + void finishWrite(); + void registerTunnel(const MPPTaskId & id, const TunnelPtr & tunnel); - uint16_t getPartitionNum() const { return tunnels.size(); } + TunnelPtr getTunnelByReceiverTaskId(const MPPTaskId & id); - void addTunnel(const TunnelPtr & tunnel) - { - tunnels.push_back(tunnel); - if (!tunnel->isLocal()) - { - remote_tunnel_cnt++; - } - } + uint16_t getPartitionNum() const { return tunnels.size(); } int getRemoteTunnelCnt() { @@ -71,6 +72,8 @@ class MPPTunnelSetBase : private boost::noncopyable private: std::vector tunnels; + std::unordered_map receiver_task_id_to_index_map; + const LoggerPtr log; int remote_tunnel_cnt = 0; }; diff --git a/dbms/src/Flash/Mpp/MinTSOScheduler.h b/dbms/src/Flash/Mpp/MinTSOScheduler.h index 17ab1f4dfa3..dbc0cb84cc3 100644 --- a/dbms/src/Flash/Mpp/MinTSOScheduler.h +++ b/dbms/src/Flash/Mpp/MinTSOScheduler.h @@ -15,12 +15,19 @@ #pragma once #include -#include -#include #include namespace DB { +class MinTSOScheduler; +using MPPTaskSchedulerPtr = std::unique_ptr; + +class MPPTaskManager; +using MPPTaskManagerPtr = std::shared_ptr; + +struct MPPQueryTaskSet; +using MPPQueryTaskSetPtr = std::shared_ptr; + /// scheduling tasks in the set according to the tso order under the soft limit of threads, but allow the min_tso query to preempt threads under the hard limit of threads. /// The min_tso query avoids the deadlock resulted from threads competition among nodes. /// schedule tasks under the lock protection of the task manager. diff --git a/dbms/src/Flash/tests/gtest_executor.cpp b/dbms/src/Flash/tests/gtest_executor.cpp new file mode 100644 index 00000000000..64c60f14bb6 --- /dev/null +++ b/dbms/src/Flash/tests/gtest_executor.cpp @@ -0,0 +1,230 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +namespace DB +{ +namespace tests +{ +class ExecutorTestRunner : public DB::tests::ExecutorTest +{ +public: + void initializeContext() override + { + ExecutorTest::initializeContext(); + context.addMockTable({"test_db", "test_table"}, + {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}, + {toNullableVec("s1", {"banana", {}, "banana"}), + toNullableVec("s2", {"apple", {}, "banana"})}); + context.addExchangeReceiver("exchange1", + {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}, + {toNullableVec("s1", {"banana", {}, "banana"}), + toNullableVec("s2", {"apple", {}, "banana"})}); + + context.addExchangeReceiver("exchange_r_table", + {{"s1", TiDB::TP::TypeString}, {"join_c", TiDB::TP::TypeString}}, + {toNullableVec("s", {"banana", "banana"}), + toNullableVec("join_c", {"apple", "banana"})}); + + context.addExchangeReceiver("exchange_l_table", + {{"s1", TiDB::TP::TypeString}, {"join_c", TiDB::TP::TypeString}}, + {toNullableVec("s", {"banana", "banana"}), + toNullableVec("join_c", {"apple", "banana"})}); + + context.addMockTable({"test_db", "r_table"}, + {{"s", TiDB::TP::TypeString}, {"join_c", TiDB::TP::TypeString}}, + {toVec("s", {"banana", "banana"}), + toVec("join_c", {"apple", "banana"})}); + + context.addMockTable({"test_db", "r_table_2"}, + {{"s", TiDB::TP::TypeString}, {"join_c", TiDB::TP::TypeString}}, + {toVec("s", {"banana", "banana", "banana"}), + toVec("join_c", {"apple", "apple", "apple"})}); + + context.addMockTable({"test_db", "l_table"}, + {{"s", TiDB::TP::TypeString}, {"join_c", TiDB::TP::TypeString}}, + {toVec("s", {"banana", "banana"}), + toVec("join_c", {"apple", "banana"})}); + } +}; + +TEST_F(ExecutorTestRunner, Filter) +try +{ + auto request = context + .scan("test_db", "test_table") + .filter(eq(col("s1"), col("s2"))) + .build(context); + { + executeStreams(request, + {toNullableVec({"banana"}), + toNullableVec({"banana"})}); + } + + request = context.receive("exchange1") + .filter(eq(col("s1"), col("s2"))) + .build(context); + { + executeStreams(request, + {toNullableVec({"banana"}), + toNullableVec({"banana"})}); + } +} +CATCH + +TEST_F(ExecutorTestRunner, JoinWithTableScan) +try +{ + auto request = context + .scan("test_db", "l_table") + .join(context.scan("test_db", "r_table"), {col("join_c")}, ASTTableJoin::Kind::Left) + .topN("join_c", false, 2) + .build(context); + { + String expected = "topn_3 | order_by: {(<1, String>, desc: false)}, limit: 2\n" + " Join_2 | LeftOuterJoin, HashJoin. left_join_keys: {<0, String>}, right_join_keys: {<0, String>}\n" + " table_scan_0 | {<0, String>, <1, String>}\n" + " table_scan_1 | {<0, String>, <1, String>}\n"; + ASSERT_DAGREQUEST_EQAUL(expected, request); + executeStreams(request, + {toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"}), + toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"})}, + 2); + + executeStreams(request, + {toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"}), + toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"})}, + 5); + + executeStreams(request, + {toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"}), + toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"})}); + } + request = context + .scan("test_db", "l_table") + .join(context.scan("test_db", "r_table"), {col("join_c")}, ASTTableJoin::Kind::Left) + .project({"s", "join_c"}) + .topN("join_c", false, 2) + .build(context); + { + String expected = "topn_4 | order_by: {(<1, String>, desc: false)}, limit: 2\n" + " project_3 | {<0, String>, <1, String>}\n" + " Join_2 | LeftOuterJoin, HashJoin. left_join_keys: {<0, String>}, right_join_keys: {<0, String>}\n" + " table_scan_0 | {<0, String>, <1, String>}\n" + " table_scan_1 | {<0, String>, <1, String>}\n"; + ASSERT_DAGREQUEST_EQAUL(expected, request); + executeStreams(request, + {toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"})}, + 2); + } + + request = context + .scan("test_db", "l_table") + .join(context.scan("test_db", "r_table_2"), {col("join_c")}, ASTTableJoin::Kind::Left) + .topN("join_c", false, 4) + .build(context); + { + String expected = "topn_3 | order_by: {(<1, String>, desc: false)}, limit: 4\n" + " Join_2 | LeftOuterJoin, HashJoin. left_join_keys: {<0, String>}, right_join_keys: {<0, String>}\n" + " table_scan_0 | {<0, String>, <1, String>}\n" + " table_scan_1 | {<0, String>, <1, String>}\n"; + ASSERT_DAGREQUEST_EQAUL(expected, request); + executeStreams(request, + {toNullableVec({"banana", "banana", "banana", "banana"}), + toNullableVec({"apple", "apple", "apple", "banana"}), + toNullableVec({"banana", "banana", "banana", {}}), + toNullableVec({"apple", "apple", "apple", {}})}, + 2); + executeStreams(request, + {toNullableVec({"banana", "banana", "banana", "banana"}), + toNullableVec({"apple", "apple", "apple", "banana"}), + toNullableVec({"banana", "banana", "banana", {}}), + toNullableVec({"apple", "apple", "apple", {}})}, + 3); + } +} +CATCH + +TEST_F(ExecutorTestRunner, JoinWithExchangeReceiver) +try +{ + auto request = context + .receive("exchange_l_table") + .join(context.receive("exchange_r_table"), {col("join_c")}, ASTTableJoin::Kind::Left) + .topN("join_c", false, 2) + .build(context); + { + String expected = "topn_3 | order_by: {(<1, String>, desc: false)}, limit: 2\n" + " Join_2 | LeftOuterJoin, HashJoin. left_join_keys: {<0, String>}, right_join_keys: {<0, String>}\n" + " exchange_receiver_0 | type:PassThrough, {<0, String>, <1, String>}\n" + " exchange_receiver_1 | type:PassThrough, {<0, String>, <1, String>}\n"; + ASSERT_DAGREQUEST_EQAUL(expected, request); + executeStreams(request, + {toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"}), + toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"})}, + 2); + + executeStreams(request, + {toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"}), + toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"})}, + 5); + + executeStreams(request, + {toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"}), + toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"})}); + } +} +CATCH + +TEST_F(ExecutorTestRunner, JoinWithTableScanAndReceiver) +try +{ + auto request = context + .scan("test_db", "l_table") + .join(context.receive("exchange_r_table"), {col("join_c")}, ASTTableJoin::Kind::Left) + .topN("join_c", false, 2) + .build(context); + { + String expected = "topn_3 | order_by: {(<1, String>, desc: false)}, limit: 2\n" + " Join_2 | LeftOuterJoin, HashJoin. left_join_keys: {<0, String>}, right_join_keys: {<0, String>}\n" + " table_scan_0 | {<0, String>, <1, String>}\n" + " exchange_receiver_1 | type:PassThrough, {<0, String>, <1, String>}\n"; + ASSERT_DAGREQUEST_EQAUL(expected, request); + executeStreams(request, + {toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"}), + toNullableVec({"banana", "banana"}), + toNullableVec({"apple", "banana"})}, + 2); + } +} +CATCH + +} // namespace tests +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Flash/tests/gtest_interpreter.cpp b/dbms/src/Flash/tests/gtest_interpreter.cpp index aed9d9e90f9..ba7d8fd15ee 100644 --- a/dbms/src/Flash/tests/gtest_interpreter.cpp +++ b/dbms/src/Flash/tests/gtest_interpreter.cpp @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include namespace DB { namespace tests { -class InterpreterExecuteTest : public DB::tests::InterpreterTest +class InterpreterExecuteTest : public DB::tests::ExecutorTest { public: void initializeContext() override { - InterpreterTest::initializeContext(); + ExecutorTest::initializeContext(); context.addMockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}); context.addMockTable({"test_db", "test_table_1"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}); @@ -47,7 +47,7 @@ try .build(context); { String expected = R"( -Union: +Union: SharedQuery x 10: Expression: MergeSorting, limit = 10 @@ -72,7 +72,7 @@ Union: { String expected = R"( -Union: +Union: SharedQuery x 10: Limit, limit = 10 Union: @@ -100,7 +100,7 @@ try .build(context); { String expected = R"( -Union: +Union: Expression x 10: Expression: Expression: @@ -122,7 +122,7 @@ Union: .build(context); { String expected = R"( -Union: +Union: Expression x 10: Expression: Expression: @@ -147,7 +147,7 @@ Union: .build(context); { String expected = R"( -Union: +Union: Expression x 10: Expression: Expression: @@ -181,7 +181,7 @@ Union: .build(context); { String expected = R"( -Union: +Union: SharedQuery x 10: Limit, limit = 10 Union: @@ -244,7 +244,7 @@ CreatingSets HashJoinProbe: Expression: MockTableScan - Union: + Union: Expression x 10: Expression: HashJoinProbe: @@ -260,7 +260,7 @@ CreatingSets .build(context); { String expected = R"( -Union: +Union: Expression x 10: Expression: Expression: @@ -283,7 +283,7 @@ Union: .build(context); { String expected = R"( -Union: +Union: MockExchangeSender x 10 Expression: Expression: @@ -331,7 +331,7 @@ CreatingSets HashJoinProbe: Expression: MockExchangeReceiver - Union: + Union: Expression x 10: Expression: HashJoinProbe: @@ -373,7 +373,7 @@ CreatingSets HashJoinProbe: Expression: MockExchangeReceiver - Union: + Union: MockExchangeSender x 10 Expression: Expression: @@ -385,5 +385,85 @@ CreatingSets } CATCH +TEST_F(InterpreterExecuteTest, Window) +try +{ + auto request = context + .scan("test_db", "test_table") + .sort({{"s1", true}, {"s2", false}}, true) + .window(RowNumber(), {"s1", true}, {"s2", false}, buildDefaultRowsFrame()) + .build(context); + { + String expected = R"( +Union: + Expression x 10: + SharedQuery: + Expression: + Window, function: {row_number}, frame: {type: Rows, boundary_begin: Current, boundary_end: Current} + Expression: + MergeSorting, limit = 0 + Union: + PartialSorting x 10: limit = 0 + Expression: + MockTableScan)"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); + } + + request = context.scan("test_db", "test_table") + .sort({{"s1", true}, {"s2", false}}, true) + .window(RowNumber(), {"s1", true}, {"s2", false}, buildDefaultRowsFrame()) + .project({"s1", "s2", "RowNumber()"}) + .build(context); + { + String expected = R"( +Union: + Expression x 10: + Expression: + Expression: + Expression: + SharedQuery: + Expression: + Window, function: {row_number}, frame: {type: Rows, boundary_begin: Current, boundary_end: Current} + Expression: + MergeSorting, limit = 0 + Union: + PartialSorting x 10: limit = 0 + Expression: + MockTableScan)"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); + } + + request = context.scan("test_db", "test_table_1") + .sort({{"s1", true}, {"s2", false}}, true) + .project({"s1", "s2", "s3"}) + .window(RowNumber(), {"s1", true}, {"s1", false}, buildDefaultRowsFrame()) + .project({"s1", "s2", "s3", "RowNumber()"}) + .build(context); + { + String expected = R"( +Union: + Expression x 10: + Expression: + Expression: + Expression: + SharedQuery: + Expression: + Window, function: {row_number}, frame: {type: Rows, boundary_begin: Current, boundary_end: Current} + Union: + Expression x 10: + Expression: + Expression: + SharedQuery: + Expression: + MergeSorting, limit = 0 + Union: + PartialSorting x 10: limit = 0 + Expression: + MockTableScan)"; + ASSERT_BLOCKINPUTSTREAM_EQAUL(expected, request, 10); + } +} +CATCH + } // namespace tests } // namespace DB \ No newline at end of file diff --git a/dbms/src/Functions/tests/gtest_arithmetic_functions.cpp b/dbms/src/Functions/tests/gtest_arithmetic_functions.cpp index 1a542c908ee..1d548a4c2d2 100644 --- a/dbms/src/Functions/tests/gtest_arithmetic_functions.cpp +++ b/dbms/src/Functions/tests/gtest_arithmetic_functions.cpp @@ -128,6 +128,10 @@ try null_or_zero_field.push_back(Field(DecimalField(0, 0))); std::vector values{10, 2, 20, 8, 10, 0, 30, 8, 16, 4}; + /// The precision of a non-zero DecimalField must not less than minDecimalPrecision + /// the decimal_128_factor is used to make sure the precision is big enough + Int128 decimal_128_factor = 10000000000; + decimal_128_factor *= 1000000000; const size_t size = 10; @@ -172,7 +176,7 @@ try if (col2_type->onlyNull()) continue; auto c1 = nullable_decimal_type_1->createColumnConst(size, Null()); - auto c2 = col2_type->createColumnConst(size, Field(DecimalField(2, 0))); + auto c2 = col2_type->createColumnConst(size, Field(DecimalField(2 * decimal_128_factor, 0))); auto col1 = ColumnWithTypeAndName(std::move(c1), nullable_decimal_type_1, "col1"); auto col2 = ColumnWithTypeAndName(std::move(c2), col2_type, "col2"); auto result = executeFunction(func_name, {col1, col2}); @@ -193,7 +197,7 @@ try continue; if (!col2_value.isNull() && col2_type->onlyNull()) continue; - auto c1 = col1_type->createColumnConst(size, Field(DecimalField(100, 2))); + auto c1 = col1_type->createColumnConst(size, Field(DecimalField(100 * decimal_128_factor, 2))); auto c2 = col2_type->createColumnConst(size, col2_value); auto col1 = ColumnWithTypeAndName(std::move(c1), col1_type, "col1"); auto col2 = ColumnWithTypeAndName(std::move(c2), col2_type, "col2"); @@ -211,11 +215,11 @@ try { if (col1_type->onlyNull() || col2_type->onlyNull()) continue; - auto c1 = col1_type->createColumnConst(size, Field(DecimalField(1000, 2))); - auto c2 = col2_type->createColumnConst(size, Field(DecimalField(2, 0))); + auto c1 = col1_type->createColumnConst(size, Field(DecimalField(1000 * decimal_128_factor, 2))); + auto c2 = col2_type->createColumnConst(size, Field(DecimalField(2 * decimal_128_factor, 0))); auto col1 = ColumnWithTypeAndName(std::move(c1), col1_type, "col1"); auto col2 = ColumnWithTypeAndName(std::move(c2), col2_type, "col2"); - auto res_col = executeFunction(func_name, {col1, col2}).column; + auto res_col = executeFunction(func_name, {col1, col2}, nullptr, false).column; ASSERT_TRUE(size == res_col->size()); Field res_field; for (size_t i = 0; i < size; i++) @@ -246,7 +250,7 @@ try if (col1_type->isNullable() && col1_null_map[i]) c1_mutable->insert(Null()); else - c1_mutable->insert(Field(DecimalField(values[i], 2))); + c1_mutable->insert(Field(DecimalField(values[i] * decimal_128_factor, 2))); } auto c2 = col2_type->createColumnConst(values.size(), col2_value); @@ -271,9 +275,9 @@ try if (col1_type->isNullable() && col1_null_map[i]) c1_mutable->insert(Null()); else - c1_mutable->insert(Field(DecimalField(values[i], 2))); + c1_mutable->insert(Field(DecimalField(values[i] * decimal_128_factor, 2))); } - auto c2 = col2_type->createColumnConst(values.size(), Field(DecimalField(2, 0))); + auto c2 = col2_type->createColumnConst(values.size(), Field(DecimalField(2 * decimal_128_factor, 0))); auto col1 = ColumnWithTypeAndName(std::move(c1_mutable), col1_type, "col1"); auto col2 = ColumnWithTypeAndName(std::move(c2), col2_type, "col2"); @@ -312,7 +316,7 @@ try if (col2_type->isNullable() && col2_null_map[i]) c2->insert(Null()); else - c2->insert(Field(DecimalField(values[i], 2))); + c2->insert(Field(DecimalField(values[i] * decimal_128_factor, 2))); } auto col1 = ColumnWithTypeAndName(std::move(c1), col1_type, "col1"); auto col2 = ColumnWithTypeAndName(std::move(c2), col2_type, "col2"); @@ -334,14 +338,14 @@ try if (values[i] != 0) value *= values[i]; } - auto c1 = col1_type->createColumnConst(size, Field(DecimalField(value, 2))); + auto c1 = col1_type->createColumnConst(size, Field(DecimalField(value * decimal_128_factor, 2))); auto c2 = col2_type->createColumn(); for (size_t i = 0; i < values.size(); i++) { if (col2_type->isNullable() && col2_null_map[i]) c2->insert(Null()); else - c2->insert(Field(DecimalField(values[i], 0))); + c2->insert(Field(DecimalField(values[i] * decimal_128_factor, 0))); } auto col1 = ColumnWithTypeAndName(std::move(c1), col1_type, "col1"); auto col2 = ColumnWithTypeAndName(std::move(c2), col2_type, "col2"); @@ -377,11 +381,11 @@ try if (col1_type->isNullable() && col1_null_map[i]) c1->insert(Null()); else - c1->insert(Field(DecimalField(values[i], 2))); + c1->insert(Field(DecimalField(values[i] * decimal_128_factor, 2))); if (col2_type->isNullable() && col2_null_map[i]) c2->insert(Null()); else - c2->insert(Field(DecimalField(values[i], 0))); + c2->insert(Field(DecimalField(values[i] * decimal_128_factor, 0))); } auto col1 = ColumnWithTypeAndName(std::move(c1), col1_type, "col1"); auto col2 = ColumnWithTypeAndName(std::move(c2), col2_type, "col2"); diff --git a/dbms/src/Functions/tests/gtest_bitand.cpp b/dbms/src/Functions/tests/gtest_bitand.cpp index b77bc5e8547..88c70847d98 100644 --- a/dbms/src/Functions/tests/gtest_bitand.cpp +++ b/dbms/src/Functions/tests/gtest_bitand.cpp @@ -40,7 +40,7 @@ class TestFunctionBitAnd : public DB::tests::FunctionTest TEST_F(TestFunctionBitAnd, Simple) try { - ASSERT_BITAND(createColumn>({-1, 1}), createColumn>({0, 0}), createColumn>({0, 0})); + ASSERT_BITAND(createColumn>({-1, 1}), createColumn>({0, 0}), createColumn>({0, 0})); } CATCH @@ -49,21 +49,21 @@ TEST_F(TestFunctionBitAnd, TypePromotion) try { // Type Promotion - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); // Type Promotion across signed/unsigned - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn>({0}), createColumn>({0})); } CATCH @@ -71,51 +71,51 @@ TEST_F(TestFunctionBitAnd, Nullable) try { // Non Nullable - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); - ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); + ASSERT_BITAND(createColumn({1}), createColumn({0}), createColumn({0})); // Across Nullable and non-Nullable - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); + ASSERT_BITAND(createColumn({1}), createColumn>({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); - ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); + ASSERT_BITAND(createColumn>({1}), createColumn({0}), createColumn>({0})); } CATCH @@ -129,48 +129,48 @@ try /// 4. ColumnConst, value != null /// 5. ColumnConst, value = null - ASSERT_BITAND(createColumn({0, 0, 1, 1}), createColumn({0, 1, 0, 1}), createColumn({0, 0, 0, 1})); - ASSERT_BITAND(createColumn({0, 0, 1, 1}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 0, std::nullopt, std::nullopt})); - ASSERT_BITAND(createColumn({0, 0, 1, 1}), createConstColumn(4, 0), createColumn({0, 0, 0, 0})); - ASSERT_BITAND(createColumn({0, 0, 1, 1}), createConstColumn>(4, 0), createColumn>({0, 0, 0, 0})); - ASSERT_BITAND(createColumn({0, 0, 1, 1}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); // become const in wrapInNullable - - ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn({0, 1, 0, 1}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn(4, 0), createColumn>({0, 0, std::nullopt, std::nullopt})); - ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, 0), createColumn>({0, 0, std::nullopt, std::nullopt})); - ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); - - ASSERT_BITAND(createConstColumn(4, 0), createColumn({0, 1, 0, 1}), createColumn({0, 0, 0, 0})); - ASSERT_BITAND(createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 0, std::nullopt, std::nullopt})); - ASSERT_BITAND(createConstColumn(4, 0), createConstColumn(4, 0), createConstColumn(4, 0)); - ASSERT_BITAND(createConstColumn(4, 0), createConstColumn>(4, 0), createConstColumn>(4, 0)); - ASSERT_BITAND(createConstColumn(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); - - ASSERT_BITAND(createConstColumn>(4, 0), createColumn({0, 1, 0, 1}), createColumn>({0, 0, 0, 0})); - ASSERT_BITAND(createConstColumn>(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 0, std::nullopt, std::nullopt})); - ASSERT_BITAND(createConstColumn>(4, 0), createConstColumn(4, 0), createConstColumn>(4, 0)); - ASSERT_BITAND(createConstColumn>(4, 0), createConstColumn>(4, 0), createConstColumn>(4, 0)); - ASSERT_BITAND(createConstColumn>(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); - - ASSERT_BITAND(createConstColumn>(4, std::nullopt), createColumn({0, 1, 0, 1}), createConstColumn>(4, std::nullopt)); - ASSERT_BITAND(createConstColumn>(4, std::nullopt), createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt)); - ASSERT_BITAND(createConstColumn>(4, std::nullopt), createConstColumn(4, 0), createConstColumn>(4, std::nullopt)); - ASSERT_BITAND(createConstColumn>(4, std::nullopt), createConstColumn>(4, 0), createConstColumn>(4, std::nullopt)); - ASSERT_BITAND(createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + ASSERT_BITAND(createColumn({0, 0, 1, 1}), createColumn({0, 1, 0, 1}), createColumn({0, 0, 0, 1})); + ASSERT_BITAND(createColumn({0, 0, 1, 1}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 0, std::nullopt, std::nullopt})); + ASSERT_BITAND(createColumn({0, 0, 1, 1}), createConstColumn(4, 0), createColumn({0, 0, 0, 0})); + ASSERT_BITAND(createColumn({0, 0, 1, 1}), createConstColumn>(4, 0), createColumn({0, 0, 0, 0})); + ASSERT_BITAND(createColumn({0, 0, 1, 1}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); // become const in wrapInNullable + + ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn({0, 1, 0, 1}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn(4, 0), createColumn>({0, 0, std::nullopt, std::nullopt})); + ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, 0), createColumn>({0, 0, std::nullopt, std::nullopt})); + ASSERT_BITAND(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + + ASSERT_BITAND(createConstColumn(4, 0), createColumn({0, 1, 0, 1}), createColumn({0, 0, 0, 0})); + ASSERT_BITAND(createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 0, std::nullopt, std::nullopt})); + ASSERT_BITAND(createConstColumn(4, 0), createConstColumn(4, 0), createConstColumn(4, 0)); + ASSERT_BITAND(createConstColumn(4, 0), createConstColumn>(4, 0), createConstColumn(4, 0)); + ASSERT_BITAND(createConstColumn(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + + ASSERT_BITAND(createConstColumn>(4, 0), createColumn({0, 1, 0, 1}), createColumn({0, 0, 0, 0})); + ASSERT_BITAND(createConstColumn>(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 0, std::nullopt, std::nullopt})); + ASSERT_BITAND(createConstColumn>(4, 0), createConstColumn(4, 0), createConstColumn(4, 0)); + ASSERT_BITAND(createConstColumn>(4, 0), createConstColumn>(4, 0), createConstColumn(4, 0)); + ASSERT_BITAND(createConstColumn>(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + + ASSERT_BITAND(createConstColumn>(4, std::nullopt), createColumn({0, 1, 0, 1}), createConstColumn>(4, std::nullopt)); + ASSERT_BITAND(createConstColumn>(4, std::nullopt), createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt)); + ASSERT_BITAND(createConstColumn>(4, std::nullopt), createConstColumn(4, 0), createConstColumn>(4, std::nullopt)); + ASSERT_BITAND(createConstColumn>(4, std::nullopt), createConstColumn(4, 0), createConstColumn>(4, std::nullopt)); + ASSERT_BITAND(createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); } CATCH TEST_F(TestFunctionBitAnd, Boundary) try { - ASSERT_BITAND(createColumn({127, 127, -128, -128}), createColumn({0, 255, 0, 255}), createColumn({0, 127, 0, -128})); - ASSERT_BITAND(createColumn({127, 127, -128, -128}), createColumn({0, 65535, 0, 65535}), createColumn({0, 127, 0, -128})); - ASSERT_BITAND(createColumn({32767, 32767, -32768, -32768}), createColumn({0, 255, 0, 255}), createColumn({0, 255, 0, 0})); + ASSERT_BITAND(createColumn({127, 127, -128, -128}), createColumn({0, 255, 0, 255}), createColumn({0, 127, 0, 128})); + ASSERT_BITAND(createColumn({127, 127, -128, -128}), createColumn({0, 65535, 0, 65535}), createColumn({0, 127, 0, 65408})); + ASSERT_BITAND(createColumn({32767, 32767, -32768, -32768}), createColumn({0, 255, 0, 255}), createColumn({0, 255, 0, 0})); ASSERT_BITAND(createColumn({0, 0, 1, 1, -1, -1, INT64_MAX, INT64_MAX, INT64_MIN, INT64_MIN}), createColumn({0, UINT64_MAX, 0, UINT64_MAX, 0, UINT64_MAX, 0, UINT64_MAX, 0, UINT64_MAX}), - createColumn({0, 0, 0, 1, 0, -1, 0, INT64_MAX, 0, INT64_MIN})); + createColumn({0, 0, 0, 1, 0, UINT64_MAX, 0, INT64_MAX, 0, 9223372036854775808ull})); } CATCH diff --git a/dbms/src/Functions/tests/gtest_bitnot.cpp b/dbms/src/Functions/tests/gtest_bitnot.cpp index 6b96e59718e..0b542d87961 100644 --- a/dbms/src/Functions/tests/gtest_bitnot.cpp +++ b/dbms/src/Functions/tests/gtest_bitnot.cpp @@ -41,7 +41,7 @@ class TestFunctionBitNot : public DB::tests::FunctionTest TEST_F(TestFunctionBitNot, Simple) try { - ASSERT_BITNOT(createColumn>({-1, 1}), createColumn>({0, -2})); + ASSERT_BITNOT(createColumn>({-1, 1}), createColumn>({0, UINT64_MAX - 1})); } CATCH @@ -49,21 +49,21 @@ CATCH TEST_F(TestFunctionBitNot, TypeTest) try { - ASSERT_BITNOT(createColumn>({1}), createColumn>({-2})); - ASSERT_BITNOT(createColumn>({1}), createColumn>({-2})); - ASSERT_BITNOT(createColumn>({1}), createColumn>({-2})); - ASSERT_BITNOT(createColumn>({1}), createColumn>({-2})); - - ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT8_MAX - 1})); - ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT16_MAX - 1})); - ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT32_MAX - 1})); + ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT64_MAX - 1})); + ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT64_MAX - 1})); + ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT64_MAX - 1})); + ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT64_MAX - 1})); + + ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT64_MAX - 1})); + ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT64_MAX - 1})); + ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT64_MAX - 1})); ASSERT_BITNOT(createColumn>({1}), createColumn>({UINT64_MAX - 1})); - ASSERT_BITNOT(createColumn({0, 0, 1, 1}), createColumn({-1, -1, -2, -2})); - ASSERT_BITNOT(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({-1, -2, std::nullopt, std::nullopt})); - ASSERT_BITNOT(createConstColumn(4, 0), createConstColumn(4, -1)); - ASSERT_BITNOT(createConstColumn>(4, 0), createConstColumn>(4, -1)); - ASSERT_BITNOT(createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + ASSERT_BITNOT(createColumn({0, 0, 1, 1}), createColumn({UINT64_MAX, UINT64_MAX, UINT64_MAX - 1, UINT64_MAX - 1})); + ASSERT_BITNOT(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({UINT64_MAX, UINT64_MAX - 1, std::nullopt, std::nullopt})); + ASSERT_BITNOT(createConstColumn(4, 0), createConstColumn(4, UINT64_MAX)); + ASSERT_BITNOT(createConstColumn>(4, 0), createConstColumn(4, UINT64_MAX)); + ASSERT_BITNOT(createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); } CATCH @@ -71,7 +71,7 @@ TEST_F(TestFunctionBitNot, Boundary) try { ASSERT_BITNOT(createColumn({0, 1, -1, INT64_MAX, INT64_MIN}), - createColumn({-1, -2, 0, INT64_MIN, INT64_MAX})); + createColumn({UINT64_MAX, UINT64_MAX - 1, 0, static_cast(INT64_MAX) + 1, INT64_MAX})); ASSERT_BITNOT(createColumn({0, 1, UINT64_MAX}), createColumn({UINT64_MAX, UINT64_MAX - 1, 0})); diff --git a/dbms/src/Functions/tests/gtest_bitor.cpp b/dbms/src/Functions/tests/gtest_bitor.cpp index a1611659913..fb4794c31b2 100644 --- a/dbms/src/Functions/tests/gtest_bitor.cpp +++ b/dbms/src/Functions/tests/gtest_bitor.cpp @@ -40,7 +40,7 @@ class TestFunctionBitOr : public DB::tests::FunctionTest TEST_F(TestFunctionBitOr, Simple) try { - ASSERT_BITOR(createColumn>({-1, 1}), createColumn>({0, 0}), createColumn>({-1, 1})); + ASSERT_BITOR(createColumn>({-1, 1}), createColumn>({0, 0}), createColumn>({UINT64_MAX, 1})); } CATCH @@ -49,21 +49,21 @@ TEST_F(TestFunctionBitOr, TypePromotion) try { // Type Promotion - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); // Type Promotion across signed/unsigned - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); } CATCH @@ -71,51 +71,51 @@ TEST_F(TestFunctionBitOr, Nullable) try { // Non Nullable - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITOR(createColumn({1}), createColumn({0}), createColumn({1})); // Across Nullable and non-Nullable - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITOR(createColumn>({1}), createColumn({0}), createColumn>({1})); } CATCH @@ -129,48 +129,48 @@ try /// 4. ColumnConst, value != null /// 5. ColumnConst, value = null - ASSERT_BITOR(createColumn({0, 0, 1, 1}), createColumn({0, 1, 0, 1}), createColumn({0, 1, 1, 1})); - ASSERT_BITOR(createColumn({0, 0, 1, 1}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITOR(createColumn({0, 0, 1, 1}), createConstColumn(4, 0), createColumn({0, 0, 1, 1})); - ASSERT_BITOR(createColumn({0, 0, 1, 1}), createConstColumn>(4, 0), createColumn>({0, 0, 1, 1})); - ASSERT_BITOR(createColumn({0, 0, 1, 1}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); // become const in wrapInNullable - - ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn({0, 1, 0, 1}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); - - ASSERT_BITOR(createConstColumn(4, 0), createColumn({0, 1, 0, 1}), createColumn({0, 1, 0, 1})); - ASSERT_BITOR(createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITOR(createConstColumn(4, 0), createConstColumn(4, 0), createConstColumn(4, 0)); - ASSERT_BITOR(createConstColumn(4, 0), createConstColumn>(4, 0), createConstColumn>(4, 0)); - ASSERT_BITOR(createConstColumn(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); - - ASSERT_BITOR(createConstColumn>(4, 0), createColumn({0, 1, 0, 1}), createColumn>({0, 1, 0, 1})); - ASSERT_BITOR(createConstColumn>(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITOR(createConstColumn>(4, 0), createConstColumn(4, 0), createConstColumn>(4, 0)); - ASSERT_BITOR(createConstColumn>(4, 0), createConstColumn>(4, 0), createConstColumn>(4, 0)); - ASSERT_BITOR(createConstColumn>(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); - - ASSERT_BITOR(createConstColumn>(4, std::nullopt), createColumn({0, 1, 0, 1}), createConstColumn>(4, std::nullopt)); - ASSERT_BITOR(createConstColumn>(4, std::nullopt), createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt)); - ASSERT_BITOR(createConstColumn>(4, std::nullopt), createConstColumn(4, 0), createConstColumn>(4, std::nullopt)); - ASSERT_BITOR(createConstColumn>(4, std::nullopt), createConstColumn>(4, 0), createConstColumn>(4, std::nullopt)); - ASSERT_BITOR(createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + ASSERT_BITOR(createColumn({0, 0, 1, 1}), createColumn({0, 1, 0, 1}), createColumn({0, 1, 1, 1})); + ASSERT_BITOR(createColumn({0, 0, 1, 1}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITOR(createColumn({0, 0, 1, 1}), createConstColumn(4, 0), createColumn({0, 0, 1, 1})); + ASSERT_BITOR(createColumn({0, 0, 1, 1}), createConstColumn>(4, 0), createColumn({0, 0, 1, 1})); + ASSERT_BITOR(createColumn({0, 0, 1, 1}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); // become const in wrapInNullable + + ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn({0, 1, 0, 1}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + + ASSERT_BITOR(createConstColumn(4, 0), createColumn({0, 1, 0, 1}), createColumn({0, 1, 0, 1})); + ASSERT_BITOR(createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITOR(createConstColumn(4, 0), createConstColumn(4, 0), createConstColumn(4, 0)); + ASSERT_BITOR(createConstColumn(4, 0), createConstColumn>(4, 0), createConstColumn(4, 0)); + ASSERT_BITOR(createConstColumn(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + + ASSERT_BITOR(createConstColumn>(4, 0), createColumn({0, 1, 0, 1}), createColumn({0, 1, 0, 1})); + ASSERT_BITOR(createConstColumn>(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITOR(createConstColumn>(4, 0), createConstColumn(4, 0), createConstColumn(4, 0)); + ASSERT_BITOR(createConstColumn>(4, 0), createConstColumn>(4, 0), createConstColumn(4, 0)); + ASSERT_BITOR(createConstColumn>(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + + ASSERT_BITOR(createConstColumn>(4, std::nullopt), createColumn({0, 1, 0, 1}), createConstColumn>(4, std::nullopt)); + ASSERT_BITOR(createConstColumn>(4, std::nullopt), createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt)); + ASSERT_BITOR(createConstColumn>(4, std::nullopt), createConstColumn(4, 0), createConstColumn>(4, std::nullopt)); + ASSERT_BITOR(createConstColumn>(4, std::nullopt), createConstColumn(4, 0), createConstColumn>(4, std::nullopt)); + ASSERT_BITOR(createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); } CATCH TEST_F(TestFunctionBitOr, Boundary) try { - ASSERT_BITOR(createColumn({127, 127, -128, -128}), createColumn({0, 255, 0, 255}), createColumn({127, -1, -128, -1})); - ASSERT_BITOR(createColumn({127, 127, -128, -128}), createColumn({0, 65535, 0, 65535}), createColumn({127, -1, -128, -1})); - ASSERT_BITOR(createColumn({32767, 32767, -32768, -32768}), createColumn({0, 255, 0, 255}), createColumn({32767, 32767, -32768, -32513})); + ASSERT_BITOR(createColumn({127, 127, -128, -128}), createColumn({0, 255, 0, 255}), createColumn({127, 255, 18446744073709551488ull, UINT64_MAX})); + ASSERT_BITOR(createColumn({127, 127, -128, -128}), createColumn({0, 65535, 0, 65535}), createColumn({127, 65535, 18446744073709551488ull, UINT64_MAX})); + ASSERT_BITOR(createColumn({32767, 32767, -32768, -32768}), createColumn({0, 255, 0, 255}), createColumn({32767, 32767, 18446744073709518848ull, 18446744073709519103ull})); ASSERT_BITOR(createColumn({0, 0, 1, 1, -1, -1, INT64_MAX, INT64_MAX, INT64_MIN, INT64_MIN}), createColumn({0, UINT64_MAX, 0, UINT64_MAX, 0, UINT64_MAX, 0, UINT64_MAX, 0, UINT64_MAX}), - createColumn({0, -1, 1, -1, -1, -1, INT64_MAX, -1, INT64_MIN, -1})); + createColumn({0, UINT64_MAX, 1, UINT64_MAX, UINT64_MAX, UINT64_MAX, INT64_MAX, UINT64_MAX, 9223372036854775808ull, UINT64_MAX})); } CATCH diff --git a/dbms/src/Functions/tests/gtest_bitxor.cpp b/dbms/src/Functions/tests/gtest_bitxor.cpp index 60f3027d2b5..85cc51a8a49 100644 --- a/dbms/src/Functions/tests/gtest_bitxor.cpp +++ b/dbms/src/Functions/tests/gtest_bitxor.cpp @@ -40,7 +40,7 @@ class TestFunctionBitXor : public DB::tests::FunctionTest TEST_F(TestFunctionBitXor, Simple) try { - ASSERT_BITXOR(createColumn>({-1, 1}), createColumn>({0, 0}), createColumn>({-1, 1})); + ASSERT_BITXOR(createColumn>({-1, 1}), createColumn>({0, 0}), createColumn>({UINT64_MAX, 1})); } CATCH @@ -49,21 +49,21 @@ TEST_F(TestFunctionBitXor, TypePromotion) try { // Type Promotion - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); // Type Promotion across signed/unsigned - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn>({0}), createColumn>({1})); } CATCH @@ -71,51 +71,51 @@ TEST_F(TestFunctionBitXor, Nullable) try { // Non Nullable - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); - ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); + ASSERT_BITXOR(createColumn({1}), createColumn({0}), createColumn({1})); // Across Nullable and non-Nullable - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn({1}), createColumn>({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); - ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); + ASSERT_BITXOR(createColumn>({1}), createColumn({0}), createColumn>({1})); } CATCH @@ -129,48 +129,48 @@ try /// 4. ColumnConst, value != null /// 5. ColumnConst, value = null - ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createColumn({0, 1, 0, 1}), createColumn({0, 1, 1, 0})); - ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createConstColumn(4, 0), createColumn({0, 0, 1, 1})); - ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createConstColumn>(4, 0), createColumn>({0, 0, 1, 1})); - ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); // become const in wrapInNullable - - ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn({0, 1, 0, 1}), createColumn>({0, 0, std::nullopt, std::nullopt})); - ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 0, std::nullopt, std::nullopt})); - ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); - - ASSERT_BITXOR(createConstColumn(4, 0), createColumn({0, 1, 0, 1}), createColumn({0, 1, 0, 1})); - ASSERT_BITXOR(createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITXOR(createConstColumn(4, 0), createConstColumn(4, 0), createConstColumn(4, 0)); - ASSERT_BITXOR(createConstColumn(4, 0), createConstColumn>(4, 0), createConstColumn>(4, 0)); - ASSERT_BITXOR(createConstColumn(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); - - ASSERT_BITXOR(createConstColumn>(4, 0), createColumn({0, 1, 0, 1}), createColumn>({0, 1, 0, 1})); - ASSERT_BITXOR(createConstColumn>(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); - ASSERT_BITXOR(createConstColumn>(4, 0), createConstColumn(4, 0), createConstColumn>(4, 0)); - ASSERT_BITXOR(createConstColumn>(4, 0), createConstColumn>(4, 0), createConstColumn>(4, 0)); - ASSERT_BITXOR(createConstColumn>(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); - - ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createColumn({0, 1, 0, 1}), createConstColumn>(4, std::nullopt)); - ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt)); - ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createConstColumn(4, 0), createConstColumn>(4, std::nullopt)); - ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createConstColumn>(4, 0), createConstColumn>(4, std::nullopt)); - ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createColumn({0, 1, 0, 1}), createColumn({0, 1, 1, 0})); + ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createConstColumn(4, 0), createColumn({0, 0, 1, 1})); + ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createConstColumn>(4, 0), createColumn({0, 0, 1, 1})); + ASSERT_BITXOR(createColumn({0, 0, 1, 1}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); // become const in wrapInNullable + + ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn({0, 1, 0, 1}), createColumn>({0, 0, std::nullopt, std::nullopt})); + ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 0, std::nullopt, std::nullopt})); + ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITXOR(createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + + ASSERT_BITXOR(createConstColumn(4, 0), createColumn({0, 1, 0, 1}), createColumn({0, 1, 0, 1})); + ASSERT_BITXOR(createConstColumn(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITXOR(createConstColumn(4, 0), createConstColumn(4, 0), createConstColumn(4, 0)); + ASSERT_BITXOR(createConstColumn(4, 0), createConstColumn>(4, 0), createConstColumn(4, 0)); + ASSERT_BITXOR(createConstColumn(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + + ASSERT_BITXOR(createConstColumn>(4, 0), createColumn({0, 1, 0, 1}), createColumn({0, 1, 0, 1})); + ASSERT_BITXOR(createConstColumn>(4, 0), createColumn>({0, 1, std::nullopt, std::nullopt}), createColumn>({0, 1, std::nullopt, std::nullopt})); + ASSERT_BITXOR(createConstColumn>(4, 0), createConstColumn(4, 0), createConstColumn(4, 0)); + ASSERT_BITXOR(createConstColumn>(4, 0), createConstColumn>(4, 0), createConstColumn(4, 0)); + ASSERT_BITXOR(createConstColumn>(4, 0), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); + + ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createColumn({0, 1, 0, 1}), createConstColumn>(4, std::nullopt)); + ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createColumn>({0, 1, std::nullopt, std::nullopt}), createConstColumn>(4, std::nullopt)); + ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createConstColumn(4, 0), createConstColumn>(4, std::nullopt)); + ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createConstColumn(4, 0), createConstColumn>(4, std::nullopt)); + ASSERT_BITXOR(createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt), createConstColumn>(4, std::nullopt)); } CATCH TEST_F(TestFunctionBitXor, Boundary) try { - ASSERT_BITXOR(createColumn({127, 127, -128, -128}), createColumn({0, 255, 0, 255}), createColumn({127, -128, -128, 127})); - ASSERT_BITXOR(createColumn({127, 127, -128, -128}), createColumn({0, 65535, 0, 65535}), createColumn({127, -128, -128, 127})); - ASSERT_BITXOR(createColumn({32767, 32767, -32768, -32768}), createColumn({0, 255, 0, 255}), createColumn({32767, 32512, -32768, -32513})); + ASSERT_BITXOR(createColumn({127, 127, -128, -128}), createColumn({0, 255, 0, 255}), createColumn({127, 128, 18446744073709551488ull, 18446744073709551487ull})); + ASSERT_BITXOR(createColumn({127, 127, -128, -128}), createColumn({0, 65535, 0, 65535}), createColumn({127, 65408, 18446744073709551488ull, 18446744073709486207ull})); + ASSERT_BITXOR(createColumn({32767, 32767, -32768, -32768}), createColumn({0, 255, 0, 255}), createColumn({32767, 32512, 18446744073709518848ull, 18446744073709519103ull})); ASSERT_BITXOR(createColumn({0, 0, 1, 1, -1, -1, INT64_MAX, INT64_MAX, INT64_MIN, INT64_MIN}), createColumn({0, UINT64_MAX, 0, UINT64_MAX, 0, UINT64_MAX, 0, UINT64_MAX, 0, UINT64_MAX}), - createColumn({0, -1, 1, -2, -1, 0, INT64_MAX, INT64_MIN, INT64_MIN, INT64_MAX})); + createColumn({0, UINT64_MAX, 1, UINT64_MAX - 1, UINT64_MAX, 0, INT64_MAX, 9223372036854775808ull, 9223372036854775808ull, INT64_MAX})); } CATCH diff --git a/dbms/src/Functions/tests/gtest_date_add.cpp b/dbms/src/Functions/tests/gtest_date_add.cpp index 19effb11fb6..d87209c0805 100644 --- a/dbms/src/Functions/tests/gtest_date_add.cpp +++ b/dbms/src/Functions/tests/gtest_date_add.cpp @@ -114,58 +114,58 @@ TEST_F(Dateadd, dateAddStringRealUnitTest) { ASSERT_COLUMN_EQ( toNullableVec({"2012-12-14", "2012-12-14 12:12:12", "2012-12-14", "2012-12-14 12:12:12"}), - executeFunction("addDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.6)))); + executeFunction("addDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-13", "2012-12-13 12:12:12", "2012-12-13", "2012-12-13 12:12:12"}), - executeFunction("addDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.4)))); + executeFunction("addDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-14", "2012-12-14 12:12:12", "2012-12-14", "2012-12-14 12:12:12"}), - executeFunction("addDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toFloatVec({1.6, 1.6, 1.6, 1.6})))); + executeFunction("addDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toFloatVec({1.6, 1.6, 1.6, 1.6})))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-13", "2012-12-13 12:12:12", "2012-12-13", "2012-12-13 12:12:12"}), - executeFunction("addDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toFloatVec({1.4, 1.4, 1.4, 1.4})))); + executeFunction("addDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toFloatVec({1.4, 1.4, 1.4, 1.4})))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-26", "2012-12-26 12:12:12", "2012-12-26", "2012-12-26 12:12:12"}), - executeFunction("addWeeks", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.6)))); + executeFunction("addWeeks", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-19", "2012-12-19 12:12:12", "2012-12-19", "2012-12-19 12:12:12"}), - executeFunction("addWeeks", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.4)))); + executeFunction("addWeeks", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2013-02-12", "2013-02-12 12:12:12", "2013-02-12", "2013-02-12 12:12:12"}), - executeFunction("addMonths", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.6)))); + executeFunction("addMonths", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2013-01-12", "2013-01-12 12:12:12", "2013-01-12", "2013-01-12 12:12:12"}), - executeFunction("addMonths", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.4)))); + executeFunction("addMonths", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2014-12-12", "2014-12-12 12:12:12", "2014-12-12", "2014-12-12 12:12:12"}), - executeFunction("addYears", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.6)))); + executeFunction("addYears", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2013-12-12", "2013-12-12 12:12:12", "2013-12-12", "2013-12-12 12:12:12"}), - executeFunction("addYears", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.4)))); + executeFunction("addYears", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 02:00:00", "2012-12-12 14:12:12", "2012-12-12 02:00:00", "2012-12-12 14:12:12"}), - executeFunction("addHours", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.6)))); + executeFunction("addHours", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 01:00:00", "2012-12-12 13:12:12", "2012-12-12 01:00:00", "2012-12-12 13:12:12"}), - executeFunction("addHours", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.4)))); + executeFunction("addHours", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 00:02:00", "2012-12-12 12:14:12", "2012-12-12 00:02:00", "2012-12-12 12:14:12"}), - executeFunction("addMinutes", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.6)))); + executeFunction("addMinutes", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 00:01:00", "2012-12-12 12:13:12", "2012-12-12 00:01:00", "2012-12-12 12:13:12"}), - executeFunction("addMinutes", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.4)))); + executeFunction("addMinutes", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 00:00:02", "2012-12-12 12:12:14", "2012-12-12 00:00:02", "2012-12-12 12:12:14"}), - executeFunction("addSeconds", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.6)))); + executeFunction("addSeconds", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 00:00:01", "2012-12-12 12:12:13", "2012-12-12 00:00:01", "2012-12-12 12:12:13"}), - executeFunction("addSeconds", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(1.4)))); + executeFunction("addSeconds", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(1.4)))); } } // namespace DB::tests diff --git a/dbms/src/Functions/tests/gtest_date_or_datetime_to_something.cpp b/dbms/src/Functions/tests/gtest_date_or_datetime_to_something.cpp index 115218e5550..206c13b1ef6 100644 --- a/dbms/src/Functions/tests/gtest_date_or_datetime_to_something.cpp +++ b/dbms/src/Functions/tests/gtest_date_or_datetime_to_something.cpp @@ -66,7 +66,7 @@ try MyDateTime(2020, 10, 10, 0, 0, 0, 0).toPackedUInt()) .column; input_col = ColumnWithTypeAndName(data_col_ptr, data_type_ptr, "input"); - output_col = createConstColumn>(4, 4); + output_col = createConstColumn(4, 4); ASSERT_COLUMN_EQ(output_col, executeFunction(func_name, input_col)); // ColumnConst(null) diff --git a/dbms/src/Functions/tests/gtest_date_sub.cpp b/dbms/src/Functions/tests/gtest_date_sub.cpp index ad6751bac09..ee45e4e212b 100644 --- a/dbms/src/Functions/tests/gtest_date_sub.cpp +++ b/dbms/src/Functions/tests/gtest_date_sub.cpp @@ -114,58 +114,58 @@ TEST_F(Datesub, dateSubStringRealUnitTest) { ASSERT_COLUMN_EQ( toNullableVec({"2012-12-14", "2012-12-14 12:12:12", "2012-12-14", "2012-12-14 12:12:12"}), - executeFunction("subtractDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.6)))); + executeFunction("subtractDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-13", "2012-12-13 12:12:12", "2012-12-13", "2012-12-13 12:12:12"}), - executeFunction("subtractDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.4)))); + executeFunction("subtractDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-14", "2012-12-14 12:12:12", "2012-12-14", "2012-12-14 12:12:12"}), - executeFunction("subtractDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toFloatVec({-1.6, -1.6, -1.6, -1.6})))); + executeFunction("subtractDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toFloatVec({-1.6, -1.6, -1.6, -1.6})))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-13", "2012-12-13 12:12:12", "2012-12-13", "2012-12-13 12:12:12"}), - executeFunction("subtractDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toFloatVec({-1.4, -1.4, -1.4, -1.4})))); + executeFunction("subtractDays", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toFloatVec({-1.4, -1.4, -1.4, -1.4})))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-26", "2012-12-26 12:12:12", "2012-12-26", "2012-12-26 12:12:12"}), - executeFunction("subtractWeeks", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.6)))); + executeFunction("subtractWeeks", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-19", "2012-12-19 12:12:12", "2012-12-19", "2012-12-19 12:12:12"}), - executeFunction("subtractWeeks", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.4)))); + executeFunction("subtractWeeks", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2013-02-12", "2013-02-12 12:12:12", "2013-02-12", "2013-02-12 12:12:12"}), - executeFunction("subtractMonths", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.6)))); + executeFunction("subtractMonths", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2013-01-12", "2013-01-12 12:12:12", "2013-01-12", "2013-01-12 12:12:12"}), - executeFunction("subtractMonths", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.4)))); + executeFunction("subtractMonths", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2014-12-12", "2014-12-12 12:12:12", "2014-12-12", "2014-12-12 12:12:12"}), - executeFunction("subtractYears", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.6)))); + executeFunction("subtractYears", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2013-12-12", "2013-12-12 12:12:12", "2013-12-12", "2013-12-12 12:12:12"}), - executeFunction("subtractYears", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.4)))); + executeFunction("subtractYears", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 02:00:00", "2012-12-12 14:12:12", "2012-12-12 02:00:00", "2012-12-12 14:12:12"}), - executeFunction("subtractHours", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.6)))); + executeFunction("subtractHours", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 01:00:00", "2012-12-12 13:12:12", "2012-12-12 01:00:00", "2012-12-12 13:12:12"}), - executeFunction("subtractHours", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.4)))); + executeFunction("subtractHours", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 00:02:00", "2012-12-12 12:14:12", "2012-12-12 00:02:00", "2012-12-12 12:14:12"}), - executeFunction("subtractMinutes", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.6)))); + executeFunction("subtractMinutes", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 00:01:00", "2012-12-12 12:13:12", "2012-12-12 00:01:00", "2012-12-12 12:13:12"}), - executeFunction("subtractMinutes", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.4)))); + executeFunction("subtractMinutes", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.4)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 00:00:02", "2012-12-12 12:12:14", "2012-12-12 00:00:02", "2012-12-12 12:12:14"}), - executeFunction("subtractSeconds", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.6)))); + executeFunction("subtractSeconds", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.6)))); ASSERT_COLUMN_EQ( toNullableVec({"2012-12-12 00:00:01", "2012-12-12 12:12:13", "2012-12-12 00:00:01", "2012-12-12 12:12:13"}), - executeFunction("subtractSeconds", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("round", toConst(-1.4)))); + executeFunction("subtractSeconds", toNullableVec({"20121212", "20121212121212", "2012-12-12", "2012-12-12 12:12:12"}), executeFunction("tidbRound", toConst(-1.4)))); } } // namespace DB::tests diff --git a/dbms/src/Functions/tests/gtest_datetime_daymonthyear.cpp b/dbms/src/Functions/tests/gtest_datetime_daymonthyear.cpp index 99254821062..1cc7d799014 100644 --- a/dbms/src/Functions/tests/gtest_datetime_daymonthyear.cpp +++ b/dbms/src/Functions/tests/gtest_datetime_daymonthyear.cpp @@ -82,9 +82,9 @@ class TestDateTimeDayMonthYear : public DB::tests::FunctionTest .column, makeNullable(std::make_shared
()), "result"); - result_day = createConstColumn>(1, {day}); - result_month = createConstColumn>(1, {month}); - result_year = createConstColumn>(1, {year}); + result_day = createConstColumn(1, {day}); + result_month = createConstColumn(1, {month}); + result_year = createConstColumn(1, {year}); } else { diff --git a/dbms/src/Functions/tests/gtest_duration_pushdown.cpp b/dbms/src/Functions/tests/gtest_duration_pushdown.cpp index bd6fddaeb84..4501a4c9fae 100644 --- a/dbms/src/Functions/tests/gtest_duration_pushdown.cpp +++ b/dbms/src/Functions/tests/gtest_duration_pushdown.cpp @@ -43,8 +43,10 @@ try result_col, executeFunction( "FunctionConvertDurationFromNanos", - createColumn>({-1, 0, 1, {}, INT64_MAX, INT64_MIN, (838 * 3600 + 59 * 60 + 59) * 1000000000L, -(838 * 3600 + 59 * 60 + 59) * 1000000000L}), - createConstColumn(8, 1))); + {createColumn>({-1, 0, 1, {}, INT64_MAX, INT64_MIN, (838 * 3600 + 59 * 60 + 59) * 1000000000L, -(838 * 3600 + 59 * 60 + 59) * 1000000000L}), + createConstColumn(8, 1)}, + nullptr, + true)); ColumnWithTypeAndName result_col2( createConstColumn(3, 3).column, @@ -54,8 +56,10 @@ try result_col2, executeFunction( "FunctionConvertDurationFromNanos", - createConstColumn(3, 3), - createConstColumn(3, 2))); + {createConstColumn(3, 3), + createConstColumn(3, 2)}, + nullptr, + true)); } CATCH diff --git a/dbms/src/Functions/tests/gtest_functions_round_with_frac.cpp b/dbms/src/Functions/tests/gtest_functions_round_with_frac.cpp index 0c85285029e..403b88259a4 100644 --- a/dbms/src/Functions/tests/gtest_functions_round_with_frac.cpp +++ b/dbms/src/Functions/tests/gtest_functions_round_with_frac.cpp @@ -312,9 +312,10 @@ TEST_F(TestFunctionsRoundWithFrac, IntConstInput) // const signed - const frac for (size_t i = 0; i < size; ++i) { - ASSERT_COLUMN_EQ(createConstColumn>(1, int32_result[i]), + bool frac_data_null = !frac_data[i].has_value(); + ASSERT_COLUMN_EQ(frac_data_null ? createConstColumn>(1, int32_result[i]) : createConstColumn(1, int32_result[i].value()), execute(createConstColumn(1, int32_input), createConstColumn>(1, frac_data[i]))); - ASSERT_COLUMN_EQ(createConstColumn>(1, uint32_result[i]), + ASSERT_COLUMN_EQ(frac_data_null ? createConstColumn>(1, uint32_result[i]) : createConstColumn(1, uint32_result[i].value()), execute(createConstColumn(1, uint32_input), createConstColumn>(1, frac_data[i]))); ASSERT_COLUMN_EQ(createConstColumn>(1, {}), execute(createConstColumn>(1, {}), createConstColumn>(1, frac_data[i]))); @@ -441,22 +442,22 @@ try { auto frac = createColumn>({3, 2, 1, 0, -1, -2, -3, -4, -5, -6, {}}); - ASSERT_COLUMN_EQ(column({max_prec, 3}, - {"98765.432", "98765.430", "98765.400", "98765.000", "98770.000", "98800.000", "99000.000", "100000.000", "100000.000", "0.000", {}}), + ASSERT_COLUMN_EQ(createColumn>(std::make_tuple(9, 3), + {"98765.432", "98765.430", "98765.400", "98765.000", "98770.000", "98800.000", "99000.000", "100000.000", "100000.000", "0.000", {}}), this->execute(constColumn({max_prec - 1, 3}, 11, "98765.432"), frac)); ASSERT_COLUMN_EQ(constColumn({max_prec, 3}, 11, {}), this->execute(constColumn({max_prec - 1, 3}, 11, {}), frac)); } // const input & frac - ASSERT_COLUMN_EQ(constColumn({max_prec - 1, 2}, 1, "0.03"), + ASSERT_COLUMN_EQ(createConstColumn(std::make_tuple(3, 2), 1, "0.03"), this->execute(constColumn({max_prec - 1, 3}, 1, "0.025"), createConstColumn(1, 2))); ASSERT_COLUMN_EQ( constColumn({max_prec - 1, 2}, 1, {}), this->execute(constColumn({max_prec - 1, 3}, 1, {}), createConstColumn(1, 2))); - ASSERT_COLUMN_EQ(constColumn({max_prec - 3, 0}, 1, {}), + ASSERT_COLUMN_EQ(createConstColumn>(std::make_tuple(1, 0), 1, {}, "", 0), this->execute(constColumn({max_prec - 1, 3}, 1, "0.025"), createConstColumn>(1, {}))); - ASSERT_COLUMN_EQ(createConstColumn(std::make_tuple(max_prec, 5), 100, "1." + String(5, '0')), + ASSERT_COLUMN_EQ(createConstColumn(std::make_tuple(6, 5), 100, "1." + String(5, '0')), this->execute(createConstColumn(std::make_tuple(max_prec - 5, 0), 100, "1"), createConstColumn(100, 5))); } CATCH diff --git a/dbms/src/Functions/tests/gtest_ifnull.cpp b/dbms/src/Functions/tests/gtest_ifnull.cpp index 6d4bd17089a..bd319b59d92 100644 --- a/dbms/src/Functions/tests/gtest_ifnull.cpp +++ b/dbms/src/Functions/tests/gtest_ifnull.cpp @@ -34,19 +34,15 @@ class TestIfNull : public DB::tests::FunctionTest protected: ColumnWithTypeAndName executeIfNull(const ColumnWithTypeAndName & first_column, const ColumnWithTypeAndName & second_column) { - auto is_null_column = executeFunction("isNull", first_column); - auto not_null_column = executeFunction("assumeNotNull", first_column); - return executeFunction("multiIf", is_null_column, second_column, not_null_column); + return executeFunction("ifNull", first_column, second_column); } DataTypePtr getReturnTypeForIfNull(const DataTypePtr & type_1, const DataTypePtr & type_2) { - const static auto cond_type = std::make_shared(); ColumnsWithTypeAndName input_columns{ - {nullptr, cond_type, ""}, - {nullptr, removeNullable(type_1), ""}, + {nullptr, type_1, ""}, {nullptr, type_2, ""}, }; - return getReturnTypeForFunction(context, "multiIf", input_columns); + return getReturnTypeForFunction(context, "ifNull", input_columns); } template ColumnWithTypeAndName createIntegerColumnInternal(const std::vector & signed_input, const std::vector unsigned_input, const std::vector & null_map) @@ -150,12 +146,12 @@ try } else { - ASSERT_COLUMN_EQ(col_2.type->isNullable() ? expr_data_2_nullable_vector : expr_data_2_vector, executeIfNull(col_1, col_2)); + ASSERT_COLUMN_EQ(expr_data_2_vector, executeIfNull(col_1, col_2)); } } else { - if (col_2.type->isNullable()) + if (col_2.column->isNullAt(0)) { ASSERT_COLUMN_EQ(expr_data_1_nullable_vector, executeIfNull(col_1, col_2)); } @@ -185,14 +181,7 @@ try } else { - if (col_2.type->isNullable()) - { - ASSERT_COLUMN_EQ(createNullableColumn(vector_const_result, {0, 0, 0, 0, 0}), executeIfNull(col_1, col_2)); - } - else - { - ASSERT_COLUMN_EQ(createColumn(vector_const_result), executeIfNull(col_1, col_2)); - } + ASSERT_COLUMN_EQ(createColumn(vector_const_result), executeIfNull(col_1, col_2)); } } } diff --git a/dbms/src/Functions/tests/gtest_inet_aton_ntoa.cpp b/dbms/src/Functions/tests/gtest_inet_aton_ntoa.cpp index 756dc7e610a..cb37252d17e 100644 --- a/dbms/src/Functions/tests/gtest_inet_aton_ntoa.cpp +++ b/dbms/src/Functions/tests/gtest_inet_aton_ntoa.cpp @@ -142,7 +142,7 @@ try // const non-null column ASSERT_COLUMN_EQ( - createConstColumn>(1, "0.0.0.1"), + createConstColumn(1, "0.0.0.1"), executeFunction(func_name, createConstColumn>(1, 1))); // normal cases diff --git a/dbms/src/Functions/tests/gtest_is_true_false.cpp b/dbms/src/Functions/tests/gtest_is_true_false.cpp index 520728b4380..400166c685e 100644 --- a/dbms/src/Functions/tests/gtest_is_true_false.cpp +++ b/dbms/src/Functions/tests/gtest_is_true_false.cpp @@ -124,10 +124,10 @@ CATCH createColumn>({0, 1, 1, std::nullopt}), \ executeFunction("isTrueWithNull", createColumn>({0, 1, static_cast(-1), std::nullopt}))); \ ASSERT_COLUMN_EQ( \ - createConstColumn>(5, 0), \ + createConstColumn(5, 0), \ executeFunction("isTrueWithNull", createConstColumn>(5, 0))); \ ASSERT_COLUMN_EQ( \ - createConstColumn>(5, 1), \ + createConstColumn(5, 1), \ executeFunction("isTrueWithNull", createConstColumn>(5, 2))); \ ASSERT_COLUMN_EQ( \ createConstColumn>(5, std::nullopt), \ @@ -194,10 +194,10 @@ CATCH createColumn>({1, 0, 0, std::nullopt}), \ executeFunction("isFalseWithNull", createColumn>({0, 1, static_cast(-1), std::nullopt}))); \ ASSERT_COLUMN_EQ( \ - createConstColumn>(5, 1), \ + createConstColumn(5, 1), \ executeFunction("isFalseWithNull", createConstColumn>(5, 0))); \ ASSERT_COLUMN_EQ( \ - createConstColumn>(5, 0), \ + createConstColumn(5, 0), \ executeFunction("isFalseWithNull", createConstColumn>(5, 2))); \ ASSERT_COLUMN_EQ( \ createConstColumn>(5, std::nullopt), \ diff --git a/dbms/src/Functions/tests/gtest_least_greatest.cpp b/dbms/src/Functions/tests/gtest_least_greatest.cpp index bc57cc531a1..cbf7552fdc1 100644 --- a/dbms/src/Functions/tests/gtest_least_greatest.cpp +++ b/dbms/src/Functions/tests/gtest_least_greatest.cpp @@ -154,7 +154,7 @@ try // const-const least ASSERT_COLUMN_EQ( - createConstColumn>(1, -3), + createConstColumn(1, -3), executeFunction( func_name, createConstColumn>(1, 5), @@ -323,7 +323,7 @@ try // const-const greatest ASSERT_COLUMN_EQ( - createConstColumn>(1, 5), + createConstColumn(1, 5), executeFunction( func_name, createConstColumn>(1, 5), diff --git a/dbms/src/Functions/tests/gtest_logical.cpp b/dbms/src/Functions/tests/gtest_logical.cpp index 7988989cc88..29fd4d6e133 100644 --- a/dbms/src/Functions/tests/gtest_logical.cpp +++ b/dbms/src/Functions/tests/gtest_logical.cpp @@ -46,7 +46,7 @@ try createColumn>({1, 0}))); // const, const ASSERT_COLUMN_EQ( - createConstColumn>(1, 1), + createConstColumn(1, 1), executeFunction( func_name, createConstColumn>(1, 1), @@ -82,7 +82,7 @@ try createColumn>({1, 0}))); // const, const ASSERT_COLUMN_EQ( - createConstColumn>(1, 1), + createConstColumn(1, 1), executeFunction( func_name, createConstColumn>(1, 1), @@ -118,7 +118,7 @@ try createColumn>({1, 0}))); // const, const ASSERT_COLUMN_EQ( - createConstColumn>(1, 0), + createConstColumn(1, 0), executeFunction( func_name, createConstColumn>(1, 1), @@ -146,7 +146,7 @@ try createColumn>({0, 1, {}}))); // const ASSERT_COLUMN_EQ( - createConstColumn>(1, 0), + createConstColumn(1, 0), executeFunction( func_name, createConstColumn>(1, 1))); diff --git a/dbms/src/Functions/tests/gtest_regexp.cpp b/dbms/src/Functions/tests/gtest_regexp.cpp index 25314e9f5e7..d3eb93a0790 100644 --- a/dbms/src/Functions/tests/gtest_regexp.cpp +++ b/dbms/src/Functions/tests/gtest_regexp.cpp @@ -1828,15 +1828,15 @@ TEST_F(Regexp, testRegexp) for (size_t i = 0; i < row_size; i++) { /// test regexp(const, const) - ASSERT_COLUMN_EQ(input_string_nulls[i] || pattern_nulls[i] ? const_uint8_null_column : createConstColumn>(row_size, results[i]), + ASSERT_COLUMN_EQ(input_string_nulls[i] || pattern_nulls[i] ? const_uint8_null_column : createConstColumn(row_size, results[i]), executeFunction("regexp", input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]))); /// test regexp(const, const, const) - ASSERT_COLUMN_EQ(input_string_nulls[i] || pattern_nulls[i] || match_type_nulls[i] ? const_uint8_null_column : createConstColumn>(row_size, results_with_match_type[i]), + ASSERT_COLUMN_EQ(input_string_nulls[i] || pattern_nulls[i] || match_type_nulls[i] ? const_uint8_null_column : createConstColumn(row_size, results_with_match_type[i]), executeFunction("regexp", input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), match_type_nulls[i] ? const_string_null_column : createConstColumn>(row_size, match_types[i]))); /// test regexp(const, const, const) with binary collator - ASSERT_COLUMN_EQ(input_string_nulls[i] || pattern_nulls[i] || match_type_nulls[i] ? const_uint8_null_column : createConstColumn>(row_size, results_with_match_type_collator[i]), + ASSERT_COLUMN_EQ(input_string_nulls[i] || pattern_nulls[i] || match_type_nulls[i] ? const_uint8_null_column : createConstColumn(row_size, results_with_match_type_collator[i]), executeFunction("regexp", {input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), match_type_nulls[i] ? const_string_null_column : createConstColumn>(row_size, match_types[i])}, binary_collator)); } /// case 3 regexp(vector, const[, const]) @@ -1920,20 +1920,13 @@ TEST_F(Regexp, testRegexpCustomerCases) } else if (isColumnConstNotNull(input_column) && isColumnConstNotNull(pattern_column)) { - if (input_column.type->isNullable() || pattern_column.type->isNullable()) - { - ASSERT_COLUMN_EQ(createConstColumn>(5, 1), - executeFunction("regexp", input_column, pattern_column)); - } - else - { - ASSERT_COLUMN_EQ(createConstColumn(5, 1), - executeFunction("regexp", input_column, pattern_column)); - } + ASSERT_COLUMN_EQ(createConstColumn(5, 1), + executeFunction("regexp", input_column, pattern_column)); } else { - bool result_nullable = input_column.type->isNullable() || pattern_column.type->isNullable(); + bool result_nullable = (input_column.type->isNullable() && !isColumnConstNotNull(input_column)) + || (pattern_column.type->isNullable() && !isColumnConstNotNull(pattern_column)); if (!result_nullable) { ASSERT_COLUMN_EQ(createColumn({1, 1, 1, 1, 1}), @@ -2067,28 +2060,29 @@ TEST_F(Regexp, testRegexpReplace) auto const_string_null_column = createConstColumn>(row_size, {}); auto const_int64_null_column = createConstColumn>(row_size, {}); + /// regexp_replace is not supported in TiDB yet, so use raw function test /// case 1. regexp_replace(const, const, const [, const, const ,const]) for (size_t i = 0; i < match_types.size(); i++) { /// test regexp_replace(str, pattern, replacement) ASSERT_COLUMN_EQ(createConstColumn(row_size, results[i]), - executeFunction("replaceRegexpAll", createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i]))); + executeFunction("replaceRegexpAll", {createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos) ASSERT_COLUMN_EQ(createConstColumn(row_size, results_with_pos[i]), - executeFunction("replaceRegexpAll", createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i]), createConstColumn(row_size, pos[i]))); + executeFunction("replaceRegexpAll", {createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i]), createConstColumn(row_size, pos[i])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ) ASSERT_COLUMN_EQ(createConstColumn(row_size, results_with_pos_occ[i]), - executeFunction("replaceRegexpAll", createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i]), createConstColumn(row_size, pos[i]), createConstColumn(row_size, occ[i]))); + executeFunction("replaceRegexpAll", {createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i]), createConstColumn(row_size, pos[i]), createConstColumn(row_size, occ[i])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ, match_type) ASSERT_COLUMN_EQ(createConstColumn(row_size, results_with_pos_occ_match_type[i]), - executeFunction("replaceRegexpAll", createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i]), createConstColumn(row_size, pos[i]), createConstColumn(row_size, occ[i]), createConstColumn(row_size, match_types[i]))); + executeFunction("replaceRegexpAll", {createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i]), createConstColumn(row_size, pos[i]), createConstColumn(row_size, occ[i]), createConstColumn(row_size, match_types[i])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ, match_type) with binary collator ASSERT_COLUMN_EQ(createConstColumn(row_size, results_with_pos_occ_match_type_binary[i]), - executeFunction("replaceRegexpAll", {createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i]), createConstColumn(row_size, pos[i]), createConstColumn(row_size, occ[i]), createConstColumn(row_size, match_types[i])}, binary_collator)); + executeFunction("replaceRegexpAll", {createConstColumn(row_size, input_strings[i]), createConstColumn(row_size, patterns[i]), createConstColumn(row_size, replacements[i]), createConstColumn(row_size, pos[i]), createConstColumn(row_size, occ[i]), createConstColumn(row_size, match_types[i])}, binary_collator, true)); } /// case 2. regexp_replace(const, const, const [, const, const ,const]) with null value @@ -2097,74 +2091,74 @@ TEST_F(Regexp, testRegexpReplace) /// test regexp_replace(str, pattern, replacement) bool null_result = input_string_nulls[i] || pattern_nulls[i] || replacement_nulls[i]; ASSERT_COLUMN_EQ(null_result ? const_string_null_column : createConstColumn>(row_size, results[i]), - executeFunction("replaceRegexpAll", input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i]))); + executeFunction("replaceRegexpAll", {input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos) null_result = null_result || pos_nulls[i]; ASSERT_COLUMN_EQ(null_result ? const_string_null_column : createConstColumn>(row_size, results_with_pos[i]), - executeFunction("replaceRegexpAll", input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i]), pos_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, pos[i]))); + executeFunction("replaceRegexpAll", {input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i]), pos_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, pos[i])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ) null_result = null_result || occ_nulls[i]; ASSERT_COLUMN_EQ(null_result ? const_string_null_column : createConstColumn>(row_size, results_with_pos_occ[i]), - executeFunction("replaceRegexpAll", input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i]), pos_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, pos[i]), occ_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, occ[i]))); + executeFunction("replaceRegexpAll", {input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i]), pos_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, pos[i]), occ_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, occ[i])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ, match_type) null_result = null_result || match_type_nulls[i]; ASSERT_COLUMN_EQ(null_result ? const_string_null_column : createConstColumn>(row_size, results_with_pos_occ_match_type[i]), - executeFunction("replaceRegexpAll", input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i]), pos_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, pos[i]), occ_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, occ[i]), match_type_nulls[i] ? const_string_null_column : createConstColumn>(row_size, match_types[i]))); + executeFunction("replaceRegexpAll", {input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i]), pos_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, pos[i]), occ_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, occ[i]), match_type_nulls[i] ? const_string_null_column : createConstColumn>(row_size, match_types[i])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ, match_type) with binary collator ASSERT_COLUMN_EQ(null_result ? const_string_null_column : createConstColumn>(row_size, results_with_pos_occ_match_type_binary[i]), - executeFunction("replaceRegexpAll", {input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i]), pos_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, pos[i]), occ_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, occ[i]), match_type_nulls[i] ? const_string_null_column : createConstColumn>(row_size, match_types[i])}, binary_collator)); + executeFunction("replaceRegexpAll", {input_string_nulls[i] ? const_string_null_column : createConstColumn>(row_size, input_strings[i]), pattern_nulls[i] ? const_string_null_column : createConstColumn>(row_size, patterns[i]), replacement_nulls[i] ? const_string_null_column : createConstColumn>(row_size, replacements[i]), pos_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, pos[i]), occ_nulls[i] ? const_int64_null_column : createConstColumn>(row_size, occ[i]), match_type_nulls[i] ? const_string_null_column : createConstColumn>(row_size, match_types[i])}, binary_collator, true)); } /// case 3 regexp_replace(vector, const, const[, const, const, const]) { /// test regexp_replace(str, pattern, replacement) ASSERT_COLUMN_EQ(createColumn(vec_results), - executeFunction("replaceRegexpAll", createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]))); + executeFunction("replaceRegexpAll", {createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos) ASSERT_COLUMN_EQ(createColumn(vec_results_with_pos), - executeFunction("replaceRegexpAll", createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]))); + executeFunction("replaceRegexpAll", {createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ) ASSERT_COLUMN_EQ(createColumn(vec_results_with_pos_occ), - executeFunction("replaceRegexpAll", createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]))); + executeFunction("replaceRegexpAll", {createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ, match_type) ASSERT_COLUMN_EQ(createColumn(vec_results_with_pos_occ_match_type), - executeFunction("replaceRegexpAll", createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]), createConstColumn(row_size, match_types[0]))); + executeFunction("replaceRegexpAll", {createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]), createConstColumn(row_size, match_types[0])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ, match_type) with binary collator ASSERT_COLUMN_EQ(createColumn(vec_results_with_pos_occ_match_type_binary), - executeFunction("replaceRegexpAll", {createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]), createConstColumn(row_size, match_types[0])}, binary_collator)); + executeFunction("replaceRegexpAll", {createColumn(input_strings), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]), createConstColumn(row_size, match_types[0])}, binary_collator, true)); } /// case 4 regexp_replace(vector, const, const[, const, const, const]) with null value { /// test regexp_replace(str, pattern, replacement) ASSERT_COLUMN_EQ(createNullableVectorColumn(vec_results, input_string_nulls), - executeFunction("replaceRegexpAll", createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]))); + executeFunction("replaceRegexpAll", {createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos) ASSERT_COLUMN_EQ(createNullableVectorColumn(vec_results_with_pos, input_string_nulls), - executeFunction("replaceRegexpAll", createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]))); + executeFunction("replaceRegexpAll", {createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ) ASSERT_COLUMN_EQ(createNullableVectorColumn(vec_results_with_pos_occ, input_string_nulls), - executeFunction("replaceRegexpAll", createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]))); + executeFunction("replaceRegexpAll", {createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ, match_type) ASSERT_COLUMN_EQ(createNullableVectorColumn(vec_results_with_pos_occ_match_type, input_string_nulls), - executeFunction("replaceRegexpAll", createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]), createConstColumn(row_size, match_types[0]))); + executeFunction("replaceRegexpAll", {createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]), createConstColumn(row_size, match_types[0])}, nullptr, true)); /// test regexp_replace(str, pattern, replacement, pos, occ, match_type) with binary collator ASSERT_COLUMN_EQ(createNullableVectorColumn(vec_results_with_pos_occ_match_type_binary, input_string_nulls), - executeFunction("replaceRegexpAll", {createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]), createConstColumn(row_size, match_types[0])}, binary_collator)); + executeFunction("replaceRegexpAll", {createNullableVectorColumn(input_strings, input_string_nulls), createConstColumn(row_size, patterns[0]), createConstColumn(row_size, replacements[0]), createConstColumn(row_size, pos[0]), createConstColumn(row_size, occ[0]), createConstColumn(row_size, match_types[0])}, binary_collator, true)); } } } // namespace tests diff --git a/dbms/src/Functions/tests/gtest_string_left.cpp b/dbms/src/Functions/tests/gtest_string_left.cpp index ca491bfe6fa..42a16ee6fff 100644 --- a/dbms/src/Functions/tests/gtest_string_left.cpp +++ b/dbms/src/Functions/tests/gtest_string_left.cpp @@ -27,8 +27,7 @@ namespace tests class StringLeftTest : public DB::tests::FunctionTest { public: - // leftUTF8(str,len) = substrUTF8(str,const 1,len) - static constexpr auto func_name = "substringUTF8"; + static constexpr auto func_name = "leftUTF8"; template void testBoundary() @@ -53,14 +52,14 @@ class StringLeftTest : public DB::tests::FunctionTest template void test(const std::optional & str, const std::optional & length, const std::optional & result) { - const auto start_column = createConstColumn(1, 1); auto inner_test = [&](bool is_str_const, bool is_length_const) { bool is_one_of_args_null_const = (is_str_const && !str.has_value()) || (is_length_const && !length.has_value()); bool is_result_const = (is_str_const && is_length_const) || is_one_of_args_null_const; - auto expected_res_column = is_result_const ? createConstColumn>(1, result) : createColumn>({result}); + auto expected_res_column = is_result_const ? (is_one_of_args_null_const ? createConstColumn>(1, result) : createConstColumn(1, result.value())) + : createColumn>({result}); auto str_column = is_str_const ? createConstColumn>(1, str) : createColumn>({str}); auto length_column = is_length_const ? createConstColumn>(1, length) : createColumn>({length}); - auto actual_res_column = executeFunction(func_name, str_column, start_column, length_column); + auto actual_res_column = executeFunction(func_name, str_column, length_column); ASSERT_COLUMN_EQ(expected_res_column, actual_res_column); }; std::vector is_consts = {true, false}; @@ -78,7 +77,6 @@ class StringLeftTest : public DB::tests::FunctionTest executeFunction( func_name, is_str_const ? createConstColumn>(1, "") : createColumn>({""}), - createConstColumn(1, 1), is_length_const ? createConstColumn>(1, 0) : createColumn>({0})), Exception); }; @@ -132,7 +130,6 @@ try executeFunction( func_name, createColumn>({big_string, origin_str, origin_str, mixed_language_str}), - createConstColumn(8, 1), createColumn>({22, 12, 22, english_str.size()}))); // case 2 String second_case_string = "abc"; @@ -141,14 +138,12 @@ try executeFunction( func_name, createColumn>({second_case_string, second_case_string, second_case_string, second_case_string, second_case_string, second_case_string, second_case_string, second_case_string}), - createConstColumn(8, 1), createColumn>({0, 1, 0, 1, 0, 0, 1, 1}))); ASSERT_COLUMN_EQ( createColumn>({"", "a", "", "a", "", "", "a", "a"}), executeFunction( func_name, createConstColumn>(8, second_case_string), - createConstColumn(8, 1), createColumn>({0, 1, 0, 1, 0, 0, 1, 1}))); } CATCH diff --git a/dbms/src/Functions/tests/gtest_string_lrtrim.cpp b/dbms/src/Functions/tests/gtest_string_lrtrim.cpp index 409c8ed715b..df52257613d 100644 --- a/dbms/src/Functions/tests/gtest_string_lrtrim.cpp +++ b/dbms/src/Functions/tests/gtest_string_lrtrim.cpp @@ -39,16 +39,16 @@ try { // ltrim(const) ASSERT_COLUMN_EQ( - createConstColumn>(5, "x "), + createConstColumn(5, "x "), executeFunction("tidbLTrim", createConstColumn>(5, " x "))); ASSERT_COLUMN_EQ( - createConstColumn>(5, "测试 "), + createConstColumn(5, "测试 "), executeFunction("tidbLTrim", createConstColumn>(5, " 测试 "))); ASSERT_COLUMN_EQ( - createConstColumn>(5, "x x x"), + createConstColumn(5, "x x x"), executeFunction("tidbLTrim", createConstColumn>(5, "x x x"))); ASSERT_COLUMN_EQ( - createConstColumn>(5, "测 试 "), + createConstColumn(5, "测 试 "), executeFunction("tidbLTrim", createConstColumn>(5, "测 试 "))); ASSERT_COLUMN_EQ( createConstColumn(5, "x "), @@ -83,16 +83,16 @@ try // rtrim(const) ASSERT_COLUMN_EQ( - createConstColumn>(5, " x"), + createConstColumn(5, " x"), executeFunction("tidbRTrim", createConstColumn>(5, " x "))); ASSERT_COLUMN_EQ( - createConstColumn>(5, " 测试"), + createConstColumn(5, " 测试"), executeFunction("tidbRTrim", createConstColumn>(5, " 测试 "))); ASSERT_COLUMN_EQ( - createConstColumn>(5, "x x x"), + createConstColumn(5, "x x x"), executeFunction("tidbRTrim", createConstColumn>(5, "x x x"))); ASSERT_COLUMN_EQ( - createConstColumn>(5, "测 试"), + createConstColumn(5, "测 试"), executeFunction("tidbRTrim", createConstColumn>(5, "测 试 "))); ASSERT_COLUMN_EQ( createConstColumn(5, " x"), @@ -225,10 +225,10 @@ try input_iter++, lres_iter++, rres_iter++) { ASSERT_COLUMN_EQ( - createConstColumn>(5, *lres_iter), + createConstColumn(5, *lres_iter), executeFunction("tidbLTrim", createConstColumn>(5, *input_iter))); ASSERT_COLUMN_EQ( - createConstColumn>(5, *rres_iter), + createConstColumn(5, *rres_iter), executeFunction("tidbRTrim", createConstColumn>(5, *input_iter))); ASSERT_COLUMN_EQ( createConstColumn(5, *lres_iter), diff --git a/dbms/src/Functions/tests/gtest_strings_cmp.cpp b/dbms/src/Functions/tests/gtest_strings_cmp.cpp index ca9ae6562c7..7e94a7ed30e 100644 --- a/dbms/src/Functions/tests/gtest_strings_cmp.cpp +++ b/dbms/src/Functions/tests/gtest_strings_cmp.cpp @@ -43,7 +43,7 @@ try ASSERT_COLUMN_EQ(createColumn>({1, 0, -1, std::nullopt}), executeFunction("strcmp", {createConstColumn>(4, "b"), createColumn>({"a", "b", "c", std::nullopt})})); // constant with constant - ASSERT_COLUMN_EQ(createConstColumn>(1, -1), executeFunction("strcmp", {createConstColumn>(1, "a"), createConstColumn>(1, "b")})); + ASSERT_COLUMN_EQ(createConstColumn(1, -1), executeFunction("strcmp", {createConstColumn>(1, "a"), createConstColumn>(1, "b")})); // constant with nullable ASSERT_COLUMN_EQ(createColumn>({-1}), executeFunction("strcmp", {createColumn({"a"}), createColumn>({"b"})})); @@ -65,10 +65,10 @@ try ASSERT_COLUMN_EQ(createColumn>({-1, 1, 0, std::nullopt, std::nullopt}), executeFunction("strcmp", {createColumn>({"", "123", "", "", std::nullopt}), createColumn>({"123", "", "", std::nullopt, ""})}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); // column with constant - ASSERT_COLUMN_EQ(createColumn>({-1}), executeFunction("strcmp", {createColumn({"a"}), createConstColumn>(1, "b")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); - ASSERT_COLUMN_EQ(createColumn>({-1}), executeFunction("strcmp", {createColumn({"A"}), createConstColumn>(1, "b")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); - ASSERT_COLUMN_EQ(createColumn>({-1}), executeFunction("strcmp", {createColumn({"A"}), createConstColumn>(1, "B")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); - ASSERT_COLUMN_EQ(createColumn>({-1, 0, 1, 1}), executeFunction("strcmp", {createColumn({"A", "B", "C", "D"}), createConstColumn>(4, "B")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + ASSERT_COLUMN_EQ(createColumn({-1}), executeFunction("strcmp", {createColumn({"a"}), createConstColumn>(1, "b")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + ASSERT_COLUMN_EQ(createColumn({-1}), executeFunction("strcmp", {createColumn({"A"}), createConstColumn>(1, "b")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + ASSERT_COLUMN_EQ(createColumn({-1}), executeFunction("strcmp", {createColumn({"A"}), createConstColumn>(1, "B")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + ASSERT_COLUMN_EQ(createColumn({-1, 0, 1, 1}), executeFunction("strcmp", {createColumn({"A", "B", "C", "D"}), createConstColumn>(4, "B")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); // constant with column ASSERT_COLUMN_EQ(createColumn>({1, 0, -1, std::nullopt}), executeFunction("strcmp", {createConstColumn>(4, "b"), createColumn>({"a", "b", "c", std::nullopt})}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); @@ -76,10 +76,10 @@ try ASSERT_COLUMN_EQ(createColumn>({1, 0, -1, std::nullopt}), executeFunction("strcmp", {createConstColumn>(4, "b"), createColumn>({"A", "B", "C", std::nullopt})}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); // constant with constant - ASSERT_COLUMN_EQ(createConstColumn>(1, -1), executeFunction("strcmp", {createConstColumn>(1, "a"), createConstColumn>(1, "b")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); - ASSERT_COLUMN_EQ(createConstColumn>(1, -1), executeFunction("strcmp", {createConstColumn>(1, "A"), createConstColumn>(1, "b")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); - ASSERT_COLUMN_EQ(createConstColumn>(1, -1), executeFunction("strcmp", {createConstColumn>(1, "a"), createConstColumn>(1, "B")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); - ASSERT_COLUMN_EQ(createConstColumn>(1, -1), executeFunction("strcmp", {createConstColumn>(1, "A"), createConstColumn>(1, "B")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + ASSERT_COLUMN_EQ(createConstColumn(1, -1), executeFunction("strcmp", {createConstColumn>(1, "a"), createConstColumn>(1, "b")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + ASSERT_COLUMN_EQ(createConstColumn(1, -1), executeFunction("strcmp", {createConstColumn>(1, "A"), createConstColumn>(1, "b")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + ASSERT_COLUMN_EQ(createConstColumn(1, -1), executeFunction("strcmp", {createConstColumn>(1, "a"), createConstColumn>(1, "B")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + ASSERT_COLUMN_EQ(createConstColumn(1, -1), executeFunction("strcmp", {createConstColumn>(1, "A"), createConstColumn>(1, "B")}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); // constant with nullable ASSERT_COLUMN_EQ(createColumn>({-1}), executeFunction("strcmp", {createColumn({"a"}), createColumn>({"b"})}, TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); diff --git a/dbms/src/Functions/tests/gtest_strings_format.cpp b/dbms/src/Functions/tests/gtest_strings_format.cpp index 2adc17afb93..2d571a9bb1b 100644 --- a/dbms/src/Functions/tests/gtest_strings_format.cpp +++ b/dbms/src/Functions/tests/gtest_strings_format.cpp @@ -62,6 +62,8 @@ class StringFormat : public DB::tests::FunctionTest FieldType(static_cast(-9999999), 4), FieldType(static_cast(-3333330), 4)}), createConstColumn>(4, 3))); + /// known issue https://github.com/pingcap/tiflash/issues/4891 + /* ASSERT_COLUMN_EQ( createColumn>({"-999.9999", "-1,000", "-1,000", "-999.999900000000000000000000000000", "-999.99990", "-1,000.0", "-1,000.00"}), executeFunction( @@ -80,6 +82,7 @@ class StringFormat : public DB::tests::FunctionTest 1, FieldType(static_cast(-9999999), 4)), createConstColumn>(1, 3))); + */ ASSERT_COLUMN_EQ( createColumn>({"12,332.1000", "12,332", "12,332.300000000000000000000000000000", "-12,332.30000", "-1,000.0", "-333.33", {}}), executeFunction( @@ -105,6 +108,8 @@ class StringFormat : public DB::tests::FunctionTest FieldType(static_cast(-9999999), 4), FieldType(static_cast(-3333330), 4)}), createConstColumn>(4, 3))); + /// known issue https://github.com/pingcap/tiflash/issues/4891 + /* ASSERT_COLUMN_EQ( createColumn>({"-999.9999", "-1,000", "-999.999900000000000000000000000000", "-999.99990", "-1,000.0", "-1,000.00"}), executeFunction( @@ -123,6 +128,7 @@ class StringFormat : public DB::tests::FunctionTest 1, FieldType(static_cast(-9999999), 4)), createConstColumn>(1, 3))); + */ } template @@ -226,7 +232,7 @@ try createColumn>({4, 0, -1, 31, 5, 1, 2}))); /// const, const ASSERT_COLUMN_EQ( - createConstColumn>(1, "-1,000.000"), + createConstColumn(1, "-1,000.000"), executeFunction( func_name, createConstColumn>(1, -999.9999), @@ -256,7 +262,7 @@ try createColumn>({4, 0, 31, 5, 1, 2}))); /// const, const ASSERT_COLUMN_EQ( - createConstColumn>(1, "-1,000.000"), + createConstColumn(1, "-1,000.000"), executeFunction( func_name, createConstColumn>(1, -999.9999), @@ -265,7 +271,7 @@ try /// float32, int /// const, const ASSERT_COLUMN_EQ( - createConstColumn>(1, "12.123"), + createConstColumn(1, "12.123"), executeFunction( func_name, createConstColumn>(1, 12.1235), diff --git a/dbms/src/Functions/tests/gtest_strings_right.cpp b/dbms/src/Functions/tests/gtest_strings_right.cpp index 1dbe8c7d5e3..210cb651ec6 100644 --- a/dbms/src/Functions/tests/gtest_strings_right.cpp +++ b/dbms/src/Functions/tests/gtest_strings_right.cpp @@ -54,7 +54,8 @@ class StringRightTest : public DB::tests::FunctionTest auto inner_test = [&](bool is_str_const, bool is_length_const) { bool is_one_of_args_null_const = (is_str_const && !str.has_value()) || (is_length_const && !length.has_value()); bool is_result_const = (is_str_const && is_length_const) || is_one_of_args_null_const; - auto expected_res_column = is_result_const ? createConstColumn>(1, result) : createColumn>({result}); + auto expected_res_column = is_result_const ? (is_one_of_args_null_const ? createConstColumn>(1, result) : createConstColumn(1, result.value())) + : createColumn>({result}); auto str_column = is_str_const ? createConstColumn>(1, str) : createColumn>({str}); auto length_column = is_length_const ? createConstColumn>(1, length) : createColumn>({length}); auto actual_res_column = executeFunction(func_name, str_column, length_column); diff --git a/dbms/src/Functions/tests/gtest_strings_search.cpp b/dbms/src/Functions/tests/gtest_strings_search.cpp index 58bf1b34487..544ebc34df6 100644 --- a/dbms/src/Functions/tests/gtest_strings_search.cpp +++ b/dbms/src/Functions/tests/gtest_strings_search.cpp @@ -24,6 +24,7 @@ namespace tests class StringMatch : public FunctionTest { protected: + const String func_name = "like3Args"; const String long_str = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzab" "cdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdef" "ghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkl" @@ -33,6 +34,8 @@ class StringMatch : public FunctionTest const String long_pattern = "abcdefghijklmnopqrstuvwxyz_bcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz%abcdefghijklmnopqrstuvwxyz"; + ColumnWithTypeAndName escape = createConstColumn(1, static_cast('\\')); + static ColumnWithTypeAndName toNullableVec(const std::vector> & v) { return createColumn>(v); @@ -122,10 +125,9 @@ try auto haystack = createColumn(haystack_raw, "haystack"); auto needle = createColumn(needle_raw, "needle"); - auto escape = createConstColumn(1, static_cast('\\')); auto expected = createColumn(result_raw, "result"); - auto result = executeFunction("like3Args", {haystack, needle, escape}); + auto result = executeFunction(func_name, {haystack, needle, escape}); ASSERT_COLUMN_EQ(expected, result); @@ -155,10 +157,9 @@ try auto nullable_haystack = createColumn>(nullable_haystack_raw, "haystack"); auto nullable_needle = createColumn>(nullable_needle_raw, "needle"); - auto nullable_escape = createConstColumn>(1, static_cast('\\')); auto nullable_expected = createColumn>(nullable_result_raw, "result"); - auto nullable_result = executeFunction("like3Args", {nullable_haystack, nullable_needle, nullable_escape}); + auto nullable_result = executeFunction(func_name, {nullable_haystack, nullable_needle, escape}); ASSERT_COLUMN_EQ(nullable_expected, nullable_result); } @@ -196,10 +197,9 @@ try auto haystack = createConstColumn>(1, cas.src); auto needle = createColumn>(needle_raw); - auto escape = createConstColumn>(1, static_cast('\\')); auto expected = createColumn>(result_raw); - auto result = executeFunction("like3Args", {haystack, needle, escape}); + auto result = executeFunction(func_name, {haystack, needle, escape}); ASSERT_COLUMN_EQ(expected, result); } } @@ -213,16 +213,18 @@ TEST_F(StringMatch, LikeVectorWithVector) ASSERT_COLUMN_EQ( toNullableVec(expect), executeFunction( - "like", + func_name, toNullableVec(haystack), - toNullableVec(needle))); + toNullableVec(needle), + escape)); ASSERT_COLUMN_EQ( toVec(expect), executeFunction( - "like", + func_name, toVec(haystack), - toVec(needle))); + toVec(needle), + escape)); std::vector> haystack_null = {{}, "a"}; std::vector> needle_null = {"我_tif%", {}}; @@ -230,9 +232,10 @@ TEST_F(StringMatch, LikeVectorWithVector) ASSERT_COLUMN_EQ( toNullableVec(expect_null), executeFunction( - "like", + func_name, toNullableVec(haystack_null), - toNullableVec(needle_null))); + toNullableVec(needle_null), + escape)); } TEST_F(StringMatch, LikeConstWithVector) @@ -243,32 +246,36 @@ TEST_F(StringMatch, LikeConstWithVector) ASSERT_COLUMN_EQ( toNullableVec(expect), executeFunction( - "like", + func_name, toConst("abcaba"), - toNullableVec(needle))); + toNullableVec(needle), + escape)); ASSERT_COLUMN_EQ( toVec(expect), executeFunction( - "like", + func_name, toConst("abcaba"), - toVec(needle))); + toVec(needle), + escape)); ASSERT_COLUMN_EQ( toVec(expect1), executeFunction( - "like", + func_name, toConst(long_str), - toVec(needle))); + toVec(needle), + escape)); std::vector> needle_null = {{}}; std::vector> expect_null = {{}}; ASSERT_COLUMN_EQ( toNullableVec(expect_null), executeFunction( - "like", + func_name, toConst("abc"), - toNullableVec(needle_null))); + toNullableVec(needle_null), + escape)); } TEST_F(StringMatch, LikeVectorWithConst) @@ -281,46 +288,52 @@ TEST_F(StringMatch, LikeVectorWithConst) ASSERT_COLUMN_EQ( toNullableVec(expect), executeFunction( - "like", + func_name, toNullableVec(haystack), - toConst("%aa%"))); + toConst("%aa%"), + escape)); ASSERT_COLUMN_EQ( toVec(expect), executeFunction( - "like", + func_name, toVec(haystack), - toConst("%aa%"))); + toConst("%aa%"), + escape)); ASSERT_COLUMN_EQ( toVec(expect1), executeFunction( - "like", + func_name, toVec(haystack), - toConst("%爱tif%"))); + toConst("%爱tif%"), + escape)); ASSERT_COLUMN_EQ( toVec(expect2), executeFunction( - "like", + func_name, toVec(haystack), - toConst("%不爱tif%"))); + toConst("%不爱tif%"), + escape)); ASSERT_COLUMN_EQ( toVec(expect3), executeFunction( - "like", + func_name, toVec(haystack), - toConst(long_pattern))); + toConst(long_pattern), + escape)); std::vector> haystack_null = {{}}; std::vector> expect_null = {{}}; ASSERT_COLUMN_EQ( toNullableVec(expect_null), executeFunction( - "like", + func_name, toNullableVec(haystack_null), - toConst("abc"))); + toConst("abc"), + escape)); } TEST_F(StringMatch, LikeConstWithConst) @@ -328,30 +341,34 @@ TEST_F(StringMatch, LikeConstWithConst) ASSERT_COLUMN_EQ( toConst(1), executeFunction( - "like", + func_name, toConst("resaasfe"), - toConst("%aa%"))); + toConst("%aa%"), + escape)); ASSERT_COLUMN_EQ( toConst(0), executeFunction( - "like", + func_name, toConst("abcde"), - toConst("%aa%"))); + toConst("%aa%"), + escape)); ASSERT_COLUMN_EQ( toConst(1), executeFunction( - "like", + func_name, toConst("我爱tiflash"), - toConst("%爱tif%"))); + toConst("%爱tif%"), + escape)); ASSERT_COLUMN_EQ( toConst(0), executeFunction( - "like", + func_name, toConst("我爱tiflash"), - toConst("%不爱tif%"))); + toConst("%不爱tif%"), + escape)); } } // namespace tests diff --git a/dbms/src/Functions/tests/gtest_strings_tidb_concat.cpp b/dbms/src/Functions/tests/gtest_strings_tidb_concat.cpp index a0c06a5d6a8..24295bc079b 100644 --- a/dbms/src/Functions/tests/gtest_strings_tidb_concat.cpp +++ b/dbms/src/Functions/tests/gtest_strings_tidb_concat.cpp @@ -28,6 +28,7 @@ class StringTidbConcat : public DB::tests::FunctionTest static constexpr auto func_name = "tidbConcat"; using Type = Nullable; + using NotNullType = String; InferredDataVector test_strings = {"", "www.pingcap", "中文.测.试。。。", {}}; }; @@ -45,7 +46,7 @@ try createColumn({value}))); // const ASSERT_COLUMN_EQ( - createConstColumn(1, value), + value.has_value() ? createConstColumn(1, value.value()) : createConstColumn(1, value), executeFunction( StringTidbConcat::func_name, createConstColumn(1, value))); @@ -64,7 +65,7 @@ try // all args is const or has only null const auto is_result_const = (is_value1_const && is_value2_const) || (!value1.has_value() && is_value1_const) || (!value2.has_value() && is_value2_const); ASSERT_COLUMN_EQ( - is_result_const ? createConstColumn(1, result) : createColumn({result}), + is_result_const ? (is_result_not_null ? createConstColumn(1, result.value()) : createConstColumn(1, result)) : createColumn({result}), executeFunction( StringTidbConcat::func_name, is_value1_const ? createConstColumn(1, value1) : createColumn({value1}), diff --git a/dbms/src/Functions/tests/gtest_strings_trim.cpp b/dbms/src/Functions/tests/gtest_strings_trim.cpp index be2616c4818..55c4063abb5 100644 --- a/dbms/src/Functions/tests/gtest_strings_trim.cpp +++ b/dbms/src/Functions/tests/gtest_strings_trim.cpp @@ -600,37 +600,37 @@ try { // trim(const) ASSERT_COLUMN_EQ( - createConstColumn>(5, "x"), + createConstColumn(5, "x"), executeFunction("tidbTrim", createConstColumn>(5, " x "))); // trim(const from const) ASSERT_COLUMN_EQ( - createConstColumn>(5, "a"), + createConstColumn(5, "a"), executeFunction("tidbTrim", createConstColumn>(5, "xax"), createConstColumn>(5, "x"))); // trim(leading|trailing|both const from const) ASSERT_COLUMN_EQ( - createConstColumn>(5, "a"), + createConstColumn(5, "a"), executeFunction("tidbTrim", createConstColumn>(5, "xax"), createConstColumn>(5, "x"), createConstColumn>(5, 0))); ASSERT_COLUMN_EQ( - createConstColumn>(5, "a"), + createConstColumn(5, "a"), executeFunction("tidbTrim", createConstColumn>(5, "xax"), createConstColumn>(5, "x"), createConstColumn>(5, 1))); ASSERT_COLUMN_EQ( - createConstColumn>(5, "ax"), + createConstColumn(5, "ax"), executeFunction("tidbTrim", createConstColumn>(5, "xax"), createConstColumn>(5, "x"), createConstColumn>(5, 2))); ASSERT_COLUMN_EQ( - createConstColumn>(5, "xa"), + createConstColumn(5, "xa"), executeFunction("tidbTrim", createConstColumn>(5, "xax"), createConstColumn>(5, "x"), @@ -955,7 +955,7 @@ try res_itr++, input_itr++) { ASSERT_COLUMN_EQ( - createConstColumn>(5, *res_itr), + input_itr->has_value() ? createConstColumn(5, res_itr->value()) : createConstColumn>(5, *res_itr), executeFunction("tidbTrim", createConstColumn>(5, *input_itr), createConstColumn>(5, "啊啊"), diff --git a/dbms/src/Functions/tests/gtest_substring.cpp b/dbms/src/Functions/tests/gtest_substring.cpp index 1506b3340a6..374e38970f1 100644 --- a/dbms/src/Functions/tests/gtest_substring.cpp +++ b/dbms/src/Functions/tests/gtest_substring.cpp @@ -41,7 +41,7 @@ try createConstColumn>(4, 4))); // const, const, const ASSERT_COLUMN_EQ( - createConstColumn>(1, "www."), + createConstColumn(1, "www."), executeFunction( "substringUTF8", createConstColumn>(1, "www.pingcap.com"), diff --git a/dbms/src/Functions/tests/gtest_substring_index.cpp b/dbms/src/Functions/tests/gtest_substring_index.cpp index a8208b45042..9bb4675c329 100644 --- a/dbms/src/Functions/tests/gtest_substring_index.cpp +++ b/dbms/src/Functions/tests/gtest_substring_index.cpp @@ -75,7 +75,9 @@ class SubstringIndexTest : public DB::tests::FunctionTest auto inner_test = [&](bool is_str_const, bool is_delim_const, bool is_count_const) { bool is_one_of_args_null_const = (is_str_const && !str.has_value()) || (is_delim_const && !delim.has_value()) || (is_count_const && !count.has_value()); bool is_result_const = (is_str_const && is_delim_const && is_count_const) || is_one_of_args_null_const; - auto expected_res_column = is_result_const ? createConstColumn>(1, result) : createColumn>({result}); + if (is_result_const && !is_one_of_args_null_const && !result.has_value()) + throw Exception("Should not reach here"); + auto expected_res_column = is_result_const ? (is_one_of_args_null_const ? createConstColumn>(1, result) : createConstColumn(1, result.value())) : createColumn>({result}); auto str_column = is_str_const ? createConstColumn>(1, str) : createColumn>({str}); auto delim_column = is_delim_const ? createConstColumn>(1, delim) : createColumn>({delim}); auto count_column = is_count_const ? createConstColumn>(1, count) : createColumn>({count}); diff --git a/dbms/src/Functions/tests/gtest_unix_timestamp.cpp b/dbms/src/Functions/tests/gtest_unix_timestamp.cpp index 1fa02141c4c..15909a90af9 100644 --- a/dbms/src/Functions/tests/gtest_unix_timestamp.cpp +++ b/dbms/src/Functions/tests/gtest_unix_timestamp.cpp @@ -86,16 +86,16 @@ try executeFunction(func_name_dec, createConstColumn(std::make_tuple(6), 10, date_time_with_fsp_data[0]))); /// case 2, func(nullable(not null const)) ASSERT_COLUMN_EQ( - createConstColumn>(10, date_result[0]), + createConstColumn(10, date_result[0]), executeFunction(func_name_int, createConstColumn>(10, date_data[0]))); ASSERT_COLUMN_EQ( - createConstColumn>(10, date_time_int_result[0]), + createConstColumn(10, date_time_int_result[0]), executeFunction(func_name_int, createConstColumn>(std::make_tuple(0), 10, date_time_data[0]))); ASSERT_COLUMN_EQ( - createConstColumn>(std::make_tuple(12, 0), 10, date_time_decimal_result[0]), + createConstColumn(std::make_tuple(12, 0), 10, date_time_decimal_result[0]), executeFunction(func_name_dec, createConstColumn>(std::make_tuple(0), 10, date_time_data[0]))); ASSERT_COLUMN_EQ( - createConstColumn>(std::make_tuple(18, 6), 10, date_time_with_fsp_result[0]), + createConstColumn(std::make_tuple(18, 6), 10, date_time_with_fsp_result[0]), executeFunction(func_name_dec, createConstColumn>(std::make_tuple(6), 10, date_time_with_fsp_data[0]))); /// case 3, func(nullable(null const)) ASSERT_COLUMN_EQ( diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index a0adef5b50d..a0345daaa75 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -67,10 +68,8 @@ #include #include -#include #include -#include - +#include namespace ProfileEvents { @@ -1442,19 +1441,33 @@ void Context::dropCaches() const } BackgroundProcessingPool & Context::getBackgroundPool() +{ + // Note: shared->background_pool should be initialized first. + auto lock = getLock(); + return *shared->background_pool; +} + +BackgroundProcessingPool & Context::initializeBackgroundPool(UInt16 pool_size) { auto lock = getLock(); if (!shared->background_pool) - shared->background_pool = std::make_shared(settings.background_pool_size); + shared->background_pool = std::make_shared(pool_size); return *shared->background_pool; } BackgroundProcessingPool & Context::getBlockableBackgroundPool() { - // TODO: choose a better thread pool size and maybe a better name for the pool + // TODO: maybe a better name for the pool + // Note: shared->blockable_background_pool should be initialized first. + auto lock = getLock(); + return *shared->blockable_background_pool; +} + +BackgroundProcessingPool & Context::initializeBlockableBackgroundPool(UInt16 pool_size) +{ auto lock = getLock(); if (!shared->blockable_background_pool) - shared->blockable_background_pool = std::make_shared(settings.background_pool_size); + shared->blockable_background_pool = std::make_shared(pool_size); return *shared->blockable_background_pool; } @@ -1879,6 +1892,30 @@ SharedQueriesPtr Context::getSharedQueries() return shared->shared_queries; } +size_t Context::getMaxStreams() const +{ + size_t max_streams = settings.max_threads; + bool is_cop_request = false; + if (dag_context != nullptr) + { + if (dag_context->isTest()) + max_streams = dag_context->initialize_concurrency; + else if (!dag_context->isBatchCop() && !dag_context->isMPPTask()) + { + is_cop_request = true; + max_streams = 1; + } + } + if (max_streams > 1) + max_streams *= settings.max_streams_to_max_threads_ratio; + if (max_streams == 0) + max_streams = 1; + if (unlikely(max_streams != 1 && is_cop_request)) + /// for cop request, the max_streams should be 1 + throw Exception("Cop request only support running with max_streams = 1"); + return max_streams; +} + SessionCleaner::~SessionCleaner() { try diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 63aefcbece9..434179e1ab8 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include #include @@ -381,7 +380,9 @@ class Context bool useL0Opt() const; BackgroundProcessingPool & getBackgroundPool(); + BackgroundProcessingPool & initializeBackgroundPool(UInt16 pool_size); BackgroundProcessingPool & getBlockableBackgroundPool(); + BackgroundProcessingPool & initializeBlockableBackgroundPool(UInt16 pool_size); void createTMTContext(const TiFlashRaftConfig & raft_config, pingcap::ClusterConfig && cluster_config); @@ -460,6 +461,8 @@ class Context void reloadDeltaTreeConfig(const Poco::Util::AbstractConfiguration & config); + size_t getMaxStreams() const; + private: /** Check if the current client has access to the specified database. * If access is denied, throw an exception. @@ -504,7 +507,7 @@ class DDLGuard class SessionCleaner { public: - SessionCleaner(Context & context_) + explicit SessionCleaner(Context & context_) : context{context_} {} ~SessionCleaner(); diff --git a/dbms/src/Interpreters/IDAsPathUpgrader.cpp b/dbms/src/Interpreters/IDAsPathUpgrader.cpp index 8c807b537e9..9aa3dcb8dd0 100644 --- a/dbms/src/Interpreters/IDAsPathUpgrader.cpp +++ b/dbms/src/Interpreters/IDAsPathUpgrader.cpp @@ -487,6 +487,7 @@ bool IDAsPathUpgrader::needUpgrade() if (db_info.engine != "TiFlash") { has_old_db_engine = true; + LOG_FMT_INFO(log, "Find old style of database engine, doing upgrade [path={}] [engine={}]", database_metadata_file, db_info.engine); } } diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index 87d007c101f..f2b3bbbd7fe 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -80,8 +80,8 @@ struct Settings M(SettingBool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.") \ M(SettingBool, use_uncompressed_cache, true, "Whether to use the cache of uncompressed blocks.") \ M(SettingBool, replace_running_query, false, "Whether the running request should be canceled with the same id as the new one.") \ - M(SettingUInt64, background_pool_size, DBMS_DEFAULT_BACKGROUND_POOL_SIZE, "Number of threads performing background work for tables (for example, merging in merge tree). Only has meaning at server " \ - "startup.") \ + M(SettingUInt64, background_pool_size, 0, "Number of threads performing background work for tables (for example, merging in merge tree). Only effective at server startup. " \ + "0 means a quarter of the number of logical CPU cores of the machine.") \ \ M(SettingBool, optimize_move_to_prewhere, true, "Allows disabling WHERE to PREWHERE optimization in SELECT queries from MergeTree.") \ \ @@ -272,7 +272,7 @@ struct Settings M(SettingUInt64, dt_segment_delta_small_column_file_size, 8388608, "Determine whether a column file in delta is small or not. 8MB by default.") \ M(SettingUInt64, dt_segment_stable_pack_rows, DEFAULT_MERGE_BLOCK_SIZE, "Expected stable pack rows in DeltaTree Engine.") \ M(SettingFloat, dt_segment_wait_duration_factor, 1, "The factor of wait duration in a write stall.") \ - M(SettingUInt64, dt_bg_gc_check_interval, 5, "Background gc thread check interval, the unit is second.") \ + M(SettingUInt64, dt_bg_gc_check_interval, 60, "Background gc thread check interval, the unit is second.") \ M(SettingInt64, dt_bg_gc_max_segments_to_check_every_round, 100, "Max segments to check in every gc round, value less than or equal to 0 means gc no segments.") \ M(SettingFloat, dt_bg_gc_ratio_threhold_to_trigger_gc, 1.2, "Trigger segment's gc when the ratio of invalid version exceed this threhold. Values smaller than or equal to 1.0 means gc all " \ "segments") \ @@ -356,7 +356,7 @@ struct Settings M(SettingUInt64, elastic_threadpool_shrink_period_ms, 300000, "The shrink period(ms) of elastic thread pool.") \ M(SettingBool, enable_local_tunnel, true, "Enable local data transfer between local MPP tasks.") \ M(SettingBool, enable_async_grpc_client, true, "Enable async grpc in MPP.") \ - M(SettingUInt64, grpc_completion_queue_pool_size, 0, "The size of gRPC completion queue pool. 0 means using hardware_concurrency.")\ + M(SettingUInt64, grpc_completion_queue_pool_size, 0, "The size of gRPC completion queue pool. 0 means the number of logical CPU cores. Only effective at server startup")\ M(SettingBool, enable_async_server, true, "Enable async rpc server.") \ M(SettingUInt64, async_pollers_per_cq, 200, "grpc async pollers per cqs") \ M(SettingUInt64, async_cqs, 1, "grpc async cqs") \ diff --git a/dbms/src/Interpreters/WindowDescription.cpp b/dbms/src/Interpreters/WindowDescription.cpp index 2ab407bb18e..09d96411673 100644 --- a/dbms/src/Interpreters/WindowDescription.cpp +++ b/dbms/src/Interpreters/WindowDescription.cpp @@ -44,7 +44,7 @@ WindowFrame::FrameType getFrameTypeFromTipb(const tipb::WindowFrameType & type) return WindowFrame::FrameType::Groups; default: throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unknowed frame type {}", + "Unknown frame type {}", type); } } @@ -60,4 +60,38 @@ void WindowDescription::setWindowFrame(const tipb::WindowFrame & frame_) frame.end_preceding = (frame_.end().type() == tipb::WindowBoundType::Preceding); frame.is_default = false; } + +String frameTypeToString(const WindowFrame::FrameType & type) +{ + switch (type) + { + case WindowFrame::FrameType::Rows: + return "Rows"; + case WindowFrame::FrameType::Groups: + return "Groups"; + case WindowFrame::FrameType::Ranges: + return "Ranges"; + default: + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Unknown frame type {}", + type); + } +} + +String boundaryTypeToString(const WindowFrame::BoundaryType & type) +{ + switch (type) + { + case WindowFrame::BoundaryType::Unbounded: + return "Unbounded"; + case WindowFrame::BoundaryType::Current: + return "Current"; + case WindowFrame::BoundaryType::Offset: + return "Offset"; + default: + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Unknown boundary type {}", + type); + } +} } // namespace DB diff --git a/dbms/src/Interpreters/WindowDescription.h b/dbms/src/Interpreters/WindowDescription.h index cdcade1b750..a3c2bac5747 100644 --- a/dbms/src/Interpreters/WindowDescription.h +++ b/dbms/src/Interpreters/WindowDescription.h @@ -87,6 +87,10 @@ struct WindowFrame && other.end_preceding == end_preceding; } }; + +String frameTypeToString(const WindowFrame::FrameType & type); +String boundaryTypeToString(const WindowFrame::BoundaryType & type); + class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; struct WindowDescription diff --git a/dbms/src/Server/CLIService.h b/dbms/src/Server/CLIService.h new file mode 100644 index 00000000000..9078fa991f3 --- /dev/null +++ b/dbms/src/Server/CLIService.h @@ -0,0 +1,236 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +using RaftStoreFFIFunc = void (*)(int argc, const char * const * argv, const DB::EngineStoreServerHelper *); + +template +struct CLIService : public BaseDaemon +{ + struct TiFlashProxyConfig + { + static const std::string config_prefix; + std::vector args; + std::unordered_map val_map; + bool is_proxy_runnable = false; + + static constexpr char ENGINE_STORE_VERSION[] = "engine-version"; + static constexpr char ENGINE_STORE_GIT_HASH[] = "engine-git-hash"; + static constexpr char ENGINE_STORE_ADDRESS[] = "engine-addr"; + static constexpr char ENGINE_STORE_ADVERTISE_ADDRESS[] = "advertise-engine-addr"; + static constexpr char PD_ENDPOINTS[] = "pd-endpoints"; + static constexpr char ENGINE_LABEL[] = "engine-label"; + static constexpr char ENGINE_LABEL_VALUE[] = "tiflash"; + + explicit TiFlashProxyConfig(Poco::Util::LayeredConfiguration & config); + }; + + struct RaftStoreProxyRunner : boost::noncopyable + { + struct RunRaftStoreProxyParms + { + const DB::EngineStoreServerHelper * helper; + const TiFlashProxyConfig & conf; + const RaftStoreFFIFunc ffi_function; + + /// set big enough stack size to avoid runtime error like stack-overflow. + size_t stack_size = 1024 * 1024 * 20; + }; + + explicit RaftStoreProxyRunner(RunRaftStoreProxyParms && parms_); + + void join(); + + void run(); + + private: + static void * runRaftStoreProxyFfi(void * pv); + + private: + RunRaftStoreProxyParms parms; + pthread_t thread; + }; + + Func func; + RaftStoreFFIFunc ffi_function; + const Args & args; + std::unique_ptr global_context; + + explicit CLIService(Func func_, const Args & args_, const std::string & config_file, RaftStoreFFIFunc ffi_function = nullptr); + + int main(const std::vector &) override; +}; + +template +CLIService::TiFlashProxyConfig::TiFlashProxyConfig(Poco::Util::LayeredConfiguration & config) +{ + if (!config.has(config_prefix)) + return; + + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(config_prefix, keys); + { + std::unordered_map args_map; + for (const auto & key : keys) + { + const auto k = config_prefix + "." + key; + args_map[key] = config.getString(k); + } + args_map[PD_ENDPOINTS] = config.getString("raft.pd_addr"); + args_map[ENGINE_STORE_VERSION] = TiFlashBuildInfo::getReleaseVersion(); + args_map[ENGINE_STORE_GIT_HASH] = TiFlashBuildInfo::getGitHash(); + if (!args_map.count(ENGINE_STORE_ADDRESS)) + args_map[ENGINE_STORE_ADDRESS] = config.getString("flash.service_addr"); + else + args_map[ENGINE_STORE_ADVERTISE_ADDRESS] = args_map[ENGINE_STORE_ADDRESS]; + args_map[ENGINE_LABEL] = ENGINE_LABEL_VALUE; + + for (auto && [k, v] : args_map) + { + val_map.emplace("--" + k, std::move(v)); + } + } + + args.push_back("TiFlash Proxy"); + for (const auto & v : val_map) + { + args.push_back(v.first.data()); + args.push_back(v.second.data()); + } + is_proxy_runnable = true; +} +template +CLIService::RaftStoreProxyRunner::RaftStoreProxyRunner(CLIService::RaftStoreProxyRunner::RunRaftStoreProxyParms && parms_) + : parms(std::move(parms_)) +{} +template +void CLIService::RaftStoreProxyRunner::join() +{ + if (!parms.conf.is_proxy_runnable) + return; + pthread_join(thread, nullptr); +} +template +void CLIService::RaftStoreProxyRunner::run() +{ + if (!parms.conf.is_proxy_runnable) + return; + pthread_attr_t attribute; + pthread_attr_init(&attribute); + pthread_attr_setstacksize(&attribute, parms.stack_size); + pthread_create(&thread, &attribute, runRaftStoreProxyFfi, &parms); + pthread_attr_destroy(&attribute); +} +template +void * CLIService::RaftStoreProxyRunner::runRaftStoreProxyFfi(void * pv) +{ + auto & parms = *static_cast(pv); + if (nullptr == parms.ffi_function) + { + throw DB::Exception("proxy is not available"); + } + parms.ffi_function(static_cast(parms.conf.args.size()), parms.conf.args.data(), parms.helper); + return nullptr; +} + +template +CLIService::CLIService(Func func_, const Args & args_, const std::string & config_file, RaftStoreFFIFunc ffi_function) + : func(std::move(func_)) + , ffi_function(ffi_function) + , args(args_) +{ + config_path = config_file; + ConfigProcessor config_processor(config_file); + auto loaded_config = config_processor.loadConfig(); + BaseDaemon::config().add(loaded_config.configuration); + BaseDaemon::config().setString("config-file", config_file); +} + +template +int CLIService::main(const std::vector &) +{ + using namespace DB; + TiFlashProxyConfig proxy_conf(config()); + EngineStoreServerWrap tiflash_instance_wrap{}; + auto helper = GetEngineStoreServerHelper( + &tiflash_instance_wrap); + + typename RaftStoreProxyRunner::RunRaftStoreProxyParms parms{&helper, proxy_conf, ffi_function}; + RaftStoreProxyRunner proxy_runner(std::move(parms)); + + proxy_runner.run(); + + if (proxy_conf.is_proxy_runnable) + { + while (!tiflash_instance_wrap.proxy_helper) + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + SCOPE_EXIT({ + if (!proxy_conf.is_proxy_runnable) + { + proxy_runner.join(); + return; + } + tiflash_instance_wrap.status = EngineStoreServerStatus::Terminated; + tiflash_instance_wrap.tmt = nullptr; + proxy_runner.join(); + }); + + global_context = std::make_unique(Context::createGlobal()); + global_context->setGlobalContext(*global_context); + global_context->setApplicationType(Context::ApplicationType::SERVER); + + /// Init File Provider + if (proxy_conf.is_proxy_runnable) + { + bool enable_encryption = tiflash_instance_wrap.proxy_helper->checkEncryptionEnabled(); + if (enable_encryption) + { + auto method = tiflash_instance_wrap.proxy_helper->getEncryptionMethod(); + enable_encryption = (method != EncryptionMethod::Plaintext); + } + KeyManagerPtr key_manager = std::make_shared(&tiflash_instance_wrap); + global_context->initializeFileProvider(key_manager, enable_encryption); + } + else + { + KeyManagerPtr key_manager = std::make_shared(false); + global_context->initializeFileProvider(key_manager, false); + } + + return func(*global_context, args); +} + + +template +inline const std::string CLIService::TiFlashProxyConfig::config_prefix = "flash.proxy"; diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt index 6c3d289dea6..2948bb076db 100644 --- a/dbms/src/Server/CMakeLists.txt +++ b/dbms/src/Server/CMakeLists.txt @@ -22,6 +22,8 @@ option(ENABLE_CLICKHOUSE_SERVER "Enable server" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_CLICKHOUSE_CLIENT "Enable client" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_TIFLASH_DTTOOL "Enable dttool: tools to manage dmfile" ${ENABLE_CLICKHOUSE_ALL}) option(ENABLE_TIFLASH_DTWORKLOAD "Enable dtworkload: tools to test and stress DeltaTree" ${ENABLE_CLICKHOUSE_ALL}) +option(ENABLE_TIFLASH_PAGEWORKLOAD "Enable pageworkload: tools to test and stress PageStorage" ${ENABLE_CLICKHOUSE_ALL}) +option(ENABLE_TIFLASH_PAGECTL "Enable pagectl: tools to debug page storage" ${ENABLE_CLICKHOUSE_ALL}) configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) @@ -32,6 +34,7 @@ add_library (clickhouse-server-lib NotFoundHandler.cpp PingRequestHandler.cpp RootRequestHandler.cpp + ServerInfo.cpp Server.cpp StatusFile.cpp TCPHandler.cpp @@ -135,6 +138,12 @@ endif () if (ENABLE_TIFLASH_DTWORKLOAD) target_link_libraries(tiflash dt-workload-lib) endif () +if (ENABLE_TIFLASH_PAGEWORKLOAD) + target_link_libraries(tiflash page-workload-lib) +endif() +if (ENABLE_TIFLASH_PAGECTL) + target_link_libraries(tiflash page-ctl-lib) +endif () # install always because depian package want this files: install (TARGETS tiflash RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT tiflash) diff --git a/dbms/src/Server/DTTool/DTTool.h b/dbms/src/Server/DTTool/DTTool.h index 911c29bf98b..6236bd6cdb9 100644 --- a/dbms/src/Server/DTTool/DTTool.h +++ b/dbms/src/Server/DTTool/DTTool.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -76,205 +77,6 @@ int migrateEntry(const std::vector & opts, RaftStoreFFIFunc ffi_fun namespace DTTool { -template -struct CLIService : public BaseDaemon -{ - struct TiFlashProxyConfig - { - static const std::string config_prefix; - std::vector args; - std::unordered_map val_map; - bool is_proxy_runnable = false; - - static constexpr char ENGINE_STORE_VERSION[] = "engine-version"; - static constexpr char ENGINE_STORE_GIT_HASH[] = "engine-git-hash"; - static constexpr char ENGINE_STORE_ADDRESS[] = "engine-addr"; - static constexpr char ENGINE_STORE_ADVERTISE_ADDRESS[] = "advertise-engine-addr"; - static constexpr char PD_ENDPOINTS[] = "pd-endpoints"; - - explicit TiFlashProxyConfig(Poco::Util::LayeredConfiguration & config); - }; - - struct RaftStoreProxyRunner : boost::noncopyable - { - struct RunRaftStoreProxyParms - { - const DB::EngineStoreServerHelper * helper; - const TiFlashProxyConfig & conf; - const RaftStoreFFIFunc ffi_function; - - /// set big enough stack size to avoid runtime error like stack-overflow. - size_t stack_size = 1024 * 1024 * 20; - }; - - explicit RaftStoreProxyRunner(RunRaftStoreProxyParms && parms_); - - void join(); - - void run(); - - private: - static void * runRaftStoreProxyFfi(void * pv); - - private: - RunRaftStoreProxyParms parms; - pthread_t thread; - }; - - Func func; - RaftStoreFFIFunc ffi_function; - const Args & args; - std::unique_ptr global_context; - - explicit CLIService(Func func_, const Args & args_, const std::string & config_file, RaftStoreFFIFunc ffi_function = nullptr); - - int main(const std::vector &) override; -}; - -template -CLIService::TiFlashProxyConfig::TiFlashProxyConfig(Poco::Util::LayeredConfiguration & config) -{ - if (!config.has(config_prefix)) - return; - - Poco::Util::AbstractConfiguration::Keys keys; - config.keys(config_prefix, keys); - { - std::unordered_map args_map; - for (const auto & key : keys) - { - const auto k = config_prefix + "." + key; - args_map[key] = config.getString(k); - } - args_map[PD_ENDPOINTS] = config.getString("raft.pd_addr"); - args_map[ENGINE_STORE_VERSION] = TiFlashBuildInfo::getReleaseVersion(); - args_map[ENGINE_STORE_GIT_HASH] = TiFlashBuildInfo::getGitHash(); - if (!args_map.count(ENGINE_STORE_ADDRESS)) - args_map[ENGINE_STORE_ADDRESS] = config.getString("flash.service_addr"); - else - args_map[ENGINE_STORE_ADVERTISE_ADDRESS] = args_map[ENGINE_STORE_ADDRESS]; - - for (auto && [k, v] : args_map) - { - val_map.emplace("--" + k, std::move(v)); - } - } - - args.push_back("TiFlash Proxy"); - for (const auto & v : val_map) - { - args.push_back(v.first.data()); - args.push_back(v.second.data()); - } - is_proxy_runnable = true; -} -template -CLIService::RaftStoreProxyRunner::RaftStoreProxyRunner(CLIService::RaftStoreProxyRunner::RunRaftStoreProxyParms && parms_) - : parms(std::move(parms_)) -{} -template -void CLIService::RaftStoreProxyRunner::join() -{ - if (!parms.conf.is_proxy_runnable) - return; - pthread_join(thread, nullptr); -} -template -void CLIService::RaftStoreProxyRunner::run() -{ - if (!parms.conf.is_proxy_runnable) - return; - pthread_attr_t attribute; - pthread_attr_init(&attribute); - pthread_attr_setstacksize(&attribute, parms.stack_size); - pthread_create(&thread, &attribute, runRaftStoreProxyFfi, &parms); - pthread_attr_destroy(&attribute); -} -template -void * CLIService::RaftStoreProxyRunner::runRaftStoreProxyFfi(void * pv) -{ - auto & parms = *static_cast(pv); - if (nullptr == parms.ffi_function) - { - throw DB::Exception("proxy is not available"); - } - parms.ffi_function(static_cast(parms.conf.args.size()), parms.conf.args.data(), parms.helper); - return nullptr; -} - -template -CLIService::CLIService(Func func_, const Args & args_, const std::string & config_file, RaftStoreFFIFunc ffi_function) - : func(std::move(func_)) - , ffi_function(ffi_function) - , args(args_) -{ - config_path = config_file; - ConfigProcessor config_processor(config_file); - auto loaded_config = config_processor.loadConfig(); - BaseDaemon::config().add(loaded_config.configuration); - BaseDaemon::config().setString("config-file", config_file); -} - -template -int CLIService::main(const std::vector &) -{ - using namespace DB; - TiFlashProxyConfig proxy_conf(config()); - EngineStoreServerWrap tiflash_instance_wrap{}; - auto helper = GetEngineStoreServerHelper( - &tiflash_instance_wrap); - - typename RaftStoreProxyRunner::RunRaftStoreProxyParms parms{&helper, proxy_conf, ffi_function}; - RaftStoreProxyRunner proxy_runner(std::move(parms)); - - proxy_runner.run(); - - if (proxy_conf.is_proxy_runnable) - { - while (!tiflash_instance_wrap.proxy_helper) - std::this_thread::sleep_for(std::chrono::milliseconds(200)); - } - - SCOPE_EXIT({ - if (!proxy_conf.is_proxy_runnable) - { - proxy_runner.join(); - return; - } - tiflash_instance_wrap.status = EngineStoreServerStatus::Terminated; - tiflash_instance_wrap.tmt = nullptr; - proxy_runner.join(); - }); - - global_context = std::make_unique(Context::createGlobal()); - global_context->setGlobalContext(*global_context); - global_context->setApplicationType(Context::ApplicationType::SERVER); - - /// Init File Provider - if (proxy_conf.is_proxy_runnable) - { - bool enable_encryption = tiflash_instance_wrap.proxy_helper->checkEncryptionEnabled(); - if (enable_encryption) - { - auto method = tiflash_instance_wrap.proxy_helper->getEncryptionMethod(); - enable_encryption = (method != EncryptionMethod::Plaintext); - } - KeyManagerPtr key_manager = std::make_shared(&tiflash_instance_wrap); - global_context->initializeFileProvider(key_manager, enable_encryption); - } - else - { - KeyManagerPtr key_manager = std::make_shared(false); - global_context->initializeFileProvider(key_manager, false); - } - - return func(*global_context, args); -} - - -template -inline const std::string CLIService::TiFlashProxyConfig::config_prefix = "flash.proxy"; - namespace detail { using namespace DB; diff --git a/dbms/src/Server/RaftConfigParser.cpp b/dbms/src/Server/RaftConfigParser.cpp index 8e146dd842e..2f0a88855cd 100644 --- a/dbms/src/Server/RaftConfigParser.cpp +++ b/dbms/src/Server/RaftConfigParser.cpp @@ -92,11 +92,7 @@ TiFlashRaftConfig TiFlashRaftConfig::parseSettings(Poco::Util::LayeredConfigurat { String snapshot_method = config.getString("raft.snapshot.method"); std::transform(snapshot_method.begin(), snapshot_method.end(), snapshot_method.begin(), [](char ch) { return std::tolower(ch); }); - if (snapshot_method == "block") - { - res.snapshot_apply_method = TiDB::SnapshotApplyMethod::Block; - } - else if (snapshot_method == "file1") + if (snapshot_method == "file1") { res.snapshot_apply_method = TiDB::SnapshotApplyMethod::DTFile_Directory; } diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 04676ef969d..95c1d5d3f2a 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "Server.h" - #include #include #include @@ -55,8 +53,15 @@ #include #include #include +#include +#include +#include #include +#include +#include +#include #include +#include #include #include #include @@ -72,7 +77,6 @@ #include #include #include -#include #include #include @@ -82,12 +86,6 @@ #include #include -#include "HTTPHandlerFactory.h" -#include "MetricsPrometheus.h" -#include "MetricsTransmitter.h" -#include "StatusFile.h" -#include "TCPHandlerFactory.h" - #if Poco_NetSSL_FOUND #include #include @@ -1049,6 +1047,23 @@ int Server::main(const std::vector & /*args*/) LOG_FMT_INFO(log, "tiflash proxy thread is joined"); }); + /// get CPU/memory/disk info of this server + if (tiflash_instance_wrap.proxy_helper) + { + diagnosticspb::ServerInfoRequest request; + request.set_tp(static_cast(1)); + diagnosticspb::ServerInfoResponse response; + std::string req = request.SerializeAsString(); + auto * helper = tiflash_instance_wrap.proxy_helper; + helper->fn_server_info(helper->proxy_ptr, strIntoView(&req), &response); + server_info.parseSysInfo(response); + LOG_FMT_INFO(log, "ServerInfo: {}", server_info.debugString()); + } + else + { + LOG_FMT_INFO(log, "TiFlashRaftProxyHelper is null, failed to get server info"); + } + CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); // print necessary grpc log. @@ -1119,6 +1134,19 @@ int Server::main(const std::vector & /*args*/) global_context->getPathCapacity(), global_context->getFileProvider()); + /// if default value of background_pool_size is 0 + /// set it to the a quarter of the number of logical CPU cores of machine. + Settings & settings = global_context->getSettingsRef(); + if (settings.background_pool_size == 0) + { + global_context->setSetting("background_pool_size", std::to_string(server_info.cpu_info.logical_cores / 4)); + } + LOG_FMT_INFO(log, "Background & Blockable Background pool size: {}", settings.background_pool_size); + + /// Initialize the background & blockable background thread pool. + auto & bg_pool = global_context->initializeBackgroundPool(settings.background_pool_size); + auto & blockable_bg_pool = global_context->initializeBlockableBackgroundPool(settings.background_pool_size); + global_context->initializePageStorageMode(global_context->getPathPool(), STORAGE_FORMAT_CURRENT.page); global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool()); LOG_FMT_INFO(log, "Global PageStorage run mode is {}", static_cast(global_context->getPageStorageRunMode())); @@ -1235,13 +1263,6 @@ int Server::main(const std::vector & /*args*/) /// Load global settings from default_profile and system_profile. /// It internally depends on UserConfig::parseSettings. global_context->setDefaultProfiles(config()); - Settings & settings = global_context->getSettingsRef(); - - /// Initialize the background thread pool. - /// It internally depends on settings.background_pool_size, - /// so must be called after settings has been load. - auto & bg_pool = global_context->getBackgroundPool(); - auto & blockable_bg_pool = global_context->getBlockableBackgroundPool(); /// Initialize RateLimiter. global_context->initializeRateLimiter(config(), bg_pool, blockable_bg_pool); @@ -1393,7 +1414,7 @@ int Server::main(const std::vector & /*args*/) { auto size = settings.grpc_completion_queue_pool_size; if (size == 0) - size = std::thread::hardware_concurrency(); + size = server_info.cpu_info.logical_cores; GRPCCompletionQueuePool::global_instance = std::make_unique(size); } @@ -1410,10 +1431,10 @@ int Server::main(const std::vector & /*args*/) // on ARM processors it can show only enabled at current moment cores LOG_FMT_INFO( log, - "Available RAM = {}; physical cores = {}; threads = {}.", - formatReadableSizeWithBinarySuffix(getMemoryAmount()), - getNumberOfPhysicalCPUCores(), - std::thread::hardware_concurrency()); + "Available RAM = {}; physical cores = {}; logical cores = {}.", + server_info.memory_info.capacity, + server_info.cpu_info.physical_cores, + server_info.cpu_info.logical_cores); } LOG_FMT_INFO(log, "Ready for connections."); diff --git a/dbms/src/Server/Server.h b/dbms/src/Server/Server.h index 278349f2aa4..07c5b955a92 100644 --- a/dbms/src/Server/Server.h +++ b/dbms/src/Server/Server.h @@ -14,10 +14,10 @@ #pragma once +#include +#include #include -#include "IServer.h" - /** Server provides three interfaces: * 1. HTTP - simple interface for any applications. * 2. TCP - interface for native clickhouse-client and for server to server internal communications. @@ -39,7 +39,7 @@ class Server : public BaseDaemon return BaseDaemon::config(); } - virtual const TiFlashSecurityConfig & securityConfig() const override { return security_config; }; + const TiFlashSecurityConfig & securityConfig() const override { return security_config; }; Poco::Logger & logger() const override { @@ -70,6 +70,8 @@ class Server : public BaseDaemon TiFlashSecurityConfig security_config; + ServerInfo server_info; + class FlashGrpcServerHolder; class TcpHttpServersHolder; }; diff --git a/dbms/src/Server/ServerInfo.cpp b/dbms/src/Server/ServerInfo.cpp new file mode 100644 index 00000000000..9cba40c4775 --- /dev/null +++ b/dbms/src/Server/ServerInfo.cpp @@ -0,0 +1,199 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include + +namespace DB +{ +using diagnosticspb::ServerInfoItem; +using diagnosticspb::ServerInfoResponse; + +void ServerInfo::parseCPUInfo(const diagnosticspb::ServerInfoItem & cpu_info_item) +{ + for (const auto & pair : cpu_info_item.pairs()) + { + const auto & key = pair.key(); + if (key == "cpu-logical-cores") + { + cpu_info.logical_cores = static_cast(std::stoi(pair.value())); + } + else if (key == "cpu-physical-cores") + { + cpu_info.physical_cores = static_cast(std::stoi(pair.value())); + } + else if (key == "cpu-frequency") + { + cpu_info.frequency = pair.value(); + } + else if (key == "l1-cache-size") + { + cpu_info.l1_cache_size = static_cast(std::stoull(pair.value())); + } + else if (key == "l1-cache-line-size") + { + cpu_info.l1_cache_line_size = static_cast(std::stoi(pair.value())); + } + else if (key == "l2-cache-size") + { + cpu_info.l2_cache_size = static_cast(std::stoull(pair.value())); + } + else if (key == "l2-cache-line-size") + { + cpu_info.l2_cache_line_size = static_cast(std::stoi(pair.value())); + } + else if (key == "l3-cache-size") + { + cpu_info.l3_cache_size = static_cast(std::stoull(pair.value())); + } + else if (key == "l3-cache-line-size") + { + cpu_info.l3_cache_line_size = static_cast(std::stoi(pair.value())); + } + else if (key == "cpu-arch") + { + cpu_info.arch = pair.value(); + } + } +} + +void ServerInfo::parseDiskInfo(const diagnosticspb::ServerInfoItem & disk_info_item) +{ + Disk disk; + disk.name = disk_info_item.name(); + for (const auto & pair : disk_info_item.pairs()) + { + const auto & key = pair.key(); + if (key == "type") + { + if (pair.value() == "HDD") + { + disk.disk_type = Disk::DiskType::HDD; + } + else if (pair.value() == "SSD") + { + disk.disk_type = Disk::DiskType::SSD; + } + else + { + disk.disk_type = Disk::DiskType::UNKNOWN; + } + } + else if (key == "total") + { + disk.total_space = static_cast(std::stoull(pair.value())); + } + else if (key == "free") + { + disk.free_space = static_cast(std::stoull(pair.value())); + } + else if (key == "path") + { + disk.mount_point = pair.value(); + } + else if (key == "fstype") + { + disk.fs_type = pair.value(); + } + } + disk_infos.push_back(disk); +} + +void ServerInfo::parseMemoryInfo(const diagnosticspb::ServerInfoItem & memory_info_item) +{ + for (const auto & pair : memory_info_item.pairs()) + { + if (pair.key() == "capacity") + { + memory_info.capacity = std::stoull(pair.value()); + } + } +} + +void ServerInfo::parseSysInfo(const diagnosticspb::ServerInfoResponse & sys_info_response) +{ + for (const auto & item : sys_info_response.items()) + { + const auto & tp = item.tp(); + if (tp == "cpu") + { + parseCPUInfo(item); + } + else if (tp == "disk") + { + parseDiskInfo(item); + } + else if (tp == "memory") + { + parseMemoryInfo(item); + } + } +} + +String ServerInfo::debugString() const +{ + FmtBuffer fmt_buf; + // append cpu info + fmt_buf.fmtAppend("CPU: \n" + " logical cores: {}\n" + " physical cores: {}\n" + " frequency: {}\n" + " l1 cache size: {}\n" + " l1 cache line size: {}\n" + " l2 cache size: {}\n" + " l2 cache line size: {}\n" + " l3 cache size: {}\n" + " l3 cache line size: {}\n" + " arch: {}\n", + cpu_info.logical_cores, + cpu_info.physical_cores, + cpu_info.frequency, + cpu_info.l1_cache_size, + cpu_info.l1_cache_line_size, + cpu_info.l2_cache_size, + cpu_info.l2_cache_line_size, + cpu_info.l3_cache_size, + cpu_info.l3_cache_line_size, + cpu_info.arch); + // append disk info + { + const static String disk_type_str[] = {"UNKNOWN", "HDD", "SSD"}; + for (const auto & disk_info : disk_infos) + { + fmt_buf.fmtAppend("Disk: \n" + " name: {}\n" + " type: {}\n" + " total space: {}\n" + " free space: {}\n" + " mount point: {}\n" + " fstype: {}\n", + disk_info.name, + disk_type_str[static_cast(disk_info.disk_type)], + disk_info.total_space, + disk_info.free_space, + disk_info.mount_point, + disk_info.fs_type); + } + } + // append memory info + fmt_buf.fmtAppend("Memory: \n" + " capacity: {}\n", + memory_info.capacity); + + return fmt_buf.toString(); +} + +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Server/ServerInfo.h b/dbms/src/Server/ServerInfo.h new file mode 100644 index 00000000000..9663bd37568 --- /dev/null +++ b/dbms/src/Server/ServerInfo.h @@ -0,0 +1,99 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include +#include + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif +#include +#pragma GCC diagnostic pop + +namespace DB +{ +class ServerInfo +{ +public: + struct CPUInfo + { + /// number of logical CPU cores + UInt16 logical_cores = std::thread::hardware_concurrency(); + /// number of physical CPU cores + UInt16 physical_cores = getNumberOfPhysicalCPUCores(); + /// number of L1 cache size + /// units: Byte + UInt32 l1_cache_size = 16384; // 16KB (typical value) + /// number of L2 cache size + /// units: Byte + UInt32 l2_cache_size = 65536; // 64KB (typical value) + /// number of L3 cache size + /// units: Byte + UInt32 l3_cache_size = 2097152; // 2MB (typical value) + /// number of L1 cache line size + UInt8 l1_cache_line_size = 64; // 64B (typical value) + /// number of L2 cache line size + UInt8 l2_cache_line_size = 64; // 64B (typical value) + /// number of L3 cache line size + UInt8 l3_cache_line_size = 64; // 64B (typical value) + /// CPU architecture + String arch; + /// CPU frequency + String frequency; + }; + + struct Disk + { + String name; + enum DiskType + { + UNKNOWN = 0, + HDD = 1, + SSD = 2 + }; + DiskType disk_type; + UInt64 total_space = 0; + UInt64 free_space = 0; + String mount_point; + String fs_type; + }; + using DiskInfo = std::vector; + + struct MemoryInfo + { + /// total memory size + /// units: Byte + UInt64 capacity = getMemoryAmount(); + }; + + ServerInfo() = default; + ~ServerInfo() = default; + void parseCPUInfo(const diagnosticspb::ServerInfoItem & cpu_info_item); + void parseDiskInfo(const diagnosticspb::ServerInfoItem & disk_info_item); + void parseMemoryInfo(const diagnosticspb::ServerInfoItem & memory_info_item); + void parseSysInfo(const diagnosticspb::ServerInfoResponse & sys_info_response); + String debugString() const; + + CPUInfo cpu_info; + DiskInfo disk_infos; + MemoryInfo memory_info; +}; +} // namespace DB diff --git a/dbms/src/Server/config_tools.h.in b/dbms/src/Server/config_tools.h.in index 03df94cc8e1..03a478a6473 100644 --- a/dbms/src/Server/config_tools.h.in +++ b/dbms/src/Server/config_tools.h.in @@ -6,3 +6,5 @@ #cmakedefine01 ENABLE_CLICKHOUSE_CLIENT #cmakedefine01 ENABLE_TIFLASH_DTTOOL #cmakedefine01 ENABLE_TIFLASH_DTWORKLOAD +#cmakedefine01 ENABLE_TIFLASH_PAGEWORKLOAD +#cmakedefine01 ENABLE_TIFLASH_PAGECTL diff --git a/dbms/src/Server/main.cpp b/dbms/src/Server/main.cpp index ace9dfc80ee..dbcaa4f38fc 100644 --- a/dbms/src/Server/main.cpp +++ b/dbms/src/Server/main.cpp @@ -36,7 +36,13 @@ #include #endif #if ENABLE_TIFLASH_DTWORKLOAD -#include +#include +#endif +#if ENABLE_TIFLASH_PAGEWORKLOAD +#include +#endif +#if ENABLE_TIFLASH_PAGECTL +#include #endif #include #include @@ -103,6 +109,12 @@ std::pair clickhouse_applications[] = { #endif #if ENABLE_TIFLASH_DTWORKLOAD {"dtworkload", DB::DM::tests::DTWorkload::mainEntry}, +#endif +#if ENABLE_TIFLASH_PAGEWORKLOAD + {"pageworkload", DB::PS::tests::StressWorkload::mainEntry}, +#endif +#if ENABLE_TIFLASH_PAGECTL + {"pagectl", DB::PageStorageCtl::mainEntry}, #endif {"version", mainEntryVersion}, {"errgen", mainExportError}}; diff --git a/dbms/src/Storages/BackgroundProcessingPool.cpp b/dbms/src/Storages/BackgroundProcessingPool.cpp index 96c2c6cc622..9fb4271ea38 100644 --- a/dbms/src/Storages/BackgroundProcessingPool.cpp +++ b/dbms/src/Storages/BackgroundProcessingPool.cpp @@ -87,6 +87,9 @@ BackgroundProcessingPool::BackgroundProcessingPool(int size_) : size(size_) , thread_ids_counter(size_) { + if (size <= 0) + throw Exception("BackgroundProcessingPool size must be greater than 0", ErrorCodes::LOGICAL_ERROR); + LOG_FMT_INFO(&Poco::Logger::get("BackgroundProcessingPool"), "Create BackgroundProcessingPool with {} threads", size); threads.resize(size); diff --git a/dbms/src/Storages/BackgroundProcessingPool.h b/dbms/src/Storages/BackgroundProcessingPool.h index 1ba6c4efcf8..49a01b3a397 100644 --- a/dbms/src/Storages/BackgroundProcessingPool.h +++ b/dbms/src/Storages/BackgroundProcessingPool.h @@ -81,7 +81,7 @@ class BackgroundProcessingPool using TaskHandle = std::shared_ptr; - BackgroundProcessingPool(int size_); + explicit BackgroundProcessingPool(int size_); size_t getNumberOfThreads() const { return size; } @@ -96,7 +96,7 @@ class BackgroundProcessingPool /// 2. thread B also get the same task /// 3. thread A finish the execution of the task quickly, release the task and try to update the next schedule time of the task /// 4. thread B find the task is not occupied and execute the task again almost immediately - TaskHandle addTask(const Task & task, const bool multi = true, const size_t interval_ms = 0); + TaskHandle addTask(const Task & task, bool multi = true, size_t interval_ms = 0); void removeTask(const TaskHandle & task); ~BackgroundProcessingPool(); diff --git a/dbms/src/Storages/CMakeLists.txt b/dbms/src/Storages/CMakeLists.txt index 90cc7a01d5b..68a2e6c9a74 100644 --- a/dbms/src/Storages/CMakeLists.txt +++ b/dbms/src/Storages/CMakeLists.txt @@ -15,16 +15,15 @@ add_subdirectory (System) add_subdirectory (Page) add_subdirectory (DeltaMerge/File/dtpb) -add_subdirectory (DeltaMerge/tools) +add_subdirectory (DeltaMerge/workload) +add_subdirectory (Page/workload) if (ENABLE_TESTS) add_subdirectory (tests EXCLUDE_FROM_ALL) add_subdirectory (Transaction/tests EXCLUDE_FROM_ALL) add_subdirectory (Page/V2/tests EXCLUDE_FROM_ALL) - if (ENABLE_V3_PAGESTORAGE) - add_subdirectory (Page/V3 EXCLUDE_FROM_ALL) - add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL) - endif () + add_subdirectory (Page/V3 EXCLUDE_FROM_ALL) + add_subdirectory (Page/V3/tests EXCLUDE_FROM_ALL) add_subdirectory (DeltaMerge/tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp index a74404f3dbb..195ed5c53c2 100644 --- a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp +++ b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -1137,9 +1138,11 @@ BlockInputStreams DeltaMergeStore::readRaw(const Context & db_context, } fiu_do_on(FailPoints::force_slow_page_storage_snapshot_release, { - std::thread thread_hold_snapshots([tasks]() { + std::thread thread_hold_snapshots([this, tasks]() { + LOG_FMT_WARNING(log, "failpoint force_slow_page_storage_snapshot_release begin"); std::this_thread::sleep_for(std::chrono::seconds(5 * 60)); (void)tasks; + LOG_FMT_WARNING(log, "failpoint force_slow_page_storage_snapshot_release end"); }); thread_hold_snapshots.detach(); }); diff --git a/dbms/src/Storages/DeltaMerge/StoragePool.cpp b/dbms/src/Storages/DeltaMerge/StoragePool.cpp index fa765cd9b1d..2791a74e9e3 100644 --- a/dbms/src/Storages/DeltaMerge/StoragePool.cpp +++ b/dbms/src/Storages/DeltaMerge/StoragePool.cpp @@ -515,21 +515,9 @@ void StoragePool::dataRegisterExternalPagesCallbacks(const ExternalPageCallbacks } case PageStorageRunMode::MIX_MODE: { - // When PageStorage run as Mix Mode. - // We need both get alive pages from V2 and V3 which will feedback for the DM. - // But V2 and V3 won't GC in the same time. So V3 need proxy V2 external pages callback. - // When V3 GC happend, scan the external pages from V3, in remover will scanner all of external pages from V2. - ExternalPageCallbacks mix_mode_callbacks; - - mix_mode_callbacks.scanner = callbacks.scanner; - mix_mode_callbacks.remover = [this, callbacks](const ExternalPageCallbacks::PathAndIdsVec & path_and_ids_vec, const std::set & valid_ids) { - // ns_id won't used on V2 - auto v2_valid_page_ids = data_storage_v2->getAliveExternalPageIds(ns_id); - v2_valid_page_ids.insert(valid_ids.begin(), valid_ids.end()); - callbacks.remover(path_and_ids_vec, v2_valid_page_ids); - }; - mix_mode_callbacks.ns_id = ns_id; - data_storage_v3->registerExternalPagesCallbacks(mix_mode_callbacks); + // We have transformed all pages from V2 to V3 in `restore`, so + // only need to register callbacks for V3. + data_storage_v3->registerExternalPagesCallbacks(callbacks); break; } default: @@ -636,8 +624,8 @@ PageId StoragePool::newDataPageIdForDTFile(StableDiskDelegator & delegator, cons auto existed_path = delegator.getDTFilePath(dtfile_id, /*throw_on_not_exist=*/false); fiu_do_on(FailPoints::force_set_dtfile_exist_when_acquire_id, { - static size_t fail_point_called = 0; - if (existed_path.empty() && fail_point_called % 10 == 0) + static std::atomic fail_point_called(0); + if (existed_path.empty() && fail_point_called.load() % 10 == 0) { existed_path = ""; } diff --git a/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt b/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt index 8d5854ffb5d..fd02bcebd2f 100644 --- a/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt +++ b/dbms/src/Storages/DeltaMerge/tests/CMakeLists.txt @@ -21,7 +21,7 @@ macro(grep_gtest_sources BASE_DIR DST_VAR) endmacro() # attach all dm gtest sources grep_gtest_sources(${TiFlash_SOURCE_DIR}/dbms/src/Storages/DeltaMerge/tests dm_gtest_sources) -add_executable(gtests_dm EXCLUDE_FROM_ALL ${dm_gtest_sources} dm_basic_include.h) +add_executable(gtests_dm EXCLUDE_FROM_ALL ${dm_gtest_sources} DMTestEnv.h) target_link_libraries(gtests_dm gtest_main dbms clickhouse_functions) add_check(gtests_dm) diff --git a/dbms/src/Storages/DeltaMerge/tests/dm_basic_include.h b/dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h similarity index 100% rename from dbms/src/Storages/DeltaMerge/tests/dm_basic_include.h rename to dbms/src/Storages/DeltaMerge/tests/DMTestEnv.h diff --git a/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h b/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h index bc6b7d5c3e6..787a521ded3 100644 --- a/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h +++ b/dbms/src/Storages/DeltaMerge/tests/MultiSegmentTestUtil.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,6 @@ #include #include -#include "dm_basic_include.h" namespace DB { @@ -43,7 +43,6 @@ namespace DM { namespace tests { - /// Helper class to test with multiple segments. /// You can call `prepareSegments` to prepare multiple segments. After that, /// you can use `verifyExpectedRowsForAllSegments` to verify the expectation for each segment. @@ -89,6 +88,7 @@ class MultiSegmentTestUtil : private boost::noncopyable // Check there is only one segment ASSERT_EQ(store->segments.size(), 1); const auto & [_key, seg] = *store->segments.begin(); + (void)_key; ASSERT_EQ(seg->getDelta()->getRows(), n_avg_rows_per_segment * 4); ASSERT_EQ(seg->getStable()->getRows(), 0); @@ -109,6 +109,7 @@ class MultiSegmentTestUtil : private boost::noncopyable auto segment_idx = 0; for (auto & [_key, seg] : store->segments) { + (void)_key; LOG_FMT_INFO(log, "Segment #{}: Range = {}", segment_idx, seg->getRowKeyRange().toDebugString()); ASSERT_EQ(seg->getDelta()->getRows(), 0); ASSERT_GT(seg->getStable()->getRows(), 0); // We don't check the exact rows of each segment. @@ -148,6 +149,7 @@ class MultiSegmentTestUtil : private boost::noncopyable auto segment_idx = 0; for (auto & [_key, seg] : store->segments) { + (void)_key; ASSERT_EQ(seg->getDelta()->getRows(), expected_delta_rows[segment_idx]) << "Assert failed for segment #" << segment_idx; ASSERT_EQ(seg->getStable()->getRows(), expected_stable_rows[segment_idx]) << "Assert failed for segment #" << segment_idx; segment_idx++; @@ -157,4 +159,4 @@ class MultiSegmentTestUtil : private boost::noncopyable } // namespace tests } // namespace DM -} // namespace DB \ No newline at end of file +} // namespace DB diff --git a/dbms/src/Storages/DeltaMerge/tests/bank/DeltaMergeStoreProxy.h b/dbms/src/Storages/DeltaMerge/tests/bank/DeltaMergeStoreProxy.h index 99264c77dc6..8edbbbc55b4 100644 --- a/dbms/src/Storages/DeltaMerge/tests/bank/DeltaMergeStoreProxy.h +++ b/dbms/src/Storages/DeltaMerge/tests/bank/DeltaMergeStoreProxy.h @@ -18,9 +18,9 @@ #include #include #include +#include #include #include -#include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/bank/main.cpp b/dbms/src/Storages/DeltaMerge/tests/bank/main.cpp index 115e170c48b..b90ad132e25 100644 --- a/dbms/src/Storages/DeltaMerge/tests/bank/main.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/bank/main.cpp @@ -17,8 +17,8 @@ #include #include #include +#include #include -#include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_convert_column.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_convert_column.cpp index 928a256349b..efc67de611e 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_convert_column.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_convert_column.cpp @@ -18,10 +18,9 @@ #include #include #include +#include #include -#include "dm_basic_include.h" - namespace DB { namespace DM diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp index f0c2f49c30b..00f31bc97e7 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_data_streams.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include namespace DB { diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp index 35e6c3d00c6..d46e1b7aa36 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include #include #include @@ -36,9 +38,6 @@ #include #include -#include "MultiSegmentTestUtil.h" -#include "dm_basic_include.h" - namespace DB { namespace FailPoints @@ -3565,7 +3564,6 @@ class DeltaMergeStoreMergeDeltaBySegmentTest public: DeltaMergeStoreMergeDeltaBySegmentTest() { - log = &Poco::Logger::get(DB::base::TiFlashStorageTestBasic::getCurrentFullTestName()); std::tie(ps_ver, pk_type) = GetParam(); } @@ -3608,8 +3606,6 @@ class DeltaMergeStoreMergeDeltaBySegmentTest UInt64 ps_ver; DMTestEnv::PkType pk_type; - - [[maybe_unused]] Poco::Logger * log; }; INSTANTIATE_TEST_CASE_P( diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp index 2b64fd90c09..40c399353b6 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_value_space.cpp @@ -18,14 +18,13 @@ #include #include #include +#include #include #include #include #include -#include "dm_basic_include.h" - namespace CurrentMetrics { extern const Metric DT_SnapshotOfRead; diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp index dfd4419fe38..23062f4ffdf 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_file.cpp @@ -20,13 +20,12 @@ #include #include #include +#include #include #include #include -#include "dm_basic_include.h" - namespace DB { namespace FailPoints diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp index 31fd99faf01..96c0070b73b 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_minmax_index.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include @@ -41,7 +41,7 @@ static const String DEFAULT_COL_NAME = "2020-09-26"; class DMMinMaxIndexTest : public ::testing::Test { public: - DMMinMaxIndexTest() {} + DMMinMaxIndexTest() = default; protected: static void SetUpTestCase() {} diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp index 5726cfa132d..deec5646d33 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment.cpp @@ -18,16 +18,17 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include -#include "dm_basic_include.h" - namespace CurrentMetrics { extern const Metric DT_SnapshotOfRead; @@ -50,12 +51,10 @@ extern DMFilePtr writeIntoNewDMFile(DMContext & dm_context, // DMFileBlockOutputStream::Flags flags); namespace tests { -class Segment_test : public DB::base::TiFlashStorageTestBasic +class SegmentTest : public DB::base::TiFlashStorageTestBasic { public: - Segment_test() - : storage_pool() - {} + SegmentTest() = default; public: static void SetUpTestCase() {} @@ -63,7 +62,7 @@ class Segment_test : public DB::base::TiFlashStorageTestBasic void SetUp() override { TiFlashStorageTestBasic::SetUp(); - table_columns_ = std::make_shared(); + table_columns = std::make_shared(); segment = reload(); ASSERT_EQ(segment->segmentId(), DELTA_MERGE_FIRST_SEGMENT_ID); @@ -79,43 +78,43 @@ class Segment_test : public DB::base::TiFlashStorageTestBasic ColumnDefinesPtr cols = (!pre_define_columns) ? DMTestEnv::getDefaultColumns() : pre_define_columns; setColumns(cols); - return Segment::newSegment(*dm_context_, table_columns_, RowKeyRange::newAll(false, 1), storage_pool->newMetaPageId(), 0); + return Segment::newSegment(*dm_context, table_columns, RowKeyRange::newAll(false, 1), storage_pool->newMetaPageId(), 0); } // setColumns should update dm_context at the same time void setColumns(const ColumnDefinesPtr & columns) { - *table_columns_ = *columns; + *table_columns = *columns; - dm_context_ = std::make_unique(*db_context, - *storage_path_pool, - *storage_pool, - 0, - /*min_version_*/ 0, - settings.not_compress_columns, - false, - 1, - db_context->getSettingsRef()); + dm_context = std::make_unique(*db_context, + *storage_path_pool, + *storage_pool, + 0, + /*min_version_*/ 0, + settings.not_compress_columns, + false, + 1, + db_context->getSettingsRef()); } - const ColumnDefinesPtr & tableColumns() const { return table_columns_; } + const ColumnDefinesPtr & tableColumns() const { return table_columns; } - DMContext & dmContext() { return *dm_context_; } + DMContext & dmContext() { return *dm_context; } protected: /// all these var lives as ref in dm_context std::unique_ptr storage_path_pool; std::unique_ptr storage_pool; - ColumnDefinesPtr table_columns_; + ColumnDefinesPtr table_columns; DM::DeltaMergeStore::Settings settings; /// dm_context - std::unique_ptr dm_context_; + std::unique_ptr dm_context; // the segment we are going to test SegmentPtr segment; }; -TEST_F(Segment_test, WriteRead) +TEST_F(SegmentTest, WriteRead) try { const size_t num_rows_write = 100; @@ -124,11 +123,11 @@ try // write to segment segment->write(dmContext(), block); // estimate segment - auto estimatedRows = segment->getEstimatedRows(); - ASSERT_EQ(estimatedRows, block.rows()); + auto estimated_rows = segment->getEstimatedRows(); + ASSERT_EQ(estimated_rows, block.rows()); - auto estimatedBytes = segment->getEstimatedBytes(); - ASSERT_EQ(estimatedBytes, block.bytes()); + auto estimated_bytes = segment->getEstimatedBytes(); + ASSERT_EQ(estimated_bytes, block.bytes()); } { @@ -212,7 +211,7 @@ try } CATCH -TEST_F(Segment_test, WriteRead2) +TEST_F(SegmentTest, WriteRead2) try { const size_t num_rows_write = dmContext().stable_pack_rows; @@ -249,7 +248,7 @@ try } CATCH -TEST_F(Segment_test, WriteReadMultiRange) +TEST_F(SegmentTest, WriteReadMultiRange) try { const size_t num_rows_write = 100; @@ -258,11 +257,11 @@ try // write to segment segment->write(dmContext(), block); // estimate segment - auto estimatedRows = segment->getEstimatedRows(); - ASSERT_EQ(estimatedRows, block.rows()); + auto estimated_rows = segment->getEstimatedRows(); + ASSERT_EQ(estimated_rows, block.rows()); - auto estimatedBytes = segment->getEstimatedBytes(); - ASSERT_EQ(estimatedBytes, block.bytes()); + auto estimated_bytes = segment->getEstimatedBytes(); + ASSERT_EQ(estimated_bytes, block.bytes()); } { @@ -356,7 +355,7 @@ try } CATCH -TEST_F(Segment_test, ReadWithMoreAdvacedDeltaIndex) +TEST_F(SegmentTest, ReadWithMoreAdvacedDeltaIndex) try { // Test the case that reading rows with an advance DeltaIndex @@ -421,27 +420,21 @@ try } CATCH -class SegmentDeletionRelevantPlace_test - : public Segment_test +class SegmentDeletionRelevantPlaceTest + : public SegmentTest , public testing::WithParamInterface { - DB::Settings getSettings() - { - DB::Settings settings; - auto enable_relevant_place = GetParam(); - - if (enable_relevant_place) - settings.set("dt_enable_relevant_place", "1"); - else - settings.set("dt_enable_relevant_place", "0"); - return settings; - } }; -TEST_P(SegmentDeletionRelevantPlace_test, ShareDelteRangeIndex) +TEST_P(SegmentDeletionRelevantPlaceTest, ShareDelteRangeIndex) try { + Settings my_settings; + const auto enable_relevant_place = GetParam(); + my_settings.dt_enable_relevant_place = enable_relevant_place; + this->reload({}, std::move(my_settings)); + const size_t num_rows_write = 300; { // write to segment @@ -467,27 +460,54 @@ try { HandleRange remove(100, 200); - segment->write(dmContext(), {RowKeyRange::fromHandleRange(remove)}); + segment->write(dmContext(), /*delete_range*/ {RowKeyRange::fromHandleRange(remove)}); } // The first call of get_rows below will place the DeleteRange into delta index. + // If relevant place is enabled, the placed deletes in delta-tree-index is not + // pushed forward since we do not fully apply the delete range [100, 200). auto rows1 = get_rows(RowKeyRange::fromHandleRange(HandleRange(0, 150))); + { + auto delta = segment->getDelta(); + auto placed_rows = delta->getPlacedDeltaRows(); + auto placed_deletes = delta->getPlacedDeltaDeletes(); + ASSERT_EQ(placed_rows, num_rows_write); + EXPECT_EQ(placed_deletes, enable_relevant_place ? 0 : 1); + } auto rows2 = get_rows(RowKeyRange::fromHandleRange(HandleRange(150, 300))); + { + auto delta = segment->getDelta(); + auto placed_rows = delta->getPlacedDeltaRows(); + auto placed_deletes = delta->getPlacedDeltaDeletes(); + ASSERT_EQ(placed_rows, num_rows_write); + EXPECT_EQ(placed_deletes, enable_relevant_place ? 0 : 1); + } + // Query with range [0, 300) will push the placed deletes forward no matter + // relevant place is enable or not. + auto rows3 = get_rows(RowKeyRange::fromHandleRange(HandleRange(0, 300))); + { + auto delta = segment->getDelta(); + auto placed_rows = delta->getPlacedDeltaRows(); + auto placed_deletes = delta->getPlacedDeltaDeletes(); + ASSERT_EQ(placed_rows, num_rows_write); + EXPECT_EQ(placed_deletes, 1); + } - ASSERT_EQ(rows1, (size_t)100); - ASSERT_EQ(rows2, (size_t)100); + ASSERT_EQ(rows1, 100); + ASSERT_EQ(rows2, 100); + ASSERT_EQ(rows3, 200); } CATCH -INSTANTIATE_TEST_CASE_P(WhetherEnableRelevantPlace, SegmentDeletionRelevantPlace_test, testing::Values(true, false)); +INSTANTIATE_TEST_CASE_P(WhetherEnableRelevantPlace, SegmentDeletionRelevantPlaceTest, testing::Values(true, false)); -class SegmentDeletion_test - : public Segment_test +class SegmentDeletionTest + : public SegmentTest , public testing::WithParamInterface> { }; -TEST_P(SegmentDeletion_test, DeleteDataInDelta) +TEST_P(SegmentDeletionTest, DeleteDataInDelta) try { const size_t num_rows_write = 100; @@ -565,7 +585,7 @@ try } CATCH -TEST_P(SegmentDeletion_test, DeleteDataInStable) +TEST_P(SegmentDeletionTest, DeleteDataInStable) try { const size_t num_rows_write = 100; @@ -651,7 +671,7 @@ try } CATCH -TEST_P(SegmentDeletion_test, DeleteDataInStableAndDelta) +TEST_P(SegmentDeletionTest, DeleteDataInStableAndDelta) try { const size_t num_rows_write = 100; @@ -738,9 +758,9 @@ try } CATCH -INSTANTIATE_TEST_CASE_P(WhetherReadOrMergeDeltaBeforeDeleteRange, SegmentDeletion_test, testing::Combine(testing::Bool(), testing::Bool())); +INSTANTIATE_TEST_CASE_P(WhetherReadOrMergeDeltaBeforeDeleteRange, SegmentDeletionTest, testing::Combine(testing::Bool(), testing::Bool())); -TEST_F(Segment_test, DeleteRead) +TEST_F(SegmentTest, DeleteRead) try { const size_t num_rows_write = 64; @@ -946,7 +966,7 @@ try } CATCH -TEST_F(Segment_test, Split) +TEST_F(SegmentTest, Split) try { const size_t num_rows_write_per_batch = 100; @@ -1046,7 +1066,7 @@ try } CATCH -TEST_F(Segment_test, SplitFail) +TEST_F(SegmentTest, SplitFail) try { const size_t num_rows_write = 100; @@ -1066,7 +1086,7 @@ try } CATCH -TEST_F(Segment_test, Restore) +TEST_F(SegmentTest, Restore) try { // compare will compares the given segments. @@ -1158,7 +1178,7 @@ try } CATCH -TEST_F(Segment_test, MassiveSplit) +TEST_F(SegmentTest, MassiveSplit) try { Settings settings = dmContext().db_context.getSettings(); @@ -1242,52 +1262,51 @@ try } CATCH -enum Segment_test_Mode +enum SegmentTestMode { V1_BlockOnly, V2_BlockOnly, V2_FileOnly, }; -String testModeToString(const ::testing::TestParamInfo & info) +String testModeToString(const ::testing::TestParamInfo & info) { const auto mode = info.param; switch (mode) { - case Segment_test_Mode::V1_BlockOnly: + case SegmentTestMode::V1_BlockOnly: return "V1_BlockOnly"; - case Segment_test_Mode::V2_BlockOnly: + case SegmentTestMode::V2_BlockOnly: return "V2_BlockOnly"; - case Segment_test_Mode::V2_FileOnly: + case SegmentTestMode::V2_FileOnly: return "V2_FileOnly"; default: return "Unknown"; } } -class Segment_test_2 : public Segment_test - , public testing::WithParamInterface +class SegmentTest2 : public SegmentTest + , public testing::WithParamInterface { public: - Segment_test_2() - : Segment_test() - {} + SegmentTest2() = default; + void SetUp() override { mode = GetParam(); switch (mode) { - case Segment_test_Mode::V1_BlockOnly: + case SegmentTestMode::V1_BlockOnly: setStorageFormat(1); break; - case Segment_test_Mode::V2_BlockOnly: - case Segment_test_Mode::V2_FileOnly: + case SegmentTestMode::V2_BlockOnly: + case SegmentTestMode::V2_FileOnly: setStorageFormat(2); break; } - Segment_test::SetUp(); + SegmentTest::SetUp(); } std::pair genDMFile(DMContext & context, const Block & block) @@ -1305,7 +1324,7 @@ class Segment_test_2 : public Segment_test delegator.addDTFile(file_id, dmfile->getBytesOnDisk(), store_path); - auto & pk_column = block.getByPosition(0).column; + const auto & pk_column = block.getByPosition(0).column; auto min_pk = pk_column->getInt(0); auto max_pk = pk_column->getInt(block.rows() - 1); HandleRange range(min_pk, max_pk + 1); @@ -1313,10 +1332,10 @@ class Segment_test_2 : public Segment_test return {RowKeyRange::fromHandleRange(range), {file_id}}; } - Segment_test_Mode mode; + SegmentTestMode mode; }; -TEST_P(Segment_test_2, FlushDuringSplitAndMerge) +TEST_P(SegmentTest2, FlushDuringSplitAndMerge) try { size_t row_offset = 0; @@ -1327,11 +1346,11 @@ try row_offset += 100; switch (mode) { - case Segment_test_Mode::V1_BlockOnly: - case Segment_test_Mode::V2_BlockOnly: + case SegmentTestMode::V1_BlockOnly: + case SegmentTestMode::V2_BlockOnly: segment->write(dmContext(), std::move(block)); break; - case Segment_test_Mode::V2_FileOnly: + case SegmentTestMode::V2_FileOnly: { auto delegate = dmContext().path_pool.getStableDiskDelegator(); auto file_provider = dmContext().db_context.getFileProvider(); @@ -1430,9 +1449,9 @@ try } CATCH -INSTANTIATE_TEST_CASE_P(Segment_test_Mode, // - Segment_test_2, - testing::Values(Segment_test_Mode::V1_BlockOnly, Segment_test_Mode::V2_BlockOnly, Segment_test_Mode::V2_FileOnly), +INSTANTIATE_TEST_CASE_P(SegmentTestMode, // + SegmentTest2, + testing::Values(SegmentTestMode::V1_BlockOnly, SegmentTestMode::V2_BlockOnly, SegmentTestMode::V2_FileOnly), testModeToString); enum class SegmentWriteType @@ -1440,12 +1459,12 @@ enum class SegmentWriteType ToDisk, ToCache }; -class Segment_DDL_test - : public Segment_test +class SegmentDDLTest + : public SegmentTest , public testing::WithParamInterface> { }; -String paramToString(const ::testing::TestParamInfo & info) +String paramToString(const ::testing::TestParamInfo & info) { const auto [write_type, flush_before_ddl] = info.param; @@ -1455,7 +1474,7 @@ String paramToString(const ::testing::TestParamInfo } /// Mock a col from i8 -> i32 -TEST_P(Segment_DDL_test, AlterInt8ToInt32) +TEST_P(SegmentDDLTest, AlterInt8ToInt32) try { const String column_name_i8_to_i32 = "i8_to_i32"; @@ -1627,7 +1646,7 @@ try } CATCH -TEST_P(Segment_DDL_test, AddColumn) +TEST_P(SegmentDDLTest, AddColumn) try { const String new_column_name = "i8"; @@ -1795,7 +1814,7 @@ try } CATCH -TEST_F(Segment_test, CalculateDTFileProperty) +TEST_F(SegmentTest, CalculateDTFileProperty) try { Settings settings = dmContext().db_context.getSettings(); @@ -1836,7 +1855,7 @@ try } CATCH -TEST_F(Segment_test, CalculateDTFilePropertyWithPropertyFileDeleted) +TEST_F(SegmentTest, CalculateDTFilePropertyWithPropertyFileDeleted) try { Settings settings = dmContext().db_context.getSettings(); @@ -1857,10 +1876,10 @@ try } { - auto & stable = segment->getStable(); - auto & dmfiles = stable->getDMFiles(); + const auto & stable = segment->getStable(); + const auto & dmfiles = stable->getDMFiles(); ASSERT_GT(dmfiles[0]->getPacks(), (size_t)1); - auto & dmfile = dmfiles[0]; + const auto & dmfile = dmfiles[0]; auto file_path = dmfile->path(); // check property file exists and then delete it ASSERT_EQ(Poco::File(file_path + "/property").exists(), true); @@ -1877,7 +1896,7 @@ try // calculate the StableProperty for packs in the key range [0, num_rows_write_every_round) stable->calculateStableProperty(dmContext(), range, false); ASSERT_EQ(stable->isStablePropertyCached(), true); - auto & property = stable->getStableProperty(); + const auto & property = stable->getStableProperty(); ASSERT_EQ(property.gc_hint_version, std::numeric_limits::max()); ASSERT_EQ(property.num_versions, num_rows_write_every_round); ASSERT_EQ(property.num_puts, num_rows_write_every_round); @@ -1887,7 +1906,7 @@ try CATCH INSTANTIATE_TEST_CASE_P(SegmentWriteType, - Segment_DDL_test, + SegmentDDLTest, ::testing::Combine( // ::testing::Values(SegmentWriteType::ToDisk, SegmentWriteType::ToCache), ::testing::Bool()), diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp index 1cc61663a2f..6359a3db184 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_segment_common_handle.cpp @@ -15,14 +15,13 @@ #include #include #include +#include #include #include #include #include -#include "dm_basic_include.h" - namespace DB { namespace DM diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp index a26471cfe01..f929e153847 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_storage_delta_merge.cpp @@ -31,17 +31,18 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include #include -#include "dm_basic_include.h" - namespace DB { namespace FailPoints @@ -168,6 +169,8 @@ try } EXPECT_EQ(total_segment_rows, num_rows_read); storage->drop(); + // remove the storage from TiFlash context manually + storage->removeFromTMTContext(); } CATCH @@ -252,6 +255,8 @@ try ASSERT_EQ(storage->getDatabaseName(), new_db_name); storage->drop(); + // remove the storage from TiFlash context manually + storage->removeFromTMTContext(); } CATCH @@ -315,6 +320,8 @@ try ASSERT_EQ(sort_desc.front().nulls_direction, sort_desc2.front().nulls_direction); storage->drop(); + // remove the storage from TiFlash context manually + storage->removeFromTMTContext(); } CATCH @@ -609,6 +616,8 @@ try sample.insert(DB::tests::createColumn( Strings(100, "a"), "col2")); + constexpr TiDB::TableID table_id = 1; + const String table_name = fmt::format("t_{}", table_id); Context ctx = DMTestEnv::getContext(); std::shared_ptr storage; @@ -631,12 +640,11 @@ try path.remove(true); // primary_expr_ast - const String table_name = "t_1233"; ASTPtr astptr(new ASTIdentifier(table_name, ASTIdentifier::Kind::Table)); astptr->children.emplace_back(new ASTIdentifier("col1")); TiDB::TableInfo tidb_table_info; - tidb_table_info.id = 1; + tidb_table_info.id = table_id; storage = StorageDeltaMerge::create("TiFlash", /* db_name= */ "default", @@ -692,8 +700,8 @@ try { Field res; c->get(i, res); - ASSERT(!res.isNull()); - ASSERT(res.get() == 1); + ASSERT_TRUE(!res.isNull()); + ASSERT_EQ(res.get(), table_id); } } } @@ -701,20 +709,35 @@ try in->readSuffix(); ASSERT_EQ(num_rows_read, sample.rows()); storage->drop(); + // remove the storage from TiFlash context manually + storage->removeFromTMTContext(); } CATCH TEST(StorageDeltaMergeTest, RestoreAfterClearData) try { - Context ctx = DMTestEnv::getContext(); - auto & settings = ctx.getSettingsRef(); + auto & global_settings = ::DB::tests::TiFlashTestEnv::getGlobalContext().getSettingsRef(); + // store the old value to restore global_context settings after the test finish to avoid influence other tests + auto old_global_settings = global_settings; + SCOPE_EXIT({ + global_settings = old_global_settings; + }); + // change the settings to make it more easy to trigger splitting segments + Settings settings; settings.dt_segment_limit_rows = 11; settings.dt_segment_limit_size = 20; settings.dt_segment_delta_limit_rows = 7; settings.dt_segment_delta_limit_size = 20; settings.dt_segment_force_split_size = 100; settings.dt_segment_delta_cache_limit_size = 20; + + // we need change the settings in both the ctx we get just below and the global_context above. + // because when processing write request, `DeltaMergeStore` will call `checkSegmentUpdate` with the context we just get below. + // and when initialize `DeltaMergeStore`, it will call `checkSegmentUpdate` with the global_context above. + // so we need to make the settings in these two contexts consistent. + global_settings = settings; + Context ctx = DMTestEnv::getContext(settings); std::shared_ptr storage; DataTypes data_types; Names column_names; @@ -848,6 +871,8 @@ try ASSERT_LT(read_data(), num_rows_write); } storage->drop(); + // remove the storage from TiFlash context manually + storage->removeFromTMTContext(); } CATCH diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp index 340dad1ff6e..26a5b1132e4 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_utils.cpp @@ -14,10 +14,9 @@ #include #include +#include #include -#include "dm_basic_include.h" - namespace DB { namespace DM diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp index 59052e2e8b2..16b1729bea1 100644 --- a/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp +++ b/dbms/src/Storages/DeltaMerge/tests/gtest_version_filter.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include namespace DB { diff --git a/dbms/src/Storages/DeltaMerge/tests/stress/DMStressProxy.h b/dbms/src/Storages/DeltaMerge/tests/stress/DMStressProxy.h index eb4d937e8d4..0571eafae83 100644 --- a/dbms/src/Storages/DeltaMerge/tests/stress/DMStressProxy.h +++ b/dbms/src/Storages/DeltaMerge/tests/stress/DMStressProxy.h @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/CMakeLists.txt b/dbms/src/Storages/DeltaMerge/workload/CMakeLists.txt similarity index 86% rename from dbms/src/Storages/DeltaMerge/tools/workload/CMakeLists.txt rename to dbms/src/Storages/DeltaMerge/workload/CMakeLists.txt index 7227f1cf563..7a83cbec57c 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/CMakeLists.txt +++ b/dbms/src/Storages/DeltaMerge/workload/CMakeLists.txt @@ -18,6 +18,3 @@ set(dt-workload-src MainEntry.cpp DTWorkload.cpp KeyGenerator.cpp TableGenerator add_library(dt-workload-lib ${dt-workload-src}) target_link_libraries(dt-workload-lib dbms clickhouse_functions clickhouse-server-lib) - -add_executable(dt-workload Main.cpp ${dt-workload-src}) -target_link_libraries(dt-workload dbms gtest clickhouse_functions clickhouse-server-lib) diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.cpp b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp similarity index 94% rename from dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.cpp rename to dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp index e5c7fd30f40..a53a1b9ebbd 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.cpp @@ -19,16 +19,16 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -233,7 +233,7 @@ void DTWorkload::verifyHandle(uint64_t r) } for (size_t i = 0; i < handle_col->size(); i++) { - // Handle must be int64 or uint64. Currently, TableGenterator would ensure this limit. + // Handle must be int64 or uint64. Currently, TableGenerator would ensure this limit. uint64_t h = handle_col->getInt(i); uint64_t store_ts = ts_col->getInt(i); diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.h b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.h similarity index 97% rename from dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.h rename to dbms/src/Storages/DeltaMerge/workload/DTWorkload.h index 26cc5b6e07c..1ee5ba6b871 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/DTWorkload.h +++ b/dbms/src/Storages/DeltaMerge/workload/DTWorkload.h @@ -73,7 +73,7 @@ class ThreadStat class Statistics { public: - Statistics(int write_thread_count = 0, int read_thread_count = 0) + explicit Statistics(int write_thread_count = 0, int read_thread_count = 0) : init_ms(0) , write_stats(write_thread_count) , read_stats(read_thread_count) diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp similarity index 95% rename from dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.cpp rename to dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp index be6ff1dcbbe..479977d46d1 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.cpp @@ -13,11 +13,11 @@ // limitations under the License. #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include @@ -33,7 +33,7 @@ class RandomDataGenerator : public DataGenerator , rand_gen(std::random_device()()) {} - virtual std::tuple get(uint64_t key) override + std::tuple get(uint64_t key) override { Block block; // Generate 'rowkeys'. @@ -227,7 +227,9 @@ class RandomDataGenerator : public DataGenerator struct tm randomLocalTime() { time_t t = randomUTCTimestamp(); - struct tm res; + struct tm res + { + }; if (localtime_r(&t, &res) == nullptr) { throw std::invalid_argument(fmt::format("localtime_r({}) ret {}", t, strerror(errno))); diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.h b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.h similarity index 96% rename from dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/DataGenerator.h index e32de4591e6..cd29f1a3a80 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/DataGenerator.h +++ b/dbms/src/Storages/DeltaMerge/workload/DataGenerator.h @@ -27,7 +27,7 @@ class DataGenerator public: static std::unique_ptr create(const WorkloadOptions & opts, const TableInfo & table_info, TimestampGenerator & ts_gen); virtual std::tuple get(uint64_t key) = 0; - virtual ~DataGenerator() {} + virtual ~DataGenerator() = default; }; std::string blockToString(const Block & block); diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Handle.h b/dbms/src/Storages/DeltaMerge/workload/Handle.h similarity index 90% rename from dbms/src/Storages/DeltaMerge/tools/workload/Handle.h rename to dbms/src/Storages/DeltaMerge/workload/Handle.h index eb117a4fddd..2bbd1bd409d 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Handle.h +++ b/dbms/src/Storages/DeltaMerge/workload/Handle.h @@ -16,7 +16,7 @@ #include #include -#include +#include #include #include @@ -40,7 +40,7 @@ class HandleLock static constexpr uint64_t default_lock_count = 4096; static std::unique_ptr create(const TableInfo & table_info); - HandleLock(uint64_t lock_count = default_lock_count) + explicit HandleLock(uint64_t lock_count = default_lock_count) : rmtxs(lock_count) {} @@ -52,6 +52,7 @@ class HandleLock std::vector> getLocks(const std::vector & handles) { std::vector indexes; + indexes.reserve(handles.size()); for (const auto & h : handles) { indexes.push_back(index(h)); @@ -59,6 +60,7 @@ class HandleLock // Sort mutex indexes to avoid dead lock. sort(indexes.begin(), indexes.end()); std::vector> locks; + locks.reserve(indexes.size()); for (auto i : indexes) { locks.push_back(getLockByIndex(i)); @@ -105,7 +107,7 @@ class HandleTable std::lock_guard lock(mtx); handle_to_ts[handle] = ts; Record r{handle, ts}; - if (wal != nullptr && wal->write((char *)&r, sizeof(r)) != sizeof(r)) + if (wal != nullptr && wal->write(reinterpret_cast(&r), sizeof(r)) != sizeof(r)) { throw std::runtime_error(fmt::format("write ret {}", strerror(errno))); } @@ -134,8 +136,8 @@ class HandleTable try { PosixRandomAccessFile f(fname, -1); - Record r; - while (f.read((char *)&r, sizeof(r)) == sizeof(r)) + Record r{}; + while (f.read(reinterpret_cast(&r), sizeof(r)) == sizeof(r)) { handle_to_ts[r.handle] = r.ts; } @@ -156,7 +158,7 @@ class HandleTable for (const auto & pa : handle_to_ts) { Record r{pa.first, pa.second}; - if (f.write((char *)&r, sizeof(r)) != sizeof(r)) + if (f.write(reinterpret_cast(&r), sizeof(r)) != sizeof(r)) { throw std::runtime_error(fmt::format("write ret {}", strerror(errno))); } @@ -191,7 +193,7 @@ class SharedHandleTable public: static constexpr uint64_t default_shared_count = 4096; - SharedHandleTable(uint64_t max_key_count, const std::string & waldir = "", uint64_t shared_cnt = default_shared_count) + explicit SharedHandleTable(uint64_t max_key_count, const std::string & waldir = "", uint64_t shared_cnt = default_shared_count) : tables(shared_cnt) { uint64_t max_key_count_per_shared = max_key_count / default_shared_count + 1; diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/KeyGenerator.cpp similarity index 92% rename from dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.cpp rename to dbms/src/Storages/DeltaMerge/workload/KeyGenerator.cpp index bb2f2253279..f899ec71b4b 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/KeyGenerator.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include +#include +#include #include #include @@ -31,7 +31,7 @@ class IncrementalKeyGenerator : public KeyGenerator , key(0) {} - virtual uint64_t get64() override + uint64_t get64() override { return key.fetch_add(1, std::memory_order_relaxed) % key_count + start_key; } @@ -54,7 +54,7 @@ class UniformDistributionKeyGenerator : public KeyGenerator , uniform_dist(0, key_count) {} - virtual uint64_t get64() override + uint64_t get64() override { std::lock_guard lock(mtx); return uniform_dist(rand_gen); @@ -78,7 +78,7 @@ class NormalDistributionKeyGenerator : public KeyGenerator , normal_dist(key_count / 2.0, key_count / 20.0) {} - virtual uint64_t get64() override + uint64_t get64() override { std::lock_guard lock(mtx); return normal_dist(rand_gen); diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.h b/dbms/src/Storages/DeltaMerge/workload/KeyGenerator.h similarity index 92% rename from dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/KeyGenerator.h index 447f3ffc27a..7c8b8fd0080 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/KeyGenerator.h +++ b/dbms/src/Storages/DeltaMerge/workload/KeyGenerator.h @@ -23,8 +23,8 @@ class KeyGenerator public: static std::unique_ptr create(const WorkloadOptions & opts); - KeyGenerator() {} - virtual ~KeyGenerator() {} + KeyGenerator() = default; + virtual ~KeyGenerator() = default; virtual uint64_t get64() = 0; }; diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Limiter.cpp b/dbms/src/Storages/DeltaMerge/workload/Limiter.cpp similarity index 77% rename from dbms/src/Storages/DeltaMerge/tools/workload/Limiter.cpp rename to dbms/src/Storages/DeltaMerge/workload/Limiter.cpp index 73764d27bc5..65f9e3ce72c 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Limiter.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/Limiter.cpp @@ -13,8 +13,8 @@ // limitations under the License. #include -#include -#include +#include +#include #include #include @@ -24,10 +24,10 @@ namespace DB::DM::tests class ConstantLimiter : public Limiter { public: - ConstantLimiter(uint64_t rate_per_sec) + explicit ConstantLimiter(uint64_t rate_per_sec) : limiter(rate_per_sec, LimiterType::UNKNOW) {} - virtual void request() override + void request() override { limiter.request(1); } @@ -38,7 +38,7 @@ class ConstantLimiter : public Limiter std::unique_ptr Limiter::create(const WorkloadOptions & opts) { - uint64_t per_sec = std::ceil(static_cast(opts.max_write_per_sec / opts.write_thread_count)); + uint64_t per_sec = std::ceil(opts.max_write_per_sec * 1.0 / opts.write_thread_count); return std::make_unique(per_sec); } diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Limiter.h b/dbms/src/Storages/DeltaMerge/workload/Limiter.h similarity index 96% rename from dbms/src/Storages/DeltaMerge/tools/workload/Limiter.h rename to dbms/src/Storages/DeltaMerge/workload/Limiter.h index e2892b178a2..da2d31c7915 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Limiter.h +++ b/dbms/src/Storages/DeltaMerge/workload/Limiter.h @@ -23,6 +23,6 @@ class Limiter public: static std::unique_ptr create(const WorkloadOptions & opts); virtual void request() = 0; - virtual ~Limiter() {} + virtual ~Limiter() = default; }; } // namespace DB::DM::tests \ No newline at end of file diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/MainEntry.cpp b/dbms/src/Storages/DeltaMerge/workload/MainEntry.cpp similarity index 85% rename from dbms/src/Storages/DeltaMerge/tools/workload/MainEntry.cpp rename to dbms/src/Storages/DeltaMerge/workload/MainEntry.cpp index e18a6ef30a2..88cf0b6322f 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/MainEntry.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/MainEntry.cpp @@ -14,10 +14,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include @@ -124,13 +124,26 @@ void run(WorkloadOptions & opts) // Table Schema auto table_gen = TableGenerator::create(opts); auto table_info = table_gen->get(opts.table_id, opts.table_name); - // In this for loop, destory DeltaMergeStore gracefully and recreate it. - for (uint64_t i = 0; i < opts.verify_round; i++) + // In this for loop, destroy DeltaMergeStore gracefully and recreate it. + auto run_test = [&]() { + for (uint64_t i = 0; i < opts.verify_round; i++) + { + DTWorkload workload(opts, handle_table, table_info); + workload.run(i); + stats.push_back(workload.getStat()); + LOG_FMT_INFO(log, "No.{} Workload {} {}", i, opts.write_key_distribution, stats.back().toStrings()); + } + }; + run_test(); + + if (opts.ps_run_mode == DB::PageStorageRunMode::MIX_MODE) { - DTWorkload workload(opts, handle_table, table_info); - workload.run(i); - stats.push_back(workload.getStat()); - LOG_FMT_INFO(log, "No.{} Workload {} {}", i, opts.write_key_distribution, stats.back().toStrings()); + // clear statistic in DB::PageStorageRunMode::ONLY_V2 + stats.clear(); + auto & global_context = TiFlashTestEnv::getGlobalContext(); + global_context.setPageStorageRunMode(DB::PageStorageRunMode::MIX_MODE); + global_context.initializeGlobalStoragePoolIfNeed(global_context.getPathPool()); + run_test(); } } catch (...) @@ -254,8 +267,9 @@ int DTWorkload::mainEntry(int argc, char ** argv) // or the logging in global context won't be output to // the log file init(opts); - TiFlashTestEnv::initializeGlobalContext(opts.work_dirs, opts.enable_ps_v3); + // For mixed mode, we need to run the test in ONLY_V2 mode first. + TiFlashTestEnv::initializeGlobalContext(opts.work_dirs, opts.ps_run_mode == PageStorageRunMode::ONLY_V3 ? PageStorageRunMode::ONLY_V3 : PageStorageRunMode::ONLY_V2); if (opts.testing_type == "daily_perf") { dailyPerformanceTest(opts); @@ -277,7 +291,6 @@ int DTWorkload::mainEntry(int argc, char ** argv) runAndRandomKill(opts); } } - TiFlashTestEnv::shutdown(); return 0; } diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Options.cpp b/dbms/src/Storages/DeltaMerge/workload/Options.cpp similarity index 88% rename from dbms/src/Storages/DeltaMerge/tools/workload/Options.cpp rename to dbms/src/Storages/DeltaMerge/workload/Options.cpp index 0d2b14d916b..8545d22ca8d 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Options.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/Options.cpp @@ -13,8 +13,8 @@ // limitations under the License. #include -#include -#include +#include +#include #include #include @@ -44,7 +44,7 @@ std::string WorkloadOptions::toString(std::string seperator) const fmt::format("read_stream_count {}{}", read_stream_count, seperator) + // fmt::format("testing_type {}{}", testing_type, seperator) + // fmt::format("log_write_request {}{}", log_write_request, seperator) + // - fmt::format("enable_ps_v3 {}{}", enable_ps_v3, seperator) + // + fmt::format("ps_run_mode {}{}", ps_run_mode, seperator) + // fmt::format("bg_thread_count {}{}", bg_thread_count, seperator) + // fmt::format("table_id {}{}", table_id, seperator) + // fmt::format("table_name {}{}", table_name, seperator); @@ -88,7 +88,7 @@ std::pair WorkloadOptions::parseOptions(int argc, char * argv // ("log_write_request", value()->default_value(false), "") // // - ("enable_ps_v3", value()->default_value(true), "") // + ("ps_run_mode", value()->default_value(2, "possible value: 1(only_v2), 2(only_v3), 3(mix_mode), and note that in mix_mode, the test will run twice, first round in only_v2 mode and second round in mix_mode")) // // ("bg_thread_count", value()->default_value(4), "") // // @@ -155,8 +155,20 @@ std::pair WorkloadOptions::parseOptions(int argc, char * argv testing_type = vm["testing_type"].as(); log_write_request = vm["log_write_request"].as(); - - enable_ps_v3 = vm["enable_ps_v3"].as(); + switch (vm["ps_run_mode"].as()) + { + case static_cast(PageStorageRunMode::ONLY_V2): + ps_run_mode = PageStorageRunMode::ONLY_V2; + break; + case static_cast(PageStorageRunMode::ONLY_V3): + ps_run_mode = PageStorageRunMode::ONLY_V3; + break; + case static_cast(PageStorageRunMode::MIX_MODE): + ps_run_mode = PageStorageRunMode::MIX_MODE; + break; + default: + return {false, fmt::format("unknown ps_run_mode {}.", vm["ps_run_mode"].as())}; + } bg_thread_count = vm["bg_thread_count"].as(); diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Options.h b/dbms/src/Storages/DeltaMerge/workload/Options.h similarity index 95% rename from dbms/src/Storages/DeltaMerge/tools/workload/Options.h rename to dbms/src/Storages/DeltaMerge/workload/Options.h index 17c7a5ba61f..f017daf2d8a 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Options.h +++ b/dbms/src/Storages/DeltaMerge/workload/Options.h @@ -14,6 +14,8 @@ #pragma once +#include + #include #include @@ -53,7 +55,7 @@ struct WorkloadOptions bool log_write_request; - bool enable_ps_v3; + PageStorageRunMode ps_run_mode; uint64_t bg_thread_count; diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/ReadColumnsGenerator.h b/dbms/src/Storages/DeltaMerge/workload/ReadColumnsGenerator.h similarity index 93% rename from dbms/src/Storages/DeltaMerge/tools/workload/ReadColumnsGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/ReadColumnsGenerator.h index 180409f89e1..c881bb148a2 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/ReadColumnsGenerator.h +++ b/dbms/src/Storages/DeltaMerge/workload/ReadColumnsGenerator.h @@ -14,7 +14,7 @@ #pragma once -#include +#include #include @@ -28,7 +28,7 @@ class ReadColumnsGenerator return std::make_unique(table_info); } - ReadColumnsGenerator(const TableInfo & table_info_) + explicit ReadColumnsGenerator(const TableInfo & table_info_) : table_info(table_info_) , rand_gen(std::random_device()()) , uniform_dist(0, table_info_.columns->size() - 1) diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.cpp b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp similarity index 97% rename from dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.cpp rename to dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp index cf52e808ab1..ec29a476d6a 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.cpp @@ -15,8 +15,8 @@ #include #include #include -#include -#include +#include +#include #include #include @@ -237,7 +237,7 @@ class RandomTableGenerator : public TableGenerator , rand_gen(std::random_device()()) {} - virtual TableInfo get(int64_t table_id, std::string table_name) override + TableInfo get(int64_t table_id, std::string table_name) override { TableInfo table_info; @@ -293,7 +293,7 @@ class RandomTableGenerator : public TableGenerator class ConstantTableGenerator : public TableGenerator { - virtual TableInfo get(int64_t table_id, std::string table_name) override + TableInfo get(int64_t table_id, std::string table_name) override { TableInfo table_info; diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.h b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.h similarity index 96% rename from dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/TableGenerator.h index aba5c1590b7..b88bf2b72e2 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/TableGenerator.h +++ b/dbms/src/Storages/DeltaMerge/workload/TableGenerator.h @@ -38,6 +38,6 @@ class TableGenerator virtual TableInfo get(int64_t table_id, std::string table_name) = 0; - virtual ~TableGenerator() {} + virtual ~TableGenerator() = default; }; } // namespace DB::DM::tests \ No newline at end of file diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/TimestampGenerator.h b/dbms/src/Storages/DeltaMerge/workload/TimestampGenerator.h similarity index 100% rename from dbms/src/Storages/DeltaMerge/tools/workload/TimestampGenerator.h rename to dbms/src/Storages/DeltaMerge/workload/TimestampGenerator.h diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Utils.cpp b/dbms/src/Storages/DeltaMerge/workload/Utils.cpp similarity index 94% rename from dbms/src/Storages/DeltaMerge/tools/workload/Utils.cpp rename to dbms/src/Storages/DeltaMerge/workload/Utils.cpp index 1cefae724c6..80d9f788016 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Utils.cpp +++ b/dbms/src/Storages/DeltaMerge/workload/Utils.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include @@ -83,7 +83,7 @@ std::string fieldToString(const DataTypePtr & data_type, const Field & f) } else if (t == Field::Types::Which::Decimal256) { - auto i = f.get(); + const auto & i = f.get(); auto scale = dynamic_cast(data_type.get())->getScale(); return i.toString(scale); } @@ -105,8 +105,8 @@ std::vector colToVec(const DataTypePtr & data_type, const ColumnPtr std::string blockToString(const Block & block) { std::string s = "id name type values\n"; - auto & cols = block.getColumnsWithTypeAndName(); - for (auto & col : cols) + const auto & cols = block.getColumnsWithTypeAndName(); + for (const auto & col : cols) { s += fmt::format("{} {} {} {}\n", col.column_id, col.name, col.type->getFamilyName(), colToVec(col.type, col.column)); } diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Utils.h b/dbms/src/Storages/DeltaMerge/workload/Utils.h similarity index 100% rename from dbms/src/Storages/DeltaMerge/tools/workload/Utils.h rename to dbms/src/Storages/DeltaMerge/workload/Utils.h diff --git a/dbms/src/Storages/IManageableStorage.h b/dbms/src/Storages/IManageableStorage.h index e41d092ca87..ebf84c592e4 100644 --- a/dbms/src/Storages/IManageableStorage.h +++ b/dbms/src/Storages/IManageableStorage.h @@ -157,12 +157,15 @@ class IManageableStorage : public IStorage /// when `need_block` is true, it will try return a cached block corresponding to DecodingStorageSchemaSnapshotConstPtr, /// and `releaseDecodingBlock` need to be called when the block is free /// when `need_block` is false, it will just return an nullptr - virtual std::pair getSchemaSnapshotAndBlockForDecoding(bool /* need_block */) + /// This method must be called under the protection of table structure lock + virtual std::pair getSchemaSnapshotAndBlockForDecoding(const TableStructureLockHolder & /* table_structure_lock */, bool /* need_block */) { throw Exception("Method getDecodingSchemaSnapshot is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); }; - virtual void releaseDecodingBlock(Int64 /* schema_version */, BlockUPtr /* block */) + /// The `block_decoding_schema_version` is just an internal version for `DecodingStorageSchemaSnapshot`, + /// And it has no relation with the table schema version. + virtual void releaseDecodingBlock(Int64 /* block_decoding_schema_version */, BlockUPtr /* block */) { throw Exception("Method getDecodingSchemaSnapshot is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED); } diff --git a/dbms/src/Storages/Page/CMakeLists.txt b/dbms/src/Storages/Page/CMakeLists.txt index 1883f0bc0aa..f208dc84be2 100644 --- a/dbms/src/Storages/Page/CMakeLists.txt +++ b/dbms/src/Storages/Page/CMakeLists.txt @@ -13,13 +13,4 @@ # limitations under the License. add_subdirectory(V2) - -# PageStorage Stress test -if (ENABLE_V3_PAGESTORAGE) - add_headers_and_sources(page_stress_testing stress) - add_headers_and_sources(page_stress_testing stress/workload) - add_executable(page_stress_testing EXCLUDE_FROM_ALL ${page_stress_testing_sources}) - target_link_libraries(page_stress_testing dbms page_storage_v3) - target_include_directories(page_stress_testing PRIVATE stress) - target_compile_options(page_stress_testing PRIVATE -Wno-format -lc++) # turn off printf format check -endif() \ No newline at end of file +add_subdirectory(tools) diff --git a/dbms/src/Storages/Page/Page.h b/dbms/src/Storages/Page/Page.h index b54b25033dd..5328490e5ad 100644 --- a/dbms/src/Storages/Page/Page.h +++ b/dbms/src/Storages/Page/Page.h @@ -128,12 +128,13 @@ struct PageEntry String toDebugString() const { - return fmt::format("PageEntry{{file: {}, offset: 0x{:X}, size: {}, checksum: 0x{:X}, tag: {}, field_offsets_size: {}}}", + return fmt::format("PageEntry{{file: {}, offset: 0x{:X}, size: {}, checksum: 0x{:X}, tag: {}, ref: {}, field_offsets_size: {}}}", file_id, offset, size, checksum, tag, + ref, field_offsets.size()); } diff --git a/dbms/src/Storages/Page/PageUtil.h b/dbms/src/Storages/Page/PageUtil.h index cebcbdb27f2..b0d8f0f88c8 100644 --- a/dbms/src/Storages/Page/PageUtil.h +++ b/dbms/src/Storages/Page/PageUtil.h @@ -281,7 +281,7 @@ void readFile(T & file, } if (unlikely(bytes_read != expected_bytes)) - throw DB::TiFlashException(fmt::format("No enough data in file {}, read bytes: {} , expected bytes: {}", file->getFileName(), bytes_read, expected_bytes), + throw DB::TiFlashException(fmt::format("No enough data in file {}, read bytes: {}, expected bytes: {}, offset: {}", file->getFileName(), bytes_read, expected_bytes, offset), Errors::PageStorage::FileSizeNotMatch); } diff --git a/dbms/src/Storages/Page/V2/CMakeLists.txt b/dbms/src/Storages/Page/V2/CMakeLists.txt index e840960b653..f994358f2b1 100644 --- a/dbms/src/Storages/Page/V2/CMakeLists.txt +++ b/dbms/src/Storages/Page/V2/CMakeLists.txt @@ -42,16 +42,4 @@ add_library(page_storage_v2 EXCLUDE_FROM_ALL ) target_include_directories(page_storage_v2 PUBLIC ${TiFlash_SOURCE_DIR}/contrib/tiflash-proxy/raftstore-proxy/ffi/src) target_link_libraries(page_storage_v2 clickhouse_common_io cpptoml - kv_client tipb) # TODO: remove dependency on these libs. Now we need them for DB::Context - -### Build a control binary for PageStorage -## For `page_ctl`, we need to define `PAGE_STORAGE_UTIL_DEBUGGGING` -add_executable(page_ctl EXCLUDE_FROM_ALL - tests/page_storage_ctl.cpp - ${page_storage_v2_headers} ${page_storage_v2_sources} - ${io_base_headers} ${io_base_sources} -) -target_include_directories(page_ctl PUBLIC ${TiFlash_SOURCE_DIR}/contrib/tiflash-proxy/raftstore-proxy/ffi/src) -target_link_libraries(page_ctl clickhouse_common_io cpptoml) -target_compile_options(page_ctl PRIVATE -Wno-format) -target_compile_definitions(page_ctl PRIVATE PAGE_STORAGE_UTIL_DEBUGGGING DBMS_PUBLIC_GTEST) + kv_client tipb) # TODO: remove dependency on these libs. Now we need them for DB::Context \ No newline at end of file diff --git a/dbms/src/Storages/Page/V2/PageEntries.h b/dbms/src/Storages/Page/V2/PageEntries.h index c99e0dade6b..0a0504b0cb5 100644 --- a/dbms/src/Storages/Page/V2/PageEntries.h +++ b/dbms/src/Storages/Page/V2/PageEntries.h @@ -252,7 +252,7 @@ class PageEntriesMixin protected: template - void decreasePageRef(PageId page_id); + void decreasePageRef(PageId page_id, bool keep_tombstone); void copyEntries(const PageEntriesMixin & rhs) { @@ -370,8 +370,10 @@ void PageEntriesMixin::del(PageId page_id) const size_t num_erase = page_ref.erase(page_id); if (num_erase > 0) { - // decrease origin page's ref counting - decreasePageRef(normal_page_id); + // decrease origin page's ref counting, this method can + // only called by base, so we should remove the entry if + // the ref count down to zero + decreasePageRef(normal_page_id, /*keep_tombstone=*/false); } } @@ -392,7 +394,9 @@ void PageEntriesMixin::ref(const PageId ref_id, const PageId page_id) // if RefPage{ref-id} -> Page{normal_page_id} already exists, just ignore if (ori_ref->second == normal_page_id) return; - decreasePageRef(ori_ref->second); + // this method can only called by base, so we should remove the entry if + // the ref count down to zero + decreasePageRef(ori_ref->second, /*keep_tombstone=*/false); } // build ref page_ref[ref_id] = normal_page_id; @@ -408,7 +412,7 @@ void PageEntriesMixin::ref(const PageId ref_id, const PageId page_id) template template -void PageEntriesMixin::decreasePageRef(const PageId page_id) +void PageEntriesMixin::decreasePageRef(const PageId page_id, bool keep_tombstone) { auto iter = normal_pages.find(page_id); if constexpr (must_exist) @@ -421,8 +425,11 @@ void PageEntriesMixin::decreasePageRef(const PageId page_id) if (iter != normal_pages.end()) { auto & entry = iter->second; - entry.ref -= 1; - if (entry.ref == 0) + if (entry.ref > 0) + { + entry.ref -= 1; + } + if (!keep_tombstone && entry.ref == 0) { normal_pages.erase(iter); } @@ -620,7 +627,10 @@ class PageEntriesForDelta : public PageEntriesMixin { ref_deletions.insert(page_id); } - decreasePageRef(page_id); + // If this is the base version, we should remove the entry if + // the ref count down to zero. Otherwise it is the delta version + // we should keep a tombstone. + decreasePageRef(page_id, /*keep_tombstone=*/!this->isBase()); } for (auto it : rhs.page_ref) { diff --git a/dbms/src/Storages/Page/V2/gc/DataCompactor.h b/dbms/src/Storages/Page/V2/gc/DataCompactor.h index 1ddd36f87dd..eede1775cdf 100644 --- a/dbms/src/Storages/Page/V2/gc/DataCompactor.h +++ b/dbms/src/Storages/Page/V2/gc/DataCompactor.h @@ -60,10 +60,6 @@ class DataCompactor : private boost::noncopyable std::tuple tryMigrate(const PageFileSet & page_files, SnapshotPtr && snapshot, const WritingFilesSnapshot & writing_files); -#ifndef DBMS_PUBLIC_GTEST -private: -#endif - /** * Collect valid page of snapshot. * Return { @@ -72,6 +68,9 @@ class DataCompactor : private boost::noncopyable * } */ static ValidPages collectValidPagesInPageFile(const SnapshotPtr & snapshot); +#ifndef DBMS_PUBLIC_GTEST +private: +#endif struct CompactCandidates { diff --git a/dbms/src/Storages/Page/V2/tests/gtest_page_util.cpp b/dbms/src/Storages/Page/V2/tests/gtest_page_util.cpp index e72c7a87541..c4dd2178eb9 100644 --- a/dbms/src/Storages/Page/V2/tests/gtest_page_util.cpp +++ b/dbms/src/Storages/Page/V2/tests/gtest_page_util.cpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace DB { @@ -30,6 +31,7 @@ namespace tests static const std::string FileName = "page_util_test"; TEST(PageUtilsTest, ReadWriteFile) +try { ::remove(FileName.c_str()); @@ -52,6 +54,7 @@ TEST(PageUtilsTest, ReadWriteFile) ::remove(FileName.c_str()); } +CATCH TEST(PageUtilsTest, FileNotExists) { diff --git a/dbms/src/Storages/Page/V3/BlobStore.cpp b/dbms/src/Storages/Page/V3/BlobStore.cpp index d5f71841b91..3bd0bd9c4fa 100644 --- a/dbms/src/Storages/Page/V3/BlobStore.cpp +++ b/dbms/src/Storages/Page/V3/BlobStore.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -555,7 +556,7 @@ void BlobStore::read(PageIDAndEntriesV3 & entries, const PageHandler & handler, for (const auto & [page_id_v3, entry] : entries) { - auto blob_file = read(entry.file_id, entry.offset, data_buf, entry.size, read_limiter); + auto blob_file = read(page_id_v3, entry.file_id, entry.offset, data_buf, entry.size, read_limiter); if constexpr (BLOBSTORE_CHECKSUM_ON_READ) { @@ -635,7 +636,7 @@ PageMap BlobStore::read(FieldReadInfos & to_read, const ReadLimiterPtr & read_li // TODO: Continuously fields can read by one system call. const auto [beg_offset, end_offset] = entry.getFieldOffsets(field_index); const auto size_to_read = end_offset - beg_offset; - auto blob_file = read(entry.file_id, entry.offset + beg_offset, write_offset, size_to_read, read_limiter); + auto blob_file = read(page_id_v3, entry.file_id, entry.offset + beg_offset, write_offset, size_to_read, read_limiter); fields_offset_in_page.emplace(field_index, read_size_this_entry); if constexpr (BLOBSTORE_CHECKSUM_ON_READ) @@ -732,7 +733,7 @@ PageMap BlobStore::read(PageIDAndEntriesV3 & entries, const ReadLimiterPtr & rea PageMap page_map; for (const auto & [page_id_v3, entry] : entries) { - auto blob_file = read(entry.file_id, entry.offset, pos, entry.size, read_limiter); + auto blob_file = read(page_id_v3, entry.file_id, entry.offset, pos, entry.size, read_limiter); if constexpr (BLOBSTORE_CHECKSUM_ON_READ) { @@ -797,7 +798,7 @@ Page BlobStore::read(const PageIDAndEntryV3 & id_entry, const ReadLimiterPtr & r free(p, buf_size); }); - auto blob_file = read(entry.file_id, entry.offset, data_buf, buf_size, read_limiter); + auto blob_file = read(page_id_v3, entry.file_id, entry.offset, data_buf, buf_size, read_limiter); if constexpr (BLOBSTORE_CHECKSUM_ON_READ) { ChecksumClass digest; @@ -824,11 +825,20 @@ Page BlobStore::read(const PageIDAndEntryV3 & id_entry, const ReadLimiterPtr & r return page; } -BlobFilePtr BlobStore::read(BlobFileId blob_id, BlobFileOffset offset, char * buffers, size_t size, const ReadLimiterPtr & read_limiter, bool background) +BlobFilePtr BlobStore::read(const PageIdV3Internal & page_id_v3, BlobFileId blob_id, BlobFileOffset offset, char * buffers, size_t size, const ReadLimiterPtr & read_limiter, bool background) { assert(buffers != nullptr); - auto blob_file = getBlobFile(blob_id); - blob_file->read(buffers, offset, size, read_limiter, background); + BlobFilePtr blob_file = getBlobFile(blob_id); + try + { + blob_file->read(buffers, offset, size, read_limiter, background); + } + catch (DB::Exception & e) + { + // add debug message + e.addMessage(fmt::format("(error while reading page data [page_id={}] [blob_id={}] [offset={}] [size={}] [background={}])", page_id_v3, blob_id, offset, size, background)); + e.rethrow(); + } return blob_file; } @@ -841,8 +851,8 @@ struct BlobStoreGCInfo toTypeString("Read-Only Blob", 0), toTypeString("No GC Blob", 1), toTypeString("Full GC Blob", 2), - toTypeString("Truncated Blob", 3), - toTypeString("Big Blob", 4)); + toTypeString("Big Blob", 3), + toTypeTruncateString("Truncated Blob")); } void appendToReadOnlyBlob(const BlobFileId blob_id, double valid_rate) @@ -860,23 +870,24 @@ struct BlobStoreGCInfo blob_gc_info[2].emplace_back(std::make_pair(blob_id, valid_rate)); } - void appendToTruncatedBlob(const BlobFileId blob_id, double valid_rate) + void appendToBigBlob(const BlobFileId blob_id, double valid_rate) { blob_gc_info[3].emplace_back(std::make_pair(blob_id, valid_rate)); } - void appendToBigBlob(const BlobFileId blob_id, double valid_rate) + void appendToTruncatedBlob(const BlobFileId blob_id, UInt64 origin_size, UInt64 truncated_size, double valid_rate) { - blob_gc_info[4].emplace_back(std::make_pair(blob_id, valid_rate)); + blob_gc_truncate_info.emplace_back(std::make_tuple(blob_id, origin_size, truncated_size, valid_rate)); } private: // 1. read only blob // 2. no need gc blob // 3. full gc blob - // 4. need truncate blob - // 5. big blob - std::vector> blob_gc_info[5]; + // 4. big blob + std::vector> blob_gc_info[4]; + + std::vector> blob_gc_truncate_info; String toTypeString(const std::string_view prefix, const size_t index) const { @@ -901,6 +912,32 @@ struct BlobStoreGCInfo return fmt_buf.toString(); } + + String toTypeTruncateString(const std::string_view prefix) const + { + FmtBuffer fmt_buf; + if (blob_gc_truncate_info.empty()) + { + fmt_buf.fmtAppend("{}: [null]", prefix); + } + else + { + fmt_buf.fmtAppend("{}: [", prefix); + fmt_buf.joinStr( + blob_gc_truncate_info.begin(), + blob_gc_truncate_info.end(), + [](const auto arg, FmtBuffer & fb) { + fb.fmtAppend("{} origin: {} truncate: {} rate: {:.2f}", // + std::get<0>(arg), // blob id + std::get<1>(arg), // origin size + std::get<2>(arg), // truncated size + std::get<3>(arg)); // valid rate + }, + ", "); + fmt_buf.append("]"); + } + return fmt_buf.toString(); + } }; std::vector BlobStore::getGCStats() @@ -943,7 +980,7 @@ std::vector BlobStore::getGCStats() } auto lock = stat->lock(); - auto right_margin = stat->smap->getRightMargin(); + auto right_margin = stat->smap->getUsedBoundary(); // Avoid divide by zero if (right_margin == 0) @@ -956,14 +993,13 @@ std::vector BlobStore::getGCStats() stat->sm_valid_rate)); } - LOG_FMT_TRACE(log, "Current blob is empty [blob_id={}, total size(all invalid)={}] [valid_rate={}].", stat->id, stat->sm_total_size, stat->sm_valid_rate); - // If current blob empty, the size of in disk blob may not empty // So we need truncate current blob, and let it be reused. auto blobfile = getBlobFile(stat->id); - LOG_FMT_TRACE(log, "Truncate empty blob file [blob_id={}] to 0.", stat->id); + LOG_FMT_INFO(log, "Current blob file is empty, truncated to zero [blob_id={}] [total_size={}] [valid_rate={}]", stat->id, stat->sm_total_size, stat->sm_valid_rate); blobfile->truncate(right_margin); - blobstore_gc_info.appendToTruncatedBlob(stat->id, stat->sm_valid_rate); + blobstore_gc_info.appendToTruncatedBlob(stat->id, stat->sm_total_size, right_margin, stat->sm_valid_rate); + stat->sm_total_size = right_margin; continue; } @@ -1004,9 +1040,10 @@ std::vector BlobStore::getGCStats() auto blobfile = getBlobFile(stat->id); LOG_FMT_TRACE(log, "Truncate blob file [blob_id={}] [origin size={}] [truncated size={}]", stat->id, stat->sm_total_size, right_margin); blobfile->truncate(right_margin); + blobstore_gc_info.appendToTruncatedBlob(stat->id, stat->sm_total_size, right_margin, stat->sm_valid_rate); + stat->sm_total_size = right_margin; stat->sm_valid_rate = stat->sm_valid_size * 1.0 / stat->sm_total_size; - blobstore_gc_info.appendToTruncatedBlob(stat->id, stat->sm_valid_rate); } } } @@ -1117,21 +1154,15 @@ PageEntriesEdit BlobStore::gc(std::map & std::tie(blobfile_id, file_offset_beg) = getPosFromStats(next_alloc_size); } - PageEntryV3 new_entry; - - read(file_id, entry.offset, data_pos, entry.size, read_limiter, /*background*/ true); - - // No need do crc again, crc won't be changed. - new_entry.checksum = entry.checksum; - - // Need copy the field_offsets - new_entry.field_offsets = entry.field_offsets; - - // Entry size won't be changed. - new_entry.size = entry.size; + // Read the data into buffer by old entry + read(page_id, file_id, entry.offset, data_pos, entry.size, read_limiter, /*background*/ true); + // Most vars of the entry is not changed, but the file id and offset + // need to be updated. + PageEntryV3 new_entry = entry; new_entry.file_id = blobfile_id; new_entry.offset = file_offset_beg + offset_in_data; + new_entry.padded_size = 0; // reset padded size to be zero offset_in_data += new_entry.size; data_pos += new_entry.size; diff --git a/dbms/src/Storages/Page/V3/BlobStore.h b/dbms/src/Storages/Page/V3/BlobStore.h index 5a3e98400d1..6b139b98557 100644 --- a/dbms/src/Storages/Page/V3/BlobStore.h +++ b/dbms/src/Storages/Page/V3/BlobStore.h @@ -296,7 +296,7 @@ class BlobStore : private Allocator PageEntriesEdit handleLargeWrite(DB::WriteBatch & wb, const WriteLimiterPtr & write_limiter = nullptr); - BlobFilePtr read(BlobFileId blob_id, BlobFileOffset offset, char * buffers, size_t size, const ReadLimiterPtr & read_limiter = nullptr, bool background = false); + BlobFilePtr read(const PageIdV3Internal & page_id_v3, BlobFileId blob_id, BlobFileOffset offset, char * buffers, size_t size, const ReadLimiterPtr & read_limiter = nullptr, bool background = false); /** * Ask BlobStats to get a span from BlobStat. @@ -316,7 +316,7 @@ class BlobStore : private Allocator BlobFilePtr getBlobFile(BlobFileId blob_id); friend class PageDirectoryFactory; - friend class PageStorageControl; + friend class PageStorageControlV3; #ifndef DBMS_PUBLIC_GTEST private: diff --git a/dbms/src/Storages/Page/V3/PageDirectory.cpp b/dbms/src/Storages/Page/V3/PageDirectory.cpp index e9b754854b8..5eb275f5af5 100644 --- a/dbms/src/Storages/Page/V3/PageDirectory.cpp +++ b/dbms/src/Storages/Page/V3/PageDirectory.cpp @@ -1223,7 +1223,7 @@ bool PageDirectory::tryDumpSnapshot(const ReadLimiterPtr & read_limiter, const W // `being_ref_count` by the function `createSnapshot()`. assert(!files_snap.persisted_log_files.empty()); // should not be empty when `needSave` return true auto log_num = files_snap.persisted_log_files.rbegin()->log_num; - auto identifier = fmt::format("{}_dump_{}", wal->name(), log_num); + auto identifier = fmt::format("{}.dump_{}", wal->name(), log_num); auto snapshot_reader = wal->createReaderForFiles(identifier, files_snap.persisted_log_files, read_limiter); PageDirectoryFactory factory; // we just use the `collapsed_dir` to dump edit of the snapshot, should never call functions like `apply` that diff --git a/dbms/src/Storages/Page/V3/PageDirectory.h b/dbms/src/Storages/Page/V3/PageDirectory.h index 39b5a05a40a..bd7c433022f 100644 --- a/dbms/src/Storages/Page/V3/PageDirectory.h +++ b/dbms/src/Storages/Page/V3/PageDirectory.h @@ -255,7 +255,7 @@ class VersionedPageEntries being_ref_count, entries.size()); } - friend class PageStorageControl; + friend class PageStorageControlV3; private: mutable std::mutex m; @@ -376,7 +376,7 @@ class PageDirectory DISALLOW_COPY_AND_MOVE(PageDirectory); friend class PageDirectoryFactory; - friend class PageStorageControl; + friend class PageStorageControlV3; private: // Only `std::map` is allow for `MVCCMap`. Cause `std::map::insert` ensure that diff --git a/dbms/src/Storages/Page/V3/PageStorageImpl.h b/dbms/src/Storages/Page/V3/PageStorageImpl.h index f49601ce2ad..b4cdd425e59 100644 --- a/dbms/src/Storages/Page/V3/PageStorageImpl.h +++ b/dbms/src/Storages/Page/V3/PageStorageImpl.h @@ -115,7 +115,7 @@ class PageStorageImpl : public DB::PageStorage #endif friend class PageDirectoryFactory; - friend class PageStorageControl; + friend class PageStorageControlV3; #ifndef DBMS_PUBLIC_GTEST private: #endif diff --git a/dbms/src/Storages/Page/V3/spacemap/SpaceMap.h b/dbms/src/Storages/Page/V3/spacemap/SpaceMap.h index ae44b608de0..d230b2f3e35 100644 --- a/dbms/src/Storages/Page/V3/spacemap/SpaceMap.h +++ b/dbms/src/Storages/Page/V3/spacemap/SpaceMap.h @@ -95,9 +95,11 @@ class SpaceMap virtual std::tuple searchInsertOffset(size_t size) = 0; /** - * Get the offset of the last free block. `[margin_offset, +∞)` is not used at all. + * Get the used boundary of this SpaceMap. + * The return value (`used_boundary`) means that `[used_bounary + 1, +∞)` is safe to be truncated. + * If the `used_boundary` is equal to the `end` of this SpaceMap, it means that there is no space to be truncated. */ - virtual UInt64 getRightMargin() = 0; + virtual UInt64 getUsedBoundary() = 0; /** * Get the accurate max capacity of the space map. diff --git a/dbms/src/Storages/Page/V3/spacemap/SpaceMapBig.h b/dbms/src/Storages/Page/V3/spacemap/SpaceMapBig.h index 22128a09f30..81c2a5cb786 100644 --- a/dbms/src/Storages/Page/V3/spacemap/SpaceMapBig.h +++ b/dbms/src/Storages/Page/V3/spacemap/SpaceMapBig.h @@ -74,7 +74,7 @@ class BigSpaceMap return std::make_pair(size_in_used, size_in_used); } - UInt64 getRightMargin() override + UInt64 getUsedBoundary() override { return end; } diff --git a/dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.cpp b/dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.cpp index 54275574060..4bd53b93e07 100644 --- a/dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.cpp +++ b/dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.cpp @@ -84,7 +84,7 @@ static void rb_get_new_entry(struct SmapRbEntry ** entry, UInt64 start, UInt64 c { struct SmapRbEntry * new_entry; - new_entry = static_cast(calloc(1, sizeof(struct SmapRbEntry))); + new_entry = static_cast(calloc(1, sizeof(struct SmapRbEntry))); // NOLINT if (new_entry == nullptr) { return; @@ -115,7 +115,7 @@ inline static void rb_free_entry(struct RbPrivate * private_data, struct SmapRbE private_data->read_index_next = nullptr; } - free(entry); + free(entry); // NOLINT } @@ -419,7 +419,7 @@ std::shared_ptr RBTreeSpaceMap::create(UInt64 start, UInt64 end) { auto ptr = std::shared_ptr(new RBTreeSpaceMap(start, end)); - ptr->rb_tree = static_cast(calloc(1, sizeof(struct RbPrivate))); + ptr->rb_tree = static_cast(calloc(1, sizeof(struct RbPrivate))); // NOLINT if (ptr->rb_tree == nullptr) { return nullptr; @@ -435,7 +435,7 @@ std::shared_ptr RBTreeSpaceMap::create(UInt64 start, UInt64 end) if (!rb_insert_entry(start, end, ptr->rb_tree, ptr->log)) { LOG_FMT_ERROR(ptr->log, "Erorr happend, when mark all space free. [start={}] , [end={}]", start, end); - free(ptr->rb_tree); + free(ptr->rb_tree); // NOLINT return nullptr; } return ptr; @@ -451,7 +451,7 @@ static void rb_free_tree(struct rb_root * root) next = rb_tree_next(node); entry = node_to_entry(node); rb_node_remove(node, root); - free(entry); + free(entry); // NOLINT } } @@ -460,7 +460,7 @@ void RBTreeSpaceMap::freeSmap() if (rb_tree) { rb_free_tree(&rb_tree->root); - free(rb_tree); + free(rb_tree); // NOLINT } } @@ -734,7 +734,7 @@ std::pair RBTreeSpaceMap::getSizes() const } } -UInt64 RBTreeSpaceMap::getRightMargin() +UInt64 RBTreeSpaceMap::getUsedBoundary() { struct rb_node * node = rb_tree_last(&rb_tree->root); if (node == nullptr) @@ -743,6 +743,20 @@ UInt64 RBTreeSpaceMap::getRightMargin() } auto * entry = node_to_entry(node); + + // If the `offset+size` of the last free node is not equal to `end`, it means the range `[last_node.offset, end)` is marked as used, + // then we should return `end` as the used boundary. + // + // eg. + // 1. The spacemap manage a space of `[0, 100]` + // 2. A span {offset=90, size=10} is marked as used, then the free range in SpaceMap is `[0, 90)` + // 3. The return value should be 100 + if (entry->start + entry->count != end) + { + return end; + } + + // Else we should return the offset of last free node return entry->start; } diff --git a/dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.h b/dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.h index 0393fda081b..04691007a47 100644 --- a/dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.h +++ b/dbms/src/Storages/Page/V3/spacemap/SpaceMapRBTree.h @@ -46,7 +46,7 @@ class RBTreeSpaceMap std::pair getSizes() const override; - UInt64 getRightMargin() override; + UInt64 getUsedBoundary() override; protected: RBTreeSpaceMap(UInt64 start, UInt64 end) diff --git a/dbms/src/Storages/Page/V3/spacemap/SpaceMapSTDMap.h b/dbms/src/Storages/Page/V3/spacemap/SpaceMapSTDMap.h index b6ff8797f0f..41ddd77d03a 100644 --- a/dbms/src/Storages/Page/V3/spacemap/SpaceMapSTDMap.h +++ b/dbms/src/Storages/Page/V3/spacemap/SpaceMapSTDMap.h @@ -111,13 +111,29 @@ class STDMapSpaceMap } } - UInt64 getRightMargin() override + UInt64 getUsedBoundary() override { if (free_map.empty()) { - return end - start; + return end; } - return free_map.rbegin()->first; + + const auto & last_node_it = free_map.rbegin(); + + // If the `offset+size` of the last free node is not equal to `end`, it means the range `[last_node.offset, end)` is marked as used, + // then we should return `end` as the used boundary. + // + // eg. + // 1. The spacemap manage a space of `[0, 100]` + // 2. A span {offset=90, size=10} is marked as used, then the free range in SpaceMap is `[0, 90)` + // 3. The return value should be 100 + if (last_node_it->first + last_node_it->second != end) + { + return end; + } + + // Else we should return the offset of last free node + return last_node_it->first; } bool isMarkUnused(UInt64 offset, size_t length) override diff --git a/dbms/src/Storages/Page/V3/tests/CMakeLists.txt b/dbms/src/Storages/Page/V3/tests/CMakeLists.txt index 8bab6afcded..355247c9eba 100644 --- a/dbms/src/Storages/Page/V3/tests/CMakeLists.txt +++ b/dbms/src/Storages/Page/V3/tests/CMakeLists.txt @@ -26,10 +26,4 @@ add_executable(gtests_page_storage_v3 ${ps_v3_gtest_sources} ${TiFlash_SOURCE_DI target_link_libraries(gtests_page_storage_v3 page_storage_v3 gtest_main) target_compile_options(gtests_page_storage_v3 PRIVATE -Wno-unknown-pragmas) target_compile_definitions(gtests_page_storage_v3 PRIVATE DBMS_PUBLIC_GTEST) -add_check(gtests_page_storage_v3) - - -add_executable(page_storage_ctl EXCLUDE_FROM_ALL page_storage_ctl.cpp) -target_compile_definitions(page_storage_ctl PUBLIC DBMS_PUBLIC_GTEST) -target_link_libraries(page_storage_ctl dbms page_storage_v3) -target_compile_options(page_storage_ctl PRIVATE -Wno-format -lc++) # turn off printf format check +add_check(gtests_page_storage_v3) \ No newline at end of file diff --git a/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp b/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp index 048140ed04f..fdd08c7cb8e 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_blob_store.cpp @@ -82,6 +82,7 @@ try stats.restoreByEntry(PageEntryV3{ .file_id = file_id1, .size = 128, + .padded_size = 0, .tag = 0, .offset = 1024, .checksum = 0x4567, @@ -89,6 +90,7 @@ try stats.restoreByEntry(PageEntryV3{ .file_id = file_id1, .size = 512, + .padded_size = 0, .tag = 0, .offset = 2048, .checksum = 0x4567, @@ -96,6 +98,7 @@ try stats.restoreByEntry(PageEntryV3{ .file_id = file_id2, .size = 512, + .padded_size = 0, .tag = 0, .offset = 2048, .checksum = 0x4567, @@ -531,7 +534,8 @@ TEST_F(BlobStoreTest, testWriteRead) ASSERT_EQ(record.entry.file_id, 1); // Read directly from the file - blob_store.read(record.entry.file_id, + blob_store.read(buildV3Id(TEST_NAMESPACE_ID, page_id), + record.entry.file_id, record.entry.offset, c_buff_read + index * buff_size, record.entry.size, @@ -631,7 +635,8 @@ TEST_F(BlobStoreTest, testWriteReadWithIOLimiter) { for (const auto & record : edits[i].getRecords()) { - blob_store.read(record.entry.file_id, + blob_store.read(buildV3Id(TEST_NAMESPACE_ID, page_id), + record.entry.file_id, record.entry.offset, c_buff_read + i * buff_size, record.entry.size, @@ -809,7 +814,8 @@ TEST_F(BlobStoreTest, testFeildOffsetWriteRead) ASSERT_EQ(check_field_sizes, offsets); // Read - blob_store.read(record.entry.file_id, + blob_store.read(buildV3Id(TEST_NAMESPACE_ID, page_id), + record.entry.file_id, record.entry.offset, c_buff_read + index * buff_size, record.entry.size, diff --git a/dbms/src/Storages/Page/V3/tests/gtest_free_map.cpp b/dbms/src/Storages/Page/V3/tests/gtest_free_map.cpp index f7120f000b2..faec139920b 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_free_map.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_free_map.cpp @@ -427,6 +427,43 @@ TEST_P(SpaceMapTest, TestGetMaxCap) } } + +TEST_P(SpaceMapTest, TestGetUsedBoundary) +{ + { + auto smap = SpaceMap::createSpaceMap(test_type, 0, 100); + ASSERT_TRUE(smap->markUsed(50, 10)); + ASSERT_EQ(smap->getUsedBoundary(), 60); + ASSERT_TRUE(smap->markUsed(80, 10)); + ASSERT_EQ(smap->getUsedBoundary(), 90); + + ASSERT_TRUE(smap->markUsed(90, 10)); + ASSERT_EQ(smap->getUsedBoundary(), 100); + } + + { + auto smap = SpaceMap::createSpaceMap(test_type, 0, 100); + ASSERT_TRUE(smap->markUsed(90, 10)); + ASSERT_EQ(smap->getUsedBoundary(), 100); + + ASSERT_TRUE(smap->markUsed(20, 10)); + ASSERT_EQ(smap->getUsedBoundary(), 100); + + ASSERT_TRUE(smap->markFree(90, 10)); + ASSERT_EQ(smap->getUsedBoundary(), 30); + + ASSERT_TRUE(smap->markUsed(90, 10)); + ASSERT_EQ(smap->getUsedBoundary(), 100); + } + + { + auto smap = SpaceMap::createSpaceMap(test_type, 0, 100); + ASSERT_EQ(smap->getUsedBoundary(), 0); + ASSERT_TRUE(smap->markUsed(0, 100)); + ASSERT_EQ(smap->getUsedBoundary(), 100); + } +} + INSTANTIATE_TEST_CASE_P( Type, SpaceMapTest, diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_directory.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_directory.cpp index 6e2b0efa1ea..83e07f75d37 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_directory.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_directory.cpp @@ -75,7 +75,7 @@ try auto snap0 = dir->createSnapshot(); EXPECT_ENTRY_NOT_EXIST(dir, 1, snap0); - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry1); @@ -85,7 +85,7 @@ try auto snap1 = dir->createSnapshot(); EXPECT_ENTRY_EQ(entry1, dir, 1, snap1); - PageEntryV3 entry2{.file_id = 2, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 2, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(2, entry2); @@ -102,7 +102,7 @@ try EXPECT_ENTRIES_EQ(expected_entries, dir, ids, snap2); } - PageEntryV3 entry2_v2{.file_id = 2 + 102, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2_v2{.file_id = 2 + 102, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.del(2); @@ -123,7 +123,7 @@ try auto snap0 = dir->createSnapshot(); EXPECT_ENTRY_NOT_EXIST(dir, page_id, snap0); - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(page_id, entry1); @@ -133,7 +133,7 @@ try auto snap1 = dir->createSnapshot(); EXPECT_ENTRY_EQ(entry1, dir, page_id, snap1); - PageEntryV3 entry2{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x1234, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x1234, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(page_id, entry2); @@ -151,7 +151,7 @@ try // Put identical page within one `edit` page_id++; - PageEntryV3 entry3{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x12345, .checksum = 0x4567}; + PageEntryV3 entry3{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x12345, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(page_id, entry1); @@ -172,8 +172,8 @@ CATCH TEST_F(PageDirectoryTest, ApplyPutDelRead) try { - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry2{.file_id = 2, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 2, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry1); @@ -185,8 +185,8 @@ try EXPECT_ENTRY_EQ(entry1, dir, 1, snap1); EXPECT_ENTRY_EQ(entry2, dir, 2, snap1); - PageEntryV3 entry3{.file_id = 3, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry4{.file_id = 4, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry3{.file_id = 3, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry4{.file_id = 4, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.del(2); @@ -217,8 +217,8 @@ CATCH TEST_F(PageDirectoryTest, ApplyUpdateOnRefEntries) try { - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry2{.file_id = 2, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 2, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry1); @@ -236,14 +236,14 @@ try EXPECT_ENTRY_EQ(entry2, dir, 3, snap1); // Update on ref page is not allowed - PageEntryV3 entry_updated{.file_id = 999, .size = 16, .tag = 0, .offset = 0x123, .checksum = 0x123}; + PageEntryV3 entry_updated{.file_id = 999, .size = 16, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x123}; { PageEntriesEdit edit; edit.put(3, entry_updated); ASSERT_ANY_THROW(dir->apply(std::move(edit))); } - PageEntryV3 entry_updated2{.file_id = 777, .size = 16, .tag = 0, .offset = 0x123, .checksum = 0x123}; + PageEntryV3 entry_updated2{.file_id = 777, .size = 16, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x123}; { PageEntriesEdit edit; edit.put(2, entry_updated2); @@ -255,8 +255,8 @@ CATCH TEST_F(PageDirectoryTest, ApplyDeleteOnRefEntries) try { - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry2{.file_id = 2, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 2, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry1); @@ -305,8 +305,8 @@ CATCH TEST_F(PageDirectoryTest, ApplyRefOnRefEntries) try { - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry2{.file_id = 2, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 2, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry1); @@ -343,8 +343,8 @@ CATCH TEST_F(PageDirectoryTest, ApplyDuplicatedRefEntries) try { - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry2{.file_id = 2, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 2, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry1); @@ -410,8 +410,8 @@ CATCH TEST_F(PageDirectoryTest, ApplyCollapseDuplicatedRefEntries) try { - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry2{.file_id = 2, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 2, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry1); @@ -447,9 +447,9 @@ CATCH TEST_F(PageDirectoryTest, ApplyRefToNotExistEntry) try { - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry2{.file_id = 2, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry3{.file_id = 3, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 2, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry3{.file_id = 3, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry1); @@ -628,12 +628,12 @@ try } CATCH -#define INSERT_BLOBID_ENTRY(BLOBID, VERSION) \ - PageEntryV3 entry_v##VERSION{.file_id = (BLOBID), .size = (VERSION), .tag = 0, .offset = 0x123, .checksum = 0x4567}; \ +#define INSERT_BLOBID_ENTRY(BLOBID, VERSION) \ + PageEntryV3 entry_v##VERSION{.file_id = (BLOBID), .size = (VERSION), .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; \ entries.createNewEntry(PageVersion(VERSION), entry_v##VERSION); #define INSERT_ENTRY(VERSION) INSERT_BLOBID_ENTRY(1, VERSION) -#define INSERT_GC_ENTRY(VERSION, EPOCH) \ - PageEntryV3 entry_gc_v##VERSION##_##EPOCH{.file_id = 2, .size = 100 * (VERSION) + (EPOCH), .tag = 0, .offset = 0x234, .checksum = 0x5678}; \ +#define INSERT_GC_ENTRY(VERSION, EPOCH) \ + PageEntryV3 entry_gc_v##VERSION##_##EPOCH{.file_id = 2, .size = 100 * (VERSION) + (EPOCH), .padded_size = 0, .tag = 0, .offset = 0x234, .checksum = 0x5678}; \ entries.createNewEntry(PageVersion((VERSION), (EPOCH)), entry_gc_v##VERSION##_##EPOCH); class VersionedEntriesTest : public ::testing::Test @@ -1271,12 +1271,12 @@ class PageDirectoryGCTest : public PageDirectoryTest { }; -#define INSERT_ENTRY_TO(PAGE_ID, VERSION, BLOB_FILE_ID) \ - PageEntryV3 entry_v##VERSION{.file_id = (BLOB_FILE_ID), .size = (VERSION), .tag = 0, .offset = 0x123, .checksum = 0x4567}; \ - { \ - PageEntriesEdit edit; \ - edit.put((PAGE_ID), entry_v##VERSION); \ - dir->apply(std::move(edit)); \ +#define INSERT_ENTRY_TO(PAGE_ID, VERSION, BLOB_FILE_ID) \ + PageEntryV3 entry_v##VERSION{.file_id = (BLOB_FILE_ID), .size = (VERSION), .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; \ + { \ + PageEntriesEdit edit; \ + edit.put((PAGE_ID), entry_v##VERSION); \ + dir->apply(std::move(edit)); \ } // Insert an entry into mvcc directory #define INSERT_ENTRY(PAGE_ID, VERSION) INSERT_ENTRY_TO(PAGE_ID, VERSION, 1) @@ -1566,7 +1566,7 @@ try INSERT_ENTRY_ACQ_SNAP(page_id, 5); INSERT_ENTRY(another_page_id, 6); INSERT_ENTRY(another_page_id, 7); - PageEntryV3 entry_v8{.file_id = 1, .size = 8, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_v8{.file_id = 1, .size = 8, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.del(page_id); @@ -1756,7 +1756,7 @@ TEST_F(PageDirectoryGCTest, GCOnRefedEntries) try { // 10->entry1, 11->10=>11->entry1; del 10->entry1 - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(10, entry1); @@ -1793,7 +1793,7 @@ TEST_F(PageDirectoryGCTest, GCOnRefedEntries2) try { // 10->entry1, 11->10=>11->entry1; del 10->entry1 - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(10, entry1); @@ -1836,7 +1836,7 @@ TEST_F(PageDirectoryGCTest, UpsertOnRefedEntries) try { // 10->entry1, 11->10, 12->10 - PageEntryV3 entry1{.file_id = 1, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry1{.file_id = 1, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(10, entry1); @@ -1860,7 +1860,7 @@ try } // upsert 10->entry2 - PageEntryV3 entry2{.file_id = 2, .size = 1024, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry2{.file_id = 2, .size = 1024, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; auto full_gc_entries = dir->getEntriesByBlobIds({1}); @@ -2024,10 +2024,10 @@ try return d; }; - PageEntryV3 entry_1_v1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_1_v2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_2_v1{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_2_v2{.file_id = 2, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_1_v1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_1_v2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_2_v1{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_2_v2{.file_id = 2, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry_1_v1); @@ -2055,8 +2055,8 @@ try // 10->ext, 11->10, del 10->ext // 50->entry, 51->50, 52->51=>50, del 50 - PageEntryV3 entry_50{.file_id = 1, .size = 50, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_60{.file_id = 1, .size = 90, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_50{.file_id = 1, .size = 50, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_60{.file_id = 1, .size = 90, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; { PageEntriesEdit edit; edit.del(2); @@ -2218,9 +2218,9 @@ try Poco::File(fmt::format("{}/{}{}", path, BlobFile::BLOB_PREFIX_NAME, file_id1)).createFile(); Poco::File(fmt::format("{}/{}{}", path, BlobFile::BLOB_PREFIX_NAME, file_id2)).createFile(); - PageEntryV3 entry_1_v1{.file_id = file_id1, .size = 7890, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_5_v1{.file_id = file_id2, .size = 255, .tag = 0, .offset = 0x100, .checksum = 0x4567}; - PageEntryV3 entry_5_v2{.file_id = file_id2, .size = 255, .tag = 0, .offset = 0x400, .checksum = 0x4567}; + PageEntryV3 entry_1_v1{.file_id = file_id1, .size = 7890, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_5_v1{.file_id = file_id2, .size = 255, .padded_size = 0, .tag = 0, .offset = 0x100, .checksum = 0x4567}; + PageEntryV3 entry_5_v2{.file_id = file_id2, .size = 255, .padded_size = 0, .tag = 0, .offset = 0x400, .checksum = 0x4567}; { PageEntriesEdit edit; edit.put(1, entry_1_v1); @@ -2275,8 +2275,8 @@ CATCH TEST_F(PageDirectoryGCTest, CleanAfterDecreaseRef) try { - PageEntryV3 entry_50_1{.file_id = 1, .size = 7890, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_50_2{.file_id = 2, .size = 7890, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_50_1{.file_id = 1, .size = 7890, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_50_2{.file_id = 2, .size = 7890, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; auto restore_from_edit = [](const PageEntriesEdit & edit) { auto ctx = ::DB::tests::TiFlashTestEnv::getContext(); diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp index ce2ba0adaf4..f9ef25cb973 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp @@ -1408,6 +1408,88 @@ try } CATCH +TEST_F(PageStorageTest, TruncateBlobFile) +try +{ + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + { + WriteBatch batch; + batch.putPage(1, 0, std::make_shared(c_buff, buf_sz), buf_sz, {}); + page_storage->write(std::move(batch)); + } + + auto blob_file = Poco::File(getTemporaryPath() + "/blobfile_1"); + + page_storage = reopenWithConfig(config); + EXPECT_GT(blob_file.getSize(), 0); + + { + WriteBatch batch; + batch.delPage(1); + page_storage->write(std::move(batch)); + } + page_storage = reopenWithConfig(config); + page_storage->gc(/*not_skip*/ false, nullptr, nullptr); + EXPECT_EQ(blob_file.getSize(), 0); +} +CATCH + +TEST_F(PageStorageTest, EntryTagAfterFullGC) +try +{ + { + PageStorage::Config config; + config.blob_heavy_gc_valid_rate = 1.0; /// always run full gc + page_storage = reopenWithConfig(config); + } + + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + PageId page_id = 120; + UInt64 tag = 12345; + { + WriteBatch batch; + batch.putPage(page_id, tag, std::make_shared(c_buff, buf_sz), buf_sz, {}); + page_storage->write(std::move(batch)); + } + + { + auto entry = page_storage->getEntry(page_id); + ASSERT_EQ(entry.tag, tag); + auto page = page_storage->read(page_id); + for (size_t i = 0; i < buf_sz; ++i) + { + EXPECT_EQ(*(page.data.begin() + i), static_cast(i % 0xff)); + } + } + + auto done_full_gc = page_storage->gc(); + EXPECT_TRUE(done_full_gc); + + { + auto entry = page_storage->getEntry(page_id); + ASSERT_EQ(entry.tag, tag); + auto page = page_storage->read(page_id); + for (size_t i = 0; i < buf_sz; ++i) + { + EXPECT_EQ(*(page.data.begin() + i), static_cast(i % 0xff)); + } + } +} +CATCH } // namespace PS::V3::tests } // namespace DB diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp index e2e9ea2ffd3..74e56c929d8 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage_mix_mode.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -19,6 +20,8 @@ #include #include #include +#include +#include #include namespace DB @@ -500,6 +503,318 @@ try } CATCH +// v2 put 1, v2 ref 2->1, get snapshot s1, v3 del 1, read s1 +TEST_F(PageStorageMixedTest, RefWithSnapshot) +try +{ + const size_t buf_sz = 1024; + char c_buff[buf_sz] = {0}; + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, 0, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + ASSERT_EQ(page_reader_mix->getNormalPageId(2), 1); + } + + auto snapshot_mix_mode = page_reader_mix->getSnapshot("ReadWithSnapshotAfterDelOrigin"); + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_mode); + const auto & page1 = page_reader_mix_with_snap.read(1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + } + + { + WriteBatch batch; + batch.delPage(1); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_mode); + const auto & page1 = page_reader_mix_with_snap.read(1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + } +} +CATCH + +// v2 put 1, v2 ref 2->1, get snapshot s1, v3 del 1, v3 del 2, read s1 +TEST_F(PageStorageMixedTest, RefWithDelSnapshot) +try +{ + const size_t buf_sz = 1024; + char c_buff[buf_sz] = {0}; + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, 0, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + ASSERT_EQ(page_reader_mix->getNormalPageId(2), 1); + } + + auto snapshot_mix_mode = page_reader_mix->getSnapshot("ReadWithSnapshotAfterDelOrigin"); + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_mode); + const auto & page1 = page_reader_mix_with_snap.read(1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + } + + { + WriteBatch batch; + batch.delPage(1); + batch.delPage(2); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_mode); + const auto & page1 = page_reader_mix_with_snap.read(1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + } +} +CATCH + +// v2 put 1, v2 ref 2->1, v3 del 1, get snapshot s1, v3 del 2, use s1 read 2 +TEST_F(PageStorageMixedTest, RefWithDelSnapshot2) +try +{ + const size_t buf_sz = 1024; + char c_buff[buf_sz] = {0}; + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, 0, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + ASSERT_EQ(page_reader_mix->getNormalPageId(2), 1); + } + + { + WriteBatch batch; + batch.delPage(1); + page_writer_mix->write(std::move(batch), nullptr); + } + + auto snapshot_mix_mode = page_reader_mix->getSnapshot("ReadWithSnapshotAfterDelOrigin"); + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_mode); + const auto & page1 = page_reader_mix_with_snap.read(2); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 2); + } + + { + WriteBatch batch; + batch.delPage(2); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_mix_mode); + const auto & page1 = page_reader_mix_with_snap.read(2); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 2); + } +} +CATCH + +// v2 put 1, v2 del 1, v3 put 2, v3 del 2 +TEST_F(PageStorageMixedTest, GetMaxId) +try +{ + const size_t buf_sz = 1024; + char c_buff[buf_sz] = {0}; + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, 0, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.delPage(1); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(2, 0, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + ASSERT_EQ(storage_pool_mix->newLogPageId(), 3); + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(3, 0, buff, buf_sz); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.delPage(3); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + ASSERT_EQ(storage_pool_mix->newLogPageId(), 4); + } +} +CATCH + + +TEST_F(PageStorageMixedTest, ReuseV2ID) +try +{ + const size_t buf_sz = 1024; + char c_buff[buf_sz] = {0}; + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, 0, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.delPage(1); + page_writer_v2->write(std::move(batch), nullptr); + } + + { + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::ONLY_V3); + ASSERT_EQ(storage_pool_mix->newLogPageId(), 1); + } + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, 0, buff, buf_sz); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + WriteBatch batch; + batch.delPage(1); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::ONLY_V3); + ASSERT_EQ(storage_pool_mix->newLogPageId(), 2); + } +} +CATCH + +// v2 put 1, v3 ref 2->1, reload, check max id, get snapshot s1, v3 del 1, get snapshot s2, v3 del 2, get snapshot s3, check snapshots +TEST_F(PageStorageMixedTest, V3RefV2WithSnapshot) +try +{ + const size_t buf_sz = 1024; + char c_buff[buf_sz] = {0}; + + { + WriteBatch batch; + ReadBufferPtr buff = std::make_shared(c_buff, sizeof(c_buff)); + batch.putPage(1, 0, buff, buf_sz); + page_writer_v2->write(std::move(batch), nullptr); + } + + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + + { + WriteBatch batch; + batch.putRefPage(2, 1); + page_writer_mix->write(std::move(batch), nullptr); + } + + { + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + ASSERT_EQ(page_reader_mix->getNormalPageId(2), 1); + ASSERT_EQ(storage_pool_mix->newLogPageId(), 3); + } + + auto snapshot_before_del = page_reader_mix->getSnapshot("ReadWithSnapshotBeforeDelOrigin"); + + { + WriteBatch batch; + batch.delPage(1); + page_writer_mix->write(std::move(batch), nullptr); + } + + auto snapshot_after_del_origin = page_reader_mix->getSnapshot("ReadWithSnapshotAfterDelOrigin"); + + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_before_del); + const auto & page1 = page_reader_mix_with_snap.read(1); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 1); + } + + { + WriteBatch batch; + batch.delPage(2); + page_writer_mix->write(std::move(batch), nullptr); + } + + auto snapshot_after_del_all = page_reader_mix->getSnapshot("ReadWithSnapshotAfterDelAll"); + + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_after_del_origin); + const auto & page1 = page_reader_mix_with_snap.read(2); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 2); + } + + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_after_del_origin); + const auto & page1 = page_reader_mix_with_snap.read(2); + ASSERT_PAGE_EQ(c_buff, buf_sz, page1, 2); + } + + { + auto page_reader_mix_with_snap = storage_pool_mix->newLogReader(nullptr, snapshot_after_del_all); + ASSERT_ANY_THROW(page_reader_mix_with_snap.read(2)); + } +} +CATCH TEST_F(PageStorageMixedTest, MockDTIngest) try @@ -577,6 +892,129 @@ try } CATCH + +TEST_F(PageStorageMixedTest, RefV2External2) +try +{ + auto logger = DB::Logger::get("PageStorageMixedTest"); + { + WriteBatch batch; + batch.putExternal(100, 0); + batch.putRefPage(101, 100); + batch.delPage(100); + batch.putExternal(102, 0); + page_writer_v2->write(std::move(batch), nullptr); + } + + ASSERT_EQ(reloadMixedStoragePool(), PageStorageRunMode::MIX_MODE); + { + WriteBatch batch; + batch.putExternal(100, 0); + batch.putRefPage(101, 100); + batch.delPage(100); + batch.putExternal(102, 0); + page_writer_mix->writeIntoV3(std::move(batch), nullptr); + } + { + auto snap = storage_pool_mix->log_storage_v2->getSnapshot("zzz"); // must hold + // after transform to v3, delete these from v2 + WriteBatch batch; + batch.delPage(100); + batch.delPage(101); + batch.delPage(102); + page_writer_mix->writeIntoV2(std::move(batch), nullptr); + } + + { + LOG_FMT_INFO(logger, "first check alive id in v2"); + auto alive_dt_ids_in_v2 = storage_pool_mix->log_storage_v2->getAliveExternalPageIds(TEST_NAMESPACE_ID); + EXPECT_EQ(alive_dt_ids_in_v2.size(), 0); + + storage_pool_mix->log_storage_v3->gc(false, nullptr, nullptr); + auto alive_dt_ids_in_v3 = storage_pool_mix->log_storage_v3->getAliveExternalPageIds(TEST_NAMESPACE_ID); + ASSERT_EQ(alive_dt_ids_in_v3.size(), 2); + auto iter = alive_dt_ids_in_v3.begin(); + EXPECT_EQ(*iter, 100); + iter++; + EXPECT_EQ(*iter, 102); + } + + { + LOG_FMT_INFO(logger, "remove 100, create 105"); + StorageSnapshot snap(*storage_pool_mix, nullptr, "xxx", true); // must hold and write + // write delete again + WriteBatch batch; + batch.delPage(100); + batch.putExternal(105, 0); + page_writer_mix->write(std::move(batch), nullptr); + LOG_FMT_INFO(logger, "done"); + } + { + LOG_FMT_INFO(logger, "remove 101, create 106"); + StorageSnapshot snap(*storage_pool_mix, nullptr, "xxx", true); // must hold and write + // write delete again + WriteBatch batch; + batch.delPage(101); + batch.putExternal(106, 0); + page_writer_mix->write(std::move(batch), nullptr); + LOG_FMT_INFO(logger, "done"); + } + { + LOG_FMT_INFO(logger, "remove 102, create 107"); + StorageSnapshot snap(*storage_pool_mix, nullptr, "xxx", true); // must hold and write + // write delete again + WriteBatch batch; + batch.delPage(102); + batch.putExternal(107, 0); + page_writer_mix->write(std::move(batch), nullptr); + LOG_FMT_INFO(logger, "done"); + } + + { + LOG_FMT_INFO(logger, "second check alive id in v2"); + auto alive_dt_ids_in_v2 = storage_pool_mix->log_storage_v2->getAliveExternalPageIds(TEST_NAMESPACE_ID); + EXPECT_EQ(alive_dt_ids_in_v2.size(), 0) << fmt::format("{}", alive_dt_ids_in_v2); + + storage_pool_mix->log_storage_v3->gc(false, nullptr, nullptr); + auto alive_dt_ids_in_v3 = storage_pool_mix->log_storage_v3->getAliveExternalPageIds(TEST_NAMESPACE_ID); + ASSERT_EQ(alive_dt_ids_in_v3.size(), 3) << fmt::format("{}", alive_dt_ids_in_v3); + auto iter = alive_dt_ids_in_v3.begin(); + EXPECT_EQ(*iter, 105); + iter++; + EXPECT_EQ(*iter, 106); + iter++; + EXPECT_EQ(*iter, 107); + } + { + LOG_FMT_INFO(logger, "third check alive id in v2"); + auto alive_dt_ids_in_v2 = storage_pool_mix->log_storage_v2->getAliveExternalPageIds(TEST_NAMESPACE_ID); + EXPECT_EQ(alive_dt_ids_in_v2.size(), 0) << fmt::format("{}", alive_dt_ids_in_v2); + + storage_pool_mix->log_storage_v3->gc(false, nullptr, nullptr); + auto alive_dt_ids_in_v3 = storage_pool_mix->log_storage_v3->getAliveExternalPageIds(TEST_NAMESPACE_ID); + ASSERT_EQ(alive_dt_ids_in_v3.size(), 3) << fmt::format("{}", alive_dt_ids_in_v3); + auto iter = alive_dt_ids_in_v3.begin(); + EXPECT_EQ(*iter, 105); + iter++; + EXPECT_EQ(*iter, 106); + iter++; + EXPECT_EQ(*iter, 107); + } + + { + // cleanup v3 + WriteBatch batch; + batch.delPage(100); + batch.delPage(101); + batch.delPage(102); + batch.delPage(105); + batch.delPage(106); + batch.delPage(107); + page_writer_mix->write(std::move(batch), nullptr); + } +} +CATCH + TEST_F(PageStorageMixedTest, ReadWithSnapshotAfterMergeDelta) try { diff --git a/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp b/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp index 6d47adabbc5..b4e6c2d9204 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_wal_store.cpp @@ -34,8 +34,8 @@ namespace DB::PS::V3::tests { TEST(WALSeriTest, AllPuts) { - PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20(/*seq=*/20); PageEntriesEdit edit; edit.put(1, entry_p1); @@ -56,8 +56,8 @@ TEST(WALSeriTest, AllPuts) TEST(WALSeriTest, PutsAndRefsAndDels) try { - PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver21(/*seq=*/21); PageEntriesEdit edit; edit.put(3, entry_p3); @@ -104,9 +104,9 @@ CATCH TEST(WALSeriTest, Upserts) { - PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1); PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1); PageEntriesEdit edit; @@ -164,7 +164,7 @@ TEST(WALSeriTest, RefExternalAndEntry) { PageEntriesEdit edit; - PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; edit.varEntry(1, ver1_0, entry_p1_2, 2); edit.varDel(1, ver2_0); edit.varRef(2, ver3_0, 1); @@ -405,8 +405,8 @@ try ASSERT_NE(wal, nullptr); // Stage 2. Apply with only puts - PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20(/*seq=*/20); { PageEntriesEdit edit; @@ -435,8 +435,8 @@ try } // Stage 3. Apply with puts and refs - PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver21(/*seq=*/21); { PageEntriesEdit edit; @@ -468,9 +468,9 @@ try // Stage 4. Apply with delete and upsert - PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1); PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1); { @@ -514,8 +514,8 @@ try std::vector size_each_edit; // Stage 1. Apply with only puts - PageEntryV3 entry_p1{.file_id = 1, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p2{.file_id = 1, .size = 2, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1{.file_id = 1, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p2{.file_id = 1, .size = 2, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20(/*seq=*/20); { PageEntriesEdit edit; @@ -526,8 +526,8 @@ try } // Stage 2. Apply with puts and refs - PageEntryV3 entry_p3{.file_id = 1, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5{.file_id = 1, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3{.file_id = 1, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5{.file_id = 1, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver21(/*seq=*/21); { PageEntriesEdit edit; @@ -540,9 +540,9 @@ try } // Stage 3. Apply with delete and upsert - PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .tag = 0, .offset = 0x123, .checksum = 0x4567}; - PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p1_2{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p3_2{.file_id = 2, .size = 3, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry_p5_2{.file_id = 2, .size = 5, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageVersion ver20_1(/*seq=*/20, /*epoch*/ 1); PageVersion ver21_1(/*seq=*/21, /*epoch*/ 1); { @@ -615,7 +615,7 @@ try PageVersion ver(/*seq*/ 32); for (size_t i = 0; i < num_edits_test; ++i) { - PageEntryV3 entry{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; PageEntriesEdit edit; const size_t num_pages_put = d_20(rd); for (size_t p = 0; p < num_pages_put; ++p) @@ -660,7 +660,7 @@ try .persisted_log_files = persisted_log_files}; PageEntriesEdit snap_edit; - PageEntryV3 entry{.file_id = 2, .size = 1, .tag = 0, .offset = 0x123, .checksum = 0x4567}; + PageEntryV3 entry{.file_id = 2, .size = 1, .padded_size = 0, .tag = 0, .offset = 0x123, .checksum = 0x4567}; std::uniform_int_distribution<> d_10000(0, 10000); // just fill in some random entry for (size_t i = 0; i < 70; ++i) diff --git a/dbms/src/Storages/Page/stress/stress_page_storage.cpp b/dbms/src/Storages/Page/stress/stress_page_storage.cpp deleted file mode 100644 index 818be710363..00000000000 --- a/dbms/src/Storages/Page/stress/stress_page_storage.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -namespace DB -{ -// Define is_background_thread for this binary -// It is required for `RateLimiter` but we do not link with `BackgroundProcessingPool`. -#if __APPLE__ && __clang__ -__thread bool is_background_thread = false; -#else -thread_local bool is_background_thread = false; -#endif -} // namespace DB - -int main(int argc, char ** argv) -try -{ - StressEnv::initGlobalLogger(); - auto env = StressEnv::parse(argc, argv); - env.setup(); - - auto & mamager = StressWorkloadManger::getInstance(); - mamager.setEnv(env); - mamager.runWorkload(); - - return StressEnvStatus::getInstance().isSuccess(); -} -catch (...) -{ - DB::tryLogCurrentException(""); - exit(-1); -} diff --git a/dbms/src/Storages/DeltaMerge/tools/CMakeLists.txt b/dbms/src/Storages/Page/tools/CMakeLists.txt similarity index 93% rename from dbms/src/Storages/DeltaMerge/tools/CMakeLists.txt rename to dbms/src/Storages/Page/tools/CMakeLists.txt index 36270a0c8e4..629dedd01fc 100644 --- a/dbms/src/Storages/DeltaMerge/tools/CMakeLists.txt +++ b/dbms/src/Storages/Page/tools/CMakeLists.txt @@ -14,4 +14,4 @@ include_directories (${CMAKE_CURRENT_BINARY_DIR}) -add_subdirectory (workload EXCLUDE_FROM_ALL) +add_subdirectory (PageCtl EXCLUDE_FROM_ALL) diff --git a/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt b/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt new file mode 100644 index 00000000000..576b5e07a0f --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include_directories (${CMAKE_CURRENT_BINARY_DIR}) + +add_library(page-ctl-lib MainEntry.cpp PageStorageCtlV3.cpp PageStorageCtlV2.cpp ${page-ctl-src}) +target_include_directories(page-ctl-lib PUBLIC ${TiFlash_SOURCE_DIR}/libs/libdaemon/include) +target_link_libraries(page-ctl-lib dbms daemon tiflash-dttool-entry-object clickhouse-server-lib) +target_compile_options(page-ctl-lib PRIVATE -Wno-format) + +add_executable(page-ctl Main.cpp) +target_link_libraries(page-ctl page-ctl-lib dbms clickhouse_functions clickhouse-server-lib) +target_compile_options(page-ctl PRIVATE -Wno-format) diff --git a/dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp b/dbms/src/Storages/Page/tools/PageCtl/Main.cpp similarity index 82% rename from dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp rename to dbms/src/Storages/Page/tools/PageCtl/Main.cpp index 092c8a89a42..ae9901ec864 100644 --- a/dbms/src/Storages/DeltaMerge/tools/workload/Main.cpp +++ b/dbms/src/Storages/Page/tools/PageCtl/Main.cpp @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - -using namespace DB::DM::tests; +#include int main(int argc, char ** argv) { - return DTWorkload::mainEntry(argc, argv); + return DB::PageStorageCtl::mainEntry(argc, argv); } diff --git a/dbms/src/Storages/Page/tools/PageCtl/MainEntry.cpp b/dbms/src/Storages/Page/tools/PageCtl/MainEntry.cpp new file mode 100644 index 00000000000..69b41435c34 --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/MainEntry.cpp @@ -0,0 +1,69 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include +#include +namespace DB +{ +int PageStorageCtl::mainEntry(int argc, char ** argv) +{ + namespace po = boost::program_options; + using po::value; + po::options_description desc("Allowed commands"); + desc.add_options()("help,h", "produce help message") // + ("page_storage_version,V", value(), "PageStorage Version: 2 means PageStorage V2, 3 means PageStorage V3.\n"); + po::variables_map options; + po::parsed_options parsed = po::command_line_parser(argc, argv) + .options(desc) + .allow_unregistered() + .run(); + po::store(parsed, options); + po::notify(options); + if (options.count("page_storage_version") == 0 && options.count("help") == 0) + { + std::cerr << "Invalid arg page_storage_version." << std::endl; + std::cerr << desc << std::endl; + exit(0); + } + if (options.count("page_storage_version") > 0) + { + int ps_version = options["page_storage_version"].as(); + if (ps_version == 3) + { + pageStorageV3CtlEntry(argc - 2, argv + 2); + return 0; + } + else if (ps_version == 2) + { + return pageStorageV2CtlEntry(argc - 2, argv + 2); + } + else + { + std::cerr << "Invalid arg page_storage_version." << std::endl; + std::cerr << desc << std::endl; + exit(0); + } + } + if (options.count("help") > 0) + { + std::cerr << desc << std::endl; + exit(0); + } + return 0; +} +} // namespace DB diff --git a/dbms/src/TableFunctions/TableFunctionCatBoostPool.h b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtl.h similarity index 58% rename from dbms/src/TableFunctions/TableFunctionCatBoostPool.h rename to dbms/src/Storages/Page/tools/PageCtl/PageStorageCtl.h index 0b5b32dfffe..c8f35a7750a 100644 --- a/dbms/src/TableFunctions/TableFunctionCatBoostPool.h +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtl.h @@ -14,22 +14,11 @@ #pragma once -#include - - namespace DB { -/* catboostPool('column_descriptions_file', 'dataset_description_file') - * Create storage from CatBoost dataset. - */ -class TableFunctionCatBoostPool : public ITableFunction +class PageStorageCtl { public: - static constexpr auto name = "catBoostPool"; - std::string getName() const override { return name; } - -private: - StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context) const override; + static int mainEntry(int argc, char ** argv); }; - -} // namespace DB +} // namespace DB \ No newline at end of file diff --git a/dbms/src/Storages/Page/V2/tests/page_storage_ctl.cpp b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.cpp similarity index 93% rename from dbms/src/Storages/Page/V2/tests/page_storage_ctl.cpp rename to dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.cpp index 2fb7e31d70e..f9488c4dfd9 100644 --- a/dbms/src/Storages/Page/V2/tests/page_storage_ctl.cpp +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.cpp @@ -28,34 +28,21 @@ using namespace DB::PS::V2; DB::WriteBatch::SequenceID debugging_recover_stop_sequence = 0; -/* some exported global vars */ -namespace DB -{ -#if __APPLE__ && __clang__ -__thread bool is_background_thread = false; -#else -thread_local bool is_background_thread = false; -#endif -} // namespace DB -/* some exported global vars */ -void Usage(const char * prog) +void Usage() { fprintf(stderr, R"HELP( -Usage: %s +Usage: mode == 1 -> dump all page entries 2 -> dump valid page entries - param: %s 2 [max-recover-sequence] + param: 2 [max-recover-sequence] 3 -> check all page entries and page data checksum 4 -> list capacity of all page files 5 -> list all page files 1000 -> gc files - param: %s 1000 [run-gc-times=1] [min-gc-file-num=10] [min-gc-bytes=134217728] [max-gc-valid-rate=0.35] -)HELP", - prog, - prog, - prog); + param: 1000 [run-gc-times=1] [min-gc-file-num=10] [min-gc-bytes=134217728] [max-gc-valid-rate=0.35] + )HELP"); } void printPageEntry(const DB::PageId pid, const DB::PageEntry & entry) @@ -117,7 +104,7 @@ PageStorage::Config parse_storage_config(int argc, char ** argv, Poco::Logger * return config; } -int main(int argc, char ** argv) +int pageStorageV2CtlEntry(int argc, char ** argv) try { (void)argc; @@ -125,7 +112,7 @@ try if (argc < 3) { - Usage(argv[0]); + Usage(); return 1; } @@ -153,7 +140,7 @@ try LOG_FMT_INFO(logger, "Running with [mode={}]", mode); break; default: - Usage(argv[0]); + Usage(); return 1; } @@ -271,13 +258,13 @@ void dump_all_entries(PageFileSet & page_files, int32_t mode) id_and_caches.emplace_back(std::make_pair(record.page_id, record.entry)); break; case DB::WriteBatch::WriteType::DEL: - printf("DEL\t%lld\n", // + printf("DEL\t%lld\t%llu\t%u\n", // record.page_id, page_file.getFileId(), page_file.getLevel()); break; case DB::WriteBatch::WriteType::REF: - printf("REF\t%lld\t%lld\t\n", // + printf("REF\t%lld\t%lld\t\t%llu\t%u\n", // record.page_id, record.ori_page_id, page_file.getFileId(), diff --git a/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.h b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.h new file mode 100644 index 00000000000..6d573ffaba7 --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV2.h @@ -0,0 +1,17 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +int pageStorageV2CtlEntry(int argc, char ** argv); diff --git a/dbms/src/Storages/Page/V3/tests/page_storage_ctl.cpp b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp similarity index 87% rename from dbms/src/Storages/Page/V3/tests/page_storage_ctl.cpp rename to dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp index 7ea8da6892a..c9b871c67d2 100644 --- a/dbms/src/Storages/Page/V3/tests/page_storage_ctl.cpp +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.cpp @@ -12,19 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include +#include #include -#include -#include #include -#include -#include -#include +#include #include #include #include -#include #include #include @@ -32,6 +27,9 @@ namespace DB::PS::V3 { +extern "C" { +void run_raftstore_proxy_ffi(int argc, const char * const * argv, const DB::EngineStoreServerHelper *); +} struct ControlOptions { enum DisplayType @@ -49,6 +47,8 @@ struct ControlOptions UInt64 query_ns_id = DB::TEST_NAMESPACE_ID; UInt64 check_page_id = UINT64_MAX; bool enable_fo_check = true; + bool is_imitative = true; + String config_file_path; static ControlOptions parse(int argc, char ** argv); }; @@ -65,12 +65,12 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) ("display_mode,D", value()->default_value(1), "Display Mode: 1 is summary information,\n 2 is display all of stored page and version chaim(will be very long),\n 3 is display all blobs(in disk) data distribution. \n 4 is check every data is valid.") // ("enable_fo_check,E", value()->default_value(true), "Also check the evert field offsets. This options only works when `display_mode` is 4.") // ("query_ns_id,N", value()->default_value(DB::TEST_NAMESPACE_ID), "When used `check_page_id`/`query_page_id`/`query_blob_id` to query results. You can specify a namespace id.")("check_page_id,C", value()->default_value(UINT64_MAX), "Check a single Page id, display the exception if meet. And also will check the field offsets.") // - ("query_page_id,W", value()->default_value(UINT64_MAX), "Quert a single Page id, and print its version chaim.") // - ("query_blob_id,B", value()->default_value(UINT32_MAX), "Quert a single Blob id, and print its data distribution."); + ("query_page_id,W", value()->default_value(UINT64_MAX), "Query a single Page id, and print its version chaim.") // + ("query_blob_id,B", value()->default_value(UINT32_MAX), "Query a single Blob id, and print its data distribution.")("imitative,I", value()->default_value(true), "Use imitative context instead. (encryption is not supported in this mode so that no need to set config_file_path)")("config_file_path", value(), "Path to TiFlash config (tiflash.toml)."); static_assert(sizeof(DB::PageId) == sizeof(UInt64)); - static_assert(sizeof(DB::BlobFileId) == sizeof(UInt32)); + static_assert(sizeof(DB::BlobFileId) == sizeof(UInt64)); po::variables_map options; po::store(po::parse_command_line(argc, argv, desc), options); @@ -97,6 +97,21 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) opt.enable_fo_check = options["enable_fo_check"].as(); opt.check_page_id = options["check_page_id"].as(); opt.query_ns_id = options["query_ns_id"].as(); + opt.is_imitative = options["imitative"].as(); + if (opt.is_imitative && options.count("config_file_path") != 0) + { + std::cerr << "config_file_path is not allowed in imitative mode" << std::endl; + exit(0); + } + else if (!opt.is_imitative && options.count("config_file_path") == 0) + { + std::cerr << "config_file_path is required in proxy mode" << std::endl; + exit(0); + } + if (options.count("config_file_path") != 0) + { + opt.config_file_path = options["config_file_path"].as(); + } if (opt.display_mode < DisplayType::DISPLAY_SUMMARY_INFO || opt.display_mode > DisplayType::CHECK_ALL_DATA_CRC) { @@ -108,15 +123,39 @@ ControlOptions ControlOptions::parse(int argc, char ** argv) return opt; } -class PageStorageControl +class PageStorageControlV3 { public: - explicit PageStorageControl(const ControlOptions & options_) + explicit PageStorageControlV3(const ControlOptions & options_) : options(options_) { } void run() + { + try + { + if (options.is_imitative) + { + Context context = Context::createGlobal(); + getPageStorageV3Info(context, options); + } + else + { + PageDirectory::MVCCMapType type; + CLIService service(getPageStorageV3Info, options, options.config_file_path, run_raftstore_proxy_ffi); + service.run({""}); + } + } + catch (...) + { + DB::tryLogCurrentException("exception thrown"); + std::abort(); // Finish testing if some error happened. + } + } + +private: + static int getPageStorageV3Info(Context & context, const ControlOptions & options) { DB::PSDiskDelegatorPtr delegator; if (options.paths.size() == 1) @@ -128,13 +167,20 @@ class PageStorageControl delegator = std::make_shared(options.paths); } - auto key_manager = std::make_shared(false); - auto file_provider = std::make_shared(key_manager, false); - + FileProviderPtr file_provider_ptr; + if (options.is_imitative) + { + auto key_manager = std::make_shared(false); + file_provider_ptr = std::make_shared(key_manager, false); + } + else + { + file_provider_ptr = context.getFileProvider(); + } BlobStore::Config blob_config; PageStorage::Config config; - PageStorageImpl ps_v3("PageStorageControl", delegator, config, file_provider); + PageStorageImpl ps_v3("PageStorageControlV3", delegator, config, file_provider_ptr); ps_v3.restore(); PageDirectory::MVCCMapType & mvcc_table_directory = ps_v3.page_directory->mvcc_table_directory; @@ -171,9 +217,9 @@ class PageStorageControl std::cout << "Invalid display mode." << std::endl; break; } + return 0; } -private: static String getBlobsInfo(BlobStore & blob_store, UInt32 blob_id) { auto stat_info = [](const BlobStore::BlobStats::BlobStatPtr & stat, const String & path) { @@ -469,9 +515,9 @@ class PageStorageControl } // namespace DB::PS::V3 using namespace DB::PS::V3; -int main(int argc, char ** argv) + +void pageStorageV3CtlEntry(int argc, char ** argv) { const auto & options = ControlOptions::parse(argc, argv); - PageStorageControl(options).run(); - return 0; + PageStorageControlV3(options).run(); } diff --git a/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.h b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.h new file mode 100644 index 00000000000..21a929ee599 --- /dev/null +++ b/dbms/src/Storages/Page/tools/PageCtl/PageStorageCtlV3.h @@ -0,0 +1,17 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +void pageStorageV3CtlEntry(int argc, char ** argv); diff --git a/dbms/src/Storages/Page/workload/CMakeLists.txt b/dbms/src/Storages/Page/workload/CMakeLists.txt new file mode 100644 index 00000000000..5c8ecb34d97 --- /dev/null +++ b/dbms/src/Storages/Page/workload/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2022 PingCAP, Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include_directories (${CMAKE_CURRENT_BINARY_DIR}) + +set (page-workload-src HeavyMemoryCostInGC.cpp HeavyRead.cpp HeavySkewWriteRead.cpp HeavyWrite.cpp HighValidBigFileGC.cpp HoldSnapshotsLongTime.cpp Normal.cpp + PageStorageInMemoryCapacity.cpp ThousandsOfOffset.cpp MainEntry.cpp Normal.cpp PageStorageInMemoryCapacity.cpp PSBackground.cpp PSRunnable.cpp PSStressEnv.cpp PSWorkload.cpp) + +add_library (page-workload-lib ${page-workload-src}) +target_link_libraries (page-workload-lib dbms clickhouse_functions clickhouse-server-lib) +target_compile_options (page-workload-lib PRIVATE -Wno-format -lc++) \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp b/dbms/src/Storages/Page/workload/HeavyMemoryCostInGC.cpp similarity index 96% rename from dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp rename to dbms/src/Storages/Page/workload/HeavyMemoryCostInGC.cpp index 40595f0cb59..7e745e29fc2 100644 --- a/dbms/src/Storages/Page/stress/workload/HeavyMemoryCostInGC.cpp +++ b/dbms/src/Storages/Page/workload/HeavyMemoryCostInGC.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HeavyMemoryCostInGC : public StressWorkload , public StressWorkloadFunc @@ -81,3 +83,4 @@ class HeavyMemoryCostInGC }; REGISTER_WORKLOAD(HeavyMemoryCostInGC) +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/workload/HeavyRead.cpp b/dbms/src/Storages/Page/workload/HeavyRead.cpp similarity index 94% rename from dbms/src/Storages/Page/stress/workload/HeavyRead.cpp rename to dbms/src/Storages/Page/workload/HeavyRead.cpp index 15aeb1320cf..a67c435e84c 100644 --- a/dbms/src/Storages/Page/stress/workload/HeavyRead.cpp +++ b/dbms/src/Storages/Page/workload/HeavyRead.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HeavyRead : public StressWorkload , public StressWorkloadFunc { @@ -69,4 +71,5 @@ class HeavyRead : public StressWorkload } }; -REGISTER_WORKLOAD(HeavyRead) \ No newline at end of file +REGISTER_WORKLOAD(HeavyRead) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp b/dbms/src/Storages/Page/workload/HeavySkewWriteRead.cpp similarity index 95% rename from dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp rename to dbms/src/Storages/Page/workload/HeavySkewWriteRead.cpp index 78ffa5b60e0..805bf105358 100644 --- a/dbms/src/Storages/Page/stress/workload/HeavySkewWriteRead.cpp +++ b/dbms/src/Storages/Page/workload/HeavySkewWriteRead.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HeavySkewWriteRead : public StressWorkload , public StressWorkloadFunc { @@ -84,4 +86,5 @@ class HeavySkewWriteRead : public StressWorkload } }; -REGISTER_WORKLOAD(HeavySkewWriteRead) \ No newline at end of file +REGISTER_WORKLOAD(HeavySkewWriteRead) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp b/dbms/src/Storages/Page/workload/HeavyWrite.cpp similarity index 94% rename from dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp rename to dbms/src/Storages/Page/workload/HeavyWrite.cpp index 265b289db56..8dfd7f810f7 100644 --- a/dbms/src/Storages/Page/stress/workload/HeavyWrite.cpp +++ b/dbms/src/Storages/Page/workload/HeavyWrite.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HeavyWrite : public StressWorkload , public StressWorkloadFunc { @@ -71,4 +73,5 @@ class HeavyWrite : public StressWorkload } }; -REGISTER_WORKLOAD(HeavyWrite) \ No newline at end of file +REGISTER_WORKLOAD(HeavyWrite) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp b/dbms/src/Storages/Page/workload/HighValidBigFileGC.cpp similarity index 97% rename from dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp rename to dbms/src/Storages/Page/workload/HighValidBigFileGC.cpp index 866782c9578..a9af6aebb76 100644 --- a/dbms/src/Storages/Page/stress/workload/HighValidBigFileGC.cpp +++ b/dbms/src/Storages/Page/workload/HighValidBigFileGC.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HighValidBigFileGCWorkload : public StressWorkload , public StressWorkloadFunc @@ -129,3 +131,4 @@ class HighValidBigFileGCWorkload }; REGISTER_WORKLOAD(HighValidBigFileGCWorkload) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp b/dbms/src/Storages/Page/workload/HoldSnapshotsLongTime.cpp similarity index 95% rename from dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp rename to dbms/src/Storages/Page/workload/HoldSnapshotsLongTime.cpp index b49347fc858..f02fbf65bcd 100644 --- a/dbms/src/Storages/Page/stress/workload/HoldSnapshotsLongTime.cpp +++ b/dbms/src/Storages/Page/workload/HoldSnapshotsLongTime.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class HoldSnapshotsLongTime : public StressWorkload , public StressWorkloadFunc { @@ -93,4 +95,5 @@ class HoldSnapshotsLongTime : public StressWorkload } }; -REGISTER_WORKLOAD(HoldSnapshotsLongTime) \ No newline at end of file +REGISTER_WORKLOAD(HoldSnapshotsLongTime) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/workload/MainEntry.cpp b/dbms/src/Storages/Page/workload/MainEntry.cpp new file mode 100644 index 00000000000..ac82e1ea4bc --- /dev/null +++ b/dbms/src/Storages/Page/workload/MainEntry.cpp @@ -0,0 +1,70 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include + +using namespace DB::PS::tests; + +int StressWorkload::mainEntry(int argc, char ** argv) +{ + { + // maybe due to sequence of linking, REGISTER_WORKLOAD is not visible to main function in dbms/src/Server/main.cpp + // cause that REGISTER_WORKLOAD will not be triggered before mainEntry + // we do this to trigger REGISTER_WORKLOAD explicitly. + void _work_load_register_named_HeavyMemoryCostInGC(); + void (*f)() = _work_load_register_named_HeavyMemoryCostInGC; + (void)f; + void _work_load_register_named_HeavyRead(); + f = _work_load_register_named_HeavyRead; + (void)f; + void _work_load_register_named_HeavySkewWriteRead(); + f = _work_load_register_named_HeavySkewWriteRead; + (void)f; + void _work_load_register_named_HeavyWrite(); + f = _work_load_register_named_HeavyWrite; + (void)f; + void _work_load_register_named_HighValidBigFileGCWorkload(); + f = _work_load_register_named_HighValidBigFileGCWorkload; + (void)f; + void _work_load_register_named_HoldSnapshotsLongTime(); + f = _work_load_register_named_HoldSnapshotsLongTime; + (void)f; + void _work_load_register_named_PageStorageInMemoryCapacity(); + f = _work_load_register_named_PageStorageInMemoryCapacity; + (void)f; + void _work_load_register_named_NormalWorkload(); + f = _work_load_register_named_NormalWorkload; + (void)f; + void _work_load_register_named_ThousandsOfOffset(); + f = _work_load_register_named_ThousandsOfOffset; + (void)f; + } + try + { + StressEnv::initGlobalLogger(); + auto env = StressEnv::parse(argc, argv); + env.setup(); + + auto & mamager = StressWorkloadManger::getInstance(); + mamager.setEnv(env); + mamager.runWorkload(); + + return StressEnvStatus::getInstance().isSuccess(); + } + catch (...) + { + DB::tryLogCurrentException(""); + exit(-1); + } +} \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/Normal.cpp b/dbms/src/Storages/Page/workload/Normal.cpp similarity index 95% rename from dbms/src/Storages/Page/stress/workload/Normal.cpp rename to dbms/src/Storages/Page/workload/Normal.cpp index 0323b857613..57229395809 100644 --- a/dbms/src/Storages/Page/stress/workload/Normal.cpp +++ b/dbms/src/Storages/Page/workload/Normal.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class NormalWorkload : public StressWorkload , public StressWorkloadFunc @@ -77,3 +79,4 @@ class NormalWorkload }; REGISTER_WORKLOAD(NormalWorkload) +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSBackground.cpp b/dbms/src/Storages/Page/workload/PSBackground.cpp similarity index 96% rename from dbms/src/Storages/Page/stress/PSBackground.cpp rename to dbms/src/Storages/Page/workload/PSBackground.cpp index af7329e8348..247bea23dcc 100644 --- a/dbms/src/Storages/Page/stress/PSBackground.cpp +++ b/dbms/src/Storages/Page/workload/PSBackground.cpp @@ -13,11 +13,14 @@ // limitations under the License. #include -#include #include #include +#include #include + +namespace DB::PS::tests +{ void PSMetricsDumper::onTime(Poco::Timer & /*timer*/) { for (auto & metric : metrics) @@ -107,3 +110,4 @@ void StressTimeout::start() { timeout_timer.start(Poco::TimerCallback(*this, &StressTimeout::onTime)); } +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSBackground.h b/dbms/src/Storages/Page/workload/PSBackground.h similarity index 97% rename from dbms/src/Storages/Page/stress/PSBackground.h rename to dbms/src/Storages/Page/workload/PSBackground.h index 8c22458c5e8..c91dad1361f 100644 --- a/dbms/src/Storages/Page/stress/PSBackground.h +++ b/dbms/src/Storages/Page/workload/PSBackground.h @@ -15,14 +15,16 @@ #pragma once #include #include -#include #include +#include namespace CurrentMetrics { extern const Metric PSMVCCSnapshotsList; } +namespace DB::PS::tests +{ class PSMetricsDumper { public: @@ -162,3 +164,4 @@ class StressTimeout Poco::Timer timeout_timer; }; using StressTimeoutPtr = std::shared_ptr; +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSRunnable.cpp b/dbms/src/Storages/Page/workload/PSRunnable.cpp similarity index 97% rename from dbms/src/Storages/Page/stress/PSRunnable.cpp rename to dbms/src/Storages/Page/workload/PSRunnable.cpp index 5d6c8ecc5c6..5e9774ccc99 100644 --- a/dbms/src/Storages/Page/stress/PSRunnable.cpp +++ b/dbms/src/Storages/Page/workload/PSRunnable.cpp @@ -16,14 +16,16 @@ #include #include #include -#include #include #include +#include #include #include #include +namespace DB::PS::tests +{ void PSRunnable::run() try { @@ -69,7 +71,7 @@ DB::ReadBufferPtr PSWriter::genRandomData(const DB::PageId pageId, DB::MemHolder std::uniform_int_distribution<> dist(0, 3000); const size_t buff_sz = approx_page_mb * DB::MB + dist(size_gen); - char * buff = static_cast(malloc(buff_sz)); + char * buff = static_cast(malloc(buff_sz)); // NOLINT if (buff == nullptr) { throw DB::Exception("Alloc fix memory failed.", DB::ErrorCodes::LOGICAL_ERROR); @@ -78,7 +80,7 @@ DB::ReadBufferPtr PSWriter::genRandomData(const DB::PageId pageId, DB::MemHolder const char buff_ch = pageId % 0xFF; memset(buff, buff_ch, buff_sz); - holder = DB::createMemHolder(buff, [&](char * p) { free(p); }); + holder = DB::createMemHolder(buff, [&](char * p) { free(p); }); // NOLINT return std::make_shared(const_cast(buff), buff_sz); } @@ -88,7 +90,7 @@ void PSWriter::updatedRandomData() size_t memory_size = approx_page_mb * DB::MB * 2; if (memory == nullptr) { - memory = static_cast(malloc(memory_size)); + memory = static_cast(malloc(memory_size)); // NOLINT if (memory == nullptr) { throw DB::Exception("Alloc fix memory failed.", DB::ErrorCodes::LOGICAL_ERROR); @@ -147,7 +149,7 @@ void PSCommonWriter::updatedRandomData() if (memory == nullptr) { - memory = static_cast(malloc(memory_size)); + memory = static_cast(malloc(memory_size)); // NOLINT if (memory == nullptr) { throw DB::Exception("Alloc fix memory failed.", DB::ErrorCodes::LOGICAL_ERROR); @@ -415,3 +417,4 @@ DB::PageId PSIncreaseWriter::genRandomPageId() { return static_cast(begin_page_id++); } +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSRunnable.h b/dbms/src/Storages/Page/workload/PSRunnable.h similarity index 90% rename from dbms/src/Storages/Page/stress/PSRunnable.h rename to dbms/src/Storages/Page/workload/PSRunnable.h index 3ddcd73c093..b723236391d 100644 --- a/dbms/src/Storages/Page/stress/PSRunnable.h +++ b/dbms/src/Storages/Page/workload/PSRunnable.h @@ -13,12 +13,14 @@ // limitations under the License. #pragma once -#include #include #include +#include const DB::PageId MAX_PAGE_ID_DEFAULT = 1000; +namespace DB::PS::tests +{ class PSRunnable : public Poco::Runnable { public: @@ -46,7 +48,7 @@ class PSWriter : public PSRunnable gen.seed(time(nullptr)); } - virtual ~PSWriter() + ~PSWriter() override { if (memory != nullptr) { @@ -54,7 +56,7 @@ class PSWriter : public PSRunnable } } - virtual String description() override + String description() override { return fmt::format("(Stress Test Writer {})", index); } @@ -67,7 +69,7 @@ class PSWriter : public PSRunnable static void fillAllPages(const PSPtr & ps); - virtual bool runImpl() override; + bool runImpl() override; protected: virtual DB::PageId genRandomPageId(); @@ -91,11 +93,11 @@ class PSCommonWriter : public PSWriter : PSWriter(ps_, index_) {} - virtual void updatedRandomData() override; + void updatedRandomData() override; - virtual String description() override { return fmt::format("(Stress Test Common Writer {})", index); } + String description() override { return fmt::format("(Stress Test Common Writer {})", index); } - virtual bool runImpl() override; + bool runImpl() override; void setBatchBufferNums(size_t numbers); @@ -120,7 +122,7 @@ class PSCommonWriter : public PSWriter DB::PageFieldSizes data_sizes = {}; - virtual DB::PageId genRandomPageId() override; + DB::PageId genRandomPageId() override; virtual size_t genBufferSize(); }; @@ -154,7 +156,7 @@ class PSWindowWriter : public PSCommonWriter void setNormalDistributionSigma(size_t sigma); protected: - virtual DB::PageId genRandomPageId() override; + DB::PageId genRandomPageId() override; protected: size_t window_size = 100; @@ -170,12 +172,12 @@ class PSIncreaseWriter : public PSCommonWriter String description() override { return fmt::format("(Stress Test Increase Writer {})", index); } - virtual bool runImpl() override; + bool runImpl() override; void setPageRange(size_t page_range); protected: - virtual DB::PageId genRandomPageId() override; + DB::PageId genRandomPageId() override; protected: size_t begin_page_id = 1; @@ -192,9 +194,9 @@ class PSReader : public PSRunnable gen.seed(time(nullptr)); } - virtual String description() override { return fmt::format("(Stress Test PSReader {})", index); } + String description() override { return fmt::format("(Stress Test PSReader {})", index); } - virtual bool runImpl() override; + bool runImpl() override; void setPageReadOnce(size_t page_read_once); @@ -242,7 +244,7 @@ class PSWindowReader : public PSReader void setWriterNums(size_t writer_nums); protected: - virtual DB::PageIds genRandomPageIds() override; + DB::PageIds genRandomPageIds() override; protected: size_t window_size = 100; @@ -261,12 +263,13 @@ class PSSnapshotReader : public PSReader : PSReader(ps_, index_) {} - virtual bool runImpl() override; + bool runImpl() override; void setSnapshotGetIntervalMs(size_t snapshot_get_interval_ms_); protected: - size_t snapshots_hold_num; + size_t snapshots_hold_num = 0; size_t snapshot_get_interval_ms = 0; std::list snapshots; -}; \ No newline at end of file +}; +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSStressEnv.cpp b/dbms/src/Storages/Page/workload/PSStressEnv.cpp similarity index 97% rename from dbms/src/Storages/Page/stress/PSStressEnv.cpp rename to dbms/src/Storages/Page/workload/PSStressEnv.cpp index 7d680cd43c0..f5cead0a158 100644 --- a/dbms/src/Storages/Page/stress/PSStressEnv.cpp +++ b/dbms/src/Storages/Page/workload/PSStressEnv.cpp @@ -16,18 +16,20 @@ #include #include #include -#include -#include #include #include #include #include #include #include +#include +#include #include #include +namespace DB::PS::tests +{ Poco::Logger * StressEnv::logger; void StressEnv::initGlobalLogger() { @@ -146,3 +148,4 @@ void StressEnv::setup() init_pages = true; setupSignal(); } +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSStressEnv.h b/dbms/src/Storages/Page/workload/PSStressEnv.h similarity index 98% rename from dbms/src/Storages/Page/stress/PSStressEnv.h rename to dbms/src/Storages/Page/workload/PSStressEnv.h index 1c7d8ee761f..e67cb325430 100644 --- a/dbms/src/Storages/Page/stress/PSStressEnv.h +++ b/dbms/src/Storages/Page/workload/PSStressEnv.h @@ -25,6 +25,8 @@ namespace Poco class Logger; } +namespace DB::PS::tests +{ using PSPtr = std::shared_ptr; enum StressEnvStat @@ -124,3 +126,4 @@ struct StressEnv void setup(); }; +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSWorkload.cpp b/dbms/src/Storages/Page/workload/PSWorkload.cpp similarity index 98% rename from dbms/src/Storages/Page/stress/PSWorkload.cpp rename to dbms/src/Storages/Page/workload/PSWorkload.cpp index ce1f8d92ce0..81f13527f48 100644 --- a/dbms/src/Storages/Page/stress/PSWorkload.cpp +++ b/dbms/src/Storages/Page/workload/PSWorkload.cpp @@ -14,12 +14,14 @@ #include #include -#include #include #include #include +#include #include +namespace DB::PS::tests +{ void StressWorkload::onDumpResult() { UInt64 time_interval = stop_watch.elapsedMilliseconds(); @@ -177,3 +179,4 @@ void StressWorkloadManger::runWorkload() } } } +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/PSWorkload.h b/dbms/src/Storages/Page/workload/PSWorkload.h similarity index 92% rename from dbms/src/Storages/Page/stress/PSWorkload.h rename to dbms/src/Storages/Page/workload/PSWorkload.h index cb099b4203a..eaaaf4eba5b 100644 --- a/dbms/src/Storages/Page/stress/PSWorkload.h +++ b/dbms/src/Storages/Page/workload/PSWorkload.h @@ -16,15 +16,17 @@ #include #include -#include -#include -#include #include #include #include +#include +#include +#include #include #define NORMAL_WORKLOAD 0 +namespace DB::PS::tests +{ template class StressWorkloadFunc { @@ -45,6 +47,8 @@ class StressWorkloadFunc class StressWorkload { public: + static int mainEntry(int argc, char ** argv); + explicit StressWorkload(StressEnv options_) : options(options_) {} @@ -189,13 +193,15 @@ class StressWorkloadManger StressEnv options; }; -#define REGISTER_WORKLOAD(WORKLOAD) \ - static void __attribute__((constructor)) _work_load_register_named_##WORKLOAD(void) \ - { \ - StressWorkloadManger::getInstance().reg( \ - WORKLOAD::nameFunc(), \ - WORKLOAD::maskFunc(), \ - [](const StressEnv & opts) -> std::shared_ptr { \ - return std::make_shared(opts); \ - }); \ +#define REGISTER_WORKLOAD(WORKLOAD) \ + void __attribute__((constructor)) _work_load_register_named_##WORKLOAD(void) \ + { \ + StressWorkloadManger::getInstance().reg( \ + WORKLOAD::nameFunc(), \ + WORKLOAD::maskFunc(), \ + [](const StressEnv & opts) -> std::shared_ptr { \ + return std::make_shared(opts); \ + }); \ } + +} // namespace DB::PS::tests diff --git a/dbms/src/Storages/Page/stress/workload/PageStorageInMemoryCapacity.cpp b/dbms/src/Storages/Page/workload/PageStorageInMemoryCapacity.cpp similarity index 96% rename from dbms/src/Storages/Page/stress/workload/PageStorageInMemoryCapacity.cpp rename to dbms/src/Storages/Page/workload/PageStorageInMemoryCapacity.cpp index 190cbf6b323..6ab321d1a10 100644 --- a/dbms/src/Storages/Page/stress/workload/PageStorageInMemoryCapacity.cpp +++ b/dbms/src/Storages/Page/workload/PageStorageInMemoryCapacity.cpp @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include #include #include - #ifdef __APPLE__ #include @@ -27,6 +26,8 @@ #include #endif +namespace DB::PS::tests +{ class PageStorageInMemoryCapacity : public StressWorkload , public StressWorkloadFunc { @@ -89,14 +90,14 @@ class PageStorageInMemoryCapacity : public StressWorkload } FILE * file = fopen("/proc/meminfo", "r"); - if (file != NULL) + if (file != nullptr) { char buffer[128]; #define MEMORY_TOTAL_LABEL "MemTotal:" while (fgets(buffer, 128, file)) { if ((strncmp((buffer), (MEMORY_TOTAL_LABEL), strlen(MEMORY_TOTAL_LABEL)) == 0) - && sscanf(buffer + strlen(MEMORY_TOTAL_LABEL), " %32llu kB", &total_mem)) + && sscanf(buffer + strlen(MEMORY_TOTAL_LABEL), " %32llu kB", &total_mem)) // NOLINT { break; } @@ -174,4 +175,5 @@ class PageStorageInMemoryCapacity : public StressWorkload } }; -REGISTER_WORKLOAD(PageStorageInMemoryCapacity) \ No newline at end of file +REGISTER_WORKLOAD(PageStorageInMemoryCapacity) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/Page/stress/workload/ThousandsOfOffset.cpp b/dbms/src/Storages/Page/workload/ThousandsOfOffset.cpp similarity index 97% rename from dbms/src/Storages/Page/stress/workload/ThousandsOfOffset.cpp rename to dbms/src/Storages/Page/workload/ThousandsOfOffset.cpp index 3a215f76769..5a02ef48d68 100644 --- a/dbms/src/Storages/Page/stress/workload/ThousandsOfOffset.cpp +++ b/dbms/src/Storages/Page/workload/ThousandsOfOffset.cpp @@ -12,8 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +namespace DB::PS::tests +{ class ThousandsOfOffset : public StressWorkload , public StressWorkloadFunc { @@ -168,4 +170,5 @@ class ThousandsOfOffset : public StressWorkload } }; -REGISTER_WORKLOAD(ThousandsOfOffset) \ No newline at end of file +REGISTER_WORKLOAD(ThousandsOfOffset) +} // namespace DB::PS::tests \ No newline at end of file diff --git a/dbms/src/Storages/StorageCatBoostPool.cpp b/dbms/src/Storages/StorageCatBoostPool.cpp deleted file mode 100644 index 317cac21d52..00000000000 --- a/dbms/src/Storages/StorageCatBoostPool.cpp +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int CANNOT_OPEN_FILE; - extern const int CANNOT_PARSE_TEXT; - extern const int DATABASE_ACCESS_DENIED; -} - -namespace -{ -class CatBoostDatasetBlockInputStream : public IProfilingBlockInputStream -{ -public: - - CatBoostDatasetBlockInputStream(const std::string & file_name, const std::string & format_name, - const Block & sample_block, const Context & context, size_t max_block_size) - : file_name(file_name), format_name(format_name) - { - read_buf = std::make_unique(file_name); - reader = FormatFactory().getInput(format_name, *read_buf, sample_block, context, max_block_size); - } - - String getName() const override - { - return "CatBoostDataset"; - } - - Block readImpl() override - { - return reader->read(); - } - - void readPrefixImpl() override - { - reader->readPrefix(); - } - - void readSuffixImpl() override - { - reader->readSuffix(); - } - - Block getHeader() const override { return sample_block; }; - -private: - Block sample_block; - std::unique_ptr read_buf; - BlockInputStreamPtr reader; - std::string file_name; - std::string format_name; -}; - -} - -static boost::filesystem::path canonicalPath(std::string && path) -{ - return boost::filesystem::canonical(boost::filesystem::path(path)); -} - -static std::string resolvePath(const boost::filesystem::path & base_path, std::string && path) -{ - boost::filesystem::path resolved_path(path); - if (!resolved_path.is_absolute()) - return (base_path / resolved_path).string(); - return resolved_path.string(); -} - -static void checkCreationIsAllowed(const String & base_path, const String & path) -{ - if (base_path != path.substr(0, base_path.size())) - throw Exception( - "Using file descriptor or user specified path as source of storage isn't allowed for server daemons", - ErrorCodes::DATABASE_ACCESS_DENIED); -} - - -StorageCatBoostPool::StorageCatBoostPool(const Context & context, - String column_description_file_name_, - String data_description_file_name_) - : column_description_file_name(std::move(column_description_file_name_)), - data_description_file_name(std::move(data_description_file_name_)) -{ - auto base_path = canonicalPath(context.getPath()); - column_description_file_name = resolvePath(base_path, std::move(column_description_file_name)); - data_description_file_name = resolvePath(base_path, std::move(data_description_file_name)); - if (context.getApplicationType() == Context::ApplicationType::SERVER) - { - const auto & base_path_str = base_path.string(); - checkCreationIsAllowed(base_path_str, column_description_file_name); - checkCreationIsAllowed(base_path_str, data_description_file_name); - } - - parseColumnDescription(); - createSampleBlockAndColumns(); -} - -std::string StorageCatBoostPool::getColumnTypesString(const ColumnTypesMap & columnTypesMap) -{ - std::string types_string; - bool first = true; - for (const auto & value : columnTypesMap) - { - if (!first) - types_string.append(", "); - - first = false; - types_string += value.first; - } - - return types_string; -} - -void StorageCatBoostPool::checkDatasetDescription() -{ - std::ifstream in(data_description_file_name); - if (!in.good()) - throw Exception("Cannot open file: " + data_description_file_name, ErrorCodes::CANNOT_OPEN_FILE); - - std::string line; - if (!std::getline(in, line)) - throw Exception("File is empty: " + data_description_file_name, ErrorCodes::CANNOT_PARSE_TEXT); - - size_t columns_count = 1; - for (char sym : line) - if (sym == '\t') - ++columns_count; - - columns_description.resize(columns_count); -} - -void StorageCatBoostPool::parseColumnDescription() -{ - /// NOTE: simple parsing - /// TODO: use ReadBufferFromFile - - checkDatasetDescription(); - - std::ifstream in(column_description_file_name); - if (!in.good()) - throw Exception("Cannot open file: " + column_description_file_name, ErrorCodes::CANNOT_OPEN_FILE); - - std::string line; - size_t line_num = 0; - auto column_types_map = getColumnTypesMap(); - auto column_types_string = getColumnTypesString(column_types_map); - - /// Enumerate default names for columns as Auxiliary, Auxiliary1, Auxiliary2, ... - std::map columns_per_type_count; - - while (std::getline(in, line)) - { - ++line_num; - std::string str_line_num = std::to_string(line_num); - - if (line.empty()) - continue; - - std::istringstream iss(line); - std::vector tokens; - std::string token; - while (std::getline(iss, token, '\t')) - tokens.push_back(token); - - if (tokens.size() != 2 && tokens.size() != 3) - throw Exception("Cannot parse column description at line " + str_line_num + " '" + line + "' " - + ": expected 2 or 3 columns, got " + std::to_string(tokens.size()), - ErrorCodes::CANNOT_PARSE_TEXT); - - std::string str_id = tokens[0]; - std::string col_type = tokens[1]; - std::string col_alias = tokens.size() > 2 ? tokens[2] : ""; - - size_t num_id; - try - { - num_id = std::stoull(str_id); - } - catch (std::exception & e) - { - throw Exception("Cannot parse column index at row " + str_line_num + ": " + e.what(), - ErrorCodes::CANNOT_PARSE_TEXT); - } - - if (num_id >= columns_description.size()) - throw Exception("Invalid index at row " + str_line_num + ": " + str_id - + ", expected in range [0, " + std::to_string(columns_description.size()) + ")", - ErrorCodes::CANNOT_PARSE_TEXT); - - if (column_types_map.count(col_type) == 0) - throw Exception("Invalid column type: " + col_type + ", expected: " + column_types_string, - ErrorCodes::CANNOT_PARSE_TEXT); - - auto type = column_types_map[col_type]; - - std::string col_name; - - bool is_feature_column = type == DatasetColumnType::Num || type == DatasetColumnType::Categ; - auto & col_number = columns_per_type_count[type]; - /// If column is not feature skip '0' after the name (to use 'Target' instead of 'Target0'). - col_name = col_type + (is_feature_column || col_number ? std::to_string(col_number) : ""); - ++col_number; - - columns_description[num_id] = ColumnDescription(col_name, col_alias, type); - } -} - -void StorageCatBoostPool::createSampleBlockAndColumns() -{ - ColumnsDescription columns; - NamesAndTypesList cat_columns; - NamesAndTypesList num_columns; - sample_block.clear(); - for (auto & desc : columns_description) - { - DataTypePtr type; - if (desc.column_type == DatasetColumnType::Categ - || desc.column_type == DatasetColumnType::Auxiliary - || desc.column_type == DatasetColumnType::DocId) - type = std::make_shared(); - else - type = std::make_shared(); - - if (desc.column_type == DatasetColumnType::Categ) - cat_columns.emplace_back(desc.column_name, type); - else if (desc.column_type == DatasetColumnType::Num) - num_columns.emplace_back(desc.column_name, type); - else - columns.materialized.emplace_back(desc.column_name, type); - - if (!desc.alias.empty()) - { - auto alias = std::make_shared(desc.column_name); - columns.defaults[desc.alias] = {ColumnDefaultKind::Alias, alias}; - columns.aliases.emplace_back(desc.alias, type); - } - - sample_block.insert(ColumnWithTypeAndName(type, desc.column_name)); - } - columns.ordinary.insert(columns.ordinary.end(), num_columns.begin(), num_columns.end()); - columns.ordinary.insert(columns.ordinary.end(), cat_columns.begin(), cat_columns.end()); - - setColumns(columns); -} - -BlockInputStreams StorageCatBoostPool::read(const Names & column_names, - const SelectQueryInfo & /*query_info*/, - const Context & context, - QueryProcessingStage::Enum & /*processed_stage*/, - size_t max_block_size, - unsigned /*threads*/) -{ - auto stream = std::make_shared( - data_description_file_name, "TSV", sample_block, context, max_block_size); - - auto filter_stream = std::make_shared(stream, column_names, false); - return { filter_stream }; -} - -} diff --git a/dbms/src/Storages/StorageCatBoostPool.h b/dbms/src/Storages/StorageCatBoostPool.h deleted file mode 100644 index 0f4f7c2cede..00000000000 --- a/dbms/src/Storages/StorageCatBoostPool.h +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include - -namespace DB -{ -class StorageCatBoostPool : public ext::SharedPtrHelper - , public IStorage -{ -public: - std::string getName() const override { return "CatBoostPool"; } - - std::string getTableName() const override { return table_name; } - - BlockInputStreams read(const Names & column_names, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum & processed_stage, - size_t max_block_size, - unsigned threads) override; - -private: - String table_name; - - String column_description_file_name; - String data_description_file_name; - Block sample_block; - - enum class DatasetColumnType - { - Target, - Num, - Categ, - Auxiliary, - DocId, - Weight, - Baseline - }; - - using ColumnTypesMap = std::map; - - ColumnTypesMap getColumnTypesMap() const - { - return { - {"Target", DatasetColumnType::Target}, - {"Num", DatasetColumnType::Num}, - {"Categ", DatasetColumnType::Categ}, - {"Auxiliary", DatasetColumnType::Auxiliary}, - {"DocId", DatasetColumnType::DocId}, - {"Weight", DatasetColumnType::Weight}, - {"Baseline", DatasetColumnType::Baseline}, - }; - }; - - std::string getColumnTypesString(const ColumnTypesMap & columnTypesMap); - - struct ColumnDescription - { - std::string column_name; - std::string alias; - DatasetColumnType column_type; - - ColumnDescription() - : column_type(DatasetColumnType::Num) - {} - ColumnDescription(std::string column_name, std::string alias, DatasetColumnType column_type) - : column_name(std::move(column_name)) - , alias(std::move(alias)) - , column_type(column_type) - {} - }; - - std::vector columns_description; - - void checkDatasetDescription(); - void parseColumnDescription(); - void createSampleBlockAndColumns(); - -protected: - StorageCatBoostPool(const Context & context, String column_description_file_name, String data_description_file_name); -}; - -} // namespace DB diff --git a/dbms/src/Storages/StorageDeltaMerge.cpp b/dbms/src/Storages/StorageDeltaMerge.cpp index fc73e28e23a..67d32c73a05 100644 --- a/dbms/src/Storages/StorageDeltaMerge.cpp +++ b/dbms/src/Storages/StorageDeltaMerge.cpp @@ -901,14 +901,16 @@ void StorageDeltaMerge::deleteRows(const Context & context, size_t delete_rows) LOG_FMT_ERROR(log, "Rows after delete range not match, expected: {}, got: {}", (total_rows - delete_rows), after_delete_rows); } -std::pair StorageDeltaMerge::getSchemaSnapshotAndBlockForDecoding(bool need_block) +std::pair StorageDeltaMerge::getSchemaSnapshotAndBlockForDecoding(const TableStructureLockHolder & table_structure_lock, bool need_block) { + (void)table_structure_lock; std::lock_guard lock{decode_schema_mutex}; - if (!decoding_schema_snapshot || decoding_schema_snapshot->schema_version < tidb_table_info.schema_version) + if (!decoding_schema_snapshot || decoding_schema_changed) { auto & store = getAndMaybeInitStore(); - decoding_schema_snapshot = std::make_shared(store->getStoreColumns(), tidb_table_info, store->getHandle()); + decoding_schema_snapshot = std::make_shared(store->getStoreColumns(), tidb_table_info, store->getHandle(), decoding_schema_version++); cache_blocks.clear(); + decoding_schema_changed = false; } if (need_block) @@ -930,10 +932,10 @@ std::pair StorageDeltaMerg } } -void StorageDeltaMerge::releaseDecodingBlock(Int64 schema_version, BlockUPtr block_ptr) +void StorageDeltaMerge::releaseDecodingBlock(Int64 block_decoding_schema_version, BlockUPtr block_ptr) { std::lock_guard lock{decode_schema_mutex}; - if (!decoding_schema_snapshot || schema_version < decoding_schema_snapshot->schema_version) + if (!decoding_schema_snapshot || block_decoding_schema_version < decoding_schema_snapshot->decoding_schema_version) return; if (cache_blocks.size() >= max_cached_blocks_num) return; @@ -1113,6 +1115,7 @@ try updateTableColumnInfo(); } } + decoding_schema_changed = true; SortDescription pk_desc = getPrimarySortDescription(); ColumnDefines store_columns = getStoreColumnDefines(); diff --git a/dbms/src/Storages/StorageDeltaMerge.h b/dbms/src/Storages/StorageDeltaMerge.h index e304c713b7b..79ee225d237 100644 --- a/dbms/src/Storages/StorageDeltaMerge.h +++ b/dbms/src/Storages/StorageDeltaMerge.h @@ -151,9 +151,9 @@ class StorageDeltaMerge size_t getRowKeyColumnSize() const override { return rowkey_column_size; } - std::pair getSchemaSnapshotAndBlockForDecoding(bool /* need_block */) override; + std::pair getSchemaSnapshotAndBlockForDecoding(const TableStructureLockHolder & table_structure_lock, bool /* need_block */) override; - void releaseDecodingBlock(Int64 schema_version, BlockUPtr block) override; + void releaseDecodingBlock(Int64 block_decoding_schema_version, BlockUPtr block) override; bool initStoreIfDataDirExist() override; @@ -238,6 +238,11 @@ class StorageDeltaMerge mutable std::mutex decode_schema_mutex; DecodingStorageSchemaSnapshotPtr decoding_schema_snapshot; + // The following two members must be used under the protection of table structure lock + bool decoding_schema_changed = false; + // internal version for `decoding_schema_snapshot` + Int64 decoding_schema_version = 1; + // avoid creating block every time when decoding row std::vector cache_blocks; // avoid creating too many cached blocks(the typical num should be less and equal than raft apply thread) diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp deleted file mode 100644 index 4dec4fd5ea0..00000000000 --- a/dbms/src/Storages/StorageFile.cpp +++ /dev/null @@ -1,348 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -#include -#include - -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include -#include - -#include - -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int CANNOT_WRITE_TO_FILE_DESCRIPTOR; - extern const int CANNOT_SEEK_THROUGH_FILE; - extern const int DATABASE_ACCESS_DENIED; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int UNKNOWN_IDENTIFIER; - extern const int INCORRECT_FILE_NAME; - extern const int FILE_DOESNT_EXIST; - extern const int EMPTY_LIST_OF_COLUMNS_PASSED; -}; - - -static std::string getTablePath(const std::string & db_dir_path, const std::string & table_name, const std::string & format_name) -{ - return db_dir_path + escapeForFileName(table_name) + "/data." + escapeForFileName(format_name); -} - -/// Both db_dir_path and table_path must be converted to absolute paths (in particular, path cannot contain '..'). -static void checkCreationIsAllowed(Context & context_global, const std::string & db_dir_path, const std::string & table_path, int table_fd) -{ - if (context_global.getApplicationType() != Context::ApplicationType::SERVER) - return; - - if (table_fd >= 0) - throw Exception("Using file descriptor as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED); - else if (!startsWith(table_path, db_dir_path)) - throw Exception("Part path " + table_path + " is not inside " + db_dir_path, ErrorCodes::DATABASE_ACCESS_DENIED); - - Poco::File table_path_poco_file = Poco::File(table_path); - if (!table_path_poco_file.exists()) - throw Exception("File " + table_path + " is not exist", ErrorCodes::FILE_DOESNT_EXIST); - else if (table_path_poco_file.isDirectory()) - throw Exception("File " + table_path + " must not be a directory", ErrorCodes::INCORRECT_FILE_NAME); -} - - -StorageFile::StorageFile( - const std::string & table_path_, - int table_fd_, - const std::string & db_dir_path, - const std::string & table_name_, - const std::string & format_name_, - const ColumnsDescription & columns_, - Context & context_) - : IStorage(columns_), - table_name(table_name_), format_name(format_name_), context_global(context_), table_fd(table_fd_) -{ - if (table_fd < 0) /// Will use file - { - use_table_fd = false; - - if (!table_path_.empty()) /// Is user's file - { - Poco::Path poco_path = Poco::Path(table_path_); - if (poco_path.isRelative()) - poco_path = Poco::Path(db_dir_path, poco_path); - - path = poco_path.absolute().toString(); - checkCreationIsAllowed(context_global, db_dir_path, path, table_fd); - is_db_table = false; - } - else /// Is DB's file - { - if (db_dir_path.empty()) - throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME); - - path = getTablePath(db_dir_path, table_name, format_name); - is_db_table = true; - Poco::File(Poco::Path(path).parent()).createDirectories(); - } - } - else /// Will use FD - { - checkCreationIsAllowed(context_global, db_dir_path, path, table_fd); - - is_db_table = false; - use_table_fd = true; - - /// Save initial offset, it will be used for repeating SELECTs - /// If FD isn't seekable (lseek returns -1), then the second and subsequent SELECTs will fail. - table_fd_init_offset = lseek(table_fd, 0, SEEK_CUR); - } -} - - -class StorageFileBlockInputStream : public IProfilingBlockInputStream -{ -public: - StorageFileBlockInputStream(StorageFile & storage_, const Context & context, size_t max_block_size) - : storage(storage_) - { - if (storage.use_table_fd) - { - storage.rwlock.lock(); - - /// We could use common ReadBuffer and WriteBuffer in storage to leverage cache - /// and add ability to seek unseekable files, but cache sync isn't supported. - - if (storage.table_fd_was_used) /// We need seek to initial position - { - if (storage.table_fd_init_offset < 0) - throw Exception("File descriptor isn't seekable, inside " + storage.getName(), ErrorCodes::CANNOT_SEEK_THROUGH_FILE); - - /// ReadBuffer's seek() doesn't make sence, since cache is empty - if (lseek(storage.table_fd, storage.table_fd_init_offset, SEEK_SET) < 0) - throwFromErrno("Cannot seek file descriptor, inside " + storage.getName(), ErrorCodes::CANNOT_SEEK_THROUGH_FILE); - } - - storage.table_fd_was_used = true; - read_buf = std::make_unique(storage.table_fd); - } - else - { - storage.rwlock.lock_shared(); - - read_buf = std::make_unique(storage.path); - } - - reader = FormatFactory().getInput(storage.format_name, *read_buf, storage.getSampleBlock(), context, max_block_size); - } - - ~StorageFileBlockInputStream() override - { - if (storage.use_table_fd) - storage.rwlock.unlock(); - else - storage.rwlock.unlock_shared(); - } - - String getName() const override - { - return storage.getName(); - } - - Block readImpl() override - { - return reader->read(); - } - - Block getHeader() const override { return reader->getHeader(); }; - - void readPrefixImpl() override - { - reader->readPrefix(); - } - - void readSuffixImpl() override - { - reader->readSuffix(); - } - -private: - StorageFile & storage; - Block sample_block; - std::unique_ptr read_buf; - BlockInputStreamPtr reader; -}; - - -BlockInputStreams StorageFile::read( - const Names & /*column_names*/, - const SelectQueryInfo & /*query_info*/, - const Context & context, - QueryProcessingStage::Enum & /*processed_stage*/, - size_t max_block_size, - unsigned /*num_streams*/) -{ - return BlockInputStreams(1, std::make_shared(*this, context, max_block_size)); -} - - -class StorageFileBlockOutputStream : public IBlockOutputStream -{ -public: - explicit StorageFileBlockOutputStream(StorageFile & storage_) - : storage(storage_), lock(storage.rwlock) - { - if (storage.use_table_fd) - { - /** NOTE: Using real file binded to FD may be misleading: - * SELECT *; INSERT insert_data; SELECT *; last SELECT returns initil_fd_data + insert_data - * INSERT data; SELECT *; last SELECT returns only insert_data - */ - storage.table_fd_was_used = true; - write_buf = std::make_unique(storage.table_fd); - } - else - { - write_buf = std::make_unique(storage.path, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT); - } - - writer = FormatFactory().getOutput(storage.format_name, *write_buf, storage.getSampleBlock(), storage.context_global); - } - - Block getHeader() const override { return storage.getSampleBlock(); } - - void write(const Block & block) override - { - writer->write(block); - } - - void writePrefix() override - { - writer->writePrefix(); - } - - void writeSuffix() override - { - writer->writeSuffix(); - } - - void flush() override - { - writer->flush(); - } - -private: - StorageFile & storage; - std::unique_lock lock; - std::unique_ptr write_buf; - BlockOutputStreamPtr writer; -}; - -BlockOutputStreamPtr StorageFile::write( - const ASTPtr & /*query*/, - const Settings & /*settings*/) -{ - return std::make_shared(*this); -} - - -void StorageFile::drop() -{ - /// Extra actions are not required. -} - - -void StorageFile::rename(const String & new_path_to_db, const String & /*new_database_name*/, const String & new_table_name) -{ - if (!is_db_table) - throw Exception("Can't rename table '" + table_name + "' binded to user-defined file (or FD)", ErrorCodes::DATABASE_ACCESS_DENIED); - - std::unique_lock lock(rwlock); - - std::string path_new = getTablePath(new_path_to_db, new_table_name, format_name); - Poco::File(Poco::Path(path_new).parent()).createDirectories(); - Poco::File(path).renameTo(path_new); - - path = std::move(path_new); -} - - -void registerStorageFile(StorageFactory & factory) -{ - factory.registerStorage("File", [](const StorageFactory::Arguments & args) - { - ASTs & engine_args = args.engine_args; - - if (!(engine_args.size() == 1 || engine_args.size() == 2)) - throw Exception( - "Storage File requires 1 or 2 arguments: name of used format and source.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context); - String format_name = static_cast(*engine_args[0]).value.safeGet(); - - int source_fd = -1; - String source_path; - if (engine_args.size() >= 2) - { - /// Will use FD if engine_args[1] is int literal or identifier with std* name - - if (const ASTIdentifier * identifier = typeid_cast(engine_args[1].get())) - { - if (identifier->name == "stdin") - source_fd = STDIN_FILENO; - else if (identifier->name == "stdout") - source_fd = STDOUT_FILENO; - else if (identifier->name == "stderr") - source_fd = STDERR_FILENO; - else - throw Exception("Unknown identifier '" + identifier->name + "' in second arg of File storage constructor", - ErrorCodes::UNKNOWN_IDENTIFIER); - } - else if (const ASTLiteral * literal = typeid_cast(engine_args[1].get())) - { - auto type = literal->value.getType(); - if (type == Field::Types::Int64) - source_fd = static_cast(literal->value.get()); - else if (type == Field::Types::UInt64) - source_fd = static_cast(literal->value.get()); - else if (type == Field::Types::String) - source_path = literal->value.get(); - } - } - - return StorageFile::create( - source_path, source_fd, - args.data_path, - args.table_name, format_name, args.columns, - args.context); - }); -} - -} diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h deleted file mode 100644 index ca46f7f366e..00000000000 --- a/dbms/src/Storages/StorageFile.h +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include - -#include -#include -#include - - -namespace DB -{ -class StorageFileBlockInputStream; -class StorageFileBlockOutputStream; - -class StorageFile : public ext::SharedPtrHelper - , public IStorage -{ -public: - std::string getName() const override - { - return "File"; - } - - std::string getTableName() const override - { - return table_name; - } - - BlockInputStreams read( - const Names & column_names, - const SelectQueryInfo & query_info, - const Context & context, - QueryProcessingStage::Enum & processed_stage, - size_t max_block_size, - unsigned num_streams) override; - - BlockOutputStreamPtr write( - const ASTPtr & query, - const Settings & settings) override; - - void drop() override; - - void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; - - String getDataPath() const override { return path; } - -protected: - friend class StorageFileBlockInputStream; - friend class StorageFileBlockOutputStream; - - /** there are three options (ordered by priority): - - use specified file descriptor if (fd >= 0) - - use specified table_path if it isn't empty - - create own table inside data/db/table/ - */ - StorageFile( - const std::string & table_path_, - int table_fd_, - const std::string & db_dir_path, - const std::string & table_name_, - const std::string & format_name_, - const ColumnsDescription & columns_, - Context & context_); - -private: - std::string table_name; - std::string format_name; - Context & context_global; - - std::string path; - int table_fd = -1; - - bool is_db_table = true; /// Table is stored in real database, not user's file - bool use_table_fd = false; /// Use table_fd insted of path - std::atomic table_fd_was_used{false}; /// To detect repeating reads from stdin - off_t table_fd_init_offset = -1; /// Initial position of fd, used for repeating reads - - mutable std::shared_mutex rwlock; - - Poco::Logger * log = &Poco::Logger::get("StorageFile"); -}; - -} // namespace DB diff --git a/dbms/src/Storages/Transaction/ApplySnapshot.cpp b/dbms/src/Storages/Transaction/ApplySnapshot.cpp index 3ed04d5ecbf..2df95fead93 100644 --- a/dbms/src/Storages/Transaction/ApplySnapshot.cpp +++ b/dbms/src/Storages/Transaction/ApplySnapshot.cpp @@ -261,75 +261,6 @@ void KVStore::onSnapshot(const RegionPtrWrap & new_region_wrap, RegionPtr old_re extern RegionPtrWithBlock::CachePtr GenRegionPreDecodeBlockData(const RegionPtr &, Context &); -/// `preHandleSnapshotToBlock` read data from SSTFiles and predoced the data as a block -RegionPreDecodeBlockDataPtr KVStore::preHandleSnapshotToBlock( - RegionPtr new_region, - const SSTViewVec snaps, - uint64_t /*index*/, - uint64_t /*term*/, - TMTContext & tmt) -{ - RegionPreDecodeBlockDataPtr cache{nullptr}; - { - decltype(bg_gc_region_data)::value_type tmp; - std::lock_guard lock(bg_gc_region_data_mutex); - if (!bg_gc_region_data.empty()) - { - tmp.swap(bg_gc_region_data.back()); - bg_gc_region_data.pop_back(); - } - } - - Stopwatch watch; - auto & ctx = tmt.getContext(); - SCOPE_EXIT({ GET_METRIC(tiflash_raft_command_duration_seconds, type_apply_snapshot_predecode).Observe(watch.elapsedSeconds()); }); - - { - LOG_FMT_INFO(log, "Pre-handle snapshot {} with {} TiKV sst files", new_region->toString(false), snaps.len); - // Iterator over all SST files and insert key-values into `new_region` - for (UInt64 i = 0; i < snaps.len; ++i) - { - const auto & snapshot = snaps.views[i]; - auto sst_reader = SSTReader{proxy_helper, snapshot}; - - uint64_t kv_size = 0; - while (sst_reader.remained()) - { - auto key = sst_reader.key(); - auto value = sst_reader.value(); - new_region->insert(snaps.views[i].type, TiKVKey(key.data, key.len), TiKVValue(value.data, value.len)); - ++kv_size; - sst_reader.next(); - } - - LOG_FMT_INFO(log, - "Decode {} got [cf: {}, kv size: {}]", - std::string_view(snapshot.path.data, snapshot.path.len), - CFToName(snapshot.type), - kv_size); - // Note that number of keys in different cf will be aggregated into one metrics - GET_METRIC(tiflash_raft_process_keys, type_apply_snapshot).Increment(kv_size); - } - { - LOG_FMT_INFO(log, "Start to pre-decode {} into block", new_region->toString()); - auto block_cache = GenRegionPreDecodeBlockData(new_region, ctx); - if (block_cache) - { - std::stringstream ss; - block_cache->toString(ss); - LOG_FMT_INFO(log, "Got pre-decode block cache {}", ss.str()); - } - else - LOG_FMT_INFO(log, "Got empty pre-decode block cache"); - - cache = std::move(block_cache); - } - LOG_FMT_INFO(log, "Pre-handle snapshot {} cost {}ms", new_region->toString(false), watch.elapsedMilliseconds()); - } - - return cache; -} - std::vector KVStore::preHandleSnapshotToFiles( RegionPtr new_region, const SSTViewVec snaps, @@ -473,8 +404,8 @@ void KVStore::handlePreApplySnapshot(const RegionPtrWrap & new_region, TMTContex LOG_FMT_INFO(log, "{} apply snapshot success", new_region->toString(false)); } -template void KVStore::handlePreApplySnapshot(const RegionPtrWithBlock &, TMTContext &); template void KVStore::handlePreApplySnapshot(const RegionPtrWithSnapshotFiles &, TMTContext &); + template void KVStore::checkAndApplySnapshot(const RegionPtrWithBlock &, TMTContext &); template void KVStore::checkAndApplySnapshot(const RegionPtrWithSnapshotFiles &, TMTContext &); template void KVStore::onSnapshot(const RegionPtrWithBlock &, RegionPtr, UInt64, TMTContext &); @@ -521,10 +452,7 @@ void KVStore::handleApplySnapshot( TMTContext & tmt) { auto new_region = genRegionPtr(std::move(region), peer_id, index, term); - if (snapshot_apply_method == TiDB::SnapshotApplyMethod::Block) - handlePreApplySnapshot(RegionPtrWithBlock{new_region, preHandleSnapshotToBlock(new_region, snaps, index, term, tmt)}, tmt); - else - handlePreApplySnapshot(RegionPtrWithSnapshotFiles{new_region, preHandleSnapshotToFiles(new_region, snaps, index, term, tmt)}, tmt); + handlePreApplySnapshot(RegionPtrWithSnapshotFiles{new_region, preHandleSnapshotToFiles(new_region, snaps, index, term, tmt)}, tmt); } EngineStoreApplyRes KVStore::handleIngestSST(UInt64 region_id, const SSTViewVec snaps, UInt64 index, UInt64 term, TMTContext & tmt) @@ -543,15 +471,12 @@ EngineStoreApplyRes KVStore::handleIngestSST(UInt64 region_id, const SSTViewVec fiu_do_on(FailPoints::force_set_sst_decode_rand, { static int num_call = 0; - switch (num_call++ % 3) + switch (num_call++ % 2) { case 0: - snapshot_apply_method = TiDB::SnapshotApplyMethod::Block; - break; - case 1: snapshot_apply_method = TiDB::SnapshotApplyMethod::DTFile_Directory; break; - case 2: + case 1: snapshot_apply_method = TiDB::SnapshotApplyMethod::DTFile_Single; break; default: @@ -576,15 +501,6 @@ EngineStoreApplyRes KVStore::handleIngestSST(UInt64 region_id, const SSTViewVec } }; - if (snapshot_apply_method == TiDB::SnapshotApplyMethod::Block) - { - // try to flush remain data in memory. - func_try_flush(); - region->handleIngestSSTInMemory(snaps, index, term); - // after `handleIngestSSTInMemory`, all data are stored in `region`, try to flush committed data into storage - func_try_flush(); - } - else { // try to flush remain data in memory. func_try_flush(); diff --git a/dbms/src/Storages/Transaction/DecodingStorageSchemaSnapshot.h b/dbms/src/Storages/Transaction/DecodingStorageSchemaSnapshot.h index 6cedbe3f0c0..c636d9e60ab 100644 --- a/dbms/src/Storages/Transaction/DecodingStorageSchemaSnapshot.h +++ b/dbms/src/Storages/Transaction/DecodingStorageSchemaSnapshot.h @@ -67,13 +67,14 @@ struct DecodingStorageSchemaSnapshot bool pk_is_handle; bool is_common_handle; TMTPKType pk_type = TMTPKType::UNSPECIFIED; - Int64 schema_version = DEFAULT_UNSPECIFIED_SCHEMA_VERSION; + // an internal increasing version for `DecodingStorageSchemaSnapshot`, has no relation with the table schema version + Int64 decoding_schema_version; - DecodingStorageSchemaSnapshot(DM::ColumnDefinesPtr column_defines_, const TiDB::TableInfo & table_info_, const DM::ColumnDefine & original_handle_) + DecodingStorageSchemaSnapshot(DM::ColumnDefinesPtr column_defines_, const TiDB::TableInfo & table_info_, const DM::ColumnDefine & original_handle_, Int64 decoding_schema_version_) : column_defines{std::move(column_defines_)} , pk_is_handle{table_info_.pk_is_handle} , is_common_handle{table_info_.is_common_handle} - , schema_version{table_info_.schema_version} + , decoding_schema_version{decoding_schema_version_} { std::unordered_map column_lut; for (size_t i = 0; i < table_info_.columns.size(); i++) diff --git a/dbms/src/Storages/Transaction/KVStore.cpp b/dbms/src/Storages/Transaction/KVStore.cpp index 2b82220cbc4..318a04c6ed9 100644 --- a/dbms/src/Storages/Transaction/KVStore.cpp +++ b/dbms/src/Storages/Transaction/KVStore.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -38,15 +39,15 @@ extern const int TABLE_IS_DROPPED; } // namespace ErrorCodes KVStore::KVStore(Context & context, TiDB::SnapshotApplyMethod snapshot_apply_method_) - : region_persister(context, region_manager) + : region_persister(std::make_unique(context, region_manager)) , raft_cmd_res(std::make_unique()) , snapshot_apply_method(snapshot_apply_method_) , log(&Poco::Logger::get("KVStore")) + , region_compact_log_period(120) + , region_compact_log_min_rows(40 * 1024) + , region_compact_log_min_bytes(32 * 1024 * 1024) { // default config about compact-log: period 120s, rows 40k, bytes 32MB. - REGION_COMPACT_LOG_PERIOD = 120; - REGION_COMPACT_LOG_MIN_ROWS = 40 * 1024; - REGION_COMPACT_LOG_MIN_BYTES = 32 * 1024 * 1024; } void KVStore::restore(const TiFlashRaftProxyHelper * proxy_helper) @@ -55,7 +56,7 @@ void KVStore::restore(const TiFlashRaftProxyHelper * proxy_helper) auto manage_lock = genRegionWriteLock(task_lock); this->proxy_helper = proxy_helper; - manage_lock.regions = region_persister.restore(proxy_helper); + manage_lock.regions = region_persister->restore(proxy_helper); LOG_FMT_INFO(log, "Restored {} regions", manage_lock.regions.size()); @@ -166,7 +167,7 @@ void KVStore::tryPersist(RegionID region_id) if (region) { LOG_FMT_INFO(log, "Try to persist {}", region->toString(false)); - region_persister.persist(*region); + region_persister->persist(*region); LOG_FMT_INFO(log, "After persisted {}, cache {} bytes", region->toString(false), region->dataSize()); } } @@ -182,7 +183,7 @@ void KVStore::gcRegionPersistedCache(Seconds gc_persist_period) if (now < (last_gc_time.load() + gc_persist_period)) return; last_gc_time = now; - region_persister.gc(); + region_persister->gc(); } void KVStore::removeRegion(RegionID region_id, bool remove_data, RegionTable & region_table, const KVStoreTaskLock & task_lock, const RegionTaskLock & region_lock) @@ -203,7 +204,7 @@ void KVStore::removeRegion(RegionID region_id, bool remove_data, RegionTable & r } } - region_persister.drop(region_id, region_lock); + region_persister->drop(region_id, region_lock); LOG_FMT_INFO(log, "Persisted [region {}] deleted", region_id); region_table.removeRegion(region_id, remove_data, region_lock); @@ -306,9 +307,9 @@ void KVStore::handleDestroy(UInt64 region_id, TMTContext & tmt, const KVStoreTas void KVStore::setRegionCompactLogConfig(UInt64 sec, UInt64 rows, UInt64 bytes) { - REGION_COMPACT_LOG_PERIOD = sec; - REGION_COMPACT_LOG_MIN_ROWS = rows; - REGION_COMPACT_LOG_MIN_BYTES = bytes; + region_compact_log_period = sec; + region_compact_log_min_rows = rows; + region_compact_log_min_bytes = bytes; LOG_FMT_INFO( log, @@ -321,7 +322,7 @@ void KVStore::setRegionCompactLogConfig(UInt64 sec, UInt64 rows, UInt64 bytes) void KVStore::persistRegion(const Region & region, const RegionTaskLock & region_task_lock, const char * caller) { LOG_FMT_INFO(log, "Start to persist {}, cache size: {} bytes for `{}`", region.toString(true), region.dataSize(), caller); - region_persister.persist(region, region_task_lock); + region_persister->persist(region, region_task_lock); LOG_FMT_DEBUG(log, "Persist {} done", region.toString(false)); } @@ -362,8 +363,8 @@ EngineStoreApplyRes KVStore::handleUselessAdminRaftCmd( LOG_FMT_DEBUG(log, "{} approx mem cache info: rows {}, bytes {}", curr_region.toString(false), rows, size_bytes); - if (rows >= REGION_COMPACT_LOG_MIN_ROWS.load(std::memory_order_relaxed) - || size_bytes >= REGION_COMPACT_LOG_MIN_BYTES.load(std::memory_order_relaxed)) + if (rows >= region_compact_log_min_rows.load(std::memory_order_relaxed) + || size_bytes >= region_compact_log_min_bytes.load(std::memory_order_relaxed)) { // if rows or bytes more than threshold, flush cache and perist mem data. return true; @@ -372,7 +373,7 @@ EngineStoreApplyRes KVStore::handleUselessAdminRaftCmd( { // if thhere is little data in mem, wait until time interval reached threshold. // use random period so that lots of regions will not be persisted at same time. - auto compact_log_period = std::rand() % REGION_COMPACT_LOG_PERIOD.load(std::memory_order_relaxed); // NOLINT + auto compact_log_period = std::rand() % region_compact_log_period.load(std::memory_order_relaxed); // NOLINT return !(curr_region.lastCompactLogTime() + Seconds{compact_log_period} > Clock::now()); } } @@ -765,4 +766,9 @@ KVStore::~KVStore() releaseReadIndexWorkers(); } +FileUsageStatistics KVStore::getFileUsageStatistics() const +{ + return region_persister->getFileUsageStatistics(); +} + } // namespace DB diff --git a/dbms/src/Storages/Transaction/KVStore.h b/dbms/src/Storages/Transaction/KVStore.h index ef851d67958..bb45e65d18b 100644 --- a/dbms/src/Storages/Transaction/KVStore.h +++ b/dbms/src/Storages/Transaction/KVStore.h @@ -16,20 +16,19 @@ #include #include -#include #include - namespace TiDB { struct TableInfo; } namespace DB { +class Context; namespace RegionBench { extern void concurrentBatchInsert(const TiDB::TableInfo &, Int64, Int64, Int64, UInt64, UInt64, Context &); -} +} // namespace RegionBench namespace DM { enum class FileConvertJobType; @@ -40,7 +39,6 @@ namespace tests class RegionKVStoreTest; } -class Context; class IAST; using ASTPtr = std::shared_ptr; using ASTs = std::vector; @@ -71,6 +69,8 @@ using RegionPreDecodeBlockDataPtr = std::unique_ptr; class ReadIndexWorkerManager; using BatchReadIndexRes = std::vector>; class ReadIndexStressTest; +struct FileUsageStatistics; +class RegionPersister; /// TODO: brief design document. class KVStore final : private boost::noncopyable @@ -109,12 +109,7 @@ class KVStore final : private boost::noncopyable EngineStoreApplyRes handleWriteRaftCmd(const WriteCmdsView & cmds, UInt64 region_id, UInt64 index, UInt64 term, TMTContext & tmt); void handleApplySnapshot(metapb::Region && region, uint64_t peer_id, const SSTViewVec, uint64_t index, uint64_t term, TMTContext & tmt); - RegionPreDecodeBlockDataPtr preHandleSnapshotToBlock( - RegionPtr new_region, - const SSTViewVec, - uint64_t index, - uint64_t term, - TMTContext & tmt); + std::vector /* */ preHandleSnapshotToFiles( RegionPtr new_region, const SSTViewVec, @@ -162,10 +157,7 @@ class KVStore final : private boost::noncopyable ~KVStore(); - FileUsageStatistics getFileUsageStatistics() const - { - return region_persister.getFileUsageStatistics(); - } + FileUsageStatistics getFileUsageStatistics() const; private: friend class MockTiDB; @@ -234,7 +226,7 @@ class KVStore final : private boost::noncopyable private: RegionManager region_manager; - RegionPersister region_persister; + std::unique_ptr region_persister; std::atomic last_gc_time = Timepoint::min(); @@ -247,9 +239,9 @@ class KVStore final : private boost::noncopyable Poco::Logger * log; - std::atomic REGION_COMPACT_LOG_PERIOD; - std::atomic REGION_COMPACT_LOG_MIN_ROWS; - std::atomic REGION_COMPACT_LOG_MIN_BYTES; + std::atomic region_compact_log_period; + std::atomic region_compact_log_min_rows; + std::atomic region_compact_log_min_bytes; mutable std::mutex bg_gc_region_data_mutex; std::list bg_gc_region_data; diff --git a/dbms/src/Storages/Transaction/PDTiKVClient.cpp b/dbms/src/Storages/Transaction/PDTiKVClient.cpp index 5a4b751fd9c..a06f1a3ae64 100644 --- a/dbms/src/Storages/Transaction/PDTiKVClient.cpp +++ b/dbms/src/Storages/Transaction/PDTiKVClient.cpp @@ -22,7 +22,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -Timestamp PDClientHelper::cached_gc_safe_point = 0; -std::chrono::time_point PDClientHelper::safe_point_last_update_time; +std::atomic PDClientHelper::cached_gc_safe_point = 0; +std::atomic> PDClientHelper::safe_point_last_update_time; } // namespace DB diff --git a/dbms/src/Storages/Transaction/PDTiKVClient.h b/dbms/src/Storages/Transaction/PDTiKVClient.h index 4986c28f4ac..e5801cc7fae 100644 --- a/dbms/src/Storages/Transaction/PDTiKVClient.h +++ b/dbms/src/Storages/Transaction/PDTiKVClient.h @@ -29,6 +29,8 @@ #include #include +#include + // We define a shared ptr here, because TMTContext / SchemaSyncer / IndexReader all need to // `share` the resource of cluster. using KVClusterPtr = std::shared_ptr; @@ -49,7 +51,7 @@ struct PDClientHelper { // In case we cost too much to update safe point from PD. std::chrono::time_point now = std::chrono::system_clock::now(); - const auto duration = std::chrono::duration_cast(now - safe_point_last_update_time); + const auto duration = std::chrono::duration_cast(now - safe_point_last_update_time.load()); const auto min_interval = std::max(Int64(1), safe_point_update_interval_seconds); // at least one second if (duration.count() < min_interval) return cached_gc_safe_point; @@ -73,8 +75,8 @@ struct PDClientHelper } private: - static Timestamp cached_gc_safe_point; - static std::chrono::time_point safe_point_last_update_time; + static std::atomic cached_gc_safe_point; + static std::atomic> safe_point_last_update_time; }; diff --git a/dbms/src/Storages/Transaction/PartitionStreams.cpp b/dbms/src/Storages/Transaction/PartitionStreams.cpp index 13840159ebb..4b2ca6c07a8 100644 --- a/dbms/src/Storages/Transaction/PartitionStreams.cpp +++ b/dbms/src/Storages/Transaction/PartitionStreams.cpp @@ -114,14 +114,14 @@ static void writeRegionDataToStorage( /// Read region data as block. Stopwatch watch; - Int64 block_schema_version = DEFAULT_UNSPECIFIED_SCHEMA_VERSION; + Int64 block_decoding_schema_version = -1; BlockUPtr block_ptr = nullptr; if (need_decode) { LOG_FMT_TRACE(log, "{} begin to decode table {}, region {}", FUNCTION_NAME, table_id, region->id()); DecodingStorageSchemaSnapshotConstPtr decoding_schema_snapshot; - std::tie(decoding_schema_snapshot, block_ptr) = storage->getSchemaSnapshotAndBlockForDecoding(true); - block_schema_version = decoding_schema_snapshot->schema_version; + std::tie(decoding_schema_snapshot, block_ptr) = storage->getSchemaSnapshotAndBlockForDecoding(lock, true); + block_decoding_schema_version = decoding_schema_snapshot->decoding_schema_version; auto reader = RegionBlockReader(decoding_schema_snapshot); if (!reader.read(*block_ptr, data_list_read, force_decode)) @@ -153,7 +153,7 @@ static void writeRegionDataToStorage( write_part_cost = watch.elapsedMilliseconds(); GET_METRIC(tiflash_raft_write_data_to_storage_duration_seconds, type_write).Observe(write_part_cost / 1000.0); if (need_decode) - storage->releaseDecodingBlock(block_schema_version, std::move(block_ptr)); + storage->releaseDecodingBlock(block_decoding_schema_version, std::move(block_ptr)); LOG_FMT_TRACE(log, "{}: table {}, region {}, cost [region decode {}, write part {}] ms", FUNCTION_NAME, table_id, region->id(), region_decode_cost, write_part_cost); return true; @@ -353,60 +353,6 @@ void RegionTable::writeBlockByRegion( data_list_to_remove = std::move(*data_list_read); } -RegionTable::ReadBlockByRegionRes RegionTable::readBlockByRegion(const TiDB::TableInfo & table_info, - const ColumnsDescription & columns [[maybe_unused]], - const Names & column_names_to_read, - const RegionPtr & region, - RegionVersion region_version, - RegionVersion conf_version, - bool resolve_locks, - Timestamp start_ts, - const std::unordered_set * bypass_lock_ts, - RegionScanFilterPtr scan_filter) -{ - if (!region) - throw Exception(std::string(__PRETTY_FUNCTION__) + ": region is null", ErrorCodes::LOGICAL_ERROR); - - // Tiny optimization for queries that need only handle, tso, delmark. - bool need_value = column_names_to_read.size() != 3; - auto region_data_lock = resolveLocksAndReadRegionData( - table_info.id, - region, - start_ts, - bypass_lock_ts, - region_version, - conf_version, - resolve_locks, - need_value); - - return std::visit(variant_op::overloaded{ - [&](RegionDataReadInfoList & data_list_read) -> ReadBlockByRegionRes { - /// Read region data as block. - Block block; - // FIXME: remove this deprecated function - assert(0); - { - auto reader = RegionBlockReader(nullptr); - bool ok = reader.setStartTs(start_ts) - .setFilter(scan_filter) - .read(block, data_list_read, /*force_decode*/ true); - if (!ok) - // TODO: Enrich exception message. - throw Exception("Read region " + std::to_string(region->id()) + " of table " - + std::to_string(table_info.id) + " failed", - ErrorCodes::LOGICAL_ERROR); - } - return block; - }, - [&](LockInfoPtr & lock_value) -> ReadBlockByRegionRes { - assert(lock_value); - throw LockException(region->id(), std::move(lock_value)); - }, - [](RegionException::RegionReadStatus & s) -> ReadBlockByRegionRes { return s; }, - }, - region_data_lock); -} - RegionTable::ResolveLocksAndWriteRegionRes RegionTable::resolveLocksAndWriteRegion(TMTContext & tmt, const TiDB::TableID table_id, const RegionPtr & region, @@ -509,7 +455,7 @@ RegionPtrWithBlock::CachePtr GenRegionPreDecodeBlockData(const RegionPtr & regio } DecodingStorageSchemaSnapshotConstPtr decoding_schema_snapshot; - std::tie(decoding_schema_snapshot, std::ignore) = storage->getSchemaSnapshotAndBlockForDecoding(false); + std::tie(decoding_schema_snapshot, std::ignore) = storage->getSchemaSnapshotAndBlockForDecoding(lock, false); res_block = createBlockSortByColumnID(decoding_schema_snapshot); auto reader = RegionBlockReader(decoding_schema_snapshot); if (!reader.read(res_block, *data_list_read, force_decode)) @@ -562,7 +508,7 @@ AtomicGetStorageSchema(const RegionPtr & region, TMTContext & tmt) auto table_lock = storage->lockStructureForShare(getThreadName()); dm_storage = std::dynamic_pointer_cast(storage); // only dt storage engine support `getSchemaSnapshotAndBlockForDecoding`, other engine will throw exception - std::tie(schema_snapshot, std::ignore) = storage->getSchemaSnapshotAndBlockForDecoding(false); + std::tie(schema_snapshot, std::ignore) = storage->getSchemaSnapshotAndBlockForDecoding(table_lock, false); std::tie(std::ignore, drop_lock) = std::move(table_lock).release(); return true; }; diff --git a/dbms/src/Storages/Transaction/ProxyFFI.cpp b/dbms/src/Storages/Transaction/ProxyFFI.cpp index 58e7f5ad2e5..8a40ca9b15e 100644 --- a/dbms/src/Storages/Transaction/ProxyFFI.cpp +++ b/dbms/src/Storages/Transaction/ProxyFFI.cpp @@ -24,6 +24,8 @@ #include #include +#include + #define CHECK_PARSE_PB_BUFF_IMPL(n, a, b, c) \ do \ { \ @@ -304,25 +306,13 @@ RawRustPtrWrap::~RawRustPtrWrap() RustGcHelper::instance().gcRustPtr(ptr, type); } RawRustPtrWrap::RawRustPtrWrap(RawRustPtrWrap && src) + : RawRustPtr() { RawRustPtr & tar = (*this); tar = src; src.ptr = nullptr; } -struct PreHandledSnapshotWithBlock -{ - ~PreHandledSnapshotWithBlock() { CurrentMetrics::sub(CurrentMetrics::RaftNumSnapshotsPendingApply); } - PreHandledSnapshotWithBlock(const RegionPtr & region_, RegionPtrWithBlock::CachePtr && cache_) - : region(region_) - , cache(std::move(cache_)) - { - CurrentMetrics::add(CurrentMetrics::RaftNumSnapshotsPendingApply); - } - RegionPtr region; - RegionPtrWithBlock::CachePtr cache; -}; - struct PreHandledSnapshotWithFiles { ~PreHandledSnapshotWithFiles() { CurrentMetrics::sub(CurrentMetrics::RaftNumSnapshotsPendingApply); } @@ -362,13 +352,6 @@ RawCppPtr PreHandleSnapshot( switch (kvstore->applyMethod()) { - case TiDB::SnapshotApplyMethod::Block: - { - // Pre-decode as a block - auto new_region_block_cache = kvstore->preHandleSnapshotToBlock(new_region, snaps, index, term, tmt); - auto * res = new PreHandledSnapshotWithBlock{new_region, std::move(new_region_block_cache)}; - return GenRawCppPtr(res, RawCppPtrTypeImpl::PreHandledSnapshotWithBlock); - } case TiDB::SnapshotApplyMethod::DTFile_Directory: case TiDB::SnapshotApplyMethod::DTFile_Single: { @@ -391,18 +374,12 @@ RawCppPtr PreHandleSnapshot( template void ApplyPreHandledSnapshot(EngineStoreServerWrap * server, PreHandledSnapshot * snap) { - static_assert( - std::is_same_v || std::is_same_v, - "Unknown pre-handled snapshot type"); + static_assert(std::is_same_v, "Unknown pre-handled snapshot type"); try { auto & kvstore = server->tmt->getKVStore(); - if constexpr (std::is_same_v) - { - kvstore->handlePreApplySnapshot(RegionPtrWithBlock{snap->region, std::move(snap->cache)}, *server->tmt); - } - else if constexpr (std::is_same_v) + if constexpr (std::is_same_v) { kvstore->handlePreApplySnapshot(RegionPtrWithSnapshotFiles{snap->region, std::move(snap->ingest_ids)}, *server->tmt); } @@ -418,12 +395,6 @@ void ApplyPreHandledSnapshot(EngineStoreServerWrap * server, RawVoidPtr res, Raw { switch (static_cast(type)) { - case RawCppPtrTypeImpl::PreHandledSnapshotWithBlock: - { - auto * snap = reinterpret_cast(res); - ApplyPreHandledSnapshot(server, snap); - break; - } case RawCppPtrTypeImpl::PreHandledSnapshotWithFiles: { auto * snap = reinterpret_cast(res); @@ -445,9 +416,6 @@ void GcRawCppPtr(RawVoidPtr ptr, RawCppPtrType type) case RawCppPtrTypeImpl::String: delete reinterpret_cast(ptr); break; - case RawCppPtrTypeImpl::PreHandledSnapshotWithBlock: - delete reinterpret_cast(ptr); - break; case RawCppPtrTypeImpl::PreHandledSnapshotWithFiles: delete reinterpret_cast(ptr); break; diff --git a/dbms/src/Storages/Transaction/ProxyFFI.h b/dbms/src/Storages/Transaction/ProxyFFI.h index 5d87af94f30..e1c01599275 100644 --- a/dbms/src/Storages/Transaction/ProxyFFI.h +++ b/dbms/src/Storages/Transaction/ProxyFFI.h @@ -56,7 +56,6 @@ enum class RawCppPtrTypeImpl : RawCppPtrType { None = 0, String, - PreHandledSnapshotWithBlock, PreHandledSnapshotWithFiles, WakerNotifier, }; diff --git a/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp b/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp index dafacd8947d..792f149f588 100644 --- a/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp +++ b/dbms/src/Storages/Transaction/ProxyFFIStatusService.cpp @@ -22,26 +22,6 @@ namespace DB { -HttpRequestRes HandleHttpRequestTestShow( - EngineStoreServerWrap *, - std::string_view path, - const std::string & api_name, - std::string_view query, - std::string_view body) -{ - auto * res = RawCppString::New(fmt::format( - "api_name: {}\npath: {}\nquery: {}\nbody: {}", - api_name, - path, - query, - body)); - return HttpRequestRes{ - .status = HttpRequestStatus::Ok, - .res = CppStrWithView{ - .inner = GenRawCppPtr(res, RawCppPtrTypeImpl::String), - .view = BaseBuffView{res->data(), res->size()}}}; -} - HttpRequestRes HandleHttpRequestSyncStatus( EngineStoreServerWrap * server, std::string_view path, @@ -112,8 +92,7 @@ using HANDLE_HTTP_URI_METHOD = HttpRequestRes (*)(EngineStoreServerWrap *, std:: static const std::map AVAILABLE_HTTP_URI = { {"/tiflash/sync-status/", HandleHttpRequestSyncStatus}, - {"/tiflash/store-status", HandleHttpRequestStoreStatus}, - {"/tiflash/test-show", HandleHttpRequestTestShow}}; + {"/tiflash/store-status", HandleHttpRequestStoreStatus}}; uint8_t CheckHttpUriAvailable(BaseBuffView path_) { diff --git a/dbms/src/Storages/Transaction/ReadIndexWorker.cpp b/dbms/src/Storages/Transaction/ReadIndexWorker.cpp index 97a8f4b3e0b..3223c815989 100644 --- a/dbms/src/Storages/Transaction/ReadIndexWorker.cpp +++ b/dbms/src/Storages/Transaction/ReadIndexWorker.cpp @@ -20,6 +20,7 @@ #include #include +#include namespace DB { @@ -174,7 +175,7 @@ struct BlockedReadIndexHelper : BlockedReadIndexHelperTrait return waker.waitFor(tm); } - virtual ~BlockedReadIndexHelper() = default; + ~BlockedReadIndexHelper() override = default; private: AsyncWaker & waker; @@ -193,7 +194,7 @@ struct BlockedReadIndexHelperV3 : BlockedReadIndexHelperTrait return notifier.blockedWaitFor(tm); } - virtual ~BlockedReadIndexHelperV3() = default; + ~BlockedReadIndexHelperV3() override = default; private: AsyncWaker::Notifier & notifier; @@ -342,7 +343,7 @@ struct RegionReadIndexNotifier : AsyncNotifier notify->wake(); } - virtual ~RegionReadIndexNotifier() = default; + ~RegionReadIndexNotifier() override = default; RegionReadIndexNotifier( RegionID region_id_, diff --git a/dbms/src/Storages/Transaction/Region.cpp b/dbms/src/Storages/Transaction/Region.cpp index e021de3d978..aa75eabb4b9 100644 --- a/dbms/src/Storages/Transaction/Region.cpp +++ b/dbms/src/Storages/Transaction/Region.cpp @@ -720,47 +720,6 @@ EngineStoreApplyRes Region::handleWriteRaftCmd(const WriteCmdsView & cmds, UInt6 return EngineStoreApplyRes::None; } -void Region::handleIngestSSTInMemory(const SSTViewVec snaps, UInt64 index, UInt64 term) -{ - if (index <= appliedIndex()) - return; - - { - std::unique_lock lock(mutex); - - for (UInt64 i = 0; i < snaps.len; ++i) - { - const auto & snapshot = snaps.views[i]; - auto sst_reader = SSTReader{proxy_helper, snapshot}; - - LOG_FMT_INFO(log, - "{} begin to ingest sst of cf {} at [term: {}, index: {}]", - this->toString(false), - CFToName(snapshot.type), - term, - index); - - uint64_t kv_size = 0; - while (sst_reader.remained()) - { - auto key = sst_reader.key(); - auto value = sst_reader.value(); - doInsert(snaps.views[i].type, TiKVKey(key.data, key.len), TiKVValue(value.data, value.len)); - ++kv_size; - sst_reader.next(); - } - - LOG_FMT_INFO(log, - "{} finish to ingest sst of kv count {}", - this->toString(false), - kv_size); - GET_METRIC(tiflash_raft_process_keys, type_ingest_sst).Increment(kv_size); - } - meta.setApplied(index, term); - } - meta.notifyAll(); -} - void Region::finishIngestSSTByDTFile(RegionPtr && rhs, UInt64 index, UInt64 term) { if (index <= appliedIndex()) diff --git a/dbms/src/Storages/Transaction/Region.h b/dbms/src/Storages/Transaction/Region.h index b31ae0cdc49..06b18de379a 100644 --- a/dbms/src/Storages/Transaction/Region.h +++ b/dbms/src/Storages/Transaction/Region.h @@ -191,7 +191,6 @@ class Region : public std::enable_shared_from_this TableID getMappedTableID() const; EngineStoreApplyRes handleWriteRaftCmd(const WriteCmdsView & cmds, UInt64 index, UInt64 term, TMTContext & tmt); - void handleIngestSSTInMemory(const SSTViewVec snaps, UInt64 index, UInt64 term); void finishIngestSSTByDTFile(RegionPtr && rhs, UInt64 index, UInt64 term); UInt64 getSnapshotEventFlag() const { return snapshot_event_flag; } diff --git a/dbms/src/Storages/Transaction/RegionBlockReader.cpp b/dbms/src/Storages/Transaction/RegionBlockReader.cpp index 32be7302775..af351f4a6b0 100644 --- a/dbms/src/Storages/Transaction/RegionBlockReader.cpp +++ b/dbms/src/Storages/Transaction/RegionBlockReader.cpp @@ -58,7 +58,7 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d const auto & pk_column_ids = schema_snapshot->pk_column_ids; const auto & pk_pos_map = schema_snapshot->pk_pos_map; - SortedColumnIDWithPosConstIter column_ids_iter = read_column_ids.begin(); + auto column_ids_iter = read_column_ids.begin(); size_t next_column_pos = 0; /// every table in tiflash must have an extra handle column, it either @@ -112,25 +112,6 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d size_t index = 0; for (const auto & [pk, write_type, commit_ts, value_ptr] : data_list) { - // Ignore data after the start_ts. - if (commit_ts > start_ts) - continue; - - bool should_skip = false; - if constexpr (pk_type != TMTPKType::STRING) - { - if constexpr (pk_type == TMTPKType::UINT64) - { - should_skip = scan_filter != nullptr && scan_filter->filter(static_cast(pk)); - } - else - { - should_skip = scan_filter != nullptr && scan_filter->filter(static_cast(pk)); - } - } - if (should_skip) - continue; - /// set delmark and version column delmark_data.emplace_back(write_type == Region::DelFlag); version_data.emplace_back(commit_ts); @@ -186,7 +167,7 @@ bool RegionBlockReader::readImpl(Block & block, const RegionDataReadInfoList & d { // The pk_type must be Int32/Uint32 or more narrow type // so cannot tell its' exact type here, just use `insert(Field)` - HandleID handle_value(static_cast(pk)); + auto handle_value(static_cast(pk)); raw_pk_column->insert(Field(handle_value)); if (unlikely(raw_pk_column->getInt(index) != handle_value)) { diff --git a/dbms/src/Storages/Transaction/RegionBlockReader.h b/dbms/src/Storages/Transaction/RegionBlockReader.h index 860e0d149e6..ec633e805c0 100644 --- a/dbms/src/Storages/Transaction/RegionBlockReader.h +++ b/dbms/src/Storages/Transaction/RegionBlockReader.h @@ -37,79 +37,12 @@ using ManageableStoragePtr = std::shared_ptr; struct ColumnsDescription; class Block; -class RegionScanFilter -{ - bool is_full_range_scan; - std::vector> int64_ranges; - std::vector> uint64_ranges; - - bool isValidHandle(UInt64 handle) - { - for (const auto & range : uint64_ranges) - { - if (handle >= range.first && handle < range.second) - { - return true; - } - } - return false; - } - bool isValidHandle(Int64 handle) - { - for (const auto & range : int64_ranges) - { - if (handle >= range.first && handle < range.second) - { - return true; - } - } - return false; - } - -public: - RegionScanFilter( - bool is_full_range_scan_, - std::vector> int64_ranges_, - std::vector> uint64_ranges_) - : is_full_range_scan(is_full_range_scan_) - , int64_ranges(std::move(int64_ranges_)) - , uint64_ranges(std::move(uint64_ranges_)) - {} - bool filter(UInt64 handle) { return !is_full_range_scan && !isValidHandle(handle); } - bool filter(Int64 handle) { return !is_full_range_scan && !isValidHandle(handle); } - bool isFullRangeScan() { return is_full_range_scan; } - const std::vector> & getUInt64Ranges() { return uint64_ranges; } - const std::vector> & getInt64Ranges() { return int64_ranges; } -}; - -using RegionScanFilterPtr = std::shared_ptr; - /// The Reader to read the region data in `data_list` and decode based on the given table_info and columns, as a block. class RegionBlockReader : private boost::noncopyable { - RegionScanFilterPtr scan_filter; - Timestamp start_ts = std::numeric_limits::max(); - public: RegionBlockReader(DecodingStorageSchemaSnapshotConstPtr schema_snapshot_); - inline RegionBlockReader & setFilter(RegionScanFilterPtr filter) - { - scan_filter = std::move(filter); - return *this; - } - - /// Set the `start_ts` for reading data. The `start_ts` is `Timestamp::max` if not set. - /// - /// Data with commit_ts > start_ts will be ignored. This is for the sake of decode safety on read, - /// i.e. as data keeps being synced to region cache while the schema for a specific read is fixed, - /// we'll always have newer data than schema, only ignoring them can guarantee the decode safety. - inline RegionBlockReader & setStartTs(Timestamp tso) - { - start_ts = tso; - return *this; - } - /// Read `data_list` as a block. /// /// On decode error, i.e. column number/type mismatch, will do force apply schema, @@ -117,7 +50,7 @@ class RegionBlockReader : private boost::noncopyable /// Moreover, exception will be thrown if we see fatal decode error meanwhile `force_decode` is true. /// /// `RegionBlockReader::read` is the common routine used by both 'flush' and 'read' processes of TXN engine (Delta-Tree, TXN-MergeTree), - /// each of which will use carefully adjusted 'start_ts' and 'force_decode' with appropriate error handling/retry to get what they want. + /// each of which will use carefully adjusted 'force_decode' with appropriate error handling/retry to get what they want. bool read(Block & block, const RegionDataReadInfoList & data_list, bool force_decode); private: diff --git a/dbms/src/Storages/Transaction/RegionTable.h b/dbms/src/Storages/Transaction/RegionTable.h index b30a905541a..717b1cd568f 100644 --- a/dbms/src/Storages/Transaction/RegionTable.h +++ b/dbms/src/Storages/Transaction/RegionTable.h @@ -146,21 +146,6 @@ class RegionTable : private boost::noncopyable Poco::Logger * log, bool lock_region = true); - /// Read the data of the given region into block, take good care of learner read and locks. - /// Assuming that the schema has been properly synced by outer, i.e. being new enough to decode data before start_ts, - /// we directly ask RegionBlockReader::read to perform a read with the given start_ts and force_decode being true. - using ReadBlockByRegionRes = std::variant; - static ReadBlockByRegionRes readBlockByRegion(const TiDB::TableInfo & table_info, - const ColumnsDescription & columns, - const Names & column_names_to_read, - const RegionPtr & region, - RegionVersion region_version, - RegionVersion conf_version, - bool resolve_locks, - Timestamp start_ts, - const std::unordered_set * bypass_lock_ts, - RegionScanFilterPtr scan_filter = nullptr); - /// Check transaction locks in region, and write committed data in it into storage engine if check passed. Otherwise throw an LockException. /// The write logic is the same as #writeBlockByRegion, with some extra checks about region version and conf_version. using ResolveLocksAndWriteRegionRes = std::variant; diff --git a/dbms/src/Storages/Transaction/StorageEngineType.h b/dbms/src/Storages/Transaction/StorageEngineType.h index f202d15a769..3d103ca60c1 100644 --- a/dbms/src/Storages/Transaction/StorageEngineType.h +++ b/dbms/src/Storages/Transaction/StorageEngineType.h @@ -33,7 +33,7 @@ enum class StorageEngine enum class SnapshotApplyMethod : std::int32_t { - Block = 1, + DEPRECATED_Block = 1, // Invalid if the storage engine is not DeltaTree DTFile_Directory, DTFile_Single, @@ -43,14 +43,12 @@ inline const std::string applyMethodToString(SnapshotApplyMethod method) { switch (method) { - case SnapshotApplyMethod::Block: - return "block"; - case SnapshotApplyMethod::DTFile_Directory: - return "file1"; - case SnapshotApplyMethod::DTFile_Single: - return "file2"; - default: - return "unknown(" + std::to_string(static_cast(method)) + ")"; + case SnapshotApplyMethod::DTFile_Directory: + return "file1"; + case SnapshotApplyMethod::DTFile_Single: + return "file2"; + default: + return "unknown(" + std::to_string(static_cast(method)) + ")"; } return "unknown"; } diff --git a/dbms/src/Storages/Transaction/TMTContext.cpp b/dbms/src/Storages/Transaction/TMTContext.cpp index 719784edaf2..3c7468cbd64 100644 --- a/dbms/src/Storages/Transaction/TMTContext.cpp +++ b/dbms/src/Storages/Transaction/TMTContext.cpp @@ -36,6 +36,8 @@ extern const uint64_t DEFAULT_WAIT_INDEX_TIMEOUT_MS = 5 * 60 * 1000; const int64_t DEFAULT_WAIT_REGION_READY_TIMEOUT_SEC = 20 * 60; +const int64_t DEFAULT_READ_INDEX_WORKER_TICK_MS = 10; + TMTContext::TMTContext(Context & context_, const TiFlashRaftConfig & raft_config, const pingcap::ClusterConfig & cluster_config) : context(context_) , kvstore(std::make_shared(context, raft_config.snapshot_apply_method)) @@ -56,7 +58,7 @@ TMTContext::TMTContext(Context & context_, const TiFlashRaftConfig & raft_config , replica_read_max_thread(1) , batch_read_index_timeout_ms(DEFAULT_BATCH_READ_INDEX_TIMEOUT_MS) , wait_index_timeout_ms(DEFAULT_WAIT_INDEX_TIMEOUT_MS) - , read_index_worker_tick_ms(10) + , read_index_worker_tick_ms(DEFAULT_READ_INDEX_WORKER_TICK_MS) , wait_region_ready_timeout_sec(DEFAULT_WAIT_REGION_READY_TIMEOUT_SEC) {} @@ -149,12 +151,6 @@ SchemaSyncerPtr TMTContext::getSchemaSyncer() const return schema_syncer; } -void TMTContext::setSchemaSyncer(SchemaSyncerPtr rhs) -{ - std::lock_guard lock(mutex); - schema_syncer = rhs; -} - pingcap::pd::ClientPtr TMTContext::getPDClient() const { return cluster->pd_client; @@ -194,7 +190,7 @@ void TMTContext::reloadConfig(const Poco::Util::AbstractConfiguration & config) t = t >= 0 ? t : std::numeric_limits::max(); // set -1 to wait infinitely t; }); - read_index_worker_tick_ms = config.getUInt64(READ_INDEX_WORKER_TICK_MS, 10 /*10ms*/); + read_index_worker_tick_ms = config.getUInt64(READ_INDEX_WORKER_TICK_MS, DEFAULT_READ_INDEX_WORKER_TICK_MS); } { LOG_FMT_INFO( diff --git a/dbms/src/Storages/Transaction/TMTContext.h b/dbms/src/Storages/Transaction/TMTContext.h index 27e0482b787..8e26c0da88c 100644 --- a/dbms/src/Storages/Transaction/TMTContext.h +++ b/dbms/src/Storages/Transaction/TMTContext.h @@ -34,15 +34,9 @@ using SchemaSyncerPtr = std::shared_ptr; class BackgroundService; using BackGroundServicePtr = std::unique_ptr; -class MinTSOScheduler; -using MPPTaskSchedulerPtr = std::unique_ptr; - class MPPTaskManager; using MPPTaskManagerPtr = std::shared_ptr; -struct MPPQueryTaskSet; -using MPPQueryTaskSetPtr = std::shared_ptr; - class GCManager; using GCManagerPtr = std::shared_ptr; @@ -84,7 +78,6 @@ class TMTContext : private boost::noncopyable explicit TMTContext(Context & context_, const TiFlashRaftConfig & raft_config, const pingcap::ClusterConfig & cluster_config_); SchemaSyncerPtr getSchemaSyncer() const; - void setSchemaSyncer(SchemaSyncerPtr); pingcap::pd::ClientPtr getPDClient() const; diff --git a/dbms/src/Storages/Transaction/tests/RowCodecTestUtils.h b/dbms/src/Storages/Transaction/tests/RowCodecTestUtils.h index c28ea531afe..20b395a9952 100644 --- a/dbms/src/Storages/Transaction/tests/RowCodecTestUtils.h +++ b/dbms/src/Storages/Transaction/tests/RowCodecTestUtils.h @@ -285,11 +285,11 @@ inline DecodingStorageSchemaSnapshotConstPtr getDecodingStorageSchemaSnapshot(co if (handle_id != EXTRA_HANDLE_COLUMN_ID) { auto iter = std::find_if(store_columns.begin(), store_columns.end(), [&](const ColumnDefine & cd) { return cd.id == handle_id; }); - return std::make_shared(std::make_shared(store_columns), table_info, *iter); + return std::make_shared(std::make_shared(store_columns), table_info, *iter, /* decoding_schema_version_ */ 1); } else { - return std::make_shared(std::make_shared(store_columns), table_info, store_columns[0]); + return std::make_shared(std::make_shared(store_columns), table_info, store_columns[0], /* decoding_schema_version_ */ 1); } } diff --git a/dbms/src/Storages/Transaction/tests/gtest_kvstore.cpp b/dbms/src/Storages/Transaction/tests/gtest_kvstore.cpp index e93a117cc1c..f0cafce3914 100644 --- a/dbms/src/Storages/Transaction/tests/gtest_kvstore.cpp +++ b/dbms/src/Storages/Transaction/tests/gtest_kvstore.cpp @@ -47,8 +47,7 @@ RegionPtr makeRegion(UInt64 id, const std::string start_key, const std::string e class RegionKVStoreTest : public ::testing::Test { public: - RegionKVStoreTest() - = default; + RegionKVStoreTest() = default; static void SetUpTestCase() {} static void testBasic(); @@ -311,10 +310,13 @@ void RegionKVStoreTest::testRaftMergeRollback(KVStore & kvs, TMTContext & tmt) } } region->setStateApplying(); + try { - kvs.handleAdminRaftCmd(std::move(request), - std::move(response), + raft_cmdpb::AdminRequest first_request = request; + raft_cmdpb::AdminResponse first_response = response; + kvs.handleAdminRaftCmd(std::move(first_request), + std::move(first_response), region_id, 32, 6, @@ -926,12 +928,14 @@ void RegionKVStoreTest::testKVStore() TiKVValue lock_value = RecordKVFormat::encodeLockCfValue(Region::DelFlag, "pk", 77, 0); RegionBench::setupDelRequest(request.add_requests(), ColumnFamilyName::Lock, lock_key); } - ASSERT_EQ(kvs.handleWriteRaftCmd(std::move(request), 1, 7, 6, ctx.getTMTContext()), + raft_cmdpb::RaftCmdRequest first_request = request; + ASSERT_EQ(kvs.handleWriteRaftCmd(std::move(first_request), 1, 7, 6, ctx.getTMTContext()), EngineStoreApplyRes::None); RegionBench::setupDelRequest(request.add_requests(), ColumnFamilyName::Write, TiKVKey("illegal key")); // index <= appliedIndex(), ignore - ASSERT_EQ(kvs.handleWriteRaftCmd(std::move(request), 1, 7, 6, ctx.getTMTContext()), + raft_cmdpb::RaftCmdRequest second_request; + ASSERT_EQ(kvs.handleWriteRaftCmd(std::move(second_request), 1, 7, 6, ctx.getTMTContext()), EngineStoreApplyRes::None); try { @@ -973,13 +977,24 @@ void RegionKVStoreTest::testKVStore() request.mutable_compact_log(); request.set_cmd_type(::raft_cmdpb::AdminCmdType::CompactLog); - ASSERT_EQ(kvs.handleAdminRaftCmd(std::move(request), std::move(response), 7, 22, 6, ctx.getTMTContext()), EngineStoreApplyRes::Persist); - ASSERT_EQ(kvs.handleAdminRaftCmd(raft_cmdpb::AdminRequest{request}, std::move(response), 7, 23, 6, ctx.getTMTContext()), EngineStoreApplyRes::None); + raft_cmdpb::AdminRequest first_request = request; + raft_cmdpb::AdminResponse first_response = response; + + ASSERT_EQ(kvs.handleAdminRaftCmd(std::move(first_request), std::move(first_response), 7, 22, 6, ctx.getTMTContext()), EngineStoreApplyRes::Persist); + + raft_cmdpb::AdminResponse second_response = response; + ASSERT_EQ(kvs.handleAdminRaftCmd(raft_cmdpb::AdminRequest{request}, std::move(second_response), 7, 23, 6, ctx.getTMTContext()), EngineStoreApplyRes::None); request.set_cmd_type(::raft_cmdpb::AdminCmdType::ComputeHash); - ASSERT_EQ(kvs.handleAdminRaftCmd(raft_cmdpb::AdminRequest{request}, std::move(response), 7, 24, 6, ctx.getTMTContext()), EngineStoreApplyRes::None); + + raft_cmdpb::AdminResponse third_response = response; + ASSERT_EQ(kvs.handleAdminRaftCmd(raft_cmdpb::AdminRequest{request}, std::move(third_response), 7, 24, 6, ctx.getTMTContext()), EngineStoreApplyRes::None); request.set_cmd_type(::raft_cmdpb::AdminCmdType::VerifyHash); - ASSERT_EQ(kvs.handleAdminRaftCmd(raft_cmdpb::AdminRequest{request}, std::move(response), 7, 25, 6, ctx.getTMTContext()), EngineStoreApplyRes::None); - ASSERT_EQ(kvs.handleAdminRaftCmd(raft_cmdpb::AdminRequest{request}, std::move(response), 8192, 5, 6, ctx.getTMTContext()), EngineStoreApplyRes::NotFound); + + raft_cmdpb::AdminResponse fourth_response = response; + ASSERT_EQ(kvs.handleAdminRaftCmd(raft_cmdpb::AdminRequest{request}, std::move(fourth_response), 7, 25, 6, ctx.getTMTContext()), EngineStoreApplyRes::None); + + raft_cmdpb::AdminResponse fifth_response = response; + ASSERT_EQ(kvs.handleAdminRaftCmd(raft_cmdpb::AdminRequest{request}, std::move(fifth_response), 8192, 5, 6, ctx.getTMTContext()), EngineStoreApplyRes::NotFound); { kvs.setRegionCompactLogConfig(0, 0, 0); request.set_cmd_type(::raft_cmdpb::AdminCmdType::CompactLog); @@ -995,62 +1010,12 @@ void RegionKVStoreTest::testKVStore() } { auto ori_snapshot_apply_method = kvs.snapshot_apply_method; - kvs.snapshot_apply_method = TiDB::SnapshotApplyMethod::Block; + kvs.snapshot_apply_method = TiDB::SnapshotApplyMethod::DTFile_Single; SCOPE_EXIT({ kvs.snapshot_apply_method = ori_snapshot_apply_method; }); - { - { - auto region = makeRegion(22, RecordKVFormat::genKey(1, 55), RecordKVFormat::genKey(1, 65)); - kvs.checkAndApplySnapshot(region, ctx.getTMTContext()); - } - try - { - auto region = makeRegion(20, RecordKVFormat::genKey(1, 55), RecordKVFormat::genKey(1, 65)); - kvs.checkAndApplySnapshot(region, ctx.getTMTContext()); // overlap, but not tombstone - ASSERT_TRUE(false); - } - catch (Exception & e) - { - ASSERT_EQ(e.message(), "range of region 20 is overlapped with 22, state: region { id: 22 }"); - } - { - const auto * ori_ptr = proxy_helper.proxy_ptr.inner; - proxy_helper.proxy_ptr.inner = nullptr; - SCOPE_EXIT({ - proxy_helper.proxy_ptr.inner = ori_ptr; - }); - try - { - auto region = makeRegion(20, RecordKVFormat::genKey(1, 55), RecordKVFormat::genKey(1, 65)); - kvs.checkAndApplySnapshot(region, ctx.getTMTContext()); - ASSERT_TRUE(false); - } - catch (Exception & e) - { - ASSERT_EQ(e.message(), "getRegionLocalState meet internal error: RaftStoreProxyPtr is none"); - } - } - - { - proxy_instance.getRegion(22)->setSate(({ - raft_serverpb::RegionLocalState s; - s.set_state(::raft_serverpb::PeerState::Tombstone); - s; - })); - auto region = makeRegion(20, RecordKVFormat::genKey(1, 55), RecordKVFormat::genKey(1, 65)); - kvs.checkAndApplySnapshot(region, ctx.getTMTContext()); // overlap, tombstone, remove previous one - ASSERT_EQ(nullptr, kvs.getRegion(22)); - ASSERT_NE(nullptr, kvs.getRegion(20)); - - auto state = proxy_helper.getRegionLocalState(8192); - ASSERT_EQ(state.state(), raft_serverpb::PeerState::Tombstone); - } - - kvs.handleDestroy(20, ctx.getTMTContext()); - } auto region_id = 19; auto region = makeRegion(region_id, RecordKVFormat::genKey(1, 50), RecordKVFormat::genKey(1, 60)); auto region_id_str = std::to_string(19); @@ -1077,7 +1042,7 @@ void RegionKVStoreTest::testKVStore() 8, 5, ctx.getTMTContext()); - ASSERT_EQ(kvs.getRegion(19)->dataInfo(), "[default 2 ]"); + ASSERT_EQ(kvs.getRegion(19)->checkIndex(8), true); try { kvs.handleApplySnapshot( @@ -1092,22 +1057,85 @@ void RegionKVStoreTest::testKVStore() catch (Exception & e) { ASSERT_EQ(e.message(), "[region 19] already has newer apply-index 8 than 6, should not happen"); - ASSERT_EQ(kvs.getRegion(19)->dataInfo(), "[default 2 ]"); // apply-snapshot do not work } - kvs.handleApplySnapshot( - region->getMetaRegion(), - 2, - {}, // empty - 8, // same index - 5, - ctx.getTMTContext()); - ASSERT_EQ(kvs.getRegion(19)->dataInfo(), "[default 2 ]"); // apply-snapshot do not work - region = makeRegion(19, RecordKVFormat::genKey(1, 50), RecordKVFormat::genKey(1, 60)); - region->handleWriteRaftCmd({}, 10, 10, ctx.getTMTContext()); - kvs.checkAndApplySnapshot(region, ctx.getTMTContext()); - ASSERT_EQ(kvs.getRegion(19)->dataInfo(), "[]"); + } + + { + { + auto region = makeRegion(22, RecordKVFormat::genKey(55, 50), RecordKVFormat::genKey(55, 100)); + auto ingest_ids = kvs.preHandleSnapshotToFiles( + region, + {}, + 9, + 5, + ctx.getTMTContext()); + kvs.checkAndApplySnapshot(RegionPtrWithSnapshotFiles{region, std::move(ingest_ids)}, ctx.getTMTContext()); + } + try + { + auto region = makeRegion(20, RecordKVFormat::genKey(55, 50), RecordKVFormat::genKey(55, 100)); + auto ingest_ids = kvs.preHandleSnapshotToFiles( + region, + {}, + 9, + 5, + ctx.getTMTContext()); + kvs.checkAndApplySnapshot(RegionPtrWithSnapshotFiles{region, std::move(ingest_ids)}, ctx.getTMTContext()); // overlap, but not tombstone + ASSERT_TRUE(false); + } + catch (Exception & e) + { + ASSERT_EQ(e.message(), "range of region 20 is overlapped with 22, state: region { id: 22 }"); + } + + { + const auto * ori_ptr = proxy_helper.proxy_ptr.inner; + proxy_helper.proxy_ptr.inner = nullptr; + SCOPE_EXIT({ + proxy_helper.proxy_ptr.inner = ori_ptr; + }); + + try + { + auto region = makeRegion(20, RecordKVFormat::genKey(55, 50), RecordKVFormat::genKey(55, 100)); + auto ingest_ids = kvs.preHandleSnapshotToFiles( + region, + {}, + 10, + 5, + ctx.getTMTContext()); + kvs.checkAndApplySnapshot(RegionPtrWithSnapshotFiles{region, std::move(ingest_ids)}, ctx.getTMTContext()); + ASSERT_TRUE(false); + } + catch (Exception & e) + { + ASSERT_EQ(e.message(), "getRegionLocalState meet internal error: RaftStoreProxyPtr is none"); + } + } + + { + proxy_instance.getRegion(22)->setSate(({ + raft_serverpb::RegionLocalState s; + s.set_state(::raft_serverpb::PeerState::Tombstone); + s; + })); + auto region = makeRegion(20, RecordKVFormat::genKey(55, 50), RecordKVFormat::genKey(55, 100)); + auto ingest_ids = kvs.preHandleSnapshotToFiles( + region, + {}, + 10, + 5, + ctx.getTMTContext()); + kvs.checkAndApplySnapshot(RegionPtrWithSnapshotFiles{region, std::move(ingest_ids)}, ctx.getTMTContext()); // overlap, tombstone, remove previous one + + auto state = proxy_helper.getRegionLocalState(8192); + ASSERT_EQ(state.state(), raft_serverpb::PeerState::Tombstone); + } + + kvs.handleDestroy(20, ctx.getTMTContext()); } } + { auto region_id = 19; auto region_id_str = std::to_string(19); @@ -1126,11 +1154,6 @@ void RegionKVStoreTest::testKVStore() RegionMockTest mock_test(ctx.getTMTContext().getKVStore(), region); { - auto ori_snapshot_apply_method = kvs.snapshot_apply_method; - kvs.snapshot_apply_method = TiDB::SnapshotApplyMethod::Block; - SCOPE_EXIT({ - kvs.snapshot_apply_method = ori_snapshot_apply_method; - }); // Mocking ingest a SST for column family "Write" std::vector sst_views; sst_views.push_back(SSTView{ @@ -1143,9 +1166,10 @@ void RegionKVStoreTest::testKVStore() 100, 1, ctx.getTMTContext()); - ASSERT_EQ(kvs.getRegion(19)->dataInfo(), "[default 2 ]"); + ASSERT_EQ(kvs.getRegion(19)->checkIndex(100), true); } } + { raft_cmdpb::AdminRequest request; raft_cmdpb::AdminResponse response; @@ -1155,7 +1179,7 @@ void RegionKVStoreTest::testKVStore() try { - kvs.handleAdminRaftCmd(std::move(request), std::move(response), 19, 110, 6, ctx.getTMTContext()); + kvs.handleAdminRaftCmd(std::move(request), std::move(response), 1, 110, 6, ctx.getTMTContext()); ASSERT_TRUE(false); } catch (Exception & e) @@ -1372,12 +1396,30 @@ void RegionKVStoreTest::testBasic() } } -TEST_F(RegionKVStoreTest, run) +TEST_F(RegionKVStoreTest, Basic) try { testBasic(); +} +CATCH + +TEST_F(RegionKVStoreTest, KVStore) +try +{ testKVStore(); +} +CATCH + +TEST_F(RegionKVStoreTest, Region) +try +{ testRegion(); +} +CATCH + +TEST_F(RegionKVStoreTest, ReadIndex) +try +{ testReadIndex(); } CATCH diff --git a/dbms/src/Storages/registerStorages.cpp b/dbms/src/Storages/registerStorages.cpp index a709be0b017..ddf815316ab 100644 --- a/dbms/src/Storages/registerStorages.cpp +++ b/dbms/src/Storages/registerStorages.cpp @@ -27,7 +27,6 @@ void registerStorageNull(StorageFactory & factory); void registerStorageMerge(StorageFactory & factory); void registerStorageBuffer(StorageFactory & factory); void registerStorageMemory(StorageFactory & factory); -void registerStorageFile(StorageFactory & factory); void registerStorageDictionary(StorageFactory & factory); void registerStorageSet(StorageFactory & factory); void registerStorageJoin(StorageFactory & factory); @@ -47,7 +46,6 @@ void registerStorages() registerStorageMerge(factory); registerStorageBuffer(factory); registerStorageMemory(factory); - registerStorageFile(factory); registerStorageDictionary(factory); registerStorageSet(factory); registerStorageJoin(factory); diff --git a/dbms/src/Storages/tests/gtest_filter_parser.cpp b/dbms/src/Storages/tests/gtest_filter_parser.cpp index 8820c05d2da..3a554fcf4b6 100644 --- a/dbms/src/Storages/tests/gtest_filter_parser.cpp +++ b/dbms/src/Storages/tests/gtest_filter_parser.cpp @@ -98,7 +98,7 @@ DM::RSOperatorPtr FilterParserTest::generateRsOperator(const String table_info_j DM::ColumnDefines columns_to_read; { NamesAndTypes source_columns; - std::tie(source_columns, std::ignore) = parseColumnsFromTableInfo(table_info, log->getLog()); + std::tie(source_columns, std::ignore) = parseColumnsFromTableInfo(table_info); dag_query = std::make_unique( conditions, DAGPreparedSets(), diff --git a/dbms/src/TableFunctions/TableFunctionCatBoostPool.cpp b/dbms/src/TableFunctions/TableFunctionCatBoostPool.cpp deleted file mode 100644 index ab5cd7e5849..00000000000 --- a/dbms/src/TableFunctions/TableFunctionCatBoostPool.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ -extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -extern const int BAD_ARGUMENTS; -} // namespace ErrorCodes - - -StoragePtr TableFunctionCatBoostPool::executeImpl(const ASTPtr & ast_function, const Context & context) const -{ - ASTs & args_func = typeid_cast(*ast_function).children; - - std::string err = "Table function '" + getName() + "' requires 2 parameters: " - + "column descriptions file, dataset description file"; - - if (args_func.size() != 1) - throw Exception(err, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - ASTs & args = typeid_cast(*args_func.at(0)).children; - - if (args.size() != 2) - throw Exception(err, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - auto getStringLiteral = [](const IAST & node, const char * description) { - auto lit = typeid_cast(&node); - if (!lit) - throw Exception(description + String(" must be string literal (in single quotes)."), ErrorCodes::BAD_ARGUMENTS); - - if (lit->value.getType() != Field::Types::String) - throw Exception(description + String(" must be string literal (in single quotes)."), ErrorCodes::BAD_ARGUMENTS); - - return safeGet(lit->value); - }; - String column_descriptions_file = getStringLiteral(*args[0], "Column descriptions file"); - String dataset_description_file = getStringLiteral(*args[1], "Dataset description file"); - - return StorageCatBoostPool::create(context, column_descriptions_file, dataset_description_file); -} - -void registerTableFunctionCatBoostPool(TableFunctionFactory & factory) -{ - factory.registerFunction(); -} - -} // namespace DB diff --git a/dbms/src/TableFunctions/TableFunctionFile.cpp b/dbms/src/TableFunctions/TableFunctionFile.cpp deleted file mode 100644 index 0ff1a5b443f..00000000000 --- a/dbms/src/TableFunctions/TableFunctionFile.cpp +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace DB -{ -namespace ErrorCodes -{ -extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -extern const int DATABASE_ACCESS_DENIED; -} // namespace ErrorCodes - -StoragePtr TableFunctionFile::executeImpl(const ASTPtr & ast_function, const Context & context) const -{ - // Parse args - ASTs & args_func = typeid_cast(*ast_function).children; - - if (args_func.size() != 1) - throw Exception("Table function '" + getName() + "' must have arguments.", ErrorCodes::LOGICAL_ERROR); - - ASTs & args = typeid_cast(*args_func.at(0)).children; - - if (args.size() != 3) - throw Exception("Table function '" + getName() + "' requires exactly 3 arguments: path, format and structure.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - for (size_t i = 0; i < 3; ++i) - args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(args[i], context); - - std::string path = static_cast(*args[0]).value.safeGet(); - std::string format = static_cast(*args[1]).value.safeGet(); - std::string structure = static_cast(*args[2]).value.safeGet(); - - // Create sample block - std::vector structure_vals; - boost::split(structure_vals, structure, boost::algorithm::is_any_of(" ,"), boost::algorithm::token_compress_on); - - if (structure_vals.size() % 2 != 0) - throw Exception("Odd number of elements in section structure: must be a list of name type pairs", ErrorCodes::LOGICAL_ERROR); - - Block sample_block; - const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); - - for (size_t i = 0, size = structure_vals.size(); i < size; i += 2) - { - ColumnWithTypeAndName column; - column.name = structure_vals[i]; - column.type = data_type_factory.get(structure_vals[i + 1]); - column.column = column.type->createColumn(); - sample_block.insert(std::move(column)); - } - - // Create table - StoragePtr storage = StorageFile::create( - path, - -1, - context.getUserFilesPath(), - getName(), - format, - ColumnsDescription{sample_block.getNamesAndTypesList()}, - const_cast(context)); - - storage->startup(); - - return storage; -} - - -void registerTableFunctionFile(TableFunctionFactory & factory) -{ - factory.registerFunction(); -} - -} // namespace DB diff --git a/dbms/src/TableFunctions/registerTableFunctions.cpp b/dbms/src/TableFunctions/registerTableFunctions.cpp index 2eac0ce0548..8449077cc96 100644 --- a/dbms/src/TableFunctions/registerTableFunctions.cpp +++ b/dbms/src/TableFunctions/registerTableFunctions.cpp @@ -21,16 +21,13 @@ namespace DB { void registerTableFunctionMerge(TableFunctionFactory & factory); void registerTableFunctionNumbers(TableFunctionFactory & factory); -void registerTableFunctionCatBoostPool(TableFunctionFactory & factory); -void registerTableFunctionFile(TableFunctionFactory & factory); + void registerTableFunctions() { auto & factory = TableFunctionFactory::instance(); registerTableFunctionMerge(factory); registerTableFunctionNumbers(factory); - registerTableFunctionCatBoostPool(factory); - registerTableFunctionFile(factory); } } // namespace DB diff --git a/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp b/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp index 2c3bf243176..ea19ff08dd3 100644 --- a/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp +++ b/dbms/src/TestUtils/ColumnsToTiPBExpr.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include namespace DB @@ -35,6 +36,7 @@ void columnToTiPBExpr(tipb::Expr * expr, const ColumnWithTypeAndName column, siz if (column.column->isColumnNullable()) { auto [col, null_map] = removeNullable(column.column.get()); + (void)null_map; is_const = col->isColumnConst(); } } @@ -96,6 +98,7 @@ void columnsToTiPBExprForTiDBCast( if (type_column.column->isColumnNullable()) { auto [col, null_map] = removeNullable(type_column.column.get()); + (void)null_map; is_const = col->isColumnConst(); } } diff --git a/dbms/src/TestUtils/ExecutorTestUtils.cpp b/dbms/src/TestUtils/ExecutorTestUtils.cpp new file mode 100644 index 00000000000..881ebaf88db --- /dev/null +++ b/dbms/src/TestUtils/ExecutorTestUtils.cpp @@ -0,0 +1,149 @@ +// Copyright 2022 PingCAP, Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +#include + +namespace DB::tests +{ +DAGContext & ExecutorTest::getDAGContext() +{ + assert(dag_context_ptr != nullptr); + return *dag_context_ptr; +} + +void ExecutorTest::initializeContext() +{ + dag_context_ptr = std::make_unique(1024); + context = MockDAGRequestContext(TiFlashTestEnv::getContext()); + dag_context_ptr->log = Logger::get("executorTest"); +} + +void ExecutorTest::SetUpTestCase() +{ + auto register_func = [](std::function func) { + try + { + func(); + } + catch (DB::Exception &) + { + // Maybe another test has already registered, ignore exception here. + } + }; + + register_func(DB::registerFunctions); + register_func(DB::registerAggregateFunctions); + register_func(DB::registerWindowFunctions); +} + +void ExecutorTest::initializeClientInfo() +{ + context.context.setCurrentQueryId("test"); + ClientInfo & client_info = context.context.getClientInfo(); + client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + client_info.interface = ClientInfo::Interface::GRPC; +} + +void ExecutorTest::executeInterpreter(const String & expected_string, const std::shared_ptr & request, size_t concurrency) +{ + DAGContext dag_context(*request, "interpreter_test", concurrency); + context.context.setDAGContext(&dag_context); + // Currently, don't care about regions information in interpreter tests. + DAGQuerySource dag(context.context); + auto res = executeQuery(dag, context.context, false, QueryProcessingStage::Complete); + FmtBuffer fb; + res.in->dumpTree(fb); + ASSERT_EQ(Poco::trim(expected_string), Poco::trim(fb.toString())); +} + +namespace +{ +Block mergeBlocks(Blocks blocks) +{ + if (blocks.empty()) + return {}; + + Block sample_block = blocks.back(); + std::vector actual_cols; + for (const auto & column : sample_block.getColumnsWithTypeAndName()) + { + actual_cols.push_back(column.type->createColumn()); + } + for (const auto & block : blocks) + { + for (size_t i = 0; i < block.columns(); ++i) + { + for (size_t j = 0; j < block.rows(); ++j) + { + actual_cols[i]->insert((*(block.getColumnsWithTypeAndName())[i].column)[j]); + } + } + } + + ColumnsWithTypeAndName actual_columns; + for (size_t i = 0; i < actual_cols.size(); ++i) + actual_columns.push_back({std::move(actual_cols[i]), sample_block.getColumnsWithTypeAndName()[i].type, sample_block.getColumnsWithTypeAndName()[i].name, sample_block.getColumnsWithTypeAndName()[i].column_id}); + return Block(actual_columns); +} + +void readBlock(BlockInputStreamPtr stream, const ColumnsWithTypeAndName & expect_columns) +{ + Blocks actual_blocks; + Block except_block(expect_columns); + stream->readPrefix(); + while (auto block = stream->read()) + { + actual_blocks.push_back(block); + } + stream->readSuffix(); + Block actual_block = mergeBlocks(actual_blocks); + ASSERT_BLOCK_EQ(except_block, actual_block); +} +} // namespace + +void ExecutorTest::executeStreams(const std::shared_ptr & request, std::unordered_map & source_columns_map, const ColumnsWithTypeAndName & expect_columns, size_t concurrency) +{ + DAGContext dag_context(*request, "executor_test", concurrency); + dag_context.setColumnsForTest(source_columns_map); + context.context.setDAGContext(&dag_context); + // Currently, don't care about regions information in tests. + DAGQuerySource dag(context.context); + readBlock(executeQuery(dag, context.context, false, QueryProcessingStage::Complete).in, expect_columns); +} + +void ExecutorTest::executeStreams(const std::shared_ptr & request, const ColumnsWithTypeAndName & expect_columns, size_t concurrency) +{ + executeStreams(request, context.executorIdColumnsMap(), expect_columns, concurrency); +} + +void ExecutorTest::executeStreamsWithSingleSource(const std::shared_ptr & request, const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & expect_columns, SourceType type, size_t concurrency) +{ + std::unordered_map source_columns_map; + source_columns_map[getSourceName(type)] = source_columns; + executeStreams(request, source_columns_map, expect_columns, concurrency); +} + +void ExecutorTest::dagRequestEqual(const String & expected_string, const std::shared_ptr & actual) +{ + ASSERT_EQ(Poco::trim(expected_string), Poco::trim(ExecutorSerializer().serialize(actual.get()))); +} + +} // namespace DB::tests diff --git a/dbms/src/TestUtils/InterpreterTestUtils.h b/dbms/src/TestUtils/ExecutorTestUtils.h similarity index 57% rename from dbms/src/TestUtils/InterpreterTestUtils.h rename to dbms/src/TestUtils/ExecutorTestUtils.h index 28d44d3a5f2..87bb7115bed 100644 --- a/dbms/src/TestUtils/InterpreterTestUtils.h +++ b/dbms/src/TestUtils/ExecutorTestUtils.h @@ -15,19 +15,17 @@ #pragma once #include -#include -#include #include #include #include -#include -#include #include #include +#include + namespace DB::tests { void executeInterpreter(const std::shared_ptr & request, Context & context); -class InterpreterTest : public ::testing::Test +class ExecutorTest : public ::testing::Test { protected: void SetUp() override @@ -37,7 +35,7 @@ class InterpreterTest : public ::testing::Test } public: - InterpreterTest() + ExecutorTest() : context(TiFlashTestEnv::getContext()) {} static void SetUpTestCase(); @@ -52,6 +50,45 @@ class InterpreterTest : public ::testing::Test void executeInterpreter(const String & expected_string, const std::shared_ptr & request, size_t concurrency); + enum SourceType + { + TableScan, + ExchangeReceiver + }; + + // for single source query, the source executor name is ${type}_0 + static String getSourceName(SourceType type) + { + switch (type) + { + case TableScan: + return "table_scan_0"; + case ExchangeReceiver: + return "exchange_receiver_0"; + default: + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Unknown Executor Source type {}", + type); + } + } + + void executeStreams( + const std::shared_ptr & request, + std::unordered_map & source_columns_map, + const ColumnsWithTypeAndName & expect_columns, + size_t concurrency = 1); + void executeStreams( + const std::shared_ptr & request, + const ColumnsWithTypeAndName & expect_columns, + size_t concurrency = 1); + + void executeStreamsWithSingleSource( + const std::shared_ptr & request, + const ColumnsWithTypeAndName & source_columns, + const ColumnsWithTypeAndName & expect_columns, + SourceType type = TableScan, + size_t concurrency = 1); + protected: MockDAGRequestContext context; std::unique_ptr dag_context_ptr; diff --git a/dbms/src/TestUtils/FunctionTestUtils.cpp b/dbms/src/TestUtils/FunctionTestUtils.cpp index dae07f7123b..637fbf51c00 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.cpp +++ b/dbms/src/TestUtils/FunctionTestUtils.cpp @@ -242,5 +242,37 @@ ColumnWithTypeAndName createOnlyNullColumn(size_t size, const String & name) return {std::move(col), data_type, name}; } +ColumnWithTypeAndName toDatetimeVec(String name, const std::vector & v, int fsp) +{ + std::vector::FieldType> vec; + vec.reserve(v.size()); + for (const auto & value_str : v) + { + Field value = parseMyDateTime(value_str, fsp); + vec.push_back(value.template safeGet()); + } + DataTypePtr data_type = std::make_shared(fsp); + return {makeColumn(data_type, vec), data_type, name, 0}; +} + +ColumnWithTypeAndName toNullableDatetimeVec(String name, const std::vector & v, int fsp) +{ + std::vector::FieldType>> vec; + vec.reserve(v.size()); + for (const auto & value_str : v) + { + if (!value_str.empty()) + { + Field value = parseMyDateTime(value_str, fsp); + vec.push_back(value.template safeGet()); + } + else + { + vec.push_back({}); + } + } + DataTypePtr data_type = makeNullable(std::make_shared(fsp)); + return {makeColumn>(data_type, vec), data_type, name, 0}; +} } // namespace tests } // namespace DB diff --git a/dbms/src/TestUtils/FunctionTestUtils.h b/dbms/src/TestUtils/FunctionTestUtils.h index 615a58ebda5..d6b7351df05 100644 --- a/dbms/src/TestUtils/FunctionTestUtils.h +++ b/dbms/src/TestUtils/FunctionTestUtils.h @@ -536,7 +536,7 @@ ColumnWithTypeAndName executeFunction( const String & func_name, const ColumnsWithTypeAndName & columns, const TiDB::TiDBCollatorPtr & collator = nullptr, - bool raw_function_test = true); + bool raw_function_test = false); ColumnWithTypeAndName executeFunction( Context & context, @@ -544,7 +544,7 @@ ColumnWithTypeAndName executeFunction( const ColumnNumbers & argument_column_numbers, const ColumnsWithTypeAndName & columns, const TiDB::TiDBCollatorPtr & collator = nullptr, - bool raw_function_test = true); + bool raw_function_test = false); template ColumnWithTypeAndName executeFunction( @@ -562,7 +562,7 @@ DataTypePtr getReturnTypeForFunction( const String & func_name, const ColumnsWithTypeAndName & columns, const TiDB::TiDBCollatorPtr & collator = nullptr, - bool raw_function_test = true); + bool raw_function_test = false); template ColumnWithTypeAndName createNullableColumn(InferredDataVector init_vec, const std::vector & null_map, const String name = "") @@ -654,6 +654,33 @@ ColumnWithTypeAndName createNullableColumn( return createNullableColumn(data_type_args, vec, null_map, name, 0); } +template +ColumnWithTypeAndName toNullableVec(const std::vector::FieldType>> & v) +{ + return createColumn>(v); +} + +template +ColumnWithTypeAndName toVec(const std::vector::FieldType> & v) +{ + return createColumn(v); +} + +template +ColumnWithTypeAndName toNullableVec(String name, const std::vector::FieldType>> & v) +{ + return createColumn>(v, name); +} + +template +ColumnWithTypeAndName toVec(String name, const std::vector::FieldType> & v) +{ + return createColumn(v, name); +} + +ColumnWithTypeAndName toDatetimeVec(String name, const std::vector & v, int fsp); + +ColumnWithTypeAndName toNullableDatetimeVec(String name, const std::vector & v, int fsp); class FunctionTest : public ::testing::Test { protected: @@ -687,7 +714,7 @@ class FunctionTest : public ::testing::Test const String & func_name, const ColumnsWithTypeAndName & columns, const TiDB::TiDBCollatorPtr & collator = nullptr, - bool raw_function_test = true) + bool raw_function_test = false) { return DB::tests::executeFunction(context, func_name, columns, collator, raw_function_test); } @@ -704,7 +731,7 @@ class FunctionTest : public ::testing::Test const ColumnNumbers & argument_column_numbers, const ColumnsWithTypeAndName & columns, const TiDB::TiDBCollatorPtr & collator = nullptr, - bool raw_function_test = true) + bool raw_function_test = false) { return DB::tests::executeFunction(context, func_name, argument_column_numbers, columns, collator, raw_function_test); } diff --git a/dbms/src/TestUtils/InterpreterTestUtils.cpp b/dbms/src/TestUtils/InterpreterTestUtils.cpp deleted file mode 100644 index 2cc096d4095..00000000000 --- a/dbms/src/TestUtils/InterpreterTestUtils.cpp +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2022 PingCAP, Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -namespace DB::tests -{ -DAGContext & InterpreterTest::getDAGContext() -{ - assert(dag_context_ptr != nullptr); - return *dag_context_ptr; -} - -void InterpreterTest::initializeContext() -{ - dag_context_ptr = std::make_unique(1024); - context = MockDAGRequestContext(TiFlashTestEnv::getContext()); - dag_context_ptr->log = Logger::get("interpreterTest"); -} - -void InterpreterTest::SetUpTestCase() -{ - try - { - DB::registerFunctions(); - DB::registerAggregateFunctions(); - } - catch (DB::Exception &) - { - // Maybe another test has already registered, ignore exception here. - } -} - -void InterpreterTest::initializeClientInfo() -{ - context.context.setCurrentQueryId("test"); - ClientInfo & client_info = context.context.getClientInfo(); - client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info.interface = ClientInfo::Interface::GRPC; -} - -void InterpreterTest::executeInterpreter(const String & expected_string, const std::shared_ptr & request, size_t concurrency) -{ - DAGContext dag_context(*request, "interpreter_test", concurrency); - context.context.setDAGContext(&dag_context); - // Currently, don't care about regions information in interpreter tests. - DAGQuerySource dag(context.context); - auto res = executeQuery(dag, context.context, false, QueryProcessingStage::Complete); - FmtBuffer fb; - res.in->dumpTree(fb); - ASSERT_EQ(Poco::trim(expected_string), Poco::trim(fb.toString())); -} - -void InterpreterTest::dagRequestEqual(const String & expected_string, const std::shared_ptr & actual) -{ - ASSERT_EQ(Poco::trim(expected_string), Poco::trim(ExecutorSerializer().serialize(actual.get()))); -} - -} // namespace DB::tests diff --git a/dbms/src/TestUtils/TiFlashTestEnv.cpp b/dbms/src/TestUtils/TiFlashTestEnv.cpp index bd05e5826db..a7bcfe43d7a 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.cpp +++ b/dbms/src/TestUtils/TiFlashTestEnv.cpp @@ -24,11 +24,13 @@ #include #include +#include + namespace DB::tests { std::unique_ptr TiFlashTestEnv::global_context = nullptr; -void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ps_v3) +void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, PageStorageRunMode ps_run_mode) { // set itself as global context global_context = std::make_unique(DB::Context::createGlobal()); @@ -39,6 +41,10 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ KeyManagerPtr key_manager = std::make_shared(false); global_context->initializeFileProvider(key_manager, false); + // initialize background & blockable background thread pool + global_context->initializeBackgroundPool(std::thread::hardware_concurrency() / 4); + global_context->initializeBlockableBackgroundPool(std::thread::hardware_concurrency() / 4); + // Theses global variables should be initialized by the following order // 1. capacity // 2. path pool @@ -68,7 +74,7 @@ void TiFlashTestEnv::initializeGlobalContext(Strings testdata_path, bool enable_ global_context->getPathCapacity(), global_context->getFileProvider()); - global_context->setPageStorageRunMode(enable_ps_v3 ? PageStorageRunMode::ONLY_V3 : PageStorageRunMode::ONLY_V2); + global_context->setPageStorageRunMode(ps_run_mode); global_context->initializeGlobalStoragePoolIfNeed(global_context->getPathPool()); LOG_FMT_INFO(Logger::get("TiFlashTestEnv"), "Storage mode : {}", static_cast(global_context->getPageStorageRunMode())); diff --git a/dbms/src/TestUtils/TiFlashTestEnv.h b/dbms/src/TestUtils/TiFlashTestEnv.h index 0264d87ef9f..dafecf6e1de 100644 --- a/dbms/src/TestUtils/TiFlashTestEnv.h +++ b/dbms/src/TestUtils/TiFlashTestEnv.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -88,7 +89,7 @@ class TiFlashTestEnv static Context getContext(const DB::Settings & settings = DB::Settings(), Strings testdata_path = {}); - static void initializeGlobalContext(Strings testdata_path = {}, bool enable_ps_v3 = true); + static void initializeGlobalContext(Strings testdata_path = {}, PageStorageRunMode ps_run_mode = PageStorageRunMode::ONLY_V3); static Context & getGlobalContext() { return *global_context; } static void shutdown(); diff --git a/dbms/src/TestUtils/executorSerializer.cpp b/dbms/src/TestUtils/executorSerializer.cpp index b8d2b039bd2..a0ae4b11270 100644 --- a/dbms/src/TestUtils/executorSerializer.cpp +++ b/dbms/src/TestUtils/executorSerializer.cpp @@ -204,6 +204,66 @@ void serializeExchangeReceiver(const String & executor_id, const tipb::ExchangeR buf.append("}\n"); } +void serializeWindow(const String & executor_id, const tipb::Window & window [[maybe_unused]], FmtBuffer & buf) +{ + buf.fmtAppend("{} | partition_by: {{", executor_id); + buf.joinStr( + window.partition_by().begin(), + window.partition_by().end(), + [&](const auto & partition_by, FmtBuffer & fb) { + fb.append("("); + serializeExpression(partition_by.expr(), buf); + fb.fmtAppend(", desc: {})", partition_by.desc()); + }, + ", "); + buf.append("}}, order_by: {"); + buf.joinStr( + window.order_by().begin(), + window.order_by().end(), + [&](const auto & order_by, FmtBuffer & fb) { + fb.append("("); + serializeExpression(order_by.expr(), buf); + fb.fmtAppend(", desc: {})", order_by.desc()); + }, + ", "); + buf.append("}, func_desc: {"); + buf.joinStr( + window.func_desc().begin(), + window.func_desc().end(), + [&](const auto & func, FmtBuffer &) { + serializeExpression(func, buf); + }, + ", "); + if (window.has_frame()) + { + buf.append("}, frame: {"); + if (window.frame().has_start()) + { + buf.fmtAppend("start<{}, {}, {}>", window.frame().start().type(), window.frame().start().unbounded(), window.frame().start().offset()); + } + if (window.frame().has_end()) + { + buf.fmtAppend(", end<{}, {}, {}>", window.frame().end().type(), window.frame().end().unbounded(), window.frame().end().offset()); + } + } + buf.append("}\n"); +} + +void serializeSort(const String & executor_id, const tipb::Sort & sort [[maybe_unused]], FmtBuffer & buf) +{ + buf.fmtAppend("{} | isPartialSort: {}, partition_by: {{", executor_id, sort.ispartialsort()); + buf.joinStr( + sort.byitems().begin(), + sort.byitems().end(), + [&](const auto & by, FmtBuffer & fb) { + fb.append("("); + serializeExpression(by.expr(), buf); + fb.fmtAppend(", desc: {})", by.desc()); + }, + ", "); + buf.append("}\n"); +} + void ExecutorSerializer::serialize(const tipb::Executor & root_executor, size_t level) { auto append_str = [&level, this](const tipb::Executor & executor) { @@ -248,9 +308,11 @@ void ExecutorSerializer::serialize(const tipb::Executor & root_executor, size_t serializeExchangeSender(executor.executor_id(), executor.exchange_sender(), buf); break; case tipb::ExecType::TypeSort: - throw TiFlashException("Sort executor is not supported", Errors::Coprocessor::Unimplemented); // todo support sort executor. + serializeSort(executor.executor_id(), executor.sort(), buf); + break; case tipb::ExecType::TypeWindow: - throw TiFlashException("Window executor is not supported", Errors::Coprocessor::Unimplemented); // todo support window executor. + serializeWindow(executor.executor_id(), executor.window(), buf); + break; default: throw TiFlashException("Should not reach here", Errors::Coprocessor::Internal); } diff --git a/dbms/src/TestUtils/mockExecutor.cpp b/dbms/src/TestUtils/mockExecutor.cpp index 3313aae6a93..e1ccbdbb010 100644 --- a/dbms/src/TestUtils/mockExecutor.cpp +++ b/dbms/src/TestUtils/mockExecutor.cpp @@ -52,6 +52,15 @@ ASTPtr buildOrderByItemList(MockOrderByItems order_by_items) return exp_list; } +MockWindowFrame buildDefaultRowsFrame() +{ + MockWindowFrame frame; + frame.type = tipb::WindowFrameType::Rows; + frame.end = {tipb::WindowBoundType::CurrentRow, false, 0}; + frame.start = {tipb::WindowBoundType::CurrentRow, false, 0}; + return frame; +} + // a mock DAGRequest should prepare its time_zone, flags, encode_type and output_schema. void DAGRequestBuilder::initDAGRequest(tipb::DAGRequest & dag_request) { @@ -94,6 +103,9 @@ DAGRequestBuilder & DAGRequestBuilder::mockTable(const String & db, const String TiDB::ColumnInfo ret; ret.tp = column.second; ret.name = column.first; + // TODO: find a way to assign decimal field's flen. + if (ret.tp == TiDB::TP::TypeNewDecimal) + ret.flen = 65; ret.id = i++; table_info.columns.push_back(std::move(ret)); } @@ -274,52 +286,158 @@ DAGRequestBuilder & DAGRequestBuilder::buildAggregation(ASTPtr agg_funcs, ASTPtr return *this; } -void MockDAGRequestContext::addMockTable(const MockTableName & name, const MockColumnInfoList & columns) +DAGRequestBuilder & DAGRequestBuilder::window(ASTPtr window_func, MockOrderByItem order_by, MockPartitionByItem partition_by, MockWindowFrame frame) +{ + assert(root); + auto window_func_list = std::make_shared(); + window_func_list->children.push_back(window_func); + root = compileWindow(root, getExecutorIndex(), window_func_list, buildOrderByItemList({partition_by}), buildOrderByItemList({order_by}), frame); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::window(ASTPtr window_func, MockOrderByItems order_by_list, MockPartitionByItems partition_by_list, MockWindowFrame frame) +{ + assert(root); + auto window_func_list = std::make_shared(); + window_func_list->children.push_back(window_func); + root = compileWindow(root, getExecutorIndex(), window_func_list, buildOrderByItemList(partition_by_list), buildOrderByItemList(order_by_list), frame); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::window(MockAsts window_funcs, MockOrderByItems order_by_list, MockPartitionByItems partition_by_list, MockWindowFrame frame) +{ + assert(root); + auto window_func_list = std::make_shared(); + for (const auto & func : window_funcs) + window_func_list->children.push_back(func); + root = compileWindow(root, getExecutorIndex(), window_func_list, buildOrderByItemList(partition_by_list), buildOrderByItemList(order_by_list), frame); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::sort(MockOrderByItem order_by, bool is_partial_sort) +{ + assert(root); + root = compileSort(root, getExecutorIndex(), buildOrderByItemList({order_by}), is_partial_sort); + return *this; +} + +DAGRequestBuilder & DAGRequestBuilder::sort(MockOrderByItems order_by_list, bool is_partial_sort) { - std::vector v_column_info(columns.size()); + assert(root); + root = compileSort(root, getExecutorIndex(), buildOrderByItemList(order_by_list), is_partial_sort); + return *this; +} + +void MockDAGRequestContext::addMockTable(const MockTableName & name, const MockColumnInfoList & columnInfos) +{ + std::vector v_column_info(columnInfos.size()); size_t i = 0; - for (const auto & info : columns) + for (const auto & info : columnInfos) { v_column_info[i++] = std::move(info); } mock_tables[name.first + "." + name.second] = v_column_info; } -void MockDAGRequestContext::addMockTable(const String & db, const String & table, const MockColumnInfos & columns) +void MockDAGRequestContext::addMockTable(const String & db, const String & table, const MockColumnInfos & columnInfos) { - mock_tables[db + "." + table] = columns; + mock_tables[db + "." + table] = columnInfos; } -void MockDAGRequestContext::addMockTable(const MockTableName & name, const MockColumnInfos & columns) +void MockDAGRequestContext::addMockTable(const MockTableName & name, const MockColumnInfos & columnInfos) { - mock_tables[name.first + "." + name.second] = columns; + mock_tables[name.first + "." + name.second] = columnInfos; } -void MockDAGRequestContext::addExchangeRelationSchema(String name, const MockColumnInfos & columns) +void MockDAGRequestContext::addExchangeRelationSchema(String name, const MockColumnInfos & columnInfos) { - exchange_schemas[name] = columns; + exchange_schemas[name] = columnInfos; } -void MockDAGRequestContext::addExchangeRelationSchema(String name, const MockColumnInfoList & columns) +void MockDAGRequestContext::addExchangeRelationSchema(String name, const MockColumnInfoList & columnInfos) { - std::vector v_column_info(columns.size()); + std::vector v_column_info(columnInfos.size()); size_t i = 0; - for (const auto & info : columns) + for (const auto & info : columnInfos) { v_column_info[i++] = std::move(info); } exchange_schemas[name] = v_column_info; } +void MockDAGRequestContext::addMockTableColumnData(const String & db, const String & table, ColumnsWithTypeAndName columns) +{ + mock_table_columns[db + "." + table] = columns; +} + +void MockDAGRequestContext::addMockTableColumnData(const MockTableName & name, ColumnsWithTypeAndName columns) +{ + mock_table_columns[name.first + "." + name.second] = columns; +} + +void MockDAGRequestContext::addExchangeReceiverColumnData(const String & name, ColumnsWithTypeAndName columns) +{ + mock_exchange_columns[name] = columns; +} + +void MockDAGRequestContext::addMockTable(const String & db, const String & table, const MockColumnInfoList & columnInfos, ColumnsWithTypeAndName columns) +{ + addMockTable(db, table, columnInfos); + addMockTableColumnData(db, table, columns); +} + +void MockDAGRequestContext::addMockTable(const String & db, const String & table, const MockColumnInfos & columnInfos, ColumnsWithTypeAndName columns) +{ + addMockTable(db, table, columnInfos); + addMockTableColumnData(db, table, columns); +} + +void MockDAGRequestContext::addMockTable(const MockTableName & name, const MockColumnInfoList & columnInfos, ColumnsWithTypeAndName columns) +{ + addMockTable(name, columnInfos); + addMockTableColumnData(name, columns); +} + +void MockDAGRequestContext::addMockTable(const MockTableName & name, const MockColumnInfos & columnInfos, ColumnsWithTypeAndName columns) +{ + addMockTable(name, columnInfos); + addMockTableColumnData(name, columns); +} + +void MockDAGRequestContext::addExchangeReceiver(const String & name, MockColumnInfos columnInfos, ColumnsWithTypeAndName columns) +{ + addExchangeRelationSchema(name, columnInfos); + addExchangeReceiverColumnData(name, columns); +} + +void MockDAGRequestContext::addExchangeReceiver(const String & name, MockColumnInfoList columnInfos, ColumnsWithTypeAndName columns) +{ + addExchangeRelationSchema(name, columnInfos); + addExchangeReceiverColumnData(name, columns); +} + DAGRequestBuilder MockDAGRequestContext::scan(String db_name, String table_name) { - return DAGRequestBuilder(index).mockTable({db_name, table_name}, mock_tables[db_name + "." + table_name]); + auto builder = DAGRequestBuilder(index).mockTable({db_name, table_name}, mock_tables[db_name + "." + table_name]); + // If don't have related columns, user must pass input columns as argument of executeStreams in order to run Executors Tests. + // If user don't want to test executors, it will be safe to run Interpreter Tests. + if (mock_table_columns.find(db_name + "." + table_name) != mock_table_columns.end()) + { + executor_id_columns_map[builder.getRoot()->name] = mock_table_columns[db_name + "." + table_name]; + } + return builder; } DAGRequestBuilder MockDAGRequestContext::receive(String exchange_name) { auto builder = DAGRequestBuilder(index).exchangeReceiver(exchange_schemas[exchange_name]); receiver_source_task_ids_map[builder.getRoot()->name] = {}; + // If don't have related columns, user must pass input columns as argument of executeStreams in order to run Executors Tests. + // If user don't want to test executors, it will be safe to run Interpreter Tests. + if (mock_exchange_columns.find(exchange_name) != mock_exchange_columns.end()) + { + executor_id_columns_map[builder.getRoot()->name] = mock_exchange_columns[exchange_name]; + } return builder; } } // namespace DB::tests \ No newline at end of file diff --git a/dbms/src/TestUtils/mockExecutor.h b/dbms/src/TestUtils/mockExecutor.h index 2f6d3542ebb..d52b5ec674a 100644 --- a/dbms/src/TestUtils/mockExecutor.h +++ b/dbms/src/TestUtils/mockExecutor.h @@ -14,14 +14,12 @@ #pragma once +#include #include #include #include #include -#include -#include - namespace DB::tests { using MockColumnInfo = std::pair; @@ -30,8 +28,11 @@ using MockColumnInfoList = std::initializer_list; using MockTableName = std::pair; using MockOrderByItem = std::pair; using MockOrderByItems = std::initializer_list; +using MockPartitionByItem = std::pair; +using MockPartitionByItems = std::initializer_list; using MockColumnNames = std::initializer_list; using MockAsts = std::initializer_list; +using MockWindowFrame = mock::MockWindowFrame; class MockDAGRequestContext; @@ -95,6 +96,13 @@ class DAGRequestBuilder DAGRequestBuilder & aggregation(ASTPtr agg_func, ASTPtr group_by_expr); DAGRequestBuilder & aggregation(MockAsts agg_funcs, MockAsts group_by_exprs); + // window + DAGRequestBuilder & window(ASTPtr window_func, MockOrderByItem order_by, MockPartitionByItem partition_by, MockWindowFrame frame); + DAGRequestBuilder & window(MockAsts window_funcs, MockOrderByItems order_by_list, MockPartitionByItems partition_by_list, MockWindowFrame frame); + DAGRequestBuilder & window(ASTPtr window_func, MockOrderByItems order_by_list, MockPartitionByItems partition_by_list, MockWindowFrame frame); + DAGRequestBuilder & sort(MockOrderByItem order_by, bool is_partial_sort); + DAGRequestBuilder & sort(MockOrderByItems order_by_list, bool is_partial_sort); + private: void initDAGRequest(tipb::DAGRequest & dag_request); DAGRequestBuilder & buildAggregation(ASTPtr agg_funcs, ASTPtr group_by_exprs); @@ -122,11 +130,23 @@ class MockDAGRequestContext return DAGRequestBuilder(index); } - void addMockTable(const MockTableName & name, const MockColumnInfoList & columns); - void addMockTable(const String & db, const String & table, const MockColumnInfos & columns); - void addMockTable(const MockTableName & name, const MockColumnInfos & columns); - void addExchangeRelationSchema(String name, const MockColumnInfos & columns); - void addExchangeRelationSchema(String name, const MockColumnInfoList & columns); + void addMockTable(const MockTableName & name, const MockColumnInfoList & columnInfos); + void addMockTable(const String & db, const String & table, const MockColumnInfos & columnInfos); + void addMockTable(const MockTableName & name, const MockColumnInfos & columnInfos); + void addExchangeRelationSchema(String name, const MockColumnInfos & columnInfos); + void addExchangeRelationSchema(String name, const MockColumnInfoList & columnInfos); + void addMockTableColumnData(const String & db, const String & table, ColumnsWithTypeAndName columns); + void addMockTable(const String & db, const String & table, const MockColumnInfoList & columnInfos, ColumnsWithTypeAndName columns); + void addMockTable(const String & db, const String & table, const MockColumnInfos & columnInfos, ColumnsWithTypeAndName columns); + void addMockTable(const MockTableName & name, const MockColumnInfoList & columnInfos, ColumnsWithTypeAndName columns); + void addMockTable(const MockTableName & name, const MockColumnInfos & columnInfos, ColumnsWithTypeAndName columns); + void addMockTableColumnData(const MockTableName & name, ColumnsWithTypeAndName columns); + void addExchangeReceiverColumnData(const String & name, ColumnsWithTypeAndName columns); + void addExchangeReceiver(const String & name, MockColumnInfos columnInfos, ColumnsWithTypeAndName columns); + void addExchangeReceiver(const String & name, MockColumnInfoList columnInfos, ColumnsWithTypeAndName columns); + + std::unordered_map & executorIdColumnsMap() { return executor_id_columns_map; } + DAGRequestBuilder scan(String db_name, String table_name); DAGRequestBuilder receive(String exchange_name); @@ -134,6 +154,9 @@ class MockDAGRequestContext size_t index; std::unordered_map mock_tables; std::unordered_map exchange_schemas; + std::unordered_map mock_table_columns; + std::unordered_map mock_exchange_columns; + std::unordered_map executor_id_columns_map; /// public: // Currently don't support task_id, so the following to structure is useless, @@ -148,6 +171,8 @@ ASTPtr buildLiteral(const Field & field); ASTPtr buildFunction(MockAsts exprs, const String & name); ASTPtr buildOrderByItemList(MockOrderByItems order_by_items); +MockWindowFrame buildDefaultRowsFrame(); + #define col(name) buildColumn((name)) #define lit(field) buildLiteral((field)) #define eq(expr1, expr2) makeASTFunction("equals", (expr1), (expr2)) @@ -158,5 +183,9 @@ ASTPtr buildOrderByItemList(MockOrderByItems order_by_items); #define Or(expr1, expr2) makeASTFunction("or", (expr1), (expr2)) #define NOT(expr) makeASTFunction("not", (expr1), (expr2)) #define Max(expr) makeASTFunction("max", expr) +/// Window functions +#define RowNumber() makeASTFunction("RowNumber") +#define Rank() makeASTFunction("Rank") +#define DenseRank() makeASTFunction("DenseRank") } // namespace DB::tests \ No newline at end of file diff --git a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp index 6dbf791669f..8bed0f2fc6c 100644 --- a/dbms/src/TestUtils/tests/gtest_mock_executors.cpp +++ b/dbms/src/TestUtils/tests/gtest_mock_executors.cpp @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include namespace DB { namespace tests { -class MockDAGRequestTest : public DB::tests::InterpreterTest +class MockDAGRequestTest : public DB::tests::ExecutorTest { public: void initializeContext() override { - InterpreterTest::initializeContext(); + ExecutorTest::initializeContext(); context.addMockTable({"test_db", "test_table"}, {{"s1", TiDB::TP::TypeString}, {"s2", TiDB::TP::TypeString}}); context.addMockTable({"test_db", "test_table_1"}, {{"s1", TiDB::TP::TypeLong}, {"s2", TiDB::TP::TypeString}, {"s3", TiDB::TP::TypeString}}); @@ -252,5 +252,17 @@ try } CATCH +TEST_F(MockDAGRequestTest, MockWindow) +try +{ + auto request = context.scan("test_db", "test_table").sort({"s1", false}, true).window(RowNumber(), {"s1", true}, {"s2", false}, buildDefaultRowsFrame()).build(context); + { + String expected = "window_2 | partition_by: {(<1, String>, desc: false)}}, order_by: {(<0, String>, desc: true)}, func_desc: {row_number()}, frame: {start<2, false, 0>, end<2, false, 0>}\n" + " sort_1 | isPartialSort: true, partition_by: {(<0, String>, desc: false)}\n" + " table_scan_0 | {<0, String>, <1, String>}\n"; + ASSERT_DAGREQUEST_EQAUL(expected, request); + } +} +CATCH } // namespace tests } // namespace DB \ No newline at end of file diff --git a/dbms/src/TiDB/Schema/SchemaBuilder-internal.h b/dbms/src/TiDB/Schema/SchemaBuilder-internal.h index a331205ce8c..94edcbea204 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder-internal.h +++ b/dbms/src/TiDB/Schema/SchemaBuilder-internal.h @@ -35,7 +35,7 @@ struct TableInfo; } namespace DB { -std::tuple parseColumnsFromTableInfo(const TiDB::TableInfo & table_info, Poco::Logger * log); +std::tuple parseColumnsFromTableInfo(const TiDB::TableInfo & table_info); constexpr char tmpNamePrefix[] = "_tiflash_tmp_"; diff --git a/dbms/src/TiDB/Schema/SchemaBuilder.cpp b/dbms/src/TiDB/Schema/SchemaBuilder.cpp index 64d118eec3e..ae78923fc61 100644 --- a/dbms/src/TiDB/Schema/SchemaBuilder.cpp +++ b/dbms/src/TiDB/Schema/SchemaBuilder.cpp @@ -61,6 +61,7 @@ extern const char exception_before_step_2_rename_in_exchange_partition[]; extern const char exception_after_step_2_in_exchange_partition[]; extern const char exception_before_step_3_rename_in_exchange_partition[]; extern const char exception_after_step_3_in_exchange_partition[]; +extern const char exception_between_schema_change_in_the_same_diff[]; } // namespace FailPoints bool isReservedDatabase(Context & context, const String & database_name) @@ -336,6 +337,7 @@ void SchemaBuilder::applyAlterPhysicalTable(DBInfoPtr db_inf FmtBuffer fmt_buf; fmt_buf.fmtAppend("Detected schema changes: {}: ", name_mapper.debugCanonicalName(*db_info, *table_info)); for (const auto & schema_change : schema_changes) + { for (const auto & command : schema_change.first) { if (command.type == AlterCommand::ADD_COLUMN) @@ -347,6 +349,7 @@ void SchemaBuilder::applyAlterPhysicalTable(DBInfoPtr db_inf else if (command.type == AlterCommand::RENAME_COLUMN) fmt_buf.fmtAppend("RENAME COLUMN from {} to {}, ", command.column_name, command.new_column_name); } + } return fmt_buf.toString(); }; LOG_DEBUG(log, log_str()); @@ -355,11 +358,27 @@ void SchemaBuilder::applyAlterPhysicalTable(DBInfoPtr db_inf // Using original table info with updated columns instead of using new_table_info directly, // so that other changes (RENAME commands) won't be saved. // Also, updating schema_version as altering column is structural. - for (const auto & schema_change : schema_changes) + for (size_t i = 0; i < schema_changes.size(); i++) { + if (i > 0) + { + /// If there are multiple schema change in the same diff, + /// the table schema version will be set to the latest schema version after the first schema change is applied. + /// Throw exception in the middle of the schema change to mock the case that there is a race between data decoding and applying different schema change. + FAIL_POINT_TRIGGER_EXCEPTION(FailPoints::exception_between_schema_change_in_the_same_diff); + } + const auto & schema_change = schema_changes[i]; /// Update column infos by applying schema change in this step. schema_change.second(orig_table_info); - /// Update schema version aggressively for the sake of correctness. + /// Update schema version aggressively for the sake of correctness（for read part). + /// In read action, we will use table_info.schema_version(storage_version) and TiDBSchemaSyncer.cur_version(global_version) to compare with query_version, to decide whether we can read under this query_version, or we need to make the schema newer. + /// In our comparison logic, we only serve the query when the query schema version meet the criterion: storage_version <= query_version <= global_version(The more detail info you can refer the comments in DAGStorageInterpreter::getAndLockStorages.) + /// And when apply multi diffs here, we only update global_version when all diffs have been applied. + /// So the global_version may be less than the actual "global_version" of the local schema in the process of applying schema changes. + /// And if we don't update the storage_version ahead of time, we may meet the following case when apply multiple diffs: storage_version <= global_version < actual "global_version". + /// If we receive a query with the same version as global_version, we can have the following scenario: storage_version <= global_version == query_version < actual "global_version". + /// And because storage_version <= global_version == query_version meet the criterion of serving the query, the query will be served. But query_version < actual "global_version" indicates that we use a newer schema to server an older query which may cause some inconsistency issue. + /// So we update storage_version aggressively to prevent the above scenario happens. orig_table_info.schema_version = target_version; auto alter_lock = storage->lockForAlter(getThreadName()); storage->alterFromTiDB( @@ -952,13 +971,12 @@ void SchemaBuilder::applyDropSchema(const String & db_name) } std::tuple -parseColumnsFromTableInfo(const TiDB::TableInfo & table_info, Poco::Logger * log) +parseColumnsFromTableInfo(const TiDB::TableInfo & table_info) { NamesAndTypes columns; std::vector primary_keys; for (const auto & column : table_info.columns) { - LOG_FMT_DEBUG(log, "Analyzing column: {}, type: {}", column.name, static_cast(column.tp)); DataTypePtr type = getDataTypeByColumnInfo(column); columns.emplace_back(column.name, type); if (column.hasPriKeyFlag()) @@ -988,7 +1006,7 @@ String createTableStmt( Poco::Logger * log) { LOG_FMT_DEBUG(log, "Analyzing table info : {}", table_info.serialize()); - auto [columns, pks] = parseColumnsFromTableInfo(table_info, log); + auto [columns, pks] = parseColumnsFromTableInfo(table_info); String stmt; WriteBufferFromString stmt_buf(stmt); diff --git a/dbms/src/TiDB/Schema/SchemaGetter.h b/dbms/src/TiDB/Schema/SchemaGetter.h index cfa5e1c6335..02d2f7a7c88 100644 --- a/dbms/src/TiDB/Schema/SchemaGetter.h +++ b/dbms/src/TiDB/Schema/SchemaGetter.h @@ -28,6 +28,7 @@ namespace DB { +// The enum results are completely the same as the DDL Action listed in the "parser/model/ddl.go" of TiDB codebase, which must be keeping in sync. enum class SchemaActionType : Int8 { None = 0, diff --git a/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h b/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h index b682abf1af4..4fdba195acb 100644 --- a/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h +++ b/dbms/src/TiDB/Schema/TiDBSchemaSyncer.h @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -28,6 +27,11 @@ namespace DB { +namespace ErrorCodes +{ +extern const int FAIL_POINT_ERROR; +}; + template struct TiDBSchemaSyncer : public SchemaSyncer { @@ -177,6 +181,10 @@ struct TiDBSchemaSyncer : public SchemaSyncer } catch (Exception & e) { + if (e.code() == ErrorCodes::FAIL_POINT_ERROR) + { + throw; + } GET_METRIC(tiflash_schema_apply_count, type_failed).Increment(); LOG_FMT_WARNING(log, "apply diff meets exception : {} \n stack is {}", e.displayText(), e.getStackTrace().toString()); return false; diff --git a/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp b/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp index e4205f6f938..3addf73a642 100644 --- a/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp +++ b/dbms/src/WindowFunctions/tests/gtest_window_functions.cpp @@ -12,334 +12,192 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include -#include -#include -#include -#include -#include -#include +#include namespace DB::tests { -class WindowFunction : public DB::tests::FunctionTest +class WindowExecutorTestRunner : public DB::tests::ExecutorTest { -protected: - std::shared_ptr mock_interpreter; - - void SetUp() override - { - DB::tests::FunctionTest::SetUp(); - DB::registerWindowFunctions(); - } - - template - ColumnWithTypeAndName toNullableVec(String name, const std::vector::FieldType>> & v) - { - return createColumn>(v, name); - } - - template - ColumnWithTypeAndName toVec(String name, const std::vector::FieldType> & v) - { - return createColumn(v, name); - } - - template - static ColumnWithTypeAndName toConst(const T s) - { - return createConstColumn(1, s); - } - - static ColumnWithTypeAndName toDatetimeVec(String name, const std::vector & v, int fsp) - { - std::vector::FieldType> vec; - for (const auto & value_str : v) - { - Field value = parseMyDateTime(value_str, fsp); - vec.push_back(value.template safeGet()); - } - DataTypePtr data_type = std::make_shared(fsp); - return {makeColumn(data_type, vec), data_type, name, 0}; - } - - static ColumnWithTypeAndName toNullableDatetimeVec(String name, const std::vector & v, int fsp) - { - std::vector::FieldType>> vec; - for (const auto & value_str : v) - { - if (!value_str.empty()) - { - Field value = parseMyDateTime(value_str, fsp); - vec.push_back(value.template safeGet()); - } - else - { - vec.push_back({}); - } - } - DataTypePtr data_type = makeNullable(std::make_shared(fsp)); - return {makeColumn>(data_type, vec), data_type, name, 0}; - } - - void setMaxBlockSize(int size) - { - context.getSettingsRef().max_block_size.set(size); - } - - void mockInterpreter(std::vector source_columns, Context context) - { - std::vector mock_input_streams_vec = {}; - DAGQueryBlock mock_query_block(0, static_cast>(nullptr)); - std::vector mock_subqueries_for_sets = {}; - mock_interpreter = std::make_shared(context, - mock_input_streams_vec, - mock_query_block, - 1); - - mock_interpreter->analyzer = std::make_unique(std::move(source_columns), context); - } - - void mockExecuteTableScan(DAGPipeline & pipeline, ColumnsWithTypeAndName columns) - { - pipeline.streams.push_back(std::make_shared(columns, context.getSettingsRef().max_block_size)); - mock_interpreter->input_streams_vec.push_back(pipeline.streams); - } - - void mockExecuteWindowOrder(DAGPipeline & pipeline, std::string sort_json_str) +public: + void initializeContext() override { - tipb::Sort sort; - google::protobuf::util::JsonStringToMessage(sort_json_str, &sort); - mock_interpreter->handleWindowOrder(pipeline, sort); - mock_interpreter->input_streams_vec[0] = pipeline.streams; - NamesWithAliases final_project; - for (const auto & column : (*mock_interpreter->analyzer).source_columns) - { - final_project.push_back({column.name, ""}); - } - mockExecuteProject(pipeline, final_project); - } - - void mockExecuteWindow(DAGPipeline & pipeline, std::string window_json_str) - { - tipb::Window window; - google::protobuf::util::JsonStringToMessage(window_json_str, &window); - mock_interpreter->handleWindow(pipeline, window); - mock_interpreter->input_streams_vec[0] = pipeline.streams; - NamesWithAliases final_project; - for (const auto & column : (*mock_interpreter->analyzer).source_columns) - { - final_project.push_back({column.name, ""}); - } - mockExecuteProject(pipeline, final_project); - } - - void mockExecuteProject(DAGPipeline & pipeline, NamesWithAliases & final_project) - { - mock_interpreter->executeProject(pipeline, final_project); - } - - static Block mergeBlocks(Blocks blocks) - { - if (blocks.empty()) - { - return {}; - } - Block sample_block; - std::vector actual_cols; - - for (const auto & block : blocks) - { - if (!sample_block) - { - sample_block = block; - for (const auto & column : block.getColumnsWithTypeAndName()) - { - actual_cols.push_back(column.type->createColumn()); - } - } - - for (size_t i = 0; i < block.columns(); ++i) - { - for (size_t j = 0; j < block.rows(); ++j) - { - actual_cols[i]->insert((*(block.getColumnsWithTypeAndName())[i].column)[j]); - } - } - } - - ColumnsWithTypeAndName actual_columns; - - for (size_t i = 0; i < actual_cols.size(); ++i) - { - actual_columns.push_back({std::move(actual_cols[i]), sample_block.getColumnsWithTypeAndName()[i].type, sample_block.getColumnsWithTypeAndName()[i].name, sample_block.getColumnsWithTypeAndName()[i].column_id}); - } - return Block(actual_columns); - } - - void testOneWindowFunction(const std::vector & source_column_types, const ColumnsWithTypeAndName & source_columns, const ColumnsWithTypeAndName & expect_columns, const std::string window_json_str, const std::string sort_json_str) - { - mockInterpreter(source_column_types, context); - DAGPipeline pipeline; - ExpressionActionsChain chain; - Block except_block(expect_columns); - - mockExecuteTableScan(pipeline, source_columns); - - mockExecuteWindowOrder(pipeline, sort_json_str); - - mockExecuteWindow(pipeline, window_json_str); - - auto stream = pipeline.firstStream(); - - Blocks actual_blocks; - while (Block block = stream->read()) - { - actual_blocks.push_back(block); - } - - Block actual_block = mergeBlocks(actual_blocks); - - if (actual_block) - { - // Check that input columns is properly split to many blocks - ASSERT_EQ(actual_blocks.size(), (actual_block.rows() - 1) / context.getSettingsRef().max_block_size + 1); - } - ASSERT_BLOCK_EQ(except_block, actual_block); + ExecutorTest::initializeContext(); + context.addMockTable( + {"test_db", "test_table"}, + {{"partition", TiDB::TP::TypeLongLong}, {"order", TiDB::TP::TypeLongLong}}, + {toVec("partition", {1, 1, 1, 1, 2, 2, 2, 2}), + toVec("order", {1, 1, 2, 2, 1, 1, 2, 2})}); + context.addMockTable( + {"test_db", "test_table_string"}, + {{"partition", TiDB::TP::TypeString}, {"order", TiDB::TP::TypeString}}, + {toVec("partition", {"banana", "banana", "banana", "banana", "apple", "apple", "apple", "apple"}), + toVec("order", {"apple", "apple", "banana", "banana", "apple", "apple", "banana", "banana"})}); + + context.addMockTable( + {"test_db", "test_table_more_cols"}, + {{"partition1", TiDB::TP::TypeLongLong}, {"partition2", TiDB::TP::TypeLongLong}, {"order1", TiDB::TP::TypeLongLong}, {"order2", TiDB::TP::TypeLongLong}}, + {toVec("partition1", {1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2}), + toVec("partition2", {1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 2}), + toVec("order1", {2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1}), + toVec("order2", {2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 1})}); + + context.addMockTable( + {"test_db", "test_table_float64"}, + {{"partition", TiDB::TP::TypeDouble}, {"order", TiDB::TP::TypeDouble}}, + {toVec("partition", {1.00, 1.00, 1.00, 1.00, 2.00, 2.00, 2.00, 2.00}), + toVec("order", {1.00, 1.00, 2.00, 2.00, 1.00, 1.00, 2.00, 2.00})}); + + context.addMockTable( + {"test_db", "test_table_datetime"}, + {{"partition", TiDB::TP::TypeDatetime}, {"order", TiDB::TP::TypeDatetime}}); + + context.addMockTable( + {"test_db", "test_table_for_rank"}, + {{"partition", TiDB::TP::TypeLongLong}, {"order", TiDB::TP::TypeLongLong}}, + {toVec("partition", {1, 1, 1, 1, 2, 2, 2, 2}), + toVec("order", {1, 1, 2, 2, 1, 1, 2, 2})}); } }; -TEST_F(WindowFunction, testWindowFunctionByPartitionAndOrder) +TEST_F(WindowExecutorTestRunner, testWindowFunctionByPartitionAndOrder) try { - setMaxBlockSize(3); - - std::string window_json; - std::string sort_json; - /***** row_number with different types of input *****/ // int - sql : select *, row_number() over w1 from test1 window w1 as (partition by partition_int order by order_int) - window_json = R"({"funcDesc":[{"tp":"RowNumber","sig":"Unspecified","fieldType":{"tp":8,"flag":128,"flen":21,"decimal":-1,"collate":63,"charset":"binary"},"hasDistinct":false}],"partitionBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"orderBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"frame":{"type":"Rows","start":{"type":"CurrentRow","unbounded":false,"offset":"0"},"end":{"type":"CurrentRow","unbounded":false,"offset":"0"}},"child":{"tp":"TypeSort","executorId":"Sort_12","sort":{"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGAkMCV6NP+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}}}})"; - sort_json = R"({"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGAkMCV6NP+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}})"; - testOneWindowFunction( - {NameAndTypePair("partition", std::make_shared()), NameAndTypePair("order", std::make_shared())}, - {toVec("partition", {1, 1, 1, 1, 2, 2, 2, 2}), toVec("order", {1, 1, 2, 2, 1, 1, 2, 2})}, - {toVec("partition", {1, 1, 1, 1, 2, 2, 2, 2}), toVec("order", {1, 1, 2, 2, 1, 1, 2, 2}), toNullableVec("row_number", {1, 2, 3, 4, 1, 2, 3, 4})}, - window_json, - sort_json); + auto request = context + .scan("test_db", "test_table") + .sort({{"partition", false}, {"order", false}, {"partition", false}, {"order", false}}, true) + .window(RowNumber(), {"order", false}, {"partition", false}, buildDefaultRowsFrame()) + .build(context); + executeStreams( + request, + {toNullableVec("partition", {1, 1, 1, 1, 2, 2, 2, 2}), + toNullableVec("order", {1, 1, 2, 2, 1, 1, 2, 2}), + toNullableVec("row_number", {1, 2, 3, 4, 1, 2, 3, 4})}); // null input - testOneWindowFunction( - {NameAndTypePair("partition", makeNullable(std::make_shared())), NameAndTypePair("order", makeNullable(std::make_shared()))}, + executeStreamsWithSingleSource( + request, {toNullableVec("partition", {}), toNullableVec("order", {})}, - {}, - window_json, - sort_json); + {}); // nullable - testOneWindowFunction( - {NameAndTypePair("partition", makeNullable(std::make_shared())), NameAndTypePair("order", makeNullable(std::make_shared()))}, - {toNullableVec("partition", {{}, 1, 1, 1, 1, 2, 2, 2, 2}), toNullableVec("order", {{}, 1, 1, 2, 2, 1, 1, 2, 2})}, - {toNullableVec("partition", {{}, 1, 1, 1, 1, 2, 2, 2, 2}), toNullableVec("order", {{}, 1, 1, 2, 2, 1, 1, 2, 2}), toNullableVec("row_number", {1, 1, 2, 3, 4, 1, 2, 3, 4})}, - window_json, - sort_json); + executeStreamsWithSingleSource( + request, + {toNullableVec("partition", {{}, 1, 1, 1, 1, 2, 2, 2, 2}), {toNullableVec("order", {{}, 1, 1, 2, 2, 1, 1, 2, 2})}}, + {toNullableVec("partition", {{}, 1, 1, 1, 1, 2, 2, 2, 2}), toNullableVec("order", {{}, 1, 1, 2, 2, 1, 1, 2, 2}), toNullableVec("row_number", {1, 1, 2, 3, 4, 1, 2, 3, 4})}); // string - sql : select *, row_number() over w1 from test2 window w1 as (partition by partition_string order by order_string) - window_json = R"({"funcDesc":[{"tp":"RowNumber","sig":"Unspecified","fieldType":{"tp":8,"flag":128,"flen":21,"decimal":-1,"collate":63,"charset":"binary"},"hasDistinct":false}],"partitionBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false}],"orderBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false}],"frame":{"type":"Rows","start":{"type":"CurrentRow","unbounded":false,"offset":"0"},"end":{"type":"CurrentRow","unbounded":false,"offset":"0"}},"child":{"tp":"TypeSort","executorId":"Sort_12","sort":{"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGA8Nz57tP+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"}]},"executorId":"ExchangeReceiver_11"}}}})"; - sort_json = R"({"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGA8Nz57tP+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"},{"tp":254,"flag":0,"flen":32,"decimal":0,"collate":46,"charset":"utf8mb4"}]},"executorId":"ExchangeReceiver_11"}})"; - testOneWindowFunction( - {NameAndTypePair("partition", std::make_shared()), NameAndTypePair("order", std::make_shared())}, - {toVec("partition", {"banana", "banana", "banana", "banana", "apple", "apple", "apple", "apple"}), toVec("order", {"apple", "apple", "banana", "banana", "apple", "apple", "banana", "banana"})}, - {toVec("partition", {"apple", "apple", "apple", "apple", "banana", "banana", "banana", "banana"}), toVec("order", {"apple", "apple", "banana", "banana", "apple", "apple", "banana", "banana"}), toNullableVec("row_number", {1, 2, 3, 4, 1, 2, 3, 4})}, - window_json, - sort_json); + request = context + .scan("test_db", "test_table_string") + .sort({{"partition", false}, {"order", false}, {"partition", false}, {"order", false}}, true) + .window(RowNumber(), {"order", false}, {"partition", false}, buildDefaultRowsFrame()) + .build(context); + + executeStreams( + request, + {toNullableVec("partition", {"apple", "apple", "apple", "apple", "banana", "banana", "banana", "banana"}), + toNullableVec("order", {"apple", "apple", "banana", "banana", "apple", "apple", "banana", "banana"}), + toNullableVec("row_number", {1, 2, 3, 4, 1, 2, 3, 4})}); // nullable - testOneWindowFunction( - {NameAndTypePair("partition", makeNullable(std::make_shared())), NameAndTypePair("order", makeNullable(std::make_shared()))}, - {toNullableVec("partition", {"banana", "banana", "banana", "banana", {}, "apple", "apple", "apple", "apple"}), toNullableVec("order", {"apple", "apple", "banana", "banana", {}, "apple", "apple", "banana", "banana"})}, - {toNullableVec("partition", {{}, "apple", "apple", "apple", "apple", "banana", "banana", "banana", "banana"}), toNullableVec("order", {{}, "apple", "apple", "banana", "banana", "apple", "apple", "banana", "banana"}), toNullableVec("row_number", {1, 1, 2, 3, 4, 1, 2, 3, 4})}, - window_json, - sort_json); - - // decimal - sql : select *, row_number() over w1 from test3 window w1 as (partition by partition_float order by order_decimal) - window_json = R"({"funcDesc":[{"tp":"RowNumber","sig":"Unspecified","fieldType":{"tp":8,"flag":128,"flen":21,"decimal":-1,"collate":63,"charset":"binary"},"hasDistinct":false}],"partitionBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"orderBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"frame":{"type":"Rows","start":{"type":"CurrentRow","unbounded":false,"offset":"0"},"end":{"type":"CurrentRow","unbounded":false,"offset":"0"}},"child":{"tp":"TypeSort","executorId":"Sort_12","sort":{"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGAoN3M99P+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}}}})"; - sort_json = R"({"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGAoN3M99P+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"},{"tp":246,"flag":0,"flen":6,"decimal":2,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}})"; - testOneWindowFunction( - {NameAndTypePair("partition", std::make_shared()), NameAndTypePair("order", std::make_shared())}, - {toVec("partition", {1.00, 1.00, 1.00, 1.00, 2.00, 2.00, 2.00, 2.00}), toVec("order", {1.00, 1.00, 2.00, 2.00, 1.00, 1.00, 2.00, 2.00})}, - {toVec("partition", {1.00, 1.00, 1.00, 1.00, 2.00, 2.00, 2.00, 2.00}), toVec("order", {1.00, 1.00, 2.00, 2.00, 1.00, 1.00, 2.00, 2.00}), toNullableVec("row_number", {1, 2, 3, 4, 1, 2, 3, 4})}, - window_json, - sort_json); + executeStreamsWithSingleSource( + request, + {toNullableVec("partition", {"banana", "banana", "banana", "banana", {}, "apple", "apple", "apple", "apple"}), + toNullableVec("order", {"apple", "apple", "banana", "banana", {}, "apple", "apple", "banana", "banana"})}, + {toNullableVec("partition", {{}, "apple", "apple", "apple", "apple", "banana", "banana", "banana", "banana"}), + toNullableVec("order", {{}, "apple", "apple", "banana", "banana", "apple", "apple", "banana", "banana"}), + toNullableVec("row_number", {1, 1, 2, 3, 4, 1, 2, 3, 4})}); + + // float64 - sql : select *, row_number() over w1 from test3 window w1 as (partition by partition_float order by order_float64) + request = context + .scan("test_db", "test_table_float64") + .sort({{"partition", false}, {"order", false}, {"partition", false}, {"order", false}}, true) + .window(RowNumber(), {"order", false}, {"partition", false}, buildDefaultRowsFrame()) + .build(context); + + executeStreams( + request, + {toNullableVec("partition", {1.00, 1.00, 1.00, 1.00, 2.00, 2.00, 2.00, 2.00}), + toNullableVec("order", {1.00, 1.00, 2.00, 2.00, 1.00, 1.00, 2.00, 2.00}), + toNullableVec("row_number", {1, 2, 3, 4, 1, 2, 3, 4})}); // nullable - testOneWindowFunction( - {NameAndTypePair("partition", makeNullable(std::make_shared())), NameAndTypePair("order", makeNullable(std::make_shared()))}, - {toNullableVec("partition", {{}, 1.00, 1.00, 1.00, 1.00, 2.00, 2.00, 2.00, 2.00}), toNullableVec("order", {{}, 1.00, 1.00, 2.00, 2.00, 1.00, 1.00, 2.00, 2.00})}, - {toNullableVec("partition", {{}, 1.00, 1.00, 1.00, 1.00, 2.00, 2.00, 2.00, 2.00}), toNullableVec("order", {{}, 1.00, 1.00, 2.00, 2.00, 1.00, 1.00, 2.00, 2.00}), toNullableVec("row_number", {1, 1, 2, 3, 4, 1, 2, 3, 4})}, - window_json, - sort_json); + executeStreamsWithSingleSource( + request, + {toNullableVec("partition", {{}, 1.00, 1.00, 1.00, 1.00, 2.00, 2.00, 2.00, 2.00}), + toNullableVec("order", {{}, 1.00, 1.00, 2.00, 2.00, 1.00, 1.00, 2.00, 2.00})}, + {toNullableVec("partition", {{}, 1.00, 1.00, 1.00, 1.00, 2.00, 2.00, 2.00, 2.00}), + toNullableVec("order", {{}, 1.00, 1.00, 2.00, 2.00, 1.00, 1.00, 2.00, 2.00}), + toNullableVec("row_number", {1, 1, 2, 3, 4, 1, 2, 3, 4})}); // datetime - select *, row_number() over w1 from test4 window w1 as (partition by partition_datetime order by order_datetime); - window_json = R"({"funcDesc":[{"tp":"RowNumber","sig":"Unspecified","fieldType":{"tp":8,"flag":128,"flen":21,"decimal":-1,"collate":63,"charset":"binary"},"hasDistinct":false}],"partitionBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"orderBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"frame":{"type":"Rows","start":{"type":"CurrentRow","unbounded":false,"offset":"0"},"end":{"type":"CurrentRow","unbounded":false,"offset":"0"}},"child":{"tp":"TypeSort","executorId":"Sort_12","sort":{"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGAsNmBhdT+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}}}})"; - sort_json = R"({"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGAsNmBhdT+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"},{"tp":12,"flag":128,"flen":26,"decimal":6,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}})"; - testOneWindowFunction( - {NameAndTypePair("partition", std::make_shared()), NameAndTypePair("order", std::make_shared())}, - {toDatetimeVec("partition", {"20220101010102", "20220101010102", "20220101010102", "20220101010102", "20220101010101", "20220101010101", "20220101010101", "20220101010101"}, 0), + request = context + .scan("test_db", "test_table_datetime") + .sort({{"partition", false}, {"order", false}, {"partition", false}, {"order", false}}, true) + .window(RowNumber(), {"order", false}, {"partition", false}, buildDefaultRowsFrame()) + .build(context); + executeStreamsWithSingleSource( + request, + {toNullableDatetimeVec("partition", {"20220101010102", "20220101010102", "20220101010102", "20220101010102", "20220101010101", "20220101010101", "20220101010101", "20220101010101"}, 0), toDatetimeVec("order", {"20220101010101", "20220101010101", "20220101010102", "20220101010102", "20220101010101", "20220101010101", "20220101010102", "20220101010102"}, 0)}, - {toDatetimeVec("partition", {"20220101010101", "20220101010101", "20220101010101", "20220101010101", "20220101010102", "20220101010102", "20220101010102", "20220101010102"}, 0), - toDatetimeVec("order", {"20220101010101", "20220101010101", "20220101010102", "20220101010102", "20220101010101", "20220101010101", "20220101010102", "20220101010102"}, 0), - toNullableVec("row_number", {1, 2, 3, 4, 1, 2, 3, 4})}, - window_json, - sort_json); + {toNullableDatetimeVec("partition", {"20220101010101", "20220101010101", "20220101010101", "20220101010101", "20220101010102", "20220101010102", "20220101010102", "20220101010102"}, 0), + toNullableDatetimeVec("order", {"20220101010101", "20220101010101", "20220101010102", "20220101010102", "20220101010101", "20220101010101", "20220101010102", "20220101010102"}, 0), + toNullableVec("row_number", {1, 2, 3, 4, 1, 2, 3, 4})}); // nullable - testOneWindowFunction( - {NameAndTypePair("partition", makeNullable(std::make_shared())), NameAndTypePair("order", makeNullable(std::make_shared()))}, + executeStreamsWithSingleSource( + request, {toNullableDatetimeVec("partition", {"20220101010102", {}, "20220101010102", "20220101010102", "20220101010102", "20220101010101", "20220101010101", "20220101010101", "20220101010101"}, 0), toNullableDatetimeVec("order", {"20220101010101", {}, "20220101010101", "20220101010102", "20220101010102", "20220101010101", "20220101010101", "20220101010102", "20220101010102"}, 0)}, {toNullableDatetimeVec("partition", {{}, "20220101010101", "20220101010101", "20220101010101", "20220101010101", "20220101010102", "20220101010102", "20220101010102", "20220101010102"}, 0), toNullableDatetimeVec("order", {{}, "20220101010101", "20220101010101", "20220101010102", "20220101010102", "20220101010101", "20220101010101", "20220101010102", "20220101010102"}, 0), - toNullableVec("row_number", {1, 1, 2, 3, 4, 1, 2, 3, 4})}, - window_json, - sort_json); + toNullableVec("row_number", {1, 1, 2, 3, 4, 1, 2, 3, 4})}); // 2 partiton key and 2 order key // sql : select *, row_number() over w1 from test6 window w1 as (partition by partition_int1, partition_int2 order by order_int1,order_int2) - window_json = R"({"funcDesc":[{"tp":"RowNumber","sig":"Unspecified","fieldType":{"tp":8,"flag":128,"flen":21,"decimal":-1,"collate":63,"charset":"binary"},"hasDistinct":false}],"partitionBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"orderBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAI=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAM=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"frame":{"type":"Rows","start":{"type":"CurrentRow","unbounded":false,"offset":"0"},"end":{"type":"CurrentRow","unbounded":false,"offset":"0"}},"child":{"tp":"TypeSort","executorId":"Sort_12","sort":{"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAI=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAM=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAI=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAM=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIKA0Img1If/BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}}}})"; - sort_json = R"({"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAI=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAM=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAI=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAM=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIKA0Img1If/BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}})"; - testOneWindowFunction( - {NameAndTypePair("partition1", std::make_shared()), NameAndTypePair("partition2", std::make_shared()), NameAndTypePair("order1", std::make_shared()), NameAndTypePair("order2", std::make_shared())}, - {toVec("partition1", {1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2}), toVec("partition2", {1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 2}), toVec("order1", {2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1}), toVec("order2", {2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 1})}, - {toVec("partition1", {1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2}), toVec("partition2", {1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 2}), toVec("order1", {1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2}), toVec("order2", {1, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2}), toNullableVec("row_number", {1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3})}, - window_json, - sort_json); + request = context + .scan("test_db", "test_table_more_cols") + .sort({{"partition1", false}, {"partition2", false}, {"order1", false}, {"order2", false}}, true) + .window(RowNumber(), {{"order1", false}, {"order2", false}}, {{"partition1", false}, {"partition2", false}}, buildDefaultRowsFrame()) + .build(context); + + executeStreams( + request, + {toNullableVec("partition1", {1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2}), + toNullableVec("partition2", {1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 2}), + toNullableVec("order1", {1, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2}), + toNullableVec("order2", {1, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2}), + toNullableVec("row_number", {1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3})}); /***** rank, dense_rank *****/ - window_json = R"({"funcDesc":[{"tp":"Rank","sig":"Unspecified","fieldType":{"tp":8,"flag":128,"flen":21,"decimal":-1,"collate":63,"charset":"binary"},"hasDistinct":false},{"tp":"DenseRank","sig":"Unspecified","fieldType":{"tp":8,"flag":128,"flen":21,"decimal":-1,"collate":63,"charset":"binary"},"hasDistinct":false}],"partitionBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"orderBy":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"child":{"tp":"TypeSort","executorId":"Sort_12","sort":{"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGAsOnl3NP+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}}}})"; - sort_json = R"({"byItems":[{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAA=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false},{"expr":{"tp":"ColumnRef","val":"gAAAAAAAAAE=","sig":"Unspecified","fieldType":{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},"hasDistinct":false},"desc":false}],"isPartialSort":true,"child":{"tp":"TypeExchangeReceiver","exchangeReceiver":{"encodedTaskMeta":["CIGAsOnl3NP+BRABIg4xMjcuMC4wLjE6MzkzMA=="],"fieldTypes":[{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"},{"tp":3,"flag":0,"flen":11,"decimal":0,"collate":63,"charset":"binary"}]},"executorId":"ExchangeReceiver_11"}})"; - testOneWindowFunction( - {NameAndTypePair("partition", std::make_shared()), NameAndTypePair("order", std::make_shared())}, - {toVec("partition", {1, 1, 1, 1, 2, 2, 2, 2}), toVec("order", {1, 1, 2, 2, 1, 1, 2, 2})}, - {toVec("partition", {1, 1, 1, 1, 2, 2, 2, 2}), toVec("order", {1, 1, 2, 2, 1, 1, 2, 2}), toNullableVec("rank", {1, 1, 3, 3, 1, 1, 3, 3}), toNullableVec("dense_rank", {1, 1, 2, 2, 1, 1, 2, 2})}, - window_json, - sort_json); + request = context.scan("test_db", "test_table_for_rank").sort({{"partition", false}, {"order", false}}, true).window({Rank(), DenseRank()}, {{"order", false}}, {{"partition", false}}, MockWindowFrame{}).build(context); + executeStreams( + request, + {toNullableVec("partition", {1, 1, 1, 1, 2, 2, 2, 2}), + toNullableVec("order", {1, 1, 2, 2, 1, 1, 2, 2}), + toNullableVec("rank", {1, 1, 3, 3, 1, 1, 3, 3}), + toNullableVec("dense_rank", {1, 1, 2, 2, 1, 1, 2, 2})}); // nullable - testOneWindowFunction( - {NameAndTypePair("partition", makeNullable(std::make_shared())), NameAndTypePair("order", makeNullable(std::make_shared()))}, - {toNullableVec("partition", {{}, 1, 1, 1, 1, 2, 2, 2, 2}), toNullableVec("order", {{}, 1, 1, 2, 2, 1, 1, 2, 2})}, - {toNullableVec("partition", {{}, 1, 1, 1, 1, 2, 2, 2, 2}), toNullableVec("order", {{}, 1, 1, 2, 2, 1, 1, 2, 2}), toNullableVec("rank", {1, 1, 1, 3, 3, 1, 1, 3, 3}), toNullableVec("dense_rank", {1, 1, 1, 2, 2, 1, 1, 2, 2})}, - window_json, - sort_json); - - testOneWindowFunction( - {NameAndTypePair("partition", makeNullable(std::make_shared())), NameAndTypePair("order", makeNullable(std::make_shared()))}, - {toNullableVec("partition", {{}, {}, 1, 1, 1, 1, 2, 2, 2, 2}), toNullableVec("order", {{}, 1, 1, 1, 2, 2, 1, 1, 2, 2})}, - {toNullableVec("partition", {{}, {}, 1, 1, 1, 1, 2, 2, 2, 2}), toNullableVec("order", {{}, 1, 1, 1, 2, 2, 1, 1, 2, 2}), toNullableVec("rank", {1, 2, 1, 1, 3, 3, 1, 1, 3, 3}), toNullableVec("dense_rank", {1, 2, 1, 1, 2, 2, 1, 1, 2, 2})}, - window_json, - sort_json); + executeStreamsWithSingleSource( + request, + {toNullableVec("partition", {{}, 1, 1, 1, 1, 2, 2, 2, 2}), + toNullableVec("order", {{}, 1, 1, 2, 2, 1, 1, 2, 2})}, + {toNullableVec("partition", {{}, 1, 1, 1, 1, 2, 2, 2, 2}), + toNullableVec("order", {{}, 1, 1, 2, 2, 1, 1, 2, 2}), + toNullableVec("rank", {1, 1, 1, 3, 3, 1, 1, 3, 3}), + toNullableVec("dense_rank", {1, 1, 1, 2, 2, 1, 1, 2, 2})}); + + executeStreamsWithSingleSource( + request, + {toNullableVec("partition", {{}, {}, 1, 1, 1, 1, 2, 2, 2, 2}), + toNullableVec("order", {{}, 1, 1, 1, 2, 2, 1, 1, 2, 2})}, + {toNullableVec("partition", {{}, {}, 1, 1, 1, 1, 2, 2, 2, 2}), + toNullableVec("order", {{}, 1, 1, 1, 2, 2, 1, 1, 2, 2}), + toNullableVec("rank", {1, 2, 1, 1, 3, 3, 1, 1, 3, 3}), + toNullableVec("dense_rank", {1, 2, 1, 1, 2, 2, 1, 1, 2, 2})}); } CATCH + } // namespace DB::tests diff --git a/etc/config-template.toml b/etc/config-template.toml index cad45dc8105..f56a6a095d4 100644 --- a/etc/config-template.toml +++ b/etc/config-template.toml @@ -130,10 +130,6 @@ # pd_addr = "pd0:2379" # specify which storage engine we use. tmt or dt TODO: Remove deprecated tmt engine # storage_engine = "dt" -[raft.snapshot] -# The way to apply snapshot data -# The value is one of "block" / "file1" -# method = "file1" [status] # The port through which Prometheus pulls metrics information. diff --git a/libs/libcommon/include/common/getMemoryAmount.h b/libs/libcommon/include/common/getMemoryAmount.h index 98aa87661c3..0807c6f8e12 100644 --- a/libs/libcommon/include/common/getMemoryAmount.h +++ b/libs/libcommon/include/common/getMemoryAmount.h @@ -19,5 +19,6 @@ /** * Returns the size of physical memory (RAM) in bytes. * Returns 0 on unsupported platform +* Note: do not support environment under resource isolation mechanism like Docker, CGroup. */ uint64_t getMemoryAmount(); diff --git a/release-centos7-llvm/scripts/build-tiflash-release.sh b/release-centos7-llvm/scripts/build-tiflash-release.sh index bb62e4743f6..42993b51afe 100755 --- a/release-centos7-llvm/scripts/build-tiflash-release.sh +++ b/release-centos7-llvm/scripts/build-tiflash-release.sh @@ -42,6 +42,7 @@ SRCPATH=$( ) NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} ENABLE_THINLTO=${ENABLE_THINLTO:-ON} +ENABLE_PCH=${ENABLE_PCH:-ON} INSTALL_DIR="${SRCPATH}/release-centos7-llvm/tiflash" rm -rf ${INSTALL_DIR} && mkdir -p ${INSTALL_DIR} @@ -59,6 +60,7 @@ cmake -S "${SRCPATH}" \ -DRUN_HAVE_STD_REGEX=0 \ -DENABLE_THINLTO=${ENABLE_THINLTO} \ -DTHINLTO_JOBS=${NPROC} \ + -DENABLE_PCH=${ENABLE_PCH} \ -GNinja cmake --build . --target tiflash --parallel ${NPROC} diff --git a/tests/docker/config/tics_dt.toml b/tests/docker/config/tics_dt.toml index 89147f80c7d..56bef659cb7 100644 --- a/tests/docker/config/tics_dt.toml +++ b/tests/docker/config/tics_dt.toml @@ -13,25 +13,16 @@ # limitations under the License. tmp_path = "/tmp/tiflash/data/tmp" -display_name = "TiFlash" -# specify paths used for store data, multiple path should be seperated by comma path = "/tmp/tiflash/data/db" -capacity = "107374182400" -# multi-paths example -# path = "/tmp/tiflash/data/db,/tmp/tiflash1,/tmp/tiflash2" -# capacity = "0,0,0" +capacity = "107374182400" # 100GB mark_cache_size = 5368709120 minmax_index_cache_size = 5368709120 tcp_port = 9000 http_port = 8123 + [logger] count = 10 errorlog = "/tmp/tiflash/log/error.log" size = "1000M" log = "/tmp/tiflash/log/server.log" level = "trace" -[application] -runAsDaemon = true -[raft] -# specify which storage engine we use. tmt or dt -storage_engine = "dt" diff --git a/tests/docker/config/tiflash_dt_async_grpc.toml b/tests/docker/config/tiflash_dt_async_grpc.toml index 3c67c37db33..bf31c61cfa8 100644 --- a/tests/docker/config/tiflash_dt_async_grpc.toml +++ b/tests/docker/config/tiflash_dt_async_grpc.toml @@ -13,71 +13,15 @@ # limitations under the License. tmp_path = "/tmp/tiflash/data/tmp" -display_name = "TiFlash" -## Deprecated storage path setting style. Check [storage] section for new style. path = "/tmp/tiflash/data/db" capacity = "10737418240" -## Deprecated storage path setting style of multi-disks. Check [storage] section for new style. -# path = "/tmp/tiflash/data/db,/tmp/tiflash1,/tmp/tiflash2" -# capacity = "0" mark_cache_size = 5368709120 minmax_index_cache_size = 5368709120 tcp_port = 9000 http_port = 8123 -## Storage paths settings. -# [storage] -## The storage format version in storage engine. Valid values: 1, 2 (experimental). -## format_version = 1 - -## If there are multiple SSD disks on the machine, -## specify the path list on `storage.main.dir` can improve TiFlash performance. - -## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs) -## on the machine, -## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics) -## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics) -## can improve TiFlash performance. - -# [storage.main] -## The path to store main data. -# e.g. -# dir = [ "/data0/tiflash" ] -# or -# dir = [ "/data0/tiflash", "/data1/tiflash" ] - -## Store capacity of each path, i.e. max data size allowed. -## If it is not set, or is set to 0s, the actual disk capacity is used. -## Note that we don't support human-readable big numbers(like "10GB") yet. -## Please set in the specified number of bytes. -# e.g. -# capacity = [ 10737418240, 10737418240 ] - -# [storage.latest] -## The path(s) to store latest data. -## If not set, it will be the same with `storage.main.dir`. -# dir = [ ] - -## Store capacity of each path, i.e. max data size allowed. -## If it is not set, or is set to 0s, the actual disk capacity is used. -# e.g. -# capacity = [ 10737418240, 10737418240 ] - -# [storage.raft] -## The path(s) to store Raft data. -## If not set, it will be the paths in `storage.latest.dir` appended with "/kvstore". -# dir = [ ] - -# [storage.io_rate_limit] -## The max I/O bandwith. Default value is 0 and I/O rate limit is disabled. -# max_bytes_per_sec = 268435456 -## max_read_bytes_per_sec and max_write_bytes_per_sec are the same meaning as max_bytes_per_sec, -## but for disk that read bandwidth and write bandwith are calculated separatly, such as GCP's persistent disks. -# max_read_bytes_per_sec = 0 -# max_write_bytes_per_sec = 0 - [flash] tidb_status_addr = "tidb0:10080" service_addr = "0.0.0.0:3930" @@ -100,22 +44,9 @@ size = "1000M" log = "/tmp/tiflash/log/server.log" level = "trace" -[application] -runAsDaemon = true - [raft] pd_addr = "pd0:2379" ignore_databases = "system,default" -# specify which storage engine we use. tmt or dt -storage_engine = "dt" -# Deprecated Raft data storage path setting style. Check [storage.raft] section for new style. -# If it is not set, it will be the first path of "path" appended with "/kvstore". -# kvstore_path = "" - -[raft.snapshot] -# The way to apply snapshot data -# The value is one of "block" / "file1" / "file2". -# method = "file1" [profiles] [profiles.default] diff --git a/tests/docker/config/tiflash_dt_disable_local_tunnel.toml b/tests/docker/config/tiflash_dt_disable_local_tunnel.toml index 23b82909776..1fb166a9a19 100644 --- a/tests/docker/config/tiflash_dt_disable_local_tunnel.toml +++ b/tests/docker/config/tiflash_dt_disable_local_tunnel.toml @@ -13,71 +13,15 @@ # limitations under the License. tmp_path = "/tmp/tiflash/data/tmp" -display_name = "TiFlash" -## Deprecated storage path setting style. Check [storage] section for new style. path = "/tmp/tiflash/data/db" capacity = "10737418240" -## Deprecated storage path setting style of multi-disks. Check [storage] section for new style. -# path = "/tmp/tiflash/data/db,/tmp/tiflash1,/tmp/tiflash2" -# capacity = "0" mark_cache_size = 5368709120 minmax_index_cache_size = 5368709120 tcp_port = 9000 http_port = 8123 -## Storage paths settings. -# [storage] -## The storage format version in storage engine. Valid values: 1, 2 (experimental). -## format_version = 1 - -## If there are multiple SSD disks on the machine, -## specify the path list on `storage.main.dir` can improve TiFlash performance. - -## If there are multiple disks with different IO metrics (e.g. one SSD and some HDDs) -## on the machine, -## set `storage.latest.dir` to store the latest data on SSD (disks with higher IOPS metrics) -## set `storage.main.dir` to store the main data on HDD (disks with lower IOPS metrics) -## can improve TiFlash performance. - -# [storage.main] -## The path to store main data. -# e.g. -# dir = [ "/data0/tiflash" ] -# or -# dir = [ "/data0/tiflash", "/data1/tiflash" ] - -## Store capacity of each path, i.e. max data size allowed. -## If it is not set, or is set to 0s, the actual disk capacity is used. -## Note that we don't support human-readable big numbers(like "10GB") yet. -## Please set in the specified number of bytes. -# e.g. -# capacity = [ 10737418240, 10737418240 ] - -# [storage.latest] -## The path(s) to store latest data. -## If not set, it will be the same with `storage.main.dir`. -# dir = [ ] - -## Store capacity of each path, i.e. max data size allowed. -## If it is not set, or is set to 0s, the actual disk capacity is used. -# e.g. -# capacity = [ 10737418240, 10737418240 ] - -# [storage.raft] -## The path(s) to store Raft data. -## If not set, it will be the paths in `storage.latest.dir` appended with "/kvstore". -# dir = [ ] - -# [storage.io_rate_limit] -## The max I/O bandwith. Default value is 0 and I/O rate limit is disabled. -# max_bytes_per_sec = 268435456 -## max_read_bytes_per_sec and max_write_bytes_per_sec are the same meaning as max_bytes_per_sec, -## but for disk that read bandwidth and write bandwith are calculated separatly, such as GCP's persistent disks. -# max_read_bytes_per_sec = 0 -# max_write_bytes_per_sec = 0 - [flash] tidb_status_addr = "tidb0:10080" service_addr = "0.0.0.0:3930" @@ -100,22 +44,9 @@ size = "1000M" log = "/tmp/tiflash/log/server.log" level = "trace" -[application] -runAsDaemon = true - [raft] pd_addr = "pd0:2379" ignore_databases = "system,default" -# specify which storage engine we use. tmt or dt -storage_engine = "dt" -# Deprecated Raft data storage path setting style. Check [storage.raft] section for new style. -# If it is not set, it will be the first path of "path" appended with "/kvstore". -# kvstore_path = "" - -[raft.snapshot] -# The way to apply snapshot data -# The value is one of "block" / "file1" / "file2". -# method = "file1" [profiles] [profiles.default] diff --git a/tests/fullstack-test2/ddl/alter_column_when_pk_is_handle.test b/tests/fullstack-test2/ddl/alter_column_when_pk_is_handle.test index ca92828e6cf..df0aa13823a 100644 --- a/tests/fullstack-test2/ddl/alter_column_when_pk_is_handle.test +++ b/tests/fullstack-test2/ddl/alter_column_when_pk_is_handle.test @@ -37,9 +37,25 @@ mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t | 1 | world | 0.00 | 2 | NULL | +---+-------+------+------+------+ -# Need to apply a lossy type change to reorganize data. issue#3714 +=> DBGInvoke __enable_schema_sync_service('false') + +>> DBGInvoke __enable_fail_point(exception_between_schema_change_in_the_same_diff) + +# stop decoding data +>> DBGInvoke __enable_fail_point(pause_before_apply_raft_cmd) + +# Need to apply a lossy type change to reorganize data. issue#3714 mysql> alter table test.t modify c decimal(6,3) +# refresh schema and hit the `exception_between_schema_change_in_the_same_diff` failpoint +>> DBGInvoke __refresh_schemas() + +>> DBGInvoke __disable_fail_point(exception_between_schema_change_in_the_same_diff) + +>> DBGInvoke __disable_fail_point(pause_before_apply_raft_cmd) + +=> DBGInvoke __enable_schema_sync_service('true') + mysql> set session tidb_isolation_read_engines='tiflash'; select * from test.t +---+-------+-------+------+------+ | a | b | c | d | e |