Merge branch 'master' into add-more-test-in-mix-mode

CMakeLists.txt

@@ -48,6 +48,14 @@ endif ()
 set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 
 option (USE_CCACHE "Set to OFF to disable ccache" ON)
+if (USE_CCACHE)
+    set(NOT_USE_CCACHE 0)
+else()
+    set(NOT_USE_CCACHE 1)
+endif()
+
+option(ENABLE_PCH "Enable `Precompiled header`" OFF)
+
 include (cmake/find_ccache.cmake)
 
 if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
@@ -424,9 +432,6 @@ else (ENABLE_FAILPOINTS)
     message (STATUS "Failpoints are disabled")
 endif (ENABLE_FAILPOINTS)
 
-# Enable PageStorage V3 test.
-option (ENABLE_V3_PAGESTORAGE "Enables V3 PageStorage" ON)
-
 # Flags for test coverage
 option (TEST_COVERAGE "Enables flags for test coverage" OFF)
 option (TEST_COVERAGE_XML "Output XML report for test coverage" OFF)

README.md

@@ -12,7 +12,7 @@ TiFlash repository is based on [ClickHouse](https://github.com/ClickHouse/ClickH
 
 ### Start with TiDB Cloud
 
-Quickly explore TiFlash with [a free trial of TiDB Cloud](https://tidbcloud.com/signup).
+Quickly explore TiFlash with [a free trial of TiDB Cloud](https://tidbcloud.com/free-trial).
 
 See [TiDB Cloud Quick Start Guide](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart).
 

cmake/find_ccache.cmake

@@ -24,6 +24,20 @@ if (USE_CCACHE AND CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "cca
     message ("${CCACHE_CONFIG}")
     set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
     set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
+
+    if (ENABLE_PCH)
+        execute_process (COMMAND ${CCACHE_FOUND} --get-config sloppiness OUTPUT_VARIABLE _CCACHE_SLOPPINESS OUTPUT_STRIP_TRAILING_WHITESPACE)
+        string (FIND "${_CCACHE_SLOPPINESS}" "pch_defines" _CCACHE_SLOPPINESS_RES)
+        if (NOT _CCACHE_SLOPPINESS_RES STREQUAL "-1")
+            string (FIND "${_CCACHE_SLOPPINESS}" "time_macros" _CCACHE_SLOPPINESS_RES)
+        endif ()
+
+        if (_CCACHE_SLOPPINESS_RES STREQUAL "-1")
+            message(WARNING "`Precompiled header` won't be cached by ccache, sloppiness = `${CCACHE_SLOPPINESS}`,please execute `ccache -o sloppiness=pch_defines,time_macros`")
+            set (ENABLE_PCH FALSE CACHE BOOL "" FORCE)
+        endif ()
+    endif ()
+
 else ()
     message (STATUS "Not using ccache ${CCACHE_FOUND}, USE_CCACHE=${USE_CCACHE}")
 endif ()

dbms/CMakeLists.txt

@@ -91,12 +91,10 @@ add_headers_and_sources(dbms src/Storages/Page/V2/VersionSet)
 add_headers_and_sources(dbms src/Storages/Page/V2/gc)
 add_headers_and_sources(dbms src/WindowFunctions)
 add_headers_and_sources(dbms src/TiDB/Schema)
-if (ENABLE_V3_PAGESTORAGE)
-    add_headers_and_sources(dbms src/Storages/Page/V3)
-    add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
-    add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
-    add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
-endif()
+add_headers_and_sources(dbms src/Storages/Page/V3)
+add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
+add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
+add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
 add_headers_and_sources(dbms src/Storages/Page/)
 add_headers_and_sources(dbms src/TiDB)
 add_headers_and_sources(dbms src/Client)
@@ -259,6 +257,16 @@ target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE
 # https://cmake.org/pipermail/cmake/2016-May/063400.html
 target_link_libraries (clickhouse_common_io PUBLIC ${TIFLASH_XXHASH_LIBRARY})
 
+function(add_target_pch context target)
+    if (ENABLE_PCH)
+        message(STATUS "Add PCH `${context}` for target `${target}`")
+        target_precompile_headers(${target} PRIVATE ${context})
+    endif ()
+    if(${ARGC} GREATER 2)
+        add_target_pch(${context} ${ARGN})
+    endif()
+endfunction()
+
 if (ENABLE_TESTS)
     include (${TiFlash_SOURCE_DIR}/cmake/find_gtest.cmake)
 
@@ -297,6 +305,8 @@ if (ENABLE_TESTS)
     target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy)
     add_check(gtests_dbms)
 
+    add_target_pch("pch-dbms.h" gtests_dbms)
+
     grep_bench_sources(${TiFlash_SOURCE_DIR}/dbms dbms_bench_sources)
     add_executable(bench_dbms EXCLUDE_FROM_ALL
         ${dbms_bench_sources}
@@ -311,6 +321,9 @@ if (ENABLE_TESTS)
     if (ENABLE_TIFLASH_DTWORKLOAD)
         target_link_libraries(bench_dbms dt-workload-lib)
     endif ()
+    if (ENABLE_TIFLASH_PAGEWORKLOAD)
+        target_link_libraries(bench_dbms page-workload-lib)
+    endif ()
 
     add_check(bench_dbms)
 endif ()
@@ -342,3 +355,23 @@ if (TEST_COVERAGE AND CMAKE_BUILD_TYPE STREQUAL "Debug")
         )
     endif ()
 endif ()
+
+# dbms
+add_target_pch("pch-dbms.h" dbms flash_service)
+add_target_pch("pch-common.h" clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions)
+add_target_pch("pch-common.h" clickhouse_parsers clickhouse_storages_system dt-workload-lib clickhouse-server-lib)
+
+# common
+add_target_pch("pch-kvpb.h" kv_client)
+
+add_target_pch("pch-stl.h" ${Boost_SYSTEM_LIBRARY} cctz ${RE2_LIBRARY} ${RE2_ST_LIBRARY})
+
+# grpc
+add_target_pch("$<$<COMPILE_LANGUAGE:CXX>:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" grpc grpc++)
+
+# pb
+add_target_pch("pch-stl.h" libprotobuf kvproto tipb libprotoc)
+
+# poco
+add_target_pch("pch-stl.h" Net Crypto Util Data NetSSL)
+add_target_pch("$<$<COMPILE_LANGUAGE:CXX>:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" XML Foundation JSON)

dbms/src/TableFunctions/TableFunctionFile.h → dbms/pch-common.h

@@ -14,26 +14,10 @@
 
 #pragma once
 
-#include <TableFunctions/ITableFunction.h>
+#include <Common/Exception.h>
+#include <Common/FmtUtils.h>
+#include <Common/StackTrace.h>
+#include <common/StringRef.h>
+#include <common/types.h>
 
-
-namespace DB
-{
-/* file(path, format, structure) - creates a temporary storage from file
- *
- *
- * The file must be in the clickhouse data directory.
- * The relative path begins with the clickhouse data directory.
- */
-class TableFunctionFile : public ITableFunction
-{
-public:
-    static constexpr auto name = "file";
-    std::string getName() const override { return name; }
-
-private:
-    StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context) const override;
-};
-
-
-} // namespace DB
+#include "pch-stl.h"

dbms/pch-dbms.h

@@ -0,0 +1,18 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "pch-common.h"
+#include "pch-kvpb.h"

dbms/pch-kvpb.h

@@ -0,0 +1,26 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+/** 
+There may be unexpected behaviors for `ccache` to deal with PCH which includes those header files generated by tools: 
+    'xxx' has been modified since the precompiled header 'xxx' was built: mtime changed
+    `Precompiled header includes xxx.h, which has a new mtime`
+*/
+#include <kvproto/enginepb.pb.h>
+#include <kvproto/pdpb.grpc.pb.h>
+#include <kvproto/tikvpb.grpc.pb.h>
+
+#include "pch-stl.h"

dbms/pch-stl.h

@@ -0,0 +1,34 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <algorithm>
+#include <cassert>
+#include <chrono>
+#include <cstdint>
+#include <fstream>
+#include <functional>
+#include <future>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <ostream>
+#include <queue>
+#include <random>
+#include <shared_mutex>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>

dbms/src/Common/FailPoint.cpp

@@ -65,7 +65,8 @@ std::unordered_map<String, std::shared_ptr<FailPointChannel>> FailPointHelper::f
     M(exception_when_read_from_log)                               \
     M(exception_mpp_hash_build)                                   \
     M(exception_before_drop_segment)                              \
-    M(exception_after_drop_segment)
+    M(exception_after_drop_segment)                               \
+    M(exception_between_schema_change_in_the_same_diff)
 
 #define APPLY_FOR_FAILPOINTS(M)                              \
     M(skip_check_segment_update)                             \

dbms/src/Common/FmtUtils.h

@@ -14,8 +14,6 @@
 
 #pragma once
 
-#include <common/StringRef.h>
-#include <fmt/core.h>
 #include <fmt/format.h>
 
 namespace DB
@@ -32,9 +30,9 @@ class FmtBuffer
         return *this;
     }
 
-    FmtBuffer & append(StringRef s)
+    FmtBuffer & append(std::string_view s)
     {
-        buffer.append(s.data, s.data + s.size);
+        buffer.append(s.data(), s.data() + s.size());
         return *this;
     }
 
@@ -55,7 +53,7 @@ class FmtBuffer
     FmtBuffer & joinStr(
         Iter first,
         Iter end,
-        StringRef delimiter)
+        std::string_view delimiter)
     {
         auto func = [](const auto & s, FmtBuffer & fb) {
             fb.append(s);
@@ -68,7 +66,7 @@ class FmtBuffer
         Iter first,
         Iter end,
         FF && toStringFunc, // void (const auto &, FmtBuffer &)
-        StringRef delimiter)
+        std::string_view delimiter)
     {
         if (first == end)
             return *this;

dbms/src/Common/TiFlashException.cpp

@@ -18,6 +18,32 @@
 
 namespace DB
 {
+struct TiFlashErrorRegistry::Errors : std::map<std::pair<std::string, std::string>, TiFlashError>
+{
+};
+
+TiFlashErrorRegistry::Errors & TiFlashErrorRegistry::errors()
+{
+    return *inner_data;
+}
+
+TiFlashErrorRegistry::Errors & TiFlashErrorRegistry::errors() const
+{
+    return *inner_data;
+}
+
+TiFlashErrorRegistry::TiFlashErrorRegistry()
+    : inner_data(new Errors{})
+{
+    initialize();
+}
+
+TiFlashErrorRegistry::~TiFlashErrorRegistry()
+{
+    delete inner_data;
+    inner_data = nullptr;
+}
+
 void TiFlashErrorRegistry::initialize()
 {
     // Used to check uniqueness of classes
@@ -46,9 +72,9 @@ void TiFlashErrorRegistry::initialize()
 void TiFlashErrorRegistry::registerError(const std::string & error_class, const std::string & error_code, const std::string & description, const std::string & workaround, const std::string & message_template)
 {
     TiFlashError error{error_class, error_code, description, workaround, message_template};
-    if (all_errors.find({error_class, error_code}) == all_errors.end())
+    if (errors().find({error_class, error_code}) == errors().end())
     {
-        all_errors.emplace(std::make_pair(error_class, error_code), std::move(error));
+        errors().emplace(std::make_pair(error_class, error_code), std::move(error));
     }
     else
     {
@@ -77,4 +103,51 @@ std::string TiFlashException::standardText() const
     return text;
 }
 
+std::optional<TiFlashError> TiFlashErrorRegistry::get(const std::string & error_class, const std::string & error_code) const
+{
+    auto error = errors().find({error_class, error_code});
+    if (error != errors().end())
+    {
+        return error->second;
+    }
+    else
+    {
+        return {};
+    }
+}
+std::optional<TiFlashError> TiFlashErrorRegistry::get(const std::string & error_class, int error_code) const
+{
+    return get(error_class, std::to_string(error_code));
+}
+
+std::vector<TiFlashError> TiFlashErrorRegistry::allErrors() const
+{
+    std::vector<TiFlashError> res;
+    res.reserve(errors().size());
+    for (const auto & error : errors())
+    {
+        res.push_back(error.second);
+    }
+    return res;
+}
+
+TiFlashError TiFlashErrorRegistry::simpleGet(const std::string & error_class, const std::string & error_code)
+{
+    auto & m_instance = instance();
+    auto error = m_instance.get(error_class, error_code);
+    if (error.has_value())
+    {
+        return error.value();
+    }
+    else
+    {
+        throw Exception("Unregistered TiFlashError: FLASH:" + error_class + ":" + error_code);
+    }
+}
+TiFlashError TiFlashErrorRegistry::simpleGet(const std::string & error_class, int error_code)
+{
+    return simpleGet(error_class, std::to_string(error_code));
+}
+
+
 } // namespace DB