fix conflict

.clang-tidy

@@ -18,9 +18,11 @@ Checks: '-*,
     modernize-redundant-void-arg,
     modernize-replace-auto-ptr,
     modernize-replace-random-shuffle,
+    modernize-use-auto,
     modernize-use-bool-literals,
     modernize-use-nullptr,
     modernize-use-using,
+    modernize-use-override,
     modernize-use-equals-default,
     modernize-use-equals-delete,
 
@@ -160,6 +162,11 @@ Checks: '-*,
     clang-analyzer-unix.cstring.NullArg,
 
     boost-use-to-string,
+    
+    cppcoreguidelines-pro-type-cstyle-cast,
+    cppcoreguidelines-pro-type-member-init,
+    cppcoreguidelines-no-malloc,
+    cppcoreguidelines-virtual-class-destructor,
 '
 WarningsAsErrors: '*'
 

CMakeLists.txt

@@ -48,6 +48,14 @@ endif ()
 set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 
 option (USE_CCACHE "Set to OFF to disable ccache" ON)
+if (USE_CCACHE)
+    set(NOT_USE_CCACHE 0)
+else()
+    set(NOT_USE_CCACHE 1)
+endif()
+
+option(ENABLE_PCH "Enable `Precompiled header`" OFF)
+
 include (cmake/find_ccache.cmake)
 
 if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
@@ -102,6 +110,28 @@ if (COMPILER_CLANG)
     # Clang doesn't have int128 predefined macros, workaround by manually defining them
     # Reference: https://stackoverflow.com/questions/41198673/uint128-t-not-working-with-clang-and-libstdc
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__GLIBCXX_BITSIZE_INT_N_0=128 -D__GLIBCXX_TYPE_INT_N_0=__int128")
+
+    option(ENABLE_TIME_TRACES "Enable clang feature time traces" OFF)
+    if (ENABLE_TIME_TRACES)
+        set (CLANG_TIME_TRACES_FLAGS "-ftime-trace")
+        message (STATUS "Using clang time traces flag `${CLANG_TIME_TRACES_FLAGS}`. Generates JSON file based on output filename. Results can be analyzed with chrome://tracing or https://www.speedscope.app for flamegraph visualization.")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CLANG_TIME_TRACES_FLAGS}")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_TIME_TRACES_FLAGS}")
+    endif ()
+
+    # https://clang.llvm.org/docs/ThinLTO.html
+    # Applies to clang only.
+    option(ENABLE_THINLTO "Clang-specific link time optimization" OFF)
+
+    if (ENABLE_THINLTO AND NOT ENABLE_TESTS)
+        # Link time optimization
+        set (THINLTO_JOBS "0" CACHE STRING "ThinLTO compilation parallelism")
+        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -flto=thin -fvisibility=hidden -fvisibility-inlines-hidden -fsplit-lto-unit")
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin -fvisibility=hidden -fvisibility-inlines-hidden -fwhole-program-vtables -fsplit-lto-unit")
+        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto=thin -flto-jobs=${THINLTO_JOBS} -fvisibility=hidden -fvisibility-inlines-hidden -fwhole-program-vtables -fsplit-lto-unit")
+    elseif (ENABLE_THINLTO)
+        message (WARNING "Cannot enable ThinLTO")
+    endif ()
 endif ()
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
@@ -402,9 +432,6 @@ else (ENABLE_FAILPOINTS)
     message (STATUS "Failpoints are disabled")
 endif (ENABLE_FAILPOINTS)
 
-# Enable PageStorage V3 test.
-option (ENABLE_V3_PAGESTORAGE "Enables V3 PageStorage" ON)
-
 # Flags for test coverage
 option (TEST_COVERAGE "Enables flags for test coverage" OFF)
 option (TEST_COVERAGE_XML "Output XML report for test coverage" OFF)

README.md

@@ -1,29 +1,44 @@
 # TiFlash
+
 ![tiflash-architecture](tiflash-architecture.png)
 
-[TiFlash](https://docs.pingcap.com/tidb/stable/tiflash-overview) is a columnar storage component of [TiDB](https://docs.pingcap.com/tidb/stable). It mainly plays the role of Analytical Processing (AP) in the Hybrid Transactional/Analytical Processing (HTAP) architecture of TiDB.
+[TiFlash](https://docs.pingcap.com/tidb/stable/tiflash-overview) is a columnar storage component of [TiDB](https://docs.pingcap.com/tidb/stable) and [TiDB Cloud](https://en.pingcap.com/tidb-cloud/), the fully-managed service of TiDB. It mainly plays the role of Analytical Processing (AP) in the Hybrid Transactional/Analytical Processing (HTAP) architecture of TiDB.
 
 TiFlash stores data in columnar format and synchronizes data updates in real-time from [TiKV](https://github.com/tikv/tikv) by Raft logs with sub-second latency. Reads in TiFlash are guaranteed transactionally consistent with Snapshot Isolation level. TiFlash utilizes Massively Parallel Processing (MPP) computing architecture to accelerate the analytical workloads.
 
-TiFlash repository is based on [ClickHouse](https://github.com/ClickHouse/ClickHouse). We appreciate the excellent work of ClickHouse team.
+TiFlash repository is based on [ClickHouse](https://github.com/ClickHouse/ClickHouse). We appreciate the excellent work of the ClickHouse team.
+
+## Quick Start
+
+### Start with TiDB Cloud
+
+Quickly explore TiFlash with [a free trial of TiDB Cloud](https://tidbcloud.com/free-trial).
+
+See [TiDB Cloud Quick Start Guide](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart).
+
+### Start with TiDB
+
+See [Quick Start with HTAP](https://docs.pingcap.com/tidb/stable/quick-start-with-htap) and [Use TiFlash](https://docs.pingcap.com/tidb/stable/use-tiflash).
 
 ## Build TiFlash
 
 TiFlash supports building on the following hardware architectures:
-* x86-64/amd64
-* aarch64
+
+- x86-64/amd64
+- aarch64
 
 And the following operating systems:
-* Linux
-* MacOS
+
+- Linux
+- MacOS
 
 ### 1. Checkout Source Code
 
 Assume `$WORKSPACE` to be the directory under which the TiFlash repo is placed.
 
 ```shell
 cd $WORKSPACE
-git clone --recursive https://github.com/pingcap/tiflash.git
+git clone https://github.com/pingcap/tiflash.git --recursive -j 20
 ```
 
 ### 2. Prepare Prerequisites
@@ -44,19 +59,20 @@ The following packages are needed for all platforms:
 - Ninja or GNU Make
 
 The following are platform-specific prerequisites. Click to expand details:
+
 <details>
 <summary><b>Linux specific prerequisites</b></summary>
 
-  TiFlash can be built using either LLVM or GCC toolchain on Linux. LLVM toolchain is our official one for releasing.
-  > But for GCC, only GCC 7.x is supported as far, and is not planned to be a long term support. So it may get broken some day, silently.
-  
+TiFlash can be built using either LLVM or GCC toolchain on Linux. LLVM toolchain is our official one for releasing.
+
+> But for GCC, only GCC 7.x is supported as far, and is not planned to be a long term support. So it may get broken some day, silently.
+
 - LLVM 13.0.0+
 
   TiFlash compiles using full LLVM toolchain (`clang/compiler-rt/libc++/libc++abi`) by default. You can use a system-wise toolchain if `clang/compiler-rt/libc++/libc++abi` can be installed in your environment.
 
   Click sections below to see detailed instructions:
 
-
   <details>
   <summary><b>Set up LLVM via package managers in Debian/Ubuntu</b></summary>
 
@@ -123,6 +139,8 @@ cmake $WORKSPACE/tiflash
 make tiflash -j
 ```
 
+> **NOTE**: Option `-j` (defaults to your system CPU core count, otherwise you can optionally specify a number) is used to control the build parallelism. Higher parallelism consumes more memory. If you encounter compiler OOM or hang, try to lower the parallelism by specifying a reasonable number, e.g., half of your system CPU core count or even smaller, after `-j`, depending on the available memory in your system.
+
 After building, you can get TiFlash binary under `$BUILD/dbms/src/Server/tiflash`.
 
 ### Build Options
@@ -195,11 +213,12 @@ To run unit tests, you need to build with `-DCMAKE_BUILD_TYPE=DEBUG`:
 ```shell
 cd $BUILD
 cmake $WORKSPACE/tiflash -GNinja -DCMAKE_BUILD_TYPE=DEBUG
-ninja gtests_dbms
+ninja gtests_dbms       # Most TiFlash unit tests
+ninja gtests_libdaemon  # Settings related tests
 ninja gtests_libcommon
-ninja gtests_libdaemon
 ```
-And the unit-test executables are at `$BUILD/dbms/gtests_dbms`, `$BUILD/libs/libcommon/src/tests/gtests_libcommon` and `$BUILD/libs/libdaemon/src/tests/gtests_libdaemon`.
+
+And the unit-test executables are at `$BUILD/dbms/gtests_dbms`, `$BUILD/libs/libdaemon/src/tests/gtests_libdaemon` and `$BUILD/libs/libcommon/src/tests/gtests_libcommon`.
 
 ## Run Sanitizer Tests
 
@@ -211,19 +230,42 @@ To generate unit test executables with sanitizer enabled:
 cd $BUILD
 cmake $WORKSPACE/tiflash -GNinja -DENABLE_TESTS=ON -DCMAKE_BUILD_TYPE=ASan # or TSan
 ninja gtests_dbms
-ninja gtests_libcommon
 ninja gtests_libdaemon
+ninja gtests_libcommon
 ```
 
 There are known false positives reported from leak sanitizer (which is included in address sanitizer). To suppress these errors, set the following environment variables before running the executables:
 
-```
+```shell
 LSAN_OPTIONS=suppressions=$WORKSPACE/tiflash/test/sanitize/asan.suppression
 ```
 
 ## Run Integration Tests
 
-TBD.
+1. Build your own tiflash binary in $BUILD with `-DCMAKE_BUILD_TYPE=DEBUG`.
+```
+cd $BUILD
+cmake $WORKSPACE/tiflash -GNinja -DCMAKE_BUILD_TYPE=DEBUG
+ninja tiflash
+```
+2. Run tidb cluster locally using tiup playgroud or other tools. 
+```
+tiup playground nightly --tiflash.binpath $BUILD/dbms/src/Server/tiflash
+```
+3. Check $WORKSPACE/tests/_env.sh to make the port and build dir right.
+4. Run your integration tests using commands like "./run-test.sh fullstack-test2/ddl" under $WORKSPACE dir
+
+## Run MicroBenchmark Tests
+
+To run micro benchmark tests, you need to build with -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_TESTS=ON:
+
+```shell
+cd $BUILD
+cmake $WORKSPACE/tiflash -GNinja -DCMAKE_BUILD_TYPE=DEBUG -DENABLE_TESTS=ON
+ninja bench_dbms       
+```
+
+And the microbenchmark-test executables are at `$BUILD/dbms/bench_dbms`, you can run it with `./bench_dbms` or `./bench_dbms --benchmark_filter=xxx` . More usage please check with `./bench_dbms --help`.
 
 ## Generate LLVM Coverage Report
 

cmake/find_ccache.cmake

@@ -24,6 +24,20 @@ if (USE_CCACHE AND CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "cca
     message ("${CCACHE_CONFIG}")
     set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
     set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
+
+    if (ENABLE_PCH)
+        execute_process (COMMAND ${CCACHE_FOUND} --get-config sloppiness OUTPUT_VARIABLE _CCACHE_SLOPPINESS OUTPUT_STRIP_TRAILING_WHITESPACE)
+        string (FIND "${_CCACHE_SLOPPINESS}" "pch_defines" _CCACHE_SLOPPINESS_RES)
+        if (NOT _CCACHE_SLOPPINESS_RES STREQUAL "-1")
+            string (FIND "${_CCACHE_SLOPPINESS}" "time_macros" _CCACHE_SLOPPINESS_RES)
+        endif ()
+
+        if (_CCACHE_SLOPPINESS_RES STREQUAL "-1")
+            message(WARNING "`Precompiled header` won't be cached by ccache, sloppiness = `${CCACHE_SLOPPINESS}`,please execute `ccache -o sloppiness=pch_defines,time_macros`")
+            set (ENABLE_PCH FALSE CACHE BOOL "" FORCE)
+        endif ()
+    endif ()
+
 else ()
     message (STATUS "Not using ccache ${CCACHE_FOUND}, USE_CCACHE=${USE_CCACHE}")
 endif ()

dbms/CMakeLists.txt

@@ -90,12 +90,11 @@ add_headers_and_sources(dbms src/Storages/Page/V2/mvcc)
 add_headers_and_sources(dbms src/Storages/Page/V2/VersionSet)
 add_headers_and_sources(dbms src/Storages/Page/V2/gc)
 add_headers_and_sources(dbms src/WindowFunctions)
-if (ENABLE_V3_PAGESTORAGE)
-    add_headers_and_sources(dbms src/Storages/Page/V3)
-    add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
-    add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
-    add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
-endif()
+add_headers_and_sources(dbms src/TiDB/Schema)
+add_headers_and_sources(dbms src/Storages/Page/V3)
+add_headers_and_sources(dbms src/Storages/Page/V3/LogFile)
+add_headers_and_sources(dbms src/Storages/Page/V3/WAL)
+add_headers_and_sources(dbms src/Storages/Page/V3/spacemap)
 add_headers_and_sources(dbms src/Storages/Page/)
 add_headers_and_sources(dbms src/TiDB)
 add_headers_and_sources(dbms src/Client)
@@ -258,6 +257,16 @@ target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE
 # https://cmake.org/pipermail/cmake/2016-May/063400.html
 target_link_libraries (clickhouse_common_io PUBLIC ${TIFLASH_XXHASH_LIBRARY})
 
+function(add_target_pch context target)
+    if (ENABLE_PCH)
+        message(STATUS "Add PCH `${context}` for target `${target}`")
+        target_precompile_headers(${target} PRIVATE ${context})
+    endif ()
+    if(${ARGC} GREATER 2)
+        add_target_pch(${context} ${ARGN})
+    endif()
+endfunction()
+
 if (ENABLE_TESTS)
     include (${TiFlash_SOURCE_DIR}/cmake/find_gtest.cmake)
 
@@ -296,6 +305,8 @@ if (ENABLE_TESTS)
     target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy)
     add_check(gtests_dbms)
 
+    add_target_pch("pch-dbms.h" gtests_dbms)
+
     grep_bench_sources(${TiFlash_SOURCE_DIR}/dbms dbms_bench_sources)
     add_executable(bench_dbms EXCLUDE_FROM_ALL
         ${dbms_bench_sources}
@@ -305,11 +316,15 @@ if (ENABLE_TESTS)
         ${TiFlash_SOURCE_DIR}/dbms/src/AggregateFunctions/AggregateFunctionSum.cpp
         )
     target_include_directories(bench_dbms BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR} ${benchmark_SOURCE_DIR}/include)
+    target_compile_definitions(bench_dbms PUBLIC DBMS_PUBLIC_GTEST)
 
     target_link_libraries(bench_dbms gtest dbms test_util_bench_main benchmark clickhouse_functions)
     if (ENABLE_TIFLASH_DTWORKLOAD)
         target_link_libraries(bench_dbms dt-workload-lib)
     endif ()
+    if (ENABLE_TIFLASH_PAGEWORKLOAD)
+        target_link_libraries(bench_dbms page-workload-lib)
+    endif ()
 
     add_check(bench_dbms)
 endif ()
@@ -341,3 +356,23 @@ if (TEST_COVERAGE AND CMAKE_BUILD_TYPE STREQUAL "Debug")
         )
     endif ()
 endif ()
+
+# dbms
+add_target_pch("pch-dbms.h" dbms flash_service)
+add_target_pch("pch-common.h" clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions)
+add_target_pch("pch-common.h" clickhouse_parsers clickhouse_storages_system dt-workload-lib clickhouse-server-lib)
+
+# common
+add_target_pch("pch-kvpb.h" kv_client)
+
+add_target_pch("pch-stl.h" ${Boost_SYSTEM_LIBRARY} cctz ${RE2_LIBRARY} ${RE2_ST_LIBRARY})
+
+# grpc
+add_target_pch("$<$<COMPILE_LANGUAGE:CXX>:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" grpc grpc++)
+
+# pb
+add_target_pch("pch-stl.h" libprotobuf kvproto tipb libprotoc)
+
+# poco
+add_target_pch("pch-stl.h" Net Crypto Util Data NetSSL)
+add_target_pch("$<$<COMPILE_LANGUAGE:CXX>:${CMAKE_CURRENT_SOURCE_DIR}/pch-stl.h>" XML Foundation JSON)

dbms/src/Parsers/ParserCheckQuery.h → dbms/pch-common.h

@@ -14,18 +14,10 @@
 
 #pragma once
 
-#include <Parsers/ParserQueryWithOutput.h>
+#include <Common/Exception.h>
+#include <Common/FmtUtils.h>
+#include <Common/StackTrace.h>
+#include <common/StringRef.h>
+#include <common/types.h>
 
-namespace DB
-{
-/** Query of form
- * CHECK [TABLE] [database.]table
- */
-class ParserCheckQuery : public IParserBase
-{
-protected:
-    const char * getName() const { return "ALTER query"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected);
-};
-
-}
+#include "pch-stl.h"

dbms/pch-dbms.h

@@ -0,0 +1,18 @@
+// Copyright 2022 PingCAP, Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "pch-common.h"
+#include "pch-kvpb.h"