Merge branch 'master' into remove_summary_delte

pingcap · Nov 9, 2022 · e19e9d9 · e19e9d9
2 parents f0b440e + 16cf524
commit e19e9d9
Show file tree

Hide file tree

Showing 54 changed files with 1,014 additions and 1,021 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -59,36 +59,28 @@ if (NOT MSVC)
     set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall")    # -Werror is also added inside directories with our own code.
 endif ()
 
-if (COMPILER_GCC OR COMPILER_CLANG)
-    set (CXX_WARNING_FLAGS "${CXX_WARNING_FLAGS} -Wnon-virtual-dtor")
+set (CXX_WARNING_FLAGS "${CXX_WARNING_FLAGS} -Wnon-virtual-dtor")
+
+option(ENABLE_TIME_TRACES "Enable clang feature time traces" OFF)
+if (ENABLE_TIME_TRACES)
+    set (CLANG_TIME_TRACES_FLAGS "-ftime-trace")
+    message (STATUS "Using clang time traces flag `${CLANG_TIME_TRACES_FLAGS}`. Generates JSON file based on output filename. Results can be analyzed with chrome://tracing or https://www.speedscope.app for flamegraph visualization.")
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CLANG_TIME_TRACES_FLAGS}")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_TIME_TRACES_FLAGS}")
 endif ()
 
-if (COMPILER_CLANG)
-    # Clang doesn't have int128 predefined macros, workaround by manually defining them
-    # Reference: https://stackoverflow.com/questions/41198673/uint128-t-not-working-with-clang-and-libstdc
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__GLIBCXX_BITSIZE_INT_N_0=128 -D__GLIBCXX_TYPE_INT_N_0=__int128")
-
-    option(ENABLE_TIME_TRACES "Enable clang feature time traces" OFF)
-    if (ENABLE_TIME_TRACES)
-        set (CLANG_TIME_TRACES_FLAGS "-ftime-trace")
-        message (STATUS "Using clang time traces flag `${CLANG_TIME_TRACES_FLAGS}`. Generates JSON file based on output filename. Results can be analyzed with chrome://tracing or https://www.speedscope.app for flamegraph visualization.")
-        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CLANG_TIME_TRACES_FLAGS}")
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_TIME_TRACES_FLAGS}")
-    endif ()
-
-    # https://clang.llvm.org/docs/ThinLTO.html
-    # Applies to clang only.
-    option(ENABLE_THINLTO "Clang-specific link time optimization" OFF)
-
-    if (ENABLE_THINLTO AND NOT ENABLE_TESTS)
-        # Link time optimization
-        set (THINLTO_JOBS "0" CACHE STRING "ThinLTO compilation parallelism")
-        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -flto=thin -fvisibility=hidden -fvisibility-inlines-hidden -fsplit-lto-unit")
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin -fvisibility=hidden -fvisibility-inlines-hidden -fwhole-program-vtables -fsplit-lto-unit")
-        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto=thin -flto-jobs=${THINLTO_JOBS} -fvisibility=hidden -fvisibility-inlines-hidden -fwhole-program-vtables -fsplit-lto-unit")
-    elseif (ENABLE_THINLTO)
-        message (WARNING "Cannot enable ThinLTO")
-    endif ()
+# https://clang.llvm.org/docs/ThinLTO.html
+# Applies to clang only.
+option(ENABLE_THINLTO "Clang-specific link time optimization" OFF)
+
+if (ENABLE_THINLTO AND NOT ENABLE_TESTS)
+    # Link time optimization
+    set (THINLTO_JOBS "0" CACHE STRING "ThinLTO compilation parallelism")
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -flto=thin -fvisibility=hidden -fvisibility-inlines-hidden -fsplit-lto-unit")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin -fvisibility=hidden -fvisibility-inlines-hidden -fwhole-program-vtables -fsplit-lto-unit")
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto=thin -flto-jobs=${THINLTO_JOBS} -fvisibility=hidden -fvisibility-inlines-hidden -fwhole-program-vtables -fsplit-lto-unit")
+elseif (ENABLE_THINLTO)
+    message (WARNING "Cannot enable ThinLTO")
 endif ()
 
 option (ENABLE_LLVM_PROFILE_INSTR "Generate instrumented code to collect execution counts" OFF)
@@ -133,11 +125,9 @@ if (ENABLE_LLVM_PGO)
     endif ()
 endif ()
 
-if (COMPILER_CLANG)
-    # clang: warning: argument unused during compilation: '-stdlib=libc++'
-    # clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument]
-    set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wno-unused-command-line-argument")
-endif ()
+# clang: warning: argument unused during compilation: '-stdlib=libc++'
+# clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument]
+set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wno-unused-command-line-argument")
 
 option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON)
 option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES})
@@ -172,13 +162,9 @@ endif ()
 
 include (cmake/cpu_features.cmake)
 
-if (CMAKE_VERSION VERSION_LESS "3.8.0")
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z")
-else ()
-    set (CMAKE_CXX_STANDARD 17)
-    set (CMAKE_CXX_EXTENSIONS 1) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
-    set (CMAKE_CXX_STANDARD_REQUIRED ON)
-endif ()
+set (CMAKE_CXX_STANDARD 17)
+set (CMAKE_CXX_EXTENSIONS 1) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
+set (CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3")
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3")
@@ -202,15 +188,11 @@ set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fn
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} ${MOST_DEBUGGABLE_LEVEL} -fverbose-asm -fno-inline ${CMAKE_C_FLAGS_ADD}")
 
-if (MAKE_STATIC_LIBRARIES AND NOT OS_DARWIN AND NOT COMPILER_CLANG)
-    set (CMAKE_EXE_LINKER_FLAGS          "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
-endif ()
-
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 
 include (cmake/test_compiler.cmake)
 
-if (OS_LINUX AND COMPILER_CLANG)
+if (OS_LINUX)
     set (TIFLASH_LLVM_TOOLCHAIN 1)
     set (CMAKE_EXE_LINKER_FLAGS          "${CMAKE_EXE_LINKER_FLAGS} ${GLIBC_COMPATIBILITY_LINK_FLAGS}")
 
@@ -262,9 +244,6 @@ if (USE_INCLUDE_WHAT_YOU_USE)
     if (NOT IWYU_PATH)
         message(FATAL_ERROR "Could not find the program include-what-you-use")
     endif()
-    if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
-        message(FATAL_ERROR "include-what-you-use requires CMake version at least 3.3.")
-    endif()
 endif ()
 
 option (UNBUNDLED "Try find all libraries in system (if fail - use bundled from contrib/)" OFF)

diff --git a/README.md b/README.md
@@ -147,7 +147,7 @@ cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG
 ninja tiflash
 ```
 
-Note: In Linux, usually you need to explicitly specify to use LLVM. Otherwise, the default compiler will be GCC:
+Note: In Linux, usually you need to explicitly specify to use LLVM.
 
 ```shell
 # In cmake-build-debug directory:

diff --git a/cmake/test_compiler.cmake b/cmake/test_compiler.cmake
@@ -17,44 +17,23 @@ include (CMakePushCheckState)
 
 cmake_push_check_state ()
 
-if (COMPILER_CLANG)
 # clang4 : -no-pie cause error
 # clang6 : -no-pie cause warning
 
-    if (MAKE_STATIC_LIBRARIES)
-        set (TEST_FLAG "-Wl,-Bstatic -stdlib=libc++ -lc++ -lc++abi -Wl,-Bdynamic")
-    else ()
-        set (TEST_FLAG "-stdlib=libc++ -lc++ -lc++abi")
-    endif ()
-
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
-
-    check_cxx_source_compiles("
-        #include <iostream>
-        int main() {
-            std::cerr << std::endl;
-            return 0;
-        }
-        " HAVE_LIBCXX)
-
+if (MAKE_STATIC_LIBRARIES)
+    set (TEST_FLAG "-Wl,-Bstatic -stdlib=libc++ -lc++ -lc++abi -Wl,-Bdynamic")
 else ()
+    set (TEST_FLAG "-stdlib=libc++ -lc++ -lc++abi")
+endif ()
 
-    cmake_push_check_state ()
-
-    set (TEST_FLAG "-no-pie")
-    set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
-
-    check_cxx_source_compiles("
-        int main() {
-            return 0;
-        }
-        " HAVE_NO_PIE)
-
-    if (HAVE_NO_PIE)
-        set (FLAG_NO_PIE ${TEST_FLAG})
-    endif ()
-
+set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
 
-endif ()
+check_cxx_source_compiles("
+    #include <iostream>
+    int main() {
+        std::cerr << std::endl;
+        return 0;
+    }
+    " HAVE_LIBCXX)
 
 cmake_pop_check_state ()
diff --git a/cmake/tools.cmake b/cmake/tools.cmake
@@ -16,7 +16,6 @@
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set (COMPILER_GCC 1)
-    set (MOST_DEBUGGABLE_LEVEL -Og)
 elseif (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
     set (COMPILER_APPLE_CLANG 1)
     set (COMPILER_CLANG 1) # Safe to treat AppleClang as a regular Clang, in general.
@@ -33,14 +32,10 @@ execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE COMPILER
 message (STATUS "Using compiler:\n${COMPILER_SELF_IDENTIFICATION}")
 
 # Require minimum compiler versions
-set (CLANG_MINIMUM_VERSION 5)
-set (GCC_MINIMUM_VERSION 7)
+set (CLANG_MINIMUM_VERSION 12)
 
 if (COMPILER_GCC)
-    message (WARNING "GCC is not officially supported, and may not work.")
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION})
-        message (FATAL_ERROR "Compilation with GCC version ${CMAKE_CXX_COMPILER_VERSION} is not supported, minimum required version is ${GCC_MINIMUM_VERSION}")
-    endif ()
+    message (FATAL_ERROR "GCC is not officially supported.")
 elseif (COMPILER_CLANG)
     if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
         message (FATAL_ERROR "Compilation with Clang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${CLANG_MINIMUM_VERSION}.")
@@ -64,9 +59,7 @@ endif ()
 if (NOT LINKER_NAME)
     if (OS_DARWIN AND COMPILER_APPLE_CLANG)
         set (LINKER_NAME "ld")
-    elseif (COMPILER_CLANG AND LLD_PATH)
-        set (LINKER_NAME "lld")
-    elseif (COMPILER_GCC AND LLD_PATH)
+    elseif (LLD_PATH)
         set (LINKER_NAME "lld")
     elseif (GOLD_PATH)
         message (WARNING "Linking with gold is not recommended. Please use lld.")
@@ -83,11 +76,7 @@ endif()
 
 # Archiver
 
-if (COMPILER_GCC)
-    find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-15" "llvm-ar-14" "llvm-ar-13" "llvm-ar-12")
-else ()
-    find_program (LLVM_AR_PATH NAMES "llvm-ar-${COMPILER_VERSION_MAJOR}" "llvm-ar")
-endif ()
+find_program (LLVM_AR_PATH NAMES "llvm-ar-${COMPILER_VERSION_MAJOR}" "llvm-ar")
 
 if (LLVM_AR_PATH)
     set (CMAKE_AR "${LLVM_AR_PATH}")
@@ -101,11 +90,7 @@ endif ()
 
 # Ranlib
 
-if (COMPILER_GCC)
-    find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-15" "llvm-ranlib-14" "llvm-ranlib-13" "llvm-ranlib-12")
-else ()
-    find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib-${COMPILER_VERSION_MAJOR}" "llvm-ranlib")
-endif ()
+find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib-${COMPILER_VERSION_MAJOR}" "llvm-ranlib")
 
 if (LLVM_RANLIB_PATH)
     set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}")
@@ -119,11 +104,7 @@ endif ()
 
 # Objcopy
 
-if (COMPILER_GCC)
-    find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-15" "llvm-objcopy-14" "llvm-objcopy-13" "llvm-objcopy-12" "objcopy")
-else ()
-    find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy")
-endif ()
+find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy")
 
 if (OBJCOPY_PATH)
     set (CMAKE_OBJCOPY "${OBJCOPY_PATH}")

diff --git a/contrib/tiflash-proxy b/contrib/tiflash-proxy
diff --git a/dbms/src/Common/FailPoint.cpp b/dbms/src/Common/FailPoint.cpp
@@ -70,7 +70,8 @@ std::unordered_map<String, std::shared_ptr<FailPointChannel>> FailPointHelper::f
     M(exception_before_drop_segment)                              \
     M(exception_after_drop_segment)                               \
     M(exception_between_schema_change_in_the_same_diff)           \
-    M(force_ps_wal_compact)
+    M(force_ps_wal_compact)                                       \
+    M(pause_before_full_gc_prepare)
 
 #define APPLY_FOR_FAILPOINTS(M)                              \
     M(skip_check_segment_update)                             \

diff --git a/dbms/src/Common/TiFlashMetrics.h b/dbms/src/Common/TiFlashMetrics.h
@@ -236,11 +236,14 @@ namespace DB
         F(type_mpp_query_count, {"type", "mpp_query_count"}))                                                                             \
 // clang-format on
 
+/// Buckets with boundaries [start * base^0, start * base^1, ..., start * base^(size-1)]
 struct ExpBuckets
 {
     const double start;
     const int base;
     const size_t size;
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
     inline operator prometheus::Histogram::BucketBoundaries() const &&
     {
         prometheus::Histogram::BucketBoundaries buckets(size);
@@ -259,6 +262,8 @@ struct EqualWidthBuckets
     const size_t start;
     const int num_buckets;
     const size_t step;
+
+    // NOLINTNEXTLINE(google-explicit-constructor)
     inline operator prometheus::Histogram::BucketBoundaries() const &&
     {
         // up to `num_buckets` * `step`
@@ -384,12 +389,17 @@ class TiFlashMetrics
 APPLY_FOR_METRICS(MAKE_METRIC_ENUM_M, MAKE_METRIC_ENUM_F)
 #undef APPLY_FOR_METRICS
 
+// NOLINTNEXTLINE(bugprone-reserved-identifier)
 #define __GET_METRIC_MACRO(_1, _2, NAME, ...) NAME
 #ifndef GTEST_TIFLASH_METRICS
+// NOLINTNEXTLINE(bugprone-reserved-identifier)
 #define __GET_METRIC_0(family) TiFlashMetrics::instance().family.get()
+// NOLINTNEXTLINE(bugprone-reserved-identifier)
 #define __GET_METRIC_1(family, metric) TiFlashMetrics::instance().family.get(family##_metrics::metric)
 #else
+// NOLINTNEXTLINE(bugprone-reserved-identifier)
 #define __GET_METRIC_0(family) TestMetrics::instance().family.get()
+// NOLINTNEXTLINE(bugprone-reserved-identifier)
 #define __GET_METRIC_1(family, metric) TestMetrics::instance().family.get(family##_metrics::metric)
 #endif
 #define GET_METRIC(...)                                             \
+4 −4		.github/workflows/pr-ci.yml
+2 −2		Cargo.toml
+5 −4		Makefile
+1 −1		components/batch-system/src/config.rs
+8 −13		components/raftstore/src/store/fsm/apply.rs
+2 −6		components/raftstore/src/store/snap.rs
+1 −1		components/sst_importer/src/config.rs
+0 −2		components/test_util/src/lib.rs
+48 −51		engine_store_ffi/src/lib.rs
+16 −7		engine_store_ffi/src/observer.rs
+4 −1		engine_store_ffi/src/read_index_helper.rs
+1 −0		engine_store_ffi/src/utils.rs
+6 −6		engine_tiflash/src/proxy_utils.rs
+2 −2		engine_tiflash/src/write_batch.rs
+1 −1		gen-proxy-ffi/Cargo.toml
+31 −27		new-mock-engine-store/src/lib.rs
+2 −2		new-mock-engine-store/src/mock_cluster.rs
+1 −0		new-mock-engine-store/src/node.rs
+4 −6		new-mock-engine-store/src/server.rs
+7 −3		new-mock-engine-store/src/transport_simulate.rs
+0 −0		proxy_scripts/build.sh
+6 −1		proxy_scripts/ci_check.sh
+6 −3		proxy_server/src/config.rs
+3 −2		proxy_server/src/hacked_lock_mgr.rs
+2 −3		proxy_server/src/proxy.rs
+6 −4		proxy_server/src/run.rs
+6 −3		proxy_server/src/status_server.rs
+2 −15		proxy_tests/proxy/flashback.rs
+0 −2		proxy_tests/proxy/ingest.rs
+6 −222		proxy_tests/proxy/normal.rs
+25 −11		proxy_tests/proxy/proxy.rs
+6 −10		proxy_tests/proxy/server_cluster_test.rs