Merge remote-tracking branch 'upstream/master' into feature/vector-index

CMakeLists.txt

@@ -24,6 +24,10 @@ set(CMAKE_MACOSX_RPATH 1)
 
 option(TIFLASH_ENABLE_LLVM_DEVELOPMENT "enable facilities for development with LLVM" OFF)
 
+if(NOT CMAKE_PREFIX_PATH AND DEFINED ENV{CMAKE_PREFIX_PATH})
+    message(STATUS "Reading CMAKE_PREFIX_PATH from env... $ENV{CMAKE_PREFIX_PATH}")
+    set(CMAKE_PREFIX_PATH "$ENV{CMAKE_PREFIX_PATH}")
+endif()
 if(CMAKE_PREFIX_PATH)
     # append paths for cmake to check libs
     set(ENV{LD_LIBRARY_PATH}

CMakePresets.json

@@ -1,5 +1,5 @@
 {
-  "version": 3,
+  "version": 6,
   "cmakeMinimumRequired": {
     "major": 3,
     "minor": 23,
@@ -127,5 +127,201 @@
       "cacheVariables": {},
       "binaryDir": "${sourceDir}/cmake-build-release"
     }
+  ],
+  "buildPresets": [
+    {
+      "name": "dev",
+      "displayName": "Build tiflash binary with debug info and tests enabled",
+      "configurePreset": "dev",
+      "targets": ["tiflash"]
+    },
+    {
+      "name": "unit-tests",
+      "displayName": "Build dbms unit tests",
+      "configurePreset": "dev",
+      "targets": ["gtests_dbms"]
+    },
+    {
+      "name": "unit-tests-all",
+      "displayName": "Build all unit tests",
+      "configurePreset": "dev",
+      "targets": ["gtests_dbms, gtests_libdaemon, gtests_libcommon"]
+    },
+    {
+      "name": "dev-coverage",
+      "displayName": "Build dbms unit tests with code coverage",
+      "configurePreset": "dev-coverage",
+      "targets": ["gtests_dbms"]
+    },
+    {
+      "name": "dev-coverage-all",
+      "displayName": "Build all unit tests with code coverage",
+      "configurePreset": "dev-coverage",
+      "targets": ["gtests_dbms, gtests_libdaemon, gtests_libcommon"]
+    },
+    {
+      "name": "release",
+      "displayName": "Build tiflash binary without debug info",
+      "configurePreset": "release",
+      "targets": ["tiflash"]
+    },
+    {
+      "name": "asan",
+      "displayName": "Build dbms Address Sanitizer tests",
+      "configurePreset": "asan",
+      "targets": ["gtests_dbms"]
+    },
+    {
+      "name": "asan-all",
+      "displayName": "Build all Address Sanitizer tests",
+      "configurePreset": "asan",
+      "targets": ["gtests_dbms, gtests_libdaemon, gtests_libcommon"]
+    },
+    {
+      "name": "tsan",
+      "displayName": "Build dbms Thread Sanitizer tests",
+      "configurePreset": "tsan",
+      "targets": ["gtests_dbms"]
+    },
+    {
+      "name": "tsan-all",
+      "displayName": "Build all Thread Sanitizer tests",
+      "configurePreset": "tsan",
+      "targets": ["gtests_dbms, gtests_libdaemon, gtests_libcommon"]
+    },
+    {
+      "name": "benchmarks",
+      "displayName": "Build benchmarks",
+      "configurePreset": "benchmarks",
+      "targets": ["bench_dbms"]
+    }
+  ],
+  "workflowPresets": [
+    {
+      "name": "dev",
+      "displayName": "Build debug binary workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "dev"
+        },
+        {
+          "type": "build",
+          "name": "dev"
+        }
+      ]
+    },
+    {
+      "name": "unit-tests",
+      "displayName": "Build dbms unit tests workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "dev"
+        },
+        {
+          "type": "build",
+          "name": "unit-tests"
+        }
+      ]
+    },
+    {
+      "name": "unit-tests-all",
+      "displayName": "Build all unit tests workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "dev"
+        },
+        {
+          "type": "build",
+          "name": "unit-tests-all"
+        }
+      ]
+    },
+    {
+      "name": "benchmarks",
+      "displayName": "Build benchmarks workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "benchmarks"
+        },
+        {
+          "type": "build",
+          "name": "benchmarks"
+        }
+      ]
+    },
+    {
+      "name": "asan-tests",
+      "displayName": "Build dbms Address Sanitizer tests workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "asan"
+        },
+        {
+          "type": "build",
+          "name": "asan"
+        }
+      ]
+    },
+    {
+      "name": "asan-tests-all",
+      "displayName": "Build all Address Sanitizer tests workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "asan"
+        },
+        {
+          "type": "build",
+          "name": "asan-all"
+        }
+      ]
+    },
+    {
+      "name": "tsan-tests",
+      "displayName": "Build dbms Thread Sanitizer tests workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "tsan"
+        },
+        {
+          "type": "build",
+          "name": "tsan"
+        }
+      ]
+    },
+    {
+      "name": "tsan-tests-all",
+      "displayName": "Build all Thread Sanitizer tests workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "tsan"
+        },
+        {
+          "type": "build",
+          "name": "tsan-all"
+        }
+      ]
+    },
+    {
+      "name": "release",
+      "displayName": "Build release binary workflow",
+      "steps": [
+        {
+          "type": "configure",
+          "name": "release"
+        },
+        {
+          "type": "build",
+          "name": "release"
+        }
+      ]
+    }
   ]
 }

README.md

@@ -125,7 +125,7 @@ xcode-select --install
 brew install ninja cmake [email protected] ccache
 ```
 
-If your MacOS is higher or equal to 13.0, it should work out of the box because by default Apple clang is 14.0.0. But if your MacOS is lower than 13.0, you should install llvm clang manually.
+If your MacOS is higher or equal to 13.0 (Ventura), it should work out of the box because by default Xcode 14.3 provides Apple clang 14.0.0. But if your MacOS is lower than 13.0, you should install llvm clang manually.
 
 ```shell
 brew install llvm@17
@@ -149,21 +149,14 @@ To build TiFlash for development:
 
 ```shell
 # In the TiFlash repository root:
-mkdir cmake-build-debug  # The directory name can be customized
-cd cmake-build-debug
-
-cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG
-
-ninja tiflash
+cmake --workflow --preset dev
 ```
 
 Note: In Linux, usually you need to explicitly specify to use LLVM.
 
 ```shell
-# In cmake-build-debug directory:
-cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG \
-  -DCMAKE_C_COMPILER=/usr/bin/clang-17 \
-  -DCMAKE_CXX_COMPILER=/usr/bin/clang++-17
+export CC="/usr/bin/clang-17"
+export CXX="/usr/bin/clang++-17"
 ```
 
 In MacOS, if you install llvm clang, you need to explicitly specify to use llvm clang.
@@ -177,7 +170,12 @@ export CXX="/opt/homebrew/opt/llvm/bin/clang++"
 
 Or use `CMAKE_C_COMPILER` and `CMAKE_CXX_COMPILER` to specify the compiler, like this:
 ```shell
+mkdir cmake-build-debug
+cd cmake-build-debug
+
 cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG -DCMAKE_C_COMPILER=/opt/homebrew/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm/bin/clang++
+
+ninja tiflash
 ```
 
 After building, you can get TiFlash binary in `dbms/src/Server/tiflash` in the `cmake-build-debug` directory.
@@ -270,11 +268,8 @@ cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG -DFOO=BAR
 Unit tests are automatically enabled in debug profile. To build these unit tests:
 
 ```shell
-cd cmake-build-debug
-cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG
-ninja gtests_dbms       # Most TiFlash unit tests
-ninja gtests_libdaemon  # Settings related tests
-ninja gtests_libcommon
+# In the TiFlash repository root:
+cmake --workflow --preset unit-tests-all
 ```
 
 Then, to run these unit tests:
@@ -296,12 +291,7 @@ To build unit test executables with sanitizer enabled:
 
 ```shell
 # In the TiFlash repository root:
-mkdir cmake-build-sanitizer
-cd cmake-build-sanitizer
-cmake .. -GNinja -DENABLE_TESTS=ON -DCMAKE_BUILD_TYPE=ASan # or TSan
-ninja gtests_dbms
-ninja gtests_libdaemon
-ninja gtests_libcommon
+cmake --workflow --preset asan-tests-all # or tsan-tests-all
 ```
 
 There are known false positives reported from leak sanitizer (which is included in address sanitizer). To suppress these errors, set the following environment variables before running the executables:
@@ -359,10 +349,7 @@ To build micro benchmark tests, you need release profile and tests enabled:
 
 ```shell
 # In the TiFlash repository root:
-mkdir cmake-build-release
-cd cmake-build-release
-cmake .. -GNinja -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_TESTS=ON
-ninja bench_dbms
+cmake --workflow --preset benchmarks
 ```
 
 Then, to run these micro benchmarks:

dbms/src/Common/FailPoint.cpp

@@ -117,17 +117,18 @@ namespace DB
     M(force_agg_two_level_hash_table_before_merge)           \
     M(force_thread_0_no_agg_spill)
 
-#define APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M) \
-    M(pause_with_alter_locks_acquired)         \
-    M(hang_in_execution)                       \
-    M(pause_before_dt_background_delta_merge)  \
-    M(pause_until_dt_background_delta_merge)   \
-    M(pause_before_apply_raft_cmd)             \
-    M(pause_before_apply_raft_snapshot)        \
-    M(pause_until_apply_raft_snapshot)         \
-    M(pause_after_copr_streams_acquired_once)  \
-    M(pause_before_register_non_root_mpp_task) \
-    M(pause_before_make_non_root_mpp_task_active)
+#define APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M)    \
+    M(pause_with_alter_locks_acquired)            \
+    M(hang_in_execution)                          \
+    M(pause_before_dt_background_delta_merge)     \
+    M(pause_until_dt_background_delta_merge)      \
+    M(pause_before_apply_raft_cmd)                \
+    M(pause_before_apply_raft_snapshot)           \
+    M(pause_until_apply_raft_snapshot)            \
+    M(pause_after_copr_streams_acquired_once)     \
+    M(pause_before_register_non_root_mpp_task)    \
+    M(pause_before_make_non_root_mpp_task_active) \
+    M(pause_before_page_dir_update_local_cache)
 
 #define APPLY_FOR_PAUSEABLE_FAILPOINTS(M) \
     M(pause_when_reading_from_dt_stream)  \

dbms/src/Common/HashTable/HashTableKeyHolder.h

@@ -27,7 +27,7 @@
   * make a persistent copy of the key in each of the following cases:
   * 1) the aggregation method doesn't use temporary keys, so they're persistent
   *    from the start;
-  * 1) the key is already present in the hash table;
+  * 2) the key is already present in the hash table;
   * 3) that particular key is stored by value, e.g. a short StringRef key in
   *    StringHashMap.
   *