Revert snapshot on release-5.0 branch

.ci/integration_test.groovy

@@ -9,8 +9,18 @@ catchError {
         return params.ghprbTargetBranch ?: 'master'
     }).call()
 
-    stage("Wait for ci build") {
-        echo "ticsTag=${params.ghprbActualCommit} tidbBranch=${tidbBranch}"
+    echo "ticsTag=${params.ghprbActualCommit} tidbBranch=${tidbBranch}"
+
+    stage("Wait for images") {
+        util.runClosure("wait-for-images") {
+            timeout(time: 60, unit: 'MINUTES') {
+                container("docker") {
+                    sh  """
+                        while ! docker pull hub.pingcap.net/tiflash/tics:${params.ghprbActualCommit}; do sleep 60; done
+                        """
+                }
+            }
+        }
     }
 
     node("${GO_BUILD_SLAVE}") {
@@ -23,20 +33,13 @@ catchError {
                         cp -R /nfs/cache/git/src-tics.tar.gz*  ./
                         mkdir -p ${curws}/tics
                         tar -xzf src-tics.tar.gz -C ${curws}/tics --strip-components=1
-                        """
+                    """
                     }
                 }
             }
             dir("${curws}/tics") {
                 util.checkoutTiCS("${params.ghprbActualCommit}", "${params.ghprbPullId}")
             }
-            timeout(time: 60, unit: 'MINUTES') {
-                container("golang") {
-                    sh  """
-                        COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=${curws}/tics/tests/.build bash -e ${curws}/tics/release-centos7/build/fetch-ci-build.sh
-                        """
-                }
-            }
         }
         stash includes: "tics/**", name: "git-code-tics", useDefaultExcludes: false
     }
@@ -71,9 +74,10 @@ catchError {
 
 stage('Summary') {
     def duration = ((System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 / 60).setScale(2, BigDecimal.ROUND_HALF_UP)
-    def msg = "Result: `${currentBuild.currentResult}`" + "\n" +
+    def msg = "Build Result: `${currentBuild.currentResult}`" + "\n" +
             "Elapsed Time: `${duration} mins`" + "\n" +
             "${env.RUN_DISPLAY_URL}"
 
     echo "${msg}"
+
 }

.ci/util.groovy

@@ -1,30 +1,26 @@
-def doCheckout(commit, refspec) {
-    checkout(changelog: false, poll: false, scm: [
-        $class           : "GitSCM",
-        branches         : [
-                [name: "${commit}"],
-        ],
-        userRemoteConfigs: [
-                [
-                        url          : "[email protected]:pingcap/tics.git",
-                        refspec      : refspec,
-                        credentialsId: "github-sre-bot-ssh",
-                ]
-        ],
-        extensions       : [
-                [$class: 'PruneStaleBranch'],
-                [$class: 'CleanBeforeCheckout'],
-        ],
-    ])
-}
-
 def checkoutTiCS(commit, pullId) {
     def refspec = "+refs/heads/*:refs/remotes/origin/*"
     if (pullId) {
         refspec += " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*"
     }
     try {
-        doCheckout(commit, refspec)
+        checkout(changelog: false, poll: false, scm: [
+                $class           : "GitSCM",
+                branches         : [
+                        [name: "${commit}"],
+                ],
+                userRemoteConfigs: [
+                        [
+                                url          : "[email protected]:pingcap/tics.git",
+                                refspec      : refspec,
+                                credentialsId: "github-sre-bot-ssh",
+                        ]
+                ],
+                extensions       : [
+                        [$class: 'PruneStaleBranch'],
+                        [$class: 'CleanBeforeCheckout'],
+                ],
+        ])
     } catch (info) {
         retry(2) {
             echo "checkout failed, retry.."
@@ -33,42 +29,27 @@ def checkoutTiCS(commit, pullId) {
                 echo ".git already exist or not a valid git dir. Delete dir..."
                 deleteDir()
             }
-            doCheckout(commit, refspec)
+            checkout(changelog: false, poll: false, scm: [
+                    $class           : "GitSCM",
+                    branches         : [
+                            [name: "${commit}"],
+                    ],
+                    userRemoteConfigs: [
+                            [
+                                    url          : "[email protected]:pingcap/tics.git",
+                                    refspec      : refspec,
+                                    credentialsId: "github-sre-bot-ssh",
+                            ]
+                    ],
+                    extensions       : [
+                            [$class: 'PruneStaleBranch'],
+                            [$class: 'CleanBeforeCheckout'],
+                    ],
+            ])
         }
     }
 }
 
-def checkoutTiCSFull(commit, pullId) {
-    def refspec = "+refs/heads/*:refs/remotes/origin/*"
-    if (pullId) {
-        refspec += " +refs/pull/${pullId}/*:refs/remotes/origin/pr/${pullId}/*"
-    }
-    checkout(changelog: false, poll: false, scm: [
-            $class                           : "GitSCM",
-            branches                         : [
-                    [name: "${commit}"],
-            ],
-            userRemoteConfigs                : [
-                    [
-                            url          : "[email protected]:pingcap/tics.git",
-                            refspec      : refspec,
-                            credentialsId: "github-sre-bot-ssh",
-                    ]
-            ],
-            extensions                       : [
-                    [$class             : 'SubmoduleOption',
-                     disableSubmodules  : false,
-                     parentCredentials  : true,
-                     recursiveSubmodules: true,
-                     trackingSubmodules : false,
-                     reference          : ''],
-                    [$class: 'PruneStaleBranch'],
-                    [$class: 'CleanBeforeCheckout'],
-            ],
-            doGenerateSubmoduleConfigurations: false,
-    ])
-}
-
 def runClosure(label, Closure body) {
     podTemplate(name: label, label: label, instanceCap: 15, containers: [
             containerTemplate(name: 'dockerd', image: 'docker:18.09.6-dind', privileged: true,
@@ -89,16 +70,6 @@ def runTest(label, testPath, tidbBranch) {
     runClosure(label) {
         stage("Unstash") {
             unstash 'git-code-tics'
-            dir("tics") {
-                timeout(time: 5, unit: 'MINUTES') {
-                    container("docker") {
-                        sh """
-                        pwd
-                        DOWNLOAD_TAR=true COMMIT_HASH=${params.ghprbActualCommit} PULL_ID=${params.ghprbPullId} TAR_PATH=./tests/.build bash -e release-centos7/build/fetch-ci-build.sh
-                        """
-                    }
-                }
-            }
         }
         dir(testPath) {
             stage("Test") {

.github/pull_request_template.md

@@ -26,12 +26,10 @@ Tests <!-- At least one of them must be included. -->
 
 Side effects
 
-<!--
-- Performance regression
-    - Consumes more CPU
-    - Consumes more MEM
-- Breaking backward compatibility
--->
+# - Performance regression
+#     - Consumes more CPU
+#     - Consumes more MEM
+# - Breaking backward compatibility
 
 ### Release note <!-- bugfixes or new feature need a release note -->
 

CMakeLists.txt

@@ -22,7 +22,6 @@ endif ()
 # Write compile_commands.json
 set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
 
-option (USE_CCACHE "Set to OFF to disable ccache" ON)
 include (cmake/find_ccache.cmake)
 
 if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
@@ -149,24 +148,14 @@ if (NOT ARCH_ARM)
     set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3")
 endif ()
 
-option (DEBUG_WITHOUT_DEBUG_INFO "Set to ON to build dev target without debug info (remove flag `-g` in order to accelerate compiling speed and reduce target binary size)" OFF)
-if (DEBUG_WITHOUT_DEBUG_INFO)
-    string(REPLACE "-g" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
-    string(REPLACE "-g" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
-    message (STATUS "Build debug target without debug info")
-else ()
-    set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g3 -ggdb3")
-    set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g3 -ggdb3")
-endif ()
-
 set (CMAKE_BUILD_COLOR_MAKEFILE          ON)
 set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
 set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${CMAKE_CXX_FLAGS_ADD}")
-set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
+set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
 
 set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS} ${CMAKE_C_FLAGS_ADD}")
 set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${CMAKE_C_FLAGS_ADD}")
-set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -fno-inline ${CMAKE_C_FLAGS_ADD}")
+set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")
 
 if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND ARCH_FREEBSD))
     set (CMAKE_EXE_LINKER_FLAGS          "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
@@ -239,7 +228,7 @@ if (UNBUNDLED OR NOT (ARCH_LINUX OR APPLE) OR ARCH_32)
     option (NO_WERROR "Disable -Werror compiler option" ON)
 endif ()
 
-message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED}")
+message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
 
 include(GNUInstallDirs)
 

cmake/find_ccache.cmake

@@ -1,19 +1,14 @@
 
 find_program (CCACHE_FOUND ccache)
-if (USE_CCACHE AND CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache")
-    execute_process (COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
-    execute_process (COMMAND ${CCACHE_FOUND} "-p" OUTPUT_VARIABLE CCACHE_CONFIG)
-    string (REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})
+if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache")
+    execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
+    string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION} )
 
     if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-        message (STATUS "Using ccache: ${CCACHE_FOUND}, version ${CCACHE_VERSION}")
-        message (STATUS "Show ccache config:")
-        message ("${CCACHE_CONFIG}")
-        set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
-        set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
+       #message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}")
+       set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+       set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
     else ()
-        message (STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
+       message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
     endif ()
-else ()
-    message (STATUS "Not using ccache ${CCACHE_FOUND}, USE_CCACHE=${USE_CCACHE}")
 endif ()

dbms/src/Common/ErrorCodes.cpp

@@ -390,7 +390,6 @@ namespace ErrorCodes
     extern const int PAGE_SIZE_NOT_MATCH = 9006;
     extern const int ILLFORMED_PAGE_NAME = 9007;
     extern const int ILLFORMAT_RAFT_ROW = 9008;
-    extern const int REGION_DATA_SCHEMA_UPDATED = 9009;
 
     extern const int LOCK_EXCEPTION = 10000;
     extern const int VERSION_ERROR = 10001;

dbms/src/Common/FailPoint.cpp

@@ -44,7 +44,9 @@ std::unordered_map<String, std::shared_ptr<FailPointChannel>> FailPointHelper::f
     M(force_set_segment_ingest_packs_fail)                        \
     M(segment_merge_after_ingest_packs)
 
-#define APPLY_FOR_FAILPOINTS(M) M(force_set_page_file_write_errno)
+#define APPLY_FOR_FAILPOINTS(M)        \
+    M(force_set_page_file_write_errno) \
+    M(minimum_block_size_for_cross_join)
 
 #define APPLY_FOR_FAILPOINTS_ONCE_WITH_CHANNEL(M) \
     M(pause_after_learner_read)                   \

dbms/src/Common/TiFlashMetrics.h

@@ -84,7 +84,7 @@ namespace DB
     M(tiflash_storage_write_amplification, "The data write amplification in storage engine", Gauge)                                       \
     M(tiflash_storage_read_tasks_count, "Total number of storage engine read tasks", Counter)                                             \
     M(tiflash_storage_command_count, "Total number of storage's command, such as delete range / shutdown /startup", Counter,              \
-        F(type_delete_range, {"type", "delete_range"}), F(type_ingest, {"type", "ingest"}))                                               \
+        F(type_delete_range, {"type", "delete_range"}))                                                                                   \
     M(tiflash_storage_subtask_count, "Total number of storage's sub task", Counter, F(type_delta_merge, {"type", "delta_merge"}),         \
         F(type_delta_merge_fg, {"type", "delta_merge_fg"}), F(type_delta_compact, {"type", "delta_compact"}),                             \
         F(type_delta_flush, {"type", "delta_flush"}),F(type_seg_split, {"type", "seg_split"}), F(type_seg_merge, {"type", "seg_merge"}),  \
@@ -99,13 +99,11 @@ namespace DB
         F(type_place_index_update, {{"type", "place_index_update"}}, ExpBuckets{0.0005, 2, 20}))                                          \
     M(tiflash_storage_throughput_bytes, "Calculate the throughput of tasks of storage in bytes", Gauge,           /**/                    \
         F(type_write, {"type", "write"}),                                                                         /**/                    \
-        F(type_ingest, {"type", "ingest"}),                                                                       /**/                    \
         F(type_delta_merge, {"type", "delta_merge"}),                                                             /**/                    \
         F(type_split, {"type", "split"}),                                                                         /**/                    \
         F(type_merge, {"type", "merge"}))                                                                         /**/                    \
     M(tiflash_storage_throughput_rows, "Calculate the throughput of tasks of storage in rows", Gauge,             /**/                    \
         F(type_write, {"type", "write"}),                                                                         /**/                    \
-        F(type_ingest, {"type", "ingest"}),                                                                       /**/                    \
         F(type_delta_merge, {"type", "delta_merge"}),                                                             /**/                    \
         F(type_split, {"type", "split"}),                                                                         /**/                    \
         F(type_merge, {"type", "merge"}))                                                                         /**/                    \