Passive flush with gap between current applied index and last flushed index

dbms/src/Common/FailPoint.cpp

@@ -37,7 +37,6 @@ namespace DB
     M(exception_before_dmfile_remove_encryption)                  \
     M(exception_before_dmfile_remove_from_disk)                   \
     M(force_triggle_background_merge_delta)                       \
-    M(force_triggle_foreground_flush)                             \
     M(exception_before_mpp_make_non_root_mpp_task_active)         \
     M(exception_before_mpp_register_non_root_mpp_task)            \
     M(exception_before_mpp_register_tunnel_for_non_root_mpp_task) \
@@ -100,6 +99,7 @@ namespace DB
     M(force_stop_background_checkpoint_upload)               \
     M(skip_seek_before_read_dmfile)                          \
     M(exception_after_large_write_exceed)                    \
+    M(proactive_flush_force_set_type)                        \
     M(exception_when_fetch_disagg_pages)
 
 #define APPLY_FOR_PAUSEABLE_FAILPOINTS_ONCE(M) \
@@ -120,7 +120,8 @@ namespace DB
     M(pause_when_ingesting_to_dt_store)   \
     M(pause_when_altering_dt_store)       \
     M(pause_after_copr_streams_acquired)  \
-    M(pause_query_init)
+    M(pause_query_init)                   \
+    M(pause_passive_flush_before_persist_region)
 
 #define APPLY_FOR_RANDOM_FAILPOINTS(M)                  \
     M(random_tunnel_wait_timeout_failpoint)             \
@@ -268,8 +269,7 @@ void FailPointHelper::enableFailPoint(const String & fail_point_name, std::optio
     throw Exception(fmt::format("Cannot find fail point {}", fail_point_name), ErrorCodes::FAIL_POINT_ERROR);
 }
 
-std::optional<std::any>
-FailPointHelper::getFailPointVal(const String & fail_point_name)
+std::optional<std::any> FailPointHelper::getFailPointVal(const String & fail_point_name)
 {
     if (auto iter = fail_point_val.find(fail_point_name); iter != fail_point_val.end())
     {
@@ -312,7 +312,10 @@ void FailPointHelper::initRandomFailPoints(Poco::Util::LayeredConfiguration & co
     for (const auto & string_token : string_tokens)
     {
         Poco::StringTokenizer pair_tokens(string_token, "-");
-        RUNTIME_ASSERT((pair_tokens.count() == 2), log, "RandomFailPoints config should be FailPointA-RatioA,FailPointB-RatioB,... format");
+        RUNTIME_ASSERT(
+            (pair_tokens.count() == 2),
+            log,
+            "RandomFailPoints config should be FailPointA-RatioA,FailPointB-RatioB,... format");
         double rate = atof(pair_tokens[1].c_str()); //NOLINT(cert-err34-c): check conversion error manually
         RUNTIME_ASSERT((0 <= rate && rate <= 1.0), log, "RandomFailPoint trigger rate should in [0,1], while {}", rate);
         enableRandomFailPoint(pair_tokens[0], rate);
@@ -330,7 +333,10 @@ void FailPointHelper::disableRandomFailPoints(Poco::Util::LayeredConfiguration &
     for (const auto & string_token : string_tokens)
     {
         Poco::StringTokenizer pair_tokens(string_token, "-");
-        RUNTIME_ASSERT((pair_tokens.count() == 2), log, "RandomFailPoints config should be FailPointA-RatioA,FailPointB-RatioB,... format");
+        RUNTIME_ASSERT(
+            (pair_tokens.count() == 2),
+            log,
+            "RandomFailPoints config should be FailPointA-RatioA,FailPointB-RatioB,... format");
         disableFailPoint(pair_tokens[0]);
     }
     LOG_INFO(log, "Disable RandomFailPoints: {}", random_fail_point_cfg);

dbms/src/Common/TiFlashMetrics.h

@@ -390,6 +390,31 @@ namespace DB
       Histogram,                                                                                                                    \
       F(type_decode, {{"type", "decode"}}, ExpBuckets{0.0005, 2, 20}),                                                              \
       F(type_write, {{"type", "write"}}, ExpBuckets{0.0005, 2, 20}))                                                                \
+    M(tiflash_raft_raft_log_lag_count,                                                                                              \
+      "Bucketed histogram raft index lag",                                                                                          \
+      Histogram,                                                                                                                    \
+      F(type_compact_index, {{"type", "compact_index"}}, EqualWidthBuckets{0, 200, 5}),                                             \
+      F(type_applied_index, {{"type", "applied_index"}}, EqualWidthBuckets{0, 200, 5}),                                             \
+      F(type_unflushed_applied_index, {{"type", "unflushed_applied_index"}}, EqualWidthBuckets{0, 200, 5}))                         \
+    M(tiflash_raft_raft_events_count,                                                                                               \
+      "Raft event counter",                                                                                                         \
+      Counter,                                                                                                                      \
+      F(type_pre_exec_compact, {{"type", "pre_exec_compact"}}),                                                                     \
+      F(type_flush_apply_snapshot, {{"type", "flush_apply_snapshot"}}),                                                             \
+      F(type_flush_ingest_sst, {{"type", "flush_ingest_sst"}}),                                                                     \
+      F(type_flush_useless_admin, {{"type", "flush_useless_admin"}}),                                                               \
+      F(type_flush_useful_admin, {{"type", "flush_useful_admin"}}),                                                                 \
+      F(type_flush_passive, {{"type", "flush_passive"}}),                                                                           \
+      F(type_flush_proactive, {{"type", "flush_proactive"}}),                                                                       \
+      F(type_flush_log_gap, {{"type", "flush_log_gap"}}),                                                                           \
+      F(type_flush_size, {{"type", "flush_size"}}),                                                                                 \
+      F(type_flush_rowcount, {{"type", "flush_rowcount"}}),                                                                         \
+      F(type_exec_compact, {{"type", "exec_compact"}}))                                                                             \
+    M(tiflash_raft_region_flush_size,                                                                                               \
+      "Bucketed histogram of region flushed size",                                                                                  \
+      Histogram,                                                                                                                    \
+      F(type_flushed, {{"type", "flushed"}}, ExpBuckets{32, 2, 16}),                                                                \
+      F(type_unflushed, {{"type", "unflushed"}}, ExpBuckets{32, 2, 16}))                                                            \
     /* required by DBaaS */                                                                                                         \
     M(tiflash_server_info,                                                                                                          \
       "Indicate the tiflash server info, and the value is the start timestamp (s).",                                                \
@@ -850,4 +875,4 @@ APPLY_FOR_METRICS(MAKE_METRIC_ENUM_M, MAKE_METRIC_ENUM_F)
     GET_METRIC(family, metric_max)                                                                  \
         .Set(std::max(GET_METRIC(family, metric_max).Value(), GET_METRIC(family, metric).Value())); \
     SCOPE_EXIT({ GET_METRIC(family, metric).Decrement(); })
-} // namespace DB
+} // namespace DB