From 4b62ca1cd52c787c485b2574d3a38bb0274af41b Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Fri, 9 Nov 2018 14:40:25 +0800 Subject: [PATCH 01/11] add metrics in rocksdb --- src/server/info_collector.cpp | 11 ++ src/server/info_collector.h | 4 + src/server/pegasus_server_impl.cpp | 208 +++++++++++++++++++---------- src/server/pegasus_server_impl.h | 26 +++- src/shell/command_helper.h | 64 +++++---- 5 files changed, 199 insertions(+), 114 deletions(-) diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp index 224a2f0caa..3350f30978 100644 --- a/src/server/info_collector.cpp +++ b/src/server/info_collector.cpp @@ -97,6 +97,11 @@ void info_collector::on_app_stat() all.recent_abnormal_count += row.recent_abnormal_count; all.storage_mb += row.storage_mb; all.storage_count += row.storage_count; + all.rdb_block_cache_hit_count += row.rdb_block_cache_hit_count; + all.rdb_block_cache_total_count += row.rdb_block_cache_total_count; + all.rdb_block_cache_mem_usage += row.rdb_block_cache_mem_usage; + all.rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage; + all.rdb_memtable_mem_usage += row.rdb_memtable_mem_usage; read_qps[i] = row.get_qps + row.multi_get_qps + row.scan_qps; write_qps[i] = row.put_qps + row.multi_put_qps + row.remove_qps + row.multi_remove_qps + row.incr_qps + row.check_and_set_qps + row.check_and_mutate_qps; @@ -123,6 +128,12 @@ void info_collector::on_app_stat() counters->recent_abnormal_count->set(row.recent_abnormal_count); counters->storage_mb->set(row.storage_mb); counters->storage_count->set(row.storage_count); + counters->rdb_block_cache_hit_rate->set(row.rdb_block_cache_hit_count / + row.rdb_block_cache_total_count); + counters->rdb_block_cache_mem_usage->set(row.rdb_block_cache_mem_usage); + counters->rdb_index_and_filter_blocks_mem_usage->set( + row.rdb_index_and_filter_blocks_mem_usage); + counters->rdb_memtable_mem_usage->set(row.rdb_memtable_mem_usage); counters->read_qps->set(read_qps[i]); counters->write_qps->set(write_qps[i]); } diff --git a/src/server/info_collector.h b/src/server/info_collector.h index 52299081d5..9a354e50d6 100644 --- a/src/server/info_collector.h +++ b/src/server/info_collector.h @@ -43,6 +43,10 @@ class info_collector ::dsn::perf_counter_wrapper recent_abnormal_count; ::dsn::perf_counter_wrapper storage_mb; ::dsn::perf_counter_wrapper storage_count; + ::dsn::perf_counter_wrapper rdb_block_cache_hit_rate; + ::dsn::perf_counter_wrapper rdb_block_cache_mem_usage; + ::dsn::perf_counter_wrapper rdb_index_and_filter_blocks_mem_usage; + ::dsn::perf_counter_wrapper rdb_memtable_mem_usage; ::dsn::perf_counter_wrapper read_qps; ::dsn::perf_counter_wrapper write_qps; }; diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 8606aab298..71e3285b1b 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -7,9 +7,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -21,12 +21,20 @@ #include "pegasus_event_listener.h" #include "pegasus_server_write.h" +using namespace dsn::literals::chrono_literals; + namespace pegasus { namespace server { DEFINE_TASK_CODE(LPC_PEGASUS_SERVER_DELAY, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT) -DEFINE_TASK_CODE(LPC_UPDATING_ROCKSDB_SSTSIZE, TASK_PRIORITY_COMMON, THREAD_POOL_REPLICATION_LONG) +DEFINE_TASK_CODE(LPC_UPDATE_REPLICA_ROCKSDB_STATISTICS, + TASK_PRIORITY_COMMON, + THREAD_POOL_REPLICATION_LONG) + +DEFINE_TASK_CODE(LPC_UPDATE_SERVER_ROCKSDB_STATISTICS, + TASK_PRIORITY_COMMON, + THREAD_POOL_REPLICATION_LONG) static std::string chkpt_get_dir_name(int64_t decree) { @@ -41,6 +49,9 @@ static bool chkpt_init_from_dir(const char *name, int64_t &decree) std::string(name) == chkpt_get_dir_name(decree); } +::dsn::task_ptr pegasus_server_impl::_update_server_rdb_stat; +::dsn::perf_counter_wrapper pegasus_server_impl::_pfc_rdb_block_cache_mem_usage; + pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) : dsn::apps::rrdb_service(r), _db(nullptr), @@ -198,15 +209,13 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) } } - // read rocksdb::BlockBasedTableOptions configurations - rocksdb::BlockBasedTableOptions tbl_opts; // disable table block cache, default: false if (dsn_config_get_value_bool("pegasus.server", "rocksdb_disable_table_block_cache", false, "rocksdb tbl_opts.no_block_cache, default false")) { - tbl_opts.no_block_cache = true; - tbl_opts.block_restart_interval = 4; + _tbl_opts.no_block_cache = true; + _tbl_opts.block_restart_interval = 4; } else { // block cache capacity, default 10G static uint64_t capacity = dsn_config_get_value_uint64( @@ -225,7 +234,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) // init block cache static std::shared_ptr cache = rocksdb::NewLRUCache(capacity, num_shard_bits); - tbl_opts.block_cache = cache; + _tbl_opts.block_cache = cache; } // disable bloom filter, default: false @@ -233,13 +242,17 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) "rocksdb_disable_bloom_filter", false, "rocksdb tbl_opts.filter_policy, default nullptr")) { - tbl_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); + _tbl_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); } - _db_opts.table_factory.reset(NewBlockBasedTableFactory(tbl_opts)); + _db_opts.table_factory.reset(NewBlockBasedTableFactory(_tbl_opts)); _key_ttl_compaction_filter_factory = std::make_shared(); _db_opts.compaction_filter_factory = _key_ttl_compaction_filter_factory; + _statistics = rocksdb::CreateDBStatistics(); + _statistics->stats_level_ = rocksdb::kExceptDetailedTimers; + _db_opts.statistics = _statistics; + _db_opts.listeners.emplace_back(new pegasus_event_listener()); // disable write ahead logging as replication handles logging instead now @@ -254,12 +267,12 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) 0, "checkpoint_reserve_time_seconds, 0 means no check"); - // get the _updating_sstsize_inteval_seconds. - _updating_rocksdb_sstsize_interval_seconds = - (uint32_t)dsn_config_get_value_uint64("pegasus.server", - "updating_rocksdb_sstsize_interval_seconds", - 600, - "updating_rocksdb_sstsize_interval_seconds"); + // get the _update_rdb_stat_interval. + _update_rdb_stat_interval = + std::chrono::seconds(dsn_config_get_value_uint64("pegasus.server", + "updating_rocksdb_statistics_interval", + 600, + "updating_rocksdb_statistics_interval")); // TODO: move the qps/latency counters and it's statistics to replication_app_base layer char str_gpid[128], buf[256]; @@ -315,14 +328,44 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) "statistic the recent abnormal read count"); snprintf(buf, 255, "disk.storage.sst.count@%s", str_gpid); - _pfc_sst_count.init_app_counter( + _pfc_rdb_sst_count.init_app_counter( "app.pegasus", buf, COUNTER_TYPE_NUMBER, "statistic the count of sstable files"); snprintf(buf, 255, "disk.storage.sst(MB)@%s", str_gpid); - _pfc_sst_size.init_app_counter( + _pfc_rdb_sst_size.init_app_counter( "app.pegasus", buf, COUNTER_TYPE_NUMBER, "statistic the size of sstable files"); - updating_rocksdb_sstsize(); + snprintf(buf, 255, "rdb.block_cache.hit_count@%s", str_gpid); + _pfc_rdb_block_cache_hit_count.init_app_counter( + "app.pegasus", buf, COUNTER_TYPE_NUMBER, "statistic the hit count of rocksdb block cache"); + + snprintf(buf, 255, "rdb.block_cache.total_count@%s", str_gpid); + _pfc_rdb_block_cache_total_count.init_app_counter( + "app.pegasus", + buf, + COUNTER_TYPE_NUMBER, + "statistic the total count of rocksdb block cache"); + + static std::once_flag flag; + std::call_once(flag, [&]() { + _pfc_rdb_block_cache_mem_usage.init_global_counter( + "replica", + "app.pegasus", + "rdb.block_cache.memory_usage", + COUNTER_TYPE_NUMBER, + "statistic the memory usage of rocksdb block cache"); + }); + + snprintf(buf, 255, "rdb.index_and_filter_blocks.memory_usage@%s", str_gpid); + _pfc_rdb_index_and_filter_blocks_mem_usage.init_app_counter( + "app.pegasus", + buf, + COUNTER_TYPE_NUMBER, + "statistic the memory usage of rocksdb index and filter blocks"); + + snprintf(buf, 255, "rdb.memtable.memory_usage@%s", str_gpid); + _pfc_rdb_memtable_mem_usage.init_app_counter( + "app.pegasus", buf, COUNTER_TYPE_NUMBER, "statistic the memory usage of rocksdb memtable"); } void pegasus_server_impl::parse_checkpoints() @@ -1523,14 +1566,25 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) _is_open = true; - dinfo("%s: start the updating sstsize timer task", replica_name()); - _updating_rocksdb_sstsize_timer_task = ::dsn::tasking::enqueue_timer( - LPC_UPDATING_ROCKSDB_SSTSIZE, - &_tracker, - [this]() { this->updating_rocksdb_sstsize(); }, - std::chrono::seconds(_updating_rocksdb_sstsize_interval_seconds), - 0, - std::chrono::seconds(30)); + dinfo("%s: start the update rocksdb statistics timer task", replica_name()); + _update_replica_rdb_stat = + ::dsn::tasking::enqueue_timer(LPC_UPDATING_ROCKSDB_STATISTICS, + &_tracker, + [this]() { this->updating_rocksdb_statistics(); }, + _update_rdb_stat_interval, + 0, + 30_s); + + static std::once_flag flag; + std::call_once(flag, [&]() { + _update_server_rdb_stat = ::dsn::tasking::enqueue_timer( + LPC_UPDATING_ROCKSDB_STATISTICS_STATIC, + &_tracker, + [this]() { this->updating_rocksdb_statistics_static(); }, + _update_rdb_stat_interval, + 0, + 30_s); + }); // initialize write service after server being initialized. _server_write = dsn::make_unique(this, _verbose_log); @@ -1566,9 +1620,9 @@ ::dsn::error_code pegasus_server_impl::stop(bool clear_state) } // stop all tracked tasks when pegasus server is stopped. - if (_updating_rocksdb_sstsize_timer_task != nullptr) { - _updating_rocksdb_sstsize_timer_task->cancel(true); - _updating_rocksdb_sstsize_timer_task = nullptr; + if (_update_replica_rdb_stat != nullptr) { + _update_replica_rdb_stat->cancel(true); + _update_replica_rdb_stat = nullptr; } _tracker.cancel_outstanding_tasks(); @@ -1600,8 +1654,13 @@ ::dsn::error_code pegasus_server_impl::stop(bool clear_state) derror("%s: rmdir %s failed when stop app", replica_name(), data_dir().c_str()); return ::dsn::ERR_FILE_OPERATION_FAILED; } - _pfc_sst_count->set(0); - _pfc_sst_size->set(0); + _pfc_rdb_sst_count->set(0); + _pfc_rdb_sst_size->set(0); + _pfc_rdb_block_cache_hit_count->set(0); + _pfc_rdb_block_cache_total_count->set(0); + _pfc_rdb_block_cache_mem_usage->set(0); + _pfc_rdb_index_and_filter_blocks_mem_usage->set(0); + _pfc_rdb_memtable_mem_usage->set(0); } ddebug( @@ -2161,54 +2220,55 @@ int pegasus_server_impl::append_key_value_for_multi_get( return 1; } -// statistic the count and size of files of this type. return (-1,-1) if failed. -static std::pair get_type_file_size(const std::string &path, - const std::string &type) +void pegasus_server_impl::updating_rocksdb_statistics() { - std::vector files; - if (!::dsn::utils::filesystem::get_subfiles(path, files, false)) { - dwarn("get subfiles of dir %s failed", path.c_str()); - return std::pair(-1, -1); - } - int64_t res = 0; - int64_t cnt = 0; - for (auto &f : files) { - if (f.length() > type.length() && f.substr(f.length() - type.length()) == type) { - int64_t tsize = 0; - if (::dsn::utils::filesystem::file_size(f, tsize)) { - res += tsize; - cnt++; - } else { - dwarn("get size of file %s failed", f.c_str()); - return std::pair(-1, -1); - } + std::string str_val; + uint64_t val = 0; + for (int i = 0; i < _db_opts.num_levels; ++i) { + int cur_level_count = 0; + if (_db->GetProperty(rocksdb::DB::Properties::kNumFilesAtLevelPrefix + std::to_string(i), + &str_val) && + dsn::buf2int32(str_val, cur_level_count)) { + val += cur_level_count; } } - return std::pair(cnt, res); -} + _pfc_rdb_sst_count->set(val); + ddebug_replica("_pfc_rdb_sst_count: {}", val); -std::pair pegasus_server_impl::statistic_sst_size() -{ - // dir = data_dir()/rdb - return get_type_file_size(::dsn::utils::filesystem::path_combine(data_dir(), "rdb"), ".sst"); + if (_db->GetProperty(rocksdb::DB::Properties::kTotalSstFilesSize, &str_val) && + dsn::buf2uint64(str_val, val)) { + static uint64_t bytes_per_mb = 1U << 20U; + _pfc_rdb_sst_size->set(val / bytes_per_mb); + ddebug_replica("_pfc_rdb_sst_size: {} bytes", val); + } + + uint64_t block_cache_hit = _statistics->getTickerCount(rocksdb::BLOCK_CACHE_HIT); + _pfc_rdb_block_cache_hit_count->set(block_cache_hit); + ddebug_replica("_pfc_rdb_block_cache_hit_count: {}", block_cache_hit); + + uint64_t block_cache_miss = _statistics->getTickerCount(rocksdb::BLOCK_CACHE_MISS); + uint64_t block_cache_total = block_cache_hit + block_cache_miss; + _pfc_rdb_block_cache_total_count->set(block_cache_total); + ddebug_replica("_pfc_rdb_block_cache_total_count: {}", block_cache_total); + + if (_db->GetProperty(rocksdb::DB::Properties::kEstimateTableReadersMem, &str_val) && + dsn::buf2uint64(str_val, val)) { + _pfc_rdb_index_and_filter_blocks_mem_usage->set(val); + ddebug_replica("_pfc_rdb_index_and_filter_blocks_mem_usage: {} bytes", val); + } + + if (_db->GetProperty(rocksdb::DB::Properties::kCurSizeAllMemTables, &str_val) && + dsn::buf2uint64(str_val, val)) { + _pfc_rdb_memtable_mem_usage->set(val); + ddebug_replica("_pfc_rdb_memtable_mem_usage: {} bytes", val); + } } -void pegasus_server_impl::updating_rocksdb_sstsize() +void pegasus_server_impl::updating_rocksdb_statistics_static() { - std::pair sst_size = statistic_sst_size(); - if (sst_size.first == -1) { - dwarn("%s: statistic sst file size failed", replica_name()); - } else { - int64_t sst_size_mb = sst_size.second / 1048576; - ddebug("%s: statistic sst file size succeed, sst_count = %" PRId64 ", sst_size = %" PRId64 - "(%" PRId64 "MB)", - replica_name(), - sst_size.first, - sst_size.second, - sst_size_mb); - _pfc_sst_count->set(sst_size.first); - _pfc_sst_size->set(sst_size_mb); - } + uint64_t val = _tbl_opts.block_cache->GetUsage(); + _pfc_rdb_block_cache_mem_usage->set(val); + ddebug_replica("_pfc_rdb_block_cache_mem_usage: {} bytes", val); } std::pair @@ -2438,8 +2498,8 @@ uint64_t pegasus_server_impl::do_manual_compact(const rocksdb::CompactRangeOptio status.ToString().c_str(), dsn_now_ms() - start_time); - // update size immediately - updating_rocksdb_sstsize(); + // update rocksdb statistics immediately + updating_rocksdb_statistics(); return _db->GetLastManualCompactFinishTime(); } diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index bfc3c7f462..c7cf14a54a 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -198,10 +199,9 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service const ::dsn::blob &filter_pattern, const ::dsn::blob &value); - // statistic the sst file info for this replica. return (-1,-1) if failed. - std::pair statistic_sst_size(); + void updating_rocksdb_statistics(); - void updating_rocksdb_sstsize(); + void updating_rocksdb_statistics_static(); // get the absolute path of restore directory and the flag whether force restore from env // return @@ -250,6 +250,8 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service uint64_t _abnormal_multi_get_iterate_count_threshold; std::shared_ptr _key_ttl_compaction_filter_factory; + std::shared_ptr _statistics; + rocksdb::BlockBasedTableOptions _tbl_opts; rocksdb::Options _db_opts; rocksdb::WriteOptions _wt_opts; rocksdb::ReadOptions _rd_opts; @@ -270,8 +272,9 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service pegasus_context_cache _context_cache; - ::dsn::task_ptr _updating_rocksdb_sstsize_timer_task; - uint32_t _updating_rocksdb_sstsize_interval_seconds; + std::chrono::seconds _update_rdb_stat_interval; + ::dsn::task_ptr _update_replica_rdb_stat; + static ::dsn::task_ptr _update_server_rdb_stat; pegasus_manual_compact_service _manual_compact_svc; @@ -289,8 +292,17 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service ::dsn::perf_counter_wrapper _pfc_recent_expire_count; ::dsn::perf_counter_wrapper _pfc_recent_filter_count; ::dsn::perf_counter_wrapper _pfc_recent_abnormal_count; - ::dsn::perf_counter_wrapper _pfc_sst_count; - ::dsn::perf_counter_wrapper _pfc_sst_size; + + // rocksdb internal statistics + // server level + static ::dsn::perf_counter_wrapper _pfc_rdb_block_cache_mem_usage; + // replica level + ::dsn::perf_counter_wrapper _pfc_rdb_sst_count; + ::dsn::perf_counter_wrapper _pfc_rdb_sst_size; + ::dsn::perf_counter_wrapper _pfc_rdb_block_cache_hit_count; + ::dsn::perf_counter_wrapper _pfc_rdb_block_cache_total_count; + ::dsn::perf_counter_wrapper _pfc_rdb_index_and_filter_blocks_mem_usage; + ::dsn::perf_counter_wrapper _pfc_rdb_memtable_mem_usage; }; } // namespace server diff --git a/src/shell/command_helper.h b/src/shell/command_helper.h index faf66b7b31..15278dba52 100644 --- a/src/shell/command_helper.h +++ b/src/shell/command_helper.h @@ -368,40 +368,28 @@ inline bool parse_app_pegasus_perf_counter_name(const std::string &name, struct row_data { std::string row_name; - double get_qps; - double multi_get_qps; - double put_qps; - double multi_put_qps; - double remove_qps; - double multi_remove_qps; - double incr_qps; - double check_and_set_qps; - double check_and_mutate_qps; - double scan_qps; - double recent_expire_count; - double recent_filter_count; - double recent_abnormal_count; - double storage_mb; - double storage_count; - row_data() - : get_qps(0), - multi_get_qps(0), - put_qps(0), - multi_put_qps(0), - remove_qps(0), - multi_remove_qps(0), - incr_qps(0), - check_and_set_qps(0), - check_and_mutate_qps(0), - scan_qps(0), - recent_expire_count(0), - recent_filter_count(0), - recent_abnormal_count(0), - storage_mb(0), - storage_count(0) - { - } + double get_qps = 0; + double multi_get_qps = 0; + double put_qps = 0; + double multi_put_qps = 0; + double remove_qps = 0; + double multi_remove_qps = 0; + double incr_qps = 0; + double check_and_set_qps = 0; + double check_and_mutate_qps = 0; + double scan_qps = 0; + double recent_expire_count = 0; + double recent_filter_count = 0; + double recent_abnormal_count = 0; + double storage_mb = 0; + double storage_count = 0; + double rdb_block_cache_hit_count = 0; + double rdb_block_cache_total_count = 0; + double rdb_block_cache_mem_usage = 0; + double rdb_index_and_filter_blocks_mem_usage = 0; + double rdb_memtable_mem_usage = 0; }; + inline bool update_app_pegasus_perf_counter(row_data &row, const std::string &counter_name, double value) { @@ -435,6 +423,16 @@ update_app_pegasus_perf_counter(row_data &row, const std::string &counter_name, row.storage_mb += value; else if (counter_name == "disk.storage.sst.count") row.storage_count += value; + else if (counter_name == "rdb.block_cache.hit_count") + row.rdb_block_cache_hit_count += value; + else if (counter_name == "rdb.block_cache.total_count") + row.rdb_block_cache_total_count += value; + else if (counter_name == "rdb.block_cache.memory_usage") + row.rdb_block_cache_mem_usage += value; + else if (counter_name == "rdb.index_and_filter_blocks.memory_usage") + row.rdb_index_and_filter_blocks_mem_usage += value; + else if (counter_name == "rdb.memtable.memory_usage") + row.rdb_memtable_mem_usage += value; else return false; return true; From 25d196f26c37143c6b7b0b0f00e345d5f2f623a0 Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Fri, 9 Nov 2018 16:10:53 +0800 Subject: [PATCH 02/11] add metrics in rocksdb --- src/server/info_collector.cpp | 4 ++++ src/server/pegasus_server_impl.cpp | 4 ++-- src/shell/commands.h | 22 ++++++++++++++++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp index 3350f30978..3f15758a72 100644 --- a/src/server/info_collector.cpp +++ b/src/server/info_collector.cpp @@ -177,6 +177,10 @@ info_collector::AppStatCounters *info_collector::get_app_counters(const std::str INIT_COUNER(recent_abnormal_count); INIT_COUNER(storage_mb); INIT_COUNER(storage_count); + INIT_COUNER(rdb_block_cache_hit_rate); + INIT_COUNER(rdb_block_cache_mem_usage); + INIT_COUNER(rdb_index_and_filter_blocks_mem_usage); + INIT_COUNER(rdb_memtable_mem_usage); INIT_COUNER(read_qps); INIT_COUNER(write_qps); _app_stat_counters[app_name] = counters; diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 71e3285b1b..9f3ab9acb5 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -1568,7 +1568,7 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) dinfo("%s: start the update rocksdb statistics timer task", replica_name()); _update_replica_rdb_stat = - ::dsn::tasking::enqueue_timer(LPC_UPDATING_ROCKSDB_STATISTICS, + ::dsn::tasking::enqueue_timer(LPC_UPDATE_REPLICA_ROCKSDB_STATISTICS, &_tracker, [this]() { this->updating_rocksdb_statistics(); }, _update_rdb_stat_interval, @@ -1578,7 +1578,7 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) static std::once_flag flag; std::call_once(flag, [&]() { _update_server_rdb_stat = ::dsn::tasking::enqueue_timer( - LPC_UPDATING_ROCKSDB_STATISTICS_STATIC, + LPC_UPDATE_SERVER_ROCKSDB_STATISTICS, &_tracker, [this]() { this->updating_rocksdb_statistics_static(); }, _update_rdb_stat_interval, diff --git a/src/shell/commands.h b/src/shell/commands.h index 0ab5260105..db4c34ada9 100644 --- a/src/shell/commands.h +++ b/src/shell/commands.h @@ -3652,7 +3652,7 @@ inline bool app_stat(command_executor *e, shell_context *sc, arguments args) } std::ostream out(buf); - size_t w = 10; + static const size_t w = 14; size_t first_column_width = w; if (app_name.empty()) { for (row_data &row : rows) { @@ -3669,8 +3669,9 @@ inline bool app_stat(command_executor *e, shell_context *sc, arguments args) << std::setw(w) << std::right << "CAM" << std::setw(w) << std::right << "SCAN"; if (!only_qps) { out << std::setw(w) << std::right << "expired" << std::setw(w) << std::right << "filtered" - << std::setw(w) << std::right << "abnormal" << std::setw(w) << std::right << "file_mb" - << std::setw(w) << std::right << "file_num"; + << std::setw(w) << std::right << "abnormal" << std::setw(w) << std::right + << "file_size(MB)" << std::setw(w) << std::right << "file_num" << std::setw(w) + << std::right << "hit_rate" << std::setw(w) << std::right << "mem_usage(MB)"; } out << std::endl; rows.resize(rows.size() + 1); @@ -3692,6 +3693,11 @@ inline bool app_stat(command_executor *e, shell_context *sc, arguments args) sum.recent_abnormal_count += row.recent_abnormal_count; sum.storage_mb += row.storage_mb; sum.storage_count += row.storage_count; + sum.rdb_block_cache_hit_count += row.rdb_block_cache_hit_count; + sum.rdb_block_cache_total_count += row.rdb_block_cache_total_count; + sum.rdb_block_cache_mem_usage += row.rdb_block_cache_mem_usage; + sum.rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage; + sum.rdb_memtable_mem_usage += row.rdb_memtable_mem_usage; } #define PRINT_QPS(field) \ do { \ @@ -3714,11 +3720,19 @@ inline bool app_stat(command_executor *e, shell_context *sc, arguments args) PRINT_QPS(check_and_mutate_qps); PRINT_QPS(scan_qps); if (!only_qps) { + double block_cache_hit_rate = + abs(row.rdb_block_cache_total_count) < 1e-6 + ? 100.0 + : row.rdb_block_cache_hit_count / row.rdb_block_cache_total_count; out << std::setw(w) << std::right << (int64_t)row.recent_expire_count << std::setw(w) << std::right << (int64_t)row.recent_filter_count << std::setw(w) << std::right << (int64_t)row.recent_abnormal_count << std::setw(w) << std::right << (int64_t)row.storage_mb << std::setw(w) << std::right - << (int64_t)row.storage_count; + << (int64_t)row.storage_count << std::setw(w) << std::right << block_cache_hit_rate + << std::setw(w) << std::right + << (row.rdb_block_cache_mem_usage + row.rdb_index_and_filter_blocks_mem_usage + + row.rdb_memtable_mem_usage) / + (1 << 20U); } out << std::endl; } From 62fcaf24d1314199660646b2e0818b8181a78352 Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Fri, 9 Nov 2018 17:54:08 +0800 Subject: [PATCH 03/11] add metrics in rocksdb --- src/server/brief_stat.cpp | 1 + src/server/pegasus_server_impl.cpp | 15 +++++++-------- src/server/pegasus_server_impl.h | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/server/brief_stat.cpp b/src/server/brief_stat.cpp index 1a751bd2c2..302d896d63 100644 --- a/src/server/brief_stat.cpp +++ b/src/server/brief_stat.cpp @@ -26,6 +26,7 @@ static std::map s_brief_stat_map = { {"replica*eon.replica_stub*replicas.learning.count", "learning_count"}, {"replica*app.pegasus*manual.compact.running.count", "manual_compact_running_count"}, {"replica*app.pegasus*manual.compact.enqueue.count", "manual_compact_enqueue_count"}, + {"replica*app.pegasus*rdb.block_cache.memory_usage", "rdb_block_cache_memory_usage"}, {"replica*eon.replica_stub*shared.log.size(MB)", "shared_log_size(MB)"}, {"replica*server*memused.virt(MB)", "memused_virt(MB)"}, {"replica*server*memused.res(MB)", "memused_res(MB)"}, diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 9f3ab9acb5..c974847e2d 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -267,12 +267,11 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) 0, "checkpoint_reserve_time_seconds, 0 means no check"); - // get the _update_rdb_stat_interval. _update_rdb_stat_interval = std::chrono::seconds(dsn_config_get_value_uint64("pegasus.server", - "updating_rocksdb_statistics_interval", + "update_rdb_stat_interval", 600, - "updating_rocksdb_statistics_interval")); + "update_rdb_stat_interval, in seconds")); // TODO: move the qps/latency counters and it's statistics to replication_app_base layer char str_gpid[128], buf[256]; @@ -1570,7 +1569,7 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) _update_replica_rdb_stat = ::dsn::tasking::enqueue_timer(LPC_UPDATE_REPLICA_ROCKSDB_STATISTICS, &_tracker, - [this]() { this->updating_rocksdb_statistics(); }, + [this]() { this->update_replica_rocksdb_statistics(); }, _update_rdb_stat_interval, 0, 30_s); @@ -1580,7 +1579,7 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) _update_server_rdb_stat = ::dsn::tasking::enqueue_timer( LPC_UPDATE_SERVER_ROCKSDB_STATISTICS, &_tracker, - [this]() { this->updating_rocksdb_statistics_static(); }, + [this]() { this->update_server_rocksdb_statistics(); }, _update_rdb_stat_interval, 0, 30_s); @@ -2220,7 +2219,7 @@ int pegasus_server_impl::append_key_value_for_multi_get( return 1; } -void pegasus_server_impl::updating_rocksdb_statistics() +void pegasus_server_impl::update_replica_rocksdb_statistics() { std::string str_val; uint64_t val = 0; @@ -2264,7 +2263,7 @@ void pegasus_server_impl::updating_rocksdb_statistics() } } -void pegasus_server_impl::updating_rocksdb_statistics_static() +void pegasus_server_impl::update_server_rocksdb_statistics() { uint64_t val = _tbl_opts.block_cache->GetUsage(); _pfc_rdb_block_cache_mem_usage->set(val); @@ -2499,7 +2498,7 @@ uint64_t pegasus_server_impl::do_manual_compact(const rocksdb::CompactRangeOptio dsn_now_ms() - start_time); // update rocksdb statistics immediately - updating_rocksdb_statistics(); + update_replica_rocksdb_statistics(); return _db->GetLastManualCompactFinishTime(); } diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index c7cf14a54a..69fe9e6376 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -199,9 +199,9 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service const ::dsn::blob &filter_pattern, const ::dsn::blob &value); - void updating_rocksdb_statistics(); + void update_replica_rocksdb_statistics(); - void updating_rocksdb_statistics_static(); + void update_server_rocksdb_statistics(); // get the absolute path of restore directory and the flag whether force restore from env // return From e4afe9d5a3aada3bb1873b5024b4ee8bdf105fa0 Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Fri, 9 Nov 2018 18:05:50 +0800 Subject: [PATCH 04/11] code format --- src/server/pegasus_server_impl.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index c974847e2d..aa16e60b37 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -267,11 +267,8 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) 0, "checkpoint_reserve_time_seconds, 0 means no check"); - _update_rdb_stat_interval = - std::chrono::seconds(dsn_config_get_value_uint64("pegasus.server", - "update_rdb_stat_interval", - 600, - "update_rdb_stat_interval, in seconds")); + _update_rdb_stat_interval = std::chrono::seconds(dsn_config_get_value_uint64( + "pegasus.server", "update_rdb_stat_interval", 600, "update_rdb_stat_interval, in seconds")); // TODO: move the qps/latency counters and it's statistics to replication_app_base layer char str_gpid[128], buf[256]; From cb1fb41ff9bc11839a4aa21cd52a5437b91e30cc Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Mon, 12 Nov 2018 18:31:20 +0800 Subject: [PATCH 05/11] fix by comments --- src/server/pegasus_server_impl.cpp | 48 +++++++++++++++++------------- src/shell/commands.h | 6 ++-- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index aa16e60b37..ea66724cd8 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -217,24 +217,28 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) _tbl_opts.no_block_cache = true; _tbl_opts.block_restart_interval = 4; } else { - // block cache capacity, default 10G - static uint64_t capacity = dsn_config_get_value_uint64( - "pegasus.server", - "rocksdb_block_cache_capacity", - 10 * 1024 * 1024 * 1024ULL, - "block cache capacity for one pegasus server, shared by all rocksdb instances"); - - // block cache num shard bits, default -1(auto) - static int num_shard_bits = (int)dsn_config_get_value_int64( - "pegasus.server", - "rocksdb_block_cache_num_shard_bits", - -1, - "block cache will be sharded into 2^num_shard_bits shards"); - - // init block cache - static std::shared_ptr cache = - rocksdb::NewLRUCache(capacity, num_shard_bits); - _tbl_opts.block_cache = cache; + // If block cache is enabled, all replicas on this server will share the same block cache + // object. It's convenient to control the total memory used by this server, and the LRU + // algorithm used by the block cache object can be more efficient in this way. + static std::once_flag flag; + std::call_once(flag, [&]() { + // block cache capacity, default 10G + uint64_t capacity = dsn_config_get_value_uint64( + "pegasus.server", + "rocksdb_block_cache_capacity", + 10 * 1024 * 1024 * 1024ULL, + "block cache capacity for one pegasus server, shared by all rocksdb instances"); + + // block cache num shard bits, default -1(auto) + int num_shard_bits = (int)dsn_config_get_value_int64( + "pegasus.server", + "rocksdb_block_cache_num_shard_bits", + -1, + "block cache will be sharded into 2^num_shard_bits shards"); + + // init block cache + _tbl_opts.block_cache = rocksdb::NewLRUCache(capacity, num_shard_bits); + }); } // disable bloom filter, default: false @@ -342,6 +346,8 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) COUNTER_TYPE_NUMBER, "statistic the total count of rocksdb block cache"); + // Block cache is a singleton on this server shared by all replicas, so we initialize + // `_pfc_rdb_block_cache_mem_usage` only once. static std::once_flag flag; std::call_once(flag, [&]() { _pfc_rdb_block_cache_mem_usage.init_global_counter( @@ -1571,11 +1577,13 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) 0, 30_s); + // Block cache is a singleton on this server shared by all replicas, its metrics update task + // should be scheduled once an interval on the process view. static std::once_flag flag; std::call_once(flag, [&]() { _update_server_rdb_stat = ::dsn::tasking::enqueue_timer( LPC_UPDATE_SERVER_ROCKSDB_STATISTICS, - &_tracker, + nullptr, [this]() { this->update_server_rocksdb_statistics(); }, _update_rdb_stat_interval, 0, @@ -2264,7 +2272,7 @@ void pegasus_server_impl::update_server_rocksdb_statistics() { uint64_t val = _tbl_opts.block_cache->GetUsage(); _pfc_rdb_block_cache_mem_usage->set(val); - ddebug_replica("_pfc_rdb_block_cache_mem_usage: {} bytes", val); + ddebug_f("_pfc_rdb_block_cache_mem_usage: {} bytes", val); } std::pair diff --git a/src/shell/commands.h b/src/shell/commands.h index db4c34ada9..2b8b064f0b 100644 --- a/src/shell/commands.h +++ b/src/shell/commands.h @@ -3669,9 +3669,9 @@ inline bool app_stat(command_executor *e, shell_context *sc, arguments args) << std::setw(w) << std::right << "CAM" << std::setw(w) << std::right << "SCAN"; if (!only_qps) { out << std::setw(w) << std::right << "expired" << std::setw(w) << std::right << "filtered" - << std::setw(w) << std::right << "abnormal" << std::setw(w) << std::right - << "file_size(MB)" << std::setw(w) << std::right << "file_num" << std::setw(w) - << std::right << "hit_rate" << std::setw(w) << std::right << "mem_usage(MB)"; + << std::setw(w) << std::right << "abnormal" << std::setw(w) << std::right << "file_mb" + << std::setw(w) << std::right << "file_num" << std::setw(w) << std::right << "hit_rate" + << std::setw(w) << std::right << "rdb_mem_mb"; } out << std::endl; rows.resize(rows.size() + 1); From 6620bcebdb4b82a7f46e0421bd4d05b9ad05b2b2 Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Tue, 13 Nov 2018 00:24:08 +0800 Subject: [PATCH 06/11] fix by comments --- src/server/pegasus_server_impl.cpp | 16 ++++++++-------- src/server/pegasus_server_impl.h | 1 + src/shell/commands.h | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index ea66724cd8..2cdcd439c6 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -49,6 +49,7 @@ static bool chkpt_init_from_dir(const char *name, int64_t &decree) std::string(name) == chkpt_get_dir_name(decree); } +std::shared_ptr pegasus_server_impl::_block_cache; ::dsn::task_ptr pegasus_server_impl::_update_server_rdb_stat; ::dsn::perf_counter_wrapper pegasus_server_impl::_pfc_rdb_block_cache_mem_usage; @@ -237,8 +238,11 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r) "block cache will be sharded into 2^num_shard_bits shards"); // init block cache - _tbl_opts.block_cache = rocksdb::NewLRUCache(capacity, num_shard_bits); + _block_cache = rocksdb::NewLRUCache(capacity, num_shard_bits); }); + + // every replica has the same block cache + _tbl_opts.block_cache = _block_cache; } // disable bloom filter, default: false @@ -1573,21 +1577,17 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) ::dsn::tasking::enqueue_timer(LPC_UPDATE_REPLICA_ROCKSDB_STATISTICS, &_tracker, [this]() { this->update_replica_rocksdb_statistics(); }, - _update_rdb_stat_interval, - 0, - 30_s); + _update_rdb_stat_interval); // Block cache is a singleton on this server shared by all replicas, its metrics update task - // should be scheduled once an interval on the process view. + // should be scheduled once an interval on the server view. static std::once_flag flag; std::call_once(flag, [&]() { _update_server_rdb_stat = ::dsn::tasking::enqueue_timer( LPC_UPDATE_SERVER_ROCKSDB_STATISTICS, nullptr, [this]() { this->update_server_rocksdb_statistics(); }, - _update_rdb_stat_interval, - 0, - 30_s); + _update_rdb_stat_interval); }); // initialize write service after server being initialized. diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index 69fe9e6376..bec66fb953 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -258,6 +258,7 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service std::string _usage_scenario; rocksdb::DB *_db; + static std::shared_ptr _block_cache; volatile bool _is_open; uint32_t _value_schema_version; std::atomic _last_durable_decree; diff --git a/src/shell/commands.h b/src/shell/commands.h index 2b8b064f0b..94b4ceaccb 100644 --- a/src/shell/commands.h +++ b/src/shell/commands.h @@ -3722,7 +3722,7 @@ inline bool app_stat(command_executor *e, shell_context *sc, arguments args) if (!only_qps) { double block_cache_hit_rate = abs(row.rdb_block_cache_total_count) < 1e-6 - ? 100.0 + ? 0.0 : row.rdb_block_cache_hit_count / row.rdb_block_cache_total_count; out << std::setw(w) << std::right << (int64_t)row.recent_expire_count << std::setw(w) << std::right << (int64_t)row.recent_filter_count << std::setw(w) << std::right From 3b48ea3b63ca94f369dffb0dd5c47ba33001a538 Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Tue, 13 Nov 2018 10:40:13 +0800 Subject: [PATCH 07/11] fix --- src/server/pegasus_server_impl.cpp | 2 +- src/server/pegasus_server_impl.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 2cdcd439c6..de40eafc51 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -2270,7 +2270,7 @@ void pegasus_server_impl::update_replica_rocksdb_statistics() void pegasus_server_impl::update_server_rocksdb_statistics() { - uint64_t val = _tbl_opts.block_cache->GetUsage(); + uint64_t val = _block_cache->GetUsage(); _pfc_rdb_block_cache_mem_usage->set(val); ddebug_f("_pfc_rdb_block_cache_mem_usage: {} bytes", val); } diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index bec66fb953..231f34f371 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -201,7 +201,7 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service void update_replica_rocksdb_statistics(); - void update_server_rocksdb_statistics(); + static void update_server_rocksdb_statistics(); // get the absolute path of restore directory and the flag whether force restore from env // return From 57f41b5d51c0d378d43632929cdd6f2336ae11eb Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Tue, 13 Nov 2018 10:50:59 +0800 Subject: [PATCH 08/11] fix --- src/shell/commands.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shell/commands.h b/src/shell/commands.h index 94b4ceaccb..8e6f756fba 100644 --- a/src/shell/commands.h +++ b/src/shell/commands.h @@ -3652,7 +3652,7 @@ inline bool app_stat(command_executor *e, shell_context *sc, arguments args) } std::ostream out(buf); - static const size_t w = 14; + static const size_t w = 10; size_t first_column_width = w; if (app_name.empty()) { for (row_data &row : rows) { @@ -3671,7 +3671,7 @@ inline bool app_stat(command_executor *e, shell_context *sc, arguments args) out << std::setw(w) << std::right << "expired" << std::setw(w) << std::right << "filtered" << std::setw(w) << std::right << "abnormal" << std::setw(w) << std::right << "file_mb" << std::setw(w) << std::right << "file_num" << std::setw(w) << std::right << "hit_rate" - << std::setw(w) << std::right << "rdb_mem_mb"; + << std::setw(w) << std::right << "rdbmem_mb"; } out << std::endl; rows.resize(rows.size() + 1); From 0af2f4589e1331feefd6b6e5371ca90ef84c4bc6 Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Tue, 13 Nov 2018 10:58:13 +0800 Subject: [PATCH 09/11] fix --- src/server/pegasus_server_impl.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index de40eafc51..04bf290cf8 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -1583,10 +1583,11 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) // should be scheduled once an interval on the server view. static std::once_flag flag; std::call_once(flag, [&]() { + // The timer task will always running even though there is no replicas _update_server_rdb_stat = ::dsn::tasking::enqueue_timer( LPC_UPDATE_SERVER_ROCKSDB_STATISTICS, - nullptr, - [this]() { this->update_server_rocksdb_statistics(); }, + nullptr, // TODO: the tracker is nullptr, we will fix it later + [this]() { update_server_rocksdb_statistics(); }, _update_rdb_stat_interval); }); From 1563cb30d8a0e6905b2aa094a6df037ea05a4ac2 Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Tue, 13 Nov 2018 11:11:51 +0800 Subject: [PATCH 10/11] update rdsn --- rdsn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdsn b/rdsn index a9479d629d..bb2fb74bee 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit a9479d629d2dc36ebe0c2d25230dd4713f9cd085 +Subproject commit bb2fb74bee731128e8ca88ceba1804cef967a16a From 1e03c3df142c20d416a88e1200a4aeee36fc7e1f Mon Sep 17 00:00:00 2001 From: acelyc111 <405403881@qq.com> Date: Tue, 13 Nov 2018 12:19:34 +0800 Subject: [PATCH 11/11] update task code --- src/server/pegasus_server_impl.cpp | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index 04bf290cf8..26c0566482 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "base/pegasus_key_schema.h" #include "base/pegasus_value_schema.h" @@ -28,14 +29,6 @@ namespace server { DEFINE_TASK_CODE(LPC_PEGASUS_SERVER_DELAY, TASK_PRIORITY_COMMON, ::dsn::THREAD_POOL_DEFAULT) -DEFINE_TASK_CODE(LPC_UPDATE_REPLICA_ROCKSDB_STATISTICS, - TASK_PRIORITY_COMMON, - THREAD_POOL_REPLICATION_LONG) - -DEFINE_TASK_CODE(LPC_UPDATE_SERVER_ROCKSDB_STATISTICS, - TASK_PRIORITY_COMMON, - THREAD_POOL_REPLICATION_LONG) - static std::string chkpt_get_dir_name(int64_t decree) { char buffer[256]; @@ -1574,7 +1567,7 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) dinfo("%s: start the update rocksdb statistics timer task", replica_name()); _update_replica_rdb_stat = - ::dsn::tasking::enqueue_timer(LPC_UPDATE_REPLICA_ROCKSDB_STATISTICS, + ::dsn::tasking::enqueue_timer(LPC_REPLICATION_LONG_COMMON, &_tracker, [this]() { this->update_replica_rocksdb_statistics(); }, _update_rdb_stat_interval); @@ -1585,7 +1578,7 @@ ::dsn::error_code pegasus_server_impl::start(int argc, char **argv) std::call_once(flag, [&]() { // The timer task will always running even though there is no replicas _update_server_rdb_stat = ::dsn::tasking::enqueue_timer( - LPC_UPDATE_SERVER_ROCKSDB_STATISTICS, + LPC_REPLICATION_LONG_COMMON, nullptr, // TODO: the tracker is nullptr, we will fix it later [this]() { update_server_rocksdb_statistics(); }, _update_rdb_stat_interval);