Merge branch 'master' into prefix_filter

src/include/pegasus/version.h

@@ -3,4 +3,4 @@
 // can be found in the LICENSE file in the root directory of this source tree.
 
 #pragma once
-#define PEGASUS_VERSION "1.12.SNAPSHOT"
+#define PEGASUS_VERSION "1.13.SNAPSHOT"

src/server/config.ini

@@ -239,7 +239,8 @@
   ;; WARNING: memory release may incur major performance downgrade when inproperly configured.
   ;;          ensure this feature is only enabled when it's necessary.
   mem_release_enabled = false
-  mem_release_interval_ms = 86400000
+  mem_release_check_interval_ms = 3600000
+  mem_release_max_reserved_mem_percentage = 10
 
   lb_interval_ms = 10000
 

src/server/info_collector.cpp

@@ -167,6 +167,7 @@ void info_collector::on_app_stat()
         all.rdb_block_cache_total_count += row.rdb_block_cache_total_count;
         all.rdb_index_and_filter_blocks_mem_usage += row.rdb_index_and_filter_blocks_mem_usage;
         all.rdb_memtable_mem_usage += row.rdb_memtable_mem_usage;
+        all.rdb_estimate_num_keys += row.rdb_estimate_num_keys;
         read_qps[i] = row.get_qps + row.multi_get_qps + row.scan_qps;
         write_qps[i] = row.put_qps + row.multi_put_qps + row.remove_qps + row.multi_remove_qps +
                        row.incr_qps + row.check_and_set_qps + row.check_and_mutate_qps;
@@ -205,6 +206,7 @@ void info_collector::on_app_stat()
         counters->rdb_index_and_filter_blocks_mem_usage->set(
             row.rdb_index_and_filter_blocks_mem_usage);
         counters->rdb_memtable_mem_usage->set(row.rdb_memtable_mem_usage);
+        counters->rdb_estimate_num_keys->set(row.rdb_estimate_num_keys);
         counters->read_qps->set(read_qps[i]);
         counters->write_qps->set(write_qps[i]);
     }
@@ -255,6 +257,7 @@ info_collector::AppStatCounters *info_collector::get_app_counters(const std::str
     INIT_COUNTER(rdb_block_cache_hit_rate);
     INIT_COUNTER(rdb_index_and_filter_blocks_mem_usage);
     INIT_COUNTER(rdb_memtable_mem_usage);
+    INIT_COUNTER(rdb_estimate_num_keys);
     INIT_COUNTER(read_qps);
     INIT_COUNTER(write_qps);
     _app_stat_counters[app_name] = counters;

src/server/info_collector.h

@@ -53,6 +53,7 @@ class info_collector
         ::dsn::perf_counter_wrapper rdb_block_cache_mem_usage;
         ::dsn::perf_counter_wrapper rdb_index_and_filter_blocks_mem_usage;
         ::dsn::perf_counter_wrapper rdb_memtable_mem_usage;
+        ::dsn::perf_counter_wrapper rdb_estimate_num_keys;
         ::dsn::perf_counter_wrapper read_qps;
         ::dsn::perf_counter_wrapper write_qps;
     };

src/server/pegasus_server_impl.cpp

@@ -353,6 +353,13 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
     snprintf(name, 255, "rdb.memtable.memory_usage@%s", str_gpid.c_str());
     _pfc_rdb_memtable_mem_usage.init_app_counter(
         "app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the memory usage of rocksdb memtable");
+
+    snprintf(name, 255, "rdb.estimate_num_keys@%s", str_gpid.c_str());
+    _pfc_rdb_estimate_num_keys.init_app_counter(
+        "app.pegasus",
+        name,
+        COUNTER_TYPE_NUMBER,
+        "statistics the estimated number of keys inside the rocksdb");
 }
 
 void pegasus_server_impl::parse_checkpoints()
@@ -2287,6 +2294,14 @@ void pegasus_server_impl::update_replica_rocksdb_statistics()
         _pfc_rdb_memtable_mem_usage->set(val);
         dinfo_replica("_pfc_rdb_memtable_mem_usage: {} bytes", val);
     }
+
+    // for the same n kv pairs, kEstimateNumKeys will be counted n times, you need compaction to
+    // remove duplicate
+    if (_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val) &&
+        dsn::buf2uint64(str_val, val)) {
+        _pfc_rdb_estimate_num_keys->set(val);
+        dinfo_replica("_pfc_rdb_estimate_num_keys: {}", val);
+    }
 }
 
 void pegasus_server_impl::update_server_rocksdb_statistics()

src/server/pegasus_server_impl.h

@@ -364,6 +364,7 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service
     ::dsn::perf_counter_wrapper _pfc_rdb_block_cache_total_count;
     ::dsn::perf_counter_wrapper _pfc_rdb_index_and_filter_blocks_mem_usage;
     ::dsn::perf_counter_wrapper _pfc_rdb_memtable_mem_usage;
+    ::dsn::perf_counter_wrapper _pfc_rdb_estimate_num_keys;
 };
 
 } // namespace server

src/shell/command_helper.h

@@ -538,6 +538,7 @@ struct row_data
     double rdb_block_cache_total_count = 0;
     double rdb_index_and_filter_blocks_mem_usage = 0;
     double rdb_memtable_mem_usage = 0;
+    double rdb_estimate_num_keys = 0;
 };
 
 inline bool
@@ -589,6 +590,8 @@ update_app_pegasus_perf_counter(row_data &row, const std::string &counter_name,
         row.rdb_index_and_filter_blocks_mem_usage += value;
     else if (counter_name == "rdb.memtable.memory_usage")
         row.rdb_memtable_mem_usage += value;
+    else if (counter_name == "rdb.estimate_num_keys")
+        row.rdb_estimate_num_keys += value;
     else
         return false;
     return true;