Allow to configure the compaction checker policy

kvrocks.conf

@@ -462,6 +462,18 @@ profiling-sample-record-threshold-ms 100
 # 0-7am every day.
 compaction-checker-range 0-7
 
+# When the compaction checker is triggered, the db will periodically pick the SST file
+# with the highest "deleted percentage" (i.e. the percentage of deleted keys in the SST 
+# file) to compact, in order to free disk space. 
+# However, if a specific SST file was created more than "force-compact-file-age" seconds 
+# ago, and its percentage of deleted keys is higher than 
+# "force-compact-file-min-deleted-percentage", it will be forcely compacted as well.
+
+# Default: 172800 seconds; Range: [60, INT64_MAX];
+# force-compact-file-age 172800
+# Default: 10 %; Range: [1, 100];
+# force-compact-file-min-deleted-percentage 10
+
 # Bgsave scheduler, auto bgsave at scheduled time
 # time expression format is the same as crontab(currently only support * and int)
 # e.g. bgsave-cron 0 3 * * * 0 4 * * *

src/config/config.cc

@@ -127,6 +127,9 @@ Config::Config() {
       {"replica-announce-ip", false, new StringField(&replica_announce_ip, "")},
       {"replica-announce-port", false, new UInt32Field(&replica_announce_port, 0, 0, PORT_LIMIT)},
       {"compaction-checker-range", false, new StringField(&compaction_checker_range_, "")},
+      {"force-compact-file-age", false, new Int64Field(&force_compact_file_age, 2 * 24 * 3600, 60, INT64_MAX)},
+      {"force-compact-file-min-deleted-percentage", false,
+       new IntField(&force_compact_file_min_deleted_percentage, 10, 1, 100)},
       {"db-name", true, new StringField(&db_name, "change.me.db")},
       {"dir", true, new StringField(&dir, "/tmp/kvrocks")},
       {"backup-dir", false, new StringField(&backup_dir, "")},

src/config/config.h

@@ -131,6 +131,8 @@ struct Config {
   Cron compact_cron;
   Cron bgsave_cron;
   CompactionCheckerRange compaction_checker_range{-1, -1};
+  int64_t force_compact_file_age;
+  int force_compact_file_min_deleted_percentage;
   std::map<std::string, std::string> tokens;
   std::string replica_announce_ip;
   uint32_t replica_announce_port = 0;

src/server/server.cc

@@ -175,7 +175,7 @@ Status Server::Start() {
   compaction_checker_thread_ = GET_OR_RET(Util::CreateThread("compact-check", [this] {
     uint64_t counter = 0;
     time_t last_compact_date = 0;
-    CompactionChecker compaction_checker(this->storage_);
+    CompactionChecker compaction_checker{this->storage_};
 
     while (!stop_) {
       // Sleep first

src/storage/compaction_checker.cc

@@ -55,8 +55,12 @@ void CompactionChecker::PickCompactionFiles(const std::string &cf_name) {
   if (props.size() / 360 > maxFilesToCompact) {
     maxFilesToCompact = props.size() / 360;
   }
-  int64_t forceCompactSeconds = 2 * 24 * 3600;
   int64_t now = Util::GetTimeStamp();
+
+  auto force_compact_file_age = storage_->GetConfig()->force_compact_file_age;
+  auto force_compact_min_ratio =
+      static_cast<double>(storage_->GetConfig()->force_compact_file_min_deleted_percentage / 100);
+
   std::string best_filename;
   double best_delete_ratio = 0;
   int64_t total_keys = 0, deleted_keys = 0;
@@ -76,8 +80,6 @@ void CompactionChecker::PickCompactionFiles(const std::string &cf_name) {
       }
     }
 
-    // don't compact the SST created in 1 hour
-    if (file_creation_time > static_cast<uint64_t>(now - 3600)) continue;
     for (const auto &property_iter : iter.second->user_collected_properties) {
       if (property_iter.first == "total_keys") {
         auto parse_result = ParseInt<int>(property_iter.second, 10);
@@ -104,16 +106,23 @@ void CompactionChecker::PickCompactionFiles(const std::string &cf_name) {
     }
 
     if (start_key.empty() || stop_key.empty()) continue;
-    // pick the file which was created more than 2 days
-    if (file_creation_time < static_cast<uint64_t>(now - forceCompactSeconds)) {
-      LOG(INFO) << "[compaction checker] Going to compact the key in file(created more than 2 days): " << iter.first;
-      s = storage_->Compact(&start_key, &stop_key);
-      LOG(INFO) << "[compaction checker] Compact the key in file(created more than 2 days): " << iter.first
+    double delete_ratio = static_cast<double>(deleted_keys) / static_cast<double>(total_keys);
+
+    // pick the file according to force compact policy
+    if (file_creation_time < static_cast<uint64_t>(now - force_compact_file_age) &&
+        delete_ratio >= force_compact_min_ratio) {
+      LOG(INFO) << "[compaction checker] Going to compact the key in file (force compact policy): " << iter.first;
+      auto s = storage_->Compact(&start_key, &stop_key);
+      LOG(INFO) << "[compaction checker] Compact the key in file (force compact policy): " << iter.first
                 << " finished, result: " << s.ToString();
       maxFilesToCompact--;
+      continue;
     }
+
+    // don't compact the SST created in 1 hour
+    if (file_creation_time > static_cast<uint64_t>(now - 3600)) continue;
+
     // pick the file which has highest delete ratio
-    double delete_ratio = static_cast<double>(deleted_keys) / static_cast<double>(total_keys);
     if (total_keys != 0 && delete_ratio > best_delete_ratio) {
       best_delete_ratio = delete_ratio;
       best_filename = iter.first;

src/storage/storage.h

@@ -127,6 +127,7 @@ class Storage {
   uint64_t GetCompactionCount() { return compaction_count_; }
   void IncrCompactionCount(uint64_t n) { compaction_count_.fetch_add(n); }
   bool IsSlotIdEncoded() { return config_->slot_id_encoded; }
+  const Config *GetConfig() { return config_; }
 
   Status BeginTxn();
   Status CommitTxn();