Write buffer manager supports dynamic config

include/rocksdb/write_buffer_manager.h

@@ -97,6 +97,10 @@ class WriteBufferManager final {
 
   void SetFlushSize(size_t new_size);
 
+  void SetFlushOldestFirst(bool v) {
+    flush_oldest_first_.store(v, std::memory_order_relaxed);
+  }
+
   // Below functions should be called by RocksDB internally.
 
   // This handle is the same as the one created by `DB::Open` or
@@ -191,7 +195,7 @@ class WriteBufferManager final {
   std::atomic<size_t> flush_size_;
   // Only used when flush_size is non-zero.
   std::atomic<size_t> memory_active_;
-  const bool flush_oldest_first_;
+  std::atomic<bool> flush_oldest_first_;
 
   const bool allow_stall_;
   const float stall_ratio_;

memtable/write_buffer_manager.cc

@@ -184,17 +184,39 @@ void WriteBufferManager::MaybeFlushLocked(DB* this_db) {
   uint64_t candidate_size = 0;
   uint64_t max_score = 0;
   uint64_t current_score = 0;
+
   for (auto& s : sentinels_) {
+    // TODO: move this calculation to a callback.
     uint64_t current_memory_bytes = std::numeric_limits<uint64_t>::max();
     uint64_t oldest_time = std::numeric_limits<uint64_t>::max();
     s->db->GetApproximateActiveMemTableStats(s->cf, &current_memory_bytes,
                                              &oldest_time);
-    if (flush_oldest_first_) {
+    if (flush_oldest_first_.load(std::memory_order_relaxed)) {
       // Convert oldest to highest score.
       current_score = std::numeric_limits<uint64_t>::max() - oldest_time;
     } else {
       current_score = current_memory_bytes;
     }
+    // A very mild penalty for too many L0 files.
+    uint64_t level0;
+    // 3 is to optimize the frequency of getting options, which uses mutex.
+    if (s->db->GetIntProperty(DB::Properties::kNumFilesAtLevelPrefix + "0",
+                              &level0) &&
+        level0 >= 3) {
+      auto opts = s->db->GetOptions(s->cf);
+      if (opts.level0_file_num_compaction_trigger > 0 &&
+          level0 >=
+              static_cast<uint64_t>(opts.level0_file_num_compaction_trigger)) {
+        auto diff = level0 - static_cast<uint64_t>(
+                                 opts.level0_file_num_compaction_trigger);
+        // 0->2, +1->4, +2->8, +3->12, +4->18
+        uint64_t factor = (diff + 2) * (diff + 2) / 2;
+        if (factor > 100) {
+          factor = 100;
+        }
+        current_score = current_score * (100 - factor) / factor;
+      }
+    }
     if (current_score > max_score) {
       candidate = s.get();
       max_score = current_score;