Disable compressed secondary cache if capacity is 0 (#11863)

Summary:
This PR makes disabling the compressed secondary cache by setting capacity to 0 a bit more efficient. Previously, inserts/lookups would go to the backing LRUCache before getting rejected due to 0 capacity. With this change, insert/lookup would return from ```CompressedSecondaryCache``` itself.

Tests:
Existing tests

Pull Request resolved: #11863

Reviewed By: akankshamahajan15

Differential Revision: D49476248

Pulled By: anand1976

fbshipit-source-id: f0f17a5e3df7d8bfc06709f8f23c1302056ba590

cache/compressed_secondary_cache.cc

@@ -22,7 +22,8 @@ CompressedSecondaryCache::CompressedSecondaryCache(
       cache_options_(opts),
       cache_res_mgr_(std::make_shared<ConcurrentCacheReservationManager>(
           std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
-              cache_))) {}
+              cache_))),
+      disable_cache_(opts.capacity == 0) {}
 
 CompressedSecondaryCache::~CompressedSecondaryCache() {
   assert(cache_res_mgr_->GetTotalReservedCacheSize() == 0);
@@ -33,6 +34,10 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     Cache::CreateContext* create_context, bool /*wait*/, bool advise_erase,
     bool& kept_in_sec_cache) {
   assert(helper);
+  if (disable_cache_) {
+    return nullptr;
+  }
+
   std::unique_ptr<SecondaryCacheResultHandle> handle;
   kept_in_sec_cache = false;
   Cache::Handle* lru_handle = cache_->Lookup(key);
@@ -115,6 +120,10 @@ Status CompressedSecondaryCache::Insert(const Slice& key,
     return Status::InvalidArgument();
   }
 
+  if (disable_cache_) {
+    return Status::OK();
+  }
+
   auto internal_helper = GetHelper(cache_options_.enable_custom_split_merge);
   if (!force_insert) {
     Cache::Handle* lru_handle = cache_->Lookup(key);
@@ -186,6 +195,7 @@ Status CompressedSecondaryCache::SetCapacity(size_t capacity) {
   MutexLock l(&capacity_mutex_);
   cache_options_.capacity = capacity;
   cache_->SetCapacity(capacity);
+  disable_cache_ = capacity == 0;
   return Status::OK();
 }
 

cache/compressed_secondary_cache.h

@@ -136,6 +136,7 @@ class CompressedSecondaryCache : public SecondaryCache {
   CompressedSecondaryCacheOptions cache_options_;
   mutable port::Mutex capacity_mutex_;
   std::shared_ptr<ConcurrentCacheReservationManager> cache_res_mgr_;
+  bool disable_cache_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

unreleased_history/bug_fixes/compressed_sec_cache_disable.md

@@ -0,0 +1 @@
+When the compressed secondary cache capacity is reduced to 0, it should be completely disabled. Before this fix, inserts and lookups would still go to the backing `LRUCache` before returning, thus incurring locking overhead. With this fix, inserts and lookups are no-ops and do not add any overhead.