From 105aa7d146531358aa62e9b7058029bae622a2eb Mon Sep 17 00:00:00 2001
From: trinity-1686a <trinity@quickwit.io>
Date: Fri, 25 Oct 2024 16:32:18 +0200
Subject: [PATCH] add eviction metrics to caches (#5523)

---
 .../quickwit-storage/src/cache/byte_range_cache.rs |  2 ++
 .../src/cache/memory_sized_cache.rs                |  3 +++
 .../quickwit-storage/src/file_descriptor_cache.rs  |  3 +++
 quickwit/quickwit-storage/src/metrics.rs           | 14 ++++++++++++++
 .../src/split_cache/split_table.rs                 |  8 ++++++++
 5 files changed, 30 insertions(+)

diff --git a/quickwit/quickwit-storage/src/cache/byte_range_cache.rs b/quickwit/quickwit-storage/src/cache/byte_range_cache.rs
index 9068e23657e..9ef3b7f523f 100644
--- a/quickwit/quickwit-storage/src/cache/byte_range_cache.rs
+++ b/quickwit/quickwit-storage/src/cache/byte_range_cache.rs
@@ -314,6 +314,8 @@ impl<T: 'static + ToOwned + ?Sized + Ord> NeedMutByteRangeCache<T> {
         self.num_bytes -= num_bytes as u64;
         self.cache_counters.in_cache_count.dec();
         self.cache_counters.in_cache_num_bytes.sub(num_bytes as i64);
+        self.cache_counters.evict_num_items.inc();
+        self.cache_counters.evict_num_bytes.inc_by(num_bytes as u64);
     }
 }
 
diff --git a/quickwit/quickwit-storage/src/cache/memory_sized_cache.rs b/quickwit/quickwit-storage/src/cache/memory_sized_cache.rs
index 119e0b4cbec..59cb6cb2383 100644
--- a/quickwit/quickwit-storage/src/cache/memory_sized_cache.rs
+++ b/quickwit/quickwit-storage/src/cache/memory_sized_cache.rs
@@ -74,6 +74,7 @@ struct NeedMutMemorySizedCache<K: Hash + Eq> {
 
 impl<K: Hash + Eq> Drop for NeedMutMemorySizedCache<K> {
     fn drop(&mut self) {
+        // we don't count this toward evicted entries, as we are clearing the whole cache
         self.cache_counters
             .in_cache_count
             .sub(self.num_items as i64);
@@ -110,6 +111,8 @@ impl<K: Hash + Eq> NeedMutMemorySizedCache<K> {
         self.num_bytes -= num_bytes;
         self.cache_counters.in_cache_count.dec();
         self.cache_counters.in_cache_num_bytes.sub(num_bytes as i64);
+        self.cache_counters.evict_num_items.inc();
+        self.cache_counters.evict_num_bytes.inc_by(num_bytes);
     }
 
     pub fn get<Q>(&mut self, cache_key: &Q) -> Option<OwnedBytes>
diff --git a/quickwit/quickwit-storage/src/file_descriptor_cache.rs b/quickwit/quickwit-storage/src/file_descriptor_cache.rs
index eee90a513b0..648914fed04 100644
--- a/quickwit/quickwit-storage/src/file_descriptor_cache.rs
+++ b/quickwit/quickwit-storage/src/file_descriptor_cache.rs
@@ -119,6 +119,9 @@ impl FileDescriptorCache {
         self.fd_cache_metrics
             .in_cache_count
             .set(fd_cache_lock.len() as i64);
+        self.fd_cache_metrics
+            .evict_num_items
+            .inc_by(split_ids.len() as u64);
     }
 
     pub async fn get_or_open_split_file(
diff --git a/quickwit/quickwit-storage/src/metrics.rs b/quickwit/quickwit-storage/src/metrics.rs
index 77287de891b..d346031a992 100644
--- a/quickwit/quickwit-storage/src/metrics.rs
+++ b/quickwit/quickwit-storage/src/metrics.rs
@@ -120,6 +120,8 @@ pub struct CacheMetrics {
     pub hits_num_items: IntCounter,
     pub hits_num_bytes: IntCounter,
     pub misses_num_items: IntCounter,
+    pub evict_num_items: IntCounter,
+    pub evict_num_bytes: IntCounter,
 }
 
 impl CacheMetrics {
@@ -157,6 +159,18 @@ impl CacheMetrics {
                 CACHE_METRICS_NAMESPACE,
                 &[("component_name", component_name)],
             ),
+            evict_num_items: new_counter_with_labels(
+                "cache_evict_total",
+                "Number of cache entry evicted by component",
+                CACHE_METRICS_NAMESPACE,
+                &[("component_name", component_name)],
+            ),
+            evict_num_bytes: new_counter_with_labels(
+                "cache_evict_bytes",
+                "Number of cache entry evicted in bytes by component",
+                CACHE_METRICS_NAMESPACE,
+                &[("component_name", component_name)],
+            ),
         }
     }
 }
diff --git a/quickwit/quickwit-storage/src/split_cache/split_table.rs b/quickwit/quickwit-storage/src/split_cache/split_table.rs
index f709c1a8bb1..3fccf923595 100644
--- a/quickwit/quickwit-storage/src/split_cache/split_table.rs
+++ b/quickwit/quickwit-storage/src/split_cache/split_table.rs
@@ -165,6 +165,14 @@ impl SplitTable {
                     .searcher_split_cache
                     .in_cache_num_bytes
                     .sub(num_bytes as i64);
+                crate::metrics::STORAGE_METRICS
+                    .searcher_split_cache
+                    .evict_num_items
+                    .inc();
+                crate::metrics::STORAGE_METRICS
+                    .searcher_split_cache
+                    .evict_num_bytes
+                    .inc_by(num_bytes);
                 &mut self.on_disk_splits
             }
         };