Keep daily history of hits-by-size histograms (openzfs#912)

`zcache hits` displays the hits-by-size histogram, showing the predicted
cache hit rate with varying cache size.  The data is based on cache
accesses since the last `zcache hits --clear`.

The problem is that the relevant time period for evaluating cache
effectiveness may not match the time since the histogram was cleared.

This commit makes the zettacache store a history of hits-by-size
histograms, by default one each day.  The `zcache hits` command will now
sum up the histograms for the requested time/date range, which by
default is the entire history (since the cache size was last changed by
`zcache add` or `zcache remove`).  The new `--begin`, `--end`, and
`--duration` flags can be used to select a more specific time range.

The `zcache hits --clear` command will create a new histogram, saving
the current one in the history.

The internal representation of the history can be dumped with `zcache
hits --raw`.  This can be fed into `zcache hits --from-raw` to query and
display the histogram from a different system.

The implementation required a new kind of block-based log to store the
large, variable-size histograms.  The JsonBlockBasedLog stores Rust
structs that are serializable by Serde (as opposed to a normal
BlockBasedLog which stores fixed-size C structs, anotated with
`#[repr(C)]`).

There is no mechanism to discard the historical hits-by-size histograms.
Performance evaluation found good performance even with enough entries
to cover many decades of history.  Therefore, reducing the number of
entries is left to future work.

cmd/zfs_object_agent/util/src/lib.rs

@@ -33,7 +33,7 @@ mod vec_ext;
 pub mod watch_once;
 pub mod write_stdout;
 mod zcache_devices;
-mod zcache_hits;
+pub mod zcache_hits;
 mod zcache_status;
 pub mod zettacache_stats;
 
@@ -69,7 +69,6 @@ pub use write_stdout::flush_stdout;
 pub use zcache_devices::DeviceEntry;
 pub use zcache_devices::DeviceList;
 pub use zcache_devices::VersionedDeviceList;
-pub use zcache_hits::ReportHitsResponse;
 pub use zcache_status::DeviceRemovalStatus;
 pub use zcache_status::DeviceStatus;
 pub use zcache_status::IndexStatus;

cmd/zfs_object_agent/util/src/vec_ext.rs

@@ -250,6 +250,13 @@ where
         Default::default()
     }
 
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self {
+            vec: Vec::with_capacity(capacity),
+            ..Default::default()
+        }
+    }
+
     /// Returns old value (or None if not present)
     pub fn insert(&mut self, key: K, value: V) -> Option<V> {
         let index = key.into();
@@ -384,8 +391,9 @@ where
     K: From<usize> + Into<usize> + Copy,
 {
     fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> VecMap<K, V> {
-        let mut map = VecMap::default();
-        for (k, v) in iter.into_iter() {
+        let iter = iter.into_iter();
+        let mut map = VecMap::with_capacity(iter.size_hint().0);
+        for (k, v) in iter {
             map.insert(k, v);
         }
         map

cmd/zfs_object_agent/util/src/zcache_hits.rs

@@ -5,42 +5,104 @@
 use std::cmp::min;
 use std::time::SystemTime;
 
+use derivative::Derivative;
 use num_traits::ToPrimitive;
 use serde::Deserialize;
 use serde::Serialize;
 
-// Note: This is essentially SizeHistogramPhys with live and ghost merged into combined_histogram
-#[derive(Debug, Serialize, Deserialize)]
-pub struct ReportHitsResponse {
-    pub started: SystemTime,
+use crate::from64::AsUsize;
+
+// Note, this is also part of the zettacache on-disk format, so any changes must be backwards
+// compatible.
+#[derive(Serialize, Deserialize, Clone, Derivative)]
+#[derivative(Debug)]
+pub struct SizeHistogramPhys {
+    pub start: SystemTime,
+    pub end: Option<SystemTime>,
     pub lookups: u64,
-    pub real_hits: u64, // live, not ghost hits
     pub cache_capacity: u64,
     pub bucket_size: u64,
-    #[serde(default)] // serde_nvlist omits empty Vec's
-    pub combined_histogram: Vec<u64>, // includes real and ghost
+    #[derivative(Debug(format_with = "crate::tersevec"))]
+    pub live_histogram: Vec<u64>,
+    #[derivative(Debug(format_with = "crate::tersevec"))]
+    pub ghost_histogram: Vec<u64>,
 }
 
-impl ReportHitsResponse {
+impl SizeHistogramPhys {
+    /// The histogram range is the sum of the physical cache capacity (cache_capacity)
+    /// plus the additional capacity being tracked for cache ghost hits.
+    pub fn new(histogram_range: u64, cache_capacity: u64, quantiles: usize) -> Self {
+        Self {
+            start: SystemTime::now(),
+            end: None,
+            lookups: 0,
+            cache_capacity,
+            bucket_size: histogram_range
+                .checked_div(quantiles as u64)
+                .unwrap_or_default(),
+            live_histogram: vec![0; quantiles],
+            ghost_histogram: vec![0; quantiles],
+        }
+    }
+
+    /// Record a "hit" in the appropriate size bucket
+    pub fn live_hit(&mut self, size_at_hit: u64) {
+        let index = (size_at_hit / self.bucket_size).as_usize();
+        // The histogram may not be large enough if we've expanded the capacity
+        // since the histogram was created (e.g. by adding disks).
+        if let Some(value) = self.live_histogram.get_mut(index) {
+            *value += 1;
+        }
+    }
+
+    /// Record a ghost "hit" in the appropriate size bucket
+    pub fn ghost_hit(&mut self, size_at_hit: u64) {
+        let index = (size_at_hit / self.bucket_size).as_usize();
+        // The histogram may not be large enough if we've expanded the capacity
+        // since the histogram was created (e.g. by adding disks).
+        if let Some(value) = self.ghost_histogram.get_mut(index) {
+            *value += 1;
+        }
+    }
+
+    pub fn lookup(&mut self) {
+        self.lookups += 1;
+    }
+
     /// Resample a histogram to produce a new histogram with the requested number of buckets.
-    /// This works by dividing each sample in the original histogram into "samples" chunks and
-    /// then adding the number of samples for the physical cache in the original histogram of
-    /// these chunks together for each bucket in the new histogram.
     pub fn resampled(&self, quantiles: usize) -> Self {
-        if quantiles == 0 || self.combined_histogram.is_empty() {
+        if quantiles == 0 || self.live_histogram.is_empty() {
             return Self {
+                live_histogram: Vec::new(),
+                ghost_histogram: Vec::new(),
                 bucket_size: 0,
-                combined_histogram: Vec::new(),
                 ..*self
             };
         }
 
-        let sub_samples_per_resample = self.combined_histogram.len();
-        let mut sample_iter = self.combined_histogram.iter();
+        Self {
+            live_histogram: Self::resample(&self.live_histogram, quantiles),
+            ghost_histogram: Self::resample(&self.ghost_histogram, quantiles),
+            bucket_size: self.bucket_size * self.live_histogram.len() as u64 / quantiles as u64,
+            ..*self
+        }
+    }
+
+    /// Resample a histogram to produce a new histogram with the requested number of buckets.
+    /// This works by dividing each sample in the original histogram into "samples" chunks and
+    /// then adding the number of samples for the physical cache in the original histogram of
+    /// these chunks together for each bucket in the new histogram.
+    fn resample(histogram: &[u64], quantiles: usize) -> Vec<u64> {
+        if quantiles == 0 || histogram.is_empty() {
+            return Vec::new();
+        }
+
+        let sub_samples_per_resample = histogram.len();
+        let mut sample_iter = histogram.iter();
         let mut sub_sample_value = 0.0;
         let mut samples_left = 0;
-        let mut combined_histogram: Vec<u64> = Vec::new();
-        'outer: loop {
+        let mut resampled = Vec::new();
+        loop {
             let mut accumulated_value = 0.0;
             let mut needed_samples = sub_samples_per_resample;
             while needed_samples > 0 {
@@ -49,7 +111,7 @@ impl ReportHitsResponse {
                         Some(sample) => {
                             sub_sample_value = *sample as f64 / quantiles as f64;
                         }
-                        None => break 'outer,
+                        None => return resampled,
                     }
                     samples_left = quantiles;
                 }
@@ -58,13 +120,14 @@ impl ReportHitsResponse {
                 needed_samples -= samples_to_add;
                 samples_left -= samples_to_add;
             }
-            combined_histogram.push(accumulated_value.round().to_u64().unwrap());
-        }
-        Self {
-            bucket_size: self.bucket_size * self.combined_histogram.len() as u64
-                / combined_histogram.len() as u64,
-            combined_histogram,
-            ..*self
+            resampled.push(accumulated_value.round().to_u64().unwrap());
         }
     }
 }
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ReportHitsResponse {
+    pub current: SizeHistogramPhys,
+    pub history: Vec<SizeHistogramPhys>,
+    pub manual_clear: Option<SystemTime>,
+}

cmd/zfs_object_agent/zcache/Cargo.toml

@@ -12,6 +12,7 @@ anyhow = "1.0"
 chrono = "0.4"
 async-trait = "0.1.68"
 clap = { version = "4.1.6", features = ["derive", "wrap_help"] }
+dateparser = "0.2.0"
 git-version = "0.3.5"
 humantime = "2.1.0"
 log = "0.4"