chore(bors): merge pull request #400

400: expose pool io stats from exporter r=abhilashshetty04 a=abhilashshetty04

This PR changes cache update behavior of Metrics node exporter. It would update cache when /metrics is called as opposed to having its own `pool_cycle` for cache update.

We will remove poll_period arg from helm also in the same PR.

Co-authored-by: Abhilash Shetty <[email protected]>

chart/templates/mayastor/io/io-engine-daemonset.yaml

@@ -50,9 +50,6 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
-        args:
-        - "-p{{ .Values.base.metrics.pollingInterval }}"
-        - "--api-versions={{ .Values.io_engine.api }}"
         ports:
           - containerPort: 9502
             protocol: TCP

metrics-exporter/Cargo.toml

@@ -3,7 +3,6 @@ name = "metrics-exporter"
 description = "Metrics Exporters"
 version = "0.1.0"
 edition = "2021"
-authors = ["Sahil Raja <[email protected]>"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [[bin]]

metrics-exporter/src/bin/io_engine/cache/mod.rs

@@ -1,14 +1,11 @@
 mod pool;
+mod pool_stat;
 
-use crate::{
-    client::{grpc_client::GrpcClient, pool::Pools},
-    ExporterConfig,
-};
-
+use crate::client::{grpc_client::GrpcClient, pool::Pools, pool_stat::PoolIoStats};
 use once_cell::sync::OnceCell;
 use serde::{Deserialize, Serialize};
 use std::sync::Mutex;
-use tokio::time::sleep;
+
 static CACHE: OnceCell<Mutex<Cache>> = OnceCell::new();
 
 /// Trait to be implemented by all Resource structs stored in Cache.
@@ -23,6 +20,15 @@ pub(crate) struct Cache {
     data: Data,
 }
 
+/// Wrapper over all the data that has to be stored in cache.
+#[derive(Serialize, Deserialize, Debug)]
+pub(crate) struct Data {
+    /// Contains Pool Capacity and state data.
+    pools: Pools,
+    /// Contains Pool IOStats data.
+    pool_stats: PoolIoStats,
+}
+
 impl Cache {
     /// Initialize the cache with default value.
     pub fn initialize(data: Data) {
@@ -38,13 +44,21 @@ impl Cache {
     pub fn pool_mut(&mut self) -> &mut Pools {
         &mut self.data.pools
     }
-}
 
-/// Wrapper over all the data that has to be stored in cache.
-#[derive(Serialize, Deserialize, Debug)]
-pub(crate) struct Data {
-    /// Contains Pool Capacity and state data.
-    pools: Pools,
+    /// Get mutable reference to PoolIOStats.
+    pub fn pool_iostat_mut(&mut self) -> &mut PoolIoStats {
+        &mut self.data.pool_stats
+    }
+
+    /// Get a reference to Pool.
+    pub fn pool(&self) -> &Pools {
+        &self.data.pools
+    }
+
+    /// Get a reference to PoolIoStats.
+    pub fn pool_iostat(&self) -> &PoolIoStats {
+        &self.data.pool_stats
+    }
 }
 
 impl Default for Data {
@@ -58,21 +72,13 @@ impl Data {
     fn new() -> Self {
         Self {
             pools: Pools { pools: vec![] },
+            pool_stats: PoolIoStats { pool_stats: vec![] },
         }
     }
 }
 
-/// To store data in shared variable i.e cache.
-pub(crate) async fn store_data(client: GrpcClient) {
-    tokio::spawn(async move {
-        store_resource_data(client).await;
-    });
-}
-
-/// To store pools related data in cache.
-async fn store_resource_data(client: GrpcClient) {
-    loop {
-        let _ = pool::store_pool_info_data(client.clone()).await;
-        sleep(ExporterConfig::get_config().polling_time()).await;
-    }
+/// Populates Resource cache struct.
+pub(crate) async fn store_resource_data(client: &GrpcClient) {
+    let _ = pool::store_pool_info_data(client).await;
+    let _ = pool_stat::store_pool_stats_data(client).await;
 }

metrics-exporter/src/bin/io_engine/cache/pool.rs

@@ -1,10 +1,10 @@
 use super::{Cache, ResourceOps};
 use crate::client::{
     grpc_client::GrpcClient,
-    pool::{PoolInfo, PoolOperations, Pools},
+    pool::{PoolInfo, Pools},
 };
 use std::ops::DerefMut;
-use tracing::{debug, error};
+use tracing::error;
 
 impl ResourceOps for Pools {
     type ResourceVec = Vec<PoolInfo>;
@@ -19,7 +19,7 @@ impl ResourceOps for Pools {
 }
 
 /// To store pools state and capacity data in cache.
-pub(crate) async fn store_pool_info_data(client: GrpcClient) -> Result<(), ()> {
+pub(crate) async fn store_pool_info_data(client: &GrpcClient) -> Result<(), ()> {
     let pools = client.list_pools().await;
     let mut cache = match Cache::get_cache().lock() {
         Ok(cache) => cache,
@@ -32,8 +32,7 @@ pub(crate) async fn store_pool_info_data(client: GrpcClient) -> Result<(), ()> {
     match pools {
         // set pools in the cache
         Ok(pools) => {
-            debug!("Updated pool cache with latest metrics");
-            pools_cache.pool_mut().set(pools.pools)
+            pools_cache.pool_mut().set(pools.pools);
         }
         // invalidate cache in case of error
         Err(error) => {

metrics-exporter/src/bin/io_engine/cache/pool_stat.rs

@@ -0,0 +1,45 @@
+use super::{Cache, ResourceOps};
+use crate::client::{
+    grpc_client::GrpcClient,
+    pool_stat::{PoolIoStat, PoolIoStats},
+};
+use std::ops::DerefMut;
+use tracing::error;
+
+impl ResourceOps for PoolIoStats {
+    type ResourceVec = Vec<PoolIoStat>;
+
+    fn set(&mut self, val: Self::ResourceVec) {
+        self.pool_stats = val
+    }
+
+    fn invalidate(&mut self) {
+        self.pool_stats = vec![]
+    }
+}
+
+/// To store pool iostat data in cache.
+pub(crate) async fn store_pool_stats_data(client: &GrpcClient) -> Result<(), ()> {
+    let pool_stats = client.get_pool_iostat().await;
+    let mut cache = match Cache::get_cache().lock() {
+        Ok(cache) => cache,
+        Err(error) => {
+            error!(%error, "Error while getting cache resource");
+            return Err(());
+        }
+    };
+    let pools_cache = cache.deref_mut();
+    match pool_stats {
+        // set pools in the cache
+        Ok(pools) => {
+            pools_cache.pool_iostat_mut().set(pools.pool_stats);
+        }
+        // invalidate cache in case of error
+        Err(error) => {
+            error!(?error, "Error getting pools data, invalidating pools cache");
+            pools_cache.pool_iostat_mut().invalidate();
+            return Err(());
+        }
+    };
+    Ok(())
+}