diff --git a/client/beefy/src/communication/request_response/incoming_requests_handler.rs b/client/beefy/src/communication/request_response/incoming_requests_handler.rs index d3be73dc06a43..854ece11cf93c 100644 --- a/client/beefy/src/communication/request_response/incoming_requests_handler.rs +++ b/client/beefy/src/communication/request_response/incoming_requests_handler.rs @@ -29,8 +29,13 @@ use sc_network_common::protocol::ProtocolName; use sp_runtime::traits::Block; use std::{marker::PhantomData, sync::Arc}; -use crate::communication::request_response::{ - on_demand_justifications_protocol_config, Error, JustificationRequest, BEEFY_SYNC_LOG_TARGET, +use crate::{ + communication::request_response::{ + on_demand_justifications_protocol_config, Error, JustificationRequest, + BEEFY_SYNC_LOG_TARGET, + }, + metric_inc, + metrics::{register_metrics, OnDemandIncomingRequestsMetrics}, }; /// A request coming in, including a sender for sending responses. @@ -119,6 +124,7 @@ pub struct BeefyJustifsRequestHandler { pub(crate) request_receiver: IncomingRequestReceiver, pub(crate) justif_protocol_name: ProtocolName, pub(crate) client: Arc, + pub(crate) metrics: Option, pub(crate) _block: PhantomData, } @@ -132,12 +138,16 @@ where genesis_hash: Hash, fork_id: Option<&str>, client: Arc, + prometheus_registry: Option, ) -> (Self, RequestResponseConfig) { let (request_receiver, config) = on_demand_justifications_protocol_config(genesis_hash, fork_id); let justif_protocol_name = config.name.clone(); - - (Self { request_receiver, justif_protocol_name, client, _block: PhantomData }, config) + let metrics = register_metrics(prometheus_registry); + ( + Self { request_receiver, justif_protocol_name, client, metrics, _block: PhantomData }, + config, + ) } /// Network request-response protocol name used by this handler. @@ -180,12 +190,14 @@ where let peer = request.peer; match self.handle_request(request) { Ok(()) => { + metric_inc!(self, beefy_successful_justification_responses); debug!( target: BEEFY_SYNC_LOG_TARGET, "🥩 Handled BEEFY justification request from {:?}.", peer ) }, Err(e) => { + metric_inc!(self, beefy_failed_justification_responses); // TODO (issue #12293): apply reputation changes here based on error type. debug!( target: BEEFY_SYNC_LOG_TARGET, diff --git a/client/beefy/src/communication/request_response/outgoing_requests_engine.rs b/client/beefy/src/communication/request_response/outgoing_requests_engine.rs index 766480f781ff5..3e0d6e47d9432 100644 --- a/client/beefy/src/communication/request_response/outgoing_requests_engine.rs +++ b/client/beefy/src/communication/request_response/outgoing_requests_engine.rs @@ -34,6 +34,8 @@ use std::{collections::VecDeque, result::Result, sync::Arc}; use crate::{ communication::request_response::{Error, JustificationRequest, BEEFY_SYNC_LOG_TARGET}, justification::{decode_and_verify_finality_proof, BeefyVersionedFinalityProof}, + metric_inc, + metrics::{register_metrics, OnDemandOutgoingRequestsMetrics}, KnownPeers, }; @@ -61,6 +63,7 @@ pub struct OnDemandJustificationsEngine { peers_cache: VecDeque, state: State, + metrics: Option, } impl OnDemandJustificationsEngine { @@ -68,13 +71,16 @@ impl OnDemandJustificationsEngine { network: Arc, protocol_name: ProtocolName, live_peers: Arc>>, + prometheus_registry: Option, ) -> Self { + let metrics = register_metrics(prometheus_registry); Self { network, protocol_name, live_peers, peers_cache: VecDeque::new(), state: State::Idle, + metrics, } } @@ -130,6 +136,7 @@ impl OnDemandJustificationsEngine { if let Some(peer) = self.try_next_peer() { self.request_from_peer(peer, RequestInfo { block, active_set }); } else { + metric_inc!(self, beefy_on_demand_justification_no_peer_to_request_from); debug!( target: BEEFY_SYNC_LOG_TARGET, "🥩 no good peers to request justif #{:?} from", block @@ -159,6 +166,7 @@ impl OnDemandJustificationsEngine { ) -> Result, Error> { response .map_err(|e| { + metric_inc!(self, beefy_on_demand_justification_peer_hang_up); debug!( target: BEEFY_SYNC_LOG_TARGET, "🥩 for on demand justification #{:?}, peer {:?} hung up: {:?}", @@ -169,6 +177,7 @@ impl OnDemandJustificationsEngine { Error::InvalidResponse })? .map_err(|e| { + metric_inc!(self, beefy_on_demand_justification_peer_error); debug!( target: BEEFY_SYNC_LOG_TARGET, "🥩 for on demand justification #{:?}, peer {:?} error: {:?}", @@ -185,6 +194,7 @@ impl OnDemandJustificationsEngine { &req_info.active_set, ) .map_err(|e| { + metric_inc!(self, beefy_on_demand_justification_invalid_proof); debug!( target: BEEFY_SYNC_LOG_TARGET, "🥩 for on demand justification #{:?}, peer {:?} responded with invalid proof: {:?}", @@ -224,6 +234,7 @@ impl OnDemandJustificationsEngine { } }) .map(|proof| { + metric_inc!(self, beefy_on_demand_justification_good_proof); debug!( target: BEEFY_SYNC_LOG_TARGET, "🥩 received valid on-demand justif #{:?} from {:?}", block, peer diff --git a/client/beefy/src/import.rs b/client/beefy/src/import.rs index f0fe3bfa024dc..1b5dda3795aae 100644 --- a/client/beefy/src/import.rs +++ b/client/beefy/src/import.rs @@ -35,6 +35,8 @@ use sc_consensus::{BlockCheckParams, BlockImport, BlockImportParams, ImportResul use crate::{ communication::notification::BeefyVersionedFinalityProofSender, justification::{decode_and_verify_finality_proof, BeefyVersionedFinalityProof}, + metric_inc, + metrics::BlockImportMetrics, LOG_TARGET, }; @@ -49,6 +51,7 @@ pub struct BeefyBlockImport { runtime: Arc, inner: I, justification_sender: BeefyVersionedFinalityProofSender, + metrics: Option, } impl Clone for BeefyBlockImport { @@ -58,6 +61,7 @@ impl Clone for BeefyBlockImport BeefyBlockImport { runtime: Arc, inner: I, justification_sender: BeefyVersionedFinalityProofSender, + metrics: Option, ) -> BeefyBlockImport { - BeefyBlockImport { backend, runtime, inner, justification_sender } + BeefyBlockImport { backend, runtime, inner, justification_sender, metrics } } } @@ -147,6 +152,8 @@ where self.justification_sender .notify(|| Ok::<_, ()>(proof)) .expect("forwards closure result; the closure always returns Ok; qed."); + + metric_inc!(self, beefy_good_justification_imports); } else { debug!( target: LOG_TARGET, @@ -154,6 +161,7 @@ where encoded, number, ); + metric_inc!(self, beefy_bad_justification_imports); } }, _ => (), diff --git a/client/beefy/src/lib.rs b/client/beefy/src/lib.rs index d7de7295a4a11..5f74b052e99fe 100644 --- a/client/beefy/src/lib.rs +++ b/client/beefy/src/lib.rs @@ -28,6 +28,7 @@ use crate::{ }, }, import::BeefyBlockImport, + metrics::register_metrics, round::Rounds, worker::PersistedState, }; @@ -36,7 +37,7 @@ use beefy_primitives::{ GENESIS_AUTHORITY_SET_ID, }; use futures::{stream::Fuse, StreamExt}; -use log::{debug, error, info}; +use log::{error, info}; use parking_lot::Mutex; use prometheus::Registry; use sc_client_api::{Backend, BlockBackend, BlockchainEvents, FinalityNotifications, Finalizer}; @@ -133,6 +134,7 @@ pub fn beefy_block_import_and_links( wrapped_block_import: I, backend: Arc, runtime: Arc, + prometheus_registry: Option, ) -> (BeefyBlockImport, BeefyVoterLinks, BeefyRPCLinks) where B: Block, @@ -152,10 +154,16 @@ where // BlockImport -> Voter links let (to_voter_justif_sender, from_block_import_justif_stream) = BeefyVersionedFinalityProofStream::::channel(); + let metrics = register_metrics(prometheus_registry); // BlockImport - let import = - BeefyBlockImport::new(backend, runtime, wrapped_block_import, to_voter_justif_sender); + let import = BeefyBlockImport::new( + backend, + runtime, + wrapped_block_import, + to_voter_justif_sender, + metrics, + ); let voter_links = BeefyVoterLinks { from_block_import_justif_stream, to_rpc_justif_sender, @@ -242,28 +250,16 @@ where gossip_validator.clone(), None, ); + let metrics = register_metrics(prometheus_registry.clone()); // The `GossipValidator` adds and removes known peers based on valid votes and network events. let on_demand_justifications = OnDemandJustificationsEngine::new( network.clone(), justifications_protocol_name, known_peers, + prometheus_registry.clone(), ); - let metrics = - prometheus_registry.as_ref().map(metrics::Metrics::register).and_then( - |result| match result { - Ok(metrics) => { - debug!(target: LOG_TARGET, "🥩 Registered metrics"); - Some(metrics) - }, - Err(err) => { - debug!(target: LOG_TARGET, "🥩 Failed to register metrics: {:?}", err); - None - }, - }, - ); - // Subscribe to finality notifications and justifications before waiting for runtime pallet and // reuse the streams, so we don't miss notifications while waiting for pallet to be available. let mut finality_notifications = client.finality_notification_stream().fuse(); diff --git a/client/beefy/src/metrics.rs b/client/beefy/src/metrics.rs index 55fdecc36d4b0..e3b48e6ec8d54 100644 --- a/client/beefy/src/metrics.rs +++ b/client/beefy/src/metrics.rs @@ -18,16 +18,22 @@ //! BEEFY Prometheus metrics definition +use log::debug; use prometheus::{register, Counter, Gauge, PrometheusError, Registry, U64}; -/// BEEFY metrics exposed through Prometheus -pub(crate) struct Metrics { +/// Helper trait for registering BEEFY metrics to Prometheus registry. +pub(crate) trait PrometheusRegister: Sized { + const DESCRIPTION: &'static str; + fn register(registry: &Registry) -> Result; +} + +/// BEEFY voting-related metrics exposed through Prometheus +#[derive(Clone, Debug)] +pub struct VoterMetrics { /// Current active validator set id pub beefy_validator_set_id: Gauge, /// Total number of votes sent by this node pub beefy_votes_sent: Counter, - /// Most recent concluded voting round - pub beefy_round_concluded: Gauge, /// Best block finalized by BEEFY pub beefy_best_block: Gauge, /// Best block BEEFY voted on @@ -36,10 +42,31 @@ pub(crate) struct Metrics { pub beefy_should_vote_on: Gauge, /// Number of sessions with lagging signed commitment on mandatory block pub beefy_lagging_sessions: Counter, + /// Number of times no Authority public key found in store + pub beefy_no_authority_found_in_store: Counter, + /// Number of currently buffered votes + pub beefy_buffered_votes: Gauge, + /// Number of valid but stale votes received + pub beefy_stale_votes: Counter, + /// Number of votes dropped due to full buffers + pub beefy_buffered_votes_dropped: Counter, + /// Number of currently buffered justifications + pub beefy_buffered_justifications: Gauge, + /// Number of valid but stale justifications received + pub beefy_stale_justifications: Counter, + /// Number of valid justifications successfully imported + pub beefy_imported_justifications: Counter, + /// Number of justifications dropped due to full buffers + pub beefy_buffered_justifications_dropped: Counter, + /// Trying to set Best Beefy block to old block + pub beefy_best_block_set_last_failure: Gauge, + /// Number of Successful handled votes + pub beefy_successful_handled_votes: Counter, } -impl Metrics { - pub(crate) fn register(registry: &Registry) -> Result { +impl PrometheusRegister for VoterMetrics { + const DESCRIPTION: &'static str = "voter"; + fn register(registry: &Registry) -> Result { Ok(Self { beefy_validator_set_id: register( Gauge::new( @@ -52,13 +79,6 @@ impl Metrics { Counter::new("substrate_beefy_votes_sent", "Number of votes sent by this node")?, registry, )?, - beefy_round_concluded: register( - Gauge::new( - "substrate_beefy_round_concluded", - "Voting round, that has been concluded", - )?, - registry, - )?, beefy_best_block: register( Gauge::new("substrate_beefy_best_block", "Best block finalized by BEEFY")?, registry, @@ -78,10 +98,212 @@ impl Metrics { )?, registry, )?, + beefy_no_authority_found_in_store: register( + Counter::new( + "substrate_beefy_no_authority_found_in_store", + "Number of times no Authority public key found in store", + )?, + registry, + )?, + beefy_buffered_votes: register( + Gauge::new("substrate_beefy_buffered_votes", "Number of currently buffered votes")?, + registry, + )?, + beefy_stale_votes: register( + Counter::new( + "substrate_beefy_stale_votes", + "Number of valid but stale votes received", + )?, + registry, + )?, + beefy_buffered_votes_dropped: register( + Counter::new( + "substrate_beefy_buffered_votes_dropped", + "Number of votes dropped due to full buffers", + )?, + registry, + )?, + beefy_buffered_justifications: register( + Gauge::new( + "substrate_beefy_buffered_justifications", + "Number of currently buffered justifications", + )?, + registry, + )?, + beefy_stale_justifications: register( + Counter::new( + "substrate_beefy_stale_justifications", + "Number of valid but stale justifications received", + )?, + registry, + )?, + beefy_imported_justifications: register( + Counter::new( + "substrate_beefy_imported_justifications", + "Number of valid justifications successfully imported", + )?, + registry, + )?, + beefy_buffered_justifications_dropped: register( + Counter::new( + "substrate_beefy_buffered_justifications_dropped", + "Number of justifications dropped due to full buffers", + )?, + registry, + )?, + beefy_best_block_set_last_failure: register( + Gauge::new( + "substrate_beefy_best_block_to_old_block", + "Trying to set Best Beefy block to old block", + )?, + registry, + )?, + beefy_successful_handled_votes: register( + Counter::new( + "substrate_beefy_successful_handled_votes", + "Number of Successful handled votes", + )?, + registry, + )?, }) } } +/// BEEFY block-import-related metrics exposed through Prometheus +#[derive(Clone, Debug)] +pub struct BlockImportMetrics { + /// Number of Good Justification imports + pub beefy_good_justification_imports: Counter, + /// Number of Bad Justification imports + pub beefy_bad_justification_imports: Counter, +} + +impl PrometheusRegister for BlockImportMetrics { + const DESCRIPTION: &'static str = "block-import"; + fn register(registry: &Registry) -> Result { + Ok(Self { + beefy_good_justification_imports: register( + Counter::new( + "substrate_beefy_good_justification_imports", + "Number of Good Justification imports", + )?, + registry, + )?, + beefy_bad_justification_imports: register( + Counter::new( + "substrate_beefy_bad_justification_imports", + "Number of Bad Justification imports", + )?, + registry, + )?, + }) + } +} + +/// BEEFY on-demand-justifications-related metrics exposed through Prometheus +#[derive(Clone, Debug)] +pub struct OnDemandIncomingRequestsMetrics { + /// Number of Successful Justification responses + pub beefy_successful_justification_responses: Counter, + /// Number of Failed Justification responses + pub beefy_failed_justification_responses: Counter, +} + +impl PrometheusRegister for OnDemandIncomingRequestsMetrics { + const DESCRIPTION: &'static str = "on-demand incoming justification requests"; + fn register(registry: &Registry) -> Result { + Ok(Self { + beefy_successful_justification_responses: register( + Counter::new( + "substrate_beefy_successful_justification_responses", + "Number of Successful Justification responses", + )?, + registry, + )?, + beefy_failed_justification_responses: register( + Counter::new( + "substrate_beefy_failed_justification_responses", + "Number of Failed Justification responses", + )?, + registry, + )?, + }) + } +} + +/// BEEFY on-demand-justifications-related metrics exposed through Prometheus +#[derive(Clone, Debug)] +pub struct OnDemandOutgoingRequestsMetrics { + /// Number of times there was no good peer to request justification from + pub beefy_on_demand_justification_no_peer_to_request_from: Counter, + /// Number of on-demand justification peer hang up + pub beefy_on_demand_justification_peer_hang_up: Counter, + /// Number of on-demand justification peer error + pub beefy_on_demand_justification_peer_error: Counter, + /// Number of on-demand justification invalid proof + pub beefy_on_demand_justification_invalid_proof: Counter, + /// Number of on-demand justification good proof + pub beefy_on_demand_justification_good_proof: Counter, +} + +impl PrometheusRegister for OnDemandOutgoingRequestsMetrics { + const DESCRIPTION: &'static str = "on-demand outgoing justification requests"; + fn register(registry: &Registry) -> Result { + Ok(Self { + beefy_on_demand_justification_no_peer_to_request_from: register( + Counter::new( + "substrate_beefy_on_demand_justification_no_peer_to_request_from", + "Number of times there was no good peer to request justification from", + )?, + registry, + )?, + beefy_on_demand_justification_peer_hang_up: register( + Counter::new( + "substrate_beefy_on_demand_justification_peer_hang_up", + "Number of on-demand justification peer hang up", + )?, + registry, + )?, + beefy_on_demand_justification_peer_error: register( + Counter::new( + "substrate_beefy_on_demand_justification_peer_error", + "Number of on-demand justification peer error", + )?, + registry, + )?, + beefy_on_demand_justification_invalid_proof: register( + Counter::new( + "substrate_beefy_on_demand_justification_invalid_proof", + "Number of on-demand justification invalid proof", + )?, + registry, + )?, + beefy_on_demand_justification_good_proof: register( + Counter::new( + "substrate_beefy_on_demand_justification_good_proof", + "Number of on-demand justification good proof", + )?, + registry, + )?, + }) + } +} + +pub(crate) fn register_metrics( + prometheus_registry: Option, +) -> Option { + prometheus_registry.as_ref().map(T::register).and_then(|result| match result { + Ok(metrics) => { + debug!(target: "beefy", "🥩 Registered {} metrics", T::DESCRIPTION); + Some(metrics) + }, + Err(err) => { + debug!(target: "beefy", "🥩 Failed to register {} metrics: {:?}", T::DESCRIPTION, err); + None + }, + }) +} + // Note: we use the `format` macro to convert an expr into a `u64`. This will fail, // if expr does not derive `Display`. #[macro_export] @@ -104,7 +326,6 @@ macro_rules! metric_inc { }}; } -#[cfg(test)] #[macro_export] macro_rules! metric_get { ($self:ident, $m:ident) => {{ diff --git a/client/beefy/src/tests.rs b/client/beefy/src/tests.rs index 932897c775895..007c57e4269cf 100644 --- a/client/beefy/src/tests.rs +++ b/client/beefy/src/tests.rs @@ -124,6 +124,7 @@ impl BeefyTestNet { justif_protocol_name, client, _block: PhantomData, + metrics: None, }; *net.peers[i].data.beefy_justif_req_handler.lock() = Some(justif_handler); } @@ -203,7 +204,7 @@ impl TestNetFactory for BeefyTestNet { let api = Arc::new(TestApi::with_validator_set(&validator_set)); let inner = BlockImportAdapter::new(client.clone()); let (block_import, voter_links, rpc_links) = - beefy_block_import_and_links(inner, client.as_backend(), api); + beefy_block_import_and_links(inner, client.as_backend(), api, None); let peer_data = PeerData { beefy_rpc_links: Mutex::new(Some(rpc_links)), beefy_voter_links: Mutex::new(Some(voter_links)), diff --git a/client/beefy/src/worker.rs b/client/beefy/src/worker.rs index f367c8b46d037..6528aef12508a 100644 --- a/client/beefy/src/worker.rs +++ b/client/beefy/src/worker.rs @@ -24,8 +24,8 @@ use crate::{ error::Error, justification::BeefyVersionedFinalityProof, keystore::BeefyKeystore, - metric_inc, metric_set, - metrics::Metrics, + metric_get, metric_inc, metric_set, + metrics::VoterMetrics, round::{Rounds, VoteImportResult}, BeefyVoterLinks, LOG_TARGET, }; @@ -252,7 +252,7 @@ pub(crate) struct WorkerParams { pub gossip_validator: Arc>, pub on_demand_justifications: OnDemandJustificationsEngine, pub links: BeefyVoterLinks, - pub metrics: Option, + pub metrics: Option, pub persisted_state: PersistedState, } @@ -312,7 +312,7 @@ pub(crate) struct BeefyWorker { // voter state /// BEEFY client metrics. - metrics: Option, + metrics: Option, /// Buffer holding votes for future processing. pending_votes: BTreeMap< NumberFor, @@ -407,6 +407,7 @@ where if store.intersection(&active).count() == 0 { let msg = "no authority public key found in store".to_string(); debug!(target: LOG_TARGET, "🥩 for block {:?} {}", block, msg); + metric_inc!(self, beefy_no_authority_found_in_store); Err(Error::Keystore(msg)) } else { Ok(()) @@ -494,17 +495,21 @@ where debug!(target: LOG_TARGET, "🥩 Buffer vote for round: {:?}.", block_num); if self.pending_votes.len() < MAX_BUFFERED_VOTE_ROUNDS { let votes_vec = self.pending_votes.entry(block_num).or_default(); - if votes_vec.try_push(vote).is_err() { + if votes_vec.try_push(vote).is_ok() { + metric_inc!(self, beefy_buffered_votes); + } else { warn!( target: LOG_TARGET, "🥩 Buffer vote dropped for round: {:?}", block_num - ) + ); + metric_inc!(self, beefy_buffered_votes_dropped); } } else { warn!(target: LOG_TARGET, "🥩 Buffer vote dropped for round: {:?}.", block_num); + metric_inc!(self, beefy_buffered_votes_dropped); } }, - RoundAction::Drop => (), + RoundAction::Drop => metric_inc!(self, beefy_stale_votes), }; Ok(()) } @@ -524,20 +529,23 @@ where match self.voting_oracle().triage_round(block_num, best_grandpa)? { RoundAction::Process => { debug!(target: LOG_TARGET, "🥩 Process justification for round: {:?}.", block_num); + metric_inc!(self, beefy_imported_justifications); self.finalize(justification)? }, RoundAction::Enqueue => { debug!(target: LOG_TARGET, "🥩 Buffer justification for round: {:?}.", block_num); if self.pending_justifications.len() < MAX_BUFFERED_JUSTIFICATIONS { self.pending_justifications.entry(block_num).or_insert(justification); + metric_inc!(self, beefy_buffered_justifications); } else { + metric_inc!(self, beefy_buffered_justifications_dropped); warn!( target: LOG_TARGET, "🥩 Buffer justification dropped for round: {:?}.", block_num ); } }, - RoundAction::Drop => (), + RoundAction::Drop => metric_inc!(self, beefy_stale_justifications), }; Ok(()) } @@ -555,8 +563,6 @@ where let block_number = vote.commitment.block_number; match rounds.add_vote(vote) { VoteImportResult::RoundConcluded(signed_commitment) => { - metric_set!(self, beefy_round_concluded, block_number); - let finality_proof = VersionedFinalityProof::V1(signed_commitment); info!( target: LOG_TARGET, @@ -584,6 +590,7 @@ where }, VoteImportResult::Invalid | VoteImportResult::Stale => (), }; + metric_inc!(self, beefy_successful_handled_votes); Ok(()) } @@ -637,7 +644,8 @@ where .notify(|| Ok::<_, ()>(finality_proof)) .expect("forwards closure result; the closure always returns Ok; qed."); } else { - debug!(target: LOG_TARGET, "🥩 Can't set best beefy to older: {}", block_num); + debug!(target: LOG_TARGET, "🥩 Can't set best beefy to old: {}", block_num); + metric_set!(self, beefy_best_block_set_last_failure, block_num); } Ok(()) } @@ -669,25 +677,33 @@ where let justifs_to_handle = to_process_for(&mut self.pending_justifications, interval, _ph); for (num, justification) in justifs_to_handle.into_iter() { debug!(target: LOG_TARGET, "🥩 Handle buffered justification for: {:?}.", num); + metric_inc!(self, beefy_imported_justifications); if let Err(err) = self.finalize(justification) { error!(target: LOG_TARGET, "🥩 Error finalizing block: {}", err); } } + metric_set!(self, beefy_buffered_justifications, self.pending_justifications.len()); // Possibly new interval after processing justifications. interval = self.voting_oracle().accepted_interval(best_grandpa)?; } // Process pending votes. if !self.pending_votes.is_empty() { + let mut processed = 0u64; let votes_to_handle = to_process_for(&mut self.pending_votes, interval, _ph); for (num, votes) in votes_to_handle.into_iter() { debug!(target: LOG_TARGET, "🥩 Handle buffered votes for: {:?}.", num); + processed += votes.len() as u64; for v in votes.into_iter() { if let Err(err) = self.handle_vote(v) { error!(target: LOG_TARGET, "🥩 Error handling buffered vote: {}", err); }; } } + if let Some(previous) = metric_get!(self, beefy_buffered_votes) { + previous.sub(processed); + metric_set!(self, beefy_buffered_votes, previous.get()); + } } Ok(()) } @@ -1053,10 +1069,12 @@ pub(crate) mod tests { let gossip_validator = Arc::new(GossipValidator::new(known_peers.clone())); let gossip_engine = GossipEngine::new(network.clone(), "/beefy/1", gossip_validator.clone(), None); + let metrics = None; let on_demand_justifications = OnDemandJustificationsEngine::new( network.clone(), "/beefy/justifs/1".into(), known_peers, + None, ); let genesis_header = backend .blockchain() @@ -1077,7 +1095,7 @@ pub(crate) mod tests { links, gossip_engine, gossip_validator, - metrics: None, + metrics, network, on_demand_justifications, persisted_state,