From c01894b7d6c931d0a16b0df2023e92a6cccaac43 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 13 Feb 2024 14:26:09 +0100 Subject: [PATCH 01/53] Split subsystem-bench to lib and cli tool --- polkadot/node/subsystem-bench/Cargo.toml | 16 +++- .../src/{ => cli}/approval/helpers.rs | 2 +- .../{ => cli}/approval/message_generator.rs | 18 ++-- .../approval/mock_chain_selection.rs | 0 .../src/{ => cli}/approval/mod.rs | 59 +++++++------ .../src/{ => cli}/approval/test_message.rs | 2 +- .../availability/av_store_helpers.rs | 2 +- .../src/{ => cli}/availability/cli.rs | 0 .../src/{ => cli}/availability/mod.rs | 43 +++++----- .../src/{ => cli}/subsystem-bench.rs | 84 +++++++++++++++---- .../subsystem-bench/src/{ => cli}/valgrind.rs | 0 .../src/{core => lib}/configuration.rs | 58 +------------ .../src/{core => lib}/display.rs | 7 +- .../src/{core => lib}/environment.rs | 8 +- .../src/{core => lib}/keyring.rs | 0 .../src/{core/mod.rs => lib/lib.rs} | 14 ++-- .../src/{core => lib}/mock/av_store.rs | 2 +- .../src/{core => lib}/mock/chain_api.rs | 0 .../src/{core => lib}/mock/dummy.rs | 0 .../src/{core => lib}/mock/mod.rs | 4 +- .../src/{core => lib}/mock/network_bridge.rs | 2 +- .../src/{core => lib}/mock/runtime_api.rs | 2 +- .../src/{core => lib}/network.rs | 2 +- 23 files changed, 165 insertions(+), 160 deletions(-) rename polkadot/node/subsystem-bench/src/{ => cli}/approval/helpers.rs (99%) rename polkadot/node/subsystem-bench/src/{ => cli}/approval/message_generator.rs (97%) rename polkadot/node/subsystem-bench/src/{ => cli}/approval/mock_chain_selection.rs (100%) rename polkadot/node/subsystem-bench/src/{ => cli}/approval/mod.rs (97%) rename polkadot/node/subsystem-bench/src/{ => cli}/approval/test_message.rs (99%) rename polkadot/node/subsystem-bench/src/{ => cli}/availability/av_store_helpers.rs (94%) rename polkadot/node/subsystem-bench/src/{ => cli}/availability/cli.rs (100%) rename polkadot/node/subsystem-bench/src/{ => cli}/availability/mod.rs (96%) rename polkadot/node/subsystem-bench/src/{ => cli}/subsystem-bench.rs (75%) rename polkadot/node/subsystem-bench/src/{ => cli}/valgrind.rs (100%) rename polkadot/node/subsystem-bench/src/{core => lib}/configuration.rs (77%) rename polkadot/node/subsystem-bench/src/{core => lib}/display.rs (97%) rename polkadot/node/subsystem-bench/src/{core => lib}/environment.rs (99%) rename polkadot/node/subsystem-bench/src/{core => lib}/keyring.rs (100%) rename polkadot/node/subsystem-bench/src/{core/mod.rs => lib/lib.rs} (80%) rename polkadot/node/subsystem-bench/src/{core => lib}/mock/av_store.rs (99%) rename polkadot/node/subsystem-bench/src/{core => lib}/mock/chain_api.rs (100%) rename polkadot/node/subsystem-bench/src/{core => lib}/mock/dummy.rs (100%) rename polkadot/node/subsystem-bench/src/{core => lib}/mock/mod.rs (97%) rename polkadot/node/subsystem-bench/src/{core => lib}/mock/network_bridge.rs (99%) rename polkadot/node/subsystem-bench/src/{core => lib}/mock/runtime_api.rs (99%) rename polkadot/node/subsystem-bench/src/{core => lib}/network.rs (99%) diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index bc038abd6484..433424dfab41 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -8,9 +8,13 @@ license.workspace = true readme = "README.md" publish = false +[lib] +name = "polkadot_subsystem_bench" +path = "src/lib/lib.rs" + [[bin]] name = "subsystem-bench" -path = "src/subsystem-bench.rs" +path = "src/cli/subsystem-bench.rs" # Prevent rustdoc error. Already documented from top-level Cargo.toml. doc = false @@ -22,7 +26,9 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-availability-recovery = { path = "../network/availability-recovery", features = ["subsystem-benchmarks"] } +polkadot-availability-recovery = { path = "../network/availability-recovery", features = [ + "subsystem-benchmarks", +] } polkadot-availability-distribution = { path = "../network/availability-distribution" } polkadot-node-core-av-store = { path = "../core/av-store" } polkadot-node-core-chain-api = { path = "../core/chain-api" } @@ -75,10 +81,12 @@ sp-runtime = { path = "../../../substrate/primitives/runtime", default-features sp-timestamp = { path = "../../../substrate/primitives/timestamp" } schnorrkel = { version = "0.9.1", default-features = false } -rand_core = "0.6.2" # should match schnorrkel +rand_core = "0.6.2" # should match schnorrkel rand_chacha = { version = "0.3.1" } paste = "1.0.14" -orchestra = { version = "0.3.5", default-features = false, features = ["futures_channel"] } +orchestra = { version = "0.3.5", default-features = false, features = [ + "futures_channel", +] } pyroscope = "0.5.7" pyroscope_pprofrs = "0.2.7" diff --git a/polkadot/node/subsystem-bench/src/approval/helpers.rs b/polkadot/node/subsystem-bench/src/cli/approval/helpers.rs similarity index 99% rename from polkadot/node/subsystem-bench/src/approval/helpers.rs rename to polkadot/node/subsystem-bench/src/cli/approval/helpers.rs index 623d91848f53..653f2a077e82 100644 --- a/polkadot/node/subsystem-bench/src/approval/helpers.rs +++ b/polkadot/node/subsystem-bench/src/cli/approval/helpers.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::core::configuration::TestAuthorities; use itertools::Itertools; use polkadot_node_core_approval_voting::time::{Clock, SystemClock, Tick}; use polkadot_node_network_protocol::{ @@ -30,6 +29,7 @@ use polkadot_primitives::{ Id as ParaId, Slot, ValidatorIndex, }; use polkadot_primitives_test_helpers::dummy_candidate_receipt_bad_sig; +use polkadot_subsystem_bench::configuration::TestAuthorities; use rand::{seq::SliceRandom, SeedableRng}; use rand_chacha::ChaCha20Rng; use sc_network::PeerId; diff --git a/polkadot/node/subsystem-bench/src/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/cli/approval/message_generator.rs similarity index 97% rename from polkadot/node/subsystem-bench/src/approval/message_generator.rs rename to polkadot/node/subsystem-bench/src/cli/approval/message_generator.rs index a71034013247..186bb4e69347 100644 --- a/polkadot/node/subsystem-bench/src/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/cli/approval/message_generator.rs @@ -21,12 +21,7 @@ use crate::{ ApprovalTestState, BlockTestData, GeneratedState, BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET, SLOT_DURATION_MILLIS, }, - core::{ - configuration::{TestAuthorities, TestConfiguration}, - mock::runtime_api::session_info_for_peers, - NODE_UNDER_TEST, - }, - ApprovalsOptions, TestObjective, + ApprovalsOptions, }; use futures::SinkExt; use itertools::Itertools; @@ -47,6 +42,11 @@ use polkadot_primitives::{ vstaging::ApprovalVoteMultipleCandidates, CandidateEvent, CandidateHash, CandidateIndex, CoreIndex, Hash, SessionInfo, Slot, ValidatorId, ValidatorIndex, ASSIGNMENT_KEY_TYPE_ID, }; +use polkadot_subsystem_bench::{ + configuration::{TestAuthorities, TestConfiguration}, + mock::runtime_api::session_info_for_peers, + NODE_UNDER_TEST, +}; use rand::{seq::SliceRandom, RngCore, SeedableRng}; use rand_chacha::ChaCha20Rng; use rand_distr::{Distribution, Normal}; @@ -132,11 +132,7 @@ impl PeerMessagesGenerator { options: &ApprovalsOptions, ) -> String { let mut fingerprint = options.fingerprint(); - let mut exclude_objective = configuration.clone(); - // The objective contains the full content of `ApprovalOptions`, we don't want to put all of - // that in fingerprint, so execlute it because we add it manually see above. - exclude_objective.objective = TestObjective::Unimplemented; - let configuration_bytes = bincode::serialize(&exclude_objective).unwrap(); + let configuration_bytes = bincode::serialize(&configuration).unwrap(); fingerprint.extend(configuration_bytes); let mut sha1 = sha1::Sha1::new(); sha1.update(fingerprint); diff --git a/polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs b/polkadot/node/subsystem-bench/src/cli/approval/mock_chain_selection.rs similarity index 100% rename from polkadot/node/subsystem-bench/src/approval/mock_chain_selection.rs rename to polkadot/node/subsystem-bench/src/cli/approval/mock_chain_selection.rs diff --git a/polkadot/node/subsystem-bench/src/approval/mod.rs b/polkadot/node/subsystem-bench/src/cli/approval/mod.rs similarity index 97% rename from polkadot/node/subsystem-bench/src/approval/mod.rs rename to polkadot/node/subsystem-bench/src/cli/approval/mod.rs index f07912de1887..a0da4760845c 100644 --- a/polkadot/node/subsystem-bench/src/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/cli/approval/mod.rs @@ -14,35 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::{ - approval::{ - helpers::{ - generate_babe_epoch, generate_new_session_topology, generate_peer_view_change_for, - make_header, PastSystemClock, - }, - message_generator::PeerMessagesGenerator, - mock_chain_selection::MockChainSelection, - test_message::{MessagesBundle, TestMessageInfo}, - }, - core::{ - configuration::TestAuthorities, - environment::{ - BenchmarkUsage, TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT, - }, - mock::{ - chain_api::{ChainApiState, MockChainApi}, - dummy_builder, - network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx}, - runtime_api::MockRuntimeApi, - AlwaysSupportsParachains, TestSyncOracle, - }, - network::{ - new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface, - NetworkInterfaceReceiver, - }, - NODE_UNDER_TEST, +use crate::approval::{ + helpers::{ + generate_babe_epoch, generate_new_session_topology, generate_peer_view_change_for, + make_header, PastSystemClock, }, - TestConfiguration, + message_generator::PeerMessagesGenerator, + mock_chain_selection::MockChainSelection, + test_message::{MessagesBundle, TestMessageInfo}, }; use colored::Colorize; use futures::channel::oneshot; @@ -66,6 +45,24 @@ use polkadot_primitives::{ BlockNumber, CandidateEvent, CandidateIndex, CandidateReceipt, Hash, Header, Slot, ValidatorIndex, }; +use polkadot_subsystem_bench::{ + configuration::{TestAuthorities, TestConfiguration}, + dummy_builder, + environment::{ + BenchmarkUsage, TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT, + }, + mock::{ + chain_api::{ChainApiState, MockChainApi}, + network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx}, + runtime_api::MockRuntimeApi, + AlwaysSupportsParachains, TestSyncOracle, + }, + network::{ + new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface, + NetworkInterfaceReceiver, + }, + NODE_UNDER_TEST, +}; use prometheus::Registry; use sc_keystore::LocalKeystore; use sc_service::SpawnTaskHandle; @@ -472,11 +469,11 @@ impl ApprovalTestState { impl HandleNetworkMessage for ApprovalTestState { fn handle( &self, - _message: crate::core::network::NetworkMessage, + _message: polkadot_subsystem_bench::network::NetworkMessage, _node_sender: &mut futures::channel::mpsc::UnboundedSender< - crate::core::network::NetworkMessage, + polkadot_subsystem_bench::network::NetworkMessage, >, - ) -> Option { + ) -> Option { self.total_sent_messages_from_node .as_ref() .fetch_add(1, std::sync::atomic::Ordering::SeqCst); diff --git a/polkadot/node/subsystem-bench/src/approval/test_message.rs b/polkadot/node/subsystem-bench/src/cli/approval/test_message.rs similarity index 99% rename from polkadot/node/subsystem-bench/src/approval/test_message.rs rename to polkadot/node/subsystem-bench/src/cli/approval/test_message.rs index 8aaabc3426c8..887443fa5e47 100644 --- a/polkadot/node/subsystem-bench/src/approval/test_message.rs +++ b/polkadot/node/subsystem-bench/src/cli/approval/test_message.rs @@ -16,13 +16,13 @@ use crate::{ approval::{BlockTestData, CandidateTestData}, - core::configuration::TestAuthorities, ApprovalsOptions, }; use itertools::Itertools; use parity_scale_codec::{Decode, Encode}; use polkadot_node_network_protocol::v3 as protocol_v3; use polkadot_primitives::{CandidateIndex, Hash, ValidatorIndex}; +use polkadot_subsystem_bench::configuration::TestAuthorities; use sc_network::PeerId; use std::collections::{HashMap, HashSet}; diff --git a/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs b/polkadot/node/subsystem-bench/src/cli/availability/av_store_helpers.rs similarity index 94% rename from polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs rename to polkadot/node/subsystem-bench/src/cli/availability/av_store_helpers.rs index 261dbd0376c7..9f3b7fb34d85 100644 --- a/polkadot/node/subsystem-bench/src/availability/av_store_helpers.rs +++ b/polkadot/node/subsystem-bench/src/cli/availability/av_store_helpers.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::core::{environment::TestEnvironmentDependencies, mock::TestSyncOracle}; use polkadot_node_core_av_store::{AvailabilityStoreSubsystem, Config}; use polkadot_node_metrics::metrics::Metrics; use polkadot_node_subsystem_util::database::Database; +use polkadot_subsystem_bench::{environment::TestEnvironmentDependencies, mock::TestSyncOracle}; use std::sync::Arc; mod columns { diff --git a/polkadot/node/subsystem-bench/src/availability/cli.rs b/polkadot/node/subsystem-bench/src/cli/availability/cli.rs similarity index 100% rename from polkadot/node/subsystem-bench/src/availability/cli.rs rename to polkadot/node/subsystem-bench/src/cli/availability/cli.rs diff --git a/polkadot/node/subsystem-bench/src/availability/mod.rs b/polkadot/node/subsystem-bench/src/cli/availability/mod.rs similarity index 96% rename from polkadot/node/subsystem-bench/src/availability/mod.rs rename to polkadot/node/subsystem-bench/src/cli/availability/mod.rs index ad9a17ff8f47..b30d4366ae19 100644 --- a/polkadot/node/subsystem-bench/src/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/cli/availability/mod.rs @@ -14,24 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::{ - core::{ - configuration::TestConfiguration, - environment::{BenchmarkUsage, TestEnvironmentDependencies}, - mock::{ - av_store, - av_store::MockAvailabilityStore, - chain_api::{ChainApiState, MockChainApi}, - dummy_builder, - network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx}, - runtime_api, - runtime_api::MockRuntimeApi, - AlwaysSupportsParachains, - }, - network::new_network, - }, - TestEnvironment, TestObjective, GENESIS_HASH, -}; +use crate::{TestEnvironment, TestObjective, GENESIS_HASH}; use av_store::NetworkAvailabilityState; use av_store_helpers::new_av_store; use bitvec::bitvec; @@ -68,6 +51,19 @@ use polkadot_primitives::{ Header, PersistedValidationData, Signed, SigningContext, ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; +use polkadot_subsystem_bench::{ + configuration::TestConfiguration, + dummy_builder, + environment::{BenchmarkUsage, TestEnvironmentDependencies}, + mock::{ + av_store::{self, MockAvailabilityStore}, + chain_api::{ChainApiState, MockChainApi}, + network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx}, + runtime_api::{self, MockRuntimeApi}, + AlwaysSupportsParachains, + }, + network::new_network, +}; use sc_network::{ request_responses::{IncomingRequest as RawIncomingRequest, ProtocolConfig}, PeerId, @@ -216,7 +212,7 @@ fn prepare_test_inner( let network_bridge_rx = network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); - let (overseer, overseer_handle) = match &state.config().objective { + let (overseer, overseer_handle) = match &state.objective() { TestObjective::DataAvailabilityRead(options) => { let use_fast_path = options.fetch_from_backers; @@ -304,6 +300,8 @@ fn prepare_test_inner( #[derive(Clone)] pub struct TestState { + // Test Objective + objective: TestObjective, // Full test configuration config: TestConfiguration, // A cycle iterator on all PoV sizes used in the test. @@ -326,8 +324,8 @@ pub struct TestState { } impl TestState { - fn config(&self) -> &TestConfiguration { - &self.config + fn objective(&self) -> &TestObjective { + &self.objective } pub fn next_candidate(&mut self) -> Option { @@ -367,7 +365,7 @@ impl TestState { .cycle(); } - pub fn new(config: &TestConfiguration) -> Self { + pub fn new(objective: TestObjective, config: &TestConfiguration) -> Self { let config = config.clone(); let mut chunks = Vec::new(); @@ -420,6 +418,7 @@ impl TestState { candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), backed_candidates: Vec::new(), + objective, config, }; diff --git a/polkadot/node/subsystem-bench/src/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs similarity index 75% rename from polkadot/node/subsystem-bench/src/subsystem-bench.rs rename to polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 0803f175474e..d91e876182d8 100644 --- a/polkadot/node/subsystem-bench/src/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -26,22 +26,23 @@ use clap::Parser; use clap_num::number_range; use color_eyre::eyre; use colored::Colorize; -use core::{ +use polkadot_subsystem_bench::{ configuration::TestConfiguration, display::display_configuration, environment::{TestEnvironment, GENESIS_HASH}, }; use pyroscope::PyroscopeAgent; use pyroscope_pprofrs::{pprof_backend, PprofConfig}; +use rand::thread_rng; +use rand_distr::{Distribution, Uniform}; use serde::{Deserialize, Serialize}; use std::path::Path; mod approval; mod availability; -mod core; mod valgrind; -const LOG_TARGET: &str = "subsystem-bench"; +const LOG_TARGET: &str = "subsystem-bench::cli"; fn le_100(s: &str) -> Result { number_range(s, 0, 100) @@ -79,6 +80,59 @@ impl std::fmt::Display for TestObjective { } } +/// The test input parameters +#[derive(Clone, Debug, Serialize, Deserialize)] +struct CliTestConfiguration { + /// Test Objective + pub objective: TestObjective, + /// Test Configuration + #[serde(flatten)] + pub test_config: TestConfiguration, +} + +#[derive(Serialize, Deserialize)] +pub struct TestSequence { + #[serde(rename(serialize = "TestConfiguration", deserialize = "TestConfiguration"))] + test_configurations: Vec, +} + +fn random_uniform_sample + From>(min_value: T, max_value: T) -> T { + Uniform::from(min_value.into()..=max_value.into()) + .sample(&mut thread_rng()) + .into() +} + +fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { + random_uniform_sample(min_pov_size, max_pov_size) +} + +fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec { + (0..count).map(|_| random_pov_size(min_kib * 1024, max_kib * 1024)).collect() +} + +impl TestSequence { + pub fn into_vec(self) -> Vec { + self.test_configurations + .into_iter() + .map(|mut v| { + v.test_config.pov_sizes = generate_pov_sizes( + v.test_config.n_cores, + v.test_config.min_pov_size, + v.test_config.max_pov_size, + ); + v + }) + .collect() + } +} + +impl TestSequence { + pub fn new_from_file(path: &Path) -> std::io::Result { + let string = String::from_utf8(std::fs::read(path)?).expect("File is valid UTF8"); + Ok(serde_yaml::from_str(&string).expect("File is valid test sequence YA")) + } +} + #[derive(Debug, Parser)] #[allow(missing_docs)] struct BenchCli { @@ -148,19 +202,21 @@ impl BenchCli { None }; - let test_sequence = core::configuration::TestSequence::new_from_file(Path::new(&self.path)) + let test_sequence = TestSequence::new_from_file(Path::new(&self.path)) .expect("File exists") .into_vec(); let num_steps = test_sequence.len(); gum::info!("{}", format!("Sequence contains {} step(s)", num_steps).bright_purple()); - for (index, test_config) in test_sequence.into_iter().enumerate() { - let benchmark_name = format!("{} #{} {}", &self.path, index + 1, test_config.objective); + for (index, CliTestConfiguration { objective, test_config }) in + test_sequence.into_iter().enumerate() + { + let benchmark_name = format!("{} #{} {}", &self.path, index + 1, objective); gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); display_configuration(&test_config); - let usage = match test_config.objective { + let usage = match objective { TestObjective::DataAvailabilityRead(ref _opts) => { - let mut state = TestState::new(&test_config); + let mut state = TestState::new(objective, &test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); env.runtime().block_on(availability::benchmark_availability_read( &benchmark_name, @@ -168,13 +224,8 @@ impl BenchCli { state, )) }, - TestObjective::ApprovalVoting(ref options) => { - let (mut env, state) = - approval::prepare_test(test_config.clone(), options.clone()); - env.runtime().block_on(bench_approvals(&benchmark_name, &mut env, state)) - }, TestObjective::DataAvailabilityWrite => { - let mut state = TestState::new(&test_config); + let mut state = TestState::new(objective, &test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); env.runtime().block_on(availability::benchmark_availability_write( &benchmark_name, @@ -182,6 +233,11 @@ impl BenchCli { state, )) }, + TestObjective::ApprovalVoting(ref options) => { + let (mut env, state) = + approval::prepare_test(test_config.clone(), options.clone()); + env.runtime().block_on(bench_approvals(&benchmark_name, &mut env, state)) + }, TestObjective::Unimplemented => todo!(), }; diff --git a/polkadot/node/subsystem-bench/src/valgrind.rs b/polkadot/node/subsystem-bench/src/cli/valgrind.rs similarity index 100% rename from polkadot/node/subsystem-bench/src/valgrind.rs rename to polkadot/node/subsystem-bench/src/cli/valgrind.rs diff --git a/polkadot/node/subsystem-bench/src/core/configuration.rs b/polkadot/node/subsystem-bench/src/lib/configuration.rs similarity index 77% rename from polkadot/node/subsystem-bench/src/core/configuration.rs rename to polkadot/node/subsystem-bench/src/lib/configuration.rs index 00be2a86b173..f1085082714d 100644 --- a/polkadot/node/subsystem-bench/src/core/configuration.rs +++ b/polkadot/node/subsystem-bench/src/lib/configuration.rs @@ -16,25 +16,15 @@ //! Test configuration definition and helpers. -use crate::{core::keyring::Keyring, TestObjective}; +use crate::keyring::Keyring; use itertools::Itertools; use polkadot_primitives::{AssignmentId, AuthorityDiscoveryId, ValidatorId}; use rand::thread_rng; -use rand_distr::{Distribution, Normal, Uniform}; +use rand_distr::{Distribution, Normal}; use sc_network::PeerId; use serde::{Deserialize, Serialize}; use sp_consensus_babe::AuthorityId; -use std::{collections::HashMap, path::Path}; - -pub fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { - random_uniform_sample(min_pov_size, max_pov_size) -} - -fn random_uniform_sample + From>(min_value: T, max_value: T) -> T { - Uniform::from(min_value.into()..=max_value.into()) - .sample(&mut thread_rng()) - .into() -} +use std::collections::HashMap; /// Peer networking latency configuration. #[derive(Clone, Debug, Default, Serialize, Deserialize)] @@ -87,8 +77,6 @@ fn default_no_show_slots() -> usize { /// The test input parameters #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TestConfiguration { - /// The test objective - pub objective: TestObjective, /// Number of validators pub n_validators: usize, /// Number of cores @@ -115,7 +103,7 @@ pub struct TestConfiguration { pub max_pov_size: usize, /// Randomly sampled pov_sizes #[serde(skip)] - pov_sizes: Vec, + pub pov_sizes: Vec, /// The amount of bandiwdth remote validators have. #[serde(default = "default_bandwidth")] pub peer_bandwidth: usize, @@ -133,36 +121,6 @@ pub struct TestConfiguration { pub num_blocks: usize, } -fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec { - (0..count).map(|_| random_pov_size(min_kib * 1024, max_kib * 1024)).collect() -} - -#[derive(Serialize, Deserialize)] -pub struct TestSequence { - #[serde(rename(serialize = "TestConfiguration", deserialize = "TestConfiguration"))] - test_configurations: Vec, -} - -impl TestSequence { - pub fn into_vec(self) -> Vec { - self.test_configurations - .into_iter() - .map(|mut config| { - config.pov_sizes = - generate_pov_sizes(config.n_cores, config.min_pov_size, config.max_pov_size); - config - }) - .collect() - } -} - -impl TestSequence { - pub fn new_from_file(path: &Path) -> std::io::Result { - let string = String::from_utf8(std::fs::read(path)?).expect("File is valid UTF8"); - Ok(serde_yaml::from_str(&string).expect("File is valid test sequence YA")) - } -} - /// Helper struct for authority related state. #[derive(Clone)] pub struct TestAuthorities { @@ -177,14 +135,6 @@ pub struct TestAuthorities { } impl TestConfiguration { - #[allow(unused)] - pub fn write_to_disk(&self) { - // Serialize a slice of configurations - let yaml = serde_yaml::to_string(&TestSequence { test_configurations: vec![self.clone()] }) - .unwrap(); - std::fs::write("last_test.yaml", yaml).unwrap(); - } - pub fn pov_sizes(&self) -> &[usize] { &self.pov_sizes } diff --git a/polkadot/node/subsystem-bench/src/core/display.rs b/polkadot/node/subsystem-bench/src/lib/display.rs similarity index 97% rename from polkadot/node/subsystem-bench/src/core/display.rs rename to polkadot/node/subsystem-bench/src/lib/display.rs index 13a349382e2f..0214a51d051e 100644 --- a/polkadot/node/subsystem-bench/src/core/display.rs +++ b/polkadot/node/subsystem-bench/src/lib/display.rs @@ -19,7 +19,7 @@ //! //! Currently histogram buckets are skipped. -use crate::{TestConfiguration, LOG_TARGET}; +use crate::configuration::TestConfiguration; use colored::Colorize; use prometheus::{ proto::{MetricFamily, MetricType}, @@ -27,6 +27,8 @@ use prometheus::{ }; use std::fmt::Display; +const LOG_TARGET: &str = "subsystem-bench::display"; + #[derive(Default, Debug)] pub struct MetricCollection(Vec); @@ -186,8 +188,7 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { pub fn display_configuration(test_config: &TestConfiguration) { gum::info!( - "[{}] {}, {}, {}, {}, {}", - format!("objective = {:?}", test_config.objective).green(), + "{}, {}, {}, {}, {}", format!("n_validators = {}", test_config.n_validators).blue(), format!("n_cores = {}", test_config.n_cores).blue(), format!("pov_size = {} - {}", test_config.min_pov_size, test_config.max_pov_size) diff --git a/polkadot/node/subsystem-bench/src/core/environment.rs b/polkadot/node/subsystem-bench/src/lib/environment.rs similarity index 99% rename from polkadot/node/subsystem-bench/src/core/environment.rs rename to polkadot/node/subsystem-bench/src/lib/environment.rs index ca4c41cf45f9..3341e31f54e3 100644 --- a/polkadot/node/subsystem-bench/src/core/environment.rs +++ b/polkadot/node/subsystem-bench/src/lib/environment.rs @@ -17,11 +17,9 @@ //! Test environment implementation use crate::{ - core::{ - configuration::TestAuthorities, mock::AlwaysSupportsParachains, - network::NetworkEmulatorHandle, - }, - TestConfiguration, + configuration::{TestAuthorities, TestConfiguration}, + mock::AlwaysSupportsParachains, + network::NetworkEmulatorHandle, }; use colored::Colorize; use core::time::Duration; diff --git a/polkadot/node/subsystem-bench/src/core/keyring.rs b/polkadot/node/subsystem-bench/src/lib/keyring.rs similarity index 100% rename from polkadot/node/subsystem-bench/src/core/keyring.rs rename to polkadot/node/subsystem-bench/src/lib/keyring.rs diff --git a/polkadot/node/subsystem-bench/src/core/mod.rs b/polkadot/node/subsystem-bench/src/lib/lib.rs similarity index 80% rename from polkadot/node/subsystem-bench/src/core/mod.rs rename to polkadot/node/subsystem-bench/src/lib/lib.rs index 764184c5b377..f8a0f6354690 100644 --- a/polkadot/node/subsystem-bench/src/core/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/lib.rs @@ -15,11 +15,11 @@ // along with Polkadot. If not, see . // The validator index that represent the node that is under test. -pub(crate) const NODE_UNDER_TEST: u32 = 0; +pub const NODE_UNDER_TEST: u32 = 0; -pub(crate) mod configuration; -pub(crate) mod display; -pub(crate) mod environment; -pub(crate) mod keyring; -pub(crate) mod mock; -pub(crate) mod network; +pub mod configuration; +pub mod display; +pub mod environment; +pub mod keyring; +pub mod mock; +pub mod network; diff --git a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs b/polkadot/node/subsystem-bench/src/lib/mock/av_store.rs similarity index 99% rename from polkadot/node/subsystem-bench/src/core/mock/av_store.rs rename to polkadot/node/subsystem-bench/src/lib/mock/av_store.rs index 0a7725c91e04..41c4fe2cbadc 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/av_store.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/av_store.rs @@ -16,7 +16,7 @@ //! A generic av store subsystem mockup suitable to be used in benchmarks. -use crate::core::network::{HandleNetworkMessage, NetworkMessage}; +use crate::network::{HandleNetworkMessage, NetworkMessage}; use futures::{channel::oneshot, FutureExt}; use parity_scale_codec::Encode; use polkadot_node_network_protocol::request_response::{ diff --git a/polkadot/node/subsystem-bench/src/core/mock/chain_api.rs b/polkadot/node/subsystem-bench/src/lib/mock/chain_api.rs similarity index 100% rename from polkadot/node/subsystem-bench/src/core/mock/chain_api.rs rename to polkadot/node/subsystem-bench/src/lib/mock/chain_api.rs diff --git a/polkadot/node/subsystem-bench/src/core/mock/dummy.rs b/polkadot/node/subsystem-bench/src/lib/mock/dummy.rs similarity index 100% rename from polkadot/node/subsystem-bench/src/core/mock/dummy.rs rename to polkadot/node/subsystem-bench/src/lib/mock/dummy.rs diff --git a/polkadot/node/subsystem-bench/src/core/mock/mod.rs b/polkadot/node/subsystem-bench/src/lib/mock/mod.rs similarity index 97% rename from polkadot/node/subsystem-bench/src/core/mock/mod.rs rename to polkadot/node/subsystem-bench/src/lib/mock/mod.rs index 46fdeb196c01..6dda9a47d398 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/mod.rs @@ -34,9 +34,10 @@ impl HeadSupportsParachains for AlwaysSupportsParachains { } // An orchestra with dummy subsystems +#[macro_export] macro_rules! dummy_builder { ($spawn_task_handle: ident, $metrics: ident) => {{ - use $crate::core::mock::dummy::*; + use $crate::mock::dummy::*; // Initialize a mock overseer. // All subsystem except approval_voting and approval_distribution are mock subsystems. @@ -72,7 +73,6 @@ macro_rules! dummy_builder { .spawner(SpawnGlue($spawn_task_handle)) }}; } -pub(crate) use dummy_builder; #[derive(Clone)] pub struct TestSyncOracle {} diff --git a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs b/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs similarity index 99% rename from polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs rename to polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs index 4682c7ec79ae..d598f6447d3d 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/network_bridge.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/network_bridge.rs @@ -17,7 +17,7 @@ //! Mocked `network-bridge` subsystems that uses a `NetworkInterface` to access //! the emulated network. -use crate::core::{ +use crate::{ configuration::TestAuthorities, network::{NetworkEmulatorHandle, NetworkInterfaceReceiver, NetworkMessage, RequestExt}, }; diff --git a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs similarity index 99% rename from polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs rename to polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs index 0dd76efcbaf0..53faf562f03c 100644 --- a/polkadot/node/subsystem-bench/src/core/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs @@ -16,7 +16,7 @@ //! A generic runtime api subsystem mockup suitable to be used in benchmarks. -use crate::core::configuration::{TestAuthorities, TestConfiguration}; +use crate::configuration::{TestAuthorities, TestConfiguration}; use bitvec::prelude::BitVec; use futures::FutureExt; use itertools::Itertools; diff --git a/polkadot/node/subsystem-bench/src/core/network.rs b/polkadot/node/subsystem-bench/src/lib/network.rs similarity index 99% rename from polkadot/node/subsystem-bench/src/core/network.rs rename to polkadot/node/subsystem-bench/src/lib/network.rs index e9124726d7c0..1e09441792d5 100644 --- a/polkadot/node/subsystem-bench/src/core/network.rs +++ b/polkadot/node/subsystem-bench/src/lib/network.rs @@ -33,7 +33,7 @@ // | // Subsystems under test -use crate::core::{ +use crate::{ configuration::{random_latency, TestAuthorities, TestConfiguration}, environment::TestEnvironmentDependencies, NODE_UNDER_TEST, From f71c502444701a8ab384cfd9512a7901ef67a5b4 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 13 Feb 2024 14:36:01 +0100 Subject: [PATCH 02/53] Remove unused values from BenchCli --- .../src/cli/availability/cli.rs | 9 ---- .../src/cli/subsystem-bench.rs | 54 ++----------------- 2 files changed, 4 insertions(+), 59 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/cli/availability/cli.rs b/polkadot/node/subsystem-bench/src/cli/availability/cli.rs index 65df8c1552aa..06adbe5f1a79 100644 --- a/polkadot/node/subsystem-bench/src/cli/availability/cli.rs +++ b/polkadot/node/subsystem-bench/src/cli/availability/cli.rs @@ -16,15 +16,6 @@ use serde::{Deserialize, Serialize}; -#[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq)] -#[value(rename_all = "kebab-case")] -#[non_exhaustive] -pub enum NetworkEmulation { - Ideal, - Healthy, - Degraded, -} - #[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)] #[clap(rename_all = "kebab-case")] #[allow(missing_docs)] diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index d91e876182d8..b8432ddf3902 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -18,12 +18,8 @@ //! CI regression testing. use approval::{bench_approvals, ApprovalsOptions}; -use availability::{ - cli::{DataAvailabilityReadOptions, NetworkEmulation}, - prepare_test, TestState, -}; +use availability::{cli::DataAvailabilityReadOptions, prepare_test, TestState}; use clap::Parser; -use clap_num::number_range; use color_eyre::eyre; use colored::Colorize; use polkadot_subsystem_bench::{ @@ -44,14 +40,6 @@ mod valgrind; const LOG_TARGET: &str = "subsystem-bench::cli"; -fn le_100(s: &str) -> Result { - number_range(s, 0, 100) -} - -fn le_5000(s: &str) -> Result { - number_range(s, 0, 5000) -} - /// Supported test objectives #[derive(Debug, Clone, Parser, Serialize, Deserialize)] #[command(rename_all = "kebab-case")] @@ -111,7 +99,7 @@ fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec Vec { + fn into_vec(self) -> Vec { self.test_configurations .into_iter() .map(|mut v| { @@ -127,7 +115,7 @@ impl TestSequence { } impl TestSequence { - pub fn new_from_file(path: &Path) -> std::io::Result { + fn new_from_file(path: &Path) -> std::io::Result { let string = String::from_utf8(std::fs::read(path)?).expect("File is valid UTF8"); Ok(serde_yaml::from_str(&string).expect("File is valid test sequence YA")) } @@ -136,30 +124,6 @@ impl TestSequence { #[derive(Debug, Parser)] #[allow(missing_docs)] struct BenchCli { - #[arg(long, value_enum, ignore_case = true, default_value_t = NetworkEmulation::Ideal)] - /// The type of network to be emulated - pub network: NetworkEmulation, - - #[clap(short, long)] - /// The bandwidth of emulated remote peers in KiB - pub peer_bandwidth: Option, - - #[clap(short, long)] - /// The bandwidth of our node in KiB - pub bandwidth: Option, - - #[clap(long, value_parser=le_100)] - /// Emulated peer connection ratio [0-100]. - pub connectivity: Option, - - #[clap(long, value_parser=le_5000)] - /// Mean remote peer latency in milliseconds [0-5000]. - pub peer_mean_latency: Option, - - #[clap(long, value_parser=le_5000)] - /// Remote peer latency standard deviation - pub peer_latency_std_dev: Option, - #[clap(long, default_value_t = false)] /// Enable CPU Profiling with Pyroscope pub profile: bool, @@ -176,10 +140,6 @@ struct BenchCli { /// Enable Cache Misses Profiling with Valgrind. Linux only, Valgrind must be in the PATH pub cache_misses: bool, - #[clap(long, default_value_t = false)] - /// Shows the output in YAML format - pub yaml_output: bool, - #[arg(required = true)] /// Path to the test sequence configuration file pub path: String, @@ -240,13 +200,7 @@ impl BenchCli { }, TestObjective::Unimplemented => todo!(), }; - - let output = if self.yaml_output { - serde_yaml::to_string(&vec![usage])? - } else { - usage.to_string() - }; - println!("{}", output); + println!("{}", usage); } if let Some(agent_running) = agent_running { From 733f0ebc96d645d433c85dd81e8b467106b3845f Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 13 Feb 2024 14:43:25 +0100 Subject: [PATCH 03/53] Move usage display to tests --- .../subsystem-bench/src/cli/approval/mod.rs | 13 ++++++------ .../src/cli/availability/mod.rs | 20 +++++++++++-------- .../src/cli/subsystem-bench.rs | 13 +++++------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/cli/approval/mod.rs b/polkadot/node/subsystem-bench/src/cli/approval/mod.rs index a0da4760845c..a9383db0ef66 100644 --- a/polkadot/node/subsystem-bench/src/cli/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/cli/approval/mod.rs @@ -48,9 +48,7 @@ use polkadot_primitives::{ use polkadot_subsystem_bench::{ configuration::{TestAuthorities, TestConfiguration}, dummy_builder, - environment::{ - BenchmarkUsage, TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT, - }, + environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, mock::{ chain_api::{ChainApiState, MockChainApi}, network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx}, @@ -884,7 +882,7 @@ pub async fn bench_approvals( benchmark_name: &str, env: &mut TestEnvironment, mut state: ApprovalTestState, -) -> BenchmarkUsage { +) { let producer_rx = state .start_message_production( env.network(), @@ -902,7 +900,7 @@ pub async fn bench_approvals_run( env: &mut TestEnvironment, state: ApprovalTestState, producer_rx: oneshot::Receiver<()>, -) -> BenchmarkUsage { +) { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); @@ -1063,5 +1061,8 @@ pub async fn bench_approvals_run( state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); - env.collect_resource_usage(benchmark_name, &["approval-distribution", "approval-voting"]) + println!( + "{}", + env.collect_resource_usage(benchmark_name, &["approval-distribution", "approval-voting"]) + ); } diff --git a/polkadot/node/subsystem-bench/src/cli/availability/mod.rs b/polkadot/node/subsystem-bench/src/cli/availability/mod.rs index b30d4366ae19..4d047f86c1d1 100644 --- a/polkadot/node/subsystem-bench/src/cli/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/cli/availability/mod.rs @@ -54,7 +54,7 @@ use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; use polkadot_subsystem_bench::{ configuration::TestConfiguration, dummy_builder, - environment::{BenchmarkUsage, TestEnvironmentDependencies}, + environment::TestEnvironmentDependencies, mock::{ av_store::{self, MockAvailabilityStore}, chain_api::{ChainApiState, MockChainApi}, @@ -435,7 +435,7 @@ pub async fn benchmark_availability_read( benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState, -) -> BenchmarkUsage { +) { let config = env.config().clone(); env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; @@ -496,14 +496,15 @@ pub async fn benchmark_availability_read( ); env.stop().await; - env.collect_resource_usage(benchmark_name, &["availability-recovery"]) + + println!("{}", env.collect_resource_usage(benchmark_name, &["availability-recovery"])); } pub async fn benchmark_availability_write( benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState, -) -> BenchmarkUsage { +) { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); @@ -657,10 +658,13 @@ pub async fn benchmark_availability_write( ); env.stop().await; - env.collect_resource_usage( - benchmark_name, - &["availability-distribution", "bitfield-distribution", "availability-store"], - ) + println!( + "{}", + env.collect_resource_usage( + benchmark_name, + &["availability-distribution", "bitfield-distribution", "availability-store"], + ) + ); } pub fn peer_bitfield_message_v2( diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index b8432ddf3902..16bce9e49ac4 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -12,10 +12,10 @@ // GNU General Public License for more details. // You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . +// along with Polkadot. If not, see . -//! A tool for running subsystem benchmark tests designed for development and -//! CI regression testing. +//! A tool for running subsystem benchmark tests +//! designed for development and CI regression testing. use approval::{bench_approvals, ApprovalsOptions}; use availability::{cli::DataAvailabilityReadOptions, prepare_test, TestState}; @@ -50,7 +50,6 @@ pub enum TestObjective { DataAvailabilityWrite, /// Benchmark the approval-voting and approval-distribution subsystems. ApprovalVoting(ApprovalsOptions), - Unimplemented, } impl std::fmt::Display for TestObjective { @@ -62,7 +61,7 @@ impl std::fmt::Display for TestObjective { Self::DataAvailabilityRead(_) => "DataAvailabilityRead", Self::DataAvailabilityWrite => "DataAvailabilityWrite", Self::ApprovalVoting(_) => "ApprovalVoting", - Self::Unimplemented => "Unimplemented", + // Self::Unimplemented => "Unimplemented", } ) } @@ -174,7 +173,7 @@ impl BenchCli { gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); display_configuration(&test_config); - let usage = match objective { + match objective { TestObjective::DataAvailabilityRead(ref _opts) => { let mut state = TestState::new(objective, &test_config); let (mut env, _protocol_config) = prepare_test(test_config, &mut state); @@ -198,9 +197,7 @@ impl BenchCli { approval::prepare_test(test_config.clone(), options.clone()); env.runtime().block_on(bench_approvals(&benchmark_name, &mut env, state)) }, - TestObjective::Unimplemented => todo!(), }; - println!("{}", usage); } if let Some(agent_running) = agent_running { From fdc13927e1be1cea756f25e24a1fa11caa2f07ad Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 13 Feb 2024 14:53:14 +0100 Subject: [PATCH 04/53] Update displaying of the configuration --- .../src/cli/subsystem-bench.rs | 3 ++- .../node/subsystem-bench/src/lib/display.rs | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 16bce9e49ac4..70581705b18f 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -166,12 +166,13 @@ impl BenchCli { .into_vec(); let num_steps = test_sequence.len(); gum::info!("{}", format!("Sequence contains {} step(s)", num_steps).bright_purple()); + for (index, CliTestConfiguration { objective, test_config }) in test_sequence.into_iter().enumerate() { let benchmark_name = format!("{} #{} {}", &self.path, index + 1, objective); gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); - display_configuration(&test_config); + gum::info!("[{}] {}", format!("objective = {:?}", objective).green(), test_config); match objective { TestObjective::DataAvailabilityRead(ref _opts) => { diff --git a/polkadot/node/subsystem-bench/src/lib/display.rs b/polkadot/node/subsystem-bench/src/lib/display.rs index 0214a51d051e..b153d54a7c36 100644 --- a/polkadot/node/subsystem-bench/src/lib/display.rs +++ b/polkadot/node/subsystem-bench/src/lib/display.rs @@ -87,6 +87,7 @@ impl Display for MetricCollection { Ok(()) } } + #[derive(Debug, Clone)] pub struct TestMetric { name: String, @@ -186,14 +187,16 @@ pub fn parse_metrics(registry: &Registry) -> MetricCollection { test_metrics.into() } -pub fn display_configuration(test_config: &TestConfiguration) { - gum::info!( - "{}, {}, {}, {}, {}", - format!("n_validators = {}", test_config.n_validators).blue(), - format!("n_cores = {}", test_config.n_cores).blue(), - format!("pov_size = {} - {}", test_config.min_pov_size, test_config.max_pov_size) - .bright_black(), - format!("connectivity = {}", test_config.connectivity).bright_black(), - format!("latency = {:?}", test_config.latency).bright_black(), - ); +impl Display for TestConfiguration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}, {}, {}, {}, {}", + format!("n_validators = {}", self.n_validators).blue(), + format!("n_cores = {}", self.n_cores).blue(), + format!("pov_size = {} - {}", self.min_pov_size, self.max_pov_size).bright_black(), + format!("connectivity = {}", self.connectivity).bright_black(), + format!("latency = {:?}", self.latency).bright_black(), + ) + } } From 815c023d0cad5a0dbfbaf664bc3caaa1ea73e492 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 13 Feb 2024 14:54:27 +0100 Subject: [PATCH 05/53] fixup: add log target --- polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 70581705b18f..88265f837463 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -172,7 +172,7 @@ impl BenchCli { { let benchmark_name = format!("{} #{} {}", &self.path, index + 1, objective); gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); - gum::info!("[{}] {}", format!("objective = {:?}", objective).green(), test_config); + gum::info!(target: LOG_TARGET, "[{}] {}", format!("objective = {:?}", objective).green(), test_config); match objective { TestObjective::DataAvailabilityRead(ref _opts) => { From 0a651261f5ac6ed1a13d165b219ede1766d782a0 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 13 Feb 2024 14:56:31 +0100 Subject: [PATCH 06/53] Remove toml fmt --- polkadot/node/subsystem-bench/Cargo.toml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 433424dfab41..d847472dcea1 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -26,9 +26,7 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-availability-recovery = { path = "../network/availability-recovery", features = [ - "subsystem-benchmarks", -] } +polkadot-availability-recovery = { path = "../network/availability-recovery", features = [ "subsystem-benchmarks" ] } polkadot-availability-distribution = { path = "../network/availability-distribution" } polkadot-node-core-av-store = { path = "../core/av-store" } polkadot-node-core-chain-api = { path = "../core/chain-api" } @@ -81,12 +79,10 @@ sp-runtime = { path = "../../../substrate/primitives/runtime", default-features sp-timestamp = { path = "../../../substrate/primitives/timestamp" } schnorrkel = { version = "0.9.1", default-features = false } -rand_core = "0.6.2" # should match schnorrkel +rand_core = "0.6.2" # should match schnorrkel rand_chacha = { version = "0.3.1" } paste = "1.0.14" -orchestra = { version = "0.3.5", default-features = false, features = [ - "futures_channel", -] } +orchestra = { version = "0.3.5", default-features = false, features = [ "futures_channel" ] } pyroscope = "0.5.7" pyroscope_pprofrs = "0.2.7" From 42963a946533685b8e7a2b51c73a25763affc5fc Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 13 Feb 2024 14:57:14 +0100 Subject: [PATCH 07/53] fixup: remove toml fmt --- polkadot/node/subsystem-bench/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index d847472dcea1..1ec557d77649 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -26,7 +26,7 @@ polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-primitives = { path = "../primitives" } polkadot-primitives = { path = "../../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } -polkadot-availability-recovery = { path = "../network/availability-recovery", features = [ "subsystem-benchmarks" ] } +polkadot-availability-recovery = { path = "../network/availability-recovery", features = ["subsystem-benchmarks"] } polkadot-availability-distribution = { path = "../network/availability-distribution" } polkadot-node-core-av-store = { path = "../core/av-store" } polkadot-node-core-chain-api = { path = "../core/chain-api" } @@ -82,7 +82,7 @@ schnorrkel = { version = "0.9.1", default-features = false } rand_core = "0.6.2" # should match schnorrkel rand_chacha = { version = "0.3.1" } paste = "1.0.14" -orchestra = { version = "0.3.5", default-features = false, features = [ "futures_channel" ] } +orchestra = { version = "0.3.5", default-features = false, features = ["futures_channel"] } pyroscope = "0.5.7" pyroscope_pprofrs = "0.2.7" From 98224fd2807875ce5e17f9364a0e1624675636ff Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 13 Feb 2024 15:45:36 +0100 Subject: [PATCH 08/53] Move approval to lib --- .../src/cli/subsystem-bench.rs | 5 +- .../src/{cli => lib}/approval/helpers.rs | 2 +- .../approval/message_generator.rs | 13 ++--- .../approval/mock_chain_selection.rs | 0 .../src/{cli => lib}/approval/mod.rs | 54 +++++++++---------- .../src/{cli => lib}/approval/test_message.rs | 5 +- polkadot/node/subsystem-bench/src/lib/lib.rs | 1 + 7 files changed, 36 insertions(+), 44 deletions(-) rename polkadot/node/subsystem-bench/src/{cli => lib}/approval/helpers.rs (99%) rename polkadot/node/subsystem-bench/src/{cli => lib}/approval/message_generator.rs (99%) rename polkadot/node/subsystem-bench/src/{cli => lib}/approval/mock_chain_selection.rs (100%) rename polkadot/node/subsystem-bench/src/{cli => lib}/approval/mod.rs (98%) rename polkadot/node/subsystem-bench/src/{cli => lib}/approval/test_message.rs (98%) diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 88265f837463..9eaf5f569bc6 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -17,14 +17,13 @@ //! A tool for running subsystem benchmark tests //! designed for development and CI regression testing. -use approval::{bench_approvals, ApprovalsOptions}; use availability::{cli::DataAvailabilityReadOptions, prepare_test, TestState}; use clap::Parser; use color_eyre::eyre; use colored::Colorize; use polkadot_subsystem_bench::{ + approval::{self, bench_approvals, ApprovalsOptions}, configuration::TestConfiguration, - display::display_configuration, environment::{TestEnvironment, GENESIS_HASH}, }; use pyroscope::PyroscopeAgent; @@ -34,7 +33,6 @@ use rand_distr::{Distribution, Uniform}; use serde::{Deserialize, Serialize}; use std::path::Path; -mod approval; mod availability; mod valgrind; @@ -61,7 +59,6 @@ impl std::fmt::Display for TestObjective { Self::DataAvailabilityRead(_) => "DataAvailabilityRead", Self::DataAvailabilityWrite => "DataAvailabilityWrite", Self::ApprovalVoting(_) => "ApprovalVoting", - // Self::Unimplemented => "Unimplemented", } ) } diff --git a/polkadot/node/subsystem-bench/src/cli/approval/helpers.rs b/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs similarity index 99% rename from polkadot/node/subsystem-bench/src/cli/approval/helpers.rs rename to polkadot/node/subsystem-bench/src/lib/approval/helpers.rs index 653f2a077e82..af5ff5aa1fac 100644 --- a/polkadot/node/subsystem-bench/src/cli/approval/helpers.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/helpers.rs @@ -14,6 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use crate::configuration::TestAuthorities; use itertools::Itertools; use polkadot_node_core_approval_voting::time::{Clock, SystemClock, Tick}; use polkadot_node_network_protocol::{ @@ -29,7 +30,6 @@ use polkadot_primitives::{ Id as ParaId, Slot, ValidatorIndex, }; use polkadot_primitives_test_helpers::dummy_candidate_receipt_bad_sig; -use polkadot_subsystem_bench::configuration::TestAuthorities; use rand::{seq::SliceRandom, SeedableRng}; use rand_chacha::ChaCha20Rng; use sc_network::PeerId; diff --git a/polkadot/node/subsystem-bench/src/cli/approval/message_generator.rs b/polkadot/node/subsystem-bench/src/lib/approval/message_generator.rs similarity index 99% rename from polkadot/node/subsystem-bench/src/cli/approval/message_generator.rs rename to polkadot/node/subsystem-bench/src/lib/approval/message_generator.rs index 186bb4e69347..c1b31a509f6d 100644 --- a/polkadot/node/subsystem-bench/src/cli/approval/message_generator.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/message_generator.rs @@ -18,10 +18,12 @@ use crate::{ approval::{ helpers::{generate_babe_epoch, generate_topology}, test_message::{MessagesBundle, TestMessageInfo}, - ApprovalTestState, BlockTestData, GeneratedState, BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET, - SLOT_DURATION_MILLIS, + ApprovalTestState, ApprovalsOptions, BlockTestData, GeneratedState, + BUFFER_FOR_GENERATION_MILLIS, LOG_TARGET, SLOT_DURATION_MILLIS, }, - ApprovalsOptions, + configuration::{TestAuthorities, TestConfiguration}, + mock::runtime_api::session_info_for_peers, + NODE_UNDER_TEST, }; use futures::SinkExt; use itertools::Itertools; @@ -42,11 +44,6 @@ use polkadot_primitives::{ vstaging::ApprovalVoteMultipleCandidates, CandidateEvent, CandidateHash, CandidateIndex, CoreIndex, Hash, SessionInfo, Slot, ValidatorId, ValidatorIndex, ASSIGNMENT_KEY_TYPE_ID, }; -use polkadot_subsystem_bench::{ - configuration::{TestAuthorities, TestConfiguration}, - mock::runtime_api::session_info_for_peers, - NODE_UNDER_TEST, -}; use rand::{seq::SliceRandom, RngCore, SeedableRng}; use rand_chacha::ChaCha20Rng; use rand_distr::{Distribution, Normal}; diff --git a/polkadot/node/subsystem-bench/src/cli/approval/mock_chain_selection.rs b/polkadot/node/subsystem-bench/src/lib/approval/mock_chain_selection.rs similarity index 100% rename from polkadot/node/subsystem-bench/src/cli/approval/mock_chain_selection.rs rename to polkadot/node/subsystem-bench/src/lib/approval/mock_chain_selection.rs diff --git a/polkadot/node/subsystem-bench/src/cli/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs similarity index 98% rename from polkadot/node/subsystem-bench/src/cli/approval/mod.rs rename to polkadot/node/subsystem-bench/src/lib/approval/mod.rs index a9383db0ef66..a4addbc8e251 100644 --- a/polkadot/node/subsystem-bench/src/cli/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -14,14 +14,30 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::approval::{ - helpers::{ - generate_babe_epoch, generate_new_session_topology, generate_peer_view_change_for, - make_header, PastSystemClock, +use crate::{ + approval::{ + helpers::{ + generate_babe_epoch, generate_new_session_topology, generate_peer_view_change_for, + make_header, PastSystemClock, + }, + message_generator::PeerMessagesGenerator, + mock_chain_selection::MockChainSelection, + test_message::{MessagesBundle, TestMessageInfo}, }, - message_generator::PeerMessagesGenerator, - mock_chain_selection::MockChainSelection, - test_message::{MessagesBundle, TestMessageInfo}, + configuration::{TestAuthorities, TestConfiguration}, + dummy_builder, + environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, + mock::{ + chain_api::{ChainApiState, MockChainApi}, + network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx}, + runtime_api::MockRuntimeApi, + AlwaysSupportsParachains, TestSyncOracle, + }, + network::{ + new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface, + NetworkInterfaceReceiver, + }, + NODE_UNDER_TEST, }; use colored::Colorize; use futures::channel::oneshot; @@ -45,22 +61,6 @@ use polkadot_primitives::{ BlockNumber, CandidateEvent, CandidateIndex, CandidateReceipt, Hash, Header, Slot, ValidatorIndex, }; -use polkadot_subsystem_bench::{ - configuration::{TestAuthorities, TestConfiguration}, - dummy_builder, - environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, - mock::{ - chain_api::{ChainApiState, MockChainApi}, - network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx}, - runtime_api::MockRuntimeApi, - AlwaysSupportsParachains, TestSyncOracle, - }, - network::{ - new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface, - NetworkInterfaceReceiver, - }, - NODE_UNDER_TEST, -}; use prometheus::Registry; use sc_keystore::LocalKeystore; use sc_service::SpawnTaskHandle; @@ -467,11 +467,9 @@ impl ApprovalTestState { impl HandleNetworkMessage for ApprovalTestState { fn handle( &self, - _message: polkadot_subsystem_bench::network::NetworkMessage, - _node_sender: &mut futures::channel::mpsc::UnboundedSender< - polkadot_subsystem_bench::network::NetworkMessage, - >, - ) -> Option { + _message: crate::network::NetworkMessage, + _node_sender: &mut futures::channel::mpsc::UnboundedSender, + ) -> Option { self.total_sent_messages_from_node .as_ref() .fetch_add(1, std::sync::atomic::Ordering::SeqCst); diff --git a/polkadot/node/subsystem-bench/src/cli/approval/test_message.rs b/polkadot/node/subsystem-bench/src/lib/approval/test_message.rs similarity index 98% rename from polkadot/node/subsystem-bench/src/cli/approval/test_message.rs rename to polkadot/node/subsystem-bench/src/lib/approval/test_message.rs index 887443fa5e47..63e383509be9 100644 --- a/polkadot/node/subsystem-bench/src/cli/approval/test_message.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/test_message.rs @@ -15,14 +15,13 @@ // along with Polkadot. If not, see . use crate::{ - approval::{BlockTestData, CandidateTestData}, - ApprovalsOptions, + approval::{ApprovalsOptions, BlockTestData, CandidateTestData}, + configuration::TestAuthorities, }; use itertools::Itertools; use parity_scale_codec::{Decode, Encode}; use polkadot_node_network_protocol::v3 as protocol_v3; use polkadot_primitives::{CandidateIndex, Hash, ValidatorIndex}; -use polkadot_subsystem_bench::configuration::TestAuthorities; use sc_network::PeerId; use std::collections::{HashMap, HashSet}; diff --git a/polkadot/node/subsystem-bench/src/lib/lib.rs b/polkadot/node/subsystem-bench/src/lib/lib.rs index f8a0f6354690..6c22dac391c3 100644 --- a/polkadot/node/subsystem-bench/src/lib/lib.rs +++ b/polkadot/node/subsystem-bench/src/lib/lib.rs @@ -17,6 +17,7 @@ // The validator index that represent the node that is under test. pub const NODE_UNDER_TEST: u32 = 0; +pub mod approval; pub mod configuration; pub mod display; pub mod environment; From d98b89992539c864a4eef2b03d18022d3c3db465 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 13 Feb 2024 16:10:49 +0100 Subject: [PATCH 09/53] Move availability to lib --- .../src/cli/availability/cli.rs | 28 -------- .../src/cli/subsystem-bench.rs | 38 ++++++----- .../availability/av_store_helpers.rs | 2 +- .../src/{cli => lib}/availability/mod.rs | 67 ++++++++++--------- polkadot/node/subsystem-bench/src/lib/lib.rs | 11 +-- 5 files changed, 66 insertions(+), 80 deletions(-) delete mode 100644 polkadot/node/subsystem-bench/src/cli/availability/cli.rs rename polkadot/node/subsystem-bench/src/{cli => lib}/availability/av_store_helpers.rs (94%) rename polkadot/node/subsystem-bench/src/{cli => lib}/availability/mod.rs (95%) diff --git a/polkadot/node/subsystem-bench/src/cli/availability/cli.rs b/polkadot/node/subsystem-bench/src/cli/availability/cli.rs deleted file mode 100644 index 06adbe5f1a79..000000000000 --- a/polkadot/node/subsystem-bench/src/cli/availability/cli.rs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)] -#[clap(rename_all = "kebab-case")] -#[allow(missing_docs)] -pub struct DataAvailabilityReadOptions { - #[clap(short, long, default_value_t = false)] - /// Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as - /// we don't need to re-construct from chunks. Tipically this is only faster if nodes have - /// enough bandwidth. - pub fetch_from_backers: bool, -} diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 9eaf5f569bc6..c2b5e11ae28a 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -17,15 +17,10 @@ //! A tool for running subsystem benchmark tests //! designed for development and CI regression testing. -use availability::{cli::DataAvailabilityReadOptions, prepare_test, TestState}; use clap::Parser; use color_eyre::eyre; use colored::Colorize; -use polkadot_subsystem_bench::{ - approval::{self, bench_approvals, ApprovalsOptions}, - configuration::TestConfiguration, - environment::{TestEnvironment, GENESIS_HASH}, -}; +use polkadot_subsystem_bench::{approval, availability, configuration}; use pyroscope::PyroscopeAgent; use pyroscope_pprofrs::{pprof_backend, PprofConfig}; use rand::thread_rng; @@ -33,7 +28,6 @@ use rand_distr::{Distribution, Uniform}; use serde::{Deserialize, Serialize}; use std::path::Path; -mod availability; mod valgrind; const LOG_TARGET: &str = "subsystem-bench::cli"; @@ -43,11 +37,11 @@ const LOG_TARGET: &str = "subsystem-bench::cli"; #[command(rename_all = "kebab-case")] pub enum TestObjective { /// Benchmark availability recovery strategies. - DataAvailabilityRead(DataAvailabilityReadOptions), + DataAvailabilityRead(availability::DataAvailabilityReadOptions), /// Benchmark availability and bitfield distribution. DataAvailabilityWrite, /// Benchmark the approval-voting and approval-distribution subsystems. - ApprovalVoting(ApprovalsOptions), + ApprovalVoting(approval::ApprovalsOptions), } impl std::fmt::Display for TestObjective { @@ -71,7 +65,7 @@ struct CliTestConfiguration { pub objective: TestObjective, /// Test Configuration #[serde(flatten)] - pub test_config: TestConfiguration, + pub test_config: configuration::TestConfiguration, } #[derive(Serialize, Deserialize)] @@ -172,9 +166,13 @@ impl BenchCli { gum::info!(target: LOG_TARGET, "[{}] {}", format!("objective = {:?}", objective).green(), test_config); match objective { - TestObjective::DataAvailabilityRead(ref _opts) => { - let mut state = TestState::new(objective, &test_config); - let (mut env, _protocol_config) = prepare_test(test_config, &mut state); + TestObjective::DataAvailabilityRead(opts) => { + let mut state = availability::TestState::new(&test_config); + let (mut env, _protocol_config) = availability::prepare_test( + test_config, + &mut state, + availability::TestDataAvailability::Read(opts), + ); env.runtime().block_on(availability::benchmark_availability_read( &benchmark_name, &mut env, @@ -182,8 +180,12 @@ impl BenchCli { )) }, TestObjective::DataAvailabilityWrite => { - let mut state = TestState::new(objective, &test_config); - let (mut env, _protocol_config) = prepare_test(test_config, &mut state); + let mut state = availability::TestState::new(&test_config); + let (mut env, _protocol_config) = availability::prepare_test( + test_config, + &mut state, + availability::TestDataAvailability::Write, + ); env.runtime().block_on(availability::benchmark_availability_write( &benchmark_name, &mut env, @@ -193,7 +195,11 @@ impl BenchCli { TestObjective::ApprovalVoting(ref options) => { let (mut env, state) = approval::prepare_test(test_config.clone(), options.clone()); - env.runtime().block_on(bench_approvals(&benchmark_name, &mut env, state)) + env.runtime().block_on(approval::bench_approvals( + &benchmark_name, + &mut env, + state, + )) }, }; } diff --git a/polkadot/node/subsystem-bench/src/cli/availability/av_store_helpers.rs b/polkadot/node/subsystem-bench/src/lib/availability/av_store_helpers.rs similarity index 94% rename from polkadot/node/subsystem-bench/src/cli/availability/av_store_helpers.rs rename to polkadot/node/subsystem-bench/src/lib/availability/av_store_helpers.rs index 9f3b7fb34d85..3300def2235e 100644 --- a/polkadot/node/subsystem-bench/src/cli/availability/av_store_helpers.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/av_store_helpers.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use crate::{environment::TestEnvironmentDependencies, mock::TestSyncOracle}; use polkadot_node_core_av_store::{AvailabilityStoreSubsystem, Config}; use polkadot_node_metrics::metrics::Metrics; use polkadot_node_subsystem_util::database::Database; -use polkadot_subsystem_bench::{environment::TestEnvironmentDependencies, mock::TestSyncOracle}; use std::sync::Arc; mod columns { diff --git a/polkadot/node/subsystem-bench/src/cli/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs similarity index 95% rename from polkadot/node/subsystem-bench/src/cli/availability/mod.rs rename to polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 4d047f86c1d1..877daa93abcc 100644 --- a/polkadot/node/subsystem-bench/src/cli/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -14,7 +14,19 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::{TestEnvironment, TestObjective, GENESIS_HASH}; +use crate::{ + configuration::TestConfiguration, + dummy_builder, + environment::{TestEnvironment, TestEnvironmentDependencies, GENESIS_HASH}, + mock::{ + av_store::{self, MockAvailabilityStore}, + chain_api::{ChainApiState, MockChainApi}, + network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx}, + runtime_api::{self, MockRuntimeApi}, + AlwaysSupportsParachains, + }, + network::new_network, +}; use av_store::NetworkAvailabilityState; use av_store_helpers::new_av_store; use bitvec::bitvec; @@ -51,32 +63,35 @@ use polkadot_primitives::{ Header, PersistedValidationData, Signed, SigningContext, ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; -use polkadot_subsystem_bench::{ - configuration::TestConfiguration, - dummy_builder, - environment::TestEnvironmentDependencies, - mock::{ - av_store::{self, MockAvailabilityStore}, - chain_api::{ChainApiState, MockChainApi}, - network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx}, - runtime_api::{self, MockRuntimeApi}, - AlwaysSupportsParachains, - }, - network::new_network, -}; use sc_network::{ request_responses::{IncomingRequest as RawIncomingRequest, ProtocolConfig}, PeerId, }; use sc_service::SpawnTaskHandle; +use serde::{Deserialize, Serialize}; use sp_core::H256; use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant}; mod av_store_helpers; -pub(crate) mod cli; const LOG_TARGET: &str = "subsystem-bench::availability"; +#[derive(Debug, Clone, Serialize, Deserialize, clap::Parser)] +#[clap(rename_all = "kebab-case")] +#[allow(missing_docs)] +pub struct DataAvailabilityReadOptions { + #[clap(short, long, default_value_t = false)] + /// Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU as + /// we don't need to re-construct from chunks. Tipically this is only faster if nodes have + /// enough bandwidth. + pub fetch_from_backers: bool, +} + +pub enum TestDataAvailability { + Read(DataAvailabilityReadOptions), + Write, +} + fn build_overseer_for_availability_read( spawn_task_handle: SpawnTaskHandle, runtime_api: MockRuntimeApi, @@ -137,13 +152,15 @@ fn build_overseer_for_availability_write( pub fn prepare_test( config: TestConfiguration, state: &mut TestState, + mode: TestDataAvailability, ) -> (TestEnvironment, Vec) { - prepare_test_inner(config, state, TestEnvironmentDependencies::default()) + prepare_test_inner(config, state, mode, TestEnvironmentDependencies::default()) } fn prepare_test_inner( config: TestConfiguration, state: &mut TestState, + mode: TestDataAvailability, dependencies: TestEnvironmentDependencies, ) -> (TestEnvironment, Vec) { // Generate test authorities. @@ -212,8 +229,8 @@ fn prepare_test_inner( let network_bridge_rx = network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); - let (overseer, overseer_handle) = match &state.objective() { - TestObjective::DataAvailabilityRead(options) => { + let (overseer, overseer_handle) = match &mode { + TestDataAvailability::Read(options) => { let use_fast_path = options.fetch_from_backers; let subsystem = if use_fast_path { @@ -243,7 +260,7 @@ fn prepare_test_inner( &dependencies, ) }, - TestObjective::DataAvailabilityWrite => { + TestDataAvailability::Write => { let availability_distribution = AvailabilityDistributionSubsystem::new( test_authorities.keyring.keystore(), IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver }, @@ -280,9 +297,6 @@ fn prepare_test_inner( &dependencies, ) }, - _ => { - unimplemented!("Invalid test objective") - }, }; ( @@ -300,8 +314,6 @@ fn prepare_test_inner( #[derive(Clone)] pub struct TestState { - // Test Objective - objective: TestObjective, // Full test configuration config: TestConfiguration, // A cycle iterator on all PoV sizes used in the test. @@ -324,10 +336,6 @@ pub struct TestState { } impl TestState { - fn objective(&self) -> &TestObjective { - &self.objective - } - pub fn next_candidate(&mut self) -> Option { let candidate = self.candidates.next(); let candidate_hash = candidate.as_ref().unwrap().hash(); @@ -365,7 +373,7 @@ impl TestState { .cycle(); } - pub fn new(objective: TestObjective, config: &TestConfiguration) -> Self { + pub fn new(config: &TestConfiguration) -> Self { let config = config.clone(); let mut chunks = Vec::new(); @@ -418,7 +426,6 @@ impl TestState { candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), backed_candidates: Vec::new(), - objective, config, }; diff --git a/polkadot/node/subsystem-bench/src/lib/lib.rs b/polkadot/node/subsystem-bench/src/lib/lib.rs index 6c22dac391c3..4137d662a913 100644 --- a/polkadot/node/subsystem-bench/src/lib/lib.rs +++ b/polkadot/node/subsystem-bench/src/lib/lib.rs @@ -18,9 +18,10 @@ pub const NODE_UNDER_TEST: u32 = 0; pub mod approval; +pub mod availability; pub mod configuration; -pub mod display; -pub mod environment; -pub mod keyring; -pub mod mock; -pub mod network; +pub(crate) mod display; +pub(crate) mod environment; +pub(crate) mod keyring; +pub(crate) mod mock; +pub(crate) mod network; From 5175897bd49c7005c701d7b655dc1c2e4b73ce69 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 14 Feb 2024 15:43:07 +0100 Subject: [PATCH 10/53] Return usage --- .../src/cli/subsystem-bench.rs | 33 +++++++++---------- .../subsystem-bench/src/lib/approval/mod.rs | 13 ++++---- .../src/lib/availability/mod.rs | 20 +++++------ .../subsystem-bench/src/lib/environment.rs | 6 ++-- 4 files changed, 33 insertions(+), 39 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index c2b5e11ae28a..aff402dae32b 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -74,20 +74,6 @@ pub struct TestSequence { test_configurations: Vec, } -fn random_uniform_sample + From>(min_value: T, max_value: T) -> T { - Uniform::from(min_value.into()..=max_value.into()) - .sample(&mut thread_rng()) - .into() -} - -fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { - random_uniform_sample(min_pov_size, max_pov_size) -} - -fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec { - (0..count).map(|_| random_pov_size(min_kib * 1024, max_kib * 1024)).collect() -} - impl TestSequence { fn into_vec(self) -> Vec { self.test_configurations @@ -102,9 +88,7 @@ impl TestSequence { }) .collect() } -} -impl TestSequence { fn new_from_file(path: &Path) -> std::io::Result { let string = String::from_utf8(std::fs::read(path)?).expect("File is valid UTF8"); Ok(serde_yaml::from_str(&string).expect("File is valid test sequence YA")) @@ -165,7 +149,7 @@ impl BenchCli { gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); gum::info!(target: LOG_TARGET, "[{}] {}", format!("objective = {:?}", objective).green(), test_config); - match objective { + let usage = match objective { TestObjective::DataAvailabilityRead(opts) => { let mut state = availability::TestState::new(&test_config); let (mut env, _protocol_config) = availability::prepare_test( @@ -202,6 +186,7 @@ impl BenchCli { )) }, }; + println!("{}", usage); } if let Some(agent_running) = agent_running { @@ -213,6 +198,20 @@ impl BenchCli { } } +fn random_uniform_sample + From>(min_value: T, max_value: T) -> T { + Uniform::from(min_value.into()..=max_value.into()) + .sample(&mut thread_rng()) + .into() +} + +fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { + random_uniform_sample(min_pov_size, max_pov_size) +} + +fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec { + (0..count).map(|_| random_pov_size(min_kib * 1024, max_kib * 1024)).collect() +} + fn main() -> eyre::Result<()> { color_eyre::install()?; env_logger::builder() diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index a4addbc8e251..3388837bb653 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -26,7 +26,9 @@ use crate::{ }, configuration::{TestAuthorities, TestConfiguration}, dummy_builder, - environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, + environment::{ + BenchmarkUsage, TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT, + }, mock::{ chain_api::{ChainApiState, MockChainApi}, network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx}, @@ -880,7 +882,7 @@ pub async fn bench_approvals( benchmark_name: &str, env: &mut TestEnvironment, mut state: ApprovalTestState, -) { +) -> BenchmarkUsage { let producer_rx = state .start_message_production( env.network(), @@ -898,7 +900,7 @@ pub async fn bench_approvals_run( env: &mut TestEnvironment, state: ApprovalTestState, producer_rx: oneshot::Receiver<()>, -) { +) -> BenchmarkUsage { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); @@ -1059,8 +1061,5 @@ pub async fn bench_approvals_run( state.total_unique_messages.load(std::sync::atomic::Ordering::SeqCst) ); - println!( - "{}", - env.collect_resource_usage(benchmark_name, &["approval-distribution", "approval-voting"]) - ); + env.collect_resource_usage(benchmark_name, &["approval-distribution", "approval-voting"]) } diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 877daa93abcc..645062a50ad0 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -17,7 +17,7 @@ use crate::{ configuration::TestConfiguration, dummy_builder, - environment::{TestEnvironment, TestEnvironmentDependencies, GENESIS_HASH}, + environment::{BenchmarkUsage, TestEnvironment, TestEnvironmentDependencies, GENESIS_HASH}, mock::{ av_store::{self, MockAvailabilityStore}, chain_api::{ChainApiState, MockChainApi}, @@ -442,7 +442,7 @@ pub async fn benchmark_availability_read( benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState, -) { +) -> BenchmarkUsage { let config = env.config().clone(); env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; @@ -503,15 +503,14 @@ pub async fn benchmark_availability_read( ); env.stop().await; - - println!("{}", env.collect_resource_usage(benchmark_name, &["availability-recovery"])); + env.collect_resource_usage(benchmark_name, &["availability-recovery"]) } pub async fn benchmark_availability_write( benchmark_name: &str, env: &mut TestEnvironment, mut state: TestState, -) { +) -> BenchmarkUsage { let config = env.config().clone(); env.metrics().set_n_validators(config.n_validators); @@ -665,13 +664,10 @@ pub async fn benchmark_availability_write( ); env.stop().await; - println!( - "{}", - env.collect_resource_usage( - benchmark_name, - &["availability-distribution", "bitfield-distribution", "availability-store"], - ) - ); + env.collect_resource_usage( + benchmark_name, + &["availability-distribution", "bitfield-distribution", "availability-store"], + ) } pub fn peer_bitfield_message_v2( diff --git a/polkadot/node/subsystem-bench/src/lib/environment.rs b/polkadot/node/subsystem-bench/src/lib/environment.rs index 3341e31f54e3..df30e5a74bae 100644 --- a/polkadot/node/subsystem-bench/src/lib/environment.rs +++ b/polkadot/node/subsystem-bench/src/lib/environment.rs @@ -412,9 +412,9 @@ impl TestEnvironment { #[derive(Debug, Serialize, Deserialize)] pub struct BenchmarkUsage { - benchmark_name: String, - network_usage: Vec, - cpu_usage: Vec, + pub benchmark_name: String, + pub network_usage: Vec, + pub cpu_usage: Vec, } impl std::fmt::Display for BenchmarkUsage { From a721bca9cc2bde857fbefe73fb9793517eb0dd83 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 14 Feb 2024 15:44:50 +0100 Subject: [PATCH 11/53] Add skeleton tests --- Cargo.lock | 1 + polkadot/node/core/approval-voting/Cargo.toml | 2 + .../tests/approval-voting-regression-bench.rs | 123 ++++++++++++++++++ .../examples/approvals_no_shows.yaml | 6 +- .../examples/approvals_throughput.yaml | 3 +- .../approvals_throughput_best_case.yaml | 2 +- ...s_throughput_no_optimisations_enabled.yaml | 2 +- .../subsystem-bench/src/lib/configuration.rs | 45 +++++-- 8 files changed, 166 insertions(+), 18 deletions(-) create mode 100644 polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs diff --git a/Cargo.lock b/Cargo.lock index 68d399f78a77..d95794bf94a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12352,6 +12352,7 @@ dependencies = [ "polkadot-overseer", "polkadot-primitives", "polkadot-primitives-test-helpers", + "polkadot-subsystem-bench", "rand", "rand_chacha 0.3.1", "rand_core 0.6.4", diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index 1010779a1b36..ac6e89f5ca09 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -53,3 +53,5 @@ kvdb-memorydb = "0.13.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } log = { workspace = true, default-features = true } env_logger = "0.9.0" + +polkadot-subsystem-bench = { path = "../../subsystem-bench" } diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs new file mode 100644 index 000000000000..9c7783f22cf7 --- /dev/null +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -0,0 +1,123 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use polkadot_subsystem_bench::{ + approval::{bench_approvals, prepare_test, ApprovalsOptions}, + configuration::TestConfiguration, +}; + +fn test_configuration() -> TestConfiguration { + let mut config = TestConfiguration::default(); + config.n_validators = 500; + config.n_cores = 100; + config.min_pov_size = 1120; + config.max_pov_size = 5120; + config.peer_bandwidth = 524288000000; + config.bandwidth = 524288000000; + config.num_blocks = 10; + + config +} + +#[test] +fn approvals_no_shows() { + let options = ApprovalsOptions { + coalesce_mean: 3.0, + coalesce_std_dev: 1.0, + enable_assignments_v2: true, + last_considered_tranche: 89, + stop_when_approved: true, + coalesce_tranche_diff: 12, + num_no_shows_per_candidate: 10, + workdir_prefix: "/tmp".to_string(), + }; + let (mut env, state) = prepare_test(test_configuration(), options); + + let usage = env.runtime().block_on(bench_approvals("approvals_no_shows", &mut env, state)); + + println!("{usage}"); + + assert!(false) +} + +#[test] +fn approvals_throughput() { + let options = ApprovalsOptions { + coalesce_mean: 3.0, + coalesce_std_dev: 1.0, + enable_assignments_v2: true, + last_considered_tranche: 89, + stop_when_approved: false, + coalesce_tranche_diff: 12, + num_no_shows_per_candidate: 0, + workdir_prefix: "/tmp".to_string(), + }; + let (mut env, state) = prepare_test(test_configuration(), options); + + let usage = env.runtime().block_on(bench_approvals("approvals_throughput", &mut env, state)); + + println!("{usage}"); + + assert!(false) +} + +#[test] +fn approvals_throughput_best_case() { + let options = ApprovalsOptions { + coalesce_mean: 4.0, + coalesce_std_dev: 2.0, + enable_assignments_v2: true, + last_considered_tranche: 90, + stop_when_approved: true, + coalesce_tranche_diff: 13, + num_no_shows_per_candidate: 1, + workdir_prefix: "/tmp".to_string(), + }; + let (mut env, state) = prepare_test(test_configuration(), options); + + let usage = + env.runtime() + .block_on(bench_approvals("approvals_throughput_best_case", &mut env, state)); + + println!("{usage}"); + + assert!(false) +} + +#[test] +fn approvals_throughput_no_optimisations_enabled() { + let options = ApprovalsOptions { + coalesce_mean: 1.0, + coalesce_std_dev: 0.0, + enable_assignments_v2: false, + last_considered_tranche: 89, + stop_when_approved: false, + coalesce_tranche_diff: 12, + num_no_shows_per_candidate: 0, + workdir_prefix: "/tmp".to_string(), + }; + let (mut env, state) = prepare_test(test_configuration(), options); + + let usage = env.runtime().block_on(bench_approvals( + "approvals_throughput_no_optimisations_enabled", + &mut env, + state, + )); + + println!("{usage}"); + + assert!(false) +} diff --git a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml index 758c7fbbf112..146da57d44c4 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml @@ -1,14 +1,14 @@ TestConfiguration: # Test 1 - objective: !ApprovalVoting - last_considered_tranche: 89 coalesce_mean: 3.0 coalesce_std_dev: 1.0 + enable_assignments_v2: true + last_considered_tranche: 89 stop_when_approved: true coalesce_tranche_diff: 12 - workdir_prefix: "/tmp/" - enable_assignments_v2: true num_no_shows_per_candidate: 10 + workdir_prefix: "/tmp/" n_validators: 500 n_cores: 100 min_pov_size: 1120 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml index 9eeeefc53a42..6b17e62c20aa 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput.yaml @@ -7,11 +7,10 @@ TestConfiguration: last_considered_tranche: 89 stop_when_approved: false coalesce_tranche_diff: 12 - workdir_prefix: "/tmp" num_no_shows_per_candidate: 0 + workdir_prefix: "/tmp" n_validators: 500 n_cores: 100 - n_included_candidates: 100 min_pov_size: 1120 max_pov_size: 5120 peer_bandwidth: 524288000000 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml index 370bb31a5c4c..e946c28e8ef5 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml @@ -7,8 +7,8 @@ TestConfiguration: last_considered_tranche: 89 stop_when_approved: true coalesce_tranche_diff: 12 - workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 + workdir_prefix: "/tmp/" n_validators: 500 n_cores: 100 min_pov_size: 1120 diff --git a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml index 30b9ac8dc50f..8f4b050e72f2 100644 --- a/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml +++ b/polkadot/node/subsystem-bench/examples/approvals_throughput_no_optimisations_enabled.yaml @@ -7,8 +7,8 @@ TestConfiguration: last_considered_tranche: 89 stop_when_approved: false coalesce_tranche_diff: 12 - workdir_prefix: "/tmp/" num_no_shows_per_candidate: 0 + workdir_prefix: "/tmp/" n_validators: 500 n_cores: 100 min_pov_size: 1120 diff --git a/polkadot/node/subsystem-bench/src/lib/configuration.rs b/polkadot/node/subsystem-bench/src/lib/configuration.rs index f1085082714d..f56f919fe11e 100644 --- a/polkadot/node/subsystem-bench/src/lib/configuration.rs +++ b/polkadot/node/subsystem-bench/src/lib/configuration.rs @@ -121,17 +121,27 @@ pub struct TestConfiguration { pub num_blocks: usize, } -/// Helper struct for authority related state. -#[derive(Clone)] -pub struct TestAuthorities { - pub keyring: Keyring, - pub validator_public: Vec, - pub validator_authority_id: Vec, - pub validator_babe_id: Vec, - pub validator_assignment_id: Vec, - pub key_seeds: Vec, - pub peer_ids: Vec, - pub peer_id_to_authority: HashMap, +impl Default for TestConfiguration { + fn default() -> Self { + Self { + n_validators: Default::default(), + n_cores: Default::default(), + needed_approvals: default_needed_approvals(), + zeroth_delay_tranche_width: default_zeroth_delay_tranche_width(), + relay_vrf_modulo_samples: default_relay_vrf_modulo_samples(), + n_delay_tranches: default_n_delay_tranches(), + no_show_slots: default_no_show_slots(), + max_validators_per_core: default_backing_group_size(), + min_pov_size: default_pov_size(), + max_pov_size: default_pov_size(), + pov_sizes: Default::default(), + peer_bandwidth: default_bandwidth(), + bandwidth: default_bandwidth(), + latency: Default::default(), + connectivity: default_connectivity(), + num_blocks: Default::default(), + } + } } impl TestConfiguration { @@ -189,6 +199,19 @@ impl TestConfiguration { } } +/// Helper struct for authority related state. +#[derive(Clone)] +pub struct TestAuthorities { + pub keyring: Keyring, + pub validator_public: Vec, + pub validator_authority_id: Vec, + pub validator_babe_id: Vec, + pub validator_assignment_id: Vec, + pub key_seeds: Vec, + pub peer_ids: Vec, + pub peer_id_to_authority: HashMap, +} + /// Sample latency (in milliseconds) from a normal distribution with parameters /// specified in `maybe_peer_latency`. pub fn random_latency(maybe_peer_latency: Option<&PeerLatency>) -> usize { From b27edaa6c2c7106c21d5f16116cea43f101f5fb2 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 15 Feb 2024 14:57:44 +0100 Subject: [PATCH 12/53] Add approval tests --- polkadot/node/core/approval-voting/Cargo.toml | 9 + .../tests/approval-voting-regression-bench.rs | 181 ++++++++++-------- .../src/cli/subsystem-bench.rs | 4 +- .../subsystem-bench/src/lib/approval/mod.rs | 15 +- .../src/lib/availability/mod.rs | 14 +- .../subsystem-bench/src/lib/environment.rs | 70 ++----- polkadot/node/subsystem-bench/src/lib/lib.rs | 1 + .../node/subsystem-bench/src/lib/usage.rs | 102 ++++++++++ 8 files changed, 254 insertions(+), 142 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/lib/usage.rs diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index ac6e89f5ca09..00b29894b43b 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -55,3 +55,12 @@ log = { workspace = true, default-features = true } env_logger = "0.9.0" polkadot-subsystem-bench = { path = "../../subsystem-bench" } + +[[test]] +name = "approval-voting-regression-bench" +path = "tests/approval-voting-regression-bench.rs" +harness = false +required-features = ["subsystem-bench"] + +[features] +subsystem-bench = [] diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs index 9c7783f22cf7..70808272e70e 100644 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -17,8 +17,24 @@ use polkadot_subsystem_bench::{ approval::{bench_approvals, prepare_test, ApprovalsOptions}, configuration::TestConfiguration, + usage::BenchmarkUsage, }; +fn main() -> Result<(), String> { + let mut messages = vec![]; + + messages.extend(approvals_no_shows()); + messages.extend(approvals_throughput()); + messages.extend(approvals_throughput_best_case()); + + if messages.is_empty() { + Ok(()) + } else { + eprintln!("{}", messages.join("\n")); + Err("Regressions found".to_string()) + } +} + fn test_configuration() -> TestConfiguration { let mut config = TestConfiguration::default(); config.n_validators = 500; @@ -32,92 +48,93 @@ fn test_configuration() -> TestConfiguration { config } -#[test] -fn approvals_no_shows() { - let options = ApprovalsOptions { - coalesce_mean: 3.0, - coalesce_std_dev: 1.0, - enable_assignments_v2: true, - last_considered_tranche: 89, - stop_when_approved: true, - coalesce_tranche_diff: 12, - num_no_shows_per_candidate: 10, - workdir_prefix: "/tmp".to_string(), - }; - let (mut env, state) = prepare_test(test_configuration(), options); - - let usage = env.runtime().block_on(bench_approvals("approvals_no_shows", &mut env, state)); - - println!("{usage}"); - - assert!(false) +fn run_benchmark(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage { + let (mut env, state) = prepare_test(test_configuration(), options, false); + let usage = env.runtime().block_on(bench_approvals(test_case, &mut env, state)); + println!("{}", usage); + usage } -#[test] -fn approvals_throughput() { - let options = ApprovalsOptions { - coalesce_mean: 3.0, - coalesce_std_dev: 1.0, - enable_assignments_v2: true, - last_considered_tranche: 89, - stop_when_approved: false, - coalesce_tranche_diff: 12, - num_no_shows_per_candidate: 0, - workdir_prefix: "/tmp".to_string(), - }; - let (mut env, state) = prepare_test(test_configuration(), options); - - let usage = env.runtime().block_on(bench_approvals("approvals_throughput", &mut env, state)); - - println!("{usage}"); - - assert!(false) +fn approvals_no_shows() { + let mut messages = vec![]; + let usage = run_benchmark( + "approvals_no_shows", + ApprovalsOptions { + coalesce_mean: 3.0, + coalesce_std_dev: 1.0, + enable_assignments_v2: true, + last_considered_tranche: 89, + stop_when_approved: true, + coalesce_tranche_diff: 12, + num_no_shows_per_candidate: 10, + workdir_prefix: "/tmp".to_string(), + }, + ); + + messages.extend(usage.check_network_usage(&[ + ("Received from peers", 6900.000, 7000.000), + ("Sent to peers", 8000.000, 8100.000), + ])); + messages.extend(usage.check_cpu_usage(&[ + ("approval-distribution", 0.700, 0.800), + ("approval-voting", 1.100, 1.200), + ])); + + messages } -#[test] -fn approvals_throughput_best_case() { - let options = ApprovalsOptions { - coalesce_mean: 4.0, - coalesce_std_dev: 2.0, - enable_assignments_v2: true, - last_considered_tranche: 90, - stop_when_approved: true, - coalesce_tranche_diff: 13, - num_no_shows_per_candidate: 1, - workdir_prefix: "/tmp".to_string(), - }; - let (mut env, state) = prepare_test(test_configuration(), options); - - let usage = - env.runtime() - .block_on(bench_approvals("approvals_throughput_best_case", &mut env, state)); - - println!("{usage}"); - - assert!(false) +fn approvals_throughput() { + let mut messages = vec![]; + let usage = run_benchmark( + "approvals_throughput", + ApprovalsOptions { + coalesce_mean: 3.0, + coalesce_std_dev: 1.0, + enable_assignments_v2: true, + last_considered_tranche: 89, + stop_when_approved: false, + coalesce_tranche_diff: 12, + num_no_shows_per_candidate: 0, + workdir_prefix: "/tmp".to_string(), + }, + ); + + messages.extend(usage.check_network_usage(&[ + ("Received from peers", 52900.000, 53000.000), + ("Sent to peers", 63500.000, 63600.000), + ])); + messages.extend(usage.check_cpu_usage(&[ + ("approval-distribution", 6.300, 6.400), + ("approval-voting", 9.400, 9.500), + ])); + + messages } -#[test] -fn approvals_throughput_no_optimisations_enabled() { - let options = ApprovalsOptions { - coalesce_mean: 1.0, - coalesce_std_dev: 0.0, - enable_assignments_v2: false, - last_considered_tranche: 89, - stop_when_approved: false, - coalesce_tranche_diff: 12, - num_no_shows_per_candidate: 0, - workdir_prefix: "/tmp".to_string(), - }; - let (mut env, state) = prepare_test(test_configuration(), options); - - let usage = env.runtime().block_on(bench_approvals( - "approvals_throughput_no_optimisations_enabled", - &mut env, - state, - )); - - println!("{usage}"); - - assert!(false) +fn approvals_throughput_best_case() -> Vec { + let mut messages = vec![]; + let usage = run_benchmark( + "approvals_throughput_best_case", + ApprovalsOptions { + coalesce_mean: 3.0, + coalesce_std_dev: 1.0, + enable_assignments_v2: true, + last_considered_tranche: 89, + stop_when_approved: true, + coalesce_tranche_diff: 12, + num_no_shows_per_candidate: 0, + workdir_prefix: "/tmp".to_string(), + }, + ); + + messages.extend(usage.check_network_usage(&[ + ("Received from peers", 2900.000, 3000.000), + ("Sent to peers", 3200.000, 3300.000), + ])); + messages.extend(usage.check_cpu_usage(&[ + ("approval-distribution", 0.400, 0.500), + ("approval-voting", 0.600, 0.700), + ])); + + messages } diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index aff402dae32b..563f6851851d 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -156,6 +156,7 @@ impl BenchCli { test_config, &mut state, availability::TestDataAvailability::Read(opts), + true, ); env.runtime().block_on(availability::benchmark_availability_read( &benchmark_name, @@ -169,6 +170,7 @@ impl BenchCli { test_config, &mut state, availability::TestDataAvailability::Write, + true, ); env.runtime().block_on(availability::benchmark_availability_write( &benchmark_name, @@ -178,7 +180,7 @@ impl BenchCli { }, TestObjective::ApprovalVoting(ref options) => { let (mut env, state) = - approval::prepare_test(test_config.clone(), options.clone()); + approval::prepare_test(test_config.clone(), options.clone(), true); env.runtime().block_on(approval::bench_approvals( &benchmark_name, &mut env, diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index 3388837bb653..450faf06123f 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -26,9 +26,7 @@ use crate::{ }, configuration::{TestAuthorities, TestConfiguration}, dummy_builder, - environment::{ - BenchmarkUsage, TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT, - }, + environment::{TestEnvironment, TestEnvironmentDependencies, MAX_TIME_OF_FLIGHT}, mock::{ chain_api::{ChainApiState, MockChainApi}, network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx}, @@ -39,6 +37,7 @@ use crate::{ new_network, HandleNetworkMessage, NetworkEmulatorHandle, NetworkInterface, NetworkInterfaceReceiver, }, + usage::BenchmarkUsage, NODE_UNDER_TEST, }; use colored::Colorize; @@ -836,8 +835,14 @@ fn build_overseer( pub fn prepare_test( config: TestConfiguration, options: ApprovalsOptions, + with_prometheus_endpoint: bool, ) -> (TestEnvironment, ApprovalTestState) { - prepare_test_inner(config, TestEnvironmentDependencies::default(), options) + prepare_test_inner( + config, + TestEnvironmentDependencies::default(), + options, + with_prometheus_endpoint, + ) } /// Build the test environment for an Approval benchmark. @@ -845,6 +850,7 @@ fn prepare_test_inner( config: TestConfiguration, dependencies: TestEnvironmentDependencies, options: ApprovalsOptions, + with_prometheus_endpoint: bool, ) -> (TestEnvironment, ApprovalTestState) { gum::info!("Prepare test state"); let state = ApprovalTestState::new(&config, options, &dependencies); @@ -873,6 +879,7 @@ fn prepare_test_inner( overseer, overseer_handle, state.test_authorities.clone(), + with_prometheus_endpoint, ), state, ) diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 645062a50ad0..f012a5a907ed 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -17,7 +17,7 @@ use crate::{ configuration::TestConfiguration, dummy_builder, - environment::{BenchmarkUsage, TestEnvironment, TestEnvironmentDependencies, GENESIS_HASH}, + environment::{TestEnvironment, TestEnvironmentDependencies, GENESIS_HASH}, mock::{ av_store::{self, MockAvailabilityStore}, chain_api::{ChainApiState, MockChainApi}, @@ -26,6 +26,7 @@ use crate::{ AlwaysSupportsParachains, }, network::new_network, + usage::BenchmarkUsage, }; use av_store::NetworkAvailabilityState; use av_store_helpers::new_av_store; @@ -153,8 +154,15 @@ pub fn prepare_test( config: TestConfiguration, state: &mut TestState, mode: TestDataAvailability, + with_prometheus_endpoint: bool, ) -> (TestEnvironment, Vec) { - prepare_test_inner(config, state, mode, TestEnvironmentDependencies::default()) + prepare_test_inner( + config, + state, + mode, + TestEnvironmentDependencies::default(), + with_prometheus_endpoint, + ) } fn prepare_test_inner( @@ -162,6 +170,7 @@ fn prepare_test_inner( state: &mut TestState, mode: TestDataAvailability, dependencies: TestEnvironmentDependencies, + with_prometheus_endpoint: bool, ) -> (TestEnvironment, Vec) { // Generate test authorities. let test_authorities = config.generate_authorities(); @@ -307,6 +316,7 @@ fn prepare_test_inner( overseer, overseer_handle, test_authorities, + with_prometheus_endpoint, ), req_cfgs, ) diff --git a/polkadot/node/subsystem-bench/src/lib/environment.rs b/polkadot/node/subsystem-bench/src/lib/environment.rs index df30e5a74bae..958ed50d0894 100644 --- a/polkadot/node/subsystem-bench/src/lib/environment.rs +++ b/polkadot/node/subsystem-bench/src/lib/environment.rs @@ -20,8 +20,8 @@ use crate::{ configuration::{TestAuthorities, TestConfiguration}, mock::AlwaysSupportsParachains, network::NetworkEmulatorHandle, + usage::{BenchmarkUsage, ResourceUsage}, }; -use colored::Colorize; use core::time::Duration; use futures::{Future, FutureExt}; use polkadot_node_subsystem::{messages::AllMessages, Overseer, SpawnGlue, TimeoutExt}; @@ -31,7 +31,6 @@ use polkadot_node_subsystem_util::metrics::prometheus::{ }; use polkadot_overseer::{BlockInfo, Handle as OverseerHandle}; use sc_service::{SpawnTaskHandle, TaskManager}; -use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, SocketAddr}; use tokio::runtime::Handle; @@ -202,6 +201,7 @@ impl TestEnvironment { overseer: Overseer, AlwaysSupportsParachains>, overseer_handle: OverseerHandle, authorities: TestAuthorities, + with_prometheus_endpoint: bool, ) -> Self { let metrics = TestEnvironmentMetrics::new(&dependencies.registry) .expect("Metrics need to be registered"); @@ -209,19 +209,21 @@ impl TestEnvironment { let spawn_handle = dependencies.task_manager.spawn_handle(); spawn_handle.spawn_blocking("overseer", "overseer", overseer.run().boxed()); - let registry_clone = dependencies.registry.clone(); - dependencies.task_manager.spawn_handle().spawn_blocking( - "prometheus", - "test-environment", - async move { - prometheus_endpoint::init_prometheus( - SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999), - registry_clone, - ) - .await - .unwrap(); - }, - ); + if with_prometheus_endpoint { + let registry_clone = dependencies.registry.clone(); + dependencies.task_manager.spawn_handle().spawn_blocking( + "prometheus", + "test-environment", + async move { + prometheus_endpoint::init_prometheus( + SocketAddr::new(std::net::IpAddr::V4(Ipv4Addr::LOCALHOST), 9999), + registry_clone, + ) + .await + .unwrap(); + }, + ); + } TestEnvironment { runtime_handle: dependencies.runtime.handle().clone(), @@ -409,41 +411,3 @@ impl TestEnvironment { usage } } - -#[derive(Debug, Serialize, Deserialize)] -pub struct BenchmarkUsage { - pub benchmark_name: String, - pub network_usage: Vec, - pub cpu_usage: Vec, -} - -impl std::fmt::Display for BenchmarkUsage { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "\n{}\n\n{}\n{}\n\n{}\n{}\n", - self.benchmark_name.purple(), - format!("{:<32}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(), - self.network_usage - .iter() - .map(|v| v.to_string()) - .collect::>() - .join("\n"), - format!("{:<32}{:>12}{:>12}", "CPU usage in seconds", "total", "per block").blue(), - self.cpu_usage.iter().map(|v| v.to_string()).collect::>().join("\n") - ) - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct ResourceUsage { - resource_name: String, - total: f64, - per_block: f64, -} - -impl std::fmt::Display for ResourceUsage { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{:<32}{:>12.3}{:>12.3}", self.resource_name.cyan(), self.total, self.per_block) - } -} diff --git a/polkadot/node/subsystem-bench/src/lib/lib.rs b/polkadot/node/subsystem-bench/src/lib/lib.rs index 4137d662a913..d06f2822a895 100644 --- a/polkadot/node/subsystem-bench/src/lib/lib.rs +++ b/polkadot/node/subsystem-bench/src/lib/lib.rs @@ -25,3 +25,4 @@ pub(crate) mod environment; pub(crate) mod keyring; pub(crate) mod mock; pub(crate) mod network; +pub mod usage; diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs new file mode 100644 index 000000000000..1fc6a3327928 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -0,0 +1,102 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test usage implementation + +use colored::Colorize; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchmarkUsage { + pub benchmark_name: String, + pub network_usage: Vec, + pub cpu_usage: Vec, +} + +impl std::fmt::Display for BenchmarkUsage { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "\n{}\n\n{}\n{}\n\n{}\n{}\n", + self.benchmark_name.purple(), + format!("{:<32}{:>12}{:>12}", "Network usage, KiB", "total", "per block").blue(), + self.network_usage + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n"), + format!("{:<32}{:>12}{:>12}", "CPU usage in seconds", "total", "per block").blue(), + self.cpu_usage.iter().map(|v| v.to_string()).collect::>().join("\n") + ) + } +} + +impl BenchmarkUsage { + pub fn check_network_usage(&self, checks: &[ResourceUsageCheck]) -> Vec { + check_usage(&self.benchmark_name, &self.network_usage, checks) + } + + pub fn check_cpu_usage(&self, checks: &[ResourceUsageCheck]) -> Vec { + check_usage(&self.benchmark_name, &self.cpu_usage, checks) + } +} + +fn check_usage( + benchmark_name: &str, + usage: &Vec, + checks: &[ResourceUsageCheck], +) -> Vec { + checks + .iter() + .filter_map(|check| { + check_resource_usage(usage, check) + .map(|message| format!("{}: {}", benchmark_name, message)) + }) + .collect() +} + +fn check_resource_usage( + usage: &Vec, + (resource_name, min, max): &ResourceUsageCheck, +) -> Option { + if let Some(usage) = usage.iter().find(|v| v.resource_name == *resource_name) { + if usage.per_block >= *min && usage.per_block < *max { + None + } else { + Some(format!( + "The resource `{}` is expected to be in the range of {}..{}, but the value is {}", + resource_name, min, max, usage.per_block + )) + } + } else { + Some(format!("The resource `{}` is not found", resource_name)) + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ResourceUsage { + pub resource_name: String, + pub total: f64, + pub per_block: f64, +} + +impl std::fmt::Display for ResourceUsage { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{:<32}{:>12.3}{:>12.3}", self.resource_name.cyan(), self.total, self.per_block) + } +} + +type ResourceUsageCheck<'a> = (&'a str, f64, f64); From 913695528bc8fb2dc4da0c908893bbfadd55730e Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 15 Feb 2024 18:17:41 +0100 Subject: [PATCH 13/53] Add regression tests --- Cargo.lock | 2 + polkadot/node/core/approval-voting/Cargo.toml | 4 +- .../tests/approval-voting-regression-bench.rs | 5 +- .../availability-distribution/Cargo.toml | 11 ++++ ...ilability-distribution-regression-bench.rs | 64 +++++++++++++++++++ .../network/availability-recovery/Cargo.toml | 7 ++ .../availability-recovery-regression-bench.rs | 63 ++++++++++++++++++ .../src/cli/subsystem-bench.rs | 31 +-------- .../subsystem-bench/src/lib/configuration.rs | 20 +++++- 9 files changed, 174 insertions(+), 33 deletions(-) create mode 100644 polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs create mode 100644 polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs diff --git a/Cargo.lock b/Cargo.lock index d95794bf94a2..cf715e5b426d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12086,6 +12086,7 @@ dependencies = [ "polkadot-node-subsystem-util", "polkadot-primitives", "polkadot-primitives-test-helpers", + "polkadot-subsystem-bench", "rand", "sc-network", "schnellru", @@ -12117,6 +12118,7 @@ dependencies = [ "polkadot-node-subsystem-util", "polkadot-primitives", "polkadot-primitives-test-helpers", + "polkadot-subsystem-bench", "rand", "sc-network", "schnellru", diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index 00b29894b43b..1bd6d1acab27 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -60,7 +60,7 @@ polkadot-subsystem-bench = { path = "../../subsystem-bench" } name = "approval-voting-regression-bench" path = "tests/approval-voting-regression-bench.rs" harness = false -required-features = ["subsystem-bench"] +required-features = ["subsystem-benchmarks"] [features] -subsystem-bench = [] +subsystem-benchmarks = [] diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs index 70808272e70e..1a742846990c 100644 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -44,6 +44,7 @@ fn test_configuration() -> TestConfiguration { config.peer_bandwidth = 524288000000; config.bandwidth = 524288000000; config.num_blocks = 10; + config.generate_pov_sizes(); config } @@ -55,7 +56,7 @@ fn run_benchmark(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage { usage } -fn approvals_no_shows() { +fn approvals_no_shows() -> Vec { let mut messages = vec![]; let usage = run_benchmark( "approvals_no_shows", @@ -83,7 +84,7 @@ fn approvals_no_shows() { messages } -fn approvals_throughput() { +fn approvals_throughput() -> Vec { let mut messages = vec![]; let usage = run_benchmark( "approvals_throughput", diff --git a/polkadot/node/network/availability-distribution/Cargo.toml b/polkadot/node/network/availability-distribution/Cargo.toml index a1484429ed63..39a3f6fda016 100644 --- a/polkadot/node/network/availability-distribution/Cargo.toml +++ b/polkadot/node/network/availability-distribution/Cargo.toml @@ -36,3 +36,14 @@ sc-network = { path = "../../../../substrate/client/network" } futures-timer = "3.0.2" assert_matches = "1.4.0" polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" } +polkadot-subsystem-bench = { path = "../../subsystem-bench" } + + +[[test]] +name = "availability-distribution-regression-bench" +path = "tests/availability-distribution-regression-bench.rs" +harness = false +required-features = ["subsystem-benchmarks"] + +[features] +subsystem-benchmarks = [] diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs new file mode 100644 index 000000000000..82d7431a3b72 --- /dev/null +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -0,0 +1,64 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use polkadot_subsystem_bench::{ + availability::{benchmark_availability_write, prepare_test, TestDataAvailability, TestState}, + configuration::{PeerLatency, TestConfiguration}, +}; + +fn main() -> Result<(), String> { + let mut messages = vec![]; + let mut config = TestConfiguration::default(); + config.latency = Some(PeerLatency { mean_latency_ms: 30, std_dev: 2.0 }); + config.n_validators = 1000; + config.n_cores = 200; + config.max_validators_per_core = 5; + config.min_pov_size = 5120; + config.max_pov_size = 5120; + config.peer_bandwidth = 52428800; + config.bandwidth = 52428800; + config.connectivity = 75; + config.num_blocks = 3; + config.generate_pov_sizes(); + + let mut state = TestState::new(&config); + let (mut env, _protocol_config) = + prepare_test(config, &mut state, TestDataAvailability::Write, false); + let usage = env.runtime().block_on(benchmark_availability_write( + "data_availability_write", + &mut env, + state, + )); + + println!("{}", usage); + + messages.extend(usage.check_network_usage(&[ + ("Received from peers", 4300.000, 4400.000), + ("Sent to peers", 15900.000, 16000.000), + ])); + messages.extend(usage.check_cpu_usage(&[ + ("availability-distribution", 0.000, 0.100), + ("bitfield-distribution", 0.000, 0.100), + ("availability-store", 0.100, 0.200), + ])); + + if messages.is_empty() { + Ok(()) + } else { + eprintln!("{}", messages.join("\n")); + Err("Regressions found".to_string()) + } +} diff --git a/polkadot/node/network/availability-recovery/Cargo.toml b/polkadot/node/network/availability-recovery/Cargo.toml index e86445730ffc..32dafd7a9281 100644 --- a/polkadot/node/network/availability-recovery/Cargo.toml +++ b/polkadot/node/network/availability-recovery/Cargo.toml @@ -41,6 +41,13 @@ sc-network = { path = "../../../../substrate/client/network" } polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" } +polkadot-subsystem-bench = { path = "../../subsystem-bench" } + +[[test]] +name = "availability-recovery-regression-bench" +path = "tests/availability-recovery-regression-bench.rs" +harness = false +required-features = ["subsystem-benchmarks"] [features] subsystem-benchmarks = [] diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs new file mode 100644 index 000000000000..ef75c00b490d --- /dev/null +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -0,0 +1,63 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use polkadot_subsystem_bench::{ + availability::{ + benchmark_availability_read, prepare_test, DataAvailabilityReadOptions, + TestDataAvailability, TestState, + }, + configuration::{PeerLatency, TestConfiguration}, +}; + +fn main() -> Result<(), String> { + let mut messages = vec![]; + let options = DataAvailabilityReadOptions { fetch_from_backers: true }; + let mut config = TestConfiguration::default(); + config.latency = Some(PeerLatency { mean_latency_ms: 100, std_dev: 1.0 }); + config.n_validators = 300; + config.n_cores = 20; + config.min_pov_size = 5120; + config.max_pov_size = 5120; + config.peer_bandwidth = 52428800; + config.bandwidth = 52428800; + config.num_blocks = 3; + config.connectivity = 90; + config.generate_pov_sizes(); + + let mut state = TestState::new(&config); + let (mut env, _protocol_config) = + prepare_test(config, &mut state, TestDataAvailability::Read(options), false); + let usage = env.runtime().block_on(benchmark_availability_read( + "data_availability_read", + &mut env, + state, + )); + + println!("{}", usage); + + messages.extend(usage.check_network_usage(&[ + ("Received from peers", 102400.000, 102500.000), + ("Sent to peers", 0.000, 100.000), + ])); + messages.extend(usage.check_cpu_usage(&[("availability-recovery", 4.200, 4.300)])); + + if messages.is_empty() { + Ok(()) + } else { + eprintln!("{}", messages.join("\n")); + Err("Regressions found".to_string()) + } +} diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 563f6851851d..6861ece40083 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -23,8 +23,6 @@ use colored::Colorize; use polkadot_subsystem_bench::{approval, availability, configuration}; use pyroscope::PyroscopeAgent; use pyroscope_pprofrs::{pprof_backend, PprofConfig}; -use rand::thread_rng; -use rand_distr::{Distribution, Uniform}; use serde::{Deserialize, Serialize}; use std::path::Path; @@ -76,17 +74,7 @@ pub struct TestSequence { impl TestSequence { fn into_vec(self) -> Vec { - self.test_configurations - .into_iter() - .map(|mut v| { - v.test_config.pov_sizes = generate_pov_sizes( - v.test_config.n_cores, - v.test_config.min_pov_size, - v.test_config.max_pov_size, - ); - v - }) - .collect() + self.test_configuration } fn new_from_file(path: &Path) -> std::io::Result { @@ -142,12 +130,13 @@ impl BenchCli { let num_steps = test_sequence.len(); gum::info!("{}", format!("Sequence contains {} step(s)", num_steps).bright_purple()); - for (index, CliTestConfiguration { objective, test_config }) in + for (index, CliTestConfiguration { objective, mut test_config }) in test_sequence.into_iter().enumerate() { let benchmark_name = format!("{} #{} {}", &self.path, index + 1, objective); gum::info!(target: LOG_TARGET, "{}", format!("Step {}/{}", index + 1, num_steps).bright_purple(),); gum::info!(target: LOG_TARGET, "[{}] {}", format!("objective = {:?}", objective).green(), test_config); + test_config.generate_pov_sizes(); let usage = match objective { TestObjective::DataAvailabilityRead(opts) => { @@ -200,20 +189,6 @@ impl BenchCli { } } -fn random_uniform_sample + From>(min_value: T, max_value: T) -> T { - Uniform::from(min_value.into()..=max_value.into()) - .sample(&mut thread_rng()) - .into() -} - -fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { - random_uniform_sample(min_pov_size, max_pov_size) -} - -fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec { - (0..count).map(|_| random_pov_size(min_kib * 1024, max_kib * 1024)).collect() -} - fn main() -> eyre::Result<()> { color_eyre::install()?; env_logger::builder() diff --git a/polkadot/node/subsystem-bench/src/lib/configuration.rs b/polkadot/node/subsystem-bench/src/lib/configuration.rs index f56f919fe11e..c76933085271 100644 --- a/polkadot/node/subsystem-bench/src/lib/configuration.rs +++ b/polkadot/node/subsystem-bench/src/lib/configuration.rs @@ -20,7 +20,7 @@ use crate::keyring::Keyring; use itertools::Itertools; use polkadot_primitives::{AssignmentId, AuthorityDiscoveryId, ValidatorId}; use rand::thread_rng; -use rand_distr::{Distribution, Normal}; +use rand_distr::{Distribution, Normal, Uniform}; use sc_network::PeerId; use serde::{Deserialize, Serialize}; use sp_consensus_babe::AuthorityId; @@ -145,6 +145,10 @@ impl Default for TestConfiguration { } impl TestConfiguration { + pub fn generate_pov_sizes(&mut self) { + self.pov_sizes = generate_pov_sizes(self.n_cores, self.min_pov_size, self.max_pov_size); + } + pub fn pov_sizes(&self) -> &[usize] { &self.pov_sizes } @@ -199,6 +203,20 @@ impl TestConfiguration { } } +fn random_uniform_sample + From>(min_value: T, max_value: T) -> T { + Uniform::from(min_value.into()..=max_value.into()) + .sample(&mut thread_rng()) + .into() +} + +fn random_pov_size(min_pov_size: usize, max_pov_size: usize) -> usize { + random_uniform_sample(min_pov_size, max_pov_size) +} + +fn generate_pov_sizes(count: usize, min_kib: usize, max_kib: usize) -> Vec { + (0..count).map(|_| random_pov_size(min_kib * 1024, max_kib * 1024)).collect() +} + /// Helper struct for authority related state. #[derive(Clone)] pub struct TestAuthorities { From 18a0c7efdcdb8dccf866ba8ac2f0bbe2a1510b45 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 08:46:32 +0100 Subject: [PATCH 14/53] Address clippy warnings --- polkadot/node/subsystem-bench/src/lib/usage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs index 1fc6a3327928..9b88fdd15c6b 100644 --- a/polkadot/node/subsystem-bench/src/lib/usage.rs +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -69,7 +69,7 @@ fn check_usage( } fn check_resource_usage( - usage: &Vec, + usage: &[ResourceUsage], (resource_name, min, max): &ResourceUsageCheck, ) -> Option { if let Some(usage) = usage.iter().find(|v| v.resource_name == *resource_name) { From d72a0352038e8243ed4a42c9af3b1a6356dcc8a7 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 09:00:59 +0100 Subject: [PATCH 15/53] Address clippy warnings --- polkadot/node/subsystem-bench/src/lib/usage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs index 9b88fdd15c6b..a2f0a2114556 100644 --- a/polkadot/node/subsystem-bench/src/lib/usage.rs +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -56,7 +56,7 @@ impl BenchmarkUsage { fn check_usage( benchmark_name: &str, - usage: &Vec, + usage: &[ResourceUsage], checks: &[ResourceUsageCheck], ) -> Vec { checks From d161de1a4ec36496f9de7ba1522bd7985a008611 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 09:26:27 +0100 Subject: [PATCH 16/53] Address clippy warnings --- polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 6861ece40083..023ab635181e 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -74,7 +74,7 @@ pub struct TestSequence { impl TestSequence { fn into_vec(self) -> Vec { - self.test_configuration + self.test_configurations } fn new_from_file(path: &Path) -> std::io::Result { From ab2e1794333b1ef7571822cb6017e5154a2babc4 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 17:09:14 +0100 Subject: [PATCH 17/53] Update --- polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 023ab635181e..deb351360d74 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -12,7 +12,7 @@ // GNU General Public License for more details. // You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . +// along with Polkadot. If not, see . //! A tool for running subsystem benchmark tests //! designed for development and CI regression testing. @@ -73,10 +73,6 @@ pub struct TestSequence { } impl TestSequence { - fn into_vec(self) -> Vec { - self.test_configurations - } - fn new_from_file(path: &Path) -> std::io::Result { let string = String::from_utf8(std::fs::read(path)?).expect("File is valid UTF8"); Ok(serde_yaml::from_str(&string).expect("File is valid test sequence YA")) @@ -126,7 +122,7 @@ impl BenchCli { let test_sequence = TestSequence::new_from_file(Path::new(&self.path)) .expect("File exists") - .into_vec(); + .test_configurations; let num_steps = test_sequence.len(); gum::info!("{}", format!("Sequence contains {} step(s)", num_steps).bright_purple()); From 1e59be14fe420ec70d06ac3171451e2df8f8ecc6 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 17:17:02 +0100 Subject: [PATCH 18/53] Add draft ci run --- .gitlab/pipeline/test.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index b5e26d194896..f318f4516252 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -492,3 +492,15 @@ test-syscalls: printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n"; fi allow_failure: false # this rarely triggers in practice + +subsystem-bench: + stage: test + extends: + - .docker-env + - .common-refs + script: + - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks -- --nocapture + - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks -- --nocapture + - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks -- --nocapture + tags: + - benchmark From b54b85d3fb8b1755a888e1984f426bd39fb3b694 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 17:51:13 +0100 Subject: [PATCH 19/53] Filter regression tests --- .config/nextest.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.config/nextest.toml b/.config/nextest.toml index 1e18f8b5589c..880499e96b45 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -124,3 +124,7 @@ serial-integration = { max-threads = 1 } [[profile.default.overrides]] filter = 'test(/(^ui$|_ui|ui_)/)' test-group = 'serial-integration' + +# Skip subsystems regression benchmarks +[[profile.default.overrides]] +filter = 'not deps(/polkadot-subsystem-bench/)' From 0e4432a97fcabb3243f135bdf469cb406aa5ad20 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 18:09:04 +0100 Subject: [PATCH 20/53] Update nexttest config --- .config/nextest.toml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 880499e96b45..d01cfea1554c 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -9,6 +9,9 @@ dir = "target/nextest" # This section defines the default nextest profile. Custom profiles are layered # on top of the default profile. [profile.default] +# Skip subsystems regression benchmarks +filter = 'not deps(/polkadot-subsystem-bench/)' + # "retries" defines the number of times a test should be retried. If set to a # non-zero value, tests that succeed on a subsequent attempt will be marked as # non-flaky. Can be overridden through the `--retries` option. @@ -124,7 +127,3 @@ serial-integration = { max-threads = 1 } [[profile.default.overrides]] filter = 'test(/(^ui$|_ui|ui_)/)' test-group = 'serial-integration' - -# Skip subsystems regression benchmarks -[[profile.default.overrides]] -filter = 'not deps(/polkadot-subsystem-bench/)' From 3b6a3e90cab38ae145de1c805157aac36877bb9f Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 18:38:05 +0100 Subject: [PATCH 21/53] Skip benches --- .config/nextest.toml | 3 --- .gitlab/pipeline/test.yml | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index d01cfea1554c..1e18f8b5589c 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -9,9 +9,6 @@ dir = "target/nextest" # This section defines the default nextest profile. Custom profiles are layered # on top of the default profile. [profile.default] -# Skip subsystems regression benchmarks -filter = 'not deps(/polkadot-subsystem-bench/)' - # "retries" defines the number of times a test should be retried. If set to a # non-zero value, tests that succeed on a subsequent attempt will be marked as # non-flaky. Can be overridden through the `--retries` option. diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index f318f4516252..cebfbaa08349 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -25,6 +25,7 @@ test-linux-stable: # "upgrade_version_checks_should_work" is currently failing - | time cargo nextest run \ + --filter-expr 'not deps(/polkadot-subsystem-bench/)' \ --workspace \ --locked \ --release \ From 2fec1c17b23f5ae1e6881b7557161e3d5f993fca Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 19:00:49 +0100 Subject: [PATCH 22/53] Skip benches --- .gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index cebfbaa08349..b42d6e9f6afe 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -69,7 +69,7 @@ test-linux-stable-runtime-benchmarks: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - - time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet + - time cargo nextest run --filter-expr 'not deps(/polkadot-subsystem-bench/)' --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet # can be used to run all tests # test-linux-stable-all: From 26574329dad5bcd137a61862d253073105839dfb Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 19:02:11 +0100 Subject: [PATCH 23/53] Rename test job --- .gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index b42d6e9f6afe..1f3bece73d50 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -494,7 +494,7 @@ test-syscalls: fi allow_failure: false # this rarely triggers in practice -subsystem-bench: +subsystem-regression-tests: stage: test extends: - .docker-env From c037885880d46fce2e7d60f6f356c1869904e065 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 19:03:29 +0100 Subject: [PATCH 24/53] Fix label --- polkadot/node/subsystem-bench/src/lib/usage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs index a2f0a2114556..aaf9bb33d449 100644 --- a/polkadot/node/subsystem-bench/src/lib/usage.rs +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -38,7 +38,7 @@ impl std::fmt::Display for BenchmarkUsage { .map(|v| v.to_string()) .collect::>() .join("\n"), - format!("{:<32}{:>12}{:>12}", "CPU usage in seconds", "total", "per block").blue(), + format!("{:<32}{:>12}{:>12}", "CPU usage, seconds", "total", "per block").blue(), self.cpu_usage.iter().map(|v| v.to_string()).collect::>().join("\n") ) } From d46e9fa5237f19cb2b683a80331aa53d973abffd Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 16 Feb 2024 19:15:15 +0100 Subject: [PATCH 25/53] Update values --- .../tests/approval-voting-regression-bench.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs index 1a742846990c..ba93ef648ade 100644 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -73,12 +73,12 @@ fn approvals_no_shows() -> Vec { ); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 6900.000, 7000.000), - ("Sent to peers", 8000.000, 8100.000), + ("Received from peers", 6600.000, 7200.000), + ("Sent to peers", 7500.000, 8300.000), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 0.700, 0.800), - ("approval-voting", 1.100, 1.200), + ("approval-distribution", 0.880, 0.960), + ("approval-voting", 1.200, 1.320), ])); messages @@ -101,12 +101,12 @@ fn approvals_throughput() -> Vec { ); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 52900.000, 53000.000), - ("Sent to peers", 63500.000, 63600.000), + ("Received from peers", 50300.000, 55500.000), + ("Sent to peers", 60300.000, 66700.000), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 6.300, 6.400), - ("approval-voting", 9.400, 9.500), + ("approval-distribution", 7.500, 8.290), + ("approval-voting", 9.57, 10.580), ])); messages @@ -129,12 +129,12 @@ fn approvals_throughput_best_case() -> Vec { ); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 2900.000, 3000.000), - ("Sent to peers", 3200.000, 3300.000), + ("Received from peers", 2800.000, 3100.000), + ("Sent to peers", 3100.000, 3400.000), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 0.400, 0.500), - ("approval-voting", 0.600, 0.700), + ("approval-distribution", 0.420, 0.460), + ("approval-voting", 0.530, 0.580), ])); messages From 708e00201fbbc142638f2918324ee54c0aa5b34a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Mon, 19 Feb 2024 10:04:32 +0100 Subject: [PATCH 26/53] Update availability recovery bench values --- .../tests/availability-recovery-regression-bench.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index ef75c00b490d..a95863f349ca 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -52,7 +52,7 @@ fn main() -> Result<(), String> { ("Received from peers", 102400.000, 102500.000), ("Sent to peers", 0.000, 100.000), ])); - messages.extend(usage.check_cpu_usage(&[("availability-recovery", 4.200, 4.300)])); + messages.extend(usage.check_cpu_usage(&[("availability-recovery", 3.700, 4.100)])); if messages.is_empty() { Ok(()) From d1b36312a3d49bd212e7112bb4fda4445f40492d Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Mon, 19 Feb 2024 14:09:01 +0100 Subject: [PATCH 27/53] Use average run --- .../tests/approval-voting-regression-bench.rs | 11 +++++-- ...ilability-distribution-regression-bench.rs | 25 ++++++++++------ .../availability-recovery-regression-bench.rs | 29 +++++++++++++------ .../node/subsystem-bench/src/lib/usage.rs | 29 +++++++++++++++++++ 4 files changed, 74 insertions(+), 20 deletions(-) diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs index ba93ef648ade..185534ff49d1 100644 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -20,6 +20,8 @@ use polkadot_subsystem_bench::{ usage::BenchmarkUsage, }; +const BENCH_COUNT: usize = 10; + fn main() -> Result<(), String> { let mut messages = vec![]; @@ -50,8 +52,13 @@ fn test_configuration() -> TestConfiguration { } fn run_benchmark(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage { - let (mut env, state) = prepare_test(test_configuration(), options, false); - let usage = env.runtime().block_on(bench_approvals(test_case, &mut env, state)); + let usages: Vec = (0..BENCH_COUNT) + .map(|_| { + let (mut env, state) = prepare_test(test_configuration(), options.clone(), false); + env.runtime().block_on(bench_approvals(test_case, &mut env, state)) + }) + .collect(); + let usage = BenchmarkUsage::average(&usages); println!("{}", usage); usage } diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 82d7431a3b72..ccf6e7a5de98 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -17,8 +17,11 @@ use polkadot_subsystem_bench::{ availability::{benchmark_availability_write, prepare_test, TestDataAvailability, TestState}, configuration::{PeerLatency, TestConfiguration}, + usage::BenchmarkUsage, }; +const BENCH_COUNT: usize = 10; + fn main() -> Result<(), String> { let mut messages = vec![]; let mut config = TestConfiguration::default(); @@ -34,15 +37,19 @@ fn main() -> Result<(), String> { config.num_blocks = 3; config.generate_pov_sizes(); - let mut state = TestState::new(&config); - let (mut env, _protocol_config) = - prepare_test(config, &mut state, TestDataAvailability::Write, false); - let usage = env.runtime().block_on(benchmark_availability_write( - "data_availability_write", - &mut env, - state, - )); - + let usages: Vec = (0..BENCH_COUNT) + .map(|_| { + let mut state = TestState::new(&config); + let (mut env, _protocol_config) = + prepare_test(config.clone(), &mut state, TestDataAvailability::Write, false); + env.runtime().block_on(benchmark_availability_write( + "data_availability_write", + &mut env, + state, + )) + }) + .collect(); + let usage = BenchmarkUsage::average(&usages); println!("{}", usage); messages.extend(usage.check_network_usage(&[ diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index a95863f349ca..da0ff1e7273c 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -20,8 +20,11 @@ use polkadot_subsystem_bench::{ TestDataAvailability, TestState, }, configuration::{PeerLatency, TestConfiguration}, + usage::BenchmarkUsage, }; +const BENCH_COUNT: usize = 10; + fn main() -> Result<(), String> { let mut messages = vec![]; let options = DataAvailabilityReadOptions { fetch_from_backers: true }; @@ -37,15 +40,23 @@ fn main() -> Result<(), String> { config.connectivity = 90; config.generate_pov_sizes(); - let mut state = TestState::new(&config); - let (mut env, _protocol_config) = - prepare_test(config, &mut state, TestDataAvailability::Read(options), false); - let usage = env.runtime().block_on(benchmark_availability_read( - "data_availability_read", - &mut env, - state, - )); - + let usages: Vec = (0..BENCH_COUNT) + .map(|_| { + let mut state = TestState::new(&config); + let (mut env, _protocol_config) = prepare_test( + config.clone(), + &mut state, + TestDataAvailability::Read(options.clone()), + false, + ); + env.runtime().block_on(benchmark_availability_read( + "data_availability_read", + &mut env, + state, + )) + }) + .collect(); + let usage = BenchmarkUsage::average(&usages); println!("{}", usage); messages.extend(usage.check_network_usage(&[ diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs index aaf9bb33d449..a23774da378e 100644 --- a/polkadot/node/subsystem-bench/src/lib/usage.rs +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -18,6 +18,7 @@ use colored::Colorize; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; #[derive(Debug, Serialize, Deserialize)] pub struct BenchmarkUsage { @@ -45,6 +46,18 @@ impl std::fmt::Display for BenchmarkUsage { } impl BenchmarkUsage { + pub fn average(usages: &[Self]) -> Self { + let all_network_usages: Vec<&ResourceUsage> = + usages.iter().flat_map(|v| &v.network_usage).collect(); + let all_cpu_usage: Vec<&ResourceUsage> = usages.iter().flat_map(|v| &v.cpu_usage).collect(); + + Self { + benchmark_name: usages.get(0).map(|v| v.benchmark_name.clone()).unwrap_or_default(), + network_usage: ResourceUsage::average_by_resource_name(&all_network_usages), + cpu_usage: ResourceUsage::average_by_resource_name(&all_cpu_usage), + } + } + pub fn check_network_usage(&self, checks: &[ResourceUsageCheck]) -> Vec { check_usage(&self.benchmark_name, &self.network_usage, checks) } @@ -99,4 +112,20 @@ impl std::fmt::Display for ResourceUsage { } } +impl ResourceUsage { + fn average_by_resource_name(usages: &[&Self]) -> Vec { + let mut by_name: HashMap> = Default::default(); + for usage in usages { + by_name.entry(usage.resource_name.clone()).or_default().push(usage); + } + let mut average = vec![]; + for (resource_name, values) in by_name { + let total = values.iter().map(|v| v.total).sum::() / values.len() as f64; + let per_block = values.iter().map(|v| v.per_block).sum::() / values.len() as f64; + average.push(Self { resource_name, total, per_block }); + } + average + } +} + type ResourceUsageCheck<'a> = (&'a str, f64, f64); From 282745a84afcb200616e110e6267c3ac9e94a533 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Mon, 19 Feb 2024 14:20:02 +0100 Subject: [PATCH 28/53] Address clippy errors --- polkadot/node/subsystem-bench/src/lib/usage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs index a23774da378e..8648e6e07d77 100644 --- a/polkadot/node/subsystem-bench/src/lib/usage.rs +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -52,7 +52,7 @@ impl BenchmarkUsage { let all_cpu_usage: Vec<&ResourceUsage> = usages.iter().flat_map(|v| &v.cpu_usage).collect(); Self { - benchmark_name: usages.get(0).map(|v| v.benchmark_name.clone()).unwrap_or_default(), + benchmark_name: usages.first().map(|v| v.benchmark_name.clone()).unwrap_or_default(), network_usage: ResourceUsage::average_by_resource_name(&all_network_usages), cpu_usage: ResourceUsage::average_by_resource_name(&all_cpu_usage), } From f6df907e890dd98fc03114f18ed2b964689da2af Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Mon, 19 Feb 2024 15:20:49 +0100 Subject: [PATCH 29/53] Update README --- polkadot/node/subsystem-bench/README.md | 138 +++++++++++------------- 1 file changed, 65 insertions(+), 73 deletions(-) diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index e090a0392cb7..86cfb80b613e 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -1,6 +1,6 @@ # Subsystem benchmark client -Run parachain consensus stress and performance tests on your development machine. +Run parachain consensus stress and performance tests on your development machine or in CI. ## Motivation @@ -95,39 +95,16 @@ If you are running the servers in Docker, use the following URLs: Follow [this guide](https://grafana.com/docs/grafana/latest/dashboards/manage-dashboards/#export-and-import-dashboards) to import the dashboards from the repository `grafana` folder. -## How to run a test - -To run a test, you need to first choose a test objective. Currently, we support the following: - -``` -target/testnet/subsystem-bench --help -The almighty Subsystem Benchmark Tool™️ - -Usage: subsystem-bench [OPTIONS] - -Commands: - data-availability-read Benchmark availability recovery strategies +### Standard test options ``` +$ subsystem-bench --help +Usage: subsystem-bench [OPTIONS] -Note: `test-sequence` is a special test objective that wraps up an arbitrary number of test objectives. It is tipically - used to run a suite of tests defined in a `yaml` file like in this [example](examples/availability_read.yaml). +Arguments: + Path to the test sequence configuration file -### Standard test options - -``` - --network The type of network to be emulated [default: ideal] [possible values: ideal, healthy, - degraded] - --n-cores Number of cores to fetch availability for [default: 100] - --n-validators Number of validators to fetch chunks from [default: 500] - --min-pov-size The minimum pov size in KiB [default: 5120] - --max-pov-size The maximum pov size bytes [default: 5120] - -n, --num-blocks The number of blocks the test is going to run [default: 1] - -p, --peer-bandwidth The bandwidth of emulated remote peers in KiB - -b, --bandwidth The bandwidth of our node in KiB - --connectivity Emulated peer connection ratio [0-100] - --peer-mean-latency Mean remote peer latency in milliseconds [0-5000] - --peer-latency-std-dev Remote peer latency standard deviation +Options: --profile Enable CPU Profiling with Pyroscope --pyroscope-url Pyroscope Server URL [default: http://localhost:4040] --pyroscope-sample-rate Pyroscope Sample Rate [default: 113] @@ -135,27 +112,17 @@ Note: `test-sequence` is a special test objective that wraps up an arbitrary num -h, --help Print help ``` -These apply to all test objectives, except `test-sequence` which relies on the values being specified in a file. - -### Test objectives - -Each test objective can have it's specific configuration options, in contrast with the standard test options. +## How to run a test -For `data-availability-read` the recovery strategy to be used is configurable. +To run a test, you need to use a path to a test objective: ``` -target/testnet/subsystem-bench data-availability-read --help -Benchmark availability recovery strategies - -Usage: subsystem-bench data-availability-read [OPTIONS] - -Options: - -f, --fetch-from-backers Turbo boost AD Read by fetching the full availability datafrom backers first. Saves CPU - as we don't need to re-construct from chunks. Tipically this is only faster if nodes - have enough bandwidth - -h, --help Print help +target/testnet/subsystem-bench polkadot/node/subsystem-bench/examples/availability_read.yaml ``` +Note: test objectives may be wrapped up into a test sequence. +It is tipically used to run a suite of tests like in this [example](examples/availability_read.yaml). + ### Understanding the test configuration A single test configuration `TestConfiguration` struct applies to a single run of a certain test objective. @@ -175,36 +142,61 @@ the test is started. ### Example run -Let's run an availabilty read test which will recover availability for 10 cores with max PoV size on a 500 +Let's run an availabilty read test which will recover availability for 200 cores with max PoV size on a 1000 node validator network. ``` - target/testnet/subsystem-bench --n-cores 10 data-availability-read -[2023-11-28T09:01:59Z INFO subsystem_bench::core::display] n_validators = 500, n_cores = 10, pov_size = 5120 - 5120, - latency = None -[2023-11-28T09:01:59Z INFO subsystem-bench::availability] Generating template candidate index=0 pov_size=5242880 -[2023-11-28T09:01:59Z INFO subsystem-bench::availability] Created test environment. -[2023-11-28T09:01:59Z INFO subsystem-bench::availability] Pre-generating 10 candidates. -[2023-11-28T09:02:01Z INFO subsystem-bench::core] Initializing network emulation for 500 peers. -[2023-11-28T09:02:01Z INFO substrate_prometheus_endpoint] 〽️ Prometheus exporter started at 127.0.0.1:9999 -[2023-11-28T09:02:01Z INFO subsystem-bench::availability] Current block 1/1 -[2023-11-28T09:02:01Z INFO subsystem_bench::availability] 10 recoveries pending -[2023-11-28T09:02:04Z INFO subsystem_bench::availability] Block time 3231ms -[2023-11-28T09:02:04Z INFO subsystem-bench::availability] Sleeping till end of block (2768ms) -[2023-11-28T09:02:07Z INFO subsystem_bench::availability] All blocks processed in 6001ms -[2023-11-28T09:02:07Z INFO subsystem_bench::availability] Throughput: 51200 KiB/block -[2023-11-28T09:02:07Z INFO subsystem_bench::availability] Block time: 6001 ms -[2023-11-28T09:02:07Z INFO subsystem_bench::availability] - - Total received from network: 66 MiB - Total sent to network: 58 KiB - Total subsystem CPU usage 4.16s - CPU usage per block 4.16s - Total test environment CPU usage 0.00s - CPU usage per block 0.00s +target/testnet/subsystem-bench polkadot/node/subsystem-bench/examples/availability_write.yaml +[2024-02-19T14:10:32.981Z INFO subsystem_bench] Sequence contains 1 step(s) +[2024-02-19T14:10:32.981Z INFO subsystem-bench::cli] Step 1/1 +[2024-02-19T14:10:32.981Z INFO subsystem-bench::cli] [objective = DataAvailabilityWrite] n_validators = 1000, n_cores = 200, pov_size = 5120 - 5120, connectivity = 75, latency = Some(PeerLatency { mean_latency_ms: 30, std_dev: 2.0 }) +[2024-02-19T14:10:32.982Z INFO subsystem-bench::availability] Generating template candidate index=0 pov_size=5242880 +[2024-02-19T14:10:33.106Z INFO subsystem-bench::availability] Created test environment. +[2024-02-19T14:10:33.106Z INFO subsystem-bench::availability] Pre-generating 600 candidates. +[2024-02-19T14:10:34.096Z INFO subsystem-bench::network] Initializing emulation for a 1000 peer network. +[2024-02-19T14:10:34.096Z INFO subsystem-bench::network] connectivity 75%, latency Some(PeerLatency { mean_latency_ms: 30, std_dev: 2.0 }) +[2024-02-19T14:10:34.098Z INFO subsystem-bench::network] Network created, connected validator count 749 +[2024-02-19T14:10:34.099Z INFO subsystem-bench::availability] Seeding availability store with candidates ... +[2024-02-19T14:10:34.100Z INFO substrate_prometheus_endpoint] 〽️ Prometheus exporter started at 127.0.0.1:9999 +[2024-02-19T14:10:34.387Z INFO subsystem-bench::availability] Done +[2024-02-19T14:10:34.387Z INFO subsystem-bench::availability] Current block #1 +[2024-02-19T14:10:34.389Z INFO subsystem-bench::availability] Waiting for all emulated peers to receive their chunk from us ... +[2024-02-19T14:10:34.625Z INFO subsystem-bench::availability] All chunks received in 237ms +[2024-02-19T14:10:34.626Z INFO polkadot_subsystem_bench::availability] Waiting for 749 bitfields to be received and processed +[2024-02-19T14:10:35.710Z INFO subsystem-bench::availability] All bitfields processed +[2024-02-19T14:10:35.710Z INFO subsystem-bench::availability] All work for block completed in 1322ms +[2024-02-19T14:10:35.710Z INFO subsystem-bench::availability] Current block #2 +[2024-02-19T14:10:35.712Z INFO subsystem-bench::availability] Waiting for all emulated peers to receive their chunk from us ... +[2024-02-19T14:10:35.947Z INFO subsystem-bench::availability] All chunks received in 236ms +[2024-02-19T14:10:35.947Z INFO polkadot_subsystem_bench::availability] Waiting for 749 bitfields to be received and processed +[2024-02-19T14:10:37.038Z INFO subsystem-bench::availability] All bitfields processed +[2024-02-19T14:10:37.038Z INFO subsystem-bench::availability] All work for block completed in 1328ms +[2024-02-19T14:10:37.039Z INFO subsystem-bench::availability] Current block #3 +[2024-02-19T14:10:37.040Z INFO subsystem-bench::availability] Waiting for all emulated peers to receive their chunk from us ... +[2024-02-19T14:10:37.276Z INFO subsystem-bench::availability] All chunks received in 237ms +[2024-02-19T14:10:37.276Z INFO polkadot_subsystem_bench::availability] Waiting for 749 bitfields to be received and processed +[2024-02-19T14:10:38.362Z INFO subsystem-bench::availability] All bitfields processed +[2024-02-19T14:10:38.362Z INFO subsystem-bench::availability] All work for block completed in 1323ms +[2024-02-19T14:10:38.362Z INFO subsystem-bench::availability] All blocks processed in 3974ms +[2024-02-19T14:10:38.362Z INFO subsystem-bench::availability] Avg block time: 1324 ms +[2024-02-19T14:10:38.362Z INFO parachain::availability-store] received `Conclude` signal, exiting +[2024-02-19T14:10:38.362Z INFO parachain::bitfield-distribution] Conclude +[2024-02-19T14:10:38.362Z INFO subsystem-bench::network] Downlink channel closed, network interface task exiting + +polkadot/node/subsystem-bench/examples/availability_write.yaml #1 DataAvailabilityWrite + +Network usage, KiB total per block +Received from peers 12922.000 4307.333 +Sent to peers 47705.000 15901.667 + +CPU usage, seconds total per block +availability-distribution 0.045 0.015 +bitfield-distribution 0.104 0.035 +availability-store 0.304 0.101 +Test environment 3.213 1.071 ``` -`Block time` in the context of `data-availability-read` has a different meaning. It measures the amount of time it +`Block time` in the current context has a different meaning. It measures the amount of time it took the subsystem to finish processing all of the messages sent in the context of the current test block. ### Test logs @@ -234,7 +226,7 @@ benchmark results into account. A report is saved in a file `cachegrind_report.t Example run results: ``` -$ target/testnet/subsystem-bench --n-cores 10 --cache-misses data-availability-read +$ target/testnet/subsystem-bench --cache-misses cache-misses-data-availability-read.yaml $ cat cachegrind_report.txt I refs: 64,622,081,485 I1 misses: 3,018,168 @@ -275,7 +267,7 @@ happy and negative scenarios (low bandwidth, network errors and low connectivity To faster write a new test objective you need to use some higher level wrappers and logic: `TestEnvironment`, `TestConfiguration`, `TestAuthorities`, `NetworkEmulator`. To create the `TestEnvironment` you will -need to also build an `Overseer`, but that should be easy using the mockups for subsystems in`core::mock`. +need to also build an `Overseer`, but that should be easy using the mockups for subsystems in `mock`. ### Mocking From f0aea37698c8a2cc968908ee333f4fa6d1e42c5f Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Mon, 19 Feb 2024 16:20:33 +0100 Subject: [PATCH 30/53] Update values --- .gitlab/pipeline/test.yml | 2 +- .../tests/approval-voting-regression-bench.rs | 20 +++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 1f3bece73d50..4d4ef8005666 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -500,7 +500,7 @@ subsystem-regression-tests: - .docker-env - .common-refs script: - - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks -- --nocapture + # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks -- --nocapture - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks -- --nocapture - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks -- --nocapture tags: diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs index 185534ff49d1..72a6faf8d7a0 100644 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -81,11 +81,11 @@ fn approvals_no_shows() -> Vec { messages.extend(usage.check_network_usage(&[ ("Received from peers", 6600.000, 7200.000), - ("Sent to peers", 7500.000, 8300.000), + ("Sent to peers", 7600.000, 8500.00), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 0.880, 0.960), - ("approval-voting", 1.200, 1.320), + ("approval-distribution", 0.940, 1.040), + ("approval-voting", 1.300, 1.440), ])); messages @@ -108,12 +108,12 @@ fn approvals_throughput() -> Vec { ); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 50300.000, 55500.000), + ("Received from peers", 50200.000, 55500.000), ("Sent to peers", 60300.000, 66700.000), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 7.500, 8.290), - ("approval-voting", 9.57, 10.580), + ("approval-distribution", 8.210, 9.080), + ("approval-voting", 10.530, 11.640), ])); messages @@ -135,13 +135,17 @@ fn approvals_throughput_best_case() -> Vec { }, ); + // CPU usage, seconds total per block + // approval-distribution 4.799 0.480 + // approval-voting 6.248 0.625 + // Test environment 3.057 0.306 messages.extend(usage.check_network_usage(&[ ("Received from peers", 2800.000, 3100.000), ("Sent to peers", 3100.000, 3400.000), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 0.420, 0.460), - ("approval-voting", 0.530, 0.580), + ("approval-distribution", 0.460, 0.500), + ("approval-voting", 0.590, 0.660), ])); messages From ff7dac31324a0e34495d3654312d3990736b363c Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Mon, 19 Feb 2024 17:59:54 +0100 Subject: [PATCH 31/53] Update values --- .../tests/approval-voting-regression-bench.rs | 4 ---- .../availability-distribution-regression-bench.rs | 10 +++++----- .../tests/availability-recovery-regression-bench.rs | 4 ++-- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs index 72a6faf8d7a0..b5c842e62115 100644 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -135,10 +135,6 @@ fn approvals_throughput_best_case() -> Vec { }, ); - // CPU usage, seconds total per block - // approval-distribution 4.799 0.480 - // approval-voting 6.248 0.625 - // Test environment 3.057 0.306 messages.extend(usage.check_network_usage(&[ ("Received from peers", 2800.000, 3100.000), ("Sent to peers", 3100.000, 3400.000), diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index ccf6e7a5de98..2858fb2b4596 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -53,13 +53,13 @@ fn main() -> Result<(), String> { println!("{}", usage); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 4300.000, 4400.000), - ("Sent to peers", 15900.000, 16000.000), + ("Received from peers", 4100.000, 4500.000), + ("Sent to peers", 15100.000, 16700.000), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.000, 0.100), - ("bitfield-distribution", 0.000, 0.100), - ("availability-store", 0.100, 0.200), + ("availability-distribution", 0.022, 0.024), + ("bitfield-distribution", 0.070, 0.080), + ("availability-store", 0.160, 0.180), ])); if messages.is_empty() { diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index da0ff1e7273c..5230ed3361f7 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -60,8 +60,8 @@ fn main() -> Result<(), String> { println!("{}", usage); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 102400.000, 102500.000), - ("Sent to peers", 0.000, 100.000), + ("Received from peers", 97300.000, 107500.000), + ("Sent to peers", 0.320, 0.250), ])); messages.extend(usage.check_cpu_usage(&[("availability-recovery", 3.700, 4.100)])); From e1e6f2d1ba7b49acf7365e18f58ac44f3cb0af95 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 20 Feb 2024 11:43:31 +0100 Subject: [PATCH 32/53] Fix md line length --- polkadot/node/subsystem-bench/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 86cfb80b613e..26ec29c66745 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -32,7 +32,8 @@ a local Grafana/Prometheus stack is needed. ### Run Prometheus, Pyroscope and Graphana in Docker -If docker is not usable, then follow the next sections to manually install Prometheus, Pyroscope and Graphana on your machine. +If docker is not usable, then follow the next sections to manually install Prometheus, Pyroscope and Graphana +on your machine. ```bash cd polkadot/node/subsystem-bench/docker From 68ec96416f4600b517a393860f2b416dc2e423e2 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 20 Feb 2024 11:45:35 +0100 Subject: [PATCH 33/53] Update a wrong range --- .../tests/availability-recovery-regression-bench.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index 5230ed3361f7..b5298b7931cd 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -61,7 +61,7 @@ fn main() -> Result<(), String> { messages.extend(usage.check_network_usage(&[ ("Received from peers", 97300.000, 107500.000), - ("Sent to peers", 0.320, 0.250), + ("Sent to peers", 0.250, 0.320), ])); messages.extend(usage.check_cpu_usage(&[("availability-recovery", 3.700, 4.100)])); From f03f4a742fd63d7c88823bc62283b7312a93349a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 20 Feb 2024 12:23:47 +0100 Subject: [PATCH 34/53] Disable linting in code block --- polkadot/node/subsystem-bench/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/polkadot/node/subsystem-bench/README.md b/polkadot/node/subsystem-bench/README.md index 26ec29c66745..3aac2810ad58 100644 --- a/polkadot/node/subsystem-bench/README.md +++ b/polkadot/node/subsystem-bench/README.md @@ -146,6 +146,8 @@ the test is started. Let's run an availabilty read test which will recover availability for 200 cores with max PoV size on a 1000 node validator network. + + ``` target/testnet/subsystem-bench polkadot/node/subsystem-bench/examples/availability_write.yaml [2024-02-19T14:10:32.981Z INFO subsystem_bench] Sequence contains 1 step(s) @@ -197,6 +199,8 @@ availability-store 0.304 0.101 Test environment 3.213 1.071 ``` + + `Block time` in the current context has a different meaning. It measures the amount of time it took the subsystem to finish processing all of the messages sent in the context of the current test block. @@ -226,6 +230,7 @@ Since the execution will be very slow, it's recommended not to run it together w benchmark results into account. A report is saved in a file `cachegrind_report.txt`. Example run results: + ``` $ target/testnet/subsystem-bench --cache-misses cache-misses-data-availability-read.yaml $ cat cachegrind_report.txt From 4abf5d8543b7930b90be8c3af065756e86fa19c9 Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:39:42 +0100 Subject: [PATCH 35/53] Update .gitlab/pipeline/test.yml --- .gitlab/pipeline/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 4d4ef8005666..971d08950d19 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -499,6 +499,7 @@ subsystem-regression-tests: extends: - .docker-env - .common-refs + - .run-immediately script: # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks -- --nocapture - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks -- --nocapture From 9df54eca5f7cc1276c59dc5012721c418005b42b Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 20 Feb 2024 18:03:18 +0100 Subject: [PATCH 36/53] Update values --- .../tests/availability-recovery-regression-bench.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index b5298b7931cd..f45a268f991b 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -61,9 +61,9 @@ fn main() -> Result<(), String> { messages.extend(usage.check_network_usage(&[ ("Received from peers", 97300.000, 107500.000), - ("Sent to peers", 0.250, 0.320), + ("Sent to peers", 0.320, 0.350), ])); - messages.extend(usage.check_cpu_usage(&[("availability-recovery", 3.700, 4.100)])); + messages.extend(usage.check_cpu_usage(&[("availability-recovery", 3.660, 4.040)])); if messages.is_empty() { Ok(()) From 3acfe1861d91e950b0a33f42d8483f35a92eb16b Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 20 Feb 2024 18:24:34 +0100 Subject: [PATCH 37/53] Update values --- .../tests/availability-distribution-regression-bench.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 2858fb2b4596..8956a143c5b1 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -53,13 +53,13 @@ fn main() -> Result<(), String> { println!("{}", usage); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 4100.000, 4500.000), + ("Received from peers", 4110.000, 4550.000), ("Sent to peers", 15100.000, 16700.000), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.022, 0.024), - ("bitfield-distribution", 0.070, 0.080), - ("availability-store", 0.160, 0.180), + ("availability-distribution", 0.024, 0.026), + ("bitfield-distribution", 0.082, 0.090), + ("availability-store", 0.168, 0.186), ])); if messages.is_empty() { From 386896295bbf1305f6e1257db4cf96953ee1390b Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 14:05:44 +0100 Subject: [PATCH 38/53] Try warming up --- .gitlab/pipeline/test.yml | 6 +- .../availability-recovery-regression-bench.rs | 64 ++++++++++++------- .../node/subsystem-bench/src/lib/usage.rs | 30 +++++++-- 3 files changed, 71 insertions(+), 29 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 971d08950d19..fc8495046e85 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -501,8 +501,8 @@ subsystem-regression-tests: - .common-refs - .run-immediately script: - # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks -- --nocapture - - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks -- --nocapture - - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks -- --nocapture + # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks + - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks + # - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks tags: - benchmark diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index f45a268f991b..8e9b6519a38b 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -23,7 +23,9 @@ use polkadot_subsystem_bench::{ usage::BenchmarkUsage, }; -const BENCH_COUNT: usize = 10; +const BENCH_COUNT: usize = 3; +const WARM_UP_COUNT: usize = 10; +const WARM_UP_PRECISION: f64 = 0.01; fn main() -> Result<(), String> { let mut messages = vec![]; @@ -40,30 +42,14 @@ fn main() -> Result<(), String> { config.connectivity = 90; config.generate_pov_sizes(); - let usages: Vec = (0..BENCH_COUNT) - .map(|_| { - let mut state = TestState::new(&config); - let (mut env, _protocol_config) = prepare_test( - config.clone(), - &mut state, - TestDataAvailability::Read(options.clone()), - false, - ); - env.runtime().block_on(benchmark_availability_read( - "data_availability_read", - &mut env, - state, - )) - }) - .collect(); - let usage = BenchmarkUsage::average(&usages); - println!("{}", usage); + warm_up(config.clone(), options.clone())?; + let usage = benchmark(config.clone(), options.clone()); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 97300.000, 107500.000), - ("Sent to peers", 0.320, 0.350), + ("Received from peers", 102400.000, 0.05), + ("Sent to peers", 0.335, 0.05), ])); - messages.extend(usage.check_cpu_usage(&[("availability-recovery", 3.660, 4.040)])); + messages.extend(usage.check_cpu_usage(&[("availability-recovery", 3.850, 0.05)])); if messages.is_empty() { Ok(()) @@ -72,3 +58,37 @@ fn main() -> Result<(), String> { Err("Regressions found".to_string()) } } + +fn warm_up(config: TestConfiguration, options: DataAvailabilityReadOptions) -> Result<(), String> { + println!("Warming up..."); + let mut prev_run: Option = None; + for _ in 0..WARM_UP_COUNT { + let curr = run(config.clone(), options.clone()); + if let Some(ref prev) = prev_run { + let diff = curr.cpu_usage_diff(prev, "availability-recovery").expect("Must exist"); + if diff < WARM_UP_PRECISION { + return Ok(()) + } + } + prev_run = Some(curr); + } + + Err("Can't warm up".to_string()) +} + +fn benchmark(config: TestConfiguration, options: DataAvailabilityReadOptions) -> BenchmarkUsage { + println!("Benchmarking..."); + let usages: Vec = + (0..BENCH_COUNT).map(|_| run(config, options)).collect(); + let usage = BenchmarkUsage::average(&usages); + println!("{}", usage); + usage +} + +fn run(config: TestConfiguration, options: DataAvailabilityReadOptions) -> BenchmarkUsage { + let mut state = TestState::new(&config); + let (mut env, _protocol_config) = + prepare_test(config.clone(), &mut state, TestDataAvailability::Read(options), false); + env.runtime() + .block_on(benchmark_availability_read("data_availability_read", &mut env, state)) +} diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs index 8648e6e07d77..56f39bbe70f8 100644 --- a/polkadot/node/subsystem-bench/src/lib/usage.rs +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -65,6 +65,23 @@ impl BenchmarkUsage { pub fn check_cpu_usage(&self, checks: &[ResourceUsageCheck]) -> Vec { check_usage(&self.benchmark_name, &self.cpu_usage, checks) } + + + pub fn cpu_usage_diff(&self, other: &Self, resource_name: &str) -> Option { + let self_res = self + .cpu_usage + .iter() + .find(|v| v.resource_name == resource_name); + let other_res = other + .cpu_usage + .iter() + .find(|v| v.resource_name == resource_name); + + match (self_res, other_res) { + (Some(self_res), Some(other_res)) => Some(self_res.diff(other_res)), + _ => None + } + } } fn check_usage( @@ -83,15 +100,16 @@ fn check_usage( fn check_resource_usage( usage: &[ResourceUsage], - (resource_name, min, max): &ResourceUsageCheck, + (resource_name, base, precision): &ResourceUsageCheck, ) -> Option { if let Some(usage) = usage.iter().find(|v| v.resource_name == *resource_name) { - if usage.per_block >= *min && usage.per_block < *max { + let diff = (base - usage.per_block).abs() / base; + if diff < *precision { None } else { Some(format!( - "The resource `{}` is expected to be in the range of {}..{}, but the value is {}", - resource_name, min, max, usage.per_block + "The resource `{}` is expected to be equal to {} with a precision {}, but the current value is {}", + resource_name, base, precision, usage.per_block )) } } else { @@ -126,6 +144,10 @@ impl ResourceUsage { } average } + + fn diff(&self, other: &Self) -> f64 { + (self.per_block - other.per_block).abs() / self.per_block + } } type ResourceUsageCheck<'a> = (&'a str, f64, f64); From e4f3c8798317df673d921e7363b0cfbe5c122ad0 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 14:07:37 +0100 Subject: [PATCH 39/53] Fix format --- .../tests/availability-recovery-regression-bench.rs | 3 +-- polkadot/node/subsystem-bench/src/lib/usage.rs | 13 +++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index 8e9b6519a38b..f8f5596ec7ec 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -78,8 +78,7 @@ fn warm_up(config: TestConfiguration, options: DataAvailabilityReadOptions) -> R fn benchmark(config: TestConfiguration, options: DataAvailabilityReadOptions) -> BenchmarkUsage { println!("Benchmarking..."); - let usages: Vec = - (0..BENCH_COUNT).map(|_| run(config, options)).collect(); + let usages: Vec = (0..BENCH_COUNT).map(|_| run(config, options)).collect(); let usage = BenchmarkUsage::average(&usages); println!("{}", usage); usage diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs index 56f39bbe70f8..b83ef7d98d91 100644 --- a/polkadot/node/subsystem-bench/src/lib/usage.rs +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -66,20 +66,13 @@ impl BenchmarkUsage { check_usage(&self.benchmark_name, &self.cpu_usage, checks) } - pub fn cpu_usage_diff(&self, other: &Self, resource_name: &str) -> Option { - let self_res = self - .cpu_usage - .iter() - .find(|v| v.resource_name == resource_name); - let other_res = other - .cpu_usage - .iter() - .find(|v| v.resource_name == resource_name); + let self_res = self.cpu_usage.iter().find(|v| v.resource_name == resource_name); + let other_res = other.cpu_usage.iter().find(|v| v.resource_name == resource_name); match (self_res, other_res) { (Some(self_res), Some(other_res)) => Some(self_res.diff(other_res)), - _ => None + _ => None, } } } From f55ae88f9099b92b37978adecd4b81a69f682d4a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 14:22:08 +0100 Subject: [PATCH 40/53] Address clippy errors --- .../tests/availability-recovery-regression-bench.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index f8f5596ec7ec..fbd08acbf012 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -78,7 +78,8 @@ fn warm_up(config: TestConfiguration, options: DataAvailabilityReadOptions) -> R fn benchmark(config: TestConfiguration, options: DataAvailabilityReadOptions) -> BenchmarkUsage { println!("Benchmarking..."); - let usages: Vec = (0..BENCH_COUNT).map(|_| run(config, options)).collect(); + let usages: Vec = + (0..BENCH_COUNT).map(|_| run(config.clone(), options.clone())).collect(); let usage = BenchmarkUsage::average(&usages); println!("{}", usage); usage From 222f79180a6c7d7d37a824a17b50cea3e9a510f8 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 14:45:04 +0100 Subject: [PATCH 41/53] Warm up availability-distribution-regression-bench --- ...ilability-distribution-regression-bench.rs | 71 +++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 8956a143c5b1..126f1ae76407 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -20,7 +20,9 @@ use polkadot_subsystem_bench::{ usage::BenchmarkUsage, }; -const BENCH_COUNT: usize = 10; +const BENCH_COUNT: usize = 3; +const WARM_UP_COUNT: usize = 10; +const WARM_UP_PRECISION: f64 = 0.01; fn main() -> Result<(), String> { let mut messages = vec![]; @@ -37,29 +39,17 @@ fn main() -> Result<(), String> { config.num_blocks = 3; config.generate_pov_sizes(); - let usages: Vec = (0..BENCH_COUNT) - .map(|_| { - let mut state = TestState::new(&config); - let (mut env, _protocol_config) = - prepare_test(config.clone(), &mut state, TestDataAvailability::Write, false); - env.runtime().block_on(benchmark_availability_write( - "data_availability_write", - &mut env, - state, - )) - }) - .collect(); - let usage = BenchmarkUsage::average(&usages); - println!("{}", usage); + warm_up(config.clone())?; + let usage = benchmark(config.clone()); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 4110.000, 4550.000), - ("Sent to peers", 15100.000, 16700.000), + ("Received from peers", 4330.0, 0.05), + ("Sent to peers", 15900.0, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.024, 0.026), - ("bitfield-distribution", 0.082, 0.090), - ("availability-store", 0.168, 0.186), + ("availability-distribution", 0.025, 0.05), + ("bitfield-distribution", 0.086, 0.05), + ("availability-store", 0.177, 0.05), ])); if messages.is_empty() { @@ -69,3 +59,44 @@ fn main() -> Result<(), String> { Err("Regressions found".to_string()) } } + +fn warm_up(config: TestConfiguration) -> Result<(), String> { + println!("Warming up..."); + let mut prev_run: Option = None; + for _ in 0..WARM_UP_COUNT { + let curr = run(config.clone()); + if let Some(ref prev) = prev_run { + let av_distr_diff = + curr.cpu_usage_diff(prev, "availability-distribution").expect("Must exist"); + let bitf_distr_diff = + curr.cpu_usage_diff(prev, "bitfield-distribution").expect("Must exist"); + let av_store_diff = + curr.cpu_usage_diff(prev, "availability-store").expect("Must exist"); + if av_distr_diff < WARM_UP_PRECISION && + bitf_distr_diff < WARM_UP_PRECISION && + av_store_diff < WARM_UP_PRECISION + { + return Ok(()) + } + } + prev_run = Some(curr); + } + + Err("Can't warm up".to_string()) +} + +fn benchmark(config: TestConfiguration) -> BenchmarkUsage { + println!("Benchmarking..."); + let usages: Vec = (0..BENCH_COUNT).map(|_| run(config.clone())).collect(); + let usage = BenchmarkUsage::average(&usages); + println!("{}", usage); + usage +} + +fn run(config: TestConfiguration) -> BenchmarkUsage { + let mut state = TestState::new(&config); + let (mut env, _protocol_config) = + prepare_test(config.clone(), &mut state, TestDataAvailability::Write, false); + env.runtime() + .block_on(benchmark_availability_write("data_availability_write", &mut env, state)) +} From ddf440876e591beaeb3a95e440b5a579f951048c Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 14:51:16 +0100 Subject: [PATCH 42/53] Run benches --- .gitlab/pipeline/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index fc8495046e85..b89b137a48a4 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -502,7 +502,7 @@ subsystem-regression-tests: - .run-immediately script: # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks - - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks - # - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks + # - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks + - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks tags: - benchmark From 4f40d1f4b3b30a64d653c83092375f758d861b1a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 15:02:03 +0100 Subject: [PATCH 43/53] Update warm up --- .../tests/availability-distribution-regression-bench.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 126f1ae76407..70c8b60995a6 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -21,7 +21,7 @@ use polkadot_subsystem_bench::{ }; const BENCH_COUNT: usize = 3; -const WARM_UP_COUNT: usize = 10; +const WARM_UP_COUNT: usize = 20; const WARM_UP_PRECISION: f64 = 0.01; fn main() -> Result<(), String> { From 0e1b59e70b3e8c2b187bc68579a63e4b2360d7f0 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 16:11:52 +0100 Subject: [PATCH 44/53] Run benchmarks --- .gitlab/pipeline/test.yml | 4 +- .../tests/approval-voting-regression-bench.rs | 162 ++++++++++-------- ...ilability-distribution-regression-bench.rs | 4 +- 3 files changed, 98 insertions(+), 72 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index b89b137a48a4..e5028f9ab5eb 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -501,8 +501,8 @@ subsystem-regression-tests: - .common-refs - .run-immediately script: - # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks + - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks # - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks - - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks + # - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks tags: - benchmark diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs index b5c842e62115..ca0bcfbc8564 100644 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -20,14 +20,16 @@ use polkadot_subsystem_bench::{ usage::BenchmarkUsage, }; -const BENCH_COUNT: usize = 10; +const BENCH_COUNT: usize = 3; +const WARM_UP_COUNT: usize = 20; +const WARM_UP_PRECISION: f64 = 0.01; fn main() -> Result<(), String> { let mut messages = vec![]; - messages.extend(approvals_no_shows()); - messages.extend(approvals_throughput()); - messages.extend(approvals_throughput_best_case()); + messages.extend(approvals_no_shows()?); + // messages.extend(approvals_throughput()?); + // messages.extend(approvals_throughput_best_case()?); if messages.is_empty() { Ok(()) @@ -51,98 +53,122 @@ fn test_configuration() -> TestConfiguration { config } -fn run_benchmark(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage { - let usages: Vec = (0..BENCH_COUNT) - .map(|_| { - let (mut env, state) = prepare_test(test_configuration(), options.clone(), false); - env.runtime().block_on(bench_approvals(test_case, &mut env, state)) - }) - .collect(); +fn warm_up(test_case: &str, options: ApprovalsOptions) -> Result<(), String> { + println!("Warming up..."); + let mut prev_run: Option = None; + for _ in 0..WARM_UP_COUNT { + let curr = run(test_case, options.clone()); + if let Some(ref prev) = prev_run { + let ap_distr_diff = + curr.cpu_usage_diff(prev, "approval-distribution").expect("Must exist"); + let ap_vot_diff = curr.cpu_usage_diff(prev, "approval-voting").expect("Must exist"); + if ap_distr_diff < WARM_UP_PRECISION && ap_vot_diff < WARM_UP_PRECISION { + return Ok(()) + } + } + prev_run = Some(curr); + } + + Err("Can't warm up".to_string()) +} + +fn benchmark(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage { + println!("Benchmarking..."); + let usages: Vec = + (0..BENCH_COUNT).map(|_| run(test_case, options.clone())).collect(); let usage = BenchmarkUsage::average(&usages); println!("{}", usage); usage } -fn approvals_no_shows() -> Vec { +fn run(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage { + let (mut env, state) = prepare_test(test_configuration(), options.clone(), false); + env.runtime().block_on(bench_approvals(test_case, &mut env, state)) +} + +fn approvals_no_shows() -> Result, String> { let mut messages = vec![]; - let usage = run_benchmark( - "approvals_no_shows", - ApprovalsOptions { - coalesce_mean: 3.0, - coalesce_std_dev: 1.0, - enable_assignments_v2: true, - last_considered_tranche: 89, - stop_when_approved: true, - coalesce_tranche_diff: 12, - num_no_shows_per_candidate: 10, - workdir_prefix: "/tmp".to_string(), - }, - ); + let test_case = "approvals_no_shows"; + let options = ApprovalsOptions { + coalesce_mean: 3.0, + coalesce_std_dev: 1.0, + enable_assignments_v2: true, + last_considered_tranche: 89, + stop_when_approved: true, + coalesce_tranche_diff: 12, + num_no_shows_per_candidate: 10, + workdir_prefix: "/tmp".to_string(), + }; + + warm_up(test_case, options.clone())?; + let usage = benchmark(test_case, options.clone()); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 6600.000, 7200.000), - ("Sent to peers", 7600.000, 8500.00), + ("Received from peers", 6900.0, 0.05), + ("Sent to peers", 8050.0, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 0.940, 1.040), - ("approval-voting", 1.300, 1.440), + ("approval-distribution", 0.99, 0.05), + ("approval-voting", 1.37, 0.05), ])); - messages + Ok(messages) } -fn approvals_throughput() -> Vec { +fn approvals_throughput() -> Result, String> { let mut messages = vec![]; - let usage = run_benchmark( - "approvals_throughput", - ApprovalsOptions { - coalesce_mean: 3.0, - coalesce_std_dev: 1.0, - enable_assignments_v2: true, - last_considered_tranche: 89, - stop_when_approved: false, - coalesce_tranche_diff: 12, - num_no_shows_per_candidate: 0, - workdir_prefix: "/tmp".to_string(), - }, - ); + let test_case = "approvals_throughput"; + let options = ApprovalsOptions { + coalesce_mean: 3.0, + coalesce_std_dev: 1.0, + enable_assignments_v2: true, + last_considered_tranche: 89, + stop_when_approved: false, + coalesce_tranche_diff: 12, + num_no_shows_per_candidate: 0, + workdir_prefix: "/tmp".to_string(), + }; + + warm_up(test_case, options.clone())?; + let usage = benchmark(test_case, options.clone()); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 50200.000, 55500.000), - ("Sent to peers", 60300.000, 66700.000), + ("Received from peers", 52850.0, 0.05), + ("Sent to peers", 63500.0, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 8.210, 9.080), - ("approval-voting", 10.530, 11.640), + ("approval-distribution", 8.645, 0.05), + ("approval-voting", 11.085, 0.05), ])); - messages + Ok(messages) } -fn approvals_throughput_best_case() -> Vec { +fn approvals_throughput_best_case() -> Result, String> { let mut messages = vec![]; - let usage = run_benchmark( - "approvals_throughput_best_case", - ApprovalsOptions { - coalesce_mean: 3.0, - coalesce_std_dev: 1.0, - enable_assignments_v2: true, - last_considered_tranche: 89, - stop_when_approved: true, - coalesce_tranche_diff: 12, - num_no_shows_per_candidate: 0, - workdir_prefix: "/tmp".to_string(), - }, - ); + let test_case = "approvals_throughput_best_case"; + let options = ApprovalsOptions { + coalesce_mean: 3.0, + coalesce_std_dev: 1.0, + enable_assignments_v2: true, + last_considered_tranche: 89, + stop_when_approved: true, + coalesce_tranche_diff: 12, + num_no_shows_per_candidate: 0, + workdir_prefix: "/tmp".to_string(), + }; + + warm_up(test_case, options.clone())?; + let usage = benchmark(test_case, options.clone()); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 2800.000, 3100.000), - ("Sent to peers", 3100.000, 3400.000), + ("Received from peers", 2950.0, 0.05), + ("Sent to peers", 3250.0, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 0.460, 0.500), - ("approval-voting", 0.590, 0.660), + ("approval-distribution", 0.48, 0.05), + ("approval-voting", 0.625, 0.05), ])); - messages + Ok(messages) } diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 70c8b60995a6..33d6ab1506c0 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -48,8 +48,8 @@ fn main() -> Result<(), String> { ])); messages.extend(usage.check_cpu_usage(&[ ("availability-distribution", 0.025, 0.05), - ("bitfield-distribution", 0.086, 0.05), - ("availability-store", 0.177, 0.05), + ("bitfield-distribution", 0.085, 0.05), + ("availability-store", 0.180, 0.05), ])); if messages.is_empty() { From 31a50904265cfca6cd65e0678d394c8b908e5fbd Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 16:29:19 +0100 Subject: [PATCH 45/53] Run benchmarks --- .../approval-voting/tests/approval-voting-regression-bench.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs index ca0bcfbc8564..b27579a3da28 100644 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -115,6 +115,7 @@ fn approvals_no_shows() -> Result, String> { Ok(messages) } +#[allow(dead_code)] fn approvals_throughput() -> Result, String> { let mut messages = vec![]; let test_case = "approvals_throughput"; @@ -144,6 +145,7 @@ fn approvals_throughput() -> Result, String> { Ok(messages) } +#[allow(dead_code)] fn approvals_throughput_best_case() -> Result, String> { let mut messages = vec![]; let test_case = "approvals_throughput_best_case"; From 732b4ba8177d5c3a804966f1ac42f4d7b6b73c97 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 16:50:42 +0100 Subject: [PATCH 46/53] Disable approval-voting-regression-bench --- .gitlab/pipeline/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index e5028f9ab5eb..89f663fd9b27 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -501,8 +501,8 @@ subsystem-regression-tests: - .common-refs - .run-immediately script: - - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks - # - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks - # - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks + # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks + - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks + - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks tags: - benchmark From c0e6f31778b7fb7635f66f240cc8dd8aa05dbac3 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 17:11:24 +0100 Subject: [PATCH 47/53] Update values --- .../tests/approval-voting-regression-bench.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs index b27579a3da28..00be5d736364 100644 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs @@ -104,12 +104,12 @@ fn approvals_no_shows() -> Result, String> { let usage = benchmark(test_case, options.clone()); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 6900.0, 0.05), - ("Sent to peers", 8050.0, 0.05), + ("Received from peers", 7000.0, 0.05), + ("Sent to peers", 8100.0, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 0.99, 0.05), - ("approval-voting", 1.37, 0.05), + ("approval-distribution", 0.990, 0.05), + ("approval-voting", 1.310, 0.05), ])); Ok(messages) From 02d63b85d5c86b18fbc7d205530af4bcb8cb45db Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Feb 2024 17:11:46 +0100 Subject: [PATCH 48/53] Update benches --- .gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 89f663fd9b27..a3bb103d4ed0 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -501,7 +501,7 @@ subsystem-regression-tests: - .common-refs - .run-immediately script: - # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks + - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks tags: From 341c8b7a29d2677f70c7811103b07e0a6ec56be2 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 23 Feb 2024 08:41:32 +0100 Subject: [PATCH 49/53] Disable approval-voting tests --- .gitlab/pipeline/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index a3bb103d4ed0..89f663fd9b27 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -501,7 +501,7 @@ subsystem-regression-tests: - .common-refs - .run-immediately script: - - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks + # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks tags: From 77f3da57faf0423eae5086759799ad43696c992a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 29 Feb 2024 17:42:50 +0100 Subject: [PATCH 50/53] Remove approval-voting tests --- Cargo.lock | 1 - polkadot/node/core/approval-voting/Cargo.toml | 11 -- .../tests/approval-voting-regression-bench.rs | 176 ------------------ 3 files changed, 188 deletions(-) delete mode 100644 polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs diff --git a/Cargo.lock b/Cargo.lock index d65787d9f449..27fe691452f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12427,7 +12427,6 @@ dependencies = [ "polkadot-overseer", "polkadot-primitives", "polkadot-primitives-test-helpers", - "polkadot-subsystem-bench", "rand", "rand_chacha 0.3.1", "rand_core 0.6.4", diff --git a/polkadot/node/core/approval-voting/Cargo.toml b/polkadot/node/core/approval-voting/Cargo.toml index b004be4e2222..f6d89dbc1528 100644 --- a/polkadot/node/core/approval-voting/Cargo.toml +++ b/polkadot/node/core/approval-voting/Cargo.toml @@ -53,14 +53,3 @@ kvdb-memorydb = "0.13.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" } log = { workspace = true, default-features = true } env_logger = "0.9.0" - -polkadot-subsystem-bench = { path = "../../subsystem-bench" } - -[[test]] -name = "approval-voting-regression-bench" -path = "tests/approval-voting-regression-bench.rs" -harness = false -required-features = ["subsystem-benchmarks"] - -[features] -subsystem-benchmarks = [] diff --git a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs b/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs deleted file mode 100644 index 00be5d736364..000000000000 --- a/polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use polkadot_subsystem_bench::{ - approval::{bench_approvals, prepare_test, ApprovalsOptions}, - configuration::TestConfiguration, - usage::BenchmarkUsage, -}; - -const BENCH_COUNT: usize = 3; -const WARM_UP_COUNT: usize = 20; -const WARM_UP_PRECISION: f64 = 0.01; - -fn main() -> Result<(), String> { - let mut messages = vec![]; - - messages.extend(approvals_no_shows()?); - // messages.extend(approvals_throughput()?); - // messages.extend(approvals_throughput_best_case()?); - - if messages.is_empty() { - Ok(()) - } else { - eprintln!("{}", messages.join("\n")); - Err("Regressions found".to_string()) - } -} - -fn test_configuration() -> TestConfiguration { - let mut config = TestConfiguration::default(); - config.n_validators = 500; - config.n_cores = 100; - config.min_pov_size = 1120; - config.max_pov_size = 5120; - config.peer_bandwidth = 524288000000; - config.bandwidth = 524288000000; - config.num_blocks = 10; - config.generate_pov_sizes(); - - config -} - -fn warm_up(test_case: &str, options: ApprovalsOptions) -> Result<(), String> { - println!("Warming up..."); - let mut prev_run: Option = None; - for _ in 0..WARM_UP_COUNT { - let curr = run(test_case, options.clone()); - if let Some(ref prev) = prev_run { - let ap_distr_diff = - curr.cpu_usage_diff(prev, "approval-distribution").expect("Must exist"); - let ap_vot_diff = curr.cpu_usage_diff(prev, "approval-voting").expect("Must exist"); - if ap_distr_diff < WARM_UP_PRECISION && ap_vot_diff < WARM_UP_PRECISION { - return Ok(()) - } - } - prev_run = Some(curr); - } - - Err("Can't warm up".to_string()) -} - -fn benchmark(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage { - println!("Benchmarking..."); - let usages: Vec = - (0..BENCH_COUNT).map(|_| run(test_case, options.clone())).collect(); - let usage = BenchmarkUsage::average(&usages); - println!("{}", usage); - usage -} - -fn run(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage { - let (mut env, state) = prepare_test(test_configuration(), options.clone(), false); - env.runtime().block_on(bench_approvals(test_case, &mut env, state)) -} - -fn approvals_no_shows() -> Result, String> { - let mut messages = vec![]; - let test_case = "approvals_no_shows"; - let options = ApprovalsOptions { - coalesce_mean: 3.0, - coalesce_std_dev: 1.0, - enable_assignments_v2: true, - last_considered_tranche: 89, - stop_when_approved: true, - coalesce_tranche_diff: 12, - num_no_shows_per_candidate: 10, - workdir_prefix: "/tmp".to_string(), - }; - - warm_up(test_case, options.clone())?; - let usage = benchmark(test_case, options.clone()); - - messages.extend(usage.check_network_usage(&[ - ("Received from peers", 7000.0, 0.05), - ("Sent to peers", 8100.0, 0.05), - ])); - messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 0.990, 0.05), - ("approval-voting", 1.310, 0.05), - ])); - - Ok(messages) -} - -#[allow(dead_code)] -fn approvals_throughput() -> Result, String> { - let mut messages = vec![]; - let test_case = "approvals_throughput"; - let options = ApprovalsOptions { - coalesce_mean: 3.0, - coalesce_std_dev: 1.0, - enable_assignments_v2: true, - last_considered_tranche: 89, - stop_when_approved: false, - coalesce_tranche_diff: 12, - num_no_shows_per_candidate: 0, - workdir_prefix: "/tmp".to_string(), - }; - - warm_up(test_case, options.clone())?; - let usage = benchmark(test_case, options.clone()); - - messages.extend(usage.check_network_usage(&[ - ("Received from peers", 52850.0, 0.05), - ("Sent to peers", 63500.0, 0.05), - ])); - messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 8.645, 0.05), - ("approval-voting", 11.085, 0.05), - ])); - - Ok(messages) -} - -#[allow(dead_code)] -fn approvals_throughput_best_case() -> Result, String> { - let mut messages = vec![]; - let test_case = "approvals_throughput_best_case"; - let options = ApprovalsOptions { - coalesce_mean: 3.0, - coalesce_std_dev: 1.0, - enable_assignments_v2: true, - last_considered_tranche: 89, - stop_when_approved: true, - coalesce_tranche_diff: 12, - num_no_shows_per_candidate: 0, - workdir_prefix: "/tmp".to_string(), - }; - - warm_up(test_case, options.clone())?; - let usage = benchmark(test_case, options.clone()); - - messages.extend(usage.check_network_usage(&[ - ("Received from peers", 2950.0, 0.05), - ("Sent to peers", 3250.0, 0.05), - ])); - messages.extend(usage.check_cpu_usage(&[ - ("approval-distribution", 0.48, 0.05), - ("approval-voting", 0.625, 0.05), - ])); - - Ok(messages) -} From 36b3f6c7fdce65b393f23c69f24cdd35cfc76599 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 29 Feb 2024 17:52:29 +0100 Subject: [PATCH 51/53] Remove regression tests from CI --- .gitlab/pipeline/test.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 89f663fd9b27..b6b499670194 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -493,16 +493,3 @@ test-syscalls: printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n"; fi allow_failure: false # this rarely triggers in practice - -subsystem-regression-tests: - stage: test - extends: - - .docker-env - - .common-refs - - .run-immediately - script: - # - cargo test --profile=testnet -p polkadot-node-core-approval-voting --test approval-voting-regression-bench --features subsystem-benchmarks - - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks - - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks - tags: - - benchmark From ff918193c31c5ded81dd4853578608ec28247f14 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 29 Feb 2024 18:27:12 +0100 Subject: [PATCH 52/53] Add comments --- .../availability-distribution-regression-bench.rs | 11 +++++++++++ .../tests/availability-recovery-regression-bench.rs | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 33d6ab1506c0..f2872f3c72ba 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -14,6 +14,15 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! availability-read regression tests +//! +//! TODO: Explain the test case after configuration adjusted to Kusama +//! +//! Subsystems involved: +//! - availability-distribution +//! - bitfield-distribution +//! - availability-store + use polkadot_subsystem_bench::{ availability::{benchmark_availability_write, prepare_test, TestDataAvailability, TestState}, configuration::{PeerLatency, TestConfiguration}, @@ -26,6 +35,8 @@ const WARM_UP_PRECISION: f64 = 0.01; fn main() -> Result<(), String> { let mut messages = vec![]; + + // TODO: Adjust the test configurations to Kusama values let mut config = TestConfiguration::default(); config.latency = Some(PeerLatency { mean_latency_ms: 30, std_dev: 2.0 }); config.n_validators = 1000; diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index fbd08acbf012..beb063e7ae0d 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -14,6 +14,13 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! availability-write regression tests +//! +//! TODO: Explain the test case after configuration adjusted to Kusama +//! +//! Subsystems involved: +//! - availability-recovery + use polkadot_subsystem_bench::{ availability::{ benchmark_availability_read, prepare_test, DataAvailabilityReadOptions, @@ -29,6 +36,8 @@ const WARM_UP_PRECISION: f64 = 0.01; fn main() -> Result<(), String> { let mut messages = vec![]; + + // TODO: Adjust the test configurations to Kusama values let options = DataAvailabilityReadOptions { fetch_from_backers: true }; let mut config = TestConfiguration::default(); config.latency = Some(PeerLatency { mean_latency_ms: 100, std_dev: 1.0 }); From 07b46624b211eddfc1a4d5eac042e8657564a7bf Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 1 Mar 2024 15:29:31 +0100 Subject: [PATCH 53/53] Add comments --- .gitlab/pipeline/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 5c41a3e6e08f..2d3779e403bf 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -23,6 +23,7 @@ test-linux-stable: - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}" # add experimental to features after https://github.com/paritytech/substrate/pull/14502 is merged # "upgrade_version_checks_should_work" is currently failing + # Filtered by deps to exclude subsystem regression tests that we run in another job - | time cargo nextest run \ --filter-expr 'not deps(/polkadot-subsystem-bench/)' \ @@ -70,6 +71,7 @@ test-linux-stable-runtime-benchmarks: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: + # Filtered by deps to exclude subsystem regression tests that we run in another job - time cargo nextest run --filter-expr 'not deps(/polkadot-subsystem-bench/)' --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet # can be used to run all tests