subsystem-bench: add regression tests for availability read and write

polkadot/node/core/approval-voting/Cargo.toml

@@ -53,3 +53,14 @@ kvdb-memorydb = "0.13.0"
 test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../../primitives/test-helpers" }
 log = { workspace = true, default-features = true }
 env_logger = "0.9.0"
+
+polkadot-subsystem-bench = { path = "../../subsystem-bench" }
+
+[[test]]
+name = "approval-voting-regression-bench"
+path = "tests/approval-voting-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
+
+[features]
+subsystem-benchmarks = []

polkadot/node/core/approval-voting/tests/approval-voting-regression-bench.rs

@@ -0,0 +1,141 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use polkadot_subsystem_bench::{
+	approval::{bench_approvals, prepare_test, ApprovalsOptions},
+	configuration::TestConfiguration,
+	usage::BenchmarkUsage,
+};
+
+fn main() -> Result<(), String> {
+	let mut messages = vec![];
+
+	messages.extend(approvals_no_shows());
+	messages.extend(approvals_throughput());
+	messages.extend(approvals_throughput_best_case());
+
+	if messages.is_empty() {
+		Ok(())
+	} else {
+		eprintln!("{}", messages.join("\n"));
+		Err("Regressions found".to_string())
+	}
+}
+
+fn test_configuration() -> TestConfiguration {
+	let mut config = TestConfiguration::default();
+	config.n_validators = 500;
+	config.n_cores = 100;
+	config.min_pov_size = 1120;
+	config.max_pov_size = 5120;
+	config.peer_bandwidth = 524288000000;
+	config.bandwidth = 524288000000;
+	config.num_blocks = 10;
+	config.generate_pov_sizes();
+
+	config
+}
+
+fn run_benchmark(test_case: &str, options: ApprovalsOptions) -> BenchmarkUsage {
+	let (mut env, state) = prepare_test(test_configuration(), options, false);
+	let usage = env.runtime().block_on(bench_approvals(test_case, &mut env, state));
+	println!("{}", usage);
+	usage
+}
+
+fn approvals_no_shows() -> Vec<String> {
+	let mut messages = vec![];
+	let usage = run_benchmark(
+		"approvals_no_shows",
+		ApprovalsOptions {
+			coalesce_mean: 3.0,
+			coalesce_std_dev: 1.0,
+			enable_assignments_v2: true,
+			last_considered_tranche: 89,
+			stop_when_approved: true,
+			coalesce_tranche_diff: 12,
+			num_no_shows_per_candidate: 10,
+			workdir_prefix: "/tmp".to_string(),
+		},
+	);
+
+	messages.extend(usage.check_network_usage(&[
+		("Received from peers", 6900.000, 7000.000),
+		("Sent to peers", 8000.000, 8100.000),
+	]));
+	messages.extend(usage.check_cpu_usage(&[
+		("approval-distribution", 0.700, 0.800),
+		("approval-voting", 1.100, 1.200),
+	]));
+
+	messages
+}
+
+fn approvals_throughput() -> Vec<String> {
+	let mut messages = vec![];
+	let usage = run_benchmark(
+		"approvals_throughput",
+		ApprovalsOptions {
+			coalesce_mean: 3.0,
+			coalesce_std_dev: 1.0,
+			enable_assignments_v2: true,
+			last_considered_tranche: 89,
+			stop_when_approved: false,
+			coalesce_tranche_diff: 12,
+			num_no_shows_per_candidate: 0,
+			workdir_prefix: "/tmp".to_string(),
+		},
+	);
+
+	messages.extend(usage.check_network_usage(&[
+		("Received from peers", 52900.000, 53000.000),
+		("Sent to peers", 63500.000, 63600.000),
+	]));
+	messages.extend(usage.check_cpu_usage(&[
+		("approval-distribution", 6.300, 6.400),
+		("approval-voting", 9.400, 9.500),
+	]));
+
+	messages
+}
+
+fn approvals_throughput_best_case() -> Vec<String> {
+	let mut messages = vec![];
+	let usage = run_benchmark(
+		"approvals_throughput_best_case",
+		ApprovalsOptions {
+			coalesce_mean: 3.0,
+			coalesce_std_dev: 1.0,
+			enable_assignments_v2: true,
+			last_considered_tranche: 89,
+			stop_when_approved: true,
+			coalesce_tranche_diff: 12,
+			num_no_shows_per_candidate: 0,
+			workdir_prefix: "/tmp".to_string(),
+		},
+	);
+
+	messages.extend(usage.check_network_usage(&[
+		("Received from peers", 2900.000, 3000.000),
+		("Sent to peers", 3200.000, 3300.000),
+	]));
+	messages.extend(usage.check_cpu_usage(&[
+		("approval-distribution", 0.400, 0.500),
+		("approval-voting", 0.600, 0.700),
+	]));
+
+	messages
+}

polkadot/node/network/availability-distribution/Cargo.toml

@@ -36,3 +36,14 @@ sc-network = { path = "../../../../substrate/client/network" }
 futures-timer = "3.0.2"
 assert_matches = "1.4.0"
 polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" }
+polkadot-subsystem-bench = { path = "../../subsystem-bench" }
+
+
+[[test]]
+name = "availability-distribution-regression-bench"
+path = "tests/availability-distribution-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
+
+[features]
+subsystem-benchmarks = []

polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs

@@ -0,0 +1,64 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use polkadot_subsystem_bench::{
+	availability::{benchmark_availability_write, prepare_test, TestDataAvailability, TestState},
+	configuration::{PeerLatency, TestConfiguration},
+};
+
+fn main() -> Result<(), String> {
+	let mut messages = vec![];
+	let mut config = TestConfiguration::default();
+	config.latency = Some(PeerLatency { mean_latency_ms: 30, std_dev: 2.0 });
+	config.n_validators = 1000;
+	config.n_cores = 200;
+	config.max_validators_per_core = 5;
+	config.min_pov_size = 5120;
+	config.max_pov_size = 5120;
+	config.peer_bandwidth = 52428800;
+	config.bandwidth = 52428800;
+	config.connectivity = 75;
+	config.num_blocks = 3;
+	config.generate_pov_sizes();
+
+	let mut state = TestState::new(&config);
+	let (mut env, _protocol_config) =
+		prepare_test(config, &mut state, TestDataAvailability::Write, false);
+	let usage = env.runtime().block_on(benchmark_availability_write(
+		"data_availability_write",
+		&mut env,
+		state,
+	));
+
+	println!("{}", usage);
+
+	messages.extend(usage.check_network_usage(&[
+		("Received from peers", 4300.000, 4400.000),
+		("Sent to peers", 15900.000, 16000.000),
+	]));
+	messages.extend(usage.check_cpu_usage(&[
+		("availability-distribution", 0.000, 0.100),
+		("bitfield-distribution", 0.000, 0.100),
+		("availability-store", 0.100, 0.200),
+	]));
+
+	if messages.is_empty() {
+		Ok(())
+	} else {
+		eprintln!("{}", messages.join("\n"));
+		Err("Regressions found".to_string())
+	}
+}

polkadot/node/network/availability-recovery/Cargo.toml

@@ -41,6 +41,13 @@ sc-network = { path = "../../../../substrate/client/network" }
 
 polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" }
 polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" }
+polkadot-subsystem-bench = { path = "../../subsystem-bench" }
+
+[[test]]
+name = "availability-recovery-regression-bench"
+path = "tests/availability-recovery-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
 
 [features]
 subsystem-benchmarks = []

polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs

@@ -0,0 +1,63 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+use polkadot_subsystem_bench::{
+	availability::{
+		benchmark_availability_read, prepare_test, DataAvailabilityReadOptions,
+		TestDataAvailability, TestState,
+	},
+	configuration::{PeerLatency, TestConfiguration},
+};
+
+fn main() -> Result<(), String> {
+	let mut messages = vec![];
+	let options = DataAvailabilityReadOptions { fetch_from_backers: true };
+	let mut config = TestConfiguration::default();
+	config.latency = Some(PeerLatency { mean_latency_ms: 100, std_dev: 1.0 });
+	config.n_validators = 300;
+	config.n_cores = 20;
+	config.min_pov_size = 5120;
+	config.max_pov_size = 5120;
+	config.peer_bandwidth = 52428800;
+	config.bandwidth = 52428800;
+	config.num_blocks = 3;
+	config.connectivity = 90;
+	config.generate_pov_sizes();
+
+	let mut state = TestState::new(&config);
+	let (mut env, _protocol_config) =
+		prepare_test(config, &mut state, TestDataAvailability::Read(options), false);
+	let usage = env.runtime().block_on(benchmark_availability_read(
+		"data_availability_read",
+		&mut env,
+		state,
+	));
+
+	println!("{}", usage);
+
+	messages.extend(usage.check_network_usage(&[
+		("Received from peers", 102400.000, 102500.000),
+		("Sent to peers", 0.000, 100.000),
+	]));
+	messages.extend(usage.check_cpu_usage(&[("availability-recovery", 4.200, 4.300)]));
+
+	if messages.is_empty() {
+		Ok(())
+	} else {
+		eprintln!("{}", messages.join("\n"));
+		Err("Regressions found".to_string())
+	}
+}

polkadot/node/subsystem-bench/Cargo.toml

@@ -8,9 +8,13 @@ license.workspace = true
 readme = "README.md"
 publish = false
 
+[lib]
+name = "polkadot_subsystem_bench"
+path = "src/lib/lib.rs"
+
 [[bin]]
 name = "subsystem-bench"
-path = "src/subsystem-bench.rs"
+path = "src/cli/subsystem-bench.rs"
 
 # Prevent rustdoc error. Already documented from top-level Cargo.toml.
 doc = false

polkadot/node/subsystem-bench/examples/approvals_no_shows.yaml

@@ -1,14 +1,14 @@
 TestConfiguration:
 # Test 1
 - objective: !ApprovalVoting
-    last_considered_tranche: 89
     coalesce_mean: 3.0
     coalesce_std_dev: 1.0
+    enable_assignments_v2: true
+    last_considered_tranche: 89
     stop_when_approved: true
     coalesce_tranche_diff: 12
-    workdir_prefix: "/tmp/"
-    enable_assignments_v2: true
     num_no_shows_per_candidate: 10
+    workdir_prefix: "/tmp/"
   n_validators: 500
   n_cores: 100
   min_pov_size: 1120

polkadot/node/subsystem-bench/examples/approvals_throughput.yaml

@@ -7,11 +7,10 @@ TestConfiguration:
     last_considered_tranche: 89
     stop_when_approved: false
     coalesce_tranche_diff: 12
-    workdir_prefix: "/tmp"
     num_no_shows_per_candidate: 0
+    workdir_prefix: "/tmp"
   n_validators: 500
   n_cores: 100
-  n_included_candidates: 100
   min_pov_size: 1120
   max_pov_size: 5120
   peer_bandwidth: 524288000000

polkadot/node/subsystem-bench/examples/approvals_throughput_best_case.yaml

@@ -7,8 +7,8 @@ TestConfiguration:
     last_considered_tranche: 89
     stop_when_approved: true
     coalesce_tranche_diff: 12
-    workdir_prefix: "/tmp/"
     num_no_shows_per_candidate: 0
+    workdir_prefix: "/tmp/"
   n_validators: 500
   n_cores: 100
   min_pov_size: 1120