Add benchmark extrinsic command (paritytech#11456)

* Benchmark extrinsic

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Reduce warmup and repeat

Running this 1000 times with a full block takes ~33 minutes 🙈.
Reducing it to ~3 minutes per default.

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Make ExistentialDeposit public

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Add 'bechmark extrinsic' command

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* fmt

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Add --list and cleanup

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Unrelated Clippy

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Clippy

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Fix tests and doc

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Move implementations up + fmt

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Dont use parameter_types macro

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Cache to_lowercase() call

The .to_lowercase() on the builder is actually not needes
since its already documented to only return lower case.

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Spelling

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Use correct nightly for fmt...

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Rename ED

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Fix compile

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Subtract block base weight

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Fixes

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Use tmp folder for test

This should already be the case since --dev is passed but
somehow not...

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

* Fix test

Signed-off-by: Oliver Tale-Yazdi <[email protected]>

bin/node-template/node/src/command_helper.rs → bin/node-template/node/src/benchmarking.rs

@@ -16,13 +16,14 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-//! Contains code to setup the command invocations in [`super::command`] which would
-//! otherwise bloat that module.
+//! Setup code for [`super::command`] which would otherwise bloat that module.
+//!
+//! Should only be used for benchmarking as it may break in other contexts.
 
 use crate::service::FullClient;
 
 use node_template_runtime as runtime;
-use runtime::SystemCall;
+use runtime::{AccountId, Balance, BalancesCall, SystemCall};
 use sc_cli::Result;
 use sc_client_api::BlockBackend;
 use sp_core::{Encode, Pair};
@@ -35,19 +36,27 @@ use std::{sync::Arc, time::Duration};
 /// Generates extrinsics for the `benchmark overhead` command.
 ///
 /// Note: Should only be used for benchmarking.
-pub struct BenchmarkExtrinsicBuilder {
+pub struct RemarkBuilder {
 	client: Arc<FullClient>,
 }
 
-impl BenchmarkExtrinsicBuilder {
+impl RemarkBuilder {
 	/// Creates a new [`Self`] from the given client.
 	pub fn new(client: Arc<FullClient>) -> Self {
 		Self { client }
 	}
 }
 
-impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
-	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder {
+	fn pallet(&self) -> &str {
+		"system"
+	}
+
+	fn extrinsic(&self) -> &str {
+		"remark"
+	}
+
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
 		let acc = Sr25519Keyring::Bob.pair();
 		let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic(
 			self.client.as_ref(),
@@ -61,6 +70,49 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
 	}
 }
 
+/// Generates `Balances::TransferKeepAlive` extrinsics for the benchmarks.
+///
+/// Note: Should only be used for benchmarking.
+pub struct TransferKeepAliveBuilder {
+	client: Arc<FullClient>,
+	dest: AccountId,
+	value: Balance,
+}
+
+impl TransferKeepAliveBuilder {
+	/// Creates a new [`Self`] from the given client.
+	pub fn new(client: Arc<FullClient>, dest: AccountId, value: Balance) -> Self {
+		Self { client, dest, value }
+	}
+}
+
+impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder {
+	fn pallet(&self) -> &str {
+		"balances"
+	}
+
+	fn extrinsic(&self) -> &str {
+		"transfer_keep_alive"
+	}
+
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+		let acc = Sr25519Keyring::Bob.pair();
+		let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic(
+			self.client.as_ref(),
+			acc,
+			BalancesCall::transfer_keep_alive {
+				dest: self.dest.clone().into(),
+				value: self.value.into(),
+			}
+			.into(),
+			nonce,
+		)
+		.into();
+
+		Ok(extrinsic)
+	}
+}
+
 /// Create a transaction using the given `call`.
 ///
 /// Note: Should only be used for benchmarking.

bin/node-template/node/src/command.rs

@@ -1,14 +1,14 @@
 use crate::{
+	benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder},
 	chain_spec,
 	cli::{Cli, Subcommand},
-	command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder},
 	service,
 };
-use frame_benchmarking_cli::{BenchmarkCmd, SUBSTRATE_REFERENCE_HARDWARE};
-use node_template_runtime::Block;
+use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE};
+use node_template_runtime::{Block, EXISTENTIAL_DEPOSIT};
 use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli};
 use sc_service::PartialComponents;
-use std::sync::Arc;
+use sp_keyring::Sr25519Keyring;
 
 impl SubstrateCli for Cli {
 	fn impl_name() -> String {
@@ -137,9 +137,23 @@ pub fn run() -> sc_cli::Result<()> {
 					},
 					BenchmarkCmd::Overhead(cmd) => {
 						let PartialComponents { client, .. } = service::new_partial(&config)?;
-						let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone());
+						let ext_builder = RemarkBuilder::new(client.clone());
 
-						cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder))
+						cmd.run(config, client, inherent_benchmark_data()?, &ext_builder)
+					},
+					BenchmarkCmd::Extrinsic(cmd) => {
+						let PartialComponents { client, .. } = service::new_partial(&config)?;
+						// Register the *Remark* and *TKA* builders.
+						let ext_factory = ExtrinsicFactory(vec![
+							Box::new(RemarkBuilder::new(client.clone())),
+							Box::new(TransferKeepAliveBuilder::new(
+								client.clone(),
+								Sr25519Keyring::Alice.to_account_id(),
+								EXISTENTIAL_DEPOSIT,
+							)),
+						]);
+
+						cmd.run(client, inherent_benchmark_data()?, &ext_factory)
 					},
 					BenchmarkCmd::Machine(cmd) =>
 						cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()),

bin/node-template/node/src/main.rs

@@ -4,9 +4,9 @@
 mod chain_spec;
 #[macro_use]
 mod service;
+mod benchmarking;
 mod cli;
 mod command;
-mod command_helper;
 mod rpc;
 
 fn main() -> sc_cli::Result<()> {

bin/node-template/runtime/src/lib.rs

@@ -234,6 +234,9 @@ impl pallet_timestamp::Config for Runtime {
 	type WeightInfo = ();
 }
 
+/// Existential deposit.
+pub const EXISTENTIAL_DEPOSIT: u128 = 500;
+
 impl pallet_balances::Config for Runtime {
 	type MaxLocks = ConstU32<50>;
 	type MaxReserves = ();
@@ -243,7 +246,7 @@ impl pallet_balances::Config for Runtime {
 	/// The ubiquitous event type.
 	type Event = Event;
 	type DustRemoval = ();
-	type ExistentialDeposit = ConstU128<500>;
+	type ExistentialDeposit = ConstU128<EXISTENTIAL_DEPOSIT>;
 	type AccountStore = System;
 	type WeightInfo = pallet_balances::weights::SubstrateWeight<Runtime>;
 }

bin/node/cli/src/command_helper.rs → bin/node/cli/src/benchmarking.rs

@@ -16,33 +16,45 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-//! Contains code to setup the command invocations in [`super::command`] which would
-//! otherwise bloat that module.
+//! Setup code for [`super::command`] which would otherwise bloat that module.
+//!
+//! Should only be used for benchmarking as it may break in other contexts.
 
 use crate::service::{create_extrinsic, FullClient};
 
-use node_runtime::SystemCall;
+use node_primitives::{AccountId, Balance};
+use node_runtime::{BalancesCall, SystemCall};
 use sc_cli::Result;
 use sp_inherents::{InherentData, InherentDataProvider};
 use sp_keyring::Sr25519Keyring;
 use sp_runtime::OpaqueExtrinsic;
 
 use std::{sync::Arc, time::Duration};
 
-/// Generates extrinsics for the `benchmark overhead` command.
-pub struct BenchmarkExtrinsicBuilder {
+/// Generates `System::Remark` extrinsics for the benchmarks.
+///
+/// Note: Should only be used for benchmarking.
+pub struct RemarkBuilder {
 	client: Arc<FullClient>,
 }
 
-impl BenchmarkExtrinsicBuilder {
+impl RemarkBuilder {
 	/// Creates a new [`Self`] from the given client.
 	pub fn new(client: Arc<FullClient>) -> Self {
 		Self { client }
 	}
 }
 
-impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
-	fn remark(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+impl frame_benchmarking_cli::ExtrinsicBuilder for RemarkBuilder {
+	fn pallet(&self) -> &str {
+		"system"
+	}
+
+	fn extrinsic(&self) -> &str {
+		"remark"
+	}
+
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
 		let acc = Sr25519Keyring::Bob.pair();
 		let extrinsic: OpaqueExtrinsic = create_extrinsic(
 			self.client.as_ref(),
@@ -56,6 +68,48 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder {
 	}
 }
 
+/// Generates `Balances::TransferKeepAlive` extrinsics for the benchmarks.
+///
+/// Note: Should only be used for benchmarking.
+pub struct TransferKeepAliveBuilder {
+	client: Arc<FullClient>,
+	dest: AccountId,
+	value: Balance,
+}
+
+impl TransferKeepAliveBuilder {
+	/// Creates a new [`Self`] from the given client.
+	pub fn new(client: Arc<FullClient>, dest: AccountId, value: Balance) -> Self {
+		Self { client, dest, value }
+	}
+}
+
+impl frame_benchmarking_cli::ExtrinsicBuilder for TransferKeepAliveBuilder {
+	fn pallet(&self) -> &str {
+		"balances"
+	}
+
+	fn extrinsic(&self) -> &str {
+		"transfer_keep_alive"
+	}
+
+	fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
+		let acc = Sr25519Keyring::Bob.pair();
+		let extrinsic: OpaqueExtrinsic = create_extrinsic(
+			self.client.as_ref(),
+			acc,
+			BalancesCall::transfer_keep_alive {
+				dest: self.dest.clone().into(),
+				value: self.value.into(),
+			},
+			Some(nonce),
+		)
+		.into();
+
+		Ok(extrinsic)
+	}
+}
+
 /// Generates inherent data for the `benchmark overhead` command.
 pub fn inherent_benchmark_data() -> Result<InherentData> {
 	let mut inherent_data = InherentData::new();

bin/node/cli/src/command.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use super::command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder};
+use super::benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder};
 use crate::{
 	chain_spec, service,
 	service::{new_partial, FullClient},
@@ -25,9 +25,10 @@ use crate::{
 use frame_benchmarking_cli::*;
 use node_executor::ExecutorDispatch;
 use node_primitives::Block;
-use node_runtime::RuntimeApi;
+use node_runtime::{ExistentialDeposit, RuntimeApi};
 use sc_cli::{ChainSpec, Result, RuntimeVersion, SubstrateCli};
 use sc_service::PartialComponents;
+use sp_keyring::Sr25519Keyring;
 
 use std::sync::Arc;
 
@@ -126,9 +127,23 @@ pub fn run() -> Result<()> {
 					},
 					BenchmarkCmd::Overhead(cmd) => {
 						let PartialComponents { client, .. } = new_partial(&config)?;
-						let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone());
+						let ext_builder = RemarkBuilder::new(client.clone());
 
-						cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder))
+						cmd.run(config, client, inherent_benchmark_data()?, &ext_builder)
+					},
+					BenchmarkCmd::Extrinsic(cmd) => {
+						let PartialComponents { client, .. } = service::new_partial(&config)?;
+						// Register the *Remark* and *TKA* builders.
+						let ext_factory = ExtrinsicFactory(vec![
+							Box::new(RemarkBuilder::new(client.clone())),
+							Box::new(TransferKeepAliveBuilder::new(
+								client.clone(),
+								Sr25519Keyring::Alice.to_account_id(),
+								ExistentialDeposit::get(),
+							)),
+						]);
+
+						cmd.run(client, inherent_benchmark_data()?, &ext_factory)
 					},
 					BenchmarkCmd::Machine(cmd) =>
 						cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()),

bin/node/cli/src/lib.rs

@@ -35,11 +35,11 @@ pub mod chain_spec;
 #[macro_use]
 pub mod service;
 #[cfg(feature = "cli")]
+mod benchmarking;
+#[cfg(feature = "cli")]
 mod cli;
 #[cfg(feature = "cli")]
 mod command;
-#[cfg(feature = "cli")]
-mod command_helper;
 
 #[cfg(feature = "cli")]
 pub use cli::*;

bin/node/cli/tests/benchmark_extrinsic_works.rs

@@ -0,0 +1,46 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use assert_cmd::cargo::cargo_bin;
+use std::process::Command;
+use tempfile::tempdir;
+
+/// Tests that the `benchmark extrinsic` command works for
+/// remark and transfer_keep_alive within the substrate dev runtime.
+#[test]
+fn benchmark_extrinsic_works() {
+	benchmark_extrinsic("system", "remark");
+	benchmark_extrinsic("balances", "transfer_keep_alive");
+}
+
+/// Checks that the `benchmark extrinsic` command works for the given pallet and extrinsic.
+fn benchmark_extrinsic(pallet: &str, extrinsic: &str) {
+	let base_dir = tempdir().expect("could not create a temp dir");
+
+	let status = Command::new(cargo_bin("substrate"))
+		.args(&["benchmark", "extrinsic", "--dev"])
+		.arg("-d")
+		.arg(base_dir.path())
+		.args(&["--pallet", pallet, "--extrinsic", extrinsic])
+		// Run with low repeats for faster execution.
+		.args(["--warmup=10", "--repeat=10", "--max-ext-per-block=10"])
+		.status()
+		.unwrap();
+
+	assert!(status.success());
+}