Skip to content

Commit

Permalink
Merge branch 'dev' into p-990-benchmark-fix
Browse files Browse the repository at this point in the history
  • Loading branch information
wangminqi authored Aug 29, 2024
2 parents 34b1da8 + 9e893a2 commit 03ee30d
Show file tree
Hide file tree
Showing 15 changed files with 1,276 additions and 911 deletions.
380 changes: 2 additions & 378 deletions tee-worker/cli/src/benchmark/mod.rs
Original file line number Diff line number Diff line change
@@ -1,378 +1,2 @@
/*
Copyright 2021 Integritee AG and Supercomputing Systems AG
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

use crate::{
command_utils::get_worker_api_direct,
get_layer_two_nonce,
trusted_cli::TrustedCli,
trusted_command_utils::{get_identifiers, get_keystore_path, get_pair_from_str},
trusted_operation::{get_json_request, get_state, wait_until},
Cli, CliResult, CliResultOk, SR25519_KEY_TYPE,
};
use codec::Decode;
use hdrhistogram::Histogram;
use ita_stf::{
Getter, Index, PublicGetter, TrustedCall, TrustedCallSigned, TrustedGetter, STF_TX_FEE,
};
use itc_rpc_client::direct_client::{DirectApi, DirectClient};
use itp_stf_primitives::{
traits::TrustedCallSigning,
types::{KeyPair, TrustedOperation},
};
use itp_types::{
Balance, ShardIdentifier, TrustedOperationStatus,
TrustedOperationStatus::{InSidechainBlock, Submitted},
};
use log::*;
use rand::Rng;
use rayon::prelude::*;
use sgx_crypto_helper::rsa3072::Rsa3072PubKey;
use sp_application_crypto::sr25519;
use sp_core::{sr25519 as sr25519_core, Pair};
use sp_keystore::Keystore;
use std::{
boxed::Box,
string::ToString,
sync::mpsc::{channel, Receiver},
thread, time,
time::Instant,
vec::Vec,
};
use substrate_client_keystore::LocalKeystore;

// Needs to be above the existential deposit minimum, otherwise an account will not
// be created and the state is not increased.
const EXISTENTIAL_DEPOSIT: Balance = 1000;

#[derive(Parser)]
pub struct BenchmarkCommand {
/// The number of clients (=threads) to be used in the benchmark
#[clap(default_value_t = 10)]
number_clients: u32,

/// The number of iterations to execute for each client
#[clap(default_value_t = 30)]
number_iterations: u128,

/// Adds a random wait before each transaction. This is the lower bound for the interval in ms.
#[clap(default_value_t = 0)]
random_wait_before_transaction_min_ms: u32,

/// Adds a random wait before each transaction. This is the upper bound for the interval in ms.
#[clap(default_value_t = 0)]
random_wait_before_transaction_max_ms: u32,

/// Whether to wait for "InSidechainBlock" confirmation for each transaction
#[clap(short, long)]
wait_for_confirmation: bool,

/// Account to be used for initial funding of generated accounts used in benchmark
#[clap(default_value_t = String::from("//Alice"))]
funding_account: String,
}

struct BenchmarkClient {
account: sr25519_core::Pair,
current_balance: u128,
client_api: DirectClient,
receiver: Receiver<String>,
}

impl BenchmarkClient {
fn new(
account: sr25519_core::Pair,
initial_balance: u128,
initial_request: String,
cli: &Cli,
) -> Self {
debug!("get direct api");
let client_api = get_worker_api_direct(cli);

debug!("setup sender and receiver");
let (sender, receiver) = channel();
client_api.watch(initial_request, sender);
BenchmarkClient { account, current_balance: initial_balance, client_api, receiver }
}
}

/// Stores timing information about a specific transaction
struct BenchmarkTransaction {
started: Instant,
submitted: Instant,
confirmed: Option<Instant>,
}

impl BenchmarkCommand {
pub(crate) fn run(&self, cli: &Cli, trusted_args: &TrustedCli) -> CliResult {
let random_wait_before_transaction_ms: (u32, u32) = (
self.random_wait_before_transaction_min_ms,
self.random_wait_before_transaction_max_ms,
);
let store = LocalKeystore::open(get_keystore_path(trusted_args, cli), None).unwrap();
let funding_account_keys = get_pair_from_str(trusted_args, &self.funding_account, cli);

let (mrenclave, shard) = get_identifiers(trusted_args, cli);

// Get shielding pubkey.
let worker_api_direct = get_worker_api_direct(cli);
let shielding_pubkey: Rsa3072PubKey = match worker_api_direct.get_rsa_pubkey() {
Ok(key) => key,
Err(err_msg) => panic!("{}", err_msg.to_string()),
};

let nonce_start = get_layer_two_nonce!(funding_account_keys, cli, trusted_args);
println!("Nonce for account {}: {}", self.funding_account, nonce_start);

let mut accounts = Vec::new();
let initial_balance = (self.number_iterations + 1) * (STF_TX_FEE + EXISTENTIAL_DEPOSIT);
// Setup new accounts and initialize them with money from Alice.
for i in 0..self.number_clients {
let nonce = i + nonce_start;
println!("Initializing account {} with initial amount {:?}", i, initial_balance);

// Create new account to use.
let a = LocalKeystore::sr25519_generate_new(&store, SR25519_KEY_TYPE, None).unwrap();
let account = get_pair_from_str(trusted_args, a.to_string().as_str(), cli);

// Transfer amount from Alice to new account.
let top: TrustedOperation<TrustedCallSigned, Getter> = TrustedCall::balance_transfer(
funding_account_keys.public().into(),
account.public().into(),
initial_balance,
)
.sign(
&KeyPair::Sr25519(Box::new(funding_account_keys.clone())),
nonce,
&mrenclave,
&shard,
)
.into_trusted_operation(trusted_args.direct);

// For the last account we wait for confirmation in order to ensure all accounts were setup correctly
let wait_for_confirmation = i == self.number_clients - 1;
let account_funding_request = get_json_request(shard, &top, shielding_pubkey);

let client =
BenchmarkClient::new(account, initial_balance, account_funding_request, cli);
let _result = wait_for_top_confirmation(wait_for_confirmation, &client);
accounts.push(client);
}

rayon::ThreadPoolBuilder::new()
.num_threads(self.number_clients as usize)
.build_global()
.unwrap();

let overall_start = Instant::now();

// Run actual benchmark logic, in parallel, for each account initialized above.
let outputs: Vec<Vec<BenchmarkTransaction>> = accounts
.into_par_iter()
.map(move |mut client| {
let mut output: Vec<BenchmarkTransaction> = Vec::new();

for i in 0..self.number_iterations {
println!("Iteration: {}", i);

if random_wait_before_transaction_ms.1 > 0 {
random_wait(random_wait_before_transaction_ms);
}

// Create new account.
let account_keys = LocalKeystore::sr25519_generate_new(&store, SR25519_KEY_TYPE, None).unwrap();

let new_account =
get_pair_from_str(trusted_args, account_keys.to_string().as_str(), cli);

println!(" Transfer amount: {}", EXISTENTIAL_DEPOSIT);
println!(" From: {:?}", client.account.public());
println!(" To: {:?}", new_account.public());

// Get nonce of account.
let nonce = get_nonce(client.account.clone(), shard, &client.client_api);

// Transfer money from client account to new account.
let top: TrustedOperation<TrustedCallSigned, Getter> = TrustedCall::balance_transfer(
client.account.public().into(),
new_account.public().into(),
EXISTENTIAL_DEPOSIT,
)
.sign(&KeyPair::Sr25519(Box::new(client.account.clone())), nonce, &mrenclave, &shard)
.into_trusted_operation(trusted_args.direct);

let last_iteration = i == self.number_iterations - 1;
let jsonrpc_call = get_json_request(shard, &top, shielding_pubkey);
client.client_api.send(&jsonrpc_call).unwrap();
let result = wait_for_top_confirmation(
self.wait_for_confirmation || last_iteration,
&client,
);

client.current_balance -= EXISTENTIAL_DEPOSIT;

let balance = get_balance(client.account.clone(), shard, &client.client_api);
println!("Balance: {}", balance.unwrap_or_default());
assert_eq!(client.current_balance, balance.unwrap_or_default());

output.push(result);

// FIXME: We probably should re-fund the account in this case.
if client.current_balance <= EXISTENTIAL_DEPOSIT + STF_TX_FEE {
error!("Account {:?} does not have enough balance anymore. Finishing benchmark early", client.account.public());
break;
}
}

client.client_api.close().unwrap();

output
})
.collect();

println!(
"Finished benchmark with {} clients and {} transactions in {} ms",
self.number_clients,
self.number_iterations,
overall_start.elapsed().as_millis()
);

print_benchmark_statistic(outputs, self.wait_for_confirmation);

Ok(CliResultOk::None)
}
}

fn get_balance(
account: sr25519::Pair,
shard: ShardIdentifier,
direct_client: &DirectClient,
) -> Option<u128> {
let getter = Getter::trusted(
TrustedGetter::free_balance(account.public().into())
.sign(&KeyPair::Sr25519(Box::new(account.clone()))),
);

let getter_start_timer = Instant::now();
let getter_result = direct_client.get_state(shard, &getter);
let getter_execution_time = getter_start_timer.elapsed().as_millis();

let balance = decode_balance(getter_result);
info!("Balance getter execution took {} ms", getter_execution_time,);
debug!("Retrieved {:?} Balance for {:?}", balance.unwrap_or_default(), account.public());
balance
}

fn get_nonce(
account: sr25519::Pair,
shard: ShardIdentifier,
direct_client: &DirectClient,
) -> Index {
let getter = Getter::public(PublicGetter::nonce(account.public().into()));

let getter_start_timer = Instant::now();
let nonce = get_state::<Index>(direct_client, shard, &getter).ok().unwrap_or_default();
let getter_execution_time = getter_start_timer.elapsed().as_millis();
info!("Nonce getter execution took {} ms", getter_execution_time,);
debug!("Retrieved {:?} nonce for {:?}", nonce, account.public());
nonce
}

fn print_benchmark_statistic(outputs: Vec<Vec<BenchmarkTransaction>>, wait_for_confirmation: bool) {
let mut hist = Histogram::<u64>::new(1).unwrap();
for output in outputs {
for t in output {
let benchmarked_timestamp =
if wait_for_confirmation { t.confirmed } else { Some(t.submitted) };
if let Some(confirmed) = benchmarked_timestamp {
hist += confirmed.duration_since(t.started).as_millis() as u64;
} else {
println!("Missing measurement data");
}
}
}

for i in (5..=100).step_by(5) {
let text = format!(
"{} percent are done within {} ms",
i,
hist.value_at_quantile(i as f64 / 100.0)
);
println!("{}", text);
}
}

fn random_wait(random_wait_before_transaction_ms: (u32, u32)) {
let mut rng = rand::thread_rng();
let sleep_time = time::Duration::from_millis(
rng.gen_range(random_wait_before_transaction_ms.0..=random_wait_before_transaction_ms.1)
.into(),
);
println!("Sleep for: {}ms", sleep_time.as_millis());
thread::sleep(sleep_time);
}

fn wait_for_top_confirmation(
wait_for_sidechain_block: bool,
client: &BenchmarkClient,
) -> BenchmarkTransaction {
let started = Instant::now();

let submitted = wait_until(&client.receiver, is_submitted);

let confirmed = if wait_for_sidechain_block {
// We wait for the transaction hash that actually matches the submitted hash
loop {
let transaction_information = wait_until(&client.receiver, is_sidechain_block);
if let Some((hash, _)) = transaction_information {
if hash == submitted.unwrap().0 {
break transaction_information
}
}
}
} else {
None
};
if let (Some(s), Some(c)) = (submitted, confirmed) {
// Assert the two hashes are identical
assert_eq!(s.0, c.0);
}

BenchmarkTransaction {
started,
submitted: submitted.unwrap().1,
confirmed: confirmed.map(|v| v.1),
}
}

fn is_submitted(s: TrustedOperationStatus) -> bool {
matches!(s, Submitted)
}

fn is_sidechain_block(s: TrustedOperationStatus) -> bool {
matches!(s, InSidechainBlock(_))
}

fn decode_balance(maybe_encoded_balance: Option<Vec<u8>>) -> Option<Balance> {
maybe_encoded_balance.and_then(|encoded_balance| {
if let Ok(vd) = Balance::decode(&mut encoded_balance.as_slice()) {
Some(vd)
} else {
warn!("Could not decode balance. maybe hasn't been set? {:x?}", encoded_balance);
None
}
})
}
pub mod request_vc;
pub mod stf;
Loading

0 comments on commit 03ee30d

Please sign in to comment.