From 23c02802c858da6b396fa9f9acf47c72b64367bc Mon Sep 17 00:00:00 2001 From: Emma Zhong Date: Thu, 26 Oct 2023 23:47:27 -0700 Subject: [PATCH] Check system overload using txn age in queue (#13639) ## Description This PR enhances our current system overload detection to also include the scenario where the transactions depending on an object are not great in the count, but lengthy in execution. More precisely, we keep track of how long each transaction has been in the execution queue and use this age as a proxy for the hotness of the object when determining whether to reject a transaction or not. I also added a field in the node config to make this threshold configurable and easier to test. We can also move other similar numbers here as well. ## Test Plan Added a test. --- If your changes are not user-facing and not a breaking change, you can skip the following section. Otherwise, please indicate what changed, and then add to the Release Notes section as highlighted during the release process. ### Type of Change (Check all that apply) - [ ] protocol change - [x] user-visible impact - [ ] breaking change for a client SDKs - [ ] breaking change for FNs (FN binary must upgrade) - [ ] breaking change for validators or node operators (must upgrade binaries) - [ ] breaking change for on-chain data layout - [ ] necessitate either a data wipe or data migration ### Release notes Enhance our validators' protection mechanism against congestion to stop accepting transactions when one of the following happens: - There are long-running transactions waiting for execution in a certain object's queue - There are a large number of transactions waiting in a certain object's queue, or in general --- Cargo.lock | 1 + crates/sui-config/src/node.rs | 26 ++++ crates/sui-core/Cargo.toml | 1 + crates/sui-core/src/authority.rs | 14 +- .../src/authority/test_authority_builder.rs | 10 +- crates/sui-core/src/test_utils.rs | 65 ++++++--- crates/sui-core/src/transaction_manager.rs | 53 +++++-- .../src/unit_tests/execution_driver_tests.rs | 135 +++++++++++++++++- crates/sui-node/src/lib.rs | 1 + .../src/node_config_builder.rs | 2 + ...ests__network_config_snapshot_matches.snap | 28 ++++ crates/sui-types/src/error.rs | 9 ++ 12 files changed, 307 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 809d5a118d480..ccae1fceddf03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10299,6 +10299,7 @@ dependencies = [ "fs_extra", "futures", "im", + "indexmap 1.9.3", "itertools", "lru 0.10.0", "more-asserts", diff --git a/crates/sui-config/src/node.rs b/crates/sui-config/src/node.rs index 71628dd77c898..34bc636457c6f 100644 --- a/crates/sui-config/src/node.rs +++ b/crates/sui-config/src/node.rs @@ -159,6 +159,9 @@ pub struct NodeConfig { #[serde(default = "default_zklogin_oauth_providers")] pub zklogin_oauth_providers: BTreeMap>, + + #[serde(default = "default_overload_threshold_config")] + pub overload_threshold_config: OverloadThresholdConfig, } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -671,6 +674,29 @@ pub struct TransactionKeyValueStoreWriteConfig { pub concurrency: usize, } +/// Configuration for the threshold(s) at which we consider the system +/// to be overloaded. When one of the threshold is passed, the node may +/// stop processing new transactions and/or certificates until the congestion +/// resolves. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct OverloadThresholdConfig { + pub max_txn_age_in_queue: Duration, + // TODO: Move other thresholds here as well, including `MAX_TM_QUEUE_LENGTH` + // and `MAX_PER_OBJECT_QUEUE_LENGTH`. +} + +impl Default for OverloadThresholdConfig { + fn default() -> Self { + Self { + max_txn_age_in_queue: Duration::from_secs(1), // 1 second + } + } +} + +fn default_overload_threshold_config() -> OverloadThresholdConfig { + OverloadThresholdConfig::default() +} + #[derive(Debug, Clone, PartialEq, Deserialize, Serialize, Eq)] pub struct Genesis { #[serde(flatten)] diff --git a/crates/sui-core/Cargo.toml b/crates/sui-core/Cargo.toml index 974f96337f889..c5e0648b7b880 100644 --- a/crates/sui-core/Cargo.toml +++ b/crates/sui-core/Cargo.toml @@ -19,6 +19,7 @@ enum_dispatch.workspace = true eyre.workspace = true futures.workspace = true im.workspace = true +indexmap.workspace = true itertools.workspace = true lru.workspace = true num_cpus.workspace = true diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index c43b034c5fea5..a1b3c08f69cff 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -39,7 +39,7 @@ use std::{ sync::Arc, thread, }; -use sui_config::node::StateDebugDumpConfig; +use sui_config::node::{OverloadThresholdConfig, StateDebugDumpConfig}; use sui_config::NodeConfig; use sui_types::execution::DynamicallyLoadedObjectMetadata; use tap::{TapFallible, TapOptional}; @@ -649,6 +649,9 @@ pub struct AuthorityState { /// Config for state dumping on forks debug_dump_config: StateDebugDumpConfig, + + /// Config for when we consider the node overloaded. + overload_threshold_config: OverloadThresholdConfig, } /// The authority state encapsulates all state, drives execution, and ensures safety. @@ -674,6 +677,10 @@ impl AuthorityState { self.committee_store.clone() } + pub fn max_txn_age_in_queue(&self) -> Duration { + self.overload_threshold_config.max_txn_age_in_queue + } + pub fn get_epoch_state_commitments( &self, epoch: EpochId, @@ -800,7 +807,8 @@ impl AuthorityState { consensus_adapter: &Arc, tx_data: &SenderSignedData, ) -> SuiResult { - self.transaction_manager.check_execution_overload(tx_data)?; + self.transaction_manager + .check_execution_overload(self.max_txn_age_in_queue(), tx_data)?; consensus_adapter.check_consensus_overload()?; Ok(()) } @@ -2011,6 +2019,7 @@ impl AuthorityState { certificate_deny_config: CertificateDenyConfig, indirect_objects_threshold: usize, debug_dump_config: StateDebugDumpConfig, + overload_threshold_config: OverloadThresholdConfig, archive_readers: ArchiveReaderBalancer, ) -> Arc { Self::check_protocol_version(supported_protocol_versions, epoch_store.protocol_version()); @@ -2062,6 +2071,7 @@ impl AuthorityState { transaction_deny_config, certificate_deny_config, debug_dump_config, + overload_threshold_config, }); // Start a task to execute ready certificates. diff --git a/crates/sui-core/src/authority/test_authority_builder.rs b/crates/sui-core/src/authority/test_authority_builder.rs index 41bc5a85406ad..d003d52e2cc4a 100644 --- a/crates/sui-core/src/authority/test_authority_builder.rs +++ b/crates/sui-core/src/authority/test_authority_builder.rs @@ -17,10 +17,10 @@ use std::sync::Arc; use sui_archival::reader::ArchiveReaderBalancer; use sui_config::certificate_deny_config::CertificateDenyConfig; use sui_config::genesis::Genesis; -use sui_config::node::StateDebugDumpConfig; use sui_config::node::{ AuthorityStorePruningConfig, DBCheckpointConfig, ExpensiveSafetyCheckConfig, }; +use sui_config::node::{OverloadThresholdConfig, StateDebugDumpConfig}; use sui_config::transaction_deny_config::TransactionDenyConfig; use sui_macros::nondeterministic; use sui_protocol_config::{ProtocolConfig, SupportedProtocolVersions}; @@ -52,6 +52,7 @@ pub struct TestAuthorityBuilder<'a> { accounts: Vec, /// By default, we don't insert the genesis checkpoint, which isn't needed by most tests. insert_genesis_checkpoint: bool, + overload_threshold_config: Option, } impl<'a> TestAuthorityBuilder<'a> { @@ -143,6 +144,11 @@ impl<'a> TestAuthorityBuilder<'a> { self } + pub fn with_overload_threshold_config(mut self, config: OverloadThresholdConfig) -> Self { + assert!(self.overload_threshold_config.replace(config).is_none()); + self + } + pub async fn build(self) -> Arc { let local_network_config = sui_swarm_config::network_config_builder::ConfigBuilder::new_with_temp_dir() @@ -236,6 +242,7 @@ impl<'a> TestAuthorityBuilder<'a> { }; let transaction_deny_config = self.transaction_deny_config.unwrap_or_default(); let certificate_deny_config = self.certificate_deny_config.unwrap_or_default(); + let overload_threshold_config = self.overload_threshold_config.unwrap_or_default(); let state = AuthorityState::new( name, secret, @@ -256,6 +263,7 @@ impl<'a> TestAuthorityBuilder<'a> { StateDebugDumpConfig { dump_file_directory: Some(tempdir().unwrap().into_path()), }, + overload_threshold_config, ArchiveReaderBalancer::default(), ) .await; diff --git a/crates/sui-core/src/test_utils.rs b/crates/sui-core/src/test_utils.rs index 54aeb85be2042..4a37323187d3e 100644 --- a/crates/sui-core/src/test_utils.rs +++ b/crates/sui-core/src/test_utils.rs @@ -1,13 +1,16 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use crate::authority::{AuthorityState, EffectsNotifyRead}; +use crate::authority::{ + test_authority_builder::TestAuthorityBuilder, AuthorityState, EffectsNotifyRead, +}; use crate::authority_aggregator::{AuthorityAggregator, TimeoutConfig}; use crate::epoch::committee_store::CommitteeStore; use crate::state_accumulator::StateAccumulator; use crate::test_authority_clients::LocalAuthorityClient; use fastcrypto::hash::MultisetHash; use fastcrypto::traits::KeyPair; +use futures::future::join_all; use move_core_types::account_address::AccountAddress; use move_core_types::ident_str; use prometheus::Registry; @@ -18,6 +21,7 @@ use std::sync::Arc; use std::time::Duration; use sui_config::genesis::Genesis; use sui_config::local_ip_utils; +use sui_config::node::OverloadThresholdConfig; use sui_framework::BuiltInFramework; use sui_genesis_builder::validator_info::ValidatorInfo; use sui_move_build::{BuildConfig, CompiledPackage, SuiPackageHooks}; @@ -266,26 +270,50 @@ pub async fn init_local_authorities( ObjectID, ) { let (genesis, key_pairs, framework) = init_genesis(committee_size, genesis_objects).await; - let (aggregator, authorities) = init_local_authorities_with_genesis(&genesis, key_pairs).await; + let authorities = join_all(key_pairs.iter().map(|(_, key_pair)| { + TestAuthorityBuilder::new() + .with_genesis_and_keypair(&genesis, key_pair) + .build() + })) + .await; + let aggregator = init_local_authorities_with_genesis(&genesis, authorities.clone()).await; (aggregator, authorities, genesis, framework) } -pub async fn init_local_authorities_with_genesis( - genesis: &Genesis, - key_pairs: Vec<(AuthorityPublicKeyBytes, AuthorityKeyPair)>, +pub async fn init_local_authorities_with_overload_thresholds( + committee_size: usize, + genesis_objects: Vec, + overload_thresholds: OverloadThresholdConfig, ) -> ( AuthorityAggregator, Vec>, + Genesis, + ObjectID, ) { + let (genesis, key_pairs, framework) = init_genesis(committee_size, genesis_objects).await; + let authorities = join_all(key_pairs.iter().map(|(_, key_pair)| { + TestAuthorityBuilder::new() + .with_genesis_and_keypair(&genesis, key_pair) + .with_overload_threshold_config(overload_thresholds.clone()) + .build() + })) + .await; + let aggregator = init_local_authorities_with_genesis(&genesis, authorities.clone()).await; + (aggregator, authorities, genesis, framework) +} + +pub async fn init_local_authorities_with_genesis( + genesis: &Genesis, + authorities: Vec>, +) -> AuthorityAggregator { telemetry_subscribers::init_for_testing(); let committee = genesis.committee().unwrap(); let mut clients = BTreeMap::new(); - let mut states = Vec::new(); - for (authority_name, secret) in key_pairs { - let client = LocalAuthorityClient::new(secret, genesis).await; - states.push(client.state.clone()); - clients.insert(authority_name, client); + for state in authorities { + let name = state.name; + let client = LocalAuthorityClient::new_from_authority(state); + clients.insert(name, client); } let timeouts = TimeoutConfig { pre_quorum_timeout: Duration::from_secs(5), @@ -293,16 +321,13 @@ pub async fn init_local_authorities_with_genesis( serial_authority_request_interval: Duration::from_secs(1), }; let committee_store = Arc::new(CommitteeStore::new_for_testing(&committee)); - ( - AuthorityAggregator::new_with_timeouts( - committee, - committee_store, - clients, - &Registry::new(), - Arc::new(HashMap::new()), - timeouts, - ), - states, + AuthorityAggregator::new_with_timeouts( + committee, + committee_store, + clients, + &Registry::new(), + Arc::new(HashMap::new()), + timeouts, ) } diff --git a/crates/sui-core/src/transaction_manager.rs b/crates/sui-core/src/transaction_manager.rs index aff12823ee85d..33bbdbaf799af 100644 --- a/crates/sui-core/src/transaction_manager.rs +++ b/crates/sui-core/src/transaction_manager.rs @@ -5,8 +5,10 @@ use std::{ cmp::max, collections::{BTreeMap, BTreeSet, HashMap, HashSet}, sync::Arc, + time::{Duration, Instant}, }; +use indexmap::IndexMap; use lru::LruCache; use mysten_metrics::monitored_scope; use parking_lot::RwLock; @@ -246,10 +248,10 @@ struct Inner { // Maps input objects to transactions waiting for locks on the object. lock_waiters: HashMap, - // Number of transactions that depend on each object ID. Should match exactly with total - // number of transactions per object ID prefix in the missing_inputs table. + // Stores age info for all transactions depending on each object. // Used for throttling signing and submitting transactions depending on hot objects. - input_objects: HashMap, + // An `IndexMap` is used to ensure that the insertion order is preserved. + input_objects: HashMap>, // Maps object IDs to the highest observed sequence number of the object. When the value is // None, indicates that the object is immutable, corresponding to an InputKey with no sequence @@ -312,7 +314,7 @@ impl Inner { return ready_certificates; } - let input_count = self + let input_txns = self .input_objects .get_mut(&input_key.id()) .unwrap_or_else(|| { @@ -321,8 +323,12 @@ impl Inner { input_key.id() ) }); - *input_count -= digests.len(); - if *input_count == 0 { + for digest in digests.iter() { + // The digest of the transaction must be inside the map. + assert!(input_txns.shift_remove(digest).is_some()); + } + + if input_txns.is_empty() { self.input_objects.remove(&input_key.id()); } @@ -665,8 +671,8 @@ impl TransactionManager { } if acquire { pending_cert.acquiring_locks.insert(key, lock_mode); - let input_count = inner.input_objects.entry(key.id()).or_default(); - *input_count += 1; + let input_txns = inner.input_objects.entry(key.id()).or_default(); + input_txns.insert(digest, Instant::now()); } else { pending_cert.acquired_locks.insert(key, lock_mode); } @@ -836,14 +842,20 @@ impl TransactionManager { .map(|cert| cert.acquiring_locks.keys().cloned().collect()) } - // Returns the number of transactions waiting on each object ID. - pub(crate) fn objects_queue_len(&self, keys: Vec) -> Vec<(ObjectID, usize)> { + // Returns the number of transactions waiting on each object ID, as well as the age of the oldest transaction in the queue. + pub(crate) fn objects_queue_len_and_age( + &self, + keys: Vec, + ) -> Vec<(ObjectID, usize, Option)> { let inner = self.inner.read(); keys.into_iter() .map(|key| { + let default_map = IndexMap::new(); + let txns = inner.input_objects.get(&key).unwrap_or(&default_map); ( key, - inner.input_objects.get(&key).cloned().unwrap_or_default(), + txns.len(), + txns.first().map(|(_, time)| time.elapsed()), ) }) .collect() @@ -862,7 +874,11 @@ impl TransactionManager { *inner = Inner::new(new_epoch, self.metrics.clone()); } - pub(crate) fn check_execution_overload(&self, tx_data: &SenderSignedData) -> SuiResult { + pub(crate) fn check_execution_overload( + &self, + txn_age_threshold: Duration, + tx_data: &SenderSignedData, + ) -> SuiResult { // Too many transactions are pending execution. let inflight_queue_len = self.inflight_queue_len(); fp_ensure!( @@ -873,7 +889,7 @@ impl TransactionManager { } ); - for (object_id, queue_len) in self.objects_queue_len( + for (object_id, queue_len, txn_age) in self.objects_queue_len_and_age( tx_data .transaction_data() .input_objects()? @@ -890,6 +906,17 @@ impl TransactionManager { threshold: MAX_PER_OBJECT_QUEUE_LENGTH, } ); + if let Some(age) = txn_age { + // Check that we don't have a txn that has been waiting for a long time in the queue. + fp_ensure!( + age < txn_age_threshold, + SuiError::TooOldTransactionPendingOnObject { + object_id, + txn_age_sec: age.as_secs(), + threshold: txn_age_threshold.as_secs(), + } + ); + } } Ok(()) } diff --git a/crates/sui-core/src/unit_tests/execution_driver_tests.rs b/crates/sui-core/src/unit_tests/execution_driver_tests.rs index 258ef56f197ff..e5719062ed4ad 100644 --- a/crates/sui-core/src/unit_tests/execution_driver_tests.rs +++ b/crates/sui-core/src/unit_tests/execution_driver_tests.rs @@ -8,7 +8,10 @@ use crate::authority_aggregator::authority_aggregator_tests::{ }; use crate::safe_client::SafeClient; use crate::test_authority_clients::LocalAuthorityClient; -use crate::test_utils::{init_local_authorities, make_transfer_object_move_transaction}; +use crate::test_utils::{ + init_local_authorities, init_local_authorities_with_overload_thresholds, + make_transfer_object_move_transaction, +}; use crate::transaction_manager::MAX_PER_OBJECT_QUEUE_LENGTH; use std::collections::BTreeSet; @@ -16,6 +19,7 @@ use std::sync::Arc; use std::time::Duration; use itertools::Itertools; +use sui_config::node::OverloadThresholdConfig; use sui_test_transaction_builder::TestTransactionBuilder; use sui_types::base_types::TransactionDigest; use sui_types::committee::Committee; @@ -560,7 +564,7 @@ async fn test_per_object_overload() { .build_and_sign(&key); let res = authorities[3] .transaction_manager() - .check_execution_overload(shared_txn.data()); + .check_execution_overload(authorities[3].max_txn_age_in_queue(), shared_txn.data()); let message = format!("{res:?}"); assert!( message.contains("TooManyTransactionsPendingOnObject"), @@ -568,3 +572,130 @@ async fn test_per_object_overload() { message ); } + +#[tokio::test] +async fn test_txn_age_overload() { + telemetry_subscribers::init_for_testing(); + + // Initialize a network with 1 account and 3 gas objects. + let (addr, key): (_, AccountKeyPair) = get_key_pair(); + let gas_objects = (0..3) + .map(|_| Object::with_owner_for_testing(addr)) + .collect_vec(); + let (aggregator, authorities, _genesis, package) = + init_local_authorities_with_overload_thresholds( + 4, + gas_objects.clone(), + OverloadThresholdConfig { + max_txn_age_in_queue: Duration::from_secs(5), + }, + ) + .await; + let rgp = authorities + .get(0) + .unwrap() + .reference_gas_price_for_testing() + .unwrap(); + let authority_clients: Vec<_> = authorities + .iter() + .map(|a| aggregator.authority_clients[&a.name].clone()) + .collect(); + + // Create a shared counter. + let gas_ref = get_latest_ref(authority_clients[0].clone(), gas_objects[0].id()).await; + let create_counter_txn = TestTransactionBuilder::new(addr, gas_ref, rgp) + .call_counter_create(package) + .build_and_sign(&key); + let create_counter_cert = try_sign_on_first_three_authorities( + &authority_clients, + &aggregator.committee, + &create_counter_txn, + ) + .await + .unwrap(); + for authority in authorities.iter().take(3) { + send_consensus(authority, &create_counter_cert).await; + } + for authority in authorities.iter().take(3) { + authority + .database + .notify_read_executed_effects(vec![*create_counter_cert.digest()]) + .await + .unwrap() + .pop() + .unwrap(); + } + + // Signing and executing this transaction on the last authority should succeed. + authority_clients[3] + .handle_transaction(create_counter_txn.clone()) + .await + .unwrap(); + send_consensus(&authorities[3], &create_counter_cert).await; + let create_counter_effects = authorities[3] + .database + .notify_read_executed_effects(vec![*create_counter_cert.digest()]) + .await + .unwrap() + .pop() + .unwrap(); + let (shared_counter_ref, owner) = create_counter_effects.created()[0]; + let Owner::Shared { + initial_shared_version: shared_counter_initial_version, + } = owner + else { + panic!("Not a shared object! {:?} {:?}", shared_counter_ref, owner); + }; + + // Stop execution on the last authority, to simulate having a backlog. + authorities[3].shutdown_execution_for_test(); + // Make sure execution driver has exited. + sleep(Duration::from_secs(1)).await; + + // Sign and try execute 2 txns on the first three authorities. And enqueue them on the last authority. + // First shared counter txn has input object available on authority 3. So to put a txn in the queue, we + // will need another txn. + for gas_object in gas_objects.iter().take(2) { + let gas_ref = get_latest_ref(authority_clients[0].clone(), gas_object.id()).await; + let shared_txn = TestTransactionBuilder::new(addr, gas_ref, rgp) + .call_counter_increment( + package, + shared_counter_ref.0, + shared_counter_initial_version, + ) + .build_and_sign(&key); + let shared_cert = try_sign_on_first_three_authorities( + &authority_clients, + &aggregator.committee, + &shared_txn, + ) + .await + .unwrap(); + for authority in authorities.iter().take(3) { + send_consensus(authority, &shared_cert).await; + } + send_consensus(&authorities[3], &shared_cert).await; + } + + // Sleep for 6 seconds to make sure the transaction is old enough since our threshold is 5. + tokio::time::sleep(Duration::from_secs(6)).await; + + // Trying to sign a new transaction would now fail. + let gas_ref = get_latest_ref(authority_clients[0].clone(), gas_objects[2].id()).await; + let shared_txn = TestTransactionBuilder::new(addr, gas_ref, rgp) + .call_counter_increment( + package, + shared_counter_ref.0, + shared_counter_initial_version, + ) + .build_and_sign(&key); + let res = authorities[3] + .transaction_manager() + .check_execution_overload(authorities[3].max_txn_age_in_queue(), shared_txn.data()); + let message = format!("{res:?}"); + assert!( + message.contains("TooOldTransactionPendingOnObject"), + "{}", + message + ); +} diff --git a/crates/sui-node/src/lib.rs b/crates/sui-node/src/lib.rs index d17efc5562efd..6723a092fd4c5 100644 --- a/crates/sui-node/src/lib.rs +++ b/crates/sui-node/src/lib.rs @@ -580,6 +580,7 @@ impl SuiNode { config.certificate_deny_config.clone(), config.indirect_objects_threshold, config.state_debug_dump_config.clone(), + config.overload_threshold_config.clone(), archive_readers, ) .await; diff --git a/crates/sui-swarm-config/src/node_config_builder.rs b/crates/sui-swarm-config/src/node_config_builder.rs index 01ec97ce5da6a..7ede476080f4e 100644 --- a/crates/sui-swarm-config/src/node_config_builder.rs +++ b/crates/sui-swarm-config/src/node_config_builder.rs @@ -170,6 +170,7 @@ impl ValidatorConfigBuilder { .map(|i| i.as_secs()) .unwrap_or(3600), zklogin_oauth_providers: default_zklogin_oauth_providers(), + overload_threshold_config: Default::default(), } } @@ -406,6 +407,7 @@ impl FullnodeConfigBuilder { // note: not used by fullnodes. jwk_fetch_interval_seconds: 3600, zklogin_oauth_providers: default_zklogin_oauth_providers(), + overload_threshold_config: Default::default(), } } } diff --git a/crates/sui-swarm-config/tests/snapshots/snapshot_tests__network_config_snapshot_matches.snap b/crates/sui-swarm-config/tests/snapshots/snapshot_tests__network_config_snapshot_matches.snap index 77c3f903d8116..fdebd39d3ce1a 100644 --- a/crates/sui-swarm-config/tests/snapshots/snapshot_tests__network_config_snapshot_matches.snap +++ b/crates/sui-swarm-config/tests/snapshots/snapshot_tests__network_config_snapshot_matches.snap @@ -109,6 +109,10 @@ validator_configs: - Kakao - Slack - Twitch + overload-threshold-config: + max_txn_age_in_queue: + secs: 1 + nanos: 0 - protocol-key-pair: value: avYcyVgYMXTyaUYh9IRwLK0gSzl7YF6ZQDAbrS1Bhvo= worker-key-pair: @@ -215,6 +219,10 @@ validator_configs: - Kakao - Slack - Twitch + overload-threshold-config: + max_txn_age_in_queue: + secs: 1 + nanos: 0 - protocol-key-pair: value: OXnx3yM1C/ppgnDMx/o1d49fJs7E05kq11mXNae/O+I= worker-key-pair: @@ -321,6 +329,10 @@ validator_configs: - Kakao - Slack - Twitch + overload-threshold-config: + max_txn_age_in_queue: + secs: 1 + nanos: 0 - protocol-key-pair: value: CyNkjqNVr3HrHTH7f/NLs7u5lUHJzuPAw0PqMTD2y2s= worker-key-pair: @@ -427,6 +439,10 @@ validator_configs: - Kakao - Slack - Twitch + overload-threshold-config: + max_txn_age_in_queue: + secs: 1 + nanos: 0 - protocol-key-pair: value: X/I/kM+KvHcxAKEf2UU6Sr7SpN3bhiE9nP5CuM/iIY0= worker-key-pair: @@ -533,6 +549,10 @@ validator_configs: - Kakao - Slack - Twitch + overload-threshold-config: + max_txn_age_in_queue: + secs: 1 + nanos: 0 - protocol-key-pair: value: N272EiFDyKtxRbDKbyN6ujenJ+skPcRoc/XolpOLGnU= worker-key-pair: @@ -639,6 +659,10 @@ validator_configs: - Kakao - Slack - Twitch + overload-threshold-config: + max_txn_age_in_queue: + secs: 1 + nanos: 0 - protocol-key-pair: value: a74f03IOjL8ZFSWFChFVEi+wiMwHNwNCPDGIYkGfgjs= worker-key-pair: @@ -745,6 +769,10 @@ validator_configs: - Kakao - Slack - Twitch + overload-threshold-config: + max_txn_age_in_queue: + secs: 1 + nanos: 0 account_keys: - Hloy4pnf8pWEHGP+4OFsXz56bLdIJhkD2O+OdKMqCA4= - pvMScjoMR/DaN0M5IOxS2VpGC59N6kv6gDm63ufLQ5w= diff --git a/crates/sui-types/src/error.rs b/crates/sui-types/src/error.rs index c4fbaca401ad3..498fd11c56b2b 100644 --- a/crates/sui-types/src/error.rs +++ b/crates/sui-types/src/error.rs @@ -316,6 +316,13 @@ pub enum SuiError { threshold: usize, }, + #[error("Input {object_id} has a transaction {txn_age_sec} seconds old pending, above threshold of {threshold} seconds")] + TooOldTransactionPendingOnObject { + object_id: ObjectID, + txn_age_sec: u64, + threshold: u64, + }, + // Signature verification #[error("Signature is not valid: {}", error)] InvalidSignature { error: String }, @@ -735,6 +742,7 @@ impl SuiError { // Overload errors SuiError::TooManyTransactionsPendingExecution { .. } => (true, true), SuiError::TooManyTransactionsPendingOnObject { .. } => (true, true), + SuiError::TooOldTransactionPendingOnObject { .. } => (true, true), SuiError::TooManyTransactionsPendingConsensus => (true, true), // Non retryable error @@ -769,6 +777,7 @@ impl SuiError { self, SuiError::TooManyTransactionsPendingExecution { .. } | SuiError::TooManyTransactionsPendingOnObject { .. } + | SuiError::TooOldTransactionPendingOnObject { .. } | SuiError::TooManyTransactionsPendingConsensus ) }