From e7c1dfa843085d13c05453def54d329663254d9f Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:47:37 +0000 Subject: [PATCH 01/18] feat: first working FallbackProvider refactor --- .../src/rpc_clients/fallback.rs | 84 +++++++++++-------- .../hyperlane-ethereum/src/trait_builder.rs | 40 ++++++++- rust/ethers-prometheus/src/json_rpc_client.rs | 10 +++ rust/hyperlane-core/src/lib.rs | 2 +- 4 files changed, 100 insertions(+), 36 deletions(-) diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index af1dd28ce2..f3ad4fb8e5 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -1,4 +1,5 @@ use derive_new::new; +use hyperlane_core::{error::HyperlaneCustomError, ChainCommunicationError}; use std::fmt::{Debug, Formatter}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -16,9 +17,9 @@ use tracing::{info, instrument, warn_span}; use ethers_prometheus::json_rpc_client::PrometheusJsonRpcClientConfigExt; use crate::rpc_clients::{categorize_client_response, CategorizedResponse}; +use crate::BlockNumberGetter; const MAX_BLOCK_TIME: Duration = Duration::from_secs(2 * 60); -const BLOCK_NUMBER_RPC: &str = "eth_blockNumber"; #[derive(Clone, Copy, new)] struct PrioritizedProviderInner { @@ -45,6 +46,8 @@ struct PrioritizedProviders { priorities: RwLock>, } +// : Into> + /// A provider that bundles multiple providers and attempts to call the first, /// then the second, and so on until a response is received. pub struct FallbackProvider { @@ -52,6 +55,9 @@ pub struct FallbackProvider { max_block_time: Duration, } +// in the fallback provider of the T, you want T to implement Into>, so you can cast and then get the block number. +// by requiring this Into impl means T can be anything and you can create the implementation in your own crate. + impl Clone for FallbackProvider { fn clone(&self) -> Self { Self { @@ -90,7 +96,11 @@ where } } -impl FallbackProvider { +impl FallbackProvider +where + T: Debug, + for<'a> &'a T: Into>, +{ /// Convenience method for creating a `FallbackProviderBuilder` with same /// `JsonRpcClient` types pub fn builder() -> FallbackProviderBuilder { @@ -101,16 +111,32 @@ impl FallbackProvider { pub fn new(providers: impl IntoIterator) -> Self { Self::builder().add_providers(providers).build() } -} -impl FallbackProvider -where - C: JsonRpcClient, -{ + async fn deprioritize_provider(&self, priority: PrioritizedProviderInner) { + // De-prioritize the current provider by moving it to the end of the queue + let mut priorities = self.inner.priorities.write().await; + priorities.retain(|&p| p.index != priority.index); + priorities.push(priority); + } + + async fn update_last_seen_block(&self, provider_index: usize, current_block_height: u64) { + let mut priorities = self.inner.priorities.write().await; + // Get provider position in the up-to-date priorities vec + if let Some(position) = priorities.iter().position(|p| p.index == provider_index) { + priorities[position] = + PrioritizedProviderInner::from_block_height(provider_index, current_block_height); + } + } + + async fn take_priorities_snapshot(&self) -> Vec { + let read_lock = self.inner.priorities.read().await; + (*read_lock).clone() + } + async fn handle_stalled_provider( &self, priority: &PrioritizedProviderInner, - provider: &C, + provider: &T, ) -> Result<(), ProviderError> { let now = Instant::now(); if now @@ -121,17 +147,17 @@ where return Ok(()); } - let current_block_height: u64 = provider - .request(BLOCK_NUMBER_RPC, ()) + let block_getter: Box = (provider).into(); + let current_block_height = block_getter + .get() .await - .map(|r: U64| r.as_u64()) .unwrap_or(priority.last_block_height.0); if current_block_height <= priority.last_block_height.0 { // The `max_block_time` elapsed but the block number returned by the provider has not increased self.deprioritize_provider(*priority).await; info!( provider_index=%priority.index, - ?provider, + provider=?self.inner.providers[priority.index], "Deprioritizing an inner provider in FallbackProvider", ); } else { @@ -140,29 +166,10 @@ where } Ok(()) } - - async fn deprioritize_provider(&self, priority: PrioritizedProviderInner) { - // De-prioritize the current provider by moving it to the end of the queue - let mut priorities = self.inner.priorities.write().await; - priorities.retain(|&p| p.index != priority.index); - priorities.push(priority); - } - - async fn update_last_seen_block(&self, provider_index: usize, current_block_height: u64) { - let mut priorities = self.inner.priorities.write().await; - // Get provider position in the up-to-date priorities vec - if let Some(position) = priorities.iter().position(|p| p.index == provider_index) { - priorities[position] = - PrioritizedProviderInner::from_block_height(provider_index, current_block_height); - } - } - - async fn take_priorities_snapshot(&self) -> Vec { - let read_lock = self.inner.priorities.read().await; - (*read_lock).clone() - } } +impl FallbackProvider where C: JsonRpcClient {} + /// Builder to create a new fallback provider. #[derive(Debug, Clone)] pub struct FallbackProviderBuilder { @@ -237,6 +244,7 @@ impl From for ProviderError { impl JsonRpcClient for FallbackProvider where C: JsonRpcClient + PrometheusJsonRpcClientConfigExt, + for<'a> &'a C: Into>, { type Error = ProviderError; @@ -281,10 +289,12 @@ where #[cfg(test)] mod tests { + use crate::{JsonRpcBlockGetter, BLOCK_NUMBER_RPC}; + use super::*; use std::sync::Mutex; - #[derive(Debug)] + #[derive(Debug, Clone)] struct ProviderMock { // Store requests as tuples of (method, params) // Even if the tests were single-threaded, need the arc-mutex @@ -311,6 +321,12 @@ mod tests { } } + impl Into> for &ProviderMock { + fn into(self) -> Box { + Box::new(JsonRpcBlockGetter::new(self.clone())) + } + } + fn dummy_return_value() -> Result { serde_json::from_str("0").map_err(|e| HttpClientError::SerdeJson { err: e, diff --git a/rust/chains/hyperlane-ethereum/src/trait_builder.rs b/rust/chains/hyperlane-ethereum/src/trait_builder.rs index 89e4f31d4f..92fdd68000 100644 --- a/rust/chains/hyperlane-ethereum/src/trait_builder.rs +++ b/rust/chains/hyperlane-ethereum/src/trait_builder.rs @@ -1,8 +1,9 @@ -use std::fmt::Write; +use std::fmt::{Debug, Write}; use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; +use derive_new::new; use ethers::middleware::gas_oracle::{ GasCategory, GasOracle, GasOracleMiddleware, Polygon, ProviderOracle, }; @@ -10,6 +11,8 @@ use ethers::prelude::{ Http, JsonRpcClient, Middleware, NonceManagerMiddleware, Provider, Quorum, QuorumProvider, SignerMiddleware, WeightedProvider, Ws, WsClientError, }; +use ethers::providers::ProviderError; +use ethers_core::types::U64; use hyperlane_core::metrics::agent::METRICS_SCRAPE_INTERVAL; use reqwest::{Client, Url}; use thiserror::Error; @@ -278,3 +281,38 @@ where }; Ok(GasOracleMiddleware::new(provider, gas_oracle)) } + +pub const BLOCK_NUMBER_RPC: &str = "eth_blockNumber"; + +#[async_trait] +pub trait BlockNumberGetter: Send + Sync + Debug { + async fn get(&self) -> Result; +} + +#[derive(Debug, new)] +pub struct JsonRpcBlockGetter(T); + +#[async_trait] +impl BlockNumberGetter for JsonRpcBlockGetter +where + C: JsonRpcClient + Clone, +{ + async fn get(&self) -> Result { + let res = self + .0 + .request(BLOCK_NUMBER_RPC, ()) + .await + .map(|r: U64| r.as_u64()) + .map_err(Into::into)?; + Ok(res) + } +} + +impl Into> + for &PrometheusJsonRpcClient +{ + fn into(self) -> Box { + let client = self.clone(); + Box::new(JsonRpcBlockGetter(client)) + } +} diff --git a/rust/ethers-prometheus/src/json_rpc_client.rs b/rust/ethers-prometheus/src/json_rpc_client.rs index 4116d7d63a..2d58191c77 100644 --- a/rust/ethers-prometheus/src/json_rpc_client.rs +++ b/rust/ethers-prometheus/src/json_rpc_client.rs @@ -111,6 +111,16 @@ pub struct PrometheusJsonRpcClient { config: PrometheusJsonRpcClientConfig, } +impl Clone for PrometheusJsonRpcClient { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + metrics: self.metrics.clone(), + config: self.config.clone(), + } + } +} + impl Debug for PrometheusJsonRpcClient where C: JsonRpcClient, diff --git a/rust/hyperlane-core/src/lib.rs b/rust/hyperlane-core/src/lib.rs index 6834df3951..7b9c519f87 100644 --- a/rust/hyperlane-core/src/lib.rs +++ b/rust/hyperlane-core/src/lib.rs @@ -33,7 +33,7 @@ pub mod metrics; mod types; mod chain; -mod error; +pub mod error; /// Enum for validity of a list of messages #[derive(Debug)] From 3528598dee5d18635cc1e2c0d0006311285ad531 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:57:48 +0000 Subject: [PATCH 02/18] chore: clean up --- .../hyperlane-ethereum/src/rpc_clients/fallback.rs | 13 +++++++------ rust/chains/hyperlane-ethereum/src/trait_builder.rs | 9 +++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index f3ad4fb8e5..fea327cf3e 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -98,8 +98,7 @@ where impl FallbackProvider where - T: Debug, - for<'a> &'a T: Into>, + T: Debug + Into> + Clone, { /// Convenience method for creating a `FallbackProviderBuilder` with same /// `JsonRpcClient` types @@ -147,7 +146,7 @@ where return Ok(()); } - let block_getter: Box = (provider).into(); + let block_getter: Box = provider.clone().into(); let current_block_height = block_getter .get() .await @@ -243,8 +242,10 @@ impl From for ProviderError { #[cfg_attr(not(target_arch = "wasm32"), async_trait)] impl JsonRpcClient for FallbackProvider where - C: JsonRpcClient + PrometheusJsonRpcClientConfigExt, - for<'a> &'a C: Into>, + C: JsonRpcClient + + PrometheusJsonRpcClientConfigExt + + Into> + + Clone, { type Error = ProviderError; @@ -321,7 +322,7 @@ mod tests { } } - impl Into> for &ProviderMock { + impl Into> for ProviderMock { fn into(self) -> Box { Box::new(JsonRpcBlockGetter::new(self.clone())) } diff --git a/rust/chains/hyperlane-ethereum/src/trait_builder.rs b/rust/chains/hyperlane-ethereum/src/trait_builder.rs index 92fdd68000..7c9f30d70f 100644 --- a/rust/chains/hyperlane-ethereum/src/trait_builder.rs +++ b/rust/chains/hyperlane-ethereum/src/trait_builder.rs @@ -295,7 +295,7 @@ pub struct JsonRpcBlockGetter(T); #[async_trait] impl BlockNumberGetter for JsonRpcBlockGetter where - C: JsonRpcClient + Clone, + C: JsonRpcClient, { async fn get(&self) -> Result { let res = self @@ -308,11 +308,8 @@ where } } -impl Into> - for &PrometheusJsonRpcClient -{ +impl Into> for PrometheusJsonRpcClient { fn into(self) -> Box { - let client = self.clone(); - Box::new(JsonRpcBlockGetter(client)) + Box::new(JsonRpcBlockGetter(self)) } } From 9c2b2e1eb11b6a3a88bce2a00421ec9a463e56d6 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Sun, 7 Jan 2024 21:44:30 +0000 Subject: [PATCH 03/18] progress on moving FallbackProvider logic into hyperlane-core --- rust/Cargo.lock | 2 + rust/chains/hyperlane-ethereum/src/error.rs | 19 ++ rust/chains/hyperlane-ethereum/src/lib.rs | 1 + .../src/rpc_clients/fallback.rs | 211 +++--------------- .../hyperlane-ethereum/src/trait_builder.rs | 49 +--- rust/ethers-prometheus/Cargo.toml | 1 + rust/ethers-prometheus/src/json_rpc_client.rs | 30 +++ rust/hyperlane-core/Cargo.toml | 2 + rust/hyperlane-core/src/error.rs | 4 + rust/hyperlane-core/src/lib.rs | 2 + rust/hyperlane-core/src/rpc_clients/error.rs | 11 + .../src/rpc_clients/fallback.rs | 181 +++++++++++++++ rust/hyperlane-core/src/rpc_clients/mod.rs | 2 + 13 files changed, 291 insertions(+), 224 deletions(-) create mode 100644 rust/chains/hyperlane-ethereum/src/error.rs create mode 100644 rust/hyperlane-core/src/rpc_clients/error.rs create mode 100644 rust/hyperlane-core/src/rpc_clients/fallback.rs create mode 100644 rust/hyperlane-core/src/rpc_clients/mod.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c20fa554bb..b109a8c76e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2786,6 +2786,7 @@ dependencies = [ "derive-new", "derive_builder", "ethers", + "ethers-core", "futures", "hyperlane-core", "log", @@ -4120,6 +4121,7 @@ dependencies = [ "thiserror", "tiny-keccak", "tokio", + "tracing", "uint", ] diff --git a/rust/chains/hyperlane-ethereum/src/error.rs b/rust/chains/hyperlane-ethereum/src/error.rs new file mode 100644 index 0000000000..77fae64890 --- /dev/null +++ b/rust/chains/hyperlane-ethereum/src/error.rs @@ -0,0 +1,19 @@ +use ethers::providers::ProviderError; +use hyperlane_core::ChainCommunicationError; + +/// Errors from the crates specific to the hyperlane-cosmos +/// implementation. +/// This error can then be converted into the broader error type +/// in hyperlane-core using the `From` trait impl +#[derive(Debug, thiserror::Error)] +pub enum HyperlaneEthereumError { + /// provider Error + #[error("{0}")] + ProviderError(#[from] ProviderError), +} + +impl From for ChainCommunicationError { + fn from(value: HyperlaneEthereumError) -> Self { + ChainCommunicationError::from_other(value) + } +} diff --git a/rust/chains/hyperlane-ethereum/src/lib.rs b/rust/chains/hyperlane-ethereum/src/lib.rs index 2d42850bc4..90ec70c019 100644 --- a/rust/chains/hyperlane-ethereum/src/lib.rs +++ b/rust/chains/hyperlane-ethereum/src/lib.rs @@ -75,6 +75,7 @@ mod signers; mod singleton_signer; mod config; +mod error; fn extract_fn_map(abi: &'static Lazy) -> HashMap, &'static str> { abi.functions() diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index fea327cf3e..3ec6a193c5 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -1,73 +1,34 @@ use derive_new::new; -use hyperlane_core::{error::HyperlaneCustomError, ChainCommunicationError}; +use hyperlane_core::rpc_clients::fallback::{BlockNumberGetter, FallbackProvider}; use std::fmt::{Debug, Formatter}; -use std::sync::Arc; +use std::ops::Deref; use std::time::{Duration, Instant}; -use tokio::sync::RwLock; +use thiserror::Error; use async_trait::async_trait; use ethers::providers::{HttpClientError, JsonRpcClient, ProviderError}; -use ethers_core::types::U64; use serde::{de::DeserializeOwned, Serialize}; use serde_json::Value; -use thiserror::Error; use tokio::time::sleep; use tracing::{info, instrument, warn_span}; use ethers_prometheus::json_rpc_client::PrometheusJsonRpcClientConfigExt; +use crate::error::HyperlaneEthereumError; use crate::rpc_clients::{categorize_client_response, CategorizedResponse}; -use crate::BlockNumberGetter; - -const MAX_BLOCK_TIME: Duration = Duration::from_secs(2 * 60); - -#[derive(Clone, Copy, new)] -struct PrioritizedProviderInner { - // Index into the `providers` field of `PrioritizedProviders` - index: usize, - // Tuple of the block number and the time when it was queried - #[new(value = "(0, Instant::now())")] - last_block_height: (u64, Instant), -} - -impl PrioritizedProviderInner { - fn from_block_height(index: usize, block_height: u64) -> Self { - Self { - index, - last_block_height: (block_height, Instant::now()), - } - } -} -struct PrioritizedProviders { - /// Sorted list of providers this provider calls in order of most primary to - /// most fallback. - providers: Vec, - priorities: RwLock>, -} +#[derive(new)] +pub struct EthereumFallbackProvider(FallbackProvider); -// : Into> +impl Deref for EthereumFallbackProvider { + type Target = FallbackProvider; -/// A provider that bundles multiple providers and attempts to call the first, -/// then the second, and so on until a response is received. -pub struct FallbackProvider { - inner: Arc>, - max_block_time: Duration, -} - -// in the fallback provider of the T, you want T to implement Into>, so you can cast and then get the block number. -// by requiring this Into impl means T can be anything and you can create the implementation in your own crate. - -impl Clone for FallbackProvider { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - max_block_time: self.max_block_time, - } + fn deref(&self) -> &Self::Target { + &self.0 } } -impl Debug for FallbackProvider +impl Debug for EthereumFallbackProvider where C: JsonRpcClient + PrometheusJsonRpcClientConfigExt, { @@ -76,6 +37,7 @@ where .field( "chain_name", &self + .0 .inner .providers .get(0) @@ -85,6 +47,7 @@ where .field( "hosts", &self + .0 .inner .providers .iter() @@ -96,134 +59,6 @@ where } } -impl FallbackProvider -where - T: Debug + Into> + Clone, -{ - /// Convenience method for creating a `FallbackProviderBuilder` with same - /// `JsonRpcClient` types - pub fn builder() -> FallbackProviderBuilder { - FallbackProviderBuilder::default() - } - - /// Create a new fallback provider - pub fn new(providers: impl IntoIterator) -> Self { - Self::builder().add_providers(providers).build() - } - - async fn deprioritize_provider(&self, priority: PrioritizedProviderInner) { - // De-prioritize the current provider by moving it to the end of the queue - let mut priorities = self.inner.priorities.write().await; - priorities.retain(|&p| p.index != priority.index); - priorities.push(priority); - } - - async fn update_last_seen_block(&self, provider_index: usize, current_block_height: u64) { - let mut priorities = self.inner.priorities.write().await; - // Get provider position in the up-to-date priorities vec - if let Some(position) = priorities.iter().position(|p| p.index == provider_index) { - priorities[position] = - PrioritizedProviderInner::from_block_height(provider_index, current_block_height); - } - } - - async fn take_priorities_snapshot(&self) -> Vec { - let read_lock = self.inner.priorities.read().await; - (*read_lock).clone() - } - - async fn handle_stalled_provider( - &self, - priority: &PrioritizedProviderInner, - provider: &T, - ) -> Result<(), ProviderError> { - let now = Instant::now(); - if now - .duration_since(priority.last_block_height.1) - .le(&self.max_block_time) - { - // Do nothing, it's too early to tell if the provider has stalled - return Ok(()); - } - - let block_getter: Box = provider.clone().into(); - let current_block_height = block_getter - .get() - .await - .unwrap_or(priority.last_block_height.0); - if current_block_height <= priority.last_block_height.0 { - // The `max_block_time` elapsed but the block number returned by the provider has not increased - self.deprioritize_provider(*priority).await; - info!( - provider_index=%priority.index, - provider=?self.inner.providers[priority.index], - "Deprioritizing an inner provider in FallbackProvider", - ); - } else { - self.update_last_seen_block(priority.index, current_block_height) - .await; - } - Ok(()) - } -} - -impl FallbackProvider where C: JsonRpcClient {} - -/// Builder to create a new fallback provider. -#[derive(Debug, Clone)] -pub struct FallbackProviderBuilder { - providers: Vec, - max_block_time: Duration, -} - -impl Default for FallbackProviderBuilder { - fn default() -> Self { - Self { - providers: Vec::new(), - max_block_time: MAX_BLOCK_TIME, - } - } -} - -impl FallbackProviderBuilder { - /// Add a new provider to the set. Each new provider will be a lower - /// priority than the previous. - pub fn add_provider(mut self, provider: T) -> Self { - self.providers.push(provider); - self - } - - /// Add many providers sorted by highest priority to lowest. - pub fn add_providers(mut self, providers: impl IntoIterator) -> Self { - self.providers.extend(providers); - self - } - - #[cfg(test)] - pub fn with_max_block_time(mut self, max_block_time: Duration) -> Self { - self.max_block_time = max_block_time; - self - } - - /// Create a fallback provider. - pub fn build(self) -> FallbackProvider { - let provider_count = self.providers.len(); - let prioritized_providers = PrioritizedProviders { - providers: self.providers, - // The order of `self.providers` gives the initial priority. - priorities: RwLock::new( - (0..provider_count) - .map(PrioritizedProviderInner::new) - .collect(), - ), - }; - FallbackProvider { - inner: Arc::new(prioritized_providers), - max_block_time: self.max_block_time, - } - } -} - /// Errors specific to fallback provider. #[derive(Error, Debug)] pub enum FallbackError { @@ -240,7 +75,7 @@ impl From for ProviderError { #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] #[cfg_attr(not(target_arch = "wasm32"), async_trait)] -impl JsonRpcClient for FallbackProvider +impl JsonRpcClient for EthereumFallbackProvider where C: JsonRpcClient + PrometheusJsonRpcClientConfigExt @@ -249,6 +84,7 @@ where { type Error = ProviderError; + // TODO: Refactor the reusable parts of this function when implementing the cosmos-specific logic #[instrument] async fn request(&self, method: &str, params: T) -> Result where @@ -272,7 +108,7 @@ where _ => provider.request(method, ¶ms), }; let resp = fut.await; - self.handle_stalled_provider(priority, provider).await?; + self.handle_stalled_provider(priority, provider).await; let _span = warn_span!("request_with_fallback", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); @@ -290,10 +126,11 @@ where #[cfg(test)] mod tests { - use crate::{JsonRpcBlockGetter, BLOCK_NUMBER_RPC}; + use ethers_prometheus::json_rpc_client::{JsonRpcBlockGetter, BLOCK_NUMBER_RPC}; + use hyperlane_core::rpc_clients::fallback::FallbackProviderBuilder; use super::*; - use std::sync::Mutex; + use std::sync::{Arc, Mutex}; #[derive(Debug, Clone)] struct ProviderMock { @@ -385,11 +222,12 @@ mod tests { ProviderMock::new(), ]; let fallback_provider = fallback_provider_builder.add_providers(providers).build(); - fallback_provider + let ethereum_fallback_provider = EthereumFallbackProvider::new(fallback_provider); + ethereum_fallback_provider .request::<_, u64>(BLOCK_NUMBER_RPC, ()) .await .unwrap(); - let provider_call_count: Vec<_> = get_call_counts(&fallback_provider).await; + let provider_call_count: Vec<_> = get_call_counts(ðereum_fallback_provider).await; assert_eq!(provider_call_count, vec![1, 0, 0]); } @@ -405,12 +243,13 @@ mod tests { .add_providers(providers) .with_max_block_time(Duration::from_secs(0)) .build(); - fallback_provider + let ethereum_fallback_provider = EthereumFallbackProvider::new(fallback_provider); + ethereum_fallback_provider .request::<_, u64>(BLOCK_NUMBER_RPC, ()) .await .unwrap(); - let provider_call_count: Vec<_> = get_call_counts(&fallback_provider).await; + let provider_call_count: Vec<_> = get_call_counts(ðereum_fallback_provider).await; assert_eq!(provider_call_count, vec![0, 0, 2]); } diff --git a/rust/chains/hyperlane-ethereum/src/trait_builder.rs b/rust/chains/hyperlane-ethereum/src/trait_builder.rs index 7c9f30d70f..115aa26cab 100644 --- a/rust/chains/hyperlane-ethereum/src/trait_builder.rs +++ b/rust/chains/hyperlane-ethereum/src/trait_builder.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; -use derive_new::new; use ethers::middleware::gas_oracle::{ GasCategory, GasOracle, GasOracleMiddleware, Polygon, ProviderOracle, }; @@ -11,9 +10,8 @@ use ethers::prelude::{ Http, JsonRpcClient, Middleware, NonceManagerMiddleware, Provider, Quorum, QuorumProvider, SignerMiddleware, WeightedProvider, Ws, WsClientError, }; -use ethers::providers::ProviderError; -use ethers_core::types::U64; use hyperlane_core::metrics::agent::METRICS_SCRAPE_INTERVAL; +use hyperlane_core::rpc_clients::fallback::FallbackProvider; use reqwest::{Client, Url}; use thiserror::Error; @@ -28,7 +26,8 @@ use hyperlane_core::{ ChainCommunicationError, ChainResult, ContractLocator, HyperlaneDomain, KnownHyperlaneDomain, }; -use crate::{signers::Signers, ConnectionConf, FallbackProvider, RetryingProvider}; +use crate::EthereumFallbackProvider; +use crate::{signers::Signers, ConnectionConf, RetryingProvider}; // This should be whatever the prometheus scrape interval is const HTTP_CLIENT_TIMEOUT: Duration = Duration::from_secs(60); @@ -117,8 +116,14 @@ pub trait BuildableWithProvider { builder = builder.add_provider(metrics_provider); } let fallback_provider = builder.build(); - self.build(fallback_provider, locator, signer, middleware_metrics) - .await? + let ethereum_fallback_provider = EthereumFallbackProvider::new(fallback_provider); + self.build( + ethereum_fallback_provider, + locator, + signer, + middleware_metrics, + ) + .await? } ConnectionConf::Http { url } => { let http_client = Client::builder() @@ -281,35 +286,3 @@ where }; Ok(GasOracleMiddleware::new(provider, gas_oracle)) } - -pub const BLOCK_NUMBER_RPC: &str = "eth_blockNumber"; - -#[async_trait] -pub trait BlockNumberGetter: Send + Sync + Debug { - async fn get(&self) -> Result; -} - -#[derive(Debug, new)] -pub struct JsonRpcBlockGetter(T); - -#[async_trait] -impl BlockNumberGetter for JsonRpcBlockGetter -where - C: JsonRpcClient, -{ - async fn get(&self) -> Result { - let res = self - .0 - .request(BLOCK_NUMBER_RPC, ()) - .await - .map(|r: U64| r.as_u64()) - .map_err(Into::into)?; - Ok(res) - } -} - -impl Into> for PrometheusJsonRpcClient { - fn into(self) -> Box { - Box::new(JsonRpcBlockGetter(self)) - } -} diff --git a/rust/ethers-prometheus/Cargo.toml b/rust/ethers-prometheus/Cargo.toml index 92e304d6fa..2f4344fa5d 100644 --- a/rust/ethers-prometheus/Cargo.toml +++ b/rust/ethers-prometheus/Cargo.toml @@ -14,6 +14,7 @@ async-trait.workspace = true derive-new.workspace = true derive_builder.workspace = true ethers.workspace = true +ethers-core.workspace = true futures.workspace = true log.workspace = true maplit.workspace = true diff --git a/rust/ethers-prometheus/src/json_rpc_client.rs b/rust/ethers-prometheus/src/json_rpc_client.rs index 2d58191c77..546c51fb08 100644 --- a/rust/ethers-prometheus/src/json_rpc_client.rs +++ b/rust/ethers-prometheus/src/json_rpc_client.rs @@ -8,6 +8,9 @@ use async_trait::async_trait; use derive_builder::Builder; use derive_new::new; use ethers::prelude::JsonRpcClient; +use ethers_core::types::U64; +use hyperlane_core::rpc_clients::fallback::BlockNumberGetter; +use hyperlane_core::ChainCommunicationError; use maplit::hashmap; use prometheus::{CounterVec, IntCounterVec}; use serde::{de::DeserializeOwned, Serialize}; @@ -182,3 +185,30 @@ where res } } + +impl Into> for PrometheusJsonRpcClient { + fn into(self) -> Box { + Box::new(JsonRpcBlockGetter::new(self)) + } +} + +#[derive(Debug, new)] +pub struct JsonRpcBlockGetter(T); + +pub const BLOCK_NUMBER_RPC: &str = "eth_blockNumber"; + +#[async_trait] +impl BlockNumberGetter for JsonRpcBlockGetter +where + C: JsonRpcClient, +{ + async fn get(&self) -> Result { + let res = self + .0 + .request(BLOCK_NUMBER_RPC, ()) + .await + .map(|r: U64| r.as_u64()) + .map_err(Into::into)?; + Ok(res) + } +} diff --git a/rust/hyperlane-core/Cargo.toml b/rust/hyperlane-core/Cargo.toml index d5c93b937a..00c8ce4359 100644 --- a/rust/hyperlane-core/Cargo.toml +++ b/rust/hyperlane-core/Cargo.toml @@ -35,7 +35,9 @@ serde = { workspace = true } serde_json = { workspace = true } sha3 = { workspace = true } strum = { workspace = true, optional = true, features = ["derive"] } +tokio.workspace = true thiserror = { workspace = true } +tracing.workspace = true primitive-types = { workspace = true, optional = true } solana-sdk = { workspace = true, optional = true } tiny-keccak = { workspace = true, features = ["keccak"]} diff --git a/rust/hyperlane-core/src/error.rs b/rust/hyperlane-core/src/error.rs index b3fb6ce356..5c91370ce3 100644 --- a/rust/hyperlane-core/src/error.rs +++ b/rust/hyperlane-core/src/error.rs @@ -6,6 +6,7 @@ use std::ops::Deref; use bigdecimal::ParseBigDecimalError; use crate::config::StrOrIntParseError; +use crate::rpc_clients::error::RpcClientError; use std::string::FromUtf8Error; use crate::Error as PrimitiveTypeError; @@ -128,6 +129,9 @@ pub enum ChainCommunicationError { /// Big decimal parsing error #[error(transparent)] ParseBigDecimalError(#[from] ParseBigDecimalError), + /// Rpc client error + #[error(transparent)] + RpcClientError(#[from] RpcClientError), } impl ChainCommunicationError { diff --git a/rust/hyperlane-core/src/lib.rs b/rust/hyperlane-core/src/lib.rs index 7b9c519f87..e2e2079d50 100644 --- a/rust/hyperlane-core/src/lib.rs +++ b/rust/hyperlane-core/src/lib.rs @@ -35,6 +35,8 @@ mod types; mod chain; pub mod error; +pub mod rpc_clients; + /// Enum for validity of a list of messages #[derive(Debug)] pub enum ListValidity { diff --git a/rust/hyperlane-core/src/rpc_clients/error.rs b/rust/hyperlane-core/src/rpc_clients/error.rs new file mode 100644 index 0000000000..f896fc4983 --- /dev/null +++ b/rust/hyperlane-core/src/rpc_clients/error.rs @@ -0,0 +1,11 @@ +use thiserror::Error; + +use crate::ChainCommunicationError; + +/// Errors specific to fallback provider. +#[derive(Error, Debug)] +pub enum RpcClientError { + /// Fallback providers failed + #[error("All fallback providers failed. (Errors: {0:?})")] + FallbackProvidersFailed(Vec), +} diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs new file mode 100644 index 0000000000..d51f707fd8 --- /dev/null +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -0,0 +1,181 @@ +use async_trait::async_trait; +use derive_new::new; +use std::{ + fmt::Debug, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::sync::RwLock; +use tracing::info; + +use crate::ChainCommunicationError; + +#[async_trait] +pub trait BlockNumberGetter: Send + Sync + Debug { + async fn get(&self) -> Result; +} + +const MAX_BLOCK_TIME: Duration = Duration::from_secs(2 * 60); + +#[derive(Clone, Copy, new)] +pub struct PrioritizedProviderInner { + // Index into the `providers` field of `PrioritizedProviders` + pub index: usize, + // Tuple of the block number and the time when it was queried + #[new(value = "(0, Instant::now())")] + last_block_height: (u64, Instant), +} + +impl PrioritizedProviderInner { + fn from_block_height(index: usize, block_height: u64) -> Self { + Self { + index, + last_block_height: (block_height, Instant::now()), + } + } +} + +pub struct PrioritizedProviders { + /// Sorted list of providers this provider calls in order of most primary to + /// most fallback. + pub providers: Vec, + pub priorities: RwLock>, +} + +/// A provider that bundles multiple providers and attempts to call the first, +/// then the second, and so on until a response is received. +pub struct FallbackProvider { + pub inner: Arc>, + max_block_time: Duration, +} + +impl Clone for FallbackProvider { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + max_block_time: self.max_block_time, + } + } +} + +impl FallbackProvider +where + T: Into> + Debug + Clone, +{ + /// Convenience method for creating a `FallbackProviderBuilder` with same + /// `JsonRpcClient` types + pub fn builder() -> FallbackProviderBuilder { + FallbackProviderBuilder::default() + } + + /// Create a new fallback provider + pub fn new(providers: impl IntoIterator) -> Self { + Self::builder().add_providers(providers).build() + } + + async fn deprioritize_provider(&self, priority: PrioritizedProviderInner) { + // De-prioritize the current provider by moving it to the end of the queue + let mut priorities = self.inner.priorities.write().await; + priorities.retain(|&p| p.index != priority.index); + priorities.push(priority); + } + + async fn update_last_seen_block(&self, provider_index: usize, current_block_height: u64) { + let mut priorities = self.inner.priorities.write().await; + // Get provider position in the up-to-date priorities vec + if let Some(position) = priorities.iter().position(|p| p.index == provider_index) { + priorities[position] = + PrioritizedProviderInner::from_block_height(provider_index, current_block_height); + } + } + + pub async fn take_priorities_snapshot(&self) -> Vec { + let read_lock = self.inner.priorities.read().await; + (*read_lock).clone() + } + + pub async fn handle_stalled_provider(&self, priority: &PrioritizedProviderInner, provider: &T) { + let now = Instant::now(); + if now + .duration_since(priority.last_block_height.1) + .le(&self.max_block_time) + { + // Do nothing, it's too early to tell if the provider has stalled + return; + } + + let block_getter: Box = provider.clone().into(); + let current_block_height = block_getter + .get() + .await + .unwrap_or(priority.last_block_height.0); + if current_block_height <= priority.last_block_height.0 { + // The `max_block_time` elapsed but the block number returned by the provider has not increased + self.deprioritize_provider(*priority).await; + info!( + provider_index=%priority.index, + provider=?self.inner.providers[priority.index], + "Deprioritizing an inner provider in FallbackProvider", + ); + } else { + self.update_last_seen_block(priority.index, current_block_height) + .await; + } + } +} + +/// Builder to create a new fallback provider. +#[derive(Debug, Clone)] +pub struct FallbackProviderBuilder { + providers: Vec, + max_block_time: Duration, +} + +impl Default for FallbackProviderBuilder { + fn default() -> Self { + Self { + providers: Vec::new(), + max_block_time: MAX_BLOCK_TIME, + } + } +} + +impl FallbackProviderBuilder { + /// Add a new provider to the set. Each new provider will be a lower + /// priority than the previous. + pub fn add_provider(mut self, provider: T) -> Self { + self.providers.push(provider); + self + } + + /// Add many providers sorted by highest priority to lowest. + pub fn add_providers(mut self, providers: impl IntoIterator) -> Self { + self.providers.extend(providers); + self + } + + /// Only used for testing purposes. + /// TODO: Move tests into this crate to control the visiblity with conditional compilation. + pub fn with_max_block_time(mut self, max_block_time: Duration) -> Self { + self.max_block_time = max_block_time; + self + } + + /// Create a fallback provider. + pub fn build(self) -> FallbackProvider { + let provider_count = self.providers.len(); + let prioritized_providers = PrioritizedProviders { + providers: self.providers, + // The order of `self.providers` gives the initial priority. + priorities: RwLock::new( + (0..provider_count) + .map(PrioritizedProviderInner::new) + .collect(), + ), + }; + FallbackProvider { + inner: Arc::new(prioritized_providers), + max_block_time: self.max_block_time, + } + } +} diff --git a/rust/hyperlane-core/src/rpc_clients/mod.rs b/rust/hyperlane-core/src/rpc_clients/mod.rs new file mode 100644 index 0000000000..ef7479cb9c --- /dev/null +++ b/rust/hyperlane-core/src/rpc_clients/mod.rs @@ -0,0 +1,2 @@ +pub mod error; +pub mod fallback; From 0c6eea3a3ed5215e916e30df9dcb8de7c343eab6 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 8 Jan 2024 12:39:46 +0000 Subject: [PATCH 04/18] fix: use async-rwlock instead of tokio for a solana-compatible build --- rust/Cargo.lock | 11 +++++++++++ rust/Cargo.toml | 1 + rust/hyperlane-core/Cargo.toml | 2 +- rust/hyperlane-core/src/rpc_clients/fallback.rs | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b109a8c76e..a628269d10 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -325,6 +325,16 @@ dependencies = [ "event-listener", ] +[[package]] +name = "async-rwlock" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261803dcc39ba9e72760ba6e16d0199b1eef9fc44e81bffabbebb9f5aea3906c" +dependencies = [ + "async-mutex", + "event-listener", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -4091,6 +4101,7 @@ dependencies = [ name = "hyperlane-core" version = "0.1.0" dependencies = [ + "async-rwlock", "async-trait", "auto_impl 1.1.0", "bigdecimal 0.4.2", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index ae5fd5e37e..b8e3a949aa 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -52,6 +52,7 @@ version = "0.1.0" Inflector = "0.11.4" anyhow = "1.0" async-trait = "0.1" +async-rwlock = "1.3" auto_impl = "1.0" backtrace = "0.3" base64 = "0.21.2" diff --git a/rust/hyperlane-core/Cargo.toml b/rust/hyperlane-core/Cargo.toml index 00c8ce4359..8329bce5f2 100644 --- a/rust/hyperlane-core/Cargo.toml +++ b/rust/hyperlane-core/Cargo.toml @@ -11,6 +11,7 @@ version = { workspace = true } [dependencies] async-trait.workspace = true +async-rwlock.workspace = true auto_impl.workspace = true bigdecimal.workspace = true borsh.workspace = true @@ -35,7 +36,6 @@ serde = { workspace = true } serde_json = { workspace = true } sha3 = { workspace = true } strum = { workspace = true, optional = true, features = ["derive"] } -tokio.workspace = true thiserror = { workspace = true } tracing.workspace = true primitive-types = { workspace = true, optional = true } diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index d51f707fd8..20c036892b 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -1,3 +1,4 @@ +use async_rwlock::RwLock; use async_trait::async_trait; use derive_new::new; use std::{ @@ -5,7 +6,6 @@ use std::{ sync::Arc, time::{Duration, Instant}, }; -use tokio::sync::RwLock; use tracing::info; use crate::ChainCommunicationError; From 7c22e1906ced1a82031914867d0f2b09a70f1e02 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 8 Jan 2024 14:32:27 +0000 Subject: [PATCH 05/18] fix: clippy --- .../src/rpc_clients/fallback.rs | 10 +++++----- .../hyperlane-ethereum/src/trait_builder.rs | 2 +- rust/ethers-prometheus/src/json_rpc_client.rs | 10 ++++++---- rust/hyperlane-core/src/error.rs | 2 +- rust/hyperlane-core/src/lib.rs | 4 +++- rust/hyperlane-core/src/rpc_clients/fallback.rs | 16 ++++++++++++---- rust/hyperlane-core/src/rpc_clients/mod.rs | 6 ++++-- 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index 3ec6a193c5..dcf9149606 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -1,8 +1,8 @@ use derive_new::new; -use hyperlane_core::rpc_clients::fallback::{BlockNumberGetter, FallbackProvider}; +use hyperlane_core::rpc_clients::{BlockNumberGetter, FallbackProvider}; use std::fmt::{Debug, Formatter}; use std::ops::Deref; -use std::time::{Duration, Instant}; +use std::time::Duration; use thiserror::Error; use async_trait::async_trait; @@ -10,13 +10,13 @@ use ethers::providers::{HttpClientError, JsonRpcClient, ProviderError}; use serde::{de::DeserializeOwned, Serialize}; use serde_json::Value; use tokio::time::sleep; -use tracing::{info, instrument, warn_span}; +use tracing::{instrument, warn_span}; use ethers_prometheus::json_rpc_client::PrometheusJsonRpcClientConfigExt; -use crate::error::HyperlaneEthereumError; use crate::rpc_clients::{categorize_client_response, CategorizedResponse}; +/// Wrapper of `FallbackProvider` for use in `hyperlane-ethereum` #[derive(new)] pub struct EthereumFallbackProvider(FallbackProvider); @@ -127,7 +127,7 @@ where #[cfg(test)] mod tests { use ethers_prometheus::json_rpc_client::{JsonRpcBlockGetter, BLOCK_NUMBER_RPC}; - use hyperlane_core::rpc_clients::fallback::FallbackProviderBuilder; + use hyperlane_core::rpc_clients::FallbackProviderBuilder; use super::*; use std::sync::{Arc, Mutex}; diff --git a/rust/chains/hyperlane-ethereum/src/trait_builder.rs b/rust/chains/hyperlane-ethereum/src/trait_builder.rs index 115aa26cab..31fa128d86 100644 --- a/rust/chains/hyperlane-ethereum/src/trait_builder.rs +++ b/rust/chains/hyperlane-ethereum/src/trait_builder.rs @@ -11,7 +11,7 @@ use ethers::prelude::{ SignerMiddleware, WeightedProvider, Ws, WsClientError, }; use hyperlane_core::metrics::agent::METRICS_SCRAPE_INTERVAL; -use hyperlane_core::rpc_clients::fallback::FallbackProvider; +use hyperlane_core::rpc_clients::FallbackProvider; use reqwest::{Client, Url}; use thiserror::Error; diff --git a/rust/ethers-prometheus/src/json_rpc_client.rs b/rust/ethers-prometheus/src/json_rpc_client.rs index 546c51fb08..3d9b70c4cd 100644 --- a/rust/ethers-prometheus/src/json_rpc_client.rs +++ b/rust/ethers-prometheus/src/json_rpc_client.rs @@ -9,7 +9,7 @@ use derive_builder::Builder; use derive_new::new; use ethers::prelude::JsonRpcClient; use ethers_core::types::U64; -use hyperlane_core::rpc_clients::fallback::BlockNumberGetter; +use hyperlane_core::rpc_clients::BlockNumberGetter; use hyperlane_core::ChainCommunicationError; use maplit::hashmap; use prometheus::{CounterVec, IntCounterVec}; @@ -186,15 +186,17 @@ where } } -impl Into> for PrometheusJsonRpcClient { - fn into(self) -> Box { - Box::new(JsonRpcBlockGetter::new(self)) +impl From> for Box { + fn from(val: PrometheusJsonRpcClient) -> Self { + Box::new(JsonRpcBlockGetter::new(val)) } } +/// Utility struct for implementing `BlockNumberGetter` #[derive(Debug, new)] pub struct JsonRpcBlockGetter(T); +/// RPC method for getting the latest block number pub const BLOCK_NUMBER_RPC: &str = "eth_blockNumber"; #[async_trait] diff --git a/rust/hyperlane-core/src/error.rs b/rust/hyperlane-core/src/error.rs index 5c91370ce3..f7bbf35eab 100644 --- a/rust/hyperlane-core/src/error.rs +++ b/rust/hyperlane-core/src/error.rs @@ -6,7 +6,7 @@ use std::ops::Deref; use bigdecimal::ParseBigDecimalError; use crate::config::StrOrIntParseError; -use crate::rpc_clients::error::RpcClientError; +use crate::rpc_clients::RpcClientError; use std::string::FromUtf8Error; use crate::Error as PrimitiveTypeError; diff --git a/rust/hyperlane-core/src/lib.rs b/rust/hyperlane-core/src/lib.rs index e2e2079d50..610fa3d81b 100644 --- a/rust/hyperlane-core/src/lib.rs +++ b/rust/hyperlane-core/src/lib.rs @@ -8,6 +8,7 @@ extern crate core; pub use chain::*; +pub use error::*; pub use error::{ChainCommunicationError, ChainResult, HyperlaneProtocolError}; pub use identifiers::HyperlaneIdentifier; pub use traits::*; @@ -33,8 +34,9 @@ pub mod metrics; mod types; mod chain; -pub mod error; +mod error; +/// Implementations of custom rpc client logic (e.g. fallback) pub mod rpc_clients; /// Enum for validity of a list of messages diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index 20c036892b..b63b08e7bc 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -10,18 +10,22 @@ use tracing::info; use crate::ChainCommunicationError; +/// Read the current block number from a chain. #[async_trait] pub trait BlockNumberGetter: Send + Sync + Debug { + /// Latest block number getter async fn get(&self) -> Result; } const MAX_BLOCK_TIME: Duration = Duration::from_secs(2 * 60); +/// Information about a provider in `PrioritizedProviders` + #[derive(Clone, Copy, new)] pub struct PrioritizedProviderInner { - // Index into the `providers` field of `PrioritizedProviders` + /// Index into the `providers` field of `PrioritizedProviders` pub index: usize, - // Tuple of the block number and the time when it was queried + /// Tuple of the block number and the time when it was queried #[new(value = "(0, Instant::now())")] last_block_height: (u64, Instant), } @@ -35,16 +39,18 @@ impl PrioritizedProviderInner { } } +/// Sub-providers and priority information pub struct PrioritizedProviders { - /// Sorted list of providers this provider calls in order of most primary to - /// most fallback. + /// Unsorted list of providers this provider calls pub providers: Vec, + /// Sorted list of providers this provider calls, in descending order or reliability pub priorities: RwLock>, } /// A provider that bundles multiple providers and attempts to call the first, /// then the second, and so on until a response is received. pub struct FallbackProvider { + /// The sub-providers called by this provider pub inner: Arc>, max_block_time: Duration, } @@ -89,11 +95,13 @@ where } } + /// Used to iterate the providers in a non-blocking way pub async fn take_priorities_snapshot(&self) -> Vec { let read_lock = self.inner.priorities.read().await; (*read_lock).clone() } + /// De-prioritize a provider that has either timed out or returned a bad response pub async fn handle_stalled_provider(&self, priority: &PrioritizedProviderInner, provider: &T) { let now = Instant::now(); if now diff --git a/rust/hyperlane-core/src/rpc_clients/mod.rs b/rust/hyperlane-core/src/rpc_clients/mod.rs index ef7479cb9c..78851f9f26 100644 --- a/rust/hyperlane-core/src/rpc_clients/mod.rs +++ b/rust/hyperlane-core/src/rpc_clients/mod.rs @@ -1,2 +1,4 @@ -pub mod error; -pub mod fallback; +pub use self::{error::*, fallback::*}; + +mod error; +mod fallback; From 495969602ff82ebd5b2125a2e84802af115391c8 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Tue, 9 Jan 2024 18:05:29 +0000 Subject: [PATCH 06/18] wip on working fallback provider abstraction --- .../src/rpc_clients/fallback.rs | 39 +++++++++++++++++++ .../hyperlane-cosmos/src/rpc_clients/mod.rs | 3 ++ rust/chains/hyperlane-ethereum/src/error.rs | 2 +- .../src/rpc_clients/fallback.rs | 31 ++++++++++++--- .../src/rpc_clients/fallback.rs | 33 +++++++++++++++- 5 files changed, 100 insertions(+), 8 deletions(-) create mode 100644 rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs create mode 100644 rust/chains/hyperlane-cosmos/src/rpc_clients/mod.rs diff --git a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs new file mode 100644 index 0000000000..105641d1bf --- /dev/null +++ b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs @@ -0,0 +1,39 @@ +use std::{ + ops::Deref, + task::{Context, Poll}, +}; + +use derive_new::new; +use hyperlane_core::rpc_clients::CoreFallbackProvider; +use tonic::client::GrpcService; + +/// Wrapper of `FallbackProvider` for use in `hyperlane-cosmos` +#[derive(new)] +pub struct CosmosFallbackProvider(CoreFallbackProvider); + +impl Deref for CosmosFallbackProvider { + type Target = CoreFallbackProvider; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl GrpcService for CosmosFallbackProvider +where + T: GrpcService + Clone, +{ + type ResponseBody = T::ResponseBody; + type Error = T::Error; + type Future = T::Future; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + let mut provider = (*self.inner.providers)[0].clone(); + provider.poll_ready(cx) + } + + fn call(&mut self, request: http::Request) -> Self::Future { + let mut provider = (*self.inner.providers)[0].clone(); + provider.call(request) + } +} diff --git a/rust/chains/hyperlane-cosmos/src/rpc_clients/mod.rs b/rust/chains/hyperlane-cosmos/src/rpc_clients/mod.rs new file mode 100644 index 0000000000..536845688d --- /dev/null +++ b/rust/chains/hyperlane-cosmos/src/rpc_clients/mod.rs @@ -0,0 +1,3 @@ +pub use self::fallback::*; + +mod fallback; diff --git a/rust/chains/hyperlane-ethereum/src/error.rs b/rust/chains/hyperlane-ethereum/src/error.rs index 77fae64890..bc2647017b 100644 --- a/rust/chains/hyperlane-ethereum/src/error.rs +++ b/rust/chains/hyperlane-ethereum/src/error.rs @@ -1,7 +1,7 @@ use ethers::providers::ProviderError; use hyperlane_core::ChainCommunicationError; -/// Errors from the crates specific to the hyperlane-cosmos +/// Errors from the crates specific to the hyperlane-ethereum /// implementation. /// This error can then be converted into the broader error type /// in hyperlane-core using the `From` trait impl diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index dcf9149606..22c285574f 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -28,6 +28,27 @@ impl Deref for EthereumFallbackProvider { } } +impl EthereumFallbackProvider +where + C: JsonRpcClient + + PrometheusJsonRpcClientConfigExt + + Into> + + Clone, +{ + async fn get_categorized_response( + provider: C, + method: &str, + params: &Value, + ) -> CategorizedResponse { + let fut = match params { + Value::Null => provider.request(method, ()), + _ => provider.request(method, params), + }; + let resp: Result = fut.await; + categorize_client_response(method, resp) + } +} + impl Debug for EthereumFallbackProvider where C: JsonRpcClient + PrometheusJsonRpcClientConfigExt, @@ -93,6 +114,8 @@ where { use CategorizedResponse::*; let params = serde_json::to_value(params).expect("valid"); + let categorized_response_closure = + |provider| async { Self::get_categorized_response(provider, method, ¶ms).await }; let mut errors = vec![]; // make sure we do at least 4 total retries. @@ -103,16 +126,12 @@ where let priorities_snapshot = self.take_priorities_snapshot().await; for (idx, priority) in priorities_snapshot.iter().enumerate() { let provider = &self.inner.providers[priority.index]; - let fut = match params { - Value::Null => provider.request(method, ()), - _ => provider.request(method, ¶ms), - }; - let resp = fut.await; + let cat_resp = categorized_response_closure(provider.clone()).await; self.handle_stalled_provider(priority, provider).await; let _span = warn_span!("request_with_fallback", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); - match categorize_client_response(method, resp) { + match cat_resp { IsOk(v) => return Ok(serde_json::from_value(v)?), RetryableErr(e) | RateLimitErr(e) => errors.push(e.into()), NonRetryableErr(e) => return Err(e.into()), diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index b63b08e7bc..b6919686fb 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -3,10 +3,11 @@ use async_trait::async_trait; use derive_new::new; use std::{ fmt::Debug, + future::Future, sync::Arc, time::{Duration, Instant}, }; -use tracing::info; +use tracing::{info, warn_span}; use crate::ChainCommunicationError; @@ -130,6 +131,36 @@ where .await; } } + + pub async fn request(&self, categorized_response_closure: F) + where + F: Fn(T) -> FR, + FR: Future>, + { + let mut errors = vec![]; + // make sure we do at least 4 total retries. + while errors.len() <= 3 { + if !errors.is_empty() { + sleep(Duration::from_millis(100)).await; + } + let priorities_snapshot = self.take_priorities_snapshot().await; + for (idx, priority) in priorities_snapshot.iter().enumerate() { + let provider = &self.inner.providers[priority.index]; + let cat_resp = categorized_response_closure(provider.clone()).await; + self.handle_stalled_provider(priority, provider).await; + let _span = + warn_span!("request_with_fallback", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); + + match cat_resp { + IsOk(v) => return Ok(serde_json::from_value(v)?), + RetryableErr(e) | RateLimitErr(e) => errors.push(e.into()), + NonRetryableErr(e) => return Err(e.into()), + } + } + } + + Err(FallbackError::AllProvidersFailed(errors).into()) + } } /// Builder to create a new fallback provider. From a8bba336e3eea851270cd8af887e35b5b8c91fd2 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Sun, 28 Jan 2024 16:18:20 +0000 Subject: [PATCH 07/18] feat: cosmos fallback implementation that compiles --- rust/chains/hyperlane-cosmos/src/error.rs | 4 + rust/chains/hyperlane-cosmos/src/lib.rs | 1 + .../src/rpc_clients/fallback.rs | 77 ++++++++++++++++--- .../src/rpc_clients/fallback.rs | 6 +- 4 files changed, 74 insertions(+), 14 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/error.rs b/rust/chains/hyperlane-cosmos/src/error.rs index 2c3e1e7475..a18c6f89cd 100644 --- a/rust/chains/hyperlane-cosmos/src/error.rs +++ b/rust/chains/hyperlane-cosmos/src/error.rs @@ -1,5 +1,6 @@ use cosmrs::proto::prost; use hyperlane_core::ChainCommunicationError; +use std::fmt::Debug; /// Errors from the crates specific to the hyperlane-cosmos /// implementation. @@ -37,6 +38,9 @@ pub enum HyperlaneCosmosError { /// Protobuf error #[error("{0}")] Protobuf(#[from] protobuf::ProtobufError), + /// Fallback providers failed + #[error("Fallback providers failed. (Errors: {0:?})")] + FallbackProvidersFailed(Vec), } impl From for ChainCommunicationError { diff --git a/rust/chains/hyperlane-cosmos/src/lib.rs b/rust/chains/hyperlane-cosmos/src/lib.rs index 82a4a0ece1..c0ce3ad549 100644 --- a/rust/chains/hyperlane-cosmos/src/lib.rs +++ b/rust/chains/hyperlane-cosmos/src/lib.rs @@ -16,6 +16,7 @@ mod multisig_ism; mod payloads; mod providers; mod routing_ism; +mod rpc_clients; mod signers; mod trait_builder; mod types; diff --git a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs index 105641d1bf..7ccf541a7d 100644 --- a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs @@ -1,39 +1,96 @@ use std::{ + fmt::Debug, + future::Future, ops::Deref, + pin::Pin, task::{Context, Poll}, + time::Duration, }; +use async_trait::async_trait; use derive_new::new; -use hyperlane_core::rpc_clients::CoreFallbackProvider; +use hyperlane_core::rpc_clients::{BlockNumberGetter, FallbackProvider}; +use tokio::time::sleep; use tonic::client::GrpcService; +use tracing::warn_span; + +use crate::HyperlaneCosmosError; /// Wrapper of `FallbackProvider` for use in `hyperlane-cosmos` -#[derive(new)] -pub struct CosmosFallbackProvider(CoreFallbackProvider); +#[derive(new, Clone)] +pub struct CosmosFallbackProvider(FallbackProvider); impl Deref for CosmosFallbackProvider { - type Target = CoreFallbackProvider; + type Target = FallbackProvider; fn deref(&self) -> &Self::Target { &self.0 } } +#[async_trait] impl GrpcService for CosmosFallbackProvider where - T: GrpcService + Clone, + T: GrpcService + Clone + Debug + Into> + 'static, + >::Error: Into, + ReqBody: Clone + 'static, { type ResponseBody = T::ResponseBody; - type Error = T::Error; - type Future = T::Future; + type Error = HyperlaneCosmosError; + type Future = + Pin, Self::Error>>>>; fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { let mut provider = (*self.inner.providers)[0].clone(); - provider.poll_ready(cx) + provider + .poll_ready(cx) + .map_err(Into::::into) } fn call(&mut self, request: http::Request) -> Self::Future { - let mut provider = (*self.inner.providers)[0].clone(); - provider.call(request) + // use CategorizedResponse::*; + let request = clone_request(&request); + let cloned_self = self.clone(); + let f = async move { + let mut errors = vec![]; + // make sure we do at least 4 total retries. + while errors.len() <= 3 { + if !errors.is_empty() { + sleep(Duration::from_millis(100)).await + } + let priorities_snapshot = cloned_self.take_priorities_snapshot().await; + for (idx, priority) in priorities_snapshot.iter().enumerate() { + let mut provider = cloned_self.inner.providers[priority.index].clone(); + let resp = provider.call(clone_request(&request)).await; + cloned_self + .handle_stalled_provider(priority, &provider) + .await; + let _span = + warn_span!("request", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); + + match resp { + Ok(r) => return Ok(r), + Err(e) => errors.push(e.into()), + } + } + } + + Err(HyperlaneCosmosError::FallbackProvidersFailed(errors)) + }; + Box::pin(f) } } + +fn clone_request(request: &http::Request) -> http::Request +where + ReqBody: Clone + 'static, +{ + let builder = http::Request::builder() + .uri(request.uri().clone()) + .method(request.method().clone()) + .version(request.version()); + let builder = request.headers().iter().fold(builder, |builder, (k, v)| { + builder.header(k.clone(), v.clone()) + }); + builder.body(request.body().clone()).unwrap() +} diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index eadd944b40..baac564590 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -93,8 +93,6 @@ where { use CategorizedResponse::*; let params = serde_json::to_value(params).expect("valid"); - let categorized_response_closure = - |provider| async { Self::get_categorized_response(provider, method, ¶ms).await }; let mut errors = vec![]; // make sure we do at least 4 total retries. @@ -112,9 +110,9 @@ where let resp = fut.await; self.handle_stalled_provider(priority, provider).await; let _span = - warn_span!("request_with_fallback", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); + warn_span!("request", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); - match cat_resp { + match categorize_client_response(method, resp) { IsOk(v) => return Ok(serde_json::from_value(v)?), RetryableErr(e) | RateLimitErr(e) => errors.push(e.into()), NonRetryableErr(e) => return Err(e.into()), From 6bfb0476f12026cc3e21f9d2be44c40426599cf5 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:43:42 +0000 Subject: [PATCH 08/18] wip: unit tests for cosmos fallback provider --- .../src/rpc_clients/fallback.rs | 117 ++++++++++++++++++ rust/chains/hyperlane-ethereum/Cargo.toml | 2 +- .../src/rpc_clients/fallback.rs | 79 +++++------- .../src/rpc_clients/fallback.rs | 68 ++++++++++ rust/utils/abigen/src/lib.rs | 1 + 5 files changed, 219 insertions(+), 48 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs index 7ccf541a7d..69dddc8e04 100644 --- a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs @@ -94,3 +94,120 @@ where }); builder.body(request.body().clone()).unwrap() } + +#[cfg(test)] +mod tests { + use hyperlane_core::rpc_clients::test::ProviderMock; + use hyperlane_core::rpc_clients::FallbackProviderBuilder; + + use super::*; + + #[derive(Debug, Clone)] + struct CosmosProviderMock(ProviderMock); + impl Deref for CosmosProviderMock { + type Target = ProviderMock; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + impl Default for CosmosProviderMock { + fn default() -> Self { + Self(ProviderMock::default()) + } + } + impl CosmosProviderMock { + fn new(request_sleep: Option) -> Self { + Self(ProviderMock::new(request_sleep)) + } + } + + impl Into> for CosmosProviderMock { + fn into(self) -> Box { + Box::new(JsonRpcBlockGetter::new(self.clone())) + } + } + + // fn dummy_return_value() -> Result { + // serde_json::from_str("0").map_err(|e| HttpClientError::SerdeJson { + // err: e, + // text: "".to_owned(), + // }) + // } + + // #[async_trait] + // impl JsonRpcClient for CosmosProviderMock { + // type Error = HttpClientError; + + // /// Pushes the `(method, params)` to the back of the `requests` queue, + // /// pops the responses from the back of the `responses` queue + // async fn request( + // &self, + // method: &str, + // params: T, + // ) -> Result { + // self.push(method, params); + // if let Some(sleep_duration) = self.request_sleep() { + // sleep(sleep_duration).await; + // } + // dummy_return_value() + // } + // } + + impl PrometheusJsonRpcClientConfigExt for CosmosProviderMock { + fn node_host(&self) -> &str { + todo!() + } + + fn chain_name(&self) -> &str { + todo!() + } + } + + #[tokio::test] + async fn test_first_provider_is_attempted() { + let fallback_provider_builder = FallbackProviderBuilder::default(); + let providers = vec![ + CosmosProviderMock::default(), + CosmosProviderMock::default(), + CosmosProviderMock::default(), + ]; + let fallback_provider = fallback_provider_builder.add_providers(providers).build(); + let ethereum_fallback_provider = CosmosFallbackProvider::new(fallback_provider); + ethereum_fallback_provider + .request::<_, u64>(BLOCK_NUMBER_RPC, ()) + .await + .unwrap(); + let provider_call_count: Vec<_> = + ProviderMock::get_call_counts(ðereum_fallback_provider).await; + assert_eq!(provider_call_count, vec![1, 0, 0]); + } + + #[tokio::test] + async fn test_one_stalled_provider() { + let fallback_provider_builder = FallbackProviderBuilder::default(); + let providers = vec![ + CosmosProviderMock::new(Some(Duration::from_millis(10))), + CosmosProviderMock::default(), + CosmosProviderMock::default(), + ]; + let fallback_provider = fallback_provider_builder + .add_providers(providers) + .with_max_block_time(Duration::from_secs(0)) + .build(); + let ethereum_fallback_provider = CosmosFallbackProvider::new(fallback_provider); + ethereum_fallback_provider + .request::<_, u64>(BLOCK_NUMBER_RPC, ()) + .await + .unwrap(); + + let provider_call_count: Vec<_> = + ProviderMock::get_call_counts(ðereum_fallback_provider).await; + // TODO: figure out why there are 2 BLOCK_NUMBER_RPC calls to the stalled provider instead of just one. This could be because + // of how ethers work under the hood. + assert_eq!(provider_call_count, vec![0, 0, 2]); + } + + // TODO: make `categorize_client_response` generic over `ProviderError` to allow testing + // two stalled providers (so that the for loop in `request` doesn't stop after the first provider) +} diff --git a/rust/chains/hyperlane-ethereum/Cargo.toml b/rust/chains/hyperlane-ethereum/Cargo.toml index 8d6db17f45..a37660b1ee 100644 --- a/rust/chains/hyperlane-ethereum/Cargo.toml +++ b/rust/chains/hyperlane-ethereum/Cargo.toml @@ -27,7 +27,7 @@ serde_json.workspace = true thiserror.workspace = true tokio.workspace = true tracing-futures.workspace = true -tracing.workspace = true +tracing = { workspace = true, features = ["log"] } url.workspace = true hyperlane-core = { path = "../../hyperlane-core" } diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index baac564590..c7c01d0c82 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -127,39 +127,32 @@ where #[cfg(test)] mod tests { use ethers_prometheus::json_rpc_client::{JsonRpcBlockGetter, BLOCK_NUMBER_RPC}; + use hyperlane_core::rpc_clients::test::ProviderMock; use hyperlane_core::rpc_clients::FallbackProviderBuilder; use super::*; - use std::sync::{Arc, Mutex}; #[derive(Debug, Clone)] - struct ProviderMock { - // Store requests as tuples of (method, params) - // Even if the tests were single-threaded, need the arc-mutex - // for interior mutability in `JsonRpcClient::request` - requests: Arc>>, - } + struct EthereumProviderMock(ProviderMock); + impl Deref for EthereumProviderMock { + type Target = ProviderMock; - impl ProviderMock { - fn new() -> Self { - Self { - requests: Arc::new(Mutex::new(vec![])), - } + fn deref(&self) -> &Self::Target { + &self.0 } - - fn push(&self, method: &str, params: T) { - self.requests - .lock() - .unwrap() - .push((method.to_owned(), format!("{:?}", params))); + } + impl Default for EthereumProviderMock { + fn default() -> Self { + Self(ProviderMock::default()) } - - fn requests(&self) -> Vec<(String, String)> { - self.requests.lock().unwrap().clone() + } + impl EthereumProviderMock { + fn new(request_sleep: Option) -> Self { + Self(ProviderMock::new(request_sleep)) } } - impl Into> for ProviderMock { + impl Into> for EthereumProviderMock { fn into(self) -> Box { Box::new(JsonRpcBlockGetter::new(self.clone())) } @@ -173,7 +166,7 @@ mod tests { } #[async_trait] - impl JsonRpcClient for ProviderMock { + impl JsonRpcClient for EthereumProviderMock { type Error = HttpClientError; /// Pushes the `(method, params)` to the back of the `requests` queue, @@ -184,12 +177,14 @@ mod tests { params: T, ) -> Result { self.push(method, params); - sleep(Duration::from_millis(10)).await; + if let Some(sleep_duration) = self.request_sleep() { + sleep(sleep_duration).await; + } dummy_return_value() } } - impl PrometheusJsonRpcClientConfigExt for ProviderMock { + impl PrometheusJsonRpcClientConfigExt for EthereumProviderMock { fn node_host(&self) -> &str { todo!() } @@ -199,27 +194,13 @@ mod tests { } } - async fn get_call_counts(fallback_provider: &FallbackProvider) -> Vec { - fallback_provider - .inner - .priorities - .read() - .await - .iter() - .map(|p| { - let provider = &fallback_provider.inner.providers[p.index]; - provider.requests().len() - }) - .collect() - } - #[tokio::test] async fn test_first_provider_is_attempted() { let fallback_provider_builder = FallbackProviderBuilder::default(); let providers = vec![ - ProviderMock::new(), - ProviderMock::new(), - ProviderMock::new(), + EthereumProviderMock::default(), + EthereumProviderMock::default(), + EthereumProviderMock::default(), ]; let fallback_provider = fallback_provider_builder.add_providers(providers).build(); let ethereum_fallback_provider = EthereumFallbackProvider::new(fallback_provider); @@ -227,7 +208,8 @@ mod tests { .request::<_, u64>(BLOCK_NUMBER_RPC, ()) .await .unwrap(); - let provider_call_count: Vec<_> = get_call_counts(ðereum_fallback_provider).await; + let provider_call_count: Vec<_> = + ProviderMock::get_call_counts(ðereum_fallback_provider).await; assert_eq!(provider_call_count, vec![1, 0, 0]); } @@ -235,9 +217,9 @@ mod tests { async fn test_one_stalled_provider() { let fallback_provider_builder = FallbackProviderBuilder::default(); let providers = vec![ - ProviderMock::new(), - ProviderMock::new(), - ProviderMock::new(), + EthereumProviderMock::new(Some(Duration::from_millis(10))), + EthereumProviderMock::default(), + EthereumProviderMock::default(), ]; let fallback_provider = fallback_provider_builder .add_providers(providers) @@ -249,7 +231,10 @@ mod tests { .await .unwrap(); - let provider_call_count: Vec<_> = get_call_counts(ðereum_fallback_provider).await; + let provider_call_count: Vec<_> = + ProviderMock::get_call_counts(ðereum_fallback_provider).await; + // TODO: figure out why there are 2 BLOCK_NUMBER_RPC calls to the stalled provider instead of just one. This could be because + // of how ethers work under the hood. assert_eq!(provider_call_count, vec![0, 0, 2]); } diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index 6adcb76f55..8eab515cff 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -187,3 +187,71 @@ impl FallbackProviderBuilder { } } } + +pub mod test { + use super::*; + use serde::Serialize; + use std::{ + ops::Deref, + sync::{Arc, Mutex}, + }; + + #[derive(Debug, Clone)] + pub struct ProviderMock { + // Store requests as tuples of (method, params) + // Even if the tests were single-threaded, need the arc-mutex + // for interior mutability in `JsonRpcClient::request` + requests: Arc>>, + request_sleep: Option, + } + + impl Default for ProviderMock { + fn default() -> Self { + Self { + requests: Arc::new(Mutex::new(vec![])), + request_sleep: None, + } + } + } + + impl ProviderMock { + pub fn new(request_sleep: Option) -> Self { + Self { + request_sleep, + ..Default::default() + } + } + + pub fn push(&self, method: &str, params: T) { + self.requests + .lock() + .unwrap() + .push((method.to_owned(), format!("{:?}", params))); + } + + pub fn requests(&self) -> Vec<(String, String)> { + self.requests.lock().unwrap().clone() + } + + pub fn request_sleep(&self) -> Option { + self.request_sleep + } + + pub async fn get_call_counts>( + fallback_provider: &FallbackProvider, + ) -> Vec { + fallback_provider + .inner + .priorities + .read() + .await + .iter() + .map(|p| { + let provider = &fallback_provider.inner.providers[p.index]; + println!("Provider has {:?}", provider.requests()); + provider.requests().len() + }) + .collect() + } + } +} diff --git a/rust/utils/abigen/src/lib.rs b/rust/utils/abigen/src/lib.rs index 2e8d5ca081..b4b7970fdc 100644 --- a/rust/utils/abigen/src/lib.rs +++ b/rust/utils/abigen/src/lib.rs @@ -1,3 +1,4 @@ +#[cfg(feature = "fuels")] use fuels_code_gen::ProgramType; use std::collections::BTreeSet; use std::ffi::OsStr; From ef687c96125ea903b9ff78cf5671a9ecdeb1a67e Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 29 Jan 2024 17:49:19 +0000 Subject: [PATCH 09/18] test: cosmos fallback provider --- .../hyperlane-cosmos/src/providers/grpc.rs | 10 +- .../src/rpc_clients/fallback.rs | 106 ++++++++++-------- .../src/rpc_clients/fallback.rs | 30 +++-- .../src/rpc_clients/fallback.rs | 4 +- 4 files changed, 88 insertions(+), 62 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index 3594c7398e..45beb7f21b 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -25,7 +25,8 @@ use cosmrs::{ Any, Coin, }; use hyperlane_core::{ - ChainCommunicationError, ChainResult, ContractLocator, FixedPointNumber, HyperlaneDomain, U256, + rpc_clients::BlockNumberGetter, ChainCommunicationError, ChainResult, ContractLocator, + FixedPointNumber, HyperlaneDomain, U256, }; use protobuf::Message as _; use serde::Serialize; @@ -482,3 +483,10 @@ impl WasmProvider for WasmGrpcProvider { Ok(response) } } + +#[async_trait] +impl BlockNumberGetter for WasmGrpcProvider { + async fn get_block_number(&self) -> Result { + self.latest_block_height().await + } +} diff --git a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs index 69dddc8e04..650cffbf3e 100644 --- a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs @@ -7,7 +7,6 @@ use std::{ time::Duration, }; -use async_trait::async_trait; use derive_new::new; use hyperlane_core::rpc_clients::{BlockNumberGetter, FallbackProvider}; use tokio::time::sleep; @@ -28,7 +27,6 @@ impl Deref for CosmosFallbackProvider { } } -#[async_trait] impl GrpcService for CosmosFallbackProvider where T: GrpcService + Clone + Debug + Into> + 'static, @@ -97,13 +95,16 @@ where #[cfg(test)] mod tests { + use async_trait::async_trait; use hyperlane_core::rpc_clients::test::ProviderMock; use hyperlane_core::rpc_clients::FallbackProviderBuilder; + use hyperlane_core::ChainCommunicationError; use super::*; #[derive(Debug, Clone)] struct CosmosProviderMock(ProviderMock); + impl Deref for CosmosProviderMock { type Target = ProviderMock; @@ -111,56 +112,72 @@ mod tests { &self.0 } } + impl Default for CosmosProviderMock { fn default() -> Self { Self(ProviderMock::default()) } } + impl CosmosProviderMock { fn new(request_sleep: Option) -> Self { Self(ProviderMock::new(request_sleep)) } } + #[async_trait] + impl BlockNumberGetter for CosmosProviderMock { + async fn get_block_number(&self) -> Result { + Ok(0) + } + } + impl Into> for CosmosProviderMock { fn into(self) -> Box { - Box::new(JsonRpcBlockGetter::new(self.clone())) + Box::new(self) } } - // fn dummy_return_value() -> Result { - // serde_json::from_str("0").map_err(|e| HttpClientError::SerdeJson { - // err: e, - // text: "".to_owned(), - // }) - // } - - // #[async_trait] - // impl JsonRpcClient for CosmosProviderMock { - // type Error = HttpClientError; - - // /// Pushes the `(method, params)` to the back of the `requests` queue, - // /// pops the responses from the back of the `responses` queue - // async fn request( - // &self, - // method: &str, - // params: T, - // ) -> Result { - // self.push(method, params); - // if let Some(sleep_duration) = self.request_sleep() { - // sleep(sleep_duration).await; - // } - // dummy_return_value() - // } - // } - - impl PrometheusJsonRpcClientConfigExt for CosmosProviderMock { - fn node_host(&self) -> &str { + impl GrpcService for CosmosProviderMock + where + ReqBody: Clone + 'static, + { + type ResponseBody = tonic::body::BoxBody; + type Error = HyperlaneCosmosError; + type Future = + Pin, Self::Error>>>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { todo!() } - fn chain_name(&self) -> &str { - todo!() + fn call(&mut self, request: http::Request) -> Self::Future { + self.push( + Default::default(), + format!("method: {:?}, uri: {:?}", request.method(), request.uri()), + ); + let body = tonic::body::BoxBody::default(); + let self_clone = self.clone(); + let f = async move { + let response = http::Response::builder().status(200).body(body).unwrap(); + if let Some(sleep_duration) = self_clone.request_sleep() { + sleep(sleep_duration).await; + } + Ok(response) + }; + Box::pin(f) + } + } + + impl CosmosFallbackProvider { + async fn low_level_test_call(&mut self) -> Result<(), ChainCommunicationError> { + let request = http::Request::builder() + .uri("http://localhost:1234") + .method("GET") + .body(()) + .unwrap(); + self.call(request).await?; + Ok(()) } } @@ -173,13 +190,13 @@ mod tests { CosmosProviderMock::default(), ]; let fallback_provider = fallback_provider_builder.add_providers(providers).build(); - let ethereum_fallback_provider = CosmosFallbackProvider::new(fallback_provider); - ethereum_fallback_provider - .request::<_, u64>(BLOCK_NUMBER_RPC, ()) + let mut cosmos_fallback_provider = CosmosFallbackProvider::new(fallback_provider); + cosmos_fallback_provider + .low_level_test_call() .await .unwrap(); let provider_call_count: Vec<_> = - ProviderMock::get_call_counts(ðereum_fallback_provider).await; + ProviderMock::get_call_counts(&cosmos_fallback_provider).await; assert_eq!(provider_call_count, vec![1, 0, 0]); } @@ -195,19 +212,14 @@ mod tests { .add_providers(providers) .with_max_block_time(Duration::from_secs(0)) .build(); - let ethereum_fallback_provider = CosmosFallbackProvider::new(fallback_provider); - ethereum_fallback_provider - .request::<_, u64>(BLOCK_NUMBER_RPC, ()) + let mut cosmos_fallback_provider = CosmosFallbackProvider::new(fallback_provider); + cosmos_fallback_provider + .low_level_test_call() .await .unwrap(); let provider_call_count: Vec<_> = - ProviderMock::get_call_counts(ðereum_fallback_provider).await; - // TODO: figure out why there are 2 BLOCK_NUMBER_RPC calls to the stalled provider instead of just one. This could be because - // of how ethers work under the hood. - assert_eq!(provider_call_count, vec![0, 0, 2]); + ProviderMock::get_call_counts(&cosmos_fallback_provider).await; + assert_eq!(provider_call_count, vec![0, 0, 1]); } - - // TODO: make `categorize_client_response` generic over `ProviderError` to allow testing - // two stalled providers (so that the for loop in `request` doesn't stop after the first provider) } diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index c7c01d0c82..d29396363e 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -134,6 +134,7 @@ mod tests { #[derive(Debug, Clone)] struct EthereumProviderMock(ProviderMock); + impl Deref for EthereumProviderMock { type Target = ProviderMock; @@ -141,11 +142,13 @@ mod tests { &self.0 } } + impl Default for EthereumProviderMock { fn default() -> Self { Self(ProviderMock::default()) } } + impl EthereumProviderMock { fn new(request_sleep: Option) -> Self { Self(ProviderMock::new(request_sleep)) @@ -194,6 +197,18 @@ mod tests { } } + impl EthereumFallbackProvider + where + C: JsonRpcClient + + PrometheusJsonRpcClientConfigExt + + Into> + + Clone, + { + async fn low_level_test_call(&self) { + self.request::<_, u64>(BLOCK_NUMBER_RPC, ()).await.unwrap(); + } + } + #[tokio::test] async fn test_first_provider_is_attempted() { let fallback_provider_builder = FallbackProviderBuilder::default(); @@ -204,10 +219,7 @@ mod tests { ]; let fallback_provider = fallback_provider_builder.add_providers(providers).build(); let ethereum_fallback_provider = EthereumFallbackProvider::new(fallback_provider); - ethereum_fallback_provider - .request::<_, u64>(BLOCK_NUMBER_RPC, ()) - .await - .unwrap(); + ethereum_fallback_provider.low_level_test_call().await; let provider_call_count: Vec<_> = ProviderMock::get_call_counts(ðereum_fallback_provider).await; assert_eq!(provider_call_count, vec![1, 0, 0]); @@ -226,15 +238,11 @@ mod tests { .with_max_block_time(Duration::from_secs(0)) .build(); let ethereum_fallback_provider = EthereumFallbackProvider::new(fallback_provider); - ethereum_fallback_provider - .request::<_, u64>(BLOCK_NUMBER_RPC, ()) - .await - .unwrap(); - + ethereum_fallback_provider.low_level_test_call().await; let provider_call_count: Vec<_> = ProviderMock::get_call_counts(ðereum_fallback_provider).await; - // TODO: figure out why there are 2 BLOCK_NUMBER_RPC calls to the stalled provider instead of just one. This could be because - // of how ethers work under the hood. + // TODO: figure out why there are 2 BLOCK_NUMBER_RPC calls to the stalled provider instead of just one. + // This is most likely due to how ethers works, because the cosmrs test does only have one call. assert_eq!(provider_call_count, vec![0, 0, 2]); } diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index 8eab515cff..8327e406a6 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -190,7 +190,6 @@ impl FallbackProviderBuilder { pub mod test { use super::*; - use serde::Serialize; use std::{ ops::Deref, sync::{Arc, Mutex}, @@ -222,7 +221,7 @@ pub mod test { } } - pub fn push(&self, method: &str, params: T) { + pub fn push(&self, method: &str, params: T) { self.requests .lock() .unwrap() @@ -248,7 +247,6 @@ pub mod test { .iter() .map(|p| { let provider = &fallback_provider.inner.providers[p.index]; - println!("Provider has {:?}", provider.requests()); provider.requests().len() }) .collect() From 8e009f8146426f24a5e21817d034476b5000ac77 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 29 Jan 2024 23:27:33 +0000 Subject: [PATCH 10/18] wip: end of attempts to use grpc middleware; will fall back at application layer instead --- rust/Cargo.lock | 1 + rust/chains/hyperlane-cosmos/Cargo.toml | 1 + rust/chains/hyperlane-cosmos/src/error.rs | 3 + .../hyperlane-cosmos/src/providers/grpc.rs | 70 +++++++++++++++++-- .../src/rpc_clients/fallback.rs | 64 +++++++++++------ .../src/rpc_clients/fallback.rs | 31 ++++---- 6 files changed, 131 insertions(+), 39 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8f3bf3e33a..b8460388b7 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4226,6 +4226,7 @@ dependencies = [ "hyperlane-core", "injective-protobuf", "injective-std", + "itertools 0.11.0", "once_cell", "protobuf", "ripemd", diff --git a/rust/chains/hyperlane-cosmos/Cargo.toml b/rust/chains/hyperlane-cosmos/Cargo.toml index b48aa3590d..60beb84f60 100644 --- a/rust/chains/hyperlane-cosmos/Cargo.toml +++ b/rust/chains/hyperlane-cosmos/Cargo.toml @@ -22,6 +22,7 @@ hyper = { workspace = true } hyper-tls = { workspace = true } injective-protobuf = { workspace = true } injective-std = { workspace = true } +itertools = { workspace = true } once_cell = { workspace = true } protobuf = { workspace = true } ripemd = { workspace = true } diff --git a/rust/chains/hyperlane-cosmos/src/error.rs b/rust/chains/hyperlane-cosmos/src/error.rs index a18c6f89cd..06fffaff7e 100644 --- a/rust/chains/hyperlane-cosmos/src/error.rs +++ b/rust/chains/hyperlane-cosmos/src/error.rs @@ -29,6 +29,9 @@ pub enum HyperlaneCosmosError { /// Tonic error #[error("{0}")] Tonic(#[from] tonic::transport::Error), + /// Tonic codegen error + #[error("{0}")] + TonicGenError(#[from] tonic::codegen::StdError), /// Tendermint RPC Error #[error(transparent)] TendermintError(#[from] tendermint_rpc::error::Error), diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index 45beb7f21b..747acff15c 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -1,3 +1,5 @@ +use std::ops::Deref; + use async_trait::async_trait; use cosmrs::{ proto::{ @@ -24,9 +26,10 @@ use cosmrs::{ tx::{self, Fee, MessageExt, SignDoc, SignerInfo}, Any, Coin, }; +use derive_new::new; use hyperlane_core::{ - rpc_clients::BlockNumberGetter, ChainCommunicationError, ChainResult, ContractLocator, - FixedPointNumber, HyperlaneDomain, U256, + rpc_clients::{BlockNumberGetter, FallbackProvider}, + ChainCommunicationError, ChainResult, ContractLocator, FixedPointNumber, HyperlaneDomain, U256, }; use protobuf::Message as _; use serde::Serialize; @@ -35,8 +38,8 @@ use tonic::{ GrpcMethod, IntoRequest, }; -use crate::HyperlaneCosmosError; use crate::{address::CosmosAddress, CosmosAmount}; +use crate::{rpc_clients::CosmosFallbackProvider, HyperlaneCosmosError}; use crate::{signers::Signer, ConnectionConf}; /// A multiplier applied to a simulated transaction's gas usage to @@ -46,6 +49,45 @@ const GAS_ESTIMATE_MULTIPLIER: f64 = 1.25; /// be valid for. const TIMEOUT_BLOCKS: u64 = 1000; +#[derive(Debug, Clone, new)] +struct GrpcChannel(Channel); + +impl Into for Channel { + fn into(self) -> GrpcChannel { + GrpcChannel::new(self) + } +} + +impl Deref for GrpcChannel { + type Target = Channel; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[async_trait] +impl BlockNumberGetter for GrpcChannel { + async fn get_block_number(&self) -> Result { + let mut client = ServiceClient::new(self.0.clone()); + let request = tonic::Request::new(GetLatestBlockRequest {}); + + let response = client + .get_latest_block(request) + .await + .map_err(ChainCommunicationError::from_other)? + .into_inner(); + let height = response + .block + .ok_or_else(|| ChainCommunicationError::from_other_str("block not present"))? + .header + .ok_or_else(|| ChainCommunicationError::from_other_str("header not present"))? + .height; + + Ok(height as u64) + } +} + #[async_trait] /// Cosmwasm GRPC Provider pub trait WasmProvider: Send + Sync { @@ -96,7 +138,7 @@ pub struct WasmGrpcProvider { signer: Option, /// GRPC Channel that can be cheaply cloned. /// See `` - channel: Channel, + provider: CosmosFallbackProvider, gas_price: CosmosAmount, } @@ -109,9 +151,25 @@ impl WasmGrpcProvider { locator: Option, signer: Option, ) -> ChainResult { + // get all the configured grpc urls and convert them to a Vec let endpoint = Endpoint::new(conf.get_grpc_url()).map_err(Into::::into)?; + // create a vec of channels. Replace single Client instantiations with an instantiation over all channels, and then wrapping them in a fallback provider. + // However, in this case the fallback provider wouldn't be able to memorize the prioritization across calls + // Alternatively, could create a struct Clients that wraps all client types; we'd have one Clients instance per channel and should be straightforward to read blocks this way too. + + // Alternatively, could try wrapping the channels directly in a fallback provider, and reprioritizing that way + + // Looks like the way to go is to create a (Channel, BlockReaderClient) tuple and implement `GrpcService` for it let channel = endpoint.connect_lazy(); + + // Another option is to create + + let mut builder = FallbackProvider::builder(); + builder = builder.add_provider(channel); + let fallback_provider = builder.build(); + let provider = CosmosFallbackProvider::new(fallback_provider); + let contract_address = locator .map(|l| { CosmosAddress::from_h256( @@ -127,7 +185,7 @@ impl WasmGrpcProvider { conf, contract_address, signer, - channel, + provider, gas_price, }) } @@ -227,7 +285,7 @@ impl WasmGrpcProvider { signatures: vec![vec![]], }; - let mut client = TxServiceClient::new(self.channel.clone()); + let mut client = TxServiceClient::new(self.provider.clone()); let tx_bytes = raw_tx .to_bytes() .map_err(ChainCommunicationError::from_other)?; diff --git a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs index 650cffbf3e..105543701f 100644 --- a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs @@ -1,6 +1,7 @@ use std::{ - fmt::Debug, + fmt::{Debug, Formatter}, future::Future, + marker::PhantomData, ops::Deref, pin::Pin, task::{Context, Poll}, @@ -9,43 +10,67 @@ use std::{ use derive_new::new; use hyperlane_core::rpc_clients::{BlockNumberGetter, FallbackProvider}; +use itertools::Itertools; use tokio::time::sleep; -use tonic::client::GrpcService; +use tonic::{body::BoxBody, client::GrpcService, codegen::StdError, transport::Channel}; use tracing::warn_span; use crate::HyperlaneCosmosError; - /// Wrapper of `FallbackProvider` for use in `hyperlane-cosmos` #[derive(new, Clone)] -pub struct CosmosFallbackProvider(FallbackProvider); +pub struct CosmosFallbackProvider { + fallback_provider: FallbackProvider, + _phantom: PhantomData, +} -impl Deref for CosmosFallbackProvider { - type Target = FallbackProvider; +impl Deref for CosmosFallbackProvider { + type Target = FallbackProvider; fn deref(&self) -> &Self::Target { - &self.0 + &self.fallback_provider } } -impl GrpcService for CosmosFallbackProvider +impl Debug for CosmosFallbackProvider where - T: GrpcService + Clone + Debug + Into> + 'static, - >::Error: Into, - ReqBody: Clone + 'static, + C: Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // iterate the inner providers and write them to the formatter + f.debug_struct("FallbackProvider") + .field( + "providers", + &self + .fallback_provider + .inner + .providers + .iter() + .map(|v| format!("{:?}", v)) + .join(", "), + ) + .finish() + } +} + +impl GrpcService for CosmosFallbackProvider +where + T: GrpcService + Clone + Debug + Into> + 'static, + >::Error: Into, + >::ResponseBody: + tonic::codegen::Body + Send + 'static, + ::Error: Into + Send, { type ResponseBody = T::ResponseBody; - type Error = HyperlaneCosmosError; + type Error = T::Error; type Future = Pin, Self::Error>>>>; fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { let mut provider = (*self.inner.providers)[0].clone(); - provider - .poll_ready(cx) - .map_err(Into::::into) + provider.poll_ready(cx) } - fn call(&mut self, request: http::Request) -> Self::Future { + fn call(&mut self, request: http::Request) -> Self::Future { // use CategorizedResponse::*; let request = clone_request(&request); let cloned_self = self.clone(); @@ -79,10 +104,7 @@ where } } -fn clone_request(request: &http::Request) -> http::Request -where - ReqBody: Clone + 'static, -{ +fn clone_request(request: &http::Request) -> http::Request { let builder = http::Request::builder() .uri(request.uri().clone()) .method(request.method().clone()) @@ -169,7 +191,7 @@ mod tests { } } - impl CosmosFallbackProvider { + impl CosmosFallbackProvider { async fn low_level_test_call(&mut self) -> Result<(), ChainCommunicationError> { let request = http::Request::builder() .uri("http://localhost:1234") diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index 8327e406a6..eb553e5bd3 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -3,6 +3,7 @@ use async_trait::async_trait; use derive_new::new; use std::{ fmt::Debug, + marker::PhantomData, sync::Arc, time::{Duration, Instant}, }; @@ -49,28 +50,31 @@ pub struct PrioritizedProviders { /// A provider that bundles multiple providers and attempts to call the first, /// then the second, and so on until a response is received. -pub struct FallbackProvider { +pub struct FallbackProvider { /// The sub-providers called by this provider pub inner: Arc>, max_block_time: Duration, + _phantom: PhantomData, } -impl Clone for FallbackProvider { +impl Clone for FallbackProvider { fn clone(&self) -> Self { Self { inner: self.inner.clone(), max_block_time: self.max_block_time, + _phantom: PhantomData, } } } -impl FallbackProvider +impl FallbackProvider where - T: Into> + Debug + Clone, + T: Into + Debug + Clone, + B: BlockNumberGetter, { /// Convenience method for creating a `FallbackProviderBuilder` with same /// `JsonRpcClient` types - pub fn builder() -> FallbackProviderBuilder { + pub fn builder() -> FallbackProviderBuilder { FallbackProviderBuilder::default() } @@ -112,7 +116,7 @@ where return; } - let block_getter: Box = provider.clone().into(); + let block_getter: B = provider.clone().into(); let current_block_height = block_getter .get_block_number() .await @@ -134,21 +138,23 @@ where /// Builder to create a new fallback provider. #[derive(Debug, Clone)] -pub struct FallbackProviderBuilder { +pub struct FallbackProviderBuilder { providers: Vec, max_block_time: Duration, + _phantom: PhantomData, } -impl Default for FallbackProviderBuilder { +impl Default for FallbackProviderBuilder { fn default() -> Self { Self { providers: Vec::new(), max_block_time: MAX_BLOCK_TIME, + _phantom: PhantomData, } } } -impl FallbackProviderBuilder { +impl FallbackProviderBuilder { /// Add a new provider to the set. Each new provider will be a lower /// priority than the previous. pub fn add_provider(mut self, provider: T) -> Self { @@ -170,7 +176,7 @@ impl FallbackProviderBuilder { } /// Create a fallback provider. - pub fn build(self) -> FallbackProvider { + pub fn build(self) -> FallbackProvider { let provider_count = self.providers.len(); let prioritized_providers = PrioritizedProviders { providers: self.providers, @@ -184,6 +190,7 @@ impl FallbackProviderBuilder { FallbackProvider { inner: Arc::new(prioritized_providers), max_block_time: self.max_block_time, + _phantom: PhantomData, } } } @@ -236,8 +243,8 @@ pub mod test { self.request_sleep } - pub async fn get_call_counts>( - fallback_provider: &FallbackProvider, + pub async fn get_call_counts, B>( + fallback_provider: &FallbackProvider, ) -> Vec { fallback_provider .inner From 72185c4e63d8f728ce01120299a577ec578143c8 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Tue, 30 Jan 2024 14:40:42 +0000 Subject: [PATCH 11/18] feat: potentially working cosmos fallback provider --- .../src/payloads/aggregate_ism.rs | 4 +- .../hyperlane-cosmos/src/payloads/general.rs | 2 +- .../src/payloads/ism_routes.rs | 12 +- .../hyperlane-cosmos/src/payloads/mailbox.rs | 20 +- .../src/payloads/merkle_tree_hook.rs | 8 +- .../src/payloads/multisig_ism.rs | 4 +- .../src/payloads/validator_announce.rs | 6 +- .../hyperlane-cosmos/src/providers/grpc.rs | 312 +++++++++++------- .../src/rpc_clients/fallback.rs | 198 +++++------ .../src/rpc_clients/fallback.rs | 26 +- .../hyperlane-ethereum/src/trait_builder.rs | 9 +- rust/ethers-prometheus/src/json_rpc_client.rs | 6 +- rust/hyperlane-core/Cargo.toml | 1 + .../src/rpc_clients/fallback.rs | 42 ++- 14 files changed, 368 insertions(+), 282 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/payloads/aggregate_ism.rs b/rust/chains/hyperlane-cosmos/src/payloads/aggregate_ism.rs index 8276675ff7..23bb35a8f8 100644 --- a/rust/chains/hyperlane-cosmos/src/payloads/aggregate_ism.rs +++ b/rust/chains/hyperlane-cosmos/src/payloads/aggregate_ism.rs @@ -1,11 +1,11 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct VerifyRequest { pub verify: VerifyRequestInner, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct VerifyRequestInner { pub metadata: String, pub message: String, diff --git a/rust/chains/hyperlane-cosmos/src/payloads/general.rs b/rust/chains/hyperlane-cosmos/src/payloads/general.rs index 488cae2d37..af2a4b0b6e 100644 --- a/rust/chains/hyperlane-cosmos/src/payloads/general.rs +++ b/rust/chains/hyperlane-cosmos/src/payloads/general.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct EmptyStruct {} #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/rust/chains/hyperlane-cosmos/src/payloads/ism_routes.rs b/rust/chains/hyperlane-cosmos/src/payloads/ism_routes.rs index 052a1cc48b..1f659840e4 100644 --- a/rust/chains/hyperlane-cosmos/src/payloads/ism_routes.rs +++ b/rust/chains/hyperlane-cosmos/src/payloads/ism_routes.rs @@ -1,12 +1,12 @@ use super::general::EmptyStruct; use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct IsmRouteRequest { pub route: IsmRouteRequestInner, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct IsmRouteRequestInner { pub message: String, // hexbinary } @@ -16,22 +16,22 @@ pub struct IsmRouteRespnose { pub ism: String, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct QueryRoutingIsmGeneralRequest { pub routing_ism: T, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct QueryRoutingIsmRouteResponse { pub ism: String, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct QueryIsmGeneralRequest { pub ism: T, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct QueryIsmModuleTypeRequest { pub module_type: EmptyStruct, } diff --git a/rust/chains/hyperlane-cosmos/src/payloads/mailbox.rs b/rust/chains/hyperlane-cosmos/src/payloads/mailbox.rs index 145ba5b16c..75eef04595 100644 --- a/rust/chains/hyperlane-cosmos/src/payloads/mailbox.rs +++ b/rust/chains/hyperlane-cosmos/src/payloads/mailbox.rs @@ -3,52 +3,52 @@ use serde::{Deserialize, Serialize}; use super::general::EmptyStruct; // Requests -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct GeneralMailboxQuery { pub mailbox: T, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct CountRequest { pub count: EmptyStruct, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct NonceRequest { pub nonce: EmptyStruct, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct RecipientIsmRequest { pub recipient_ism: RecipientIsmRequestInner, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct RecipientIsmRequestInner { pub recipient_addr: String, // hexbinary } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct DefaultIsmRequest { pub default_ism: EmptyStruct, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct DeliveredRequest { pub message_delivered: DeliveredRequestInner, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct DeliveredRequestInner { pub id: String, // hexbinary } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct ProcessMessageRequest { pub process: ProcessMessageRequestInner, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct ProcessMessageRequestInner { pub metadata: String, pub message: String, diff --git a/rust/chains/hyperlane-cosmos/src/payloads/merkle_tree_hook.rs b/rust/chains/hyperlane-cosmos/src/payloads/merkle_tree_hook.rs index 7635f0ef72..e960628771 100644 --- a/rust/chains/hyperlane-cosmos/src/payloads/merkle_tree_hook.rs +++ b/rust/chains/hyperlane-cosmos/src/payloads/merkle_tree_hook.rs @@ -4,24 +4,24 @@ use super::general::EmptyStruct; const TREE_DEPTH: usize = 32; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct MerkleTreeGenericRequest { pub merkle_hook: T, } // --------- Requests --------- -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct MerkleTreeRequest { pub tree: EmptyStruct, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct MerkleTreeCountRequest { pub count: EmptyStruct, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct CheckPointRequest { pub check_point: EmptyStruct, } diff --git a/rust/chains/hyperlane-cosmos/src/payloads/multisig_ism.rs b/rust/chains/hyperlane-cosmos/src/payloads/multisig_ism.rs index 204e726dc7..c56588d1d6 100644 --- a/rust/chains/hyperlane-cosmos/src/payloads/multisig_ism.rs +++ b/rust/chains/hyperlane-cosmos/src/payloads/multisig_ism.rs @@ -1,11 +1,11 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct VerifyInfoRequest { pub verify_info: VerifyInfoRequestInner, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct VerifyInfoRequestInner { pub message: String, // hexbinary } diff --git a/rust/chains/hyperlane-cosmos/src/payloads/validator_announce.rs b/rust/chains/hyperlane-cosmos/src/payloads/validator_announce.rs index fdf449c7c4..cf4e5eb1f8 100644 --- a/rust/chains/hyperlane-cosmos/src/payloads/validator_announce.rs +++ b/rust/chains/hyperlane-cosmos/src/payloads/validator_announce.rs @@ -2,17 +2,17 @@ use serde::{Deserialize, Serialize}; use super::general::EmptyStruct; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct GetAnnouncedValidatorsRequest { pub get_announced_validators: EmptyStruct, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct GetAnnounceStorageLocationsRequest { pub get_announce_storage_locations: GetAnnounceStorageLocationsRequestInner, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] pub struct GetAnnounceStorageLocationsRequestInner { pub validators: Vec, } diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index 747acff15c..5eb4ae4b01 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -1,4 +1,4 @@ -use std::ops::Deref; +use std::pin::Pin; use async_trait::async_trait; use cosmrs::{ @@ -50,26 +50,20 @@ const GAS_ESTIMATE_MULTIPLIER: f64 = 1.25; const TIMEOUT_BLOCKS: u64 = 1000; #[derive(Debug, Clone, new)] -struct GrpcChannel(Channel); - -impl Into for Channel { - fn into(self) -> GrpcChannel { - GrpcChannel::new(self) - } +struct CosmosChannel { + channel: Channel, } -impl Deref for GrpcChannel { - type Target = Channel; - - fn deref(&self) -> &Self::Target { - &self.0 +impl From for CosmosChannel { + fn from(channel: Channel) -> Self { + Self { channel } } } #[async_trait] -impl BlockNumberGetter for GrpcChannel { +impl BlockNumberGetter for CosmosChannel { async fn get_block_number(&self) -> Result { - let mut client = ServiceClient::new(self.0.clone()); + let mut client = ServiceClient::new(self.channel.clone()); let request = tonic::Request::new(GetLatestBlockRequest {}); let response = client @@ -99,14 +93,14 @@ pub trait WasmProvider: Send + Sync { async fn latest_block_height(&self) -> ChainResult; /// Perform a wasm query against the stored contract address. - async fn wasm_query( + async fn wasm_query( &self, payload: T, block_height: Option, ) -> ChainResult>; /// Perform a wasm query against a specified contract address. - async fn wasm_query_to( + async fn wasm_query_to( &self, to: String, payload: T, @@ -114,14 +108,17 @@ pub trait WasmProvider: Send + Sync { ) -> ChainResult>; /// Send a wasm tx. - async fn wasm_send( + async fn wasm_send( &self, payload: T, gas_limit: Option, ) -> ChainResult; /// Estimate gas for a wasm tx. - async fn wasm_estimate_gas(&self, payload: T) -> ChainResult; + async fn wasm_estimate_gas( + &self, + payload: T, + ) -> ChainResult; } #[derive(Debug, Clone)] @@ -138,7 +135,7 @@ pub struct WasmGrpcProvider { signer: Option, /// GRPC Channel that can be cheaply cloned. /// See `` - provider: CosmosFallbackProvider, + provider: CosmosFallbackProvider, gas_price: CosmosAmount, } @@ -155,6 +152,7 @@ impl WasmGrpcProvider { let endpoint = Endpoint::new(conf.get_grpc_url()).map_err(Into::::into)?; // create a vec of channels. Replace single Client instantiations with an instantiation over all channels, and then wrapping them in a fallback provider. + // However, in this case the fallback provider wouldn't be able to memorize the prioritization across calls // Alternatively, could create a struct Clients that wraps all client types; we'd have one Clients instance per channel and should be straightforward to read blocks this way too. @@ -166,7 +164,7 @@ impl WasmGrpcProvider { // Another option is to create let mut builder = FallbackProvider::builder(); - builder = builder.add_provider(channel); + builder = builder.add_provider(CosmosChannel::from(channel)); let fallback_provider = builder.build(); let provider = CosmosFallbackProvider::new(fallback_provider); @@ -284,21 +282,36 @@ impl WasmGrpcProvider { // https://github.com/cosmos/cosmjs/blob/44893af824f0712d1f406a8daa9fcae335422235/packages/stargate/src/modules/tx/queries.ts#L67 signatures: vec![vec![]], }; - - let mut client = TxServiceClient::new(self.provider.clone()); let tx_bytes = raw_tx .to_bytes() .map_err(ChainCommunicationError::from_other)?; - #[allow(deprecated)] - let sim_req = tonic::Request::new(SimulateRequest { tx: None, tx_bytes }); - let gas_used = client - .simulate(sim_req) - .await - .map_err(ChainCommunicationError::from_other)? - .into_inner() - .gas_info - .ok_or_else(|| ChainCommunicationError::from_other_str("gas info not present"))? - .gas_used; + let gas_used = self + .provider + .call(move |provider| { + let tx_bytes_clone = tx_bytes.clone(); + let future = async move { + let mut client = TxServiceClient::new(provider.channel.clone()); + #[allow(deprecated)] + let sim_req = tonic::Request::new(SimulateRequest { + tx: None, + tx_bytes: tx_bytes_clone, + }); + let gas_used = client + .simulate(sim_req) + .await + .map_err(ChainCommunicationError::from_other)? + .into_inner() + .gas_info + .ok_or_else(|| { + ChainCommunicationError::from_other_str("gas info not present") + })? + .gas_used; + + Ok(gas_used) + }; + Pin::from(Box::from(future)) + }) + .await?; let gas_estimate = (gas_used as f64 * GAS_ESTIMATE_MULTIPLIER) as u64; @@ -307,14 +320,25 @@ impl WasmGrpcProvider { /// Fetches balance for a given `address` and `denom` pub async fn get_balance(&self, address: String, denom: String) -> ChainResult { - let mut client = QueryBalanceClient::new(self.channel.clone()); - - let balance_request = tonic::Request::new(QueryBalanceRequest { address, denom }); - let response = client - .balance(balance_request) - .await - .map_err(ChainCommunicationError::from_other)? - .into_inner(); + let response = self + .provider + .call(move |provider| { + let address = address.clone(); + let denom = denom.clone(); + let future = async move { + let mut client = QueryBalanceClient::new(provider.channel.clone()); + let balance_request = + tonic::Request::new(QueryBalanceRequest { address, denom }); + let response = client + .balance(balance_request) + .await + .map_err(ChainCommunicationError::from_other)? + .into_inner(); + Ok(response) + }; + Pin::from(Box::from(future)) + }) + .await?; let balance = response .balance @@ -331,14 +355,23 @@ impl WasmGrpcProvider { return self.account_query_injective(account).await; } - let mut client = QueryAccountClient::new(self.channel.clone()); - - let request = tonic::Request::new(QueryAccountRequest { address: account }); - let response = client - .account(request) - .await - .map_err(ChainCommunicationError::from_other)? - .into_inner(); + let response = self + .provider + .call(move |provider| { + let address = account.clone(); + let future = async move { + let mut client = QueryAccountClient::new(provider.channel.clone()); + let request = tonic::Request::new(QueryAccountRequest { address }); + let response = client + .account(request) + .await + .map_err(ChainCommunicationError::from_other)? + .into_inner(); + Ok(response) + }; + Pin::from(Box::from(future)) + }) + .await?; let account = BaseAccount::decode( response @@ -353,32 +386,46 @@ impl WasmGrpcProvider { /// Injective-specific logic for querying an account. async fn account_query_injective(&self, account: String) -> ChainResult { - let request = tonic::Request::new( - injective_std::types::cosmos::auth::v1beta1::QueryAccountRequest { address: account }, - ); - - // Borrowed from the logic of `QueryAccountClient` in `cosmrs`, but using injective types. - - let mut grpc_client = tonic::client::Grpc::new(self.channel.clone()); - grpc_client - .ready() - .await - .map_err(Into::::into)?; - - let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/cosmos.auth.v1beta1.Query/Account"); - let mut req: tonic::Request< - injective_std::types::cosmos::auth::v1beta1::QueryAccountRequest, - > = request.into_request(); - req.extensions_mut() - .insert(GrpcMethod::new("cosmos.auth.v1beta1.Query", "Account")); - - let response: tonic::Response< - injective_std::types::cosmos::auth::v1beta1::QueryAccountResponse, - > = grpc_client - .unary(req, path, codec) - .await - .map_err(Into::::into)?; + let response = self + .provider + .call(move |provider| { + let address = account.clone(); + let future = async move { + let request = tonic::Request::new( + injective_std::types::cosmos::auth::v1beta1::QueryAccountRequest { + address, + }, + ); + + // Borrowed from the logic of `QueryAccountClient` in `cosmrs`, but using injective types. + + let mut grpc_client = tonic::client::Grpc::new(provider.channel.clone()); + grpc_client + .ready() + .await + .map_err(Into::::into)?; + + let codec = tonic::codec::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/cosmos.auth.v1beta1.Query/Account"); + let mut req: tonic::Request< + injective_std::types::cosmos::auth::v1beta1::QueryAccountRequest, + > = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("cosmos.auth.v1beta1.Query", "Account")); + + let response: tonic::Response< + injective_std::types::cosmos::auth::v1beta1::QueryAccountResponse, + > = grpc_client + .unary(req, path, codec) + .await + .map_err(Into::::into)?; + + Ok(response) + }; + Pin::from(Box::from(future)) + }) + .await?; let mut eth_account = injective_protobuf::proto::account::EthAccount::parse_from_bytes( response @@ -408,14 +455,23 @@ impl WasmGrpcProvider { #[async_trait] impl WasmProvider for WasmGrpcProvider { async fn latest_block_height(&self) -> ChainResult { - let mut client = ServiceClient::new(self.channel.clone()); - let request = tonic::Request::new(GetLatestBlockRequest {}); + let response = self + .provider + .call(move |provider| { + let future = async move { + let mut client = ServiceClient::new(provider.channel.clone()); + let request = tonic::Request::new(GetLatestBlockRequest {}); + let response = client + .get_latest_block(request) + .await + .map_err(ChainCommunicationError::from_other)? + .into_inner(); + Ok(response) + }; + Pin::from(Box::from(future)) + }) + .await?; - let response = client - .get_latest_block(request) - .await - .map_err(ChainCommunicationError::from_other)? - .into_inner(); let height = response .block .ok_or_else(|| ChainCommunicationError::from_other_str("block not present"))? @@ -428,7 +484,7 @@ impl WasmProvider for WasmGrpcProvider { async fn wasm_query(&self, payload: T, block_height: Option) -> ChainResult> where - T: Serialize + Send + Sync, + T: Serialize + Send + Sync + Clone, { let contract_address = self.contract_address.as_ref().ok_or_else(|| { ChainCommunicationError::from_other_str("No contract address available") @@ -444,39 +500,48 @@ impl WasmProvider for WasmGrpcProvider { block_height: Option, ) -> ChainResult> where - T: Serialize + Send + Sync, + T: Serialize + Send + Sync + Clone, { - let mut client = WasmQueryClient::new(self.channel.clone()); - let mut request = tonic::Request::new(QuerySmartContractStateRequest { - address: to, - query_data: serde_json::to_string(&payload)?.as_bytes().to_vec(), - }); - - if let Some(block_height) = block_height { - request - .metadata_mut() - .insert("x-cosmos-block-height", block_height.into()); - } - - let response = client - .smart_contract_state(request) - .await - .map_err(ChainCommunicationError::from_other)? - .into_inner(); + let query_data = serde_json::to_string(&payload)?.as_bytes().to_vec(); + let response = self + .provider + .call(move |provider| { + let to = to.clone(); + let query_data = query_data.clone(); + let future = async move { + let mut client = WasmQueryClient::new(provider.channel.clone()); + + let mut request = tonic::Request::new(QuerySmartContractStateRequest { + address: to, + query_data, + }); + if let Some(block_height) = block_height { + request + .metadata_mut() + .insert("x-cosmos-block-height", block_height.into()); + } + let response = client + .smart_contract_state(request) + .await + .map_err(ChainCommunicationError::from_other)? + .into_inner(); + Ok(response) + }; + Pin::from(Box::from(future)) + }) + .await?; Ok(response.data) } async fn wasm_send(&self, payload: T, gas_limit: Option) -> ChainResult where - T: Serialize + Send + Sync, + T: Serialize + Send + Sync + Clone, { let signer = self.get_signer()?; - let mut client = TxServiceClient::new(self.channel.clone()); let contract_address = self.contract_address.as_ref().ok_or_else(|| { ChainCommunicationError::from_other_str("No contract address available") })?; - let msgs = vec![MsgExecuteContract { sender: signer.address.clone(), contract: contract_address.address(), @@ -485,9 +550,6 @@ impl WasmProvider for WasmGrpcProvider { } .to_any() .map_err(ChainCommunicationError::from_other)?]; - - // We often use U256s to represent gas limits, but Cosmos expects u64s. Try to convert, - // and if it fails, just fallback to None which will result in gas estimation. let gas_limit: Option = gas_limit.and_then(|limit| match limit.try_into() { Ok(limit) => Some(limit), Err(err) => { @@ -498,20 +560,30 @@ impl WasmProvider for WasmGrpcProvider { None } }); - - let tx_req = BroadcastTxRequest { - tx_bytes: self.generate_raw_signed_tx(msgs, gas_limit).await?, - mode: BroadcastMode::Sync as i32, - }; - - let tx_res = client - .broadcast_tx(tx_req) - .await - .map_err(Into::::into)? - .into_inner() - .tx_response - .ok_or_else(|| ChainCommunicationError::from_other_str("Empty tx_response"))?; - + let tx_bytes = self.generate_raw_signed_tx(msgs, gas_limit).await?; + let tx_res = self + .provider + .call(move |provider| { + let tx_bytes = tx_bytes.clone(); + let future = async move { + let mut client = TxServiceClient::new(provider.channel.clone()); + // We often use U256s to represent gas limits, but Cosmos expects u64s. Try to convert, + // and if it fails, just fallback to None which will result in gas estimation. + let tx_req = BroadcastTxRequest { + tx_bytes, + mode: BroadcastMode::Sync as i32, + }; + client + .broadcast_tx(tx_req) + .await + .map_err(Into::::into)? + .into_inner() + .tx_response + .ok_or_else(|| ChainCommunicationError::from_other_str("Empty tx_response")) + }; + Pin::from(Box::from(future)) + }) + .await?; Ok(tx_res) } diff --git a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs index 105543701f..1daea1a2c7 100644 --- a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs @@ -1,37 +1,27 @@ use std::{ fmt::{Debug, Formatter}, - future::Future, - marker::PhantomData, ops::Deref, - pin::Pin, - task::{Context, Poll}, - time::Duration, }; use derive_new::new; -use hyperlane_core::rpc_clients::{BlockNumberGetter, FallbackProvider}; +use hyperlane_core::rpc_clients::FallbackProvider; use itertools::Itertools; -use tokio::time::sleep; -use tonic::{body::BoxBody, client::GrpcService, codegen::StdError, transport::Channel}; -use tracing::warn_span; -use crate::HyperlaneCosmosError; /// Wrapper of `FallbackProvider` for use in `hyperlane-cosmos` #[derive(new, Clone)] -pub struct CosmosFallbackProvider { - fallback_provider: FallbackProvider, - _phantom: PhantomData, +pub struct CosmosFallbackProvider { + fallback_provider: FallbackProvider, } -impl Deref for CosmosFallbackProvider { - type Target = FallbackProvider; +impl Deref for CosmosFallbackProvider { + type Target = FallbackProvider; fn deref(&self) -> &Self::Target { &self.fallback_provider } } -impl Debug for CosmosFallbackProvider +impl Debug for CosmosFallbackProvider where C: Debug, { @@ -52,68 +42,68 @@ where } } -impl GrpcService for CosmosFallbackProvider -where - T: GrpcService + Clone + Debug + Into> + 'static, - >::Error: Into, - >::ResponseBody: - tonic::codegen::Body + Send + 'static, - ::Error: Into + Send, -{ - type ResponseBody = T::ResponseBody; - type Error = T::Error; - type Future = - Pin, Self::Error>>>>; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - let mut provider = (*self.inner.providers)[0].clone(); - provider.poll_ready(cx) - } - - fn call(&mut self, request: http::Request) -> Self::Future { - // use CategorizedResponse::*; - let request = clone_request(&request); - let cloned_self = self.clone(); - let f = async move { - let mut errors = vec![]; - // make sure we do at least 4 total retries. - while errors.len() <= 3 { - if !errors.is_empty() { - sleep(Duration::from_millis(100)).await - } - let priorities_snapshot = cloned_self.take_priorities_snapshot().await; - for (idx, priority) in priorities_snapshot.iter().enumerate() { - let mut provider = cloned_self.inner.providers[priority.index].clone(); - let resp = provider.call(clone_request(&request)).await; - cloned_self - .handle_stalled_provider(priority, &provider) - .await; - let _span = - warn_span!("request", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); - - match resp { - Ok(r) => return Ok(r), - Err(e) => errors.push(e.into()), - } - } - } - - Err(HyperlaneCosmosError::FallbackProvidersFailed(errors)) - }; - Box::pin(f) - } -} - -fn clone_request(request: &http::Request) -> http::Request { - let builder = http::Request::builder() - .uri(request.uri().clone()) - .method(request.method().clone()) - .version(request.version()); - let builder = request.headers().iter().fold(builder, |builder, (k, v)| { - builder.header(k.clone(), v.clone()) - }); - builder.body(request.body().clone()).unwrap() -} +// impl GrpcService for CosmosFallbackProvider +// where +// T: GrpcService + Clone + Debug + Into> + 'static, +// >::Error: Into, +// >::ResponseBody: +// tonic::codegen::Body + Send + 'static, +// ::Error: Into + Send, +// { +// type ResponseBody = T::ResponseBody; +// type Error = T::Error; +// type Future = +// Pin, Self::Error>>>>; + +// fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { +// let mut provider = (*self.inner.providers)[0].clone(); +// provider.poll_ready(cx) +// } + +// fn call(&mut self, request: http::Request) -> Self::Future { +// // use CategorizedResponse::*; +// let request = clone_request(&request); +// let cloned_self = self.clone(); +// let f = async move { +// let mut errors = vec![]; +// // make sure we do at least 4 total retries. +// while errors.len() <= 3 { +// if !errors.is_empty() { +// sleep(Duration::from_millis(100)).await +// } +// let priorities_snapshot = cloned_self.take_priorities_snapshot().await; +// for (idx, priority) in priorities_snapshot.iter().enumerate() { +// let mut provider = cloned_self.inner.providers[priority.index].clone(); +// let resp = provider.call(clone_request(&request)).await; +// cloned_self +// .handle_stalled_provider(priority, &provider) +// .await; +// let _span = +// warn_span!("request", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); + +// match resp { +// Ok(r) => return Ok(r), +// Err(e) => errors.push(e.into()), +// } +// } +// } + +// Err(HyperlaneCosmosError::FallbackProvidersFailed(errors)) +// }; +// Box::pin(f) +// } +// } + +// fn clone_request(request: &http::Request) -> http::Request { +// let builder = http::Request::builder() +// .uri(request.uri().clone()) +// .method(request.method().clone()) +// .version(request.version()); +// let builder = request.headers().iter().fold(builder, |builder, (k, v)| { +// builder.header(k.clone(), v.clone()) +// }); +// builder.body(request.body().clone()).unwrap() +// } #[cfg(test)] mod tests { @@ -160,45 +150,21 @@ mod tests { } } - impl GrpcService for CosmosProviderMock - where - ReqBody: Clone + 'static, - { - type ResponseBody = tonic::body::BoxBody; - type Error = HyperlaneCosmosError; - type Future = - Pin, Self::Error>>>>; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - todo!() - } - - fn call(&mut self, request: http::Request) -> Self::Future { - self.push( - Default::default(), - format!("method: {:?}, uri: {:?}", request.method(), request.uri()), - ); - let body = tonic::body::BoxBody::default(); - let self_clone = self.clone(); - let f = async move { - let response = http::Response::builder().status(200).body(body).unwrap(); - if let Some(sleep_duration) = self_clone.request_sleep() { - sleep(sleep_duration).await; - } - Ok(response) - }; - Box::pin(f) - } - } - - impl CosmosFallbackProvider { + impl CosmosFallbackProvider { async fn low_level_test_call(&mut self) -> Result<(), ChainCommunicationError> { - let request = http::Request::builder() - .uri("http://localhost:1234") - .method("GET") - .body(()) - .unwrap(); - self.call(request).await?; + self.call(|provider| { + provider.push("GET", "http://localhost:1234"); + let future = async move { + let body = tonic::body::BoxBody::default(); + let response = http::Response::builder().status(200).body(body).unwrap(); + if let Some(sleep_duration) = provider.request_sleep() { + sleep(sleep_duration).await; + } + Ok(response) + }; + Pin::from(Box::from(future)) + }) + .await?; Ok(()) } } diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index d29396363e..5feac3add8 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -12,23 +12,23 @@ use serde_json::Value; use tokio::time::sleep; use tracing::{instrument, warn_span}; -use ethers_prometheus::json_rpc_client::PrometheusJsonRpcClientConfigExt; +use ethers_prometheus::json_rpc_client::{JsonRpcBlockGetter, PrometheusJsonRpcClientConfigExt}; use crate::rpc_clients::{categorize_client_response, CategorizedResponse}; /// Wrapper of `FallbackProvider` for use in `hyperlane-ethereum` #[derive(new)] -pub struct EthereumFallbackProvider(FallbackProvider); +pub struct EthereumFallbackProvider(FallbackProvider); -impl Deref for EthereumFallbackProvider { - type Target = FallbackProvider; +impl Deref for EthereumFallbackProvider { + type Target = FallbackProvider; fn deref(&self) -> &Self::Target { &self.0 } } -impl Debug for EthereumFallbackProvider +impl Debug for EthereumFallbackProvider where C: JsonRpcClient + PrometheusJsonRpcClientConfigExt, { @@ -75,12 +75,13 @@ impl From for ProviderError { #[cfg_attr(target_arch = "wasm32", async_trait(?Send))] #[cfg_attr(not(target_arch = "wasm32"), async_trait)] -impl JsonRpcClient for EthereumFallbackProvider +impl JsonRpcClient for EthereumFallbackProvider> where C: JsonRpcClient + + Into> + PrometheusJsonRpcClientConfigExt - + Into> + Clone, + JsonRpcBlockGetter: BlockNumberGetter, { type Error = ProviderError; @@ -155,9 +156,9 @@ mod tests { } } - impl Into> for EthereumProviderMock { - fn into(self) -> Box { - Box::new(JsonRpcBlockGetter::new(self.clone())) + impl Into> for EthereumProviderMock { + fn into(self) -> JsonRpcBlockGetter { + JsonRpcBlockGetter::new(self) } } @@ -197,12 +198,13 @@ mod tests { } } - impl EthereumFallbackProvider + impl EthereumFallbackProvider> where C: JsonRpcClient + PrometheusJsonRpcClientConfigExt - + Into> + + Into> + Clone, + JsonRpcBlockGetter: BlockNumberGetter, { async fn low_level_test_call(&self) { self.request::<_, u64>(BLOCK_NUMBER_RPC, ()).await.unwrap(); diff --git a/rust/chains/hyperlane-ethereum/src/trait_builder.rs b/rust/chains/hyperlane-ethereum/src/trait_builder.rs index 31fa128d86..a0ab93af48 100644 --- a/rust/chains/hyperlane-ethereum/src/trait_builder.rs +++ b/rust/chains/hyperlane-ethereum/src/trait_builder.rs @@ -16,8 +16,8 @@ use reqwest::{Client, Url}; use thiserror::Error; use ethers_prometheus::json_rpc_client::{ - JsonRpcClientMetrics, JsonRpcClientMetricsBuilder, NodeInfo, PrometheusJsonRpcClient, - PrometheusJsonRpcClientConfig, + JsonRpcBlockGetter, JsonRpcClientMetrics, JsonRpcClientMetricsBuilder, NodeInfo, + PrometheusJsonRpcClient, PrometheusJsonRpcClientConfig, }; use ethers_prometheus::middleware::{ MiddlewareMetrics, PrometheusMiddleware, PrometheusMiddlewareConf, @@ -116,7 +116,10 @@ pub trait BuildableWithProvider { builder = builder.add_provider(metrics_provider); } let fallback_provider = builder.build(); - let ethereum_fallback_provider = EthereumFallbackProvider::new(fallback_provider); + let ethereum_fallback_provider = EthereumFallbackProvider::< + _, + JsonRpcBlockGetter>, + >::new(fallback_provider); self.build( ethereum_fallback_provider, locator, diff --git a/rust/ethers-prometheus/src/json_rpc_client.rs b/rust/ethers-prometheus/src/json_rpc_client.rs index 7e0c4d1fee..2cc8defe9b 100644 --- a/rust/ethers-prometheus/src/json_rpc_client.rs +++ b/rust/ethers-prometheus/src/json_rpc_client.rs @@ -186,9 +186,11 @@ where } } -impl From> for Box { +impl From> + for JsonRpcBlockGetter> +{ fn from(val: PrometheusJsonRpcClient) -> Self { - Box::new(JsonRpcBlockGetter::new(val)) + JsonRpcBlockGetter::new(val) } } diff --git a/rust/hyperlane-core/Cargo.toml b/rust/hyperlane-core/Cargo.toml index 8329bce5f2..b49c9872b2 100644 --- a/rust/hyperlane-core/Cargo.toml +++ b/rust/hyperlane-core/Cargo.toml @@ -37,6 +37,7 @@ serde_json = { workspace = true } sha3 = { workspace = true } strum = { workspace = true, optional = true, features = ["derive"] } thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt", "time"] } tracing.workspace = true primitive-types = { workspace = true, optional = true } solana-sdk = { workspace = true, optional = true } diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index eb553e5bd3..e493d0b5ef 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -4,13 +4,17 @@ use derive_new::new; use std::{ fmt::Debug, marker::PhantomData, + pin::Pin, sync::Arc, time::{Duration, Instant}, }; +use tokio; use tracing::info; use crate::ChainCommunicationError; +use super::RpcClientError; + /// Read the current block number from a chain. #[async_trait] pub trait BlockNumberGetter: Send + Sync + Debug { @@ -39,7 +43,6 @@ impl PrioritizedProviderInner { } } } - /// Sub-providers and priority information pub struct PrioritizedProviders { /// Unsorted list of providers this provider calls @@ -134,6 +137,36 @@ where .await; } } + + /// Call the first provider, then the second, and so on (in order of priority) until a response is received. + /// If all providers fail, return an error. + pub async fn call( + &self, + mut f: impl FnMut( + T, + ) -> Pin< + Box> + Send>, + >, + ) -> Result { + let mut errors = vec![]; + // make sure we do at least 4 total retries. + while errors.len() <= 3 { + if !errors.is_empty() { + tokio::time::sleep(Duration::from_millis(100)).await; + } + let priorities_snapshot = self.take_priorities_snapshot().await; + for (_idx, priority) in priorities_snapshot.iter().enumerate() { + let provider = &self.inner.providers[priority.index]; + self.handle_stalled_provider(priority, provider).await; + match f(provider.clone()).await { + Ok(v) => return Ok(v), + Err(e) => errors.push(e), + } + } + } + + Err(RpcClientError::FallbackProvidersFailed(errors).into()) + } } /// Builder to create a new fallback provider. @@ -195,6 +228,7 @@ impl FallbackProviderBuilder { } } +/// Utilities to import when testing chain-specific fallback providers pub mod test { use super::*; use std::{ @@ -202,6 +236,7 @@ pub mod test { sync::{Arc, Mutex}, }; + /// Provider that stores requests and optionally sleeps before returning a dummy value #[derive(Debug, Clone)] pub struct ProviderMock { // Store requests as tuples of (method, params) @@ -221,6 +256,7 @@ pub mod test { } impl ProviderMock { + /// Create a new provider pub fn new(request_sleep: Option) -> Self { Self { request_sleep, @@ -228,6 +264,7 @@ pub mod test { } } + /// Push a request to the internal store for later inspection pub fn push(&self, method: &str, params: T) { self.requests .lock() @@ -235,14 +272,17 @@ pub mod test { .push((method.to_owned(), format!("{:?}", params))); } + /// Get the stored requests pub fn requests(&self) -> Vec<(String, String)> { self.requests.lock().unwrap().clone() } + /// Set the sleep duration pub fn request_sleep(&self) -> Option { self.request_sleep } + /// Get how many times each provider was called pub async fn get_call_counts, B>( fallback_provider: &FallbackProvider, ) -> Vec { From a347aff17b15bc66b0c6b40160651f878cdc620d Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Tue, 30 Jan 2024 14:56:38 +0000 Subject: [PATCH 12/18] chore: clean up --- .../hyperlane-cosmos/src/providers/grpc.rs | 11 --- .../src/rpc_clients/fallback.rs | 69 ++----------------- .../src/rpc_clients/fallback.rs | 2 - .../src/rpc_clients/fallback.rs | 9 ++- 4 files changed, 11 insertions(+), 80 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index 5eb4ae4b01..07f8c44968 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -151,18 +151,7 @@ impl WasmGrpcProvider { // get all the configured grpc urls and convert them to a Vec let endpoint = Endpoint::new(conf.get_grpc_url()).map_err(Into::::into)?; - // create a vec of channels. Replace single Client instantiations with an instantiation over all channels, and then wrapping them in a fallback provider. - - // However, in this case the fallback provider wouldn't be able to memorize the prioritization across calls - // Alternatively, could create a struct Clients that wraps all client types; we'd have one Clients instance per channel and should be straightforward to read blocks this way too. - - // Alternatively, could try wrapping the channels directly in a fallback provider, and reprioritizing that way - - // Looks like the way to go is to create a (Channel, BlockReaderClient) tuple and implement `GrpcService` for it let channel = endpoint.connect_lazy(); - - // Another option is to create - let mut builder = FallbackProvider::builder(); builder = builder.add_provider(CosmosChannel::from(channel)); let fallback_provider = builder.build(); diff --git a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs index 1daea1a2c7..47267f0a16 100644 --- a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs @@ -42,75 +42,16 @@ where } } -// impl GrpcService for CosmosFallbackProvider -// where -// T: GrpcService + Clone + Debug + Into> + 'static, -// >::Error: Into, -// >::ResponseBody: -// tonic::codegen::Body + Send + 'static, -// ::Error: Into + Send, -// { -// type ResponseBody = T::ResponseBody; -// type Error = T::Error; -// type Future = -// Pin, Self::Error>>>>; - -// fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { -// let mut provider = (*self.inner.providers)[0].clone(); -// provider.poll_ready(cx) -// } - -// fn call(&mut self, request: http::Request) -> Self::Future { -// // use CategorizedResponse::*; -// let request = clone_request(&request); -// let cloned_self = self.clone(); -// let f = async move { -// let mut errors = vec![]; -// // make sure we do at least 4 total retries. -// while errors.len() <= 3 { -// if !errors.is_empty() { -// sleep(Duration::from_millis(100)).await -// } -// let priorities_snapshot = cloned_self.take_priorities_snapshot().await; -// for (idx, priority) in priorities_snapshot.iter().enumerate() { -// let mut provider = cloned_self.inner.providers[priority.index].clone(); -// let resp = provider.call(clone_request(&request)).await; -// cloned_self -// .handle_stalled_provider(priority, &provider) -// .await; -// let _span = -// warn_span!("request", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); - -// match resp { -// Ok(r) => return Ok(r), -// Err(e) => errors.push(e.into()), -// } -// } -// } - -// Err(HyperlaneCosmosError::FallbackProvidersFailed(errors)) -// }; -// Box::pin(f) -// } -// } - -// fn clone_request(request: &http::Request) -> http::Request { -// let builder = http::Request::builder() -// .uri(request.uri().clone()) -// .method(request.method().clone()) -// .version(request.version()); -// let builder = request.headers().iter().fold(builder, |builder, (k, v)| { -// builder.header(k.clone(), v.clone()) -// }); -// builder.body(request.body().clone()).unwrap() -// } - #[cfg(test)] mod tests { + use std::pin::Pin; + use std::time::Duration; + use async_trait::async_trait; use hyperlane_core::rpc_clients::test::ProviderMock; - use hyperlane_core::rpc_clients::FallbackProviderBuilder; + use hyperlane_core::rpc_clients::{BlockNumberGetter, FallbackProviderBuilder}; use hyperlane_core::ChainCommunicationError; + use tokio::time::sleep; use super::*; diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index 5feac3add8..6246b2e13f 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -37,7 +37,6 @@ where .field( "chain_name", &self - .0 .inner .providers .get(0) @@ -47,7 +46,6 @@ where .field( "hosts", &self - .0 .inner .providers .iter() diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index e493d0b5ef..bc24c96e73 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -9,7 +9,7 @@ use std::{ time::{Duration, Instant}, }; use tokio; -use tracing::info; +use tracing::{info, warn_span}; use crate::ChainCommunicationError; @@ -155,10 +155,13 @@ where tokio::time::sleep(Duration::from_millis(100)).await; } let priorities_snapshot = self.take_priorities_snapshot().await; - for (_idx, priority) in priorities_snapshot.iter().enumerate() { + for (idx, priority) in priorities_snapshot.iter().enumerate() { let provider = &self.inner.providers[priority.index]; + let resp = f(provider.clone()).await; self.handle_stalled_provider(priority, provider).await; - match f(provider.clone()).await { + let _span = + warn_span!("FallbackProvider::call", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); + match resp { Ok(v) => return Ok(v), Err(e) => errors.push(e), } From 22cbb0960f420827cc55468253d329bdb2c4882d Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Tue, 30 Jan 2024 18:07:40 +0000 Subject: [PATCH 13/18] feat(cosmos): `grpcUrl` -> `grpcUrls` agents config --- .../hyperlane-cosmos/src/providers/grpc.rs | 12 ++++++--- .../hyperlane-cosmos/src/trait_builder.rs | 10 +++---- rust/config/mainnet3_config.json | 8 ++++-- .../templates/external-secret.yaml | 2 +- .../src/settings/parser/connection_parser.rs | 27 ++++++++++--------- rust/utils/run-locally/src/cosmos/types.rs | 6 +++-- 6 files changed, 39 insertions(+), 26 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index 07f8c44968..aad93d96b0 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -149,11 +149,15 @@ impl WasmGrpcProvider { signer: Option, ) -> ChainResult { // get all the configured grpc urls and convert them to a Vec - let endpoint = - Endpoint::new(conf.get_grpc_url()).map_err(Into::::into)?; - let channel = endpoint.connect_lazy(); + let endpoints: Result, _> = conf + .get_grpc_urls() + .into_iter() + .map(|url| Endpoint::new(url).map_err(Into::::into)) + .collect(); + let channels: Vec = + endpoints?.iter().map(|e| e.connect_lazy().into()).collect(); let mut builder = FallbackProvider::builder(); - builder = builder.add_provider(CosmosChannel::from(channel)); + builder = builder.add_providers(channels); let fallback_provider = builder.build(); let provider = CosmosFallbackProvider::new(fallback_provider); diff --git a/rust/chains/hyperlane-cosmos/src/trait_builder.rs b/rust/chains/hyperlane-cosmos/src/trait_builder.rs index 2bacb4d2f5..56f9c375c4 100644 --- a/rust/chains/hyperlane-cosmos/src/trait_builder.rs +++ b/rust/chains/hyperlane-cosmos/src/trait_builder.rs @@ -7,7 +7,7 @@ use hyperlane_core::{ChainCommunicationError, FixedPointNumber}; #[derive(Debug, Clone)] pub struct ConnectionConf { /// The GRPC url to connect to - grpc_url: String, + grpc_urls: Vec, /// The RPC url to connect to rpc_url: String, /// The chain ID @@ -76,8 +76,8 @@ pub enum ConnectionConfError { impl ConnectionConf { /// Get the GRPC url - pub fn get_grpc_url(&self) -> String { - self.grpc_url.clone() + pub fn get_grpc_urls(&self) -> Vec { + self.grpc_urls.clone() } /// Get the RPC url @@ -112,7 +112,7 @@ impl ConnectionConf { /// Create a new connection configuration pub fn new( - grpc_url: String, + grpc_urls: Vec, rpc_url: String, chain_id: String, bech32_prefix: String, @@ -121,7 +121,7 @@ impl ConnectionConf { contract_address_bytes: usize, ) -> Self { Self { - grpc_url, + grpc_urls, rpc_url, chain_id, bech32_prefix, diff --git a/rust/config/mainnet3_config.json b/rust/config/mainnet3_config.json index 9f831e686a..bc9af235c0 100644 --- a/rust/config/mainnet3_config.json +++ b/rust/config/mainnet3_config.json @@ -480,7 +480,11 @@ "http": "https://rpc-injective.goldenratiostaking.net:443" } ], - "grpcUrl": "https://injective-grpc.publicnode.com/", + "grpcUrls": [ + { + "http": "https://injective-grpc.goldenratiostaking.net:443" + } + ], "canonicalAsset": "inj", "bech32Prefix": "inj", "gasPrice": { @@ -820,4 +824,4 @@ } }, "defaultRpcConsensusType": "fallback" -} +} \ No newline at end of file diff --git a/rust/helm/hyperlane-agent/templates/external-secret.yaml b/rust/helm/hyperlane-agent/templates/external-secret.yaml index 5d0eae5ced..53bcde0c3e 100644 --- a/rust/helm/hyperlane-agent/templates/external-secret.yaml +++ b/rust/helm/hyperlane-agent/templates/external-secret.yaml @@ -29,7 +29,7 @@ spec: {{- if not .disabled }} HYP_CHAINS_{{ .name | upper }}_CUSTOMRPCURLS: {{ printf "'{{ .%s_rpcs | mustFromJson | join \",\" }}'" .name }} {{- if eq .protocol "cosmos" }} - HYP_CHAINS_{{ .name | upper }}_GRPCURL: {{ printf "'{{ .%s_grpc }}'" .name }} + HYP_CHAINS_{{ .name | upper }}_GRPCURLS: {{ printf "'{{ .%s_grpc }}'" .name }} {{- end }} {{- end }} {{- end }} diff --git a/rust/hyperlane-base/src/settings/parser/connection_parser.rs b/rust/hyperlane-base/src/settings/parser/connection_parser.rs index 5d42ce4411..4ee60ec863 100644 --- a/rust/hyperlane-base/src/settings/parser/connection_parser.rs +++ b/rust/hyperlane-base/src/settings/parser/connection_parser.rs @@ -1,6 +1,7 @@ use eyre::eyre; use hyperlane_core::config::ConfigErrResultExt; use hyperlane_core::{config::ConfigParsingError, HyperlaneDomainProtocol}; +use itertools::Itertools; use url::Url; use crate::settings::envs::*; @@ -44,18 +45,20 @@ pub fn build_cosmos_connection_conf( ) -> Option { let mut local_err = ConfigParsingError::default(); - let grpc_url = chain + let grpc_urls = chain .chain(&mut local_err) - .get_key("grpcUrl") - .parse_string() - .end() - .or_else(|| { - local_err.push( - &chain.cwp + "grpc_url", - eyre!("Missing grpc definitions for chain"), - ); - None - }); + .get_key("grpcUrls") + .into_array_iter() + .map(|urls| { + urls.filter_map(|v| { + v.chain(err) + .get_key("http") + .parse_from_str("Invalid http url") + .end() + }) + .collect_vec() + }) + .unwrap_or_default(); let chain_id = chain .chain(&mut local_err) @@ -114,7 +117,7 @@ pub fn build_cosmos_connection_conf( None } else { Some(ChainConnectionConf::Cosmos(h_cosmos::ConnectionConf::new( - grpc_url.unwrap().to_string(), + grpc_urls, rpcs.first().unwrap().to_string(), chain_id.unwrap().to_string(), prefix.unwrap().to_string(), diff --git a/rust/utils/run-locally/src/cosmos/types.rs b/rust/utils/run-locally/src/cosmos/types.rs index 795d12ff39..cc2e3745f5 100644 --- a/rust/utils/run-locally/src/cosmos/types.rs +++ b/rust/utils/run-locally/src/cosmos/types.rs @@ -118,7 +118,7 @@ pub struct AgentConfig { pub protocol: String, pub chain_id: String, pub rpc_urls: Vec, - pub grpc_url: String, + pub grpc_urls: Vec, pub bech32_prefix: String, pub signer: AgentConfigSigner, pub index: AgentConfigIndex, @@ -156,7 +156,9 @@ impl AgentConfig { network.launch_resp.endpoint.rpc_addr.replace("tcp://", "") ), }], - grpc_url: format!("http://{}", network.launch_resp.endpoint.grpc_addr), + grpc_urls: vec![AgentUrl { + http: format!("http://{}", network.launch_resp.endpoint.grpc_addr), + }], bech32_prefix: "osmo".to_string(), signer: AgentConfigSigner { typ: "cosmosKey".to_string(), From da164302ce13fc852b8db42407ad1f54b0fac70e Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Wed, 31 Jan 2024 14:59:49 +0000 Subject: [PATCH 14/18] feat(cosmos): add `customGrpcUrls` agent config --- .../src/settings/parser/connection_parser.rs | 31 +++++++++++++++++-- .../sdk/src/metadata/chainMetadataTypes.ts | 6 ++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/rust/hyperlane-base/src/settings/parser/connection_parser.rs b/rust/hyperlane-base/src/settings/parser/connection_parser.rs index 4ee60ec863..c7a563b000 100644 --- a/rust/hyperlane-base/src/settings/parser/connection_parser.rs +++ b/rust/hyperlane-base/src/settings/parser/connection_parser.rs @@ -45,7 +45,7 @@ pub fn build_cosmos_connection_conf( ) -> Option { let mut local_err = ConfigParsingError::default(); - let grpc_urls = chain + let grpcs_base = chain .chain(&mut local_err) .get_key("grpcUrls") .into_array_iter() @@ -60,6 +60,33 @@ pub fn build_cosmos_connection_conf( }) .unwrap_or_default(); + let grpc_overrides = chain + .chain(&mut local_err) + .get_opt_key("customGrpcUrls") + .parse_string() + .end() + .map(|urls| { + urls.split(',') + .filter_map(|url| { + url.parse() + .take_err(&mut local_err, || &chain.cwp + "customGrpcUrls") + }) + .collect_vec() + }); + + let grpcs = grpc_overrides.unwrap_or(grpcs_base); + + if grpcs.is_empty() { + err.push( + &chain.cwp + "grpc_urls", + eyre!("Missing base grpc definitions for chain"), + ); + err.push( + &chain.cwp + "custom_grpc_urls", + eyre!("Also missing grpc overrides for chain"), + ); + } + let chain_id = chain .chain(&mut local_err) .get_key("chainId") @@ -117,7 +144,7 @@ pub fn build_cosmos_connection_conf( None } else { Some(ChainConnectionConf::Cosmos(h_cosmos::ConnectionConf::new( - grpc_urls, + grpcs, rpcs.first().unwrap().to_string(), chain_id.unwrap().to_string(), prefix.unwrap().to_string(), diff --git a/typescript/sdk/src/metadata/chainMetadataTypes.ts b/typescript/sdk/src/metadata/chainMetadataTypes.ts index d8ee616421..983d8f0bce 100644 --- a/typescript/sdk/src/metadata/chainMetadataTypes.ts +++ b/typescript/sdk/src/metadata/chainMetadataTypes.ts @@ -115,6 +115,12 @@ export const ChainMetadataSchemaObject = z.object({ .array(RpcUrlSchema) .describe('For cosmos chains only, a list of gRPC API URLs') .optional(), + customGrpcUrls: z + .string() + .optional() + .describe( + 'Specify a comma seperated list of custom GRPC URLs to use for this chain. If not specified, the default GRPC urls will be used.', + ), blockExplorers: z .array( z.object({ From e6d060fd371216c5ecdddbb46cedc9dc48f49c89 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Wed, 31 Jan 2024 16:38:12 +0000 Subject: [PATCH 15/18] chore: improve logging and test cosmos fallbackprovider e2e --- .../hyperlane-cosmos/src/providers/grpc.rs | 21 +++++++++---------- .../src/rpc_clients/fallback.rs | 10 +++++++-- rust/utils/run-locally/src/cosmos/types.rs | 12 ++++++++--- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index aad93d96b0..d2773f5aab 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -52,12 +52,9 @@ const TIMEOUT_BLOCKS: u64 = 1000; #[derive(Debug, Clone, new)] struct CosmosChannel { channel: Channel, -} - -impl From for CosmosChannel { - fn from(channel: Channel) -> Self { - Self { channel } - } + /// The url that this channel is connected to. + /// Not explicitly used, but useful for debugging. + _url: String, } #[async_trait] @@ -149,15 +146,17 @@ impl WasmGrpcProvider { signer: Option, ) -> ChainResult { // get all the configured grpc urls and convert them to a Vec - let endpoints: Result, _> = conf + let channels: Result, _> = conf .get_grpc_urls() .into_iter() - .map(|url| Endpoint::new(url).map_err(Into::::into)) + .map(|url| { + Endpoint::new(url.clone()) + .map(|e| CosmosChannel::new(e.connect_lazy(), url)) + .map_err(Into::::into) + }) .collect(); - let channels: Vec = - endpoints?.iter().map(|e| e.connect_lazy().into()).collect(); let mut builder = FallbackProvider::builder(); - builder = builder.add_providers(channels); + builder = builder.add_providers(channels?); let fallback_provider = builder.build(); let provider = CosmosFallbackProvider::new(fallback_provider); diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index bc24c96e73..0db2a7c059 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -9,7 +9,7 @@ use std::{ time::{Duration, Instant}, }; use tokio; -use tracing::{info, warn_span}; +use tracing::{info, trace, warn_span}; use crate::ChainCommunicationError; @@ -163,7 +163,13 @@ where warn_span!("FallbackProvider::call", fallback_count=%idx, provider_index=%priority.index, ?provider).entered(); match resp { Ok(v) => return Ok(v), - Err(e) => errors.push(e), + Err(e) => { + trace!( + error=?e, + "Got error from inner fallback provider", + ); + errors.push(e) + } } } } diff --git a/rust/utils/run-locally/src/cosmos/types.rs b/rust/utils/run-locally/src/cosmos/types.rs index cc2e3745f5..ed95331cea 100644 --- a/rust/utils/run-locally/src/cosmos/types.rs +++ b/rust/utils/run-locally/src/cosmos/types.rs @@ -156,9 +156,15 @@ impl AgentConfig { network.launch_resp.endpoint.rpc_addr.replace("tcp://", "") ), }], - grpc_urls: vec![AgentUrl { - http: format!("http://{}", network.launch_resp.endpoint.grpc_addr), - }], + grpc_urls: vec![ + // The first url points to a nonexistent node, but is used for checking fallback provider logic + AgentUrl { + http: "localhost:1337".to_string(), + }, + AgentUrl { + http: format!("http://{}", network.launch_resp.endpoint.grpc_addr), + }, + ], bech32_prefix: "osmo".to_string(), signer: AgentConfigSigner { typ: "cosmosKey".to_string(), From f1ffa73ebbb2fe4df776a177e924f363c98d2afe Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 5 Feb 2024 12:25:52 +0000 Subject: [PATCH 16/18] fixes --- .../hyperlane-cosmos/src/providers/grpc.rs | 21 ++-- .../src/rpc_clients/fallback.rs | 18 +-- .../hyperlane-cosmos/src/trait_builder.rs | 7 +- rust/chains/hyperlane-ethereum/Cargo.toml | 2 +- .../src/rpc_clients/fallback.rs | 2 +- .../templates/external-secret.yaml | 6 +- .../src/settings/parser/connection_parser.rs | 47 +------- .../hyperlane-base/src/settings/parser/mod.rs | 106 +++++++++++------- .../src/rpc_clients/fallback.rs | 35 +++++- typescript/sdk/src/metadata/agentConfig.ts | 2 +- .../sdk/src/metadata/chainMetadataTypes.ts | 2 +- 11 files changed, 120 insertions(+), 128 deletions(-) diff --git a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs index d2773f5aab..a6bc070aba 100644 --- a/rust/chains/hyperlane-cosmos/src/providers/grpc.rs +++ b/rust/chains/hyperlane-cosmos/src/providers/grpc.rs @@ -1,5 +1,3 @@ -use std::pin::Pin; - use async_trait::async_trait; use cosmrs::{ proto::{ @@ -37,6 +35,7 @@ use tonic::{ transport::{Channel, Endpoint}, GrpcMethod, IntoRequest, }; +use url::Url; use crate::{address::CosmosAddress, CosmosAmount}; use crate::{rpc_clients::CosmosFallbackProvider, HyperlaneCosmosError}; @@ -54,7 +53,7 @@ struct CosmosChannel { channel: Channel, /// The url that this channel is connected to. /// Not explicitly used, but useful for debugging. - _url: String, + _url: Url, } #[async_trait] @@ -150,7 +149,7 @@ impl WasmGrpcProvider { .get_grpc_urls() .into_iter() .map(|url| { - Endpoint::new(url.clone()) + Endpoint::new(url.to_string()) .map(|e| CosmosChannel::new(e.connect_lazy(), url)) .map_err(Into::::into) }) @@ -301,7 +300,7 @@ impl WasmGrpcProvider { Ok(gas_used) }; - Pin::from(Box::from(future)) + Box::pin(future) }) .await?; @@ -328,7 +327,7 @@ impl WasmGrpcProvider { .into_inner(); Ok(response) }; - Pin::from(Box::from(future)) + Box::pin(future) }) .await?; @@ -361,7 +360,7 @@ impl WasmGrpcProvider { .into_inner(); Ok(response) }; - Pin::from(Box::from(future)) + Box::pin(future) }) .await?; @@ -415,7 +414,7 @@ impl WasmGrpcProvider { Ok(response) }; - Pin::from(Box::from(future)) + Box::pin(future) }) .await?; @@ -460,7 +459,7 @@ impl WasmProvider for WasmGrpcProvider { .into_inner(); Ok(response) }; - Pin::from(Box::from(future)) + Box::pin(future) }) .await?; @@ -519,7 +518,7 @@ impl WasmProvider for WasmGrpcProvider { .into_inner(); Ok(response) }; - Pin::from(Box::from(future)) + Box::pin(future) }) .await?; @@ -573,7 +572,7 @@ impl WasmProvider for WasmGrpcProvider { .tx_response .ok_or_else(|| ChainCommunicationError::from_other_str("Empty tx_response")) }; - Pin::from(Box::from(future)) + Box::pin(future) }) .await?; Ok(tx_res) diff --git a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs index 47267f0a16..cf933ee73d 100644 --- a/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-cosmos/src/rpc_clients/fallback.rs @@ -5,7 +5,6 @@ use std::{ use derive_new::new; use hyperlane_core::rpc_clients::FallbackProvider; -use itertools::Itertools; /// Wrapper of `FallbackProvider` for use in `hyperlane-cosmos` #[derive(new, Clone)] @@ -26,25 +25,12 @@ where C: Debug, { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - // iterate the inner providers and write them to the formatter - f.debug_struct("FallbackProvider") - .field( - "providers", - &self - .fallback_provider - .inner - .providers - .iter() - .map(|v| format!("{:?}", v)) - .join(", "), - ) - .finish() + self.fallback_provider.fmt(f) } } #[cfg(test)] mod tests { - use std::pin::Pin; use std::time::Duration; use async_trait::async_trait; @@ -103,7 +89,7 @@ mod tests { } Ok(response) }; - Pin::from(Box::from(future)) + Box::pin(future) }) .await?; Ok(()) diff --git a/rust/chains/hyperlane-cosmos/src/trait_builder.rs b/rust/chains/hyperlane-cosmos/src/trait_builder.rs index 56f9c375c4..1bb3627b9d 100644 --- a/rust/chains/hyperlane-cosmos/src/trait_builder.rs +++ b/rust/chains/hyperlane-cosmos/src/trait_builder.rs @@ -2,12 +2,13 @@ use std::str::FromStr; use derive_new::new; use hyperlane_core::{ChainCommunicationError, FixedPointNumber}; +use url::Url; /// Cosmos connection configuration #[derive(Debug, Clone)] pub struct ConnectionConf { /// The GRPC url to connect to - grpc_urls: Vec, + grpc_urls: Vec, /// The RPC url to connect to rpc_url: String, /// The chain ID @@ -76,7 +77,7 @@ pub enum ConnectionConfError { impl ConnectionConf { /// Get the GRPC url - pub fn get_grpc_urls(&self) -> Vec { + pub fn get_grpc_urls(&self) -> Vec { self.grpc_urls.clone() } @@ -112,7 +113,7 @@ impl ConnectionConf { /// Create a new connection configuration pub fn new( - grpc_urls: Vec, + grpc_urls: Vec, rpc_url: String, chain_id: String, bech32_prefix: String, diff --git a/rust/chains/hyperlane-ethereum/Cargo.toml b/rust/chains/hyperlane-ethereum/Cargo.toml index a37660b1ee..70a06eed55 100644 --- a/rust/chains/hyperlane-ethereum/Cargo.toml +++ b/rust/chains/hyperlane-ethereum/Cargo.toml @@ -27,7 +27,7 @@ serde_json.workspace = true thiserror.workspace = true tokio.workspace = true tracing-futures.workspace = true -tracing = { workspace = true, features = ["log"] } +tracing = { workspace = true } url.workspace = true hyperlane-core = { path = "../../hyperlane-core" } diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index 6246b2e13f..9660676323 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -83,7 +83,7 @@ where { type Error = ProviderError; - // TODO: Refactor the reusable parts of this function when implementing the cosmos-specific logic + // TODO: Refactor to use `FallbackProvider::call` #[instrument] async fn request(&self, method: &str, params: T) -> Result where diff --git a/rust/helm/hyperlane-agent/templates/external-secret.yaml b/rust/helm/hyperlane-agent/templates/external-secret.yaml index 53bcde0c3e..36f287eca3 100644 --- a/rust/helm/hyperlane-agent/templates/external-secret.yaml +++ b/rust/helm/hyperlane-agent/templates/external-secret.yaml @@ -29,7 +29,7 @@ spec: {{- if not .disabled }} HYP_CHAINS_{{ .name | upper }}_CUSTOMRPCURLS: {{ printf "'{{ .%s_rpcs | mustFromJson | join \",\" }}'" .name }} {{- if eq .protocol "cosmos" }} - HYP_CHAINS_{{ .name | upper }}_GRPCURLS: {{ printf "'{{ .%s_grpc }}'" .name }} + HYP_CHAINS_{{ .name | upper }}_GRPCURLS: {{ printf "'{{ .%s_grpcs | mustFromJson | join \",\" }}'" .name }} {{- end }} {{- end }} {{- end }} @@ -44,9 +44,9 @@ spec: remoteRef: key: {{ printf "%s-rpc-endpoints-%s" $.Values.hyperlane.runEnv .name }} {{- if eq .protocol "cosmos" }} - - secretKey: {{ printf "%s_grpc" .name }} + - secretKey: {{ printf "%s_grpcs" .name }} remoteRef: - key: {{ printf "%s-grpc-endpoint-%s" $.Values.hyperlane.runEnv .name }} + key: {{ printf "%s-grpc-endpoints-%s" $.Values.hyperlane.runEnv .name }} {{- end }} {{- end }} {{- end }} diff --git a/rust/hyperlane-base/src/settings/parser/connection_parser.rs b/rust/hyperlane-base/src/settings/parser/connection_parser.rs index c7a563b000..0d47d6eca9 100644 --- a/rust/hyperlane-base/src/settings/parser/connection_parser.rs +++ b/rust/hyperlane-base/src/settings/parser/connection_parser.rs @@ -1,13 +1,12 @@ use eyre::eyre; use hyperlane_core::config::ConfigErrResultExt; use hyperlane_core::{config::ConfigParsingError, HyperlaneDomainProtocol}; -use itertools::Itertools; use url::Url; use crate::settings::envs::*; use crate::settings::ChainConnectionConf; -use super::{parse_cosmos_gas_price, ValueParser}; +use super::{parse_base_and_override_urls, parse_cosmos_gas_price, ValueParser}; pub fn build_ethereum_connection_conf( rpcs: &[Url], @@ -44,48 +43,8 @@ pub fn build_cosmos_connection_conf( err: &mut ConfigParsingError, ) -> Option { let mut local_err = ConfigParsingError::default(); - - let grpcs_base = chain - .chain(&mut local_err) - .get_key("grpcUrls") - .into_array_iter() - .map(|urls| { - urls.filter_map(|v| { - v.chain(err) - .get_key("http") - .parse_from_str("Invalid http url") - .end() - }) - .collect_vec() - }) - .unwrap_or_default(); - - let grpc_overrides = chain - .chain(&mut local_err) - .get_opt_key("customGrpcUrls") - .parse_string() - .end() - .map(|urls| { - urls.split(',') - .filter_map(|url| { - url.parse() - .take_err(&mut local_err, || &chain.cwp + "customGrpcUrls") - }) - .collect_vec() - }); - - let grpcs = grpc_overrides.unwrap_or(grpcs_base); - - if grpcs.is_empty() { - err.push( - &chain.cwp + "grpc_urls", - eyre!("Missing base grpc definitions for chain"), - ); - err.push( - &chain.cwp + "custom_grpc_urls", - eyre!("Also missing grpc overrides for chain"), - ); - } + let grpcs = + parse_base_and_override_urls(chain, "grpcUrls", "customGrpcUrls", "http", &mut local_err); let chain_id = chain .chain(&mut local_err) diff --git a/rust/hyperlane-base/src/settings/parser/mod.rs b/rust/hyperlane-base/src/settings/parser/mod.rs index 76c88fe1b3..3bfb52f91f 100644 --- a/rust/hyperlane-base/src/settings/parser/mod.rs +++ b/rust/hyperlane-base/src/settings/parser/mod.rs @@ -18,6 +18,7 @@ use hyperlane_core::{ use itertools::Itertools; use serde::Deserialize; use serde_json::Value; +use url::Url; pub use self::json_value_parser::ValueParser; pub use super::envs::*; @@ -134,47 +135,7 @@ fn parse_chain( .parse_u32() .unwrap_or(1); - let rpcs_base = chain - .chain(&mut err) - .get_key("rpcUrls") - .into_array_iter() - .map(|urls| { - urls.filter_map(|v| { - v.chain(&mut err) - .get_key("http") - .parse_from_str("Invalid http url") - .end() - }) - .collect_vec() - }) - .unwrap_or_default(); - - let rpc_overrides = chain - .chain(&mut err) - .get_opt_key("customRpcUrls") - .parse_string() - .end() - .map(|urls| { - urls.split(',') - .filter_map(|url| { - url.parse() - .take_err(&mut err, || &chain.cwp + "customRpcUrls") - }) - .collect_vec() - }); - - let rpcs = rpc_overrides.unwrap_or(rpcs_base); - - if rpcs.is_empty() { - err.push( - &chain.cwp + "rpc_urls", - eyre!("Missing base rpc definitions for chain"), - ); - err.push( - &chain.cwp + "custom_rpc_urls", - eyre!("Also missing rpc overrides for chain"), - ); - } + let rpcs = parse_base_and_override_urls(&chain, "rpcUrls", "customRpcUrls", "http", &mut err); let from = chain .chain(&mut err) @@ -418,3 +379,66 @@ fn parse_cosmos_gas_price(gas_price: ValueParser) -> ConfigResult Vec { + chain + .chain(err) + .get_key(key) + .into_array_iter() + .map(|urls| { + urls.filter_map(|v| { + v.chain(err) + .get_key(protocol) + .parse_from_str("Invalid url") + .end() + }) + .collect_vec() + }) + .unwrap_or_default() +} + +fn parse_custom_urls( + chain: &ValueParser, + key: &str, + err: &mut ConfigParsingError, +) -> Option> { + chain + .chain(err) + .get_opt_key(key) + .parse_string() + .end() + .map(|urls| { + urls.split(',') + .filter_map(|url| url.parse().take_err(err, || &chain.cwp + "customGrpcUrls")) + .collect_vec() + }) +} + +fn parse_base_and_override_urls( + chain: &ValueParser, + base_key: &str, + override_key: &str, + protocol: &str, + err: &mut ConfigParsingError, +) -> Vec { + let base = parse_urls(chain, base_key, protocol, err); + let overrides = parse_custom_urls(chain, override_key, err); + let combined = overrides.unwrap_or(base); + + if combined.is_empty() { + err.push( + &chain.cwp + "rpc_urls", + eyre!("Missing base rpc definitions for chain"), + ); + err.push( + &chain.cwp + "custom_rpc_urls", + eyre!("Also missing rpc overrides for chain"), + ); + } + combined +} diff --git a/rust/hyperlane-core/src/rpc_clients/fallback.rs b/rust/hyperlane-core/src/rpc_clients/fallback.rs index 0db2a7c059..6f75cbc4c8 100644 --- a/rust/hyperlane-core/src/rpc_clients/fallback.rs +++ b/rust/hyperlane-core/src/rpc_clients/fallback.rs @@ -1,8 +1,10 @@ use async_rwlock::RwLock; use async_trait::async_trait; use derive_new::new; +use itertools::Itertools; use std::{ - fmt::Debug, + fmt::{Debug, Formatter}, + future::Future, marker::PhantomData, pin::Pin, sync::Arc, @@ -53,6 +55,11 @@ pub struct PrioritizedProviders { /// A provider that bundles multiple providers and attempts to call the first, /// then the second, and so on until a response is received. +/// +/// Although no trait bounds are used in the struct definition, the intended purpose of `B` +/// is to be bound by `BlockNumberGetter` and have `T` be convertible to `B`. That is, +/// inner providers should be able to get the current block number, or be convertible into +/// something that is. pub struct FallbackProvider { /// The sub-providers called by this provider pub inner: Arc>, @@ -70,6 +77,26 @@ impl Clone for FallbackProvider { } } +impl Debug for FallbackProvider +where + T: Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // iterate the inner providers and write them to the formatter + f.debug_struct("FallbackProvider") + .field( + "providers", + &self + .inner + .providers + .iter() + .map(|v| format!("{:?}", v)) + .join(", "), + ) + .finish() + } +} + impl FallbackProvider where T: Into + Debug + Clone, @@ -142,11 +169,7 @@ where /// If all providers fail, return an error. pub async fn call( &self, - mut f: impl FnMut( - T, - ) -> Pin< - Box> + Send>, - >, + mut f: impl FnMut(T) -> Pin> + Send>>, ) -> Result { let mut errors = vec![]; // make sure we do at least 4 total retries. diff --git a/typescript/sdk/src/metadata/agentConfig.ts b/typescript/sdk/src/metadata/agentConfig.ts index e7133b4c4a..5aa89687ff 100644 --- a/typescript/sdk/src/metadata/agentConfig.ts +++ b/typescript/sdk/src/metadata/agentConfig.ts @@ -124,7 +124,7 @@ export const AgentChainMetadataSchema = ChainMetadataSchemaObject.merge( .string() .optional() .describe( - 'Specify a comma seperated list of custom RPC URLs to use for this chain. If not specified, the default RPC urls will be used.', + 'Specify a comma separated list of custom RPC URLs to use for this chain. If not specified, the default RPC urls will be used.', ), rpcConsensusType: z .nativeEnum(RpcConsensusType) diff --git a/typescript/sdk/src/metadata/chainMetadataTypes.ts b/typescript/sdk/src/metadata/chainMetadataTypes.ts index 983d8f0bce..63b9da0d86 100644 --- a/typescript/sdk/src/metadata/chainMetadataTypes.ts +++ b/typescript/sdk/src/metadata/chainMetadataTypes.ts @@ -119,7 +119,7 @@ export const ChainMetadataSchemaObject = z.object({ .string() .optional() .describe( - 'Specify a comma seperated list of custom GRPC URLs to use for this chain. If not specified, the default GRPC urls will be used.', + 'Specify a comma separated list of custom GRPC URLs to use for this chain. If not specified, the default GRPC urls will be used.', ), blockExplorers: z .array( From d0779f674e2a8c36899117f3262cd4821f81df7d Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 5 Feb 2024 13:53:45 +0000 Subject: [PATCH 17/18] chore: rm comment --- rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs index 9660676323..1243ecc106 100644 --- a/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs +++ b/rust/chains/hyperlane-ethereum/src/rpc_clients/fallback.rs @@ -241,8 +241,6 @@ mod tests { ethereum_fallback_provider.low_level_test_call().await; let provider_call_count: Vec<_> = ProviderMock::get_call_counts(ðereum_fallback_provider).await; - // TODO: figure out why there are 2 BLOCK_NUMBER_RPC calls to the stalled provider instead of just one. - // This is most likely due to how ethers works, because the cosmrs test does only have one call. assert_eq!(provider_call_count, vec![0, 0, 2]); } From 14ccd7aaf43756b72f2409b72ca40ce930e01fd8 Mon Sep 17 00:00:00 2001 From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:39:14 +0000 Subject: [PATCH 18/18] fix: feature flag tokio --- rust/Cargo.lock | 23 +++++++--------------- rust/Cargo.toml | 2 +- rust/agents/relayer/Cargo.toml | 2 +- rust/agents/validator/Cargo.toml | 2 +- rust/chains/hyperlane-cosmos/Cargo.toml | 2 +- rust/chains/hyperlane-ethereum/Cargo.toml | 4 ++-- rust/chains/hyperlane-fuel/Cargo.toml | 2 +- rust/chains/hyperlane-sealevel/Cargo.toml | 2 +- rust/hyperlane-core/Cargo.toml | 3 ++- rust/hyperlane-core/src/rpc_clients/mod.rs | 6 +++++- 10 files changed, 22 insertions(+), 26 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b8460388b7..ae5e616edf 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4145,7 +4145,7 @@ dependencies = [ "hyperlane-fuel", "hyperlane-sealevel", "hyperlane-test", - "itertools 0.11.0", + "itertools 0.12.0", "maplit", "paste", "prometheus", @@ -4192,7 +4192,7 @@ dependencies = [ "fixed-hash 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "getrandom 0.2.11", "hex 0.4.3", - "itertools 0.11.0", + "itertools 0.12.0", "num 0.4.1", "num-derive 0.4.1", "num-traits", @@ -4226,7 +4226,7 @@ dependencies = [ "hyperlane-core", "injective-protobuf", "injective-std", - "itertools 0.11.0", + "itertools 0.12.0", "once_cell", "protobuf", "ripemd", @@ -4439,7 +4439,7 @@ dependencies = [ "hyperlane-core", "hyperlane-sealevel-interchain-security-module-interface", "hyperlane-sealevel-message-recipient-interface", - "itertools 0.11.0", + "itertools 0.12.0", "log", "num-derive 0.4.1", "num-traits", @@ -4465,7 +4465,7 @@ dependencies = [ "hyperlane-sealevel-test-ism", "hyperlane-sealevel-test-send-receiver", "hyperlane-test-utils", - "itertools 0.11.0", + "itertools 0.12.0", "log", "num-derive 0.4.1", "num-traits", @@ -4982,15 +4982,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.0" @@ -6919,7 +6910,7 @@ dependencies = [ "hyperlane-core", "hyperlane-ethereum", "hyperlane-test", - "itertools 0.11.0", + "itertools 0.12.0", "num-derive 0.4.1", "num-traits", "once_cell", @@ -7566,7 +7557,7 @@ dependencies = [ "hyperlane-base", "hyperlane-core", "hyperlane-test", - "itertools 0.11.0", + "itertools 0.12.0", "migration", "num-bigint 0.4.4", "prometheus", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 1310918ed4..1c9a2e2ff4 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -97,7 +97,7 @@ hyper = "0.14" hyper-tls = "0.5.0" injective-protobuf = "0.2.2" injective-std = "0.1.5" -itertools = "0.11.0" +itertools = "*" jobserver = "=0.1.26" jsonrpc-core = "18.0" k256 = { version = "0.13.1", features = ["std", "ecdsa"] } diff --git a/rust/agents/relayer/Cargo.toml b/rust/agents/relayer/Cargo.toml index 0bd5697971..d6eb28a153 100644 --- a/rust/agents/relayer/Cargo.toml +++ b/rust/agents/relayer/Cargo.toml @@ -34,7 +34,7 @@ tokio = { workspace = true, features = ["rt", "macros", "parking_lot"] } tracing-futures.workspace = true tracing.workspace = true -hyperlane-core = { path = "../../hyperlane-core", features = ["agent"] } +hyperlane-core = { path = "../../hyperlane-core", features = ["agent", "fallback-provider"] } hyperlane-base = { path = "../../hyperlane-base" } hyperlane-ethereum = { path = "../../chains/hyperlane-ethereum" } diff --git a/rust/agents/validator/Cargo.toml b/rust/agents/validator/Cargo.toml index f562938db2..bfea4c16a3 100644 --- a/rust/agents/validator/Cargo.toml +++ b/rust/agents/validator/Cargo.toml @@ -24,7 +24,7 @@ tokio = { workspace = true, features = ["rt", "macros", "parking_lot"] } tracing-futures.workspace = true tracing.workspace = true -hyperlane-core = { path = "../../hyperlane-core", features = ["agent"] } +hyperlane-core = { path = "../../hyperlane-core", features = ["agent", "fallback-provider"] } hyperlane-base = { path = "../../hyperlane-base" } hyperlane-ethereum = { path = "../../chains/hyperlane-ethereum" } hyperlane-cosmos = { path = "../../chains/hyperlane-cosmos" } diff --git a/rust/chains/hyperlane-cosmos/Cargo.toml b/rust/chains/hyperlane-cosmos/Cargo.toml index 60beb84f60..6115a61e6d 100644 --- a/rust/chains/hyperlane-cosmos/Cargo.toml +++ b/rust/chains/hyperlane-cosmos/Cargo.toml @@ -39,4 +39,4 @@ tracing = { workspace = true } tracing-futures = { workspace = true } url = { workspace = true } -hyperlane-core = { path = "../../hyperlane-core" } +hyperlane-core = { path = "../../hyperlane-core", features = ["fallback-provider"]} diff --git a/rust/chains/hyperlane-ethereum/Cargo.toml b/rust/chains/hyperlane-ethereum/Cargo.toml index 70a06eed55..a72855a00b 100644 --- a/rust/chains/hyperlane-ethereum/Cargo.toml +++ b/rust/chains/hyperlane-ethereum/Cargo.toml @@ -27,10 +27,10 @@ serde_json.workspace = true thiserror.workspace = true tokio.workspace = true tracing-futures.workspace = true -tracing = { workspace = true } +tracing.workspace = true url.workspace = true -hyperlane-core = { path = "../../hyperlane-core" } +hyperlane-core = { path = "../../hyperlane-core", features = ["fallback-provider"]} ethers-prometheus = { path = "../../ethers-prometheus", features = ["serde"] } [build-dependencies] diff --git a/rust/chains/hyperlane-fuel/Cargo.toml b/rust/chains/hyperlane-fuel/Cargo.toml index 7dabcdd514..82bdbc782e 100644 --- a/rust/chains/hyperlane-fuel/Cargo.toml +++ b/rust/chains/hyperlane-fuel/Cargo.toml @@ -19,7 +19,7 @@ tracing-futures.workspace = true tracing.workspace = true url.workspace = true -hyperlane-core = { path = "../../hyperlane-core" } +hyperlane-core = { path = "../../hyperlane-core", features = ["fallback-provider"]} [build-dependencies] abigen = { path = "../../utils/abigen", features = ["fuels"] } diff --git a/rust/chains/hyperlane-sealevel/Cargo.toml b/rust/chains/hyperlane-sealevel/Cargo.toml index 248e3dfac1..ab4e9b17f6 100644 --- a/rust/chains/hyperlane-sealevel/Cargo.toml +++ b/rust/chains/hyperlane-sealevel/Cargo.toml @@ -24,7 +24,7 @@ tracing.workspace = true url.workspace = true account-utils = { path = "../../sealevel/libraries/account-utils" } -hyperlane-core = { path = "../../hyperlane-core", features = ["solana"] } +hyperlane-core = { path = "../../hyperlane-core", features = ["solana", "fallback-provider"] } hyperlane-sealevel-interchain-security-module-interface = { path = "../../sealevel/libraries/interchain-security-module-interface" } hyperlane-sealevel-mailbox = { path = "../../sealevel/programs/mailbox", features = ["no-entrypoint"] } hyperlane-sealevel-igp = { path = "../../sealevel/programs/hyperlane-sealevel-igp", features = ["no-entrypoint"] } diff --git a/rust/hyperlane-core/Cargo.toml b/rust/hyperlane-core/Cargo.toml index b49c9872b2..40468bef6c 100644 --- a/rust/hyperlane-core/Cargo.toml +++ b/rust/hyperlane-core/Cargo.toml @@ -37,7 +37,7 @@ serde_json = { workspace = true } sha3 = { workspace = true } strum = { workspace = true, optional = true, features = ["derive"] } thiserror = { workspace = true } -tokio = { workspace = true, features = ["rt", "time"] } +tokio = { workspace = true, optional = true, features = ["rt", "time"] } tracing.workspace = true primitive-types = { workspace = true, optional = true } solana-sdk = { workspace = true, optional = true } @@ -55,3 +55,4 @@ agent = ["ethers", "strum"] strum = ["dep:strum"] ethers = ["dep:ethers-core", "dep:ethers-contract", "dep:ethers-providers", "dep:primitive-types"] solana = ["dep:solana-sdk"] +fallback-provider = ["tokio"] diff --git a/rust/hyperlane-core/src/rpc_clients/mod.rs b/rust/hyperlane-core/src/rpc_clients/mod.rs index 78851f9f26..02aaae99f5 100644 --- a/rust/hyperlane-core/src/rpc_clients/mod.rs +++ b/rust/hyperlane-core/src/rpc_clients/mod.rs @@ -1,4 +1,8 @@ -pub use self::{error::*, fallback::*}; +pub use self::error::*; + +#[cfg(feature = "fallback-provider")] +pub use self::fallback::*; mod error; +#[cfg(feature = "fallback-provider")] mod fallback;