From 56743e7695faf2d2c2301d01966240a1f6441e4c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 22 Jun 2023 18:54:09 +0200 Subject: [PATCH 001/128] integrate proxy with lite-rpc for testing --- Cargo.toml | 1 + lite-rpc/Cargo.toml | 2 + quic-forward-proxy/Cargo.toml | 39 +++++ quic-forward-proxy/examples/sample_client.rs | 67 ++++++++ quic-forward-proxy/src/lib.rs | 5 + quic-forward-proxy/src/main.rs | 49 ++++++ quic-forward-proxy/src/proxy.rs | 152 ++++++++++++++++++ quic-forward-proxy/src/quic_util.rs | 3 + quic-forward-proxy/src/test_client/mod.rs | 2 + .../src/test_client/quic_test_client.rs | 93 +++++++++++ .../src/test_client/sample_data_factory.rs | 41 +++++ quic-forward-proxy/src/tls_config_provicer.rs | 87 ++++++++++ 12 files changed, 541 insertions(+) create mode 100644 quic-forward-proxy/Cargo.toml create mode 100644 quic-forward-proxy/examples/sample_client.rs create mode 100644 quic-forward-proxy/src/lib.rs create mode 100644 quic-forward-proxy/src/main.rs create mode 100644 quic-forward-proxy/src/proxy.rs create mode 100644 quic-forward-proxy/src/quic_util.rs create mode 100644 quic-forward-proxy/src/test_client/mod.rs create mode 100644 quic-forward-proxy/src/test_client/quic_test_client.rs create mode 100644 quic-forward-proxy/src/test_client/sample_data_factory.rs create mode 100644 quic-forward-proxy/src/tls_config_provicer.rs diff --git a/Cargo.toml b/Cargo.toml index 1b912d22..5a86da33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "core", "services", "lite-rpc", + "quic-forward-proxy", "bench" ] diff --git a/lite-rpc/Cargo.toml b/lite-rpc/Cargo.toml index 694be264..15c32ab0 100644 --- a/lite-rpc/Cargo.toml +++ b/lite-rpc/Cargo.toml @@ -39,6 +39,8 @@ async-channel = { workspace = true } quinn = { workspace = true } solana-lite-rpc-core = { workspace = true } solana-lite-rpc-services = { workspace = true } +# TODO remove +solana-lite-rpc-quic-forward-proxy = { path = "../quic-forward-proxy" } async-trait = { workspace = true } tokio = { version = "1.28.2", features = ["full", "fs"]} tokio-postgres = { version = "0.7.8", features = ["with-chrono-0_4"] } diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml new file mode 100644 index 00000000..41f7cf61 --- /dev/null +++ b/quic-forward-proxy/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "solana-lite-rpc-quic-forward-proxy" +version = "0.1.0" +edition = "2021" +description = "A staked proxy for transaction forwarding to TPU nodes" +rust-version = "1.67.1" +#default-run = "lite-rpc" +repository = "https://github.com/blockworks-foundation/lite-rpc" +license = "AGPL" +publish = false + +[dependencies] +solana-sdk = { workspace = true } +solana-streamer = { workspace = true } +rustls = { workspace = true, features = ["dangerous_configuration"]} +serde = { workspace = true } +serde_json = { workspace = true } +bincode = { workspace = true } +bs58 = { workspace = true } +base64 = { workspace = true } +thiserror = { workspace = true } +bytes = { workspace = true } +anyhow = { workspace = true } +log = { workspace = true } +clap = { workspace = true } +dashmap = { workspace = true } +tracing-subscriber = { workspace = true } +native-tls = { workspace = true } +prometheus = { workspace = true } +lazy_static = { workspace = true } +dotenv = { workspace = true } +async-channel = { workspace = true } +quinn = { workspace = true } +solana-lite-rpc-core = { workspace = true } +solana-lite-rpc-services = { workspace = true } +async-trait = { workspace = true } +chrono = { workspace = true } +tokio = { version = "1.28.2", features = ["full", "fs"]} +rcgen = "0.9.3" diff --git a/quic-forward-proxy/examples/sample_client.rs b/quic-forward-proxy/examples/sample_client.rs new file mode 100644 index 00000000..33e3752d --- /dev/null +++ b/quic-forward-proxy/examples/sample_client.rs @@ -0,0 +1,67 @@ +// DEPRECATED: use quic-proxy main.rs + +use std::net::{IpAddr, Ipv4Addr}; +use std::sync::Arc; +use std::time::Duration; +use anyhow::anyhow; +use log::info; +use rcgen::IsCa::SelfSignedOnly; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::time::timeout; +use lite_rpc_quic_forward_proxy::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; +use solana_lite_rpc_core::quic_connection_utils::SkipServerVerification; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + + // FIXME configured insecure https://quinn-rs.github.io/quinn/quinn/certificate.html + let mut _roots = rustls::RootCertStore::empty(); + // TODO add certs + + let mut client_crypto = rustls::ClientConfig::builder() + .with_safe_defaults() + // .with_root_certificates(roots) + .with_custom_certificate_verifier(SkipServerVerification::new()) + .with_no_client_auth(); + client_crypto.enable_early_data = true; + client_crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; + + let mut endpoint = quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())?; + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(client_crypto))); + + let connection_timeout = Duration::from_secs(5); + let connecting = endpoint.connect("127.0.0.1:8080".parse().unwrap(), "localhost").unwrap(); + let connection = timeout(connection_timeout, connecting).await??; + + let (mut send, mut recv) = connection.open_bi().await?; + + if false { // Rebind + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let addr = socket.local_addr().unwrap(); + info!("rebinding to {}", addr); + endpoint.rebind(socket).expect("rebind failed"); + } + + let request = "FOO BAR"; + + send.write_all(request.as_bytes()) + .await + .map_err(|e| anyhow!("failed to send request: {}", e))?; + send.finish() + .await + .map_err(|e| anyhow!("failed to shutdown stream: {}", e))?; + let resp = recv + .read_to_end(usize::MAX) + .await + .map_err(|e| anyhow!("failed to read response: {}", e))?; + + info!("resp: {:?}", std::str::from_utf8(&resp)); + + connection.close(99u32.into(), b"done"); + + // Give the server a fair chance to receive the close packet + endpoint.wait_idle().await; + + + Ok(()) +} diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs new file mode 100644 index 00000000..a535627e --- /dev/null +++ b/quic-forward-proxy/src/lib.rs @@ -0,0 +1,5 @@ +pub mod quic_util; +pub mod tls_config_provicer; +pub mod proxy; +pub use tls_config_provicer::SelfSignedTlsConfigProvider; +// pub mod tls_config; \ No newline at end of file diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs new file mode 100644 index 00000000..2f9c125d --- /dev/null +++ b/quic-forward-proxy/src/main.rs @@ -0,0 +1,49 @@ +use std::net::{IpAddr, SocketAddr}; +use anyhow::bail; +use log::info; +use crate::proxy::QuicForwardProxy; +use crate::test_client::quic_test_client::QuicTestClient; +use crate::tls_config_provicer::SelfSignedTlsConfigProvider; + +mod proxy; +mod test_client; +mod quic_util; +mod tls_config_provicer; + + +#[tokio::main(flavor = "multi_thread", worker_threads = 16)] +pub async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt::init(); + + let proxy_listener_addr = "127.0.0.1:11111".parse().unwrap(); + let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); + + + let main_services = QuicForwardProxy::new(proxy_listener_addr, &tls_configuration) + .await? + .start_services(); + + let proxy_addr = "127.0.0.1:11111".parse().unwrap(); + let test_client = QuicTestClient::new_with_endpoint( + proxy_addr, &tls_configuration) + .await? + .start_services(); + + + let ctrl_c_signal = tokio::signal::ctrl_c(); + + tokio::select! { + res = main_services => { + bail!("Services quit unexpectedly {res:?}"); + }, + res = test_client => { + bail!("Test Client quit unexpectedly {res:?}"); + }, + _ = ctrl_c_signal => { + info!("Received ctrl+c signal"); + + Ok(()) + } + } + +} diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs new file mode 100644 index 00000000..cd8373f3 --- /dev/null +++ b/quic-forward-proxy/src/proxy.rs @@ -0,0 +1,152 @@ +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; +use anyhow::{anyhow, bail}; +use log::{error, info, warn}; +use quinn::{Connecting, Endpoint, SendStream, ServerConfig}; +use rcgen::generate_simple_self_signed; +use rustls::{Certificate, PrivateKey}; +use rustls::server::ResolvesServerCert; +use solana_sdk::signature::Keypair; +use solana_sdk::transaction::VersionedTransaction; +use tokio::net::ToSocketAddrs; +use solana_lite_rpc_core::AnyhowJoinHandle; +use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; + + +pub struct QuicForwardProxy { + endpoint: Endpoint, +} + +impl QuicForwardProxy { + pub async fn new( + proxy_listener_addr: SocketAddr, + tls_config: &SelfSignedTlsConfigProvider) -> anyhow::Result { + let server_tls_config = tls_config.get_server_tls_crypto_config(); + + let mut quinn_server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); + + let endpoint = Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap(); + info!("listening on {}", endpoint.local_addr()?); + + + + Ok(Self {endpoint}) + + } + + pub async fn start_services( + mut self, + ) -> anyhow::Result<()> { + let endpoint = self.endpoint.clone(); + let quic_proxy: AnyhowJoinHandle = tokio::spawn(async move { + info!("TPU Quic Proxy server start on {}", endpoint.local_addr()?); + + let identity_keypair = Keypair::new(); // TODO + + while let Some(conn) = endpoint.accept().await { + info!("connection incoming"); + let fut = handle_connection2(conn); + tokio::spawn(async move { + if let Err(e) = fut.await { + error!("connection failed: {reason}", reason = e.to_string()) + } + }); + } + + // while let Some(conn) = endpoint.accept().await { + // info!("connection incoming"); + // // let fut = handle_connection(conn); + // tokio::spawn(async move { + // info!("start thread"); + // handle_connection2(conn).await.unwrap(); + // // if let Err(e) = fut.await { + // // error!("connection failed: {reason}", reason = e.to_string()) + // // } + // }); + // } + + bail!("TPU Quic Proxy server stopped"); + }); + + tokio::select! { + res = quic_proxy => { + bail!("TPU Quic Proxy server exited unexpectedly {res:?}"); + }, + } + } + +} + + +// meins +async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { + let connection = connecting.await?; + info!("connection established, remote {connection}", connection = connection.remote_address()); + + info!("established"); + async { + loop { + let stream = connection.accept_bi().await; + let (mut send, recv) = match stream { + Err(quinn::ConnectionError::ApplicationClosed { .. }) => { + info!("connection closed"); + return Ok(()); + } + Err(e) => { + warn!("connection failed: {}", e); + return Err(anyhow::Error::msg("connection failed")); + } + Ok(s) => s, + }; + tokio::spawn(async move { + let raw_tx = recv.read_to_end(100000).await + .unwrap(); + // let str = std::str::from_utf8(&result).unwrap(); + info!("read raw_tx {:02X?}", raw_tx); + + let tx = match bincode::deserialize::(&raw_tx) { + Ok(tx) => tx, + Err(err) => { + bail!(err.to_string()); + } + }; + + info!("transaction details: {} sigs", tx.signatures.len()); + + // send_data(send).await; + Ok(()) + }); + // info!("stream okey {:?}", stream); + // let fut = handle_request2(stream).await; + // tokio::spawn( + // async move { + // if let Err(e) = fut.await { + // error!("failed: {reason}", reason = e.to_string()); + // } + // } + // ); + } + } + .await?; + Ok(()) +} + +async fn send_data(mut send: SendStream) -> anyhow::Result<()> { + send.write_all(b"HELLO STRANGER\r\n").await?; + send.finish().await?; + Ok(()) +} + +async fn handle_request2( + (mut send, recv): (quinn::SendStream, quinn::RecvStream), +) -> anyhow::Result<()> { + info!("handle incoming request..."); + + send.write_all(b"HELLO STRANGER\r\n").await?; + send.finish().await?; + + Ok(()) +} diff --git a/quic-forward-proxy/src/quic_util.rs b/quic-forward-proxy/src/quic_util.rs new file mode 100644 index 00000000..21f61221 --- /dev/null +++ b/quic-forward-proxy/src/quic_util.rs @@ -0,0 +1,3 @@ + +pub const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; + diff --git a/quic-forward-proxy/src/test_client/mod.rs b/quic-forward-proxy/src/test_client/mod.rs new file mode 100644 index 00000000..c86b5c98 --- /dev/null +++ b/quic-forward-proxy/src/test_client/mod.rs @@ -0,0 +1,2 @@ +pub mod quic_test_client; +mod sample_data_factory; diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs new file mode 100644 index 00000000..e1fee566 --- /dev/null +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -0,0 +1,93 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; +use anyhow::bail; +use bytes::BufMut; +use log::info; +use quinn::{Endpoint, VarInt}; +use rustls::ClientConfig; +use tokio::io::AsyncWriteExt; +use solana_lite_rpc_core::AnyhowJoinHandle; +use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; +use crate::tls_config_provicer::ProxyTlsConfigProvider; +use solana_lite_rpc_core::quic_connection_utils::SkipServerVerification; +use crate::test_client::sample_data_factory::build_raw_sample_tx; + +pub struct QuicTestClient { + pub endpoint: Endpoint, + pub proxy_addr: SocketAddr, +} + +impl QuicTestClient { + pub async fn new_with_endpoint( + proxy_addr: SocketAddr, + tls_config: &impl ProxyTlsConfigProvider + ) -> anyhow::Result { + let client_crypto = tls_config.get_client_tls_crypto_config(); + let mut endpoint = quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())?; + endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(client_crypto))); + + Ok(Self { proxy_addr, endpoint }) + } + + // connect to a server + pub async fn start_services( + mut self, + ) -> anyhow::Result<()> { + let endpoint_copy = self.endpoint.clone(); + let test_client_service: AnyhowJoinHandle = tokio::spawn(async move { + info!("Sample Quic Client starting ..."); + + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + // TODO exit signal + loop { + // create new connection everytime + let connection_timeout = Duration::from_secs(5); + let connecting = endpoint_copy.connect(self.proxy_addr, "localhost").unwrap(); + let connection = tokio::time::timeout(connection_timeout, connecting).await??; + + for si in 0..5 { + let (mut send, mut recv) = connection.open_bi().await?; + + let raw = build_raw_sample_tx(); + info!("raw: {:02X?}", raw); + send.write_all(format!("SAMPLE DATA on stream {}", si).as_bytes()).await?; + + // shutdown stream + send.finish().await?; + } + + connection.close(VarInt::from_u32(0), b"done"); + ticker.tick().await; + } + + + + Ok(()) + }); + + tokio::select! { + res = test_client_service => { + bail!("Sample client service exited unexpectedly {res:?}"); + }, + } + } + +} + +fn build_tls_config() -> ClientConfig { + // FIXME configured insecure https://quinn-rs.github.io/quinn/quinn/certificate.html + let mut _roots = rustls::RootCertStore::empty(); + // TODO add certs + + let mut client_crypto = rustls::ClientConfig::builder() + .with_safe_defaults() + // .with_root_certificates(roots) + .with_custom_certificate_verifier(SkipServerVerification::new()) + .with_no_client_auth(); + client_crypto.enable_early_data = true; + client_crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; + + return client_crypto; +} + diff --git a/quic-forward-proxy/src/test_client/sample_data_factory.rs b/quic-forward-proxy/src/test_client/sample_data_factory.rs new file mode 100644 index 00000000..a5597ede --- /dev/null +++ b/quic-forward-proxy/src/test_client/sample_data_factory.rs @@ -0,0 +1,41 @@ +use std::path::Path; +use std::str::FromStr; +use solana_sdk::hash::Hash; +use solana_sdk::instruction::Instruction; +use solana_sdk::message::Message; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::{Keypair, keypair, Signature, Signer}; +use solana_sdk::transaction::{Transaction, VersionedTransaction}; +const MEMO_PROGRAM_ID: &str = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"; + + + +pub fn build_raw_sample_tx() -> Vec { + + let payer_keypair = keypair::read_keypair_file( + Path::new("/Users/stefan/mango/solana-wallet/solana-testnet-stefantest.json") + ).unwrap(); + + + let tx = build_sample_tx(&payer_keypair); + + let raw_tx = bincode::serialize::(&tx).expect("failed to serialize tx"); + + raw_tx +} + +fn build_sample_tx(payer_keypair: &Keypair) -> VersionedTransaction { + let blockhash = Hash::default(); + create_memo_tx(b"hi", payer_keypair, blockhash).into() +} + +// from bench helpers +pub fn create_memo_tx(msg: &[u8], payer: &Keypair, blockhash: Hash) -> Transaction { + let memo = Pubkey::from_str(MEMO_PROGRAM_ID).unwrap(); + + let instruction = Instruction::new_with_bytes(memo, msg, vec![]); + let message = Message::new(&[instruction], Some(&payer.pubkey())); + Transaction::new(&[payer], message, blockhash) +} + + diff --git a/quic-forward-proxy/src/tls_config_provicer.rs b/quic-forward-proxy/src/tls_config_provicer.rs new file mode 100644 index 00000000..10109b59 --- /dev/null +++ b/quic-forward-proxy/src/tls_config_provicer.rs @@ -0,0 +1,87 @@ +use std::sync::atomic::{AtomicU32, Ordering}; +use rcgen::generate_simple_self_signed; +use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; +use solana_lite_rpc_core::quic_connection_utils::SkipServerVerification; +use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; + +// TODO integrate with tpu_service + quic_connection_utils + +pub trait ProxyTlsConfigProvider { + + fn get_client_tls_crypto_config(&self) -> ClientConfig; + fn get_server_tls_crypto_config(&self) -> ServerConfig; + +} + +impl ProxyTlsConfigProvider for SelfSignedTlsConfigProvider { + fn get_client_tls_crypto_config(&self) -> ClientConfig { + self.client_crypto.clone() + } + + fn get_server_tls_crypto_config(&self) -> ServerConfig { + self.server_crypto.clone() + } +} + +pub struct SelfSignedTlsConfigProvider { + hostnames: Vec, + certificate: Certificate, + private_key: PrivateKey, + client_crypto: ClientConfig, + server_crypto: ServerConfig, +} + +const INSTANCES: AtomicU32 = AtomicU32::new(0); + +impl SelfSignedTlsConfigProvider { + pub fn new_singleton_self_signed_localhost() -> Self { + // note: this check could be relaxed when you know what you are doing! + assert_eq!(INSTANCES.fetch_add(1, Ordering::Relaxed), 0, "should be a singleton"); + let hostnames = vec!["localhost".to_string()]; + let (certificate, private_key) = Self::gen_tls_certificate_and_key(hostnames.clone()); + let server_crypto = Self::build_server_crypto(certificate.clone(), private_key.clone()); + Self { + hostnames, + certificate, + private_key, + client_crypto: Self::build_client_crypto(), + server_crypto: server_crypto, + } + } + + fn gen_tls_certificate_and_key(hostnames: Vec) -> (Certificate, PrivateKey) { + let cert = generate_simple_self_signed(hostnames).unwrap(); + let key = cert.serialize_private_key_der(); + (Certificate(cert.serialize_der().unwrap()), PrivateKey(key)) + } + + fn build_client_crypto() -> ClientConfig { + let mut client_crypto = rustls::ClientConfig::builder() + .with_safe_defaults() + // .with_root_certificates(roots) + .with_custom_certificate_verifier(SkipServerVerification::new()) + .with_no_client_auth(); + client_crypto.enable_early_data = true; + client_crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; + return client_crypto; + } + + fn build_server_crypto(server_cert: Certificate, server_key: PrivateKey) -> ServerConfig { + // let (server_cert, server_key) = gen_tls_certificate_and_key(); + + let mut server_crypto = rustls::ServerConfig::builder() + // FIXME we want client auth + .with_safe_defaults() + .with_no_client_auth() + .with_single_cert(vec![server_cert], server_key) + .unwrap(); + server_crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; + return server_crypto; + } + + pub fn get_client_tls_crypto_config(&self) -> &ClientConfig { + &self.client_crypto + } + +} + From 23b7d75194a998d54eda9f31aedc0794fdacfbcd Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 22 Jun 2023 18:55:15 +0200 Subject: [PATCH 002/128] patch connection utils --- core/src/quic_connection_utils.rs | 7 +++- lite-rpc/src/main.rs | 14 +++++++ .../src/tpu_utils/tpu_connection_manager.rs | 40 ++++++++++++++++++- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index 5a2d5ac8..385f1653 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -38,7 +38,10 @@ impl QuicConnectionUtils { .expect("Failed to set QUIC client certificates"); crypto.enable_early_data = true; - crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; + warn!("TEMP HACK TO ALLOW PROXY PROTOCOL"); + const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; + + crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec(), ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; let mut config = ClientConfig::new(Arc::new(crypto)); let mut transport_config = TransportConfig::default(); @@ -279,7 +282,7 @@ impl QuicConnectionUtils { } } -struct SkipServerVerification; +pub struct SkipServerVerification; impl SkipServerVerification { pub fn new() -> Arc { diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 1ffe394c..584482e8 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -7,6 +7,9 @@ use lite_rpc::{bridge::LiteBridge, cli::Args}; use log::info; use solana_sdk::signature::Keypair; use std::env; +use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; +use solana_lite_rpc_quic_forward_proxy::SelfSignedTlsConfigProvider; +// use lite_rpc_quic_forward_proxy::tls_config::SelfSignedTlsConfigProvider; async fn get_identity_keypair(identity_from_cli: &String) -> Keypair { if let Ok(identity_env_var) = env::var("IDENTITY") { @@ -64,6 +67,14 @@ pub async fn main() -> anyhow::Result<()> { let retry_after = Duration::from_secs(transaction_retry_after_secs); + + let proxy_listener_addr = "127.0.0.1:11111".parse().unwrap(); + let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); + let quicproxy_service = QuicForwardProxy::new(proxy_listener_addr, &tls_configuration) + .await? + .start_services(); + + let services = LiteBridge::new( rpc_addr, ws_addr, @@ -86,6 +97,9 @@ pub async fn main() -> anyhow::Result<()> { tokio::select! { res = services => { bail!("Services quit unexpectedly {res:?}"); + }, + res = quicproxy_service => { + bail!("Quic Proxy quit unexpectedly {res:?}"); } _ = ctrl_c_signal => { info!("Received ctrl+c signal"); diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 5aaf04dc..780675ab 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -1,5 +1,5 @@ use dashmap::DashMap; -use log::{error, trace}; +use log::{error, info, trace, warn}; use prometheus::{core::GenericGauge, opts, register_int_gauge}; use quinn::{Connection, Endpoint}; use solana_lite_rpc_core::{ @@ -18,6 +18,7 @@ use std::{ time::Duration, }; use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; +use tokio::time::timeout; pub const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(1); pub const CONNECTION_RETRY_COUNT: usize = 10; @@ -169,6 +170,16 @@ impl ActiveConnection { task_counter.fetch_add(1, Ordering::Relaxed); NB_QUIC_TASKS.inc(); let connection = connection.unwrap(); + + // SOS + info!("Sending copy of transaction batch of {} to tpu with identity {} to quic proxy", + txs.len(), identity); + Self::send_copy_of_txs_to_quicproxy( + &txs, endpoint.clone(), + // proxy address + "127.0.0.1:11111".parse().unwrap()).await.unwrap(); + + QuicConnectionUtils::send_transaction_batch( connection, txs, @@ -198,6 +209,33 @@ impl ActiveConnection { NB_QUIC_ACTIVE_CONNECTIONS.dec(); } + async fn send_copy_of_txs_to_quicproxy(txs: &Vec>, endpoint: Endpoint, proxy_address: SocketAddr) -> anyhow::Result<()> { + let txs_copy = txs.clone(); + let send_result = timeout(Duration::from_millis(500), Self::send_tx(endpoint, proxy_address, &txs_copy)); + + match send_result.await { + Ok(..) => { + info!("Successfully sent data to quic proxy"); + } + Err(e) => { + warn!("Failed to send data to quic proxy: {:?}", e); + } + } + Ok(()) + } + + async fn send_tx(endpoint: Endpoint, proxy_address: SocketAddr, mut txs: &Vec>) -> anyhow::Result<()> { + let mut connecting = endpoint.connect(proxy_address, "localhost")?; + let connection = timeout(Duration::from_millis(500), connecting).await??; + let (mut send, mut recv) = connection.open_bi().await?; + for tx in txs { + send.write_all(tx).await?; + } + send.finish().await?; + + Ok(()) + } + pub fn start_listening( &self, transaction_reciever: Receiver<(String, Vec)>, From 87219da3e44fef2cac197cc5ac99d6fa1740585d Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 23 Jun 2023 09:01:17 +0200 Subject: [PATCH 003/128] Cargo.lock --- Cargo.lock | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 127ced3a..ea7743fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2321,6 +2321,7 @@ dependencies = [ "serde", "serde_json", "solana-lite-rpc-core", + "solana-lite-rpc-quic-forward-proxy", "solana-lite-rpc-services", "solana-rpc-client", "solana-rpc-client-api", @@ -3180,6 +3181,18 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rcgen" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" +dependencies = [ + "pem", + "ring", + "time 0.3.20", + "yasna", +] + [[package]] name = "rcgen" version = "0.10.0" @@ -3990,6 +4003,39 @@ dependencies = [ "tokio", ] +[[package]] +name = "solana-lite-rpc-quic-forward-proxy" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-channel", + "async-trait", + "base64 0.21.0", + "bincode", + "bs58", + "bytes", + "chrono", + "clap 4.2.4", + "dashmap", + "dotenv", + "lazy_static", + "log", + "native-tls", + "prometheus", + "quinn", + "rcgen 0.9.3", + "rustls 0.20.8", + "serde", + "serde_json", + "solana-lite-rpc-core", + "solana-lite-rpc-services", + "solana-sdk", + "solana-streamer", + "thiserror", + "tokio", + "tracing-subscriber", +] + [[package]] name = "solana-lite-rpc-services" version = "0.2.1" @@ -4420,7 +4466,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rand 0.7.3", - "rcgen", + "rcgen 0.10.0", "rustls 0.20.8", "solana-metrics", "solana-perf", From 90e04680afe949ea7e908a8442e71d51a62d148f Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 23 Jun 2023 09:16:57 +0200 Subject: [PATCH 004/128] add sample client --- lite-rpc/src/main.rs | 12 +++++++++++- quic-forward-proxy/src/lib.rs | 1 + quic-forward-proxy/src/proxy.rs | 3 +++ .../src/test_client/quic_test_client.rs | 2 +- services/src/tpu_utils/tpu_connection_manager.rs | 2 +- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 584482e8..cfdf7958 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -9,6 +9,7 @@ use solana_sdk::signature::Keypair; use std::env; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::SelfSignedTlsConfigProvider; +use solana_lite_rpc_quic_forward_proxy::test_client::quic_test_client::QuicTestClient; // use lite_rpc_quic_forward_proxy::tls_config::SelfSignedTlsConfigProvider; async fn get_identity_keypair(identity_from_cli: &String) -> Keypair { @@ -92,6 +93,12 @@ pub async fn main() -> anyhow::Result<()> { prometheus_addr, ); + let proxy_addr = "127.0.0.1:11111".parse().unwrap(); + let test_client = QuicTestClient::new_with_endpoint( + proxy_addr, &tls_configuration) + .await? + .start_services(); + let ctrl_c_signal = tokio::signal::ctrl_c(); tokio::select! { @@ -100,7 +107,10 @@ pub async fn main() -> anyhow::Result<()> { }, res = quicproxy_service => { bail!("Quic Proxy quit unexpectedly {res:?}"); - } + }, + res = test_client => { + bail!("Test Client quit unexpectedly {res:?}"); + }, _ = ctrl_c_signal => { info!("Received ctrl+c signal"); diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index a535627e..1b1ceb02 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -1,5 +1,6 @@ pub mod quic_util; pub mod tls_config_provicer; pub mod proxy; +pub mod test_client; pub use tls_config_provicer::SelfSignedTlsConfigProvider; // pub mod tls_config; \ No newline at end of file diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index cd8373f3..2cb44809 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -13,6 +13,7 @@ use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; use solana_lite_rpc_core::AnyhowJoinHandle; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::ActiveConnection; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; @@ -116,6 +117,8 @@ async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { info!("transaction details: {} sigs", tx.signatures.len()); + // ActiveConnection::new(e)new(tx).await; + // send_data(send).await; Ok(()) }); diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index e1fee566..d46d831c 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -38,7 +38,7 @@ impl QuicTestClient { let test_client_service: AnyhowJoinHandle = tokio::spawn(async move { info!("Sample Quic Client starting ..."); - let mut ticker = tokio::time::interval(Duration::from_secs(1)); + let mut ticker = tokio::time::interval(Duration::from_secs(3)); // TODO exit signal loop { // create new connection everytime diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 780675ab..9073f163 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -34,7 +34,7 @@ lazy_static::lazy_static! { register_int_gauge!(opts!("literpc_quic_tasks", "Number of connections to keep asked by tpu service")).unwrap(); } -struct ActiveConnection { +pub struct ActiveConnection { endpoint: Endpoint, identity: Pubkey, tpu_address: SocketAddr, From 4e25095744a016228d318d9df3a1336ae49af639 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 23 Jun 2023 10:37:33 +0200 Subject: [PATCH 005/128] define basic proxy wire format --- quic-forward-proxy/src/lib.rs | 2 + quic-forward-proxy/src/proxy.rs | 5 +- .../src/proxy_request_format.rs | 64 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 quic-forward-proxy/src/proxy_request_format.rs diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index 1b1ceb02..dccb004d 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -2,5 +2,7 @@ pub mod quic_util; pub mod tls_config_provicer; pub mod proxy; pub mod test_client; +pub mod proxy_request_format; + pub use tls_config_provicer::SelfSignedTlsConfigProvider; // pub mod tls_config; \ No newline at end of file diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 2cb44809..6b5caeba 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -1,4 +1,4 @@ -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -8,6 +8,8 @@ use quinn::{Connecting, Endpoint, SendStream, ServerConfig}; use rcgen::generate_simple_self_signed; use rustls::{Certificate, PrivateKey}; use rustls::server::ResolvesServerCert; +use serde::{Deserialize, Serialize}; +use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::Keypair; use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; @@ -137,6 +139,7 @@ async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { Ok(()) } + async fn send_data(mut send: SendStream) -> anyhow::Result<()> { send.write_all(b"HELLO STRANGER\r\n").await?; send.finish().await?; diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs new file mode 100644 index 00000000..26a7c089 --- /dev/null +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -0,0 +1,64 @@ +use std::net::SocketAddrV4; +use serde::{Deserialize, Serialize}; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::transaction::VersionedTransaction; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TpuForwardingRequestV1 { + pub tpu_socket_addr: SocketAddrV4, // TODO is that correct + pub identity_tpunode: Pubkey, + pub transactions: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum TpuForwardingRequest { + V1(TpuForwardingRequestV1), +} + +impl TpuForwardingRequest { + pub fn new(tpu_socket_addr: SocketAddrV4, identity_tpunode: Pubkey, + transactions: Vec) -> Self { + TpuForwardingRequest::V1( + TpuForwardingRequestV1 { + tpu_socket_addr, + identity_tpunode, + transactions, + }) + } +} + + +// TODO reame +fn deserialize_request(raw_tx: &Vec) -> TpuForwardingRequest { + todo!(); +} + +fn serialize_transactions_for_tpu( + tpu_socket_addr: SocketAddrV4, + tpu_identity: Pubkey, + transactions: Vec) -> Vec { + + let request = TpuForwardingRequest::new(tpu_socket_addr, tpu_identity, transactions); + + bincode::serialize(&request).expect("Expect to serialize transactions") +} + + +mod test { + use std::str::FromStr; + use log::info; + use solana_sdk::pubkey::Pubkey; + use crate::proxy_request_format::serialize_transactions_for_tpu; + + #[test] + fn deser() { + let wire_data = serialize_transactions_for_tpu( + "127.0.0.1:5454".parse().unwrap(), + Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), + vec![]); + + println!("wire_data: {:02X?}", wire_data); + + } +} + From 47fd8b15d33d7dd39bc7372e4750f34a5045edff Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Sun, 25 Jun 2023 21:53:19 +0200 Subject: [PATCH 006/128] rountrip test with instruction --- quic-forward-proxy/Cargo.toml | 3 ++ .../src/proxy_request_format.rs | 42 +++++++++++++------ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 41f7cf61..32be6728 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -37,3 +37,6 @@ async-trait = { workspace = true } chrono = { workspace = true } tokio = { version = "1.28.2", features = ["full", "fs"]} rcgen = "0.9.3" + +[dev-dependencies] +spl-memo = "3.0.1" diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 26a7c089..350facaa 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -1,4 +1,5 @@ use std::net::SocketAddrV4; +use anyhow::Context; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::VersionedTransaction; @@ -27,13 +28,7 @@ impl TpuForwardingRequest { } } - -// TODO reame -fn deserialize_request(raw_tx: &Vec) -> TpuForwardingRequest { - todo!(); -} - -fn serialize_transactions_for_tpu( +fn serialize_tpu_forwarding_request( tpu_socket_addr: SocketAddrV4, tpu_identity: Pubkey, transactions: Vec) -> Vec { @@ -43,22 +38,45 @@ fn serialize_transactions_for_tpu( bincode::serialize(&request).expect("Expect to serialize transactions") } +// TODO reame +fn deserialize_tpu_forwarding_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { + let request = bincode::deserialize::(&raw_proxy_request) + .context("deserialize proxy request") + .unwrap(); + + request +} mod test { use std::str::FromStr; use log::info; use solana_sdk::pubkey::Pubkey; - use crate::proxy_request_format::serialize_transactions_for_tpu; + use solana_sdk::transaction::Transaction; + use spl_memo::solana_program::message::VersionedMessage; + use crate::proxy_request_format::*; #[test] - fn deser() { - let wire_data = serialize_transactions_for_tpu( + fn roundtrip() { + + let payer_pubkey = Pubkey::new_unique(); + let signer_pubkey = Pubkey::new_unique(); + + let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&signer_pubkey]); + + let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); + + let wire_data = serialize_tpu_forwarding_request( "127.0.0.1:5454".parse().unwrap(), Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), - vec![]); + vec![tx.into()]); println!("wire_data: {:02X?}", wire_data); + let request = deserialize_tpu_forwarding_request(&wire_data); + + let TpuForwardingRequest::V1(req1) = request; + + assert_eq!(req1.transactions.len(), 1); + } } - From eb330957783ac3664faa12bb58bfb83db7fe2f27 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 26 Jun 2023 08:05:18 +0200 Subject: [PATCH 007/128] introduce proxy request --- core/src/lib.rs | 1 + .../src/proxy_request_format.rs | 59 +++++++------------ core/src/quic_connection_utils.rs | 6 +- lite-rpc/src/bridge.rs | 1 + quic-forward-proxy/Cargo.toml | 3 +- quic-forward-proxy/examples/sample_client.rs | 25 +++++++- quic-forward-proxy/src/lib.rs | 1 - quic-forward-proxy/src/proxy.rs | 19 +++--- .../src/test_client/quic_test_client.rs | 39 +++++++++++- .../tests/proxy_request_format.rs | 32 ++++++++++ .../src/tpu_utils/tpu_connection_manager.rs | 50 ++++++++++++---- 11 files changed, 168 insertions(+), 68 deletions(-) rename {quic-forward-proxy => core}/src/proxy_request_format.rs (51%) create mode 100644 quic-forward-proxy/tests/proxy_request_format.rs diff --git a/core/src/lib.rs b/core/src/lib.rs index 11a61f6e..0f22a098 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -9,5 +9,6 @@ pub mod structures; pub mod subscription_handler; pub mod subscription_sink; pub mod tx_store; +pub mod proxy_request_format; pub type AnyhowJoinHandle = tokio::task::JoinHandle>; diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/core/src/proxy_request_format.rs similarity index 51% rename from quic-forward-proxy/src/proxy_request_format.rs rename to core/src/proxy_request_format.rs index 350facaa..b045344e 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -1,4 +1,4 @@ -use std::net::SocketAddrV4; +use std::net::{SocketAddr, SocketAddrV4}; use anyhow::Context; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; @@ -6,7 +6,7 @@ use solana_sdk::transaction::VersionedTransaction; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequestV1 { - pub tpu_socket_addr: SocketAddrV4, // TODO is that correct + pub tpu_socket_addr: SocketAddr, // TODO is that correct pub identity_tpunode: Pubkey, pub transactions: Vec, } @@ -17,7 +17,7 @@ pub enum TpuForwardingRequest { } impl TpuForwardingRequest { - pub fn new(tpu_socket_addr: SocketAddrV4, identity_tpunode: Pubkey, + pub fn new(tpu_socket_addr: SocketAddr, identity_tpunode: Pubkey, transactions: Vec) -> Self { TpuForwardingRequest::V1( TpuForwardingRequestV1 { @@ -26,10 +26,28 @@ impl TpuForwardingRequest { transactions, }) } + + pub fn get_tpu_socket_addr(&self) -> SocketAddr { + match self { + TpuForwardingRequest::V1(request) => request.tpu_socket_addr, + } + } + + pub fn get_identity_tpunode(&self) -> Pubkey { + match self { + TpuForwardingRequest::V1(request) => request.identity_tpunode, + } + } + + pub fn get_transactions(&self) -> Vec { + match self { + TpuForwardingRequest::V1(request) => request.transactions.clone(), + } + } } fn serialize_tpu_forwarding_request( - tpu_socket_addr: SocketAddrV4, + tpu_socket_addr: SocketAddr, tpu_identity: Pubkey, transactions: Vec) -> Vec { @@ -47,36 +65,3 @@ fn deserialize_tpu_forwarding_request(raw_proxy_request: &Vec) -> TpuForward request } -mod test { - use std::str::FromStr; - use log::info; - use solana_sdk::pubkey::Pubkey; - use solana_sdk::transaction::Transaction; - use spl_memo::solana_program::message::VersionedMessage; - use crate::proxy_request_format::*; - - #[test] - fn roundtrip() { - - let payer_pubkey = Pubkey::new_unique(); - let signer_pubkey = Pubkey::new_unique(); - - let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&signer_pubkey]); - - let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); - - let wire_data = serialize_tpu_forwarding_request( - "127.0.0.1:5454".parse().unwrap(), - Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), - vec![tx.into()]); - - println!("wire_data: {:02X?}", wire_data); - - let request = deserialize_tpu_forwarding_request(&wire_data); - - let TpuForwardingRequest::V1(req1) = request; - - assert_eq!(req1.transactions.len(), 1); - - } -} diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index 385f1653..25fd7d2d 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -96,7 +96,7 @@ impl QuicConnectionUtils { identity: Pubkey, already_connected: bool, endpoint: Endpoint, - addr: SocketAddr, + tpu_address: SocketAddr, connection_timeout: Duration, connection_retry_count: usize, exit_signal: Arc, @@ -104,9 +104,9 @@ impl QuicConnectionUtils { ) -> Option { for _ in 0..connection_retry_count { let conn = if already_connected { - Self::make_connection_0rtt(endpoint.clone(), addr, connection_timeout).await + Self::make_connection_0rtt(endpoint.clone(), tpu_address, connection_timeout).await } else { - Self::make_connection(endpoint.clone(), addr, connection_timeout).await + Self::make_connection(endpoint.clone(), tpu_address, connection_timeout).await }; match conn { Ok(conn) => { diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 77809e51..36c829a2 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -253,6 +253,7 @@ impl LiteRpcServer for LiteBridge { .await { Ok(sig) => { + println!("sig: {}", sig); TXS_IN_CHANNEL.inc(); Ok(sig) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 32be6728..c1e66f0d 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -37,6 +37,5 @@ async-trait = { workspace = true } chrono = { workspace = true } tokio = { version = "1.28.2", features = ["full", "fs"]} rcgen = "0.9.3" - -[dev-dependencies] spl-memo = "3.0.1" + diff --git a/quic-forward-proxy/examples/sample_client.rs b/quic-forward-proxy/examples/sample_client.rs index 33e3752d..d580c55e 100644 --- a/quic-forward-proxy/examples/sample_client.rs +++ b/quic-forward-proxy/examples/sample_client.rs @@ -1,11 +1,15 @@ // DEPRECATED: use quic-proxy main.rs use std::net::{IpAddr, Ipv4Addr}; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; use anyhow::anyhow; use log::info; use rcgen::IsCa::SelfSignedOnly; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::transaction::Transaction; +use spl_memo::build_memo; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::time::timeout; use lite_rpc_quic_forward_proxy::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; @@ -42,7 +46,8 @@ async fn main() -> anyhow::Result<()> { endpoint.rebind(socket).expect("rebind failed"); } - let request = "FOO BAR"; + // let request = "FOO BAR"; + let request = build_memo_tx_raw(); send.write_all(request.as_bytes()) .await @@ -65,3 +70,21 @@ async fn main() -> anyhow::Result<()> { Ok(()) } + +fn build_memo_tx_raw() { + let payer_pubkey = Pubkey::new_unique(); + let signer_pubkey = Pubkey::new_unique(); + + let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&signer_pubkey]); + + let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); + + let wire_data = serialize_tpu_forwarding_request( + "127.0.0.1:5454".parse().unwrap(), + Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), + vec![tx.into()]); + + println!("wire_data: {:02X?}", wire_data); + + wire_data +} diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index dccb004d..69582105 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -2,7 +2,6 @@ pub mod quic_util; pub mod tls_config_provicer; pub mod proxy; pub mod test_client; -pub mod proxy_request_format; pub use tls_config_provicer::SelfSignedTlsConfigProvider; // pub mod tls_config; \ No newline at end of file diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 6b5caeba..82c1c8ec 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -15,6 +15,7 @@ use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; use solana_lite_rpc_core::AnyhowJoinHandle; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::ActiveConnection; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; @@ -105,24 +106,26 @@ async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { Ok(s) => s, }; tokio::spawn(async move { - let raw_tx = recv.read_to_end(100000).await + let raw_request = recv.read_to_end(100000).await .unwrap(); // let str = std::str::from_utf8(&result).unwrap(); - info!("read raw_tx {:02X?}", raw_tx); + info!("read proxy_request {:02X?}", raw_request); - let tx = match bincode::deserialize::(&raw_tx) { - Ok(tx) => tx, + let proxy_request = match bincode::deserialize::(&raw_request) { + Ok(raw_request) => raw_request, Err(err) => { - bail!(err.to_string()); + warn!("failed to deserialize proxy request: {:?}", err); + // bail!(err.to_string()); + return; } }; - info!("transaction details: {} sigs", tx.signatures.len()); + info!("transaction details: {} sigs", proxy_request.get_transactions().len()); // ActiveConnection::new(e)new(tx).await; // send_data(send).await; - Ok(()) + // Ok(()) }); // info!("stream okey {:?}", stream); // let fut = handle_request2(stream).await; @@ -133,7 +136,7 @@ async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { // } // } // ); - } + } // -- loop } .await?; Ok(()) diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index d46d831c..c0f33023 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -1,4 +1,5 @@ -use std::net::SocketAddr; +use std::net::{SocketAddr, SocketAddrV4}; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; use anyhow::bail; @@ -6,11 +7,14 @@ use bytes::BufMut; use log::info; use quinn::{Endpoint, VarInt}; use rustls::ClientConfig; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::transaction::{Transaction, VersionedTransaction}; use tokio::io::AsyncWriteExt; use solana_lite_rpc_core::AnyhowJoinHandle; use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; use crate::tls_config_provicer::ProxyTlsConfigProvider; use solana_lite_rpc_core::quic_connection_utils::SkipServerVerification; +use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; use crate::test_client::sample_data_factory::build_raw_sample_tx; pub struct QuicTestClient { @@ -49,9 +53,9 @@ impl QuicTestClient { for si in 0..5 { let (mut send, mut recv) = connection.open_bi().await?; - let raw = build_raw_sample_tx(); + let raw = build_memo_tx_raw(); info!("raw: {:02X?}", raw); - send.write_all(format!("SAMPLE DATA on stream {}", si).as_bytes()).await?; + // send.write_all(format!("SAMPLE DATA on stream {}", si).as_bytes()).await?; // shutdown stream send.finish().await?; @@ -91,3 +95,32 @@ fn build_tls_config() -> ClientConfig { return client_crypto; } + +fn build_memo_tx_raw() -> Vec { + let payer_pubkey = Pubkey::new_unique(); + let signer_pubkey = Pubkey::new_unique(); + + let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&signer_pubkey]); + + let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); + + let wire_data = serialize_tpu_forwarding_request( + "127.0.0.1:5454".parse().unwrap(), + Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), + vec![tx.into()]); + + println!("wire_data: {:02X?}", wire_data); + + wire_data +} + + +fn serialize_tpu_forwarding_request( + tpu_socket_addr: SocketAddr, + tpu_identity: Pubkey, + transactions: Vec) -> Vec { + + let request = TpuForwardingRequest::new(tpu_socket_addr, tpu_identity, transactions); + + bincode::serialize(&request).expect("Expect to serialize transactions") +} diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs new file mode 100644 index 00000000..71704b29 --- /dev/null +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -0,0 +1,32 @@ + +use std::str::FromStr; +use log::info; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::transaction::Transaction; +use spl_memo::solana_program::message::VersionedMessage; +use crate::proxy_request_format::*; + +#[test] +fn roundtrip() { + + let payer_pubkey = Pubkey::new_unique(); + let signer_pubkey = Pubkey::new_unique(); + + let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&signer_pubkey]); + + let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); + + let wire_data = serialize_tpu_forwarding_request( + "127.0.0.1:5454".parse().unwrap(), + Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), + vec![tx.into()]); + + println!("wire_data: {:02X?}", wire_data); + + let request = deserialize_tpu_forwarding_request(&wire_data); + + let TpuForwardingRequest::V1(req1) = request; + + assert_eq!(req1.transactions.len(), 1); + +} diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 9073f163..56d82a7c 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -17,8 +17,11 @@ use std::{ }, time::Duration, }; +use anyhow::bail; +use solana_sdk::transaction::VersionedTransaction; use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; use tokio::time::timeout; +use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; pub const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(1); pub const CONNECTION_RETRY_COUNT: usize = 10; @@ -74,7 +77,7 @@ impl ActiveConnection { transaction_reciever: Receiver<(String, Vec)>, exit_oneshot_channel: tokio::sync::mpsc::Receiver<()>, endpoint: Endpoint, - addr: SocketAddr, + tpu_address: SocketAddr, exit_signal: Arc, identity: Pubkey, identity_stakes: IdentityStakes, @@ -146,7 +149,7 @@ impl ActiveConnection { identity, false, endpoint.clone(), - addr, + tpu_address, QUIC_CONNECTION_TIMEOUT, CONNECTION_RETRY_COUNT, exit_signal.clone(), @@ -177,7 +180,9 @@ impl ActiveConnection { Self::send_copy_of_txs_to_quicproxy( &txs, endpoint.clone(), // proxy address - "127.0.0.1:11111".parse().unwrap()).await.unwrap(); + "127.0.0.1:11111".parse().unwrap(), + tpu_address, + identity.clone()).await.unwrap(); QuicConnectionUtils::send_transaction_batch( @@ -185,7 +190,7 @@ impl ActiveConnection { txs, identity, endpoint, - addr, + tpu_address, exit_signal, last_stable_id, QUIC_CONNECTION_TIMEOUT, @@ -209,9 +214,28 @@ impl ActiveConnection { NB_QUIC_ACTIVE_CONNECTIONS.dec(); } - async fn send_copy_of_txs_to_quicproxy(txs: &Vec>, endpoint: Endpoint, proxy_address: SocketAddr) -> anyhow::Result<()> { - let txs_copy = txs.clone(); - let send_result = timeout(Duration::from_millis(500), Self::send_tx(endpoint, proxy_address, &txs_copy)); + async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, endpoint: Endpoint, + proxy_address: SocketAddr, tpu_target_address: SocketAddr, + identity: Pubkey) -> anyhow::Result<()> { + let raw_tx_batch_copy = raw_tx_batch.clone(); + + let mut txs = vec![]; + + for raw_tx in raw_tx_batch_copy { + let tx = match bincode::deserialize::(&raw_tx) { + Ok(tx) => tx, + Err(err) => { + bail!(err.to_string()); + } + }; + txs.push(tx); + } + + let forwarding_request = TpuForwardingRequest::new(tpu_target_address, identity, txs); + + let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); + + let send_result = timeout(Duration::from_millis(500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)); match send_result.await { Ok(..) => { @@ -224,13 +248,13 @@ impl ActiveConnection { Ok(()) } - async fn send_tx(endpoint: Endpoint, proxy_address: SocketAddr, mut txs: &Vec>) -> anyhow::Result<()> { + async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { let mut connecting = endpoint.connect(proxy_address, "localhost")?; let connection = timeout(Duration::from_millis(500), connecting).await??; let (mut send, mut recv) = connection.open_bi().await?; - for tx in txs { - send.write_all(tx).await?; - } + + send.write_all(proxy_request_raw).await?; + send.finish().await?; Ok(()) @@ -243,7 +267,7 @@ impl ActiveConnection { identity_stakes: IdentityStakes, ) { let endpoint = self.endpoint.clone(); - let addr = self.tpu_address; + let tpu_address = self.tpu_address; let exit_signal = self.exit_signal.clone(); let identity = self.identity; let txs_sent_store = self.txs_sent_store.clone(); @@ -252,7 +276,7 @@ impl ActiveConnection { transaction_reciever, exit_oneshot_channel, endpoint, - addr, + tpu_address, exit_signal, identity, identity_stakes, From f7ad7da021a5fb4a6868a22c0592f472391961ed Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 26 Jun 2023 08:08:45 +0200 Subject: [PATCH 008/128] use quic uni instead of bi stream --- quic-forward-proxy/examples/sample_client.rs | 90 ------------------- quic-forward-proxy/src/proxy.rs | 4 +- .../src/test_client/quic_test_client.rs | 2 +- .../src/tpu_utils/tpu_connection_manager.rs | 2 +- 4 files changed, 4 insertions(+), 94 deletions(-) delete mode 100644 quic-forward-proxy/examples/sample_client.rs diff --git a/quic-forward-proxy/examples/sample_client.rs b/quic-forward-proxy/examples/sample_client.rs deleted file mode 100644 index d580c55e..00000000 --- a/quic-forward-proxy/examples/sample_client.rs +++ /dev/null @@ -1,90 +0,0 @@ -// DEPRECATED: use quic-proxy main.rs - -use std::net::{IpAddr, Ipv4Addr}; -use std::str::FromStr; -use std::sync::Arc; -use std::time::Duration; -use anyhow::anyhow; -use log::info; -use rcgen::IsCa::SelfSignedOnly; -use solana_sdk::pubkey::Pubkey; -use solana_sdk::transaction::Transaction; -use spl_memo::build_memo; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::time::timeout; -use lite_rpc_quic_forward_proxy::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; -use solana_lite_rpc_core::quic_connection_utils::SkipServerVerification; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - - // FIXME configured insecure https://quinn-rs.github.io/quinn/quinn/certificate.html - let mut _roots = rustls::RootCertStore::empty(); - // TODO add certs - - let mut client_crypto = rustls::ClientConfig::builder() - .with_safe_defaults() - // .with_root_certificates(roots) - .with_custom_certificate_verifier(SkipServerVerification::new()) - .with_no_client_auth(); - client_crypto.enable_early_data = true; - client_crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; - - let mut endpoint = quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())?; - endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(client_crypto))); - - let connection_timeout = Duration::from_secs(5); - let connecting = endpoint.connect("127.0.0.1:8080".parse().unwrap(), "localhost").unwrap(); - let connection = timeout(connection_timeout, connecting).await??; - - let (mut send, mut recv) = connection.open_bi().await?; - - if false { // Rebind - let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); - let addr = socket.local_addr().unwrap(); - info!("rebinding to {}", addr); - endpoint.rebind(socket).expect("rebind failed"); - } - - // let request = "FOO BAR"; - let request = build_memo_tx_raw(); - - send.write_all(request.as_bytes()) - .await - .map_err(|e| anyhow!("failed to send request: {}", e))?; - send.finish() - .await - .map_err(|e| anyhow!("failed to shutdown stream: {}", e))?; - let resp = recv - .read_to_end(usize::MAX) - .await - .map_err(|e| anyhow!("failed to read response: {}", e))?; - - info!("resp: {:?}", std::str::from_utf8(&resp)); - - connection.close(99u32.into(), b"done"); - - // Give the server a fair chance to receive the close packet - endpoint.wait_idle().await; - - - Ok(()) -} - -fn build_memo_tx_raw() { - let payer_pubkey = Pubkey::new_unique(); - let signer_pubkey = Pubkey::new_unique(); - - let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&signer_pubkey]); - - let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); - - let wire_data = serialize_tpu_forwarding_request( - "127.0.0.1:5454".parse().unwrap(), - Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), - vec![tx.into()]); - - println!("wire_data: {:02X?}", wire_data); - - wire_data -} diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 82c1c8ec..23efea91 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -93,8 +93,8 @@ async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { info!("established"); async { loop { - let stream = connection.accept_bi().await; - let (mut send, recv) = match stream { + let stream = connection.accept_uni().await; + let mut recv = match stream { Err(quinn::ConnectionError::ApplicationClosed { .. }) => { info!("connection closed"); return Ok(()); diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index c0f33023..87dc8b60 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -51,7 +51,7 @@ impl QuicTestClient { let connection = tokio::time::timeout(connection_timeout, connecting).await??; for si in 0..5 { - let (mut send, mut recv) = connection.open_bi().await?; + let mut send = connection.open_uni().await?; let raw = build_memo_tx_raw(); info!("raw: {:02X?}", raw); diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 56d82a7c..298698bf 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -251,7 +251,7 @@ impl ActiveConnection { async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { let mut connecting = endpoint.connect(proxy_address, "localhost")?; let connection = timeout(Duration::from_millis(500), connecting).await??; - let (mut send, mut recv) = connection.open_bi().await?; + let mut send = connection.open_uni().await?; send.write_all(proxy_request_raw).await?; From 1146589b09d5ff4c36c73616dc557f4a6e16fea2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 26 Jun 2023 08:58:48 +0200 Subject: [PATCH 009/128] proxy works --- core/src/proxy_request_format.rs | 42 +++++++++---------- quic-forward-proxy/src/proxy.rs | 4 +- .../src/test_client/quic_test_client.rs | 5 ++- .../tests/proxy_request_format.rs | 32 +++++++++++--- .../src/tpu_utils/tpu_connection_manager.rs | 3 ++ 5 files changed, 54 insertions(+), 32 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index b045344e..0b0f05e1 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -1,19 +1,29 @@ +use std::fmt; +use std::fmt::Display; use std::net::{SocketAddr, SocketAddrV4}; use anyhow::Context; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::VersionedTransaction; +// TODO define a proper discriminator #[derive(Serialize, Deserialize, Debug, Clone)] -pub struct TpuForwardingRequestV1 { - pub tpu_socket_addr: SocketAddr, // TODO is that correct - pub identity_tpunode: Pubkey, - pub transactions: Vec, +pub enum TpuForwardingRequest { + V1(TpuForwardingRequestV1), } #[derive(Serialize, Deserialize, Debug, Clone)] -pub enum TpuForwardingRequest { - V1(TpuForwardingRequestV1), +pub struct TpuForwardingRequestV1 { + tpu_socket_addr: SocketAddr, // TODO is that correct + identity_tpunode: Pubkey, + transactions: Vec, +} + +impl Display for TpuForwardingRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "TpuForwardingRequest for tpu target {} with indentity {}", + &self.get_tpu_socket_addr(), &self.get_identity_tpunode()) + } } impl TpuForwardingRequest { @@ -30,38 +40,24 @@ impl TpuForwardingRequest { pub fn get_tpu_socket_addr(&self) -> SocketAddr { match self { TpuForwardingRequest::V1(request) => request.tpu_socket_addr, + _ => panic!("format version error"), } } pub fn get_identity_tpunode(&self) -> Pubkey { match self { TpuForwardingRequest::V1(request) => request.identity_tpunode, + _ => panic!("format version error"), } } pub fn get_transactions(&self) -> Vec { match self { TpuForwardingRequest::V1(request) => request.transactions.clone(), + _ => panic!("format version error"), } } } -fn serialize_tpu_forwarding_request( - tpu_socket_addr: SocketAddr, - tpu_identity: Pubkey, - transactions: Vec) -> Vec { - let request = TpuForwardingRequest::new(tpu_socket_addr, tpu_identity, transactions); - - bincode::serialize(&request).expect("Expect to serialize transactions") -} - -// TODO reame -fn deserialize_tpu_forwarding_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { - let request = bincode::deserialize::(&raw_proxy_request) - .context("deserialize proxy request") - .unwrap(); - - request -} diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 23efea91..43c8b52d 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -109,7 +109,7 @@ async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { let raw_request = recv.read_to_end(100000).await .unwrap(); // let str = std::str::from_utf8(&result).unwrap(); - info!("read proxy_request {:02X?}", raw_request); + info!("read proxy_request {} bytes", raw_request.len()); let proxy_request = match bincode::deserialize::(&raw_request) { Ok(raw_request) => raw_request, @@ -120,7 +120,7 @@ async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { } }; - info!("transaction details: {} sigs", proxy_request.get_transactions().len()); + info!("proxy request details: {}", proxy_request); // ActiveConnection::new(e)new(tx).await; diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index 87dc8b60..530034e1 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::bail; use bytes::BufMut; -use log::info; +use log::{info, trace}; use quinn::{Endpoint, VarInt}; use rustls::ClientConfig; use solana_sdk::pubkey::Pubkey; @@ -54,8 +54,9 @@ impl QuicTestClient { let mut send = connection.open_uni().await?; let raw = build_memo_tx_raw(); - info!("raw: {:02X?}", raw); + trace!("raw: {:02X?}", raw); // send.write_all(format!("SAMPLE DATA on stream {}", si).as_bytes()).await?; + send.write_all(&raw).await?; // shutdown stream send.finish().await?; diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index 71704b29..a9673483 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -1,10 +1,13 @@ - +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; use std::str::FromStr; +use anyhow::Context; +use bincode::DefaultOptions; use log::info; use solana_sdk::pubkey::Pubkey; -use solana_sdk::transaction::Transaction; +use solana_sdk::transaction::{Transaction, VersionedTransaction}; use spl_memo::solana_program::message::VersionedMessage; -use crate::proxy_request_format::*; +use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; +use solana_lite_rpc_core::proxy_request_format::*; #[test] fn roundtrip() { @@ -25,8 +28,27 @@ fn roundtrip() { let request = deserialize_tpu_forwarding_request(&wire_data); - let TpuForwardingRequest::V1(req1) = request; + assert_eq!(request.get_tpu_socket_addr().is_ipv4(), true); + assert_eq!(request.get_transactions().len(), 1); + +} + +fn serialize_tpu_forwarding_request( + tpu_socket_addr: SocketAddr, + tpu_identity: Pubkey, + transactions: Vec) -> Vec { - assert_eq!(req1.transactions.len(), 1); + let request = TpuForwardingRequest::new(tpu_socket_addr, tpu_identity, transactions); + bincode::serialize(&request).expect("Expect to serialize transactions") } + +// TODO reame +fn deserialize_tpu_forwarding_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { + let request = bincode::deserialize::(&raw_proxy_request) + .context("deserialize proxy request") + .unwrap(); + + request +} + diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 298698bf..04b0d64f 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -174,6 +174,7 @@ impl ActiveConnection { NB_QUIC_TASKS.inc(); let connection = connection.unwrap(); + // TODO split to new service // SOS info!("Sending copy of transaction batch of {} to tpu with identity {} to quic proxy", txs.len(), identity); @@ -249,6 +250,8 @@ impl ActiveConnection { } async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { + info!("sending {} bytes to proxy", proxy_request_raw.len()); + let mut connecting = endpoint.connect(proxy_address, "localhost")?; let connection = timeout(Duration::from_millis(500), connecting).await??; let mut send = connection.open_uni().await?; From 77c801ff4572b84e015b43939033941832f8b70b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 26 Jun 2023 22:10:06 +0200 Subject: [PATCH 010/128] proxy forwarding works --- Cargo.toml | 1 + core/src/proxy_request_format.rs | 58 ++++--- core/src/quic_connection_utils.rs | 15 +- lite-rpc/src/bridge.rs | 5 +- lite-rpc/src/main.rs | 26 +-- quic-forward-proxy/Cargo.toml | 1 + quic-forward-proxy/src/cli.rs | 34 ++++ quic-forward-proxy/src/lib.rs | 2 + quic-forward-proxy/src/main.rs | 15 +- quic-forward-proxy/src/proxy.rs | 163 ++++++++++++++---- .../src/test_client/quic_test_client.rs | 5 +- .../tests/proxy_request_format.rs | 26 +-- services/Cargo.toml | 1 + .../src/tpu_utils/tpu_connection_manager.rs | 61 ++++--- services/src/tpu_utils/tpu_service.rs | 6 +- 15 files changed, 285 insertions(+), 134 deletions(-) create mode 100644 quic-forward-proxy/src/cli.rs diff --git a/Cargo.toml b/Cargo.toml index 5a86da33..105cef4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ solana-client = "1.15.2" solana-net-utils = "1.15.2" solana-pubsub-client = "1.15.2" solana-streamer = "1.14.2" +itertools = "0.10.5" serde = { version = "1.0.160", features = ["derive"] } serde_json = "1.0.96" bincode = "1.3.3" diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index 0b0f05e1..4675870d 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -6,14 +6,15 @@ use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::VersionedTransaction; -// TODO define a proper discriminator -#[derive(Serialize, Deserialize, Debug, Clone)] -pub enum TpuForwardingRequest { - V1(TpuForwardingRequestV1), -} +/// +/// lite-rpc to proxy wire format +/// compat info: non-public format ATM +/// initial version +const FORMAT_VERSION1: u16 = 2301; #[derive(Serialize, Deserialize, Debug, Clone)] -pub struct TpuForwardingRequestV1 { +pub struct TpuForwardingRequest { + format_version: u16, tpu_socket_addr: SocketAddr, // TODO is that correct identity_tpunode: Pubkey, transactions: Vec, @@ -21,7 +22,7 @@ pub struct TpuForwardingRequestV1 { impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "TpuForwardingRequest for tpu target {} with indentity {}", + write!(f, "TpuForwardingRequest for tpu target {} with identity {}", &self.get_tpu_socket_addr(), &self.get_identity_tpunode()) } } @@ -29,33 +30,40 @@ impl Display for TpuForwardingRequest { impl TpuForwardingRequest { pub fn new(tpu_socket_addr: SocketAddr, identity_tpunode: Pubkey, transactions: Vec) -> Self { - TpuForwardingRequest::V1( - TpuForwardingRequestV1 { - tpu_socket_addr, - identity_tpunode, - transactions, - }) + TpuForwardingRequest { + format_version: FORMAT_VERSION1, + tpu_socket_addr, + identity_tpunode, + transactions, + } + } + + pub fn serialize_wire_format( + &self) -> Vec { + bincode::serialize(&self).expect("Expect to serialize transactions") + } + + // TODO reame + pub fn deserialize_from_raw_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { + let request = bincode::deserialize::(&raw_proxy_request) + .context("deserialize proxy request") + .unwrap(); + + assert_eq!(request.format_version, 2301); + + request } pub fn get_tpu_socket_addr(&self) -> SocketAddr { - match self { - TpuForwardingRequest::V1(request) => request.tpu_socket_addr, - _ => panic!("format version error"), - } + self.tpu_socket_addr } pub fn get_identity_tpunode(&self) -> Pubkey { - match self { - TpuForwardingRequest::V1(request) => request.identity_tpunode, - _ => panic!("format version error"), - } + self.identity_tpunode } pub fn get_transactions(&self) -> Vec { - match self { - TpuForwardingRequest::V1(request) => request.transactions.clone(), - _ => panic!("format version error"), - } + self.transactions.clone() } } diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index 25fd7d2d..f5ad5b4d 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -1,4 +1,4 @@ -use log::{trace, warn}; +use log::{info, trace, warn}; use quinn::{ ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, @@ -13,6 +13,7 @@ use std::{ }, time::Duration, }; +use anyhow::bail; use tokio::{sync::RwLock, time::timeout}; const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; @@ -38,7 +39,7 @@ impl QuicConnectionUtils { .expect("Failed to set QUIC client certificates"); crypto.enable_early_data = true; - warn!("TEMP HACK TO ALLOW PROXY PROTOCOL"); + // FIXME TEMP HACK TO ALLOW PROXY PROTOCOL const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec(), ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; @@ -46,7 +47,8 @@ impl QuicConnectionUtils { let mut config = ClientConfig::new(Arc::new(crypto)); let mut transport_config = TransportConfig::default(); - let timeout = IdleTimeout::try_from(Duration::from_secs(1)).unwrap(); + // TODO check timing + let timeout = IdleTimeout::try_from(Duration::from_secs(5)).unwrap(); transport_config.max_idle_timeout(Some(timeout)); transport_config.keep_alive_interval(Some(Duration::from_millis(500))); config.transport_config(Arc::new(transport_config)); @@ -114,7 +116,7 @@ impl QuicConnectionUtils { return Some(conn); } Err(e) => { - trace!("Could not connect to {} because of error {}", identity, e); + warn!("Could not connect to tpu {}/{}, error: {}", tpu_address, identity, e); if exit_signal.load(Ordering::Relaxed) { break; } @@ -195,13 +197,14 @@ impl QuicConnectionUtils { txs: Vec>, identity: Pubkey, endpoint: Endpoint, - socket_addr: SocketAddr, + tpu_address: SocketAddr, exit_signal: Arc, last_stable_id: Arc, connection_timeout: Duration, connection_retry_count: usize, on_connect: fn(), ) { + info!("send transaction batch of size {} to address {}", txs.len(), tpu_address); let mut queue = VecDeque::new(); for tx in txs { queue.push_back(tx); @@ -228,7 +231,7 @@ impl QuicConnectionUtils { identity, true, endpoint.clone(), - socket_addr, + tpu_address, connection_timeout, connection_retry_count, exit_signal.clone(), diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 36c829a2..10b23e44 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -77,7 +77,7 @@ impl LiteBridge { rpc_url: String, ws_addr: String, fanout_slots: u64, - identity: Keypair, + validator_identity: Arc, retry_after: Duration, max_retries: usize, ) -> anyhow::Result { @@ -89,7 +89,7 @@ impl LiteBridge { let tpu_service = TpuService::new( current_slot, fanout_slots, - Arc::new(identity), + validator_identity, rpc_client.clone(), ws_addr, tx_store.clone(), @@ -253,7 +253,6 @@ impl LiteRpcServer for LiteBridge { .await { Ok(sig) => { - println!("sig: {}", sig); TXS_IN_CHANNEL.inc(); Ok(sig) diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index cfdf7958..0fa09c96 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -7,6 +7,8 @@ use lite_rpc::{bridge::LiteBridge, cli::Args}; use log::info; use solana_sdk::signature::Keypair; use std::env; +use std::sync::Arc; +use tokio::time::timeout; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::SelfSignedTlsConfigProvider; use solana_lite_rpc_quic_forward_proxy::test_client::quic_test_client::QuicTestClient; @@ -55,7 +57,7 @@ pub async fn main() -> anyhow::Result<()> { dotenv().ok(); - let identity = get_identity_keypair(&identity_keypair).await; + let validator_identity = Arc::new(get_identity_keypair(&identity_keypair).await); let clean_interval_ms = Duration::from_millis(clean_interval_ms); @@ -68,19 +70,17 @@ pub async fn main() -> anyhow::Result<()> { let retry_after = Duration::from_secs(transaction_retry_after_secs); - let proxy_listener_addr = "127.0.0.1:11111".parse().unwrap(); let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); - let quicproxy_service = QuicForwardProxy::new(proxy_listener_addr, &tls_configuration) + let quicproxy_service = QuicForwardProxy::new(proxy_listener_addr, &tls_configuration, validator_identity.clone()) .await? .start_services(); - let services = LiteBridge::new( rpc_addr, ws_addr, fanout_size, - identity, + validator_identity.clone(), retry_after, maximum_retries_per_tx, ) @@ -93,11 +93,11 @@ pub async fn main() -> anyhow::Result<()> { prometheus_addr, ); - let proxy_addr = "127.0.0.1:11111".parse().unwrap(); - let test_client = QuicTestClient::new_with_endpoint( - proxy_addr, &tls_configuration) - .await? - .start_services(); + // let proxy_addr = "127.0.0.1:11111".parse().unwrap(); + // let test_client = QuicTestClient::new_with_endpoint( + // proxy_addr, &tls_configuration) + // .await? + // .start_services(); let ctrl_c_signal = tokio::signal::ctrl_c(); @@ -108,9 +108,9 @@ pub async fn main() -> anyhow::Result<()> { res = quicproxy_service => { bail!("Quic Proxy quit unexpectedly {res:?}"); }, - res = test_client => { - bail!("Test Client quit unexpectedly {res:?}"); - }, + // res = test_client => { + // bail!("Test Client quit unexpectedly {res:?}"); + // }, _ = ctrl_c_signal => { info!("Received ctrl+c signal"); diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index c1e66f0d..835da929 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -24,6 +24,7 @@ anyhow = { workspace = true } log = { workspace = true } clap = { workspace = true } dashmap = { workspace = true } +itertools = { workspace = true } tracing-subscriber = { workspace = true } native-tls = { workspace = true } prometheus = { workspace = true } diff --git a/quic-forward-proxy/src/cli.rs b/quic-forward-proxy/src/cli.rs new file mode 100644 index 00000000..7042f291 --- /dev/null +++ b/quic-forward-proxy/src/cli.rs @@ -0,0 +1,34 @@ +use std::env; +use clap::Parser; +use solana_sdk::signature::Keypair; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +pub struct Args { + #[arg(short = 'k', long, default_value_t = String::new())] + pub identity_keypair: String, +} + +// note this is duplicated from lite-rpc module +pub async fn get_identity_keypair(identity_from_cli: &String) -> Keypair { + if let Ok(identity_env_var) = env::var("IDENTITY") { + if let Ok(identity_bytes) = serde_json::from_str::>(identity_env_var.as_str()) { + Keypair::from_bytes(identity_bytes.as_slice()).unwrap() + } else { + // must be a file + let identity_file = tokio::fs::read_to_string(identity_env_var.as_str()) + .await + .expect("Cannot find the identity file provided"); + let identity_bytes: Vec = serde_json::from_str(&identity_file).unwrap(); + Keypair::from_bytes(identity_bytes.as_slice()).unwrap() + } + } else if identity_from_cli.is_empty() { + Keypair::new() + } else { + let identity_file = tokio::fs::read_to_string(identity_from_cli.as_str()) + .await + .expect("Cannot find the identity file provided"); + let identity_bytes: Vec = serde_json::from_str(&identity_file).unwrap(); + Keypair::from_bytes(identity_bytes.as_slice()).unwrap() + } +} diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index 69582105..c1cce8b9 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -2,6 +2,8 @@ pub mod quic_util; pub mod tls_config_provicer; pub mod proxy; pub mod test_client; +pub mod cli; + pub use tls_config_provicer::SelfSignedTlsConfigProvider; // pub mod tls_config; \ No newline at end of file diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 2f9c125d..ea9bd2ba 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -1,6 +1,10 @@ use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; use anyhow::bail; +use clap::Parser; +use dotenv::dotenv; use log::info; +use crate::cli::{Args, get_identity_keypair}; use crate::proxy::QuicForwardProxy; use crate::test_client::quic_test_client::QuicTestClient; use crate::tls_config_provicer::SelfSignedTlsConfigProvider; @@ -9,17 +13,26 @@ mod proxy; mod test_client; mod quic_util; mod tls_config_provicer; +mod cli; #[tokio::main(flavor = "multi_thread", worker_threads = 16)] pub async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt::init(); + let Args { + identity_keypair, + } = Args::parse(); + + dotenv().ok(); + + // TODO build args struct dedicyted to proxy let proxy_listener_addr = "127.0.0.1:11111".parse().unwrap(); let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); + let validator_identity = Arc::new(get_identity_keypair(&identity_keypair).await); - let main_services = QuicForwardProxy::new(proxy_listener_addr, &tls_configuration) + let main_services = QuicForwardProxy::new(proxy_listener_addr, &tls_configuration, validator_identity) .await? .start_services(); diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 43c8b52d..12669e28 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -1,49 +1,57 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::Path; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64}; use std::time::Duration; use anyhow::{anyhow, bail}; +use itertools::Itertools; use log::{error, info, warn}; -use quinn::{Connecting, Endpoint, SendStream, ServerConfig}; +use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig}; use rcgen::generate_simple_self_signed; use rustls::{Certificate, PrivateKey}; use rustls::server::ResolvesServerCert; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::Keypair; +use solana_sdk::signer::Signer; use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; use solana_lite_rpc_core::AnyhowJoinHandle; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use tokio::sync::RwLock; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::ActiveConnection; +use solana_lite_rpc_core::quic_connection_utils::QuicConnectionUtils; +use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::{ActiveConnection, CONNECTION_RETRY_COUNT, QUIC_CONNECTION_TIMEOUT}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; pub struct QuicForwardProxy { endpoint: Endpoint, + validator_identity: Arc, } impl QuicForwardProxy { pub async fn new( proxy_listener_addr: SocketAddr, - tls_config: &SelfSignedTlsConfigProvider) -> anyhow::Result { + tls_config: &SelfSignedTlsConfigProvider, + validator_identity: Arc) -> anyhow::Result { let server_tls_config = tls_config.get_server_tls_crypto_config(); let mut quinn_server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); let endpoint = Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap(); - info!("listening on {}", endpoint.local_addr()?); + info!("tpu forward proxy listening on {}", endpoint.local_addr()?); + info!("staking from validator identity {}", validator_identity.pubkey()); - - - Ok(Self {endpoint}) + Ok(Self {endpoint, validator_identity }) } pub async fn start_services( mut self, ) -> anyhow::Result<()> { + let exit_signal = Arc::new(AtomicBool::new(false)); + let endpoint = self.endpoint.clone(); let quic_proxy: AnyhowJoinHandle = tokio::spawn(async move { info!("TPU Quic Proxy server start on {}", endpoint.local_addr()?); @@ -52,7 +60,7 @@ impl QuicForwardProxy { while let Some(conn) = endpoint.accept().await { info!("connection incoming"); - let fut = handle_connection2(conn); + let fut = handle_connection2(conn, exit_signal.clone(), self.validator_identity.clone()); tokio::spawn(async move { if let Err(e) = fut.await { error!("connection failed: {reason}", reason = e.to_string()) @@ -86,11 +94,9 @@ impl QuicForwardProxy { // meins -async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { +async fn handle_connection2(connecting: Connecting, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { let connection = connecting.await?; - info!("connection established, remote {connection}", connection = connection.remote_address()); - - info!("established"); + info!("inbound connection established, remote {connection}", connection = connection.remote_address()); async { loop { let stream = connection.accept_uni().await; @@ -105,26 +111,23 @@ async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { } Ok(s) => s, }; + let exit_signal_copy = exit_signal.clone(); + let validator_identity_copy = validator_identity.clone(); tokio::spawn(async move { - let raw_request = recv.read_to_end(100000).await + let raw_request = recv.read_to_end(10_000_000).await .unwrap(); // let str = std::str::from_utf8(&result).unwrap(); info!("read proxy_request {} bytes", raw_request.len()); - let proxy_request = match bincode::deserialize::(&raw_request) { - Ok(raw_request) => raw_request, - Err(err) => { - warn!("failed to deserialize proxy request: {:?}", err); - // bail!(err.to_string()); - return; - } - }; + let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); info!("proxy request details: {}", proxy_request); + let tpu_identity = proxy_request.get_identity_tpunode(); + let tpu_addr = proxy_request.get_tpu_socket_addr(); + let txs = proxy_request.get_transactions(); - // ActiveConnection::new(e)new(tx).await; + send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_addr, &txs).await; - // send_data(send).await; // Ok(()) }); // info!("stream okey {:?}", stream); @@ -142,20 +145,112 @@ async fn handle_connection2(connecting: Connecting) -> anyhow::Result<()> { Ok(()) } +mod test { + use std::str::FromStr; + use std::sync::Arc; + use std::sync::atomic::AtomicBool; + use solana_sdk::pubkey::Pubkey; + use crate::cli::get_identity_keypair; + use crate::proxy::send_txs_to_tpu; -async fn send_data(mut send: SendStream) -> anyhow::Result<()> { - send.write_all(b"HELLO STRANGER\r\n").await?; - send.finish().await?; - Ok(()) + #[test] + fn call() { + let exit_signal = Arc::new(AtomicBool::new(false)); + + let validator_identity = get_identity_keypair(&"/Users/stefan/mango/projects/quic-forward-proxy/local-testvalidator-stake-account.json".to_string()); + let tpu_identity = Pubkey::from_str("asdfsdf").unwrap(); + let tpu_address = "127.0.0.1:1027".parse().unwrap(); + send_txs_to_tpu(exit_signal, validator_identity, tpu_identity, tpu_address, &vec![]) + + } } -async fn handle_request2( - (mut send, recv): (quinn::SendStream, quinn::RecvStream), -) -> anyhow::Result<()> { - info!("handle incoming request..."); +async fn send_txs_to_tpu(exit_signal: Arc, validator_identity: Arc, tpu_identity: Pubkey, tpu_addr: SocketAddr, txs: &Vec) { + let (certificate, key) = new_self_signed_tls_certificate( + validator_identity.as_ref(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC client certificates"); - send.write_all(b"HELLO STRANGER\r\n").await?; - send.finish().await?; + let endpoint = QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()); + let last_stable_id: Arc = Arc::new(AtomicU64::new(0)); - Ok(()) + let connection = + Arc::new(RwLock::new( + QuicConnectionUtils::connect( + tpu_identity, + false, + endpoint.clone(), + tpu_addr, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + exit_signal.clone(), + || { + // do nothing + }, + ).await.unwrap())); + + + let txs_raw = serialize_to_vecvec(&txs); + + info!("received vecvec: {}", txs_raw.iter().map(|tx| tx.len().to_string()).into_iter().join(",")); + + + QuicConnectionUtils::send_transaction_batch( + connection.clone(), + txs_raw, + tpu_identity, + endpoint, + tpu_addr, + exit_signal.clone(), + last_stable_id, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + || { + // do nothing + } + ).await; + + { + let conn = connection.clone(); + conn.write().await.close(0u32.into(), b"done"); + } } + +fn serialize_to_vecvec(transactions: &Vec) -> Vec> { + transactions.iter().map(|tx| { + let tx_raw = bincode::serialize(tx).unwrap(); + tx_raw + }).collect_vec() +} + +// async fn send_transactions_with_retry( +// conn: Connection, +// identity: Pubkey, txs: &Vec) { +// let mut retry = false; +// for tx in txs { +// let (stream, retry_conn) = +// Self::open_unistream(conn.clone(), last_stable_id.clone(), connection_timeout) +// .await; +// if let Some(send_stream) = stream { +// let tx_raw = bincode::serialize(tx).unwrap(); +// +// retry = QuicConnectionUtils::write_all( +// send_stream, +// &tx_raw, +// identity, +// last_stable_id.clone(), +// conn.stable_id() as u64, +// connection_timeout, +// ) +// .await; +// } else { +// retry = retry_conn; +// } +// if retry { +// queue.push_back(tx); +// break; +// } +// } +// +// } diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index 530034e1..aedac8f8 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -106,8 +106,9 @@ fn build_memo_tx_raw() -> Vec { let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); let wire_data = serialize_tpu_forwarding_request( - "127.0.0.1:5454".parse().unwrap(), - Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), + // FIXME hardcoded to local test-validator + "127.0.0.1:1027".parse().unwrap(), + Pubkey::from_str("EPLzGRhibYmZ7qysF9BiPmSTRaL8GiLhrQdFTfL8h2fy").unwrap(), vec![tx.into()]); println!("wire_data: {:02X?}", wire_data); diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index a9673483..6d8ff587 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -3,6 +3,7 @@ use std::str::FromStr; use anyhow::Context; use bincode::DefaultOptions; use log::info; +use serde::Serialize; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::{Transaction, VersionedTransaction}; use spl_memo::solana_program::message::VersionedMessage; @@ -19,36 +20,19 @@ fn roundtrip() { let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); - let wire_data = serialize_tpu_forwarding_request( + let wire_data = TpuForwardingRequest::new( "127.0.0.1:5454".parse().unwrap(), Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), - vec![tx.into()]); + vec![tx.into()] + ).serialize_wire_format(); println!("wire_data: {:02X?}", wire_data); - let request = deserialize_tpu_forwarding_request(&wire_data); + let request = TpuForwardingRequest::deserialize_from_raw_request(&wire_data); assert_eq!(request.get_tpu_socket_addr().is_ipv4(), true); assert_eq!(request.get_transactions().len(), 1); } -fn serialize_tpu_forwarding_request( - tpu_socket_addr: SocketAddr, - tpu_identity: Pubkey, - transactions: Vec) -> Vec { - - let request = TpuForwardingRequest::new(tpu_socket_addr, tpu_identity, transactions); - - bincode::serialize(&request).expect("Expect to serialize transactions") -} - -// TODO reame -fn deserialize_tpu_forwarding_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { - let request = bincode::deserialize::(&raw_proxy_request) - .context("deserialize proxy request") - .unwrap(); - - request -} diff --git a/services/Cargo.toml b/services/Cargo.toml index fe51b27e..d094efc8 100644 --- a/services/Cargo.toml +++ b/services/Cargo.toml @@ -27,6 +27,7 @@ thiserror = { workspace = true } futures = { workspace = true } bytes = { workspace = true } anyhow = { workspace = true } +itertools = { workspace = true } log = { workspace = true } dashmap = { workspace = true } prometheus = { workspace = true } diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 04b0d64f..27a7bd88 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -18,12 +18,14 @@ use std::{ time::Duration, }; use anyhow::bail; +use itertools::Itertools; +use solana_client::client_error::reqwest::header::WARNING; use solana_sdk::transaction::VersionedTransaction; use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; use tokio::time::timeout; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -pub const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(1); +pub const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); pub const CONNECTION_RETRY_COUNT: usize = 10; lazy_static::lazy_static! { @@ -174,32 +176,36 @@ impl ActiveConnection { NB_QUIC_TASKS.inc(); let connection = connection.unwrap(); - // TODO split to new service - // SOS - info!("Sending copy of transaction batch of {} to tpu with identity {} to quic proxy", - txs.len(), identity); - Self::send_copy_of_txs_to_quicproxy( - &txs, endpoint.clone(), - // proxy address - "127.0.0.1:11111".parse().unwrap(), - tpu_address, - identity.clone()).await.unwrap(); + if true { + // TODO split to new service + // SOS + info!("Sending copy of transaction batch of {} to tpu with identity {} to quic proxy", + txs.len(), identity); + Self::send_copy_of_txs_to_quicproxy( + &txs, endpoint.clone(), + // proxy address + "127.0.0.1:11111".parse().unwrap(), + tpu_address, + identity.clone()).await.unwrap(); + } - QuicConnectionUtils::send_transaction_batch( - connection, - txs, - identity, - endpoint, - tpu_address, - exit_signal, - last_stable_id, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - || { - // do nothing as we are using the same connection - } - ).await; + if false { + QuicConnectionUtils::send_transaction_batch( + connection, + txs, + identity, + endpoint, + tpu_address, + exit_signal, + last_stable_id, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + || { + // do nothing as we are using the same connection + } + ).await; + } NB_QUIC_TASKS.dec(); task_counter.fetch_sub(1, Ordering::Relaxed); @@ -218,6 +224,9 @@ impl ActiveConnection { async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, endpoint: Endpoint, proxy_address: SocketAddr, tpu_target_address: SocketAddr, identity: Pubkey) -> anyhow::Result<()> { + + info!("sending vecvec: {}", raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(",")); + let raw_tx_batch_copy = raw_tx_batch.clone(); let mut txs = vec![]; @@ -236,7 +245,7 @@ impl ActiveConnection { let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); - let send_result = timeout(Duration::from_millis(500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)); + let send_result = timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)); match send_result.await { Ok(..) => { diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 43fc677b..5b008397 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -65,14 +65,14 @@ impl TpuService { pub async fn new( current_slot: Slot, fanout_slots: u64, - identity: Arc, + validator_identity: Arc, rpc_client: Arc, rpc_ws_address: String, txs_sent_store: TxStore, ) -> anyhow::Result { let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); let (certificate, key) = new_self_signed_tls_certificate( - identity.as_ref(), + validator_identity.as_ref(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) .expect("Failed to initialize QUIC client certificates"); @@ -89,7 +89,7 @@ impl TpuService { rpc_ws_address, broadcast_sender: Arc::new(sender), tpu_connection_manager: Arc::new(tpu_connection_manager), - identity, + identity: validator_identity, identity_stakes: Arc::new(RwLock::new(IdentityStakes::default())), txs_sent_store, }) From a786e190bd6901e145415a67652efbd5820526a8 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 27 Jun 2023 12:08:26 +0200 Subject: [PATCH 011/128] add readme --- quic-forward-proxy/README.md | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 quic-forward-proxy/README.md diff --git a/quic-forward-proxy/README.md b/quic-forward-proxy/README.md new file mode 100644 index 00000000..b95b6a34 --- /dev/null +++ b/quic-forward-proxy/README.md @@ -0,0 +1,43 @@ + + + + +``` + +------------+ +------------+ +------------+ +------------+ + | | | | | | | | + | bench | ---1---> | lite-rpc | ---2---> | proxy | ---3---> | validator | + | | | | | | | | + +------------+ +------------+ +------------+ +------------+ + + 1. rpc request + 2. tpu forward proxy request (QUIC): transactions, tpu address and tpu identity + 3. tpu call (QUIC), transactions: + +``` + +Local Development / Testing +--------------------------- + +1. run test-validator (tested with 1.16.1) +```bash +RUST_LOG="error,solana_streamer::nonblocking::quic=debug" solana-test-validator --log +``` +2. run lite-rpc +```bash +RUST_LOG=info cargo run --bin lite-rpc -- --identity-keypair /pathto-test-ledger/validator-keypair.json +``` +3. run rust bench tool in _lite-rpc_ +```bash +cd bench; cargo run -- --tx-count=10 +``` + + +### Example Output from _Solana Validator_: +(note: the peer type is __Staked__) +``` +[2023-06-26T15:16:18.430602000Z INFO solana_streamer::nonblocking::quic] Got a connection 127.0.0.1:8058 +[2023-06-26T15:16:18.430633000Z DEBUG solana_streamer::nonblocking::quic] Peer public key is EPLzGRhibYmZ7qysF9BiPmSTRaL8GiLhrQdFTfL8h2fy +[2023-06-26T15:16:18.430839000Z DEBUG solana_streamer::nonblocking::quic] Peer type: Staked, stake 999999997717120, total stake 999999997717120, max streams 2048 receive_window Ok(12320) from peer 127.0.0.1:8058 +[2023-06-26T15:16:18.430850000Z DEBUG solana_streamer::nonblocking::quic] quic new connection 127.0.0.1:8058 streams: 0 connections: 1 +[2023-06-26T15:16:18.430854000Z DEBUG solana_streamer::nonblocking::quic] stream error: ApplicationClosed(ApplicationClose { error_code: 0, reason: b"done" }) +``` \ No newline at end of file From eb2571149693c78668c24b0a9c7d42d52827aff1 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 27 Jun 2023 13:35:21 +0200 Subject: [PATCH 012/128] comment tpu_service input --- quic-forward-proxy/tests/proxy_request_format.rs | 8 +++++--- services/src/tpu_utils/tpu_service.rs | 12 ++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index 6d8ff587..14612011 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -4,7 +4,9 @@ use anyhow::Context; use bincode::DefaultOptions; use log::info; use serde::Serialize; +use solana_sdk::hash::{Hash, Hasher}; use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::{Keypair, Signer}; use solana_sdk::transaction::{Transaction, VersionedTransaction}; use spl_memo::solana_program::message::VersionedMessage; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; @@ -13,10 +15,10 @@ use solana_lite_rpc_core::proxy_request_format::*; #[test] fn roundtrip() { - let payer_pubkey = Pubkey::new_unique(); - let signer_pubkey = Pubkey::new_unique(); + let payer = Keypair::from_base58_string("rKiJ7H5UUp3JR18kNyTF1XPuwPKHEM7gMLWHZPWP5djrW1vSjfwjhvJrevxF9MPmUmN9gJMLHZdLMgc9ao78eKr"); + let payer_pubkey = payer.pubkey(); - let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&signer_pubkey]); + let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&payer_pubkey]); let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 5b008397..0a260182 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -46,18 +46,30 @@ lazy_static::lazy_static! { register_int_gauge!(opts!("literpc_estimated_slot", "Estimated slot seen by last rpc")).unwrap(); } +/// service (singleton) #[derive(Clone)] pub struct TpuService { + /// out current_slot: Arc, + /// out estimated_slot: Arc, + /// in fanout_slots: u64, + /// in rpc_client: Arc, + /// in rpc_ws_address: String, + /// in broadcast_sender: Arc)>>, + /// in tpu_connection_manager: Arc, + /// in identity: Arc, + /// out identity_stakes: Arc>, + /// in txs_sent_store: TxStore, + /// out leader_schedule: Arc, } From c0d4b18df55951abe881aecb36d781f2e86fd471 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 27 Jun 2023 16:53:27 +0200 Subject: [PATCH 013/128] improve error handling on accept stream --- quic-forward-proxy/src/proxy.rs | 104 ++++++++------------------------ 1 file changed, 26 insertions(+), 78 deletions(-) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 12669e28..71397dfa 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -4,9 +4,9 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::time::Duration; use anyhow::{anyhow, bail}; -use itertools::Itertools; -use log::{error, info, warn}; -use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig}; +use itertools::{any, Itertools}; +use log::{debug, error, info, trace, warn}; +use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig, VarInt}; use rcgen::generate_simple_self_signed; use rustls::{Certificate, PrivateKey}; use rustls::server::ResolvesServerCert; @@ -59,8 +59,8 @@ impl QuicForwardProxy { let identity_keypair = Keypair::new(); // TODO while let Some(conn) = endpoint.accept().await { - info!("connection incoming"); - let fut = handle_connection2(conn, exit_signal.clone(), self.validator_identity.clone()); + trace!("connection incoming"); + let fut = handle_connection(conn, exit_signal.clone(), self.validator_identity.clone()); tokio::spawn(async move { if let Err(e) = fut.await { error!("connection failed: {reason}", reason = e.to_string()) @@ -68,18 +68,6 @@ impl QuicForwardProxy { }); } - // while let Some(conn) = endpoint.accept().await { - // info!("connection incoming"); - // // let fut = handle_connection(conn); - // tokio::spawn(async move { - // info!("start thread"); - // handle_connection2(conn).await.unwrap(); - // // if let Err(e) = fut.await { - // // error!("connection failed: {reason}", reason = e.to_string()) - // // } - // }); - // } - bail!("TPU Quic Proxy server stopped"); }); @@ -93,52 +81,45 @@ impl QuicForwardProxy { } -// meins -async fn handle_connection2(connecting: Connecting, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { +async fn handle_connection(connecting: Connecting, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { let connection = connecting.await?; - info!("inbound connection established, remote {connection}", connection = connection.remote_address()); + debug!("inbound connection established, remote {connection}", connection = connection.remote_address()); async { loop { - let stream = connection.accept_uni().await; - let mut recv = match stream { - Err(quinn::ConnectionError::ApplicationClosed { .. }) => { - info!("connection closed"); + let maybe_stream = connection.accept_uni().await; + let mut recv_stream = match maybe_stream { + Err(quinn::ConnectionError::ApplicationClosed(reason)) => { + debug!("connection closed by peer - reason: {:?}", reason); + if reason.error_code != VarInt::from_u32(0) { + return Err(anyhow!("connection closed by peer with unexpected reason: {:?}", reason)); + } + debug!("connection gracefully closed by peer"); return Ok(()); - } + }, Err(e) => { - warn!("connection failed: {}", e); - return Err(anyhow::Error::msg("connection failed")); + error!("failed to accept stream: {}", e); + return Err(anyhow::Error::msg("error accepting stream")); } Ok(s) => s, }; let exit_signal_copy = exit_signal.clone(); let validator_identity_copy = validator_identity.clone(); tokio::spawn(async move { - let raw_request = recv.read_to_end(10_000_000).await + let raw_request = recv_stream.read_to_end(10_000_000).await .unwrap(); - // let str = std::str::from_utf8(&result).unwrap(); - info!("read proxy_request {} bytes", raw_request.len()); + debug!("read proxy_request {} bytes", raw_request.len()); let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); - info!("proxy request details: {}", proxy_request); + debug!("proxy request details: {}", proxy_request); let tpu_identity = proxy_request.get_identity_tpunode(); let tpu_addr = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transactions(); send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_addr, &txs).await; - // Ok(()) }); - // info!("stream okey {:?}", stream); - // let fut = handle_request2(stream).await; - // tokio::spawn( - // async move { - // if let Err(e) = fut.await { - // error!("failed: {reason}", reason = e.to_string()); - // } - // } - // ); + } // -- loop } .await?; @@ -175,6 +156,10 @@ async fn send_txs_to_tpu(exit_signal: Arc, validator_identity: Arc = Arc::new(AtomicU64::new(0)); + let txs_raw = serialize_to_vecvec(&txs); + + info!("received vecvec: {}", txs_raw.iter().map(|tx| tx.len().to_string()).into_iter().join(",")); + let connection = Arc::new(RwLock::new( QuicConnectionUtils::connect( @@ -190,12 +175,6 @@ async fn send_txs_to_tpu(exit_signal: Arc, validator_identity: Arc) -> Vec> tx_raw }).collect_vec() } - -// async fn send_transactions_with_retry( -// conn: Connection, -// identity: Pubkey, txs: &Vec) { -// let mut retry = false; -// for tx in txs { -// let (stream, retry_conn) = -// Self::open_unistream(conn.clone(), last_stable_id.clone(), connection_timeout) -// .await; -// if let Some(send_stream) = stream { -// let tx_raw = bincode::serialize(tx).unwrap(); -// -// retry = QuicConnectionUtils::write_all( -// send_stream, -// &tx_raw, -// identity, -// last_stable_id.clone(), -// conn.stable_id() as u64, -// connection_timeout, -// ) -// .await; -// } else { -// retry = retry_conn; -// } -// if retry { -// queue.push_back(tx); -// break; -// } -// } -// -// } From c406f447ccd90247800784fa77f6dab6d2d22ab0 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 27 Jun 2023 16:55:38 +0200 Subject: [PATCH 014/128] remove useless async --- quic-forward-proxy/src/proxy.rs | 64 ++++++++++++++++----------------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 71397dfa..4429e8e8 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -84,46 +84,42 @@ impl QuicForwardProxy { async fn handle_connection(connecting: Connecting, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { let connection = connecting.await?; debug!("inbound connection established, remote {connection}", connection = connection.remote_address()); - async { - loop { - let maybe_stream = connection.accept_uni().await; - let mut recv_stream = match maybe_stream { - Err(quinn::ConnectionError::ApplicationClosed(reason)) => { - debug!("connection closed by peer - reason: {:?}", reason); - if reason.error_code != VarInt::from_u32(0) { - return Err(anyhow!("connection closed by peer with unexpected reason: {:?}", reason)); - } - debug!("connection gracefully closed by peer"); - return Ok(()); - }, - Err(e) => { - error!("failed to accept stream: {}", e); - return Err(anyhow::Error::msg("error accepting stream")); + loop { + let maybe_stream = connection.accept_uni().await; + let mut recv_stream = match maybe_stream { + Err(quinn::ConnectionError::ApplicationClosed(reason)) => { + debug!("connection closed by peer - reason: {:?}", reason); + if reason.error_code != VarInt::from_u32(0) { + return Err(anyhow!("connection closed by peer with unexpected reason: {:?}", reason)); } - Ok(s) => s, - }; - let exit_signal_copy = exit_signal.clone(); - let validator_identity_copy = validator_identity.clone(); - tokio::spawn(async move { - let raw_request = recv_stream.read_to_end(10_000_000).await - .unwrap(); - debug!("read proxy_request {} bytes", raw_request.len()); + debug!("connection gracefully closed by peer"); + return Ok(()); + }, + Err(e) => { + error!("failed to accept stream: {}", e); + return Err(anyhow::Error::msg("error accepting stream")); + } + Ok(s) => s, + }; + let exit_signal_copy = exit_signal.clone(); + let validator_identity_copy = validator_identity.clone(); + tokio::spawn(async move { + let raw_request = recv_stream.read_to_end(10_000_000).await + .unwrap(); + debug!("read proxy_request {} bytes", raw_request.len()); - let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); + let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); - debug!("proxy request details: {}", proxy_request); - let tpu_identity = proxy_request.get_identity_tpunode(); - let tpu_addr = proxy_request.get_tpu_socket_addr(); - let txs = proxy_request.get_transactions(); + debug!("proxy request details: {}", proxy_request); + let tpu_identity = proxy_request.get_identity_tpunode(); + let tpu_addr = proxy_request.get_tpu_socket_addr(); + let txs = proxy_request.get_transactions(); - send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_addr, &txs).await; + send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_addr, &txs).await; - }); + }); - } // -- loop - } - .await?; - Ok(()) + } // -- loop } mod test { From f5979eb0984c72e1230f0300b774d8e6749c7bfa Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 27 Jun 2023 17:06:35 +0200 Subject: [PATCH 015/128] WIP: refactor active type connection --- quic-forward-proxy/src/proxy.rs | 143 ++++++++++++++++---------------- 1 file changed, 70 insertions(+), 73 deletions(-) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 4429e8e8..95bad738 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -60,7 +60,20 @@ impl QuicForwardProxy { while let Some(conn) = endpoint.accept().await { trace!("connection incoming"); - let fut = handle_connection(conn, exit_signal.clone(), self.validator_identity.clone()); + + let (certificate, key) = new_self_signed_tls_certificate( + self.validator_identity.as_ref(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC client certificates"); + + let endpoint_outbound = QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()); + + let active_tpu_connection = ActiveTpuConnection { + endpoint: endpoint_outbound.clone(), + }; + + let fut = handle_connection(conn, active_tpu_connection, exit_signal.clone(), self.validator_identity.clone()); tokio::spawn(async move { if let Err(e) = fut.await { error!("connection failed: {reason}", reason = e.to_string()) @@ -80,8 +93,61 @@ impl QuicForwardProxy { } +#[derive(Clone)] +struct ActiveTpuConnection { + endpoint: Endpoint, +} + +impl ActiveTpuConnection { + + pub async fn send_txs_to_tpu(&self, exit_signal: Arc, validator_identity: Arc, tpu_identity: Pubkey, tpu_addr: SocketAddr, txs: &Vec) { + + let last_stable_id: Arc = Arc::new(AtomicU64::new(0)); + + let txs_raw = serialize_to_vecvec(&txs); + + info!("received vecvec: {}", txs_raw.iter().map(|tx| tx.len().to_string()).into_iter().join(",")); + + let connection = + Arc::new(RwLock::new( + QuicConnectionUtils::connect( + tpu_identity, + false, + self.endpoint.clone(), + tpu_addr, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + exit_signal.clone(), + || { + // do nothing + }, + ).await.unwrap())); + + QuicConnectionUtils::send_transaction_batch( + connection.clone(), + txs_raw, + tpu_identity, + self.endpoint.clone(), + tpu_addr, + exit_signal.clone(), + last_stable_id, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + || { + // do nothing + } + ).await; + + { + let conn = connection.clone(); + conn.write().await.close(0u32.into(), b"done"); + } + } + +} + -async fn handle_connection(connecting: Connecting, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { +async fn handle_connection(connecting: Connecting, active_tpu_connection: ActiveTpuConnection, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { let connection = connecting.await?; debug!("inbound connection established, remote {connection}", connection = connection.remote_address()); loop { @@ -101,6 +167,7 @@ async fn handle_connection(connecting: Connecting, exit_signal: Arc, } Ok(s) => s, }; + let active_tpu_connection_copy = active_tpu_connection.clone(); let exit_signal_copy = exit_signal.clone(); let validator_identity_copy = validator_identity.clone(); tokio::spawn(async move { @@ -115,83 +182,13 @@ async fn handle_connection(connecting: Connecting, exit_signal: Arc, let tpu_addr = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transactions(); - send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_addr, &txs).await; + active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_addr, &txs).await; }); } // -- loop } -mod test { - use std::str::FromStr; - use std::sync::Arc; - use std::sync::atomic::AtomicBool; - use solana_sdk::pubkey::Pubkey; - use crate::cli::get_identity_keypair; - use crate::proxy::send_txs_to_tpu; - - #[test] - fn call() { - let exit_signal = Arc::new(AtomicBool::new(false)); - - let validator_identity = get_identity_keypair(&"/Users/stefan/mango/projects/quic-forward-proxy/local-testvalidator-stake-account.json".to_string()); - let tpu_identity = Pubkey::from_str("asdfsdf").unwrap(); - let tpu_address = "127.0.0.1:1027".parse().unwrap(); - send_txs_to_tpu(exit_signal, validator_identity, tpu_identity, tpu_address, &vec![]) - - } -} - -async fn send_txs_to_tpu(exit_signal: Arc, validator_identity: Arc, tpu_identity: Pubkey, tpu_addr: SocketAddr, txs: &Vec) { - let (certificate, key) = new_self_signed_tls_certificate( - validator_identity.as_ref(), - IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), - ) - .expect("Failed to initialize QUIC client certificates"); - - let endpoint = QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()); - let last_stable_id: Arc = Arc::new(AtomicU64::new(0)); - - let txs_raw = serialize_to_vecvec(&txs); - - info!("received vecvec: {}", txs_raw.iter().map(|tx| tx.len().to_string()).into_iter().join(",")); - - let connection = - Arc::new(RwLock::new( - QuicConnectionUtils::connect( - tpu_identity, - false, - endpoint.clone(), - tpu_addr, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - exit_signal.clone(), - || { - // do nothing - }, - ).await.unwrap())); - - QuicConnectionUtils::send_transaction_batch( - connection.clone(), - txs_raw, - tpu_identity, - endpoint, - tpu_addr, - exit_signal.clone(), - last_stable_id, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - || { - // do nothing - } - ).await; - - { - let conn = connection.clone(); - conn.write().await.close(0u32.into(), b"done"); - } -} - fn serialize_to_vecvec(transactions: &Vec) -> Vec> { transactions.iter().map(|tx| { let tx_raw = bincode::serialize(tx).unwrap(); From 7c3f7bea74944c42cb9bfd995a88d168943a6504 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 29 Jun 2023 11:05:49 +0200 Subject: [PATCH 016/128] refactor out tpu_quic_connection --- Cargo.lock | 4 + core/src/quic_connection_utils.rs | 1 + lite-rpc/src/main.rs | 1 + quic-forward-proxy/src/lib.rs | 1 + quic-forward-proxy/src/main.rs | 1 + quic-forward-proxy/src/proxy.rs | 84 ++----------- quic-forward-proxy/src/tpu_quic_connection.rs | 110 ++++++++++++++++++ services/src/tpu_utils/tpu_service.rs | 2 +- 8 files changed, 128 insertions(+), 76 deletions(-) create mode 100644 quic-forward-proxy/src/tpu_quic_connection.rs diff --git a/Cargo.lock b/Cargo.lock index ea7743fe..34e29308 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -430,6 +430,7 @@ dependencies = [ "dashmap", "dirs", "futures", + "lazy_static", "log", "rand 0.8.5", "rand_chacha 0.3.1", @@ -4018,6 +4019,7 @@ dependencies = [ "clap 4.2.4", "dashmap", "dotenv", + "itertools", "lazy_static", "log", "native-tls", @@ -4031,6 +4033,7 @@ dependencies = [ "solana-lite-rpc-services", "solana-sdk", "solana-streamer", + "spl-memo", "thiserror", "tokio", "tracing-subscriber", @@ -4049,6 +4052,7 @@ dependencies = [ "chrono", "dashmap", "futures", + "itertools", "lazy_static", "log", "prometheus", diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index f5ad5b4d..73b58271 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -187,6 +187,7 @@ impl QuicConnectionUtils { last_stable_id.store(connection.stable_id() as u64, Ordering::Relaxed); (None, true) } + // timeout Err(_) => (None, false), } } diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 0fa09c96..0ed441ca 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -14,6 +14,7 @@ use solana_lite_rpc_quic_forward_proxy::SelfSignedTlsConfigProvider; use solana_lite_rpc_quic_forward_proxy::test_client::quic_test_client::QuicTestClient; // use lite_rpc_quic_forward_proxy::tls_config::SelfSignedTlsConfigProvider; +// note: copy of this method is used in quic-forward-proxy async fn get_identity_keypair(identity_from_cli: &String) -> Keypair { if let Ok(identity_env_var) = env::var("IDENTITY") { if let Ok(identity_bytes) = serde_json::from_str::>(identity_env_var.as_str()) { diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index c1cce8b9..868eca6b 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -1,6 +1,7 @@ pub mod quic_util; pub mod tls_config_provicer; pub mod proxy; +pub mod tpu_quic_connection; pub mod test_client; pub mod cli; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index ea9bd2ba..20ce44cd 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -10,6 +10,7 @@ use crate::test_client::quic_test_client::QuicTestClient; use crate::tls_config_provicer::SelfSignedTlsConfigProvider; mod proxy; +mod tpu_quic_connection; mod test_client; mod quic_util; mod tls_config_provicer; diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 95bad738..00f81ba2 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -2,6 +2,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64}; +use std::thread::sleep; use std::time::Duration; use anyhow::{anyhow, bail}; use itertools::{any, Itertools}; @@ -19,9 +20,11 @@ use tokio::net::ToSocketAddrs; use solana_lite_rpc_core::AnyhowJoinHandle; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; +use solana_lite_rpc_core::leader_schedule::LeaderSchedule; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; use solana_lite_rpc_core::quic_connection_utils::QuicConnectionUtils; use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::{ActiveConnection, CONNECTION_RETRY_COUNT, QUIC_CONNECTION_TIMEOUT}; +use crate::tpu_quic_connection::TpuQuicConnection; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; @@ -61,17 +64,8 @@ impl QuicForwardProxy { while let Some(conn) = endpoint.accept().await { trace!("connection incoming"); - let (certificate, key) = new_self_signed_tls_certificate( - self.validator_identity.as_ref(), - IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), - ) - .expect("Failed to initialize QUIC client certificates"); - - let endpoint_outbound = QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()); - - let active_tpu_connection = ActiveTpuConnection { - endpoint: endpoint_outbound.clone(), - }; + let active_tpu_connection = + TpuQuicConnection::new_with_validator_identity(self.validator_identity.as_ref()); let fut = handle_connection(conn, active_tpu_connection, exit_signal.clone(), self.validator_identity.clone()); tokio::spawn(async move { @@ -93,61 +87,8 @@ impl QuicForwardProxy { } -#[derive(Clone)] -struct ActiveTpuConnection { - endpoint: Endpoint, -} - -impl ActiveTpuConnection { - - pub async fn send_txs_to_tpu(&self, exit_signal: Arc, validator_identity: Arc, tpu_identity: Pubkey, tpu_addr: SocketAddr, txs: &Vec) { - - let last_stable_id: Arc = Arc::new(AtomicU64::new(0)); - - let txs_raw = serialize_to_vecvec(&txs); - - info!("received vecvec: {}", txs_raw.iter().map(|tx| tx.len().to_string()).into_iter().join(",")); - - let connection = - Arc::new(RwLock::new( - QuicConnectionUtils::connect( - tpu_identity, - false, - self.endpoint.clone(), - tpu_addr, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - exit_signal.clone(), - || { - // do nothing - }, - ).await.unwrap())); - - QuicConnectionUtils::send_transaction_batch( - connection.clone(), - txs_raw, - tpu_identity, - self.endpoint.clone(), - tpu_addr, - exit_signal.clone(), - last_stable_id, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - || { - // do nothing - } - ).await; - - { - let conn = connection.clone(); - conn.write().await.close(0u32.into(), b"done"); - } - } - -} - -async fn handle_connection(connecting: Connecting, active_tpu_connection: ActiveTpuConnection, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { +async fn handle_connection(connecting: Connecting, active_tpu_connection: TpuQuicConnection, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { let connection = connecting.await?; debug!("inbound connection established, remote {connection}", connection = connection.remote_address()); loop { @@ -171,7 +112,7 @@ async fn handle_connection(connecting: Connecting, active_tpu_connection: Active let exit_signal_copy = exit_signal.clone(); let validator_identity_copy = validator_identity.clone(); tokio::spawn(async move { - let raw_request = recv_stream.read_to_end(10_000_000).await + let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const .unwrap(); debug!("read proxy_request {} bytes", raw_request.len()); @@ -179,19 +120,12 @@ async fn handle_connection(connecting: Connecting, active_tpu_connection: Active debug!("proxy request details: {}", proxy_request); let tpu_identity = proxy_request.get_identity_tpunode(); - let tpu_addr = proxy_request.get_tpu_socket_addr(); + let tpu_address = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transactions(); - active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_addr, &txs).await; + active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await; }); } // -- loop } - -fn serialize_to_vecvec(transactions: &Vec) -> Vec> { - transactions.iter().map(|tx| { - let tx_raw = bincode::serialize(tx).unwrap(); - tx_raw - }).collect_vec() -} diff --git a/quic-forward-proxy/src/tpu_quic_connection.rs b/quic-forward-proxy/src/tpu_quic_connection.rs new file mode 100644 index 00000000..c440672d --- /dev/null +++ b/quic-forward-proxy/src/tpu_quic_connection.rs @@ -0,0 +1,110 @@ +use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64}; +use std::time::Duration; +use anyhow::{anyhow, bail}; +use itertools::{any, Itertools}; +use log::{debug, error, info, trace, warn}; +use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig, VarInt}; +use rcgen::generate_simple_self_signed; +use rustls::{Certificate, PrivateKey}; +use rustls::server::ResolvesServerCert; +use serde::{Deserialize, Serialize}; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::Keypair; +use solana_sdk::signer::Signer; +use solana_sdk::transaction::VersionedTransaction; +use tokio::net::ToSocketAddrs; +use solana_lite_rpc_core::AnyhowJoinHandle; +use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use tokio::sync::RwLock; +use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; +use solana_lite_rpc_core::quic_connection_utils::QuicConnectionUtils; +use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::{ActiveConnection, CONNECTION_RETRY_COUNT, QUIC_CONNECTION_TIMEOUT}; +use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; + +/// stable connect to TPU to send transactions - optimized for proxy use case +#[derive(Clone)] +pub struct TpuQuicConnection { + endpoint: Endpoint, +} + +impl TpuQuicConnection { + + /// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU + // note: ATM the provided identity might or might not be a valid validator keypair + pub fn new_with_validator_identity(validator_identity: &Keypair) -> TpuQuicConnection { + let (certificate, key) = new_self_signed_tls_certificate( + validator_identity, + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC connection certificates"); + + let endpoint_outbound = QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()); + + let active_tpu_connection = TpuQuicConnection { + endpoint: endpoint_outbound.clone(), + }; + + active_tpu_connection + } + + pub async fn send_txs_to_tpu(&self, + exit_signal: Arc, + validator_identity: Arc, + tpu_identity: Pubkey, + tpu_address: SocketAddr, + txs: &Vec) { + + let last_stable_id: Arc = Arc::new(AtomicU64::new(0)); + + let txs_raw = serialize_to_vecvec(&txs); + + info!("received vecvec: {}", txs_raw.iter().map(|tx| tx.len().to_string()).into_iter().join(",")); + + let connection = + Arc::new(RwLock::new( + QuicConnectionUtils::connect( + tpu_identity, + false, + self.endpoint.clone(), + tpu_address, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + exit_signal.clone(), + || { + // do nothing + }, + ).await.unwrap())); + + QuicConnectionUtils::send_transaction_batch( + connection.clone(), + txs_raw, + tpu_identity, + self.endpoint.clone(), + tpu_address, + exit_signal.clone(), + last_stable_id, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + || { + // do nothing + } + ).await; + + { + let conn = connection.clone(); + conn.write().await.close(0u32.into(), b"done"); + } + } + +} + + +fn serialize_to_vecvec(transactions: &Vec) -> Vec> { + transactions.iter().map(|tx| { + let tx_raw = bincode::serialize(tx).unwrap(); + tx_raw + }).collect_vec() +} diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 0a260182..8ddaafc2 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -87,7 +87,7 @@ impl TpuService { validator_identity.as_ref(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) - .expect("Failed to initialize QUIC client certificates"); + .expect("Failed to initialize QUIC connection certificates"); let tpu_connection_manager = TpuConnectionManager::new(certificate, key, fanout_slots as usize); From 228c58d8618d7fd26c460e091e92bc92a5278d60 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 20 Jul 2023 09:28:55 +0200 Subject: [PATCH 017/128] cut dependency between quic-proxy and lite-rpc crate --- lite-rpc/Cargo.toml | 2 - lite-rpc/src/main.rs | 19 +- quic-forward-proxy/Cargo.toml | 4 +- quic-forward-proxy/README.md | 6 +- quic-forward-proxy/src/active_connection.rs | 293 +++++++++++++++++ quic-forward-proxy/src/identity_stakes.rs | 22 ++ quic-forward-proxy/src/lib.rs | 10 - quic-forward-proxy/src/main.rs | 40 ++- quic-forward-proxy/src/proxy.rs | 7 +- .../src/proxy_request_format.rs | 71 ++++ .../src/quic_connection_utils.rs | 309 ++++++++++++++++++ .../src/test_client/quic_test_client.rs | 6 +- quic-forward-proxy/src/tls_config_provicer.rs | 2 +- quic-forward-proxy/src/tpu_quic_connection.rs | 8 +- quic-forward-proxy/src/tx_store.rs | 27 ++ quic-forward-proxy/src/util.rs | 2 + .../tests/proxy_request_format.rs | 3 +- 17 files changed, 774 insertions(+), 57 deletions(-) create mode 100644 quic-forward-proxy/src/active_connection.rs create mode 100644 quic-forward-proxy/src/identity_stakes.rs delete mode 100644 quic-forward-proxy/src/lib.rs create mode 100644 quic-forward-proxy/src/proxy_request_format.rs create mode 100644 quic-forward-proxy/src/quic_connection_utils.rs create mode 100644 quic-forward-proxy/src/tx_store.rs create mode 100644 quic-forward-proxy/src/util.rs diff --git a/lite-rpc/Cargo.toml b/lite-rpc/Cargo.toml index 15c32ab0..694be264 100644 --- a/lite-rpc/Cargo.toml +++ b/lite-rpc/Cargo.toml @@ -39,8 +39,6 @@ async-channel = { workspace = true } quinn = { workspace = true } solana-lite-rpc-core = { workspace = true } solana-lite-rpc-services = { workspace = true } -# TODO remove -solana-lite-rpc-quic-forward-proxy = { path = "../quic-forward-proxy" } async-trait = { workspace = true } tokio = { version = "1.28.2", features = ["full", "fs"]} tokio-postgres = { version = "0.7.8", features = ["with-chrono-0_4"] } diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 0ed441ca..b4afa191 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -9,9 +9,6 @@ use solana_sdk::signature::Keypair; use std::env; use std::sync::Arc; use tokio::time::timeout; -use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; -use solana_lite_rpc_quic_forward_proxy::SelfSignedTlsConfigProvider; -use solana_lite_rpc_quic_forward_proxy::test_client::quic_test_client::QuicTestClient; // use lite_rpc_quic_forward_proxy::tls_config::SelfSignedTlsConfigProvider; // note: copy of this method is used in quic-forward-proxy @@ -71,11 +68,11 @@ pub async fn main() -> anyhow::Result<()> { let retry_after = Duration::from_secs(transaction_retry_after_secs); - let proxy_listener_addr = "127.0.0.1:11111".parse().unwrap(); - let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); - let quicproxy_service = QuicForwardProxy::new(proxy_listener_addr, &tls_configuration, validator_identity.clone()) - .await? - .start_services(); + // let proxy_listener_addr = "127.0.0.1:11111".parse().unwrap(); + // + // let quicproxy_service = QuicForwardProxy::new(proxy_listener_addr, validator_identity.clone()) + // .await? + // .start_services(); let services = LiteBridge::new( rpc_addr, @@ -106,9 +103,9 @@ pub async fn main() -> anyhow::Result<()> { res = services => { bail!("Services quit unexpectedly {res:?}"); }, - res = quicproxy_service => { - bail!("Quic Proxy quit unexpectedly {res:?}"); - }, + // res = quicproxy_service => { + // bail!("Quic Proxy quit unexpectedly {res:?}"); + // }, // res = test_client => { // bail!("Test Client quit unexpectedly {res:?}"); // }, diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 835da929..200a56c6 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -12,6 +12,8 @@ publish = false [dependencies] solana-sdk = { workspace = true } solana-streamer = { workspace = true } +solana-transaction-status = { workspace = true } +solana-net-utils = { workspace = true } rustls = { workspace = true, features = ["dangerous_configuration"]} serde = { workspace = true } serde_json = { workspace = true } @@ -32,8 +34,6 @@ lazy_static = { workspace = true } dotenv = { workspace = true } async-channel = { workspace = true } quinn = { workspace = true } -solana-lite-rpc-core = { workspace = true } -solana-lite-rpc-services = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } tokio = { version = "1.28.2", features = ["full", "fs"]} diff --git a/quic-forward-proxy/README.md b/quic-forward-proxy/README.md index b95b6a34..4f37a6df 100644 --- a/quic-forward-proxy/README.md +++ b/quic-forward-proxy/README.md @@ -22,9 +22,13 @@ Local Development / Testing ```bash RUST_LOG="error,solana_streamer::nonblocking::quic=debug" solana-test-validator --log ``` +3. run quic proxy +```bash +RUST_LOG=debug cargo run --bin solana-lite-rpc-quic-forward-proxy -- --identity-keypair /pathto-test-ledger/validator-keypair.json +``` 2. run lite-rpc ```bash -RUST_LOG=info cargo run --bin lite-rpc -- --identity-keypair /pathto-test-ledger/validator-keypair.json +RUST_LOG=debug cargo run --bin lite-rpc ``` 3. run rust bench tool in _lite-rpc_ ```bash diff --git a/quic-forward-proxy/src/active_connection.rs b/quic-forward-proxy/src/active_connection.rs new file mode 100644 index 00000000..9e8e7450 --- /dev/null +++ b/quic-forward-proxy/src/active_connection.rs @@ -0,0 +1,293 @@ +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::Duration; +use anyhow::bail; +use log::{error, info, warn}; +use prometheus::{opts, register_int_gauge}; +use prometheus::core::GenericGauge; +use quinn::{Connection, Endpoint}; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::transaction::VersionedTransaction; +use solana_streamer::nonblocking::quic::compute_max_allowed_uni_streams; +use tokio::sync::broadcast::Receiver; +use tokio::sync::RwLock; +use tokio::time::timeout; +use crate::identity_stakes::IdentityStakes; +use crate::proxy_request_format::TpuForwardingRequest; +use crate::quic_connection_utils::QuicConnectionUtils; +use crate::tpu_quic_connection::{CONNECTION_RETRY_COUNT, QUIC_CONNECTION_TIMEOUT}; +use crate::tx_store::TxStore; +use itertools::Itertools; + +lazy_static::lazy_static! { + // TODO rename / cleanup + static ref NB_QUIC_CONNECTIONS: GenericGauge = + register_int_gauge!(opts!("literpc_nb_active_quic_connections", "Number of quic connections open")).unwrap(); + static ref NB_QUIC_ACTIVE_CONNECTIONS: GenericGauge = + register_int_gauge!(opts!("literpc_nb_active_connections", "Number quic tasks that are running")).unwrap(); + static ref NB_CONNECTIONS_TO_KEEP: GenericGauge = + register_int_gauge!(opts!("literpc_connections_to_keep", "Number of connections to keep asked by tpu service")).unwrap(); + static ref NB_QUIC_TASKS: GenericGauge = + register_int_gauge!(opts!("literpc_quic_tasks", "Number of connections to keep asked by tpu service")).unwrap(); +} + +pub struct ActiveConnection { + endpoint: Endpoint, + identity: Pubkey, + tpu_address: SocketAddr, + exit_signal: Arc, + txs_sent_store: TxStore, +} + +impl ActiveConnection { + pub fn new( + endpoint: Endpoint, + tpu_address: SocketAddr, + identity: Pubkey, + txs_sent_store: TxStore, + ) -> Self { + Self { + endpoint, + tpu_address, + identity, + exit_signal: Arc::new(AtomicBool::new(false)), + txs_sent_store, + } + } + + fn on_connect() { + NB_QUIC_CONNECTIONS.inc(); + } + + fn check_for_confirmation(txs_sent_store: &TxStore, signature: String) -> bool { + match txs_sent_store.get(&signature) { + Some(props) => props.status.is_some(), + None => false, + } + } + + #[allow(clippy::too_many_arguments)] + async fn listen( + transaction_reciever: Receiver<(String, Vec)>, + exit_oneshot_channel: tokio::sync::mpsc::Receiver<()>, + endpoint: Endpoint, + tpu_address: SocketAddr, + exit_signal: Arc, + identity: Pubkey, + identity_stakes: IdentityStakes, + txs_sent_store: TxStore, + ) { + NB_QUIC_ACTIVE_CONNECTIONS.inc(); + let mut transaction_reciever = transaction_reciever; + let mut exit_oneshot_channel = exit_oneshot_channel; + + let max_uni_stream_connections: u64 = compute_max_allowed_uni_streams( + identity_stakes.peer_type, + identity_stakes.stakes, + identity_stakes.total_stakes, + ) as u64; + let number_of_transactions_per_unistream = 5; + + let task_counter: Arc = Arc::new(AtomicU64::new(0)); + let mut connection: Option>> = None; + let last_stable_id: Arc = Arc::new(AtomicU64::new(0)); + + loop { + // exit signal set + if exit_signal.load(Ordering::Relaxed) { + break; + } + + if task_counter.load(Ordering::Relaxed) >= max_uni_stream_connections { + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + continue; + } + + tokio::select! { + tx = transaction_reciever.recv() => { + // exit signal set + if exit_signal.load(Ordering::Relaxed) { + break; + } + + let first_tx: Vec = match tx { + Ok((sig, tx)) => { + if Self::check_for_confirmation(&txs_sent_store, sig) { + // transaction is already confirmed/ no need to send + continue; + } + tx + }, + Err(e) => { + error!( + "Broadcast channel error on recv for {} error {}", + identity, e + ); + continue; + } + }; + + let mut txs = vec![first_tx]; + for _ in 1..number_of_transactions_per_unistream { + if let Ok((signature, tx)) = transaction_reciever.try_recv() { + if Self::check_for_confirmation(&txs_sent_store, signature) { + continue; + } + txs.push(tx); + } + } + + if connection.is_none() { + // initial connection + let conn = QuicConnectionUtils::connect( + identity, + false, + endpoint.clone(), + tpu_address, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + exit_signal.clone(), + Self::on_connect).await; + + if let Some(conn) = conn { + // could connect + connection = Some(Arc::new(RwLock::new(conn))); + } else { + break; + } + } + + let task_counter = task_counter.clone(); + let endpoint = endpoint.clone(); + let exit_signal = exit_signal.clone(); + let connection = connection.clone(); + let last_stable_id = last_stable_id.clone(); + + tokio::spawn(async move { + task_counter.fetch_add(1, Ordering::Relaxed); + NB_QUIC_TASKS.inc(); + let connection = connection.unwrap(); + + if true { + // TODO split to new service + // SOS + info!("Sending copy of transaction batch of {} to tpu with identity {} to quic proxy", + txs.len(), identity); + Self::send_copy_of_txs_to_quicproxy( + &txs, endpoint.clone(), + // proxy address + "127.0.0.1:11111".parse().unwrap(), + tpu_address, + identity.clone()).await.unwrap(); + } + + + if false { + QuicConnectionUtils::send_transaction_batch( + connection, + txs, + identity, + endpoint, + tpu_address, + exit_signal, + last_stable_id, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + || { + // do nothing as we are using the same connection + } + ).await; + } + + NB_QUIC_TASKS.dec(); + task_counter.fetch_sub(1, Ordering::Relaxed); + }); + }, + _ = exit_oneshot_channel.recv() => { + break; + } + }; + } + drop(transaction_reciever); + NB_QUIC_CONNECTIONS.dec(); + NB_QUIC_ACTIVE_CONNECTIONS.dec(); + } + + async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, endpoint: Endpoint, + proxy_address: SocketAddr, tpu_target_address: SocketAddr, + identity: Pubkey) -> anyhow::Result<()> { + + info!("sending vecvec: {}", raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(",")); + + let raw_tx_batch_copy = raw_tx_batch.clone(); + + let mut txs = vec![]; + + for raw_tx in raw_tx_batch_copy { + let tx = match bincode::deserialize::(&raw_tx) { + Ok(tx) => tx, + Err(err) => { + bail!(err.to_string()); + } + }; + txs.push(tx); + } + + let forwarding_request = TpuForwardingRequest::new(tpu_target_address, identity, txs); + + let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); + + let send_result = timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)); + + match send_result.await { + Ok(..) => { + info!("Successfully sent data to quic proxy"); + } + Err(e) => { + warn!("Failed to send data to quic proxy: {:?}", e); + } + } + Ok(()) + } + + async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { + info!("sending {} bytes to proxy", proxy_request_raw.len()); + + let mut connecting = endpoint.connect(proxy_address, "localhost")?; + let connection = timeout(Duration::from_millis(500), connecting).await??; + let mut send = connection.open_uni().await?; + + send.write_all(proxy_request_raw).await?; + + send.finish().await?; + + Ok(()) + } + + pub fn start_listening( + &self, + transaction_reciever: Receiver<(String, Vec)>, + exit_oneshot_channel: tokio::sync::mpsc::Receiver<()>, + identity_stakes: IdentityStakes, + ) { + let endpoint = self.endpoint.clone(); + let tpu_address = self.tpu_address; + let exit_signal = self.exit_signal.clone(); + let identity = self.identity; + let txs_sent_store = self.txs_sent_store.clone(); + tokio::spawn(async move { + Self::listen( + transaction_reciever, + exit_oneshot_channel, + endpoint, + tpu_address, + exit_signal, + identity, + identity_stakes, + txs_sent_store, + ) + .await; + }); + } +} diff --git a/quic-forward-proxy/src/identity_stakes.rs b/quic-forward-proxy/src/identity_stakes.rs new file mode 100644 index 00000000..2b1194cd --- /dev/null +++ b/quic-forward-proxy/src/identity_stakes.rs @@ -0,0 +1,22 @@ +use solana_streamer::nonblocking::quic::ConnectionPeerType; + +#[derive(Debug, Copy, Clone)] +pub struct IdentityStakes { + pub peer_type: ConnectionPeerType, + pub stakes: u64, + pub total_stakes: u64, + pub min_stakes: u64, + pub max_stakes: u64, +} + +impl Default for IdentityStakes { + fn default() -> Self { + IdentityStakes { + peer_type: ConnectionPeerType::Unstaked, + stakes: 0, + total_stakes: 0, + max_stakes: 0, + min_stakes: 0, + } + } +} diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs deleted file mode 100644 index 868eca6b..00000000 --- a/quic-forward-proxy/src/lib.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub mod quic_util; -pub mod tls_config_provicer; -pub mod proxy; -pub mod tpu_quic_connection; -pub mod test_client; -pub mod cli; - - -pub use tls_config_provicer::SelfSignedTlsConfigProvider; -// pub mod tls_config; \ No newline at end of file diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 20ce44cd..80983b04 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -7,14 +7,21 @@ use log::info; use crate::cli::{Args, get_identity_keypair}; use crate::proxy::QuicForwardProxy; use crate::test_client::quic_test_client::QuicTestClient; -use crate::tls_config_provicer::SelfSignedTlsConfigProvider; +pub use tls_config_provicer::SelfSignedTlsConfigProvider; -mod proxy; -mod tpu_quic_connection; -mod test_client; -mod quic_util; -mod tls_config_provicer; -mod cli; + +pub mod quic_util; +pub mod tls_config_provicer; +pub mod proxy; +pub mod proxy_request_format; +pub mod tpu_quic_connection; +pub mod active_connection; +pub mod cli; +pub mod test_client; +mod util; +mod tx_store; +mod identity_stakes; +mod quic_connection_utils; #[tokio::main(flavor = "multi_thread", worker_threads = 16)] @@ -33,15 +40,16 @@ pub async fn main() -> anyhow::Result<()> { let validator_identity = Arc::new(get_identity_keypair(&identity_keypair).await); - let main_services = QuicForwardProxy::new(proxy_listener_addr, &tls_configuration, validator_identity) + let tls_config = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); + let main_services = QuicForwardProxy::new(proxy_listener_addr, &tls_config, validator_identity) .await? .start_services(); - let proxy_addr = "127.0.0.1:11111".parse().unwrap(); - let test_client = QuicTestClient::new_with_endpoint( - proxy_addr, &tls_configuration) - .await? - .start_services(); + // let proxy_addr = "127.0.0.1:11111".parse().unwrap(); + // let test_client = QuicTestClient::new_with_endpoint( + // proxy_addr, &tls_configuration) + // .await? + // .start_services(); let ctrl_c_signal = tokio::signal::ctrl_c(); @@ -50,9 +58,9 @@ pub async fn main() -> anyhow::Result<()> { res = main_services => { bail!("Services quit unexpectedly {res:?}"); }, - res = test_client => { - bail!("Test Client quit unexpectedly {res:?}"); - }, + // res = test_client => { + // bail!("Test Client quit unexpectedly {res:?}"); + // }, _ = ctrl_c_signal => { info!("Received ctrl+c signal"); diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 00f81ba2..fc5ea9ff 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -17,15 +17,12 @@ use solana_sdk::signature::Keypair; use solana_sdk::signer::Signer; use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; -use solana_lite_rpc_core::AnyhowJoinHandle; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; -use solana_lite_rpc_core::leader_schedule::LeaderSchedule; -use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -use solana_lite_rpc_core::quic_connection_utils::QuicConnectionUtils; -use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::{ActiveConnection, CONNECTION_RETRY_COUNT, QUIC_CONNECTION_TIMEOUT}; +use crate::proxy_request_format::TpuForwardingRequest; use crate::tpu_quic_connection::TpuQuicConnection; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; +use crate::util::AnyhowJoinHandle; pub struct QuicForwardProxy { diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs new file mode 100644 index 00000000..4675870d --- /dev/null +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -0,0 +1,71 @@ +use std::fmt; +use std::fmt::Display; +use std::net::{SocketAddr, SocketAddrV4}; +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::transaction::VersionedTransaction; + +/// +/// lite-rpc to proxy wire format +/// compat info: non-public format ATM +/// initial version +const FORMAT_VERSION1: u16 = 2301; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TpuForwardingRequest { + format_version: u16, + tpu_socket_addr: SocketAddr, // TODO is that correct + identity_tpunode: Pubkey, + transactions: Vec, +} + +impl Display for TpuForwardingRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "TpuForwardingRequest for tpu target {} with identity {}", + &self.get_tpu_socket_addr(), &self.get_identity_tpunode()) + } +} + +impl TpuForwardingRequest { + pub fn new(tpu_socket_addr: SocketAddr, identity_tpunode: Pubkey, + transactions: Vec) -> Self { + TpuForwardingRequest { + format_version: FORMAT_VERSION1, + tpu_socket_addr, + identity_tpunode, + transactions, + } + } + + pub fn serialize_wire_format( + &self) -> Vec { + bincode::serialize(&self).expect("Expect to serialize transactions") + } + + // TODO reame + pub fn deserialize_from_raw_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { + let request = bincode::deserialize::(&raw_proxy_request) + .context("deserialize proxy request") + .unwrap(); + + assert_eq!(request.format_version, 2301); + + request + } + + pub fn get_tpu_socket_addr(&self) -> SocketAddr { + self.tpu_socket_addr + } + + pub fn get_identity_tpunode(&self) -> Pubkey { + self.identity_tpunode + } + + pub fn get_transactions(&self) -> Vec { + self.transactions.clone() + } +} + + + diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs new file mode 100644 index 00000000..73b58271 --- /dev/null +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -0,0 +1,309 @@ +use log::{info, trace, warn}; +use quinn::{ + ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, + TokioRuntime, TransportConfig, +}; +use solana_sdk::pubkey::Pubkey; +use std::{ + collections::VecDeque, + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, + }, + time::Duration, +}; +use anyhow::bail; +use tokio::{sync::RwLock, time::timeout}; + +const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; + +pub struct QuicConnectionUtils {} + +impl QuicConnectionUtils { + pub fn create_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { + let mut endpoint = { + let client_socket = + solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::UNSPECIFIED), (8000, 10000)) + .expect("create_endpoint bind_in_range") + .1; + let config = EndpointConfig::default(); + quinn::Endpoint::new(config, None, client_socket, TokioRuntime) + .expect("create_endpoint quinn::Endpoint::new") + }; + + let mut crypto = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(SkipServerVerification::new()) + .with_single_cert(vec![certificate], key) + .expect("Failed to set QUIC client certificates"); + + crypto.enable_early_data = true; + // FIXME TEMP HACK TO ALLOW PROXY PROTOCOL + const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; + + crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec(), ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; + + let mut config = ClientConfig::new(Arc::new(crypto)); + let mut transport_config = TransportConfig::default(); + + // TODO check timing + let timeout = IdleTimeout::try_from(Duration::from_secs(5)).unwrap(); + transport_config.max_idle_timeout(Some(timeout)); + transport_config.keep_alive_interval(Some(Duration::from_millis(500))); + config.transport_config(Arc::new(transport_config)); + + endpoint.set_default_client_config(config); + + endpoint + } + + pub async fn make_connection( + endpoint: Endpoint, + addr: SocketAddr, + connection_timeout: Duration, + ) -> anyhow::Result { + let connecting = endpoint.connect(addr, "connect")?; + let res = timeout(connection_timeout, connecting).await??; + Ok(res) + } + + pub async fn make_connection_0rtt( + endpoint: Endpoint, + addr: SocketAddr, + connection_timeout: Duration, + ) -> anyhow::Result { + let connecting = endpoint.connect(addr, "connect")?; + let connection = match connecting.into_0rtt() { + Ok((connection, zero_rtt)) => { + if (timeout(connection_timeout, zero_rtt).await).is_ok() { + connection + } else { + return Err(ConnectionError::TimedOut.into()); + } + } + Err(connecting) => { + if let Ok(connecting_result) = timeout(connection_timeout, connecting).await { + connecting_result? + } else { + return Err(ConnectionError::TimedOut.into()); + } + } + }; + Ok(connection) + } + + #[allow(clippy::too_many_arguments)] + pub async fn connect( + identity: Pubkey, + already_connected: bool, + endpoint: Endpoint, + tpu_address: SocketAddr, + connection_timeout: Duration, + connection_retry_count: usize, + exit_signal: Arc, + on_connect: fn(), + ) -> Option { + for _ in 0..connection_retry_count { + let conn = if already_connected { + Self::make_connection_0rtt(endpoint.clone(), tpu_address, connection_timeout).await + } else { + Self::make_connection(endpoint.clone(), tpu_address, connection_timeout).await + }; + match conn { + Ok(conn) => { + on_connect(); + return Some(conn); + } + Err(e) => { + warn!("Could not connect to tpu {}/{}, error: {}", tpu_address, identity, e); + if exit_signal.load(Ordering::Relaxed) { + break; + } + } + } + } + None + } + + pub async fn write_all( + mut send_stream: SendStream, + tx: &Vec, + identity: Pubkey, + last_stable_id: Arc, + connection_stable_id: u64, + connection_timeout: Duration, + ) -> bool { + let write_timeout_res = + timeout(connection_timeout, send_stream.write_all(tx.as_slice())).await; + match write_timeout_res { + Ok(write_res) => { + if let Err(e) = write_res { + trace!( + "Error while writing transaction for {}, error {}", + identity, + e + ); + // retry + last_stable_id.store(connection_stable_id, Ordering::Relaxed); + return true; + } + } + Err(_) => { + warn!("timeout while writing transaction for {}", identity); + } + } + + let finish_timeout_res = timeout(connection_timeout, send_stream.finish()).await; + match finish_timeout_res { + Ok(finish_res) => { + if let Err(e) = finish_res { + last_stable_id.store(connection_stable_id, Ordering::Relaxed); + trace!( + "Error while writing transaction for {}, error {}", + identity, + e + ); + return true; + } + } + Err(_) => { + warn!("timeout while finishing transaction for {}", identity); + } + } + + false + } + + pub async fn open_unistream( + connection: Connection, + last_stable_id: Arc, + connection_timeout: Duration, + ) -> (Option, bool) { + match timeout(connection_timeout, connection.open_uni()).await { + Ok(Ok(unistream)) => (Some(unistream), false), + Ok(Err(_)) => { + // reset connection for next retry + last_stable_id.store(connection.stable_id() as u64, Ordering::Relaxed); + (None, true) + } + // timeout + Err(_) => (None, false), + } + } + + #[allow(clippy::too_many_arguments)] + pub async fn send_transaction_batch( + connection: Arc>, + txs: Vec>, + identity: Pubkey, + endpoint: Endpoint, + tpu_address: SocketAddr, + exit_signal: Arc, + last_stable_id: Arc, + connection_timeout: Duration, + connection_retry_count: usize, + on_connect: fn(), + ) { + info!("send transaction batch of size {} to address {}", txs.len(), tpu_address); + let mut queue = VecDeque::new(); + for tx in txs { + queue.push_back(tx); + } + for _ in 0..connection_retry_count { + if queue.is_empty() || exit_signal.load(Ordering::Relaxed) { + // return + return; + } + // get new connection reset if necessary + let conn = { + let last_stable_id = last_stable_id.load(Ordering::Relaxed) as usize; + let conn = connection.read().await; + if conn.stable_id() == last_stable_id { + let current_stable_id = conn.stable_id(); + // problematic connection + drop(conn); + let mut conn = connection.write().await; + // check may be already written by another thread + if conn.stable_id() != current_stable_id { + conn.clone() + } else { + let new_conn = Self::connect( + identity, + true, + endpoint.clone(), + tpu_address, + connection_timeout, + connection_retry_count, + exit_signal.clone(), + on_connect, + ) + .await; + if let Some(new_conn) = new_conn { + *conn = new_conn; + conn.clone() + } else { + // could not connect + return; + } + } + } else { + conn.clone() + } + }; + let mut retry = false; + while !queue.is_empty() { + let tx = queue.pop_front().unwrap(); + let (stream, retry_conn) = + Self::open_unistream(conn.clone(), last_stable_id.clone(), connection_timeout) + .await; + if let Some(send_stream) = stream { + if exit_signal.load(Ordering::Relaxed) { + return; + } + + retry = Self::write_all( + send_stream, + &tx, + identity, + last_stable_id.clone(), + conn.stable_id() as u64, + connection_timeout, + ) + .await; + } else { + retry = retry_conn; + } + if retry { + queue.push_back(tx); + break; + } + } + if !retry { + break; + } + } + } +} + +pub struct SkipServerVerification; + +impl SkipServerVerification { + pub fn new() -> Arc { + Arc::new(Self) + } +} + +impl rustls::client::ServerCertVerifier for SkipServerVerification { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &rustls::ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: std::time::SystemTime, + ) -> Result { + Ok(rustls::client::ServerCertVerified::assertion()) + } +} diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index aedac8f8..e9648696 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -10,12 +10,12 @@ use rustls::ClientConfig; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::{Transaction, VersionedTransaction}; use tokio::io::AsyncWriteExt; -use solana_lite_rpc_core::AnyhowJoinHandle; +use crate::proxy_request_format::TpuForwardingRequest; +use crate::quic_connection_utils::SkipServerVerification; use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; use crate::tls_config_provicer::ProxyTlsConfigProvider; -use solana_lite_rpc_core::quic_connection_utils::SkipServerVerification; -use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; use crate::test_client::sample_data_factory::build_raw_sample_tx; +use crate::util::AnyhowJoinHandle; pub struct QuicTestClient { pub endpoint: Endpoint, diff --git a/quic-forward-proxy/src/tls_config_provicer.rs b/quic-forward-proxy/src/tls_config_provicer.rs index 10109b59..30c219ca 100644 --- a/quic-forward-proxy/src/tls_config_provicer.rs +++ b/quic-forward-proxy/src/tls_config_provicer.rs @@ -1,7 +1,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use rcgen::generate_simple_self_signed; use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; -use solana_lite_rpc_core::quic_connection_utils::SkipServerVerification; +use crate::quic_connection_utils::SkipServerVerification; use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; // TODO integrate with tpu_service + quic_connection_utils diff --git a/quic-forward-proxy/src/tpu_quic_connection.rs b/quic-forward-proxy/src/tpu_quic_connection.rs index c440672d..69ef3182 100644 --- a/quic-forward-proxy/src/tpu_quic_connection.rs +++ b/quic-forward-proxy/src/tpu_quic_connection.rs @@ -16,14 +16,14 @@ use solana_sdk::signature::Keypair; use solana_sdk::signer::Signer; use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; -use solana_lite_rpc_core::AnyhowJoinHandle; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; -use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -use solana_lite_rpc_core::quic_connection_utils::QuicConnectionUtils; -use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::{ActiveConnection, CONNECTION_RETRY_COUNT, QUIC_CONNECTION_TIMEOUT}; +use crate::quic_connection_utils::QuicConnectionUtils; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; +pub const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); +pub const CONNECTION_RETRY_COUNT: usize = 10; + /// stable connect to TPU to send transactions - optimized for proxy use case #[derive(Clone)] pub struct TpuQuicConnection { diff --git a/quic-forward-proxy/src/tx_store.rs b/quic-forward-proxy/src/tx_store.rs new file mode 100644 index 00000000..3300b104 --- /dev/null +++ b/quic-forward-proxy/src/tx_store.rs @@ -0,0 +1,27 @@ +use std::sync::Arc; + +use dashmap::DashMap; +use solana_transaction_status::TransactionStatus; +use tokio::time::Instant; +/// Transaction Properties + +pub struct TxProps { + pub status: Option, + /// Time at which transaction was forwarded + pub sent_at: Instant, +} + +impl Default for TxProps { + fn default() -> Self { + Self { + status: Default::default(), + sent_at: Instant::now(), + } + } +} + +pub type TxStore = Arc>; + +pub fn empty_tx_store() -> TxStore { + Arc::new(DashMap::new()) +} diff --git a/quic-forward-proxy/src/util.rs b/quic-forward-proxy/src/util.rs new file mode 100644 index 00000000..4b6b70fd --- /dev/null +++ b/quic-forward-proxy/src/util.rs @@ -0,0 +1,2 @@ + +pub type AnyhowJoinHandle = tokio::task::JoinHandle>; diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index 14612011..926f4415 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -9,8 +9,7 @@ use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, Signer}; use solana_sdk::transaction::{Transaction, VersionedTransaction}; use spl_memo::solana_program::message::VersionedMessage; -use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -use solana_lite_rpc_core::proxy_request_format::*; +use solana_lite_rpc_quic_forward_proxy::proxy_request_format::TpuForwardingRequest; #[test] fn roundtrip() { From 0490e557deda3b995663e35aeb0207d317c5f023 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 20 Jul 2023 13:45:54 +0200 Subject: [PATCH 018/128] merge integration test branch into proxy branch --- CHANGELOG.md | 14 +- Cargo.lock | 105 +++-- Cargo.toml | 9 +- bench/Cargo.toml | 2 +- bench/src/helpers.rs | 10 +- bench/src/main.rs | 25 +- core/Cargo.toml | 2 +- core/src/block_processor.rs | 12 +- core/src/block_store.rs | 65 ++- core/src/leader_schedule.rs | 13 +- core/src/lib.rs | 1 + core/src/quic_connection.rs | 274 +++++++++++ core/src/quic_connection_utils.rs | 180 ++----- core/src/rotating_queue.rs | 60 ++- core/src/solana_utils.rs | 47 +- core/src/tx_store.rs | 10 +- lite-rpc/Cargo.toml | 2 +- lite-rpc/src/bridge.rs | 91 ++-- lite-rpc/src/cli.rs | 8 +- lite-rpc/src/lib.rs | 2 +- lite-rpc/src/main.rs | 103 ++-- lite-rpc/src/postgres.rs | 18 +- lite-rpc/src/rpc_tester.rs | 41 ++ services/Cargo.toml | 10 +- services/src/block_listenser.rs | 57 +-- services/src/cleaner.rs | 35 +- services/src/prometheus_sync.rs | 11 +- .../src/tpu_utils/tpu_connection_manager.rs | 203 +++----- services/src/tpu_utils/tpu_service.rs | 189 ++++---- services/src/transaction_replayer.rs | 79 ++-- services/src/transaction_service.rs | 164 +++---- services/src/tx_sender.rs | 97 ++-- ...literpc_tpu_quic_server_integrationtest.rs | 444 ++++++++++++++++++ 33 files changed, 1502 insertions(+), 881 deletions(-) create mode 100644 core/src/quic_connection.rs create mode 100644 lite-rpc/src/rpc_tester.rs create mode 100644 services/tests/literpc_tpu_quic_server_integrationtest.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e55fa64..22d3919a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ The minor version will be incremented upon a breaking change and the patch versi ### Features: +## [0.2.2] - 2023-06-23 + +commit : 70eb250b103c64a0e5a3159c9493e87003d046a4 + +- lite-rpc : Added restart logic. +- metrics : added more counters for related to failure of services during restart. +- tpu-client : sending transaction using multiple quic connections. +- tpu-client : removed pubsub of slot and implementing force polling using rpc. + ## [0.2.1] commit: c1eed987f29417f8a3b8d147f43a112388f02e4f @@ -38,4 +47,7 @@ Initial release. - block-listening: A mechanism to get blocks from the RPC and read them to extract transaction data - tpu-client: Mechanisms related to sending transaction to the cluster leaders - postgres: Saving transaction and block data into postgres -- metrics: Updates related to metrics used for graphana and prometheus \ No newline at end of file +- metrics: Updates related to metrics used for graphana and prometheus +- core: Core library, +- services: Services library +- lite-rpc: The lite rpc binary \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 34e29308..22f5d894 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -183,7 +183,7 @@ dependencies = [ "ark-serialize", "ark-std", "derivative", - "num-traits", + "num-traits 0.2.15", "zeroize", ] @@ -199,7 +199,7 @@ dependencies = [ "ark-std", "derivative", "num-bigint 0.4.3", - "num-traits", + "num-traits 0.2.15", "paste", "rustc_version 0.3.3", "zeroize", @@ -222,7 +222,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20" dependencies = [ "num-bigint 0.4.3", - "num-traits", + "num-traits 0.2.15", "quote 1.0.26", "syn 1.0.109", ] @@ -243,7 +243,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" dependencies = [ - "num-traits", + "num-traits 0.2.15", "rand 0.8.5", ] @@ -281,7 +281,7 @@ dependencies = [ "asn1-rs-impl", "displaydoc", "nom", - "num-traits", + "num-traits 0.2.15", "rusticata-macros", "thiserror", "time 0.3.20", @@ -422,7 +422,7 @@ dependencies = [ [[package]] name = "bench" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "clap 4.2.4", @@ -669,7 +669,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-integer", - "num-traits", + "num-traits 0.2.15", "serde", "time 0.1.45", "wasm-bindgen", @@ -896,6 +896,15 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "countmap" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ef2a403c4af585607826502480ab6e453f320c230ef67255eee21f0cc72c0a6" +dependencies = [ + "num-traits 0.1.43", +] + [[package]] name = "cpufeatures" version = "0.2.6" @@ -1144,7 +1153,7 @@ dependencies = [ "displaydoc", "nom", "num-bigint 0.4.3", - "num-traits", + "num-traits 0.2.15", "rusticata-macros", ] @@ -2296,7 +2305,7 @@ checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" [[package]] name = "lite-rpc" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "async-channel", @@ -2322,7 +2331,6 @@ dependencies = [ "serde", "serde_json", "solana-lite-rpc-core", - "solana-lite-rpc-quic-forward-proxy", "solana-lite-rpc-services", "solana-rpc-client", "solana-rpc-client-api", @@ -2354,6 +2362,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "md-5" version = "0.10.5" @@ -2522,7 +2539,7 @@ dependencies = [ "num-integer", "num-iter", "num-rational", - "num-traits", + "num-traits 0.2.15", ] [[package]] @@ -2533,7 +2550,7 @@ checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ "autocfg", "num-integer", - "num-traits", + "num-traits 0.2.15", ] [[package]] @@ -2544,7 +2561,7 @@ checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" dependencies = [ "autocfg", "num-integer", - "num-traits", + "num-traits 0.2.15", ] [[package]] @@ -2554,7 +2571,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" dependencies = [ "autocfg", - "num-traits", + "num-traits 0.2.15", ] [[package]] @@ -2575,7 +2592,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", - "num-traits", + "num-traits 0.2.15", ] [[package]] @@ -2586,7 +2603,7 @@ checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" dependencies = [ "autocfg", "num-integer", - "num-traits", + "num-traits 0.2.15", ] [[package]] @@ -2598,7 +2615,16 @@ dependencies = [ "autocfg", "num-bigint 0.2.6", "num-integer", - "num-traits", + "num-traits 0.2.15", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.15", ] [[package]] @@ -3246,6 +3272,15 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.6.28" @@ -3828,7 +3863,7 @@ dependencies = [ "bytemuck", "log", "num-derive", - "num-traits", + "num-traits 0.2.15", "rustc_version 0.4.0", "serde", "solana-frozen-abi", @@ -3975,7 +4010,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-core" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "async-trait", @@ -4029,10 +4064,10 @@ dependencies = [ "rustls 0.20.8", "serde", "serde_json", - "solana-lite-rpc-core", - "solana-lite-rpc-services", + "solana-net-utils", "solana-sdk", "solana-streamer", + "solana-transaction-status", "spl-memo", "thiserror", "tokio", @@ -4041,7 +4076,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-services" -version = "0.2.1" +version = "0.2.2" dependencies = [ "anyhow", "async-channel", @@ -4050,6 +4085,8 @@ dependencies = [ "bs58", "bytes", "chrono", + "countmap", + "crossbeam-channel", "dashmap", "futures", "itertools", @@ -4072,6 +4109,8 @@ dependencies = [ "solana-version", "thiserror", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -4191,7 +4230,7 @@ dependencies = [ "memoffset 0.8.0", "num-bigint 0.4.3", "num-derive", - "num-traits", + "num-traits 0.2.15", "parking_lot", "rand 0.7.3", "rand_chacha 0.2.2", @@ -4227,7 +4266,7 @@ dependencies = [ "libloading", "log", "num-derive", - "num-traits", + "num-traits 0.2.15", "rand 0.7.3", "rustc_version 0.4.0", "serde", @@ -4313,7 +4352,7 @@ dependencies = [ "dialoguer", "log", "num-derive", - "num-traits", + "num-traits 0.2.15", "parking_lot", "qstring", "semver 1.0.16", @@ -4411,7 +4450,7 @@ dependencies = [ "log", "memmap2", "num-derive", - "num-traits", + "num-traits 0.2.15", "num_enum", "pbkdf2 0.11.0", "qstring", @@ -4589,7 +4628,7 @@ dependencies = [ "bincode", "log", "num-derive", - "num-traits", + "num-traits 0.2.15", "rustc_version 0.4.0", "serde", "serde_derive", @@ -4621,7 +4660,7 @@ dependencies = [ "lazy_static", "merlin", "num-derive", - "num-traits", + "num-traits 0.2.15", "rand 0.7.3", "serde", "serde_json", @@ -4677,7 +4716,7 @@ dependencies = [ "assert_matches", "borsh", "num-derive", - "num-traits", + "num-traits 0.2.15", "solana-program", "spl-token", "spl-token-2022", @@ -4702,7 +4741,7 @@ dependencies = [ "arrayref", "bytemuck", "num-derive", - "num-traits", + "num-traits 0.2.15", "num_enum", "solana-program", "thiserror", @@ -4717,7 +4756,7 @@ dependencies = [ "arrayref", "bytemuck", "num-derive", - "num-traits", + "num-traits 0.2.15", "num_enum", "solana-program", "solana-zk-token-sdk", @@ -5169,10 +5208,14 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] diff --git a/Cargo.toml b/Cargo.toml index 105cef4c..db88042b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -version = "0.2.1" +version = "0.2.2" authors = ["gmgalactus ", "Aniket Prajapati "] repository = "https://github.com/blockworks-foundation/lite-rpc" license = "AGPL" @@ -39,6 +39,7 @@ clap = { version = "4.2.4", features = ["derive"] } dashmap = "5.4.0" const_env = "0.1.2" jsonrpsee = { version = "0.17.0", features = ["macros", "full"] } +tracing = "0.1.27" tracing-subscriber = "0.3.16" chrono = "0.4.24" native-tls = "0.2.11" @@ -49,6 +50,6 @@ dotenv = "0.15.0" async-channel = "1.8.0" quinn = "0.9.3" rustls = { version = "=0.20.8", default-features = false } -solana-lite-rpc-services = {path = "services", version="0.2.1"} -solana-lite-rpc-core = {path = "core", version="0.2.1"} -async-trait = "0.1.68" \ No newline at end of file +solana-lite-rpc-services = {path = "services", version="0.2.2"} +solana-lite-rpc-core = {path = "core", version="0.2.2"} +async-trait = "0.1.68" diff --git a/bench/Cargo.toml b/bench/Cargo.toml index b08ed703..2057dedc 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bench" -version = "0.2.1" +version = "0.2.2" edition = "2021" [dependencies] diff --git a/bench/src/helpers.rs b/bench/src/helpers.rs index b2071c6c..2878752a 100644 --- a/bench/src/helpers.rs +++ b/bench/src/helpers.rs @@ -1,4 +1,5 @@ use anyhow::Context; +use lazy_static::lazy_static; use rand::{distributions::Alphanumeric, prelude::Distribution, SeedableRng}; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{ @@ -12,9 +13,8 @@ use solana_sdk::{ system_instruction, transaction::Transaction, }; -use std::{str::FromStr, time::Duration}; use std::path::PathBuf; -use lazy_static::lazy_static; +use std::{str::FromStr, time::Duration}; use tokio::time::Instant; const MEMO_PROGRAM_ID: &str = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"; @@ -22,20 +22,18 @@ const WAIT_LIMIT_IN_SECONDS: u64 = 60; lazy_static! { static ref USER_KEYPAIR: PathBuf = { - dirs::home_dir().unwrap() + dirs::home_dir() + .unwrap() .join(".config") .join("solana") .join("id.json") }; } - pub struct BenchHelper; impl BenchHelper { - pub async fn get_payer() -> anyhow::Result { - let payer = tokio::fs::read_to_string(USER_KEYPAIR.as_path()) .await .context("Error reading payer file")?; diff --git a/bench/src/main.rs b/bench/src/main.rs index 35e3c920..3b278bda 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -6,7 +6,7 @@ use bench::{ use clap::Parser; use dashmap::DashMap; use futures::future::join_all; -use log::{error, info}; +use log::{error, info, warn}; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{ commitment_config::CommitmentConfig, hash::Hash, signature::Keypair, signer::Signer, @@ -169,15 +169,20 @@ async fn bench( let blockhash = { *block_hash.read().await }; let tx = BenchHelper::create_memo_tx(&rand_string, &funded_payer, blockhash); let start_time = Instant::now(); - if let Ok(signature) = rpc_client.send_transaction(&tx).await { - map_of_txs.insert( - signature, - TxSendData { - sent_duration: start_time.elapsed(), - sent_instant: Instant::now(), - sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), - }, - ); + match rpc_client.send_transaction(&tx).await { + Ok(signature) => { + map_of_txs.insert( + signature, + TxSendData { + sent_duration: start_time.elapsed(), + sent_instant: Instant::now(), + sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), + }, + ); + } + Err(e) => { + warn!("tx send failed with error {}", e); + } } } }); diff --git a/core/Cargo.toml b/core/Cargo.toml index 73568ca6..12f5ee78 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-core" -version = "0.2.1" +version = "0.2.2" edition = "2021" description = "Core classes and methods used by solana lite rpc" rust-version = "1.67.1" diff --git a/core/src/block_processor.rs b/core/src/block_processor.rs index b60886b3..86d80b36 100644 --- a/core/src/block_processor.rs +++ b/core/src/block_processor.rs @@ -1,10 +1,10 @@ -use std::sync::Arc; - +use anyhow::Context; use log::{info, warn}; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_rpc_client_api::config::RpcBlockConfig; use solana_sdk::{ borsh::try_from_slice_unchecked, + clock::MAX_RECENT_BLOCKHASHES, commitment_config::CommitmentConfig, compute_budget::{self, ComputeBudgetInstruction}, slot_history::Slot, @@ -14,7 +14,7 @@ use solana_transaction_status::{ option_serializer::OptionSerializer, RewardType, TransactionDetails, UiTransactionEncoding, UiTransactionStatusMeta, }; -use tokio::time::Instant; +use std::sync::Arc; use crate::block_store::{BlockInformation, BlockStore}; @@ -77,7 +77,8 @@ impl BlockProcessor { rewards: Some(true), }, ) - .await?; + .await + .context("failed to get block")?; let Some(block_height) = block.block_height else { return Ok(BlockProcessorResult::invalid()); @@ -97,7 +98,8 @@ impl BlockProcessor { BlockInformation { slot, block_height, - instant: Instant::now(), + last_valid_blockheight: block_height + MAX_RECENT_BLOCKHASHES as u64, + cleanup_slot: block_height + 1000, processed_local_time: None, }, commitment_config, diff --git a/core/src/block_store.rs b/core/src/block_store.rs index 4250cbd3..1118a462 100644 --- a/core/src/block_store.rs +++ b/core/src/block_store.rs @@ -1,25 +1,27 @@ -use std::sync::Arc; -use std::time::Duration; - use anyhow::Context; use chrono::{DateTime, Utc}; use dashmap::DashMap; - use log::info; use serde_json::json; -use solana_client::rpc_request::RpcRequest; -use solana_client::rpc_response::{Response, RpcBlockhash}; -use solana_client::{nonblocking::rpc_client::RpcClient, rpc_config::RpcBlockConfig}; -use solana_sdk::commitment_config::CommitmentConfig; +use solana_client::{ + nonblocking::rpc_client::RpcClient, + rpc_config::RpcBlockConfig, + rpc_request::RpcRequest, + rpc_response::{Response, RpcBlockhash}, +}; +use solana_sdk::{ + clock::MAX_RECENT_BLOCKHASHES, commitment_config::CommitmentConfig, slot_history::Slot, +}; use solana_transaction_status::TransactionDetails; -use tokio::sync::RwLock; -use tokio::time::Instant; +use std::sync::Arc; +use tokio::{sync::RwLock, time::Instant}; #[derive(Clone, Copy, Debug)] pub struct BlockInformation { pub slot: u64, pub block_height: u64, - pub instant: Instant, + pub last_valid_blockheight: u64, + pub cleanup_slot: Slot, pub processed_local_time: Option>, } @@ -66,14 +68,19 @@ impl BlockStore { RpcRequest::GetLatestBlockhash, json!([commitment_config]), ) - .await?; + .await + .context("failed to poll latest blockhash")?; let processed_blockhash = response.value.blockhash; let processed_block = BlockInformation { slot: response.context.slot, - block_height: response.value.last_valid_block_height, + last_valid_blockheight: response.value.last_valid_block_height, + block_height: response + .value + .last_valid_block_height + .saturating_sub(MAX_RECENT_BLOCKHASHES as u64), processed_local_time: Some(Utc::now()), - instant: Instant::now(), + cleanup_slot: response.value.last_valid_block_height + 700, // cleanup after 1000 slots }; Ok((processed_blockhash, processed_block)) @@ -85,7 +92,8 @@ impl BlockStore { ) -> anyhow::Result<(String, BlockInformation)> { let slot = rpc_client .get_slot_with_commitment(commitment_config) - .await?; + .await + .context("failed to fetch latest slot")?; let block = rpc_client .get_block_with_config( @@ -98,7 +106,8 @@ impl BlockStore { max_supported_transaction_version: Some(0), }, ) - .await?; + .await + .context("failed to fetch latest blockhash")?; let latest_block_hash = block.blockhash; let block_height = block @@ -110,7 +119,8 @@ impl BlockStore { BlockInformation { slot, block_height, - instant: Instant::now(), + last_valid_blockheight: block_height + MAX_RECENT_BLOCKHASHES as u64, + cleanup_slot: block_height + 1000, processed_local_time: None, }, )) @@ -194,24 +204,13 @@ impl BlockStore { } } - pub async fn clean(&self, cleanup_duration: Duration) { - let latest_processed = self - .get_latest_blockhash(CommitmentConfig::processed()) - .await; - let latest_confirmed = self - .get_latest_blockhash(CommitmentConfig::confirmed()) + pub async fn clean(&self) { + let finalized_block_information = self + .get_latest_block_info(CommitmentConfig::finalized()) .await; - let latest_finalized = self - .get_latest_blockhash(CommitmentConfig::finalized()) - .await; - let before_length = self.blocks.len(); - self.blocks.retain(|k, v| { - v.instant.elapsed() < cleanup_duration - || k.eq(&latest_processed) - || k.eq(&latest_confirmed) - || k.eq(&latest_finalized) - }); + self.blocks + .retain(|_, v| v.last_valid_blockheight >= finalized_block_information.block_height); info!( "Cleaned {} block info", diff --git a/core/src/leader_schedule.rs b/core/src/leader_schedule.rs index e2dd222b..cb3c52ad 100644 --- a/core/src/leader_schedule.rs +++ b/core/src/leader_schedule.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use std::{collections::VecDeque, str::FromStr, sync::Arc}; use dashmap::DashMap; @@ -36,7 +37,10 @@ impl LeaderSchedule { } pub async fn load_cluster_info(&self, rpc_client: Arc) -> anyhow::Result<()> { - let cluster_nodes = rpc_client.get_cluster_nodes().await?; + let cluster_nodes = rpc_client + .get_cluster_nodes() + .await + .context("failed to get cluster nodes")?; cluster_nodes.iter().for_each(|x| { if let Ok(pubkey) = Pubkey::from_str(x.pubkey.as_str()) { self.cluster_nodes.insert(pubkey, Arc::new(x.clone())); @@ -70,14 +74,17 @@ impl LeaderSchedule { let first_slot_to_fetch = queue_end_slot + 1; let leaders = rpc_client .get_slot_leaders(first_slot_to_fetch, last_slot_needed - first_slot_to_fetch) - .await?; + .await + .context("failed to get slot leaders")?; let mut leader_queue = self.leader_schedule.write().await; for i in first_slot_to_fetch..last_slot_needed { let current_leader = (i - first_slot_to_fetch) as usize; let leader = leaders[current_leader]; if !self.cluster_nodes.contains_key(&leader) { - self.load_cluster_info(rpc_client.clone()).await?; + self.load_cluster_info(rpc_client.clone()) + .await + .context("failed to load cluster info")?; } match self.cluster_nodes.get(&leader) { diff --git a/core/src/lib.rs b/core/src/lib.rs index 0f22a098..06060784 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -2,6 +2,7 @@ pub mod block_processor; pub mod block_store; pub mod leader_schedule; pub mod notifications; +pub mod quic_connection; pub mod quic_connection_utils; pub mod rotating_queue; pub mod solana_utils; diff --git a/core/src/quic_connection.rs b/core/src/quic_connection.rs new file mode 100644 index 00000000..8825f884 --- /dev/null +++ b/core/src/quic_connection.rs @@ -0,0 +1,274 @@ +use crate::{ + quic_connection_utils::{QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils}, + rotating_queue::RotatingQueue, +}; +use anyhow::bail; +use log::warn; +use quinn::{Connection, Endpoint}; +use solana_sdk::pubkey::Pubkey; +use std::{ + collections::VecDeque, + net::SocketAddr, + sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, + }, +}; +use tokio::sync::RwLock; + +pub type EndpointPool = RotatingQueue; + +#[derive(Clone)] +pub struct QuicConnection { + connection: Arc>, + last_stable_id: Arc, + endpoint: Endpoint, + identity: Pubkey, + socket_address: SocketAddr, + connection_params: QuicConnectionParameters, + exit_signal: Arc, + timeout_counters: Arc, +} + +impl QuicConnection { + pub async fn new( + identity: Pubkey, + endpoints: EndpointPool, + socket_address: SocketAddr, + connection_params: QuicConnectionParameters, + exit_signal: Arc, + ) -> anyhow::Result { + let endpoint = endpoints + .get() + .await + .expect("endpoint pool is not suppose to be empty"); + let connection = QuicConnectionUtils::connect( + identity, + false, + endpoint.clone(), + socket_address, + connection_params.connection_timeout, + connection_params.connection_retry_count, + exit_signal.clone(), + ) + .await; + + match connection { + Some(connection) => Ok(Self { + connection: Arc::new(RwLock::new(connection)), + last_stable_id: Arc::new(AtomicU64::new(0)), + endpoint, + identity, + socket_address, + connection_params, + exit_signal, + timeout_counters: Arc::new(AtomicU64::new(0)), + }), + None => { + bail!("Could not establish connection"); + } + } + } + + async fn get_connection(&self) -> Option { + // get new connection reset if necessary + let last_stable_id = self.last_stable_id.load(Ordering::Relaxed) as usize; + let conn = self.connection.read().await; + if conn.stable_id() == last_stable_id { + let current_stable_id = conn.stable_id(); + // problematic connection + drop(conn); + let mut conn = self.connection.write().await; + // check may be already written by another thread + if conn.stable_id() != current_stable_id { + Some(conn.clone()) + } else { + let new_conn = QuicConnectionUtils::connect( + self.identity, + true, + self.endpoint.clone(), + self.socket_address, + self.connection_params.connection_timeout, + self.connection_params.connection_retry_count, + self.exit_signal.clone(), + ) + .await; + if let Some(new_conn) = new_conn { + *conn = new_conn; + Some(conn.clone()) + } else { + // could not connect + None + } + } + } else { + Some(conn.clone()) + } + } + + pub async fn send_transaction_batch(&self, txs: Vec>) { + let mut queue = VecDeque::new(); + for tx in txs { + queue.push_back(tx); + } + let connection_retry_count = self.connection_params.connection_retry_count; + for _ in 0..connection_retry_count { + if queue.is_empty() || self.exit_signal.load(Ordering::Relaxed) { + // return + return; + } + + let mut do_retry = false; + while !queue.is_empty() { + let tx = queue.pop_front().unwrap(); + let connection = self.get_connection().await; + + if self.exit_signal.load(Ordering::Relaxed) { + return; + } + + if let Some(connection) = connection { + let current_stable_id = connection.stable_id() as u64; + match QuicConnectionUtils::open_unistream( + connection, + self.connection_params.unistream_timeout, + ) + .await + { + Ok(send_stream) => { + match QuicConnectionUtils::write_all( + send_stream, + &tx, + self.identity, + self.connection_params, + ) + .await + { + Ok(()) => { + // do nothing + } + Err(QuicConnectionError::ConnectionError { retry }) => { + do_retry = retry; + } + Err(QuicConnectionError::TimeOut) => { + self.timeout_counters.fetch_add(1, Ordering::Relaxed); + } + } + } + Err(QuicConnectionError::ConnectionError { retry }) => { + do_retry = retry; + } + Err(QuicConnectionError::TimeOut) => { + self.timeout_counters.fetch_add(1, Ordering::Relaxed); + } + } + if do_retry { + self.last_stable_id + .store(current_stable_id, Ordering::Relaxed); + queue.push_back(tx); + break; + } + } else { + warn!( + "Could not establish connection with {}", + self.identity.to_string() + ); + break; + } + } + if !do_retry { + break; + } + } + } + + pub fn get_timeout_count(&self) -> u64 { + self.timeout_counters.load(Ordering::Relaxed) + } + + pub fn reset_timeouts(&self) { + self.timeout_counters.store(0, Ordering::Relaxed); + } +} + +#[derive(Clone)] +pub struct QuicConnectionPool { + connections: RotatingQueue, + connection_parameters: QuicConnectionParameters, + endpoints: EndpointPool, + identity: Pubkey, + socket_address: SocketAddr, + exit_signal: Arc, +} + +impl QuicConnectionPool { + pub fn new( + identity: Pubkey, + endpoints: EndpointPool, + socket_address: SocketAddr, + connection_parameters: QuicConnectionParameters, + exit_signal: Arc, + ) -> Self { + let connections = RotatingQueue::new_empty(); + Self { + connections, + identity, + endpoints, + socket_address, + connection_parameters, + exit_signal, + } + } + + pub async fn send_transaction_batch(&self, txs: Vec>) { + let connection = match self.connections.get().await { + Some(connection) => connection, + None => { + let new_connection = QuicConnection::new( + self.identity, + self.endpoints.clone(), + self.socket_address, + self.connection_parameters, + self.exit_signal.clone(), + ) + .await; + if new_connection.is_err() { + return; + } + let new_connection = new_connection.expect("Cannot establish a connection"); + self.connections.add(new_connection.clone()).await; + new_connection + } + }; + + connection.send_transaction_batch(txs).await; + } + + pub async fn add_connection(&self) { + let new_connection = QuicConnection::new( + self.identity, + self.endpoints.clone(), + self.socket_address, + self.connection_parameters, + self.exit_signal.clone(), + ) + .await; + if let Ok(new_connection) = new_connection { + self.connections.add(new_connection).await; + } + } + + pub async fn remove_connection(&self) { + if !self.connections.is_empty() { + self.connections.remove().await; + } + } + + pub fn len(&self) -> usize { + self.connections.len() + } + + pub fn is_empty(&self) -> bool { + self.connections.is_empty() + } +} diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index 73b58271..055d2aa1 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -1,23 +1,37 @@ -use log::{info, trace, warn}; +use log::{trace, warn}; use quinn::{ ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, }; use solana_sdk::pubkey::Pubkey; use std::{ - collections::VecDeque, net::{IpAddr, Ipv4Addr, SocketAddr}, sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, Ordering}, Arc, }, time::Duration, }; -use anyhow::bail; -use tokio::{sync::RwLock, time::timeout}; +use tokio::time::timeout; const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; +pub enum QuicConnectionError { + TimeOut, + ConnectionError { retry: bool }, +} + +#[derive(Clone, Copy)] +pub struct QuicConnectionParameters { + pub connection_timeout: Duration, + pub unistream_timeout: Duration, + pub write_timeout: Duration, + pub finalize_timeout: Duration, + pub connection_retry_count: usize, + pub max_number_of_connections: usize, + pub number_of_transactions_per_unistream: usize, +} + pub struct QuicConnectionUtils {} impl QuicConnectionUtils { @@ -39,16 +53,12 @@ impl QuicConnectionUtils { .expect("Failed to set QUIC client certificates"); crypto.enable_early_data = true; - // FIXME TEMP HACK TO ALLOW PROXY PROTOCOL - const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; - - crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec(), ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; + crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; let mut config = ClientConfig::new(Arc::new(crypto)); let mut transport_config = TransportConfig::default(); - // TODO check timing - let timeout = IdleTimeout::try_from(Duration::from_secs(5)).unwrap(); + let timeout = IdleTimeout::try_from(Duration::from_secs(1)).unwrap(); transport_config.max_idle_timeout(Some(timeout)); transport_config.keep_alive_interval(Some(Duration::from_millis(500))); config.transport_config(Arc::new(transport_config)); @@ -98,25 +108,23 @@ impl QuicConnectionUtils { identity: Pubkey, already_connected: bool, endpoint: Endpoint, - tpu_address: SocketAddr, + addr: SocketAddr, connection_timeout: Duration, connection_retry_count: usize, exit_signal: Arc, - on_connect: fn(), ) -> Option { for _ in 0..connection_retry_count { let conn = if already_connected { - Self::make_connection_0rtt(endpoint.clone(), tpu_address, connection_timeout).await + Self::make_connection_0rtt(endpoint.clone(), addr, connection_timeout).await } else { - Self::make_connection(endpoint.clone(), tpu_address, connection_timeout).await + Self::make_connection(endpoint.clone(), addr, connection_timeout).await }; match conn { Ok(conn) => { - on_connect(); return Some(conn); } Err(e) => { - warn!("Could not connect to tpu {}/{}, error: {}", tpu_address, identity, e); + trace!("Could not connect to {} because of error {}", identity, e); if exit_signal.load(Ordering::Relaxed) { break; } @@ -130,12 +138,13 @@ impl QuicConnectionUtils { mut send_stream: SendStream, tx: &Vec, identity: Pubkey, - last_stable_id: Arc, - connection_stable_id: u64, - connection_timeout: Duration, - ) -> bool { - let write_timeout_res = - timeout(connection_timeout, send_stream.write_all(tx.as_slice())).await; + connection_params: QuicConnectionParameters, + ) -> Result<(), QuicConnectionError> { + let write_timeout_res = timeout( + connection_params.write_timeout, + send_stream.write_all(tx.as_slice()), + ) + .await; match write_timeout_res { Ok(write_res) => { if let Err(e) = write_res { @@ -144,149 +153,50 @@ impl QuicConnectionUtils { identity, e ); - // retry - last_stable_id.store(connection_stable_id, Ordering::Relaxed); - return true; + return Err(QuicConnectionError::ConnectionError { retry: true }); } } Err(_) => { warn!("timeout while writing transaction for {}", identity); + return Err(QuicConnectionError::TimeOut); } } - let finish_timeout_res = timeout(connection_timeout, send_stream.finish()).await; + let finish_timeout_res = + timeout(connection_params.finalize_timeout, send_stream.finish()).await; match finish_timeout_res { Ok(finish_res) => { if let Err(e) = finish_res { - last_stable_id.store(connection_stable_id, Ordering::Relaxed); trace!( - "Error while writing transaction for {}, error {}", + "Error while finishing transaction for {}, error {}", identity, e ); - return true; + return Err(QuicConnectionError::ConnectionError { retry: false }); } } Err(_) => { warn!("timeout while finishing transaction for {}", identity); + return Err(QuicConnectionError::TimeOut); } } - false + Ok(()) } pub async fn open_unistream( connection: Connection, - last_stable_id: Arc, connection_timeout: Duration, - ) -> (Option, bool) { + ) -> Result { match timeout(connection_timeout, connection.open_uni()).await { - Ok(Ok(unistream)) => (Some(unistream), false), - Ok(Err(_)) => { - // reset connection for next retry - last_stable_id.store(connection.stable_id() as u64, Ordering::Relaxed); - (None, true) - } - // timeout - Err(_) => (None, false), - } - } - - #[allow(clippy::too_many_arguments)] - pub async fn send_transaction_batch( - connection: Arc>, - txs: Vec>, - identity: Pubkey, - endpoint: Endpoint, - tpu_address: SocketAddr, - exit_signal: Arc, - last_stable_id: Arc, - connection_timeout: Duration, - connection_retry_count: usize, - on_connect: fn(), - ) { - info!("send transaction batch of size {} to address {}", txs.len(), tpu_address); - let mut queue = VecDeque::new(); - for tx in txs { - queue.push_back(tx); - } - for _ in 0..connection_retry_count { - if queue.is_empty() || exit_signal.load(Ordering::Relaxed) { - // return - return; - } - // get new connection reset if necessary - let conn = { - let last_stable_id = last_stable_id.load(Ordering::Relaxed) as usize; - let conn = connection.read().await; - if conn.stable_id() == last_stable_id { - let current_stable_id = conn.stable_id(); - // problematic connection - drop(conn); - let mut conn = connection.write().await; - // check may be already written by another thread - if conn.stable_id() != current_stable_id { - conn.clone() - } else { - let new_conn = Self::connect( - identity, - true, - endpoint.clone(), - tpu_address, - connection_timeout, - connection_retry_count, - exit_signal.clone(), - on_connect, - ) - .await; - if let Some(new_conn) = new_conn { - *conn = new_conn; - conn.clone() - } else { - // could not connect - return; - } - } - } else { - conn.clone() - } - }; - let mut retry = false; - while !queue.is_empty() { - let tx = queue.pop_front().unwrap(); - let (stream, retry_conn) = - Self::open_unistream(conn.clone(), last_stable_id.clone(), connection_timeout) - .await; - if let Some(send_stream) = stream { - if exit_signal.load(Ordering::Relaxed) { - return; - } - - retry = Self::write_all( - send_stream, - &tx, - identity, - last_stable_id.clone(), - conn.stable_id() as u64, - connection_timeout, - ) - .await; - } else { - retry = retry_conn; - } - if retry { - queue.push_back(tx); - break; - } - } - if !retry { - break; - } + Ok(Ok(unistream)) => Ok(unistream), + Ok(Err(_)) => Err(QuicConnectionError::ConnectionError { retry: true }), + Err(_) => Err(QuicConnectionError::TimeOut), } } } -pub struct SkipServerVerification; +struct SkipServerVerification; impl SkipServerVerification { pub fn new() -> Arc { diff --git a/core/src/rotating_queue.rs b/core/src/rotating_queue.rs index d0a24a4d..5280718b 100644 --- a/core/src/rotating_queue.rs +++ b/core/src/rotating_queue.rs @@ -1,34 +1,74 @@ use std::{ collections::VecDeque, - sync::{Arc, RwLock}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, }; +use tokio::sync::Mutex; #[derive(Clone)] pub struct RotatingQueue { - deque: Arc>>, + deque: Arc>>, + count: Arc, } impl RotatingQueue { - pub fn new(size: usize, creator_functor: F) -> Self + pub async fn new(size: usize, creator_functor: F) -> Self where F: Fn() -> T, { let item = Self { - deque: Arc::new(RwLock::new(VecDeque::::new())), + deque: Arc::new(Mutex::new(VecDeque::::new())), + count: Arc::new(AtomicU64::new(0)), }; { - let mut deque = item.deque.write().unwrap(); + let mut deque = item.deque.lock().await; for _i in 0..size { deque.push_back(creator_functor()); } + item.count.store(size as u64, Ordering::Relaxed); } item } - pub fn get(&self) -> T { - let mut deque = self.deque.write().unwrap(); - let current = deque.pop_front().unwrap(); - deque.push_back(current.clone()); - current + pub fn new_empty() -> Self { + Self { + deque: Arc::new(Mutex::new(VecDeque::::new())), + count: Arc::new(AtomicU64::new(0)), + } + } + + pub async fn get(&self) -> Option { + let mut deque = self.deque.lock().await; + if !deque.is_empty() { + let current = deque.pop_front().unwrap(); + deque.push_back(current.clone()); + Some(current) + } else { + None + } + } + + pub async fn add(&self, instance: T) { + let mut queue = self.deque.lock().await; + queue.push_front(instance); + self.count.fetch_add(1, Ordering::Relaxed); + } + + pub async fn remove(&self) { + if !self.is_empty() { + let mut queue = self.deque.lock().await; + queue.pop_front(); + self.count.fetch_sub(1, Ordering::Relaxed); + } + } + + pub fn len(&self) -> usize { + self.count.load(Ordering::Relaxed) as usize + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 } } diff --git a/core/src/solana_utils.rs b/core/src/solana_utils.rs index 44c6fb59..21bdc860 100644 --- a/core/src/solana_utils.rs +++ b/core/src/solana_utils.rs @@ -1,8 +1,6 @@ use crate::structures::identity_stakes::IdentityStakes; use anyhow::Context; -use futures::StreamExt; -use log::{info, warn}; -use solana_pubsub_client::nonblocking::pubsub_client::PubsubClient; +use log::info; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::pubkey::Pubkey; use solana_streamer::nonblocking::quic::ConnectionPeerType; @@ -18,7 +16,7 @@ use tokio::sync::mpsc::UnboundedReceiver; const AVERAGE_SLOT_CHANGE_TIME_IN_MILLIS: u64 = 400; -pub struct SolanaUtils {} +pub struct SolanaUtils; impl SolanaUtils { pub async fn get_stakes_for_identity( @@ -61,41 +59,20 @@ impl SolanaUtils { } pub async fn poll_slots( - rpc_client: Arc, - rpc_ws_address: &str, + rpc_client: &RpcClient, update_slot: impl Fn(u64), ) -> anyhow::Result<()> { - let pubsub_client = PubsubClient::new(rpc_ws_address) - .await - .context("Error creating pubsub client")?; - - let slot = rpc_client - .get_slot_with_commitment(solana_sdk::commitment_config::CommitmentConfig { - commitment: solana_sdk::commitment_config::CommitmentLevel::Processed, - }) - .await - .context("error getting slot")?; - - update_slot(slot); - - let (mut client, unsub) = - tokio::time::timeout(Duration::from_millis(1000), pubsub_client.slot_subscribe()) + let mut poll_frequency = tokio::time::interval(Duration::from_millis(50)); + loop { + let slot = rpc_client + .get_slot_with_commitment(solana_sdk::commitment_config::CommitmentConfig { + commitment: solana_sdk::commitment_config::CommitmentLevel::Processed, + }) .await - .context("timedout subscribing to slots")? - .context("slot pub sub disconnected")?; - - while let Ok(slot_info) = - tokio::time::timeout(Duration::from_millis(2000), client.next()).await - { - if let Some(slot_info) = slot_info { - update_slot(slot_info.slot); - } + .context("Error getting slot")?; + update_slot(slot); + poll_frequency.tick().await; } - - warn!("slot pub sub disconnected reconnecting"); - unsub(); - - Ok(()) } // Estimates the slots, either from polled slot or by forcefully updating after every 400ms diff --git a/core/src/tx_store.rs b/core/src/tx_store.rs index 3300b104..a3f5c1a6 100644 --- a/core/src/tx_store.rs +++ b/core/src/tx_store.rs @@ -2,20 +2,18 @@ use std::sync::Arc; use dashmap::DashMap; use solana_transaction_status::TransactionStatus; -use tokio::time::Instant; /// Transaction Properties pub struct TxProps { pub status: Option, - /// Time at which transaction was forwarded - pub sent_at: Instant, + pub last_valid_blockheight: u64, } -impl Default for TxProps { - fn default() -> Self { +impl TxProps { + pub fn new(last_valid_blockheight: u64) -> Self { Self { status: Default::default(), - sent_at: Instant::now(), + last_valid_blockheight, } } } diff --git a/lite-rpc/Cargo.toml b/lite-rpc/Cargo.toml index 694be264..da158be7 100644 --- a/lite-rpc/Cargo.toml +++ b/lite-rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lite-rpc" -version = "0.2.1" +version = "0.2.2" edition = "2021" description = "A lite version of solana rpc to send and confirm transactions" rust-version = "1.67.1" diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index 10b23e44..e0370c2e 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -10,19 +10,20 @@ use solana_lite_rpc_services::{ block_listenser::BlockListener, metrics_capture::MetricsCapture, prometheus_sync::PrometheusSync, - tpu_utils::tpu_service::TpuService, + tpu_utils::tpu_service::{TpuService, TpuServiceConfig}, transaction_replayer::TransactionReplayer, transaction_service::{TransactionService, TransactionServiceBuilder}, tx_sender::WireTransaction, tx_sender::{TxSender, TXS_IN_CHANNEL}, }; -use anyhow::bail; +use anyhow::{bail, Context}; use jsonrpsee::{core::SubscriptionResult, server::ServerBuilder, PendingSubscriptionSink}; -use log::info; +use log::{error, info}; use prometheus::{opts, register_int_counter, IntCounter}; use solana_lite_rpc_core::{ block_store::{BlockInformation, BlockStore}, + quic_connection_utils::QuicConnectionParameters, tx_store::{empty_tx_store, TxStore}, AnyhowJoinHandle, }; @@ -75,23 +76,43 @@ pub struct LiteBridge { impl LiteBridge { pub async fn new( rpc_url: String, - ws_addr: String, + _ws_addr: String, fanout_slots: u64, validator_identity: Arc, retry_after: Duration, max_retries: usize, ) -> anyhow::Result { let rpc_client = Arc::new(RpcClient::new(rpc_url.clone())); - let current_slot = rpc_client.get_slot().await?; + let current_slot = rpc_client + .get_slot() + .await + .context("failed to get initial slot")?; let tx_store = empty_tx_store(); + let tpu_config = TpuServiceConfig { + fanout_slots, + number_of_leaders_to_cache: 1024, + clusterinfo_refresh_time: Duration::from_secs(60 * 60), + leader_schedule_update_frequency: Duration::from_secs(10), + maximum_transaction_in_queue: 20000, + maximum_number_of_errors: 10, + quic_connection_params: QuicConnectionParameters { + connection_timeout: Duration::from_secs(1), + connection_retry_count: 10, + finalize_timeout: Duration::from_millis(200), + max_number_of_connections: 10, + unistream_timeout: Duration::from_millis(500), + write_timeout: Duration::from_secs(1), + number_of_transactions_per_unistream: 8, + }, + }; + let tpu_service = TpuService::new( + tpu_config, + Arc::new(identity), current_slot, - fanout_slots, - validator_identity, rpc_client.clone(), - ws_addr, tx_store.clone(), ) .await?; @@ -132,7 +153,6 @@ impl LiteBridge { mut self, http_addr: T, ws_addr: T, - clean_interval: Duration, enable_postgres: bool, prometheus_addr: T, ) -> anyhow::Result<()> { @@ -147,19 +167,16 @@ impl LiteBridge { }; let metrics_capture = MetricsCapture::new(self.tx_store.clone()).capture(); - let prometheus_sync = PrometheusSync.sync(prometheus_addr); + let prometheus_sync = PrometheusSync::sync(prometheus_addr); - let max_retries = self.max_retries; + // transaction services let (transaction_service, jh_transaction_services) = self .transaction_service_builder - .start( - postgres_send, - self.block_store.clone(), - max_retries, - clean_interval, - ) - .await; + .clone() + .start(postgres_send, self.block_store.clone(), self.max_retries); + self.transaction_service = Some(transaction_service); + let rpc = self.into_rpc(); let (ws_server, http_server) = { @@ -178,12 +195,14 @@ impl LiteBridge { let ws_server: AnyhowJoinHandle = tokio::spawn(async move { info!("Websocket Server started at {ws_addr:?}"); ws_server_handle.stopped().await; + error!("Websocket server stopped"); bail!("Websocket server stopped"); }); let http_server: AnyhowJoinHandle = tokio::spawn(async move { info!("HTTP Server started at {http_addr:?}"); http_server_handle.stopped().await; + error!("HTTP server stopped"); bail!("HTTP server stopped"); }); @@ -196,27 +215,29 @@ impl LiteBridge { unreachable!(); }; - postgres.await + let res = postgres.await; + error!("postgres server stopped"); + res }); tokio::select! { res = ws_server => { - bail!("WebSocket server exited unexpectedly {res:?}"); + bail!("WebSocket server {res:?}"); }, res = http_server => { - bail!("HTTP server exited unexpectedly {res:?}"); + bail!("HTTP server {res:?}"); }, res = metrics_capture => { - bail!("Metrics Capture exited unexpectedly {res:?}"); + bail!("Metrics Capture {res:?}"); }, res = prometheus_sync => { - bail!("Prometheus Service exited unexpectedly {res:?}"); + bail!("Prometheus Service {res:?}"); }, res = postgres => { - bail!("Postgres service exited unexpectedly {res:?}"); + bail!("Postgres service {res:?}"); }, res = jh_transaction_services => { - bail!("Transaction service exited unexpectedly {res:?}"); + bail!("Transaction service {res:?}"); } } } @@ -313,6 +334,7 @@ impl LiteRpcServer for LiteBridge { .rpc_client .is_blockhash_valid(&blockhash, commitment) .await + .context("failed to get blockhash validity") { Ok(is_valid) => is_valid, Err(err) => { @@ -389,16 +411,27 @@ impl LiteRpcServer for LiteBridge { .rpc_client .request_airdrop_with_config(&pubkey, lamports, config.unwrap_or_default()) .await + .context("failed to request airdrop") { Ok(airdrop_sig) => airdrop_sig.to_string(), Err(err) => { return Err(jsonrpsee::core::Error::Custom(err.to_string())); } }; - - self.tx_store - .insert(airdrop_sig.clone(), Default::default()); - + if let Ok((_, block_height)) = self + .rpc_client + .get_latest_blockhash_with_commitment(CommitmentConfig::finalized()) + .await + .context("failed to get latest blockhash") + { + self.tx_store.insert( + airdrop_sig.clone(), + solana_lite_rpc_core::tx_store::TxProps { + status: None, + last_valid_blockheight: block_height, + }, + ); + } Ok(airdrop_sig) } diff --git a/lite-rpc/src/cli.rs b/lite-rpc/src/cli.rs index cc93aba5..fb6c7f81 100644 --- a/lite-rpc/src/cli.rs +++ b/lite-rpc/src/cli.rs @@ -1,10 +1,9 @@ use crate::{ - DEFAULT_CLEAN_INTERVAL_MS, DEFAULT_FANOUT_SIZE, DEFAULT_RETRY_TIMEOUT, DEFAULT_RPC_ADDR, - DEFAULT_WS_ADDR, MAX_RETRIES, + DEFAULT_FANOUT_SIZE, DEFAULT_RETRY_TIMEOUT, DEFAULT_RPC_ADDR, DEFAULT_WS_ADDR, MAX_RETRIES, }; use clap::Parser; -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Clone)] #[command(author, version, about, long_about = None)] pub struct Args { #[arg(short, long, default_value_t = String::from(DEFAULT_RPC_ADDR))] @@ -18,9 +17,6 @@ pub struct Args { /// tpu fanout #[arg(short = 'f', long, default_value_t = DEFAULT_FANOUT_SIZE) ] pub fanout_size: u64, - /// interval between clean - #[arg(short = 'c', long, default_value_t = DEFAULT_CLEAN_INTERVAL_MS)] - pub clean_interval_ms: u64, /// enable logging to postgres #[arg(short = 'p', long)] pub enable_postgres: bool, diff --git a/lite-rpc/src/lib.rs b/lite-rpc/src/lib.rs index e77b9214..2e36b5f6 100644 --- a/lite-rpc/src/lib.rs +++ b/lite-rpc/src/lib.rs @@ -18,7 +18,7 @@ pub const DEFAULT_LITE_RPC_ADDR: &str = "http://0.0.0.0:8890"; pub const DEFAULT_WS_ADDR: &str = "ws://0.0.0.0:8900"; #[from_env] -pub const DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE: usize = 40_000; +pub const DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE: usize = 200_000; /// 25 slots in 10s send to little more leaders #[from_env] diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index b4afa191..3635197d 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -1,18 +1,18 @@ +pub mod rpc_tester; + use std::time::Duration; -use anyhow::bail; +use anyhow::{bail, Context}; use clap::Parser; use dotenv::dotenv; use lite_rpc::{bridge::LiteBridge, cli::Args}; -use log::info; + use solana_sdk::signature::Keypair; use std::env; -use std::sync::Arc; -use tokio::time::timeout; -// use lite_rpc_quic_forward_proxy::tls_config::SelfSignedTlsConfigProvider; -// note: copy of this method is used in quic-forward-proxy -async fn get_identity_keypair(identity_from_cli: &String) -> Keypair { +use crate::rpc_tester::RpcTester; + +async fn get_identity_keypair(identity_from_cli: &str) -> Keypair { if let Ok(identity_env_var) = env::var("IDENTITY") { if let Ok(identity_bytes) = serde_json::from_str::>(identity_env_var.as_str()) { Keypair::from_bytes(identity_bytes.as_slice()).unwrap() @@ -27,7 +27,7 @@ async fn get_identity_keypair(identity_from_cli: &String) -> Keypair { } else if identity_from_cli.is_empty() { Keypair::new() } else { - let identity_file = tokio::fs::read_to_string(identity_from_cli.as_str()) + let identity_file = tokio::fs::read_to_string(identity_from_cli) .await .expect("Cannot find the identity file provided"); let identity_bytes: Vec = serde_json::from_str(&identity_file).unwrap(); @@ -35,82 +35,81 @@ async fn get_identity_keypair(identity_from_cli: &String) -> Keypair { } } -#[tokio::main(flavor = "multi_thread", worker_threads = 16)] -pub async fn main() -> anyhow::Result<()> { - tracing_subscriber::fmt::init(); - +pub async fn start_lite_rpc(args: Args) -> anyhow::Result<()> { let Args { rpc_addr, ws_addr, lite_rpc_ws_addr, lite_rpc_http_addr, - clean_interval_ms, fanout_size, enable_postgres, prometheus_addr, identity_keypair, maximum_retries_per_tx, transaction_retry_after_secs, - } = Args::parse(); - - dotenv().ok(); - - let validator_identity = Arc::new(get_identity_keypair(&identity_keypair).await); + } = args; - let clean_interval_ms = Duration::from_millis(clean_interval_ms); - - let enable_postgres = enable_postgres - || if let Ok(enable_postgres_env_var) = env::var("PG_ENABLED") { - enable_postgres_env_var != "false" - } else { - false - }; + let identity = get_identity_keypair(&identity_keypair).await; let retry_after = Duration::from_secs(transaction_retry_after_secs); - // let proxy_listener_addr = "127.0.0.1:11111".parse().unwrap(); - // - // let quicproxy_service = QuicForwardProxy::new(proxy_listener_addr, validator_identity.clone()) - // .await? - // .start_services(); - - let services = LiteBridge::new( + LiteBridge::new( rpc_addr, ws_addr, fanout_size, - validator_identity.clone(), + identity, retry_after, maximum_retries_per_tx, ) - .await? + .await + .context("Error building LiteBridge")? .start_services( lite_rpc_http_addr, lite_rpc_ws_addr, - clean_interval_ms, enable_postgres, prometheus_addr, - ); + ) + .await +} - // let proxy_addr = "127.0.0.1:11111".parse().unwrap(); - // let test_client = QuicTestClient::new_with_endpoint( - // proxy_addr, &tls_configuration) - // .await? - // .start_services(); +fn get_args() -> Args { + let mut args = Args::parse(); + + dotenv().ok(); + + args.enable_postgres = args.enable_postgres + || if let Ok(enable_postgres_env_var) = env::var("PG_ENABLED") { + enable_postgres_env_var != "false" + } else { + false + }; + + args +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 16)] +pub async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt::init(); + + let args = get_args(); let ctrl_c_signal = tokio::signal::ctrl_c(); + let rpc_tester = RpcTester::from(&args).start(); + + let main = start_lite_rpc(args.clone()); tokio::select! { - res = services => { - bail!("Services quit unexpectedly {res:?}"); - }, - // res = quicproxy_service => { - // bail!("Quic Proxy quit unexpectedly {res:?}"); - // }, - // res = test_client => { - // bail!("Test Client quit unexpectedly {res:?}"); - // }, + err = rpc_tester => { + // This should never happen + unreachable!("{err:?}") + } + res = main => { + // This should never happen + log::error!("Services quit unexpectedly {res:?}"); + bail!("") + } _ = ctrl_c_signal => { - info!("Received ctrl+c signal"); + log::info!("Received ctrl+c signal"); Ok(()) } diff --git a/lite-rpc/src/postgres.rs b/lite-rpc/src/postgres.rs index 47bd3985..dd920833 100644 --- a/lite-rpc/src/postgres.rs +++ b/lite-rpc/src/postgres.rs @@ -6,18 +6,18 @@ use postgres_native_tls::MakeTlsConnector; use prometheus::{core::GenericGauge, opts, register_int_gauge}; use std::{sync::Arc, time::Duration}; -use tokio::{ - sync::{RwLock, RwLockReadGuard}, - task::JoinHandle, -}; +use tokio::sync::{RwLock, RwLockReadGuard}; use tokio_postgres::{config::SslMode, tls::MakeTlsConnect, types::ToSql, Client, NoTls, Socket}; use native_tls::{Certificate, Identity, TlsConnector}; use crate::encoding::BinaryEncoding; -use solana_lite_rpc_core::notifications::{ - BlockNotification, NotificationMsg, NotificationReciever, TransactionNotification, - TransactionUpdateNotification, +use solana_lite_rpc_core::{ + notifications::{ + BlockNotification, NotificationMsg, NotificationReciever, TransactionNotification, + TransactionUpdateNotification, + }, + AnyhowJoinHandle, }; lazy_static::lazy_static! { @@ -200,7 +200,6 @@ impl PostgresSession { log::error!("Connection to Postgres broke {err:?}"); return; } - unreachable!("Postgres thread returned") }); @@ -411,7 +410,7 @@ impl Postgres { Ok(self.session.read().await) } - pub fn start(mut self, mut recv: NotificationReciever) -> JoinHandle> { + pub fn start(mut self, mut recv: NotificationReciever) -> AnyhowJoinHandle { tokio::spawn(async move { info!("start postgres worker"); @@ -462,6 +461,7 @@ impl Postgres { } Err(tokio::sync::mpsc::error::TryRecvError::Empty) => break, Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) => { + log::error!("Postgres channel broke"); bail!("Postgres channel broke") } } diff --git a/lite-rpc/src/rpc_tester.rs b/lite-rpc/src/rpc_tester.rs new file mode 100644 index 00000000..50cd3fcc --- /dev/null +++ b/lite-rpc/src/rpc_tester.rs @@ -0,0 +1,41 @@ +use std::net::SocketAddr; + +use lite_rpc::cli::Args; +use prometheus::{opts, register_gauge, Gauge}; +use solana_rpc_client::nonblocking::rpc_client::RpcClient; + +lazy_static::lazy_static! { + static ref RPC_RESPONDING: Gauge = + register_gauge!(opts!("literpc_rpc_responding", "If LiteRpc is responding")).unwrap(); +} + +pub struct RpcTester(RpcClient); + +impl From<&Args> for RpcTester { + fn from(value: &Args) -> Self { + let addr: SocketAddr = value + .lite_rpc_http_addr + .parse() + .expect("Invalid literpc http address"); + + RpcTester(RpcClient::new(format!("http://0.0.0.0:{}", addr.port()))) + } +} + +impl RpcTester { + /// Starts a loop that checks if the rpc is responding every 5 seconds + pub async fn start(self) -> ! { + loop { + // sleep for 5 seconds + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + // do a simple request to self for getVersion + let Err(err) = self.0.get_version().await else { + RPC_RESPONDING.set(1.0); + continue; + }; + + RPC_RESPONDING.set(0.0); + log::error!("Rpc not responding {err:?}"); + } + } +} diff --git a/services/Cargo.toml b/services/Cargo.toml index d094efc8..7ef7b40d 100644 --- a/services/Cargo.toml +++ b/services/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-services" -version = "0.2.1" +version = "0.2.2" edition = "2021" description = "Services used by solana lite rpc" rust-version = "1.67.1" @@ -29,6 +29,8 @@ bytes = { workspace = true } anyhow = { workspace = true } itertools = { workspace = true } log = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["std", "env-filter"] } dashmap = { workspace = true } prometheus = { workspace = true } lazy_static = { workspace = true } @@ -36,4 +38,8 @@ async-channel = { workspace = true } quinn = { workspace = true } chrono = { workspace = true } rustls = { workspace = true } -solana-lite-rpc-core = { workspace = true } \ No newline at end of file +solana-lite-rpc-core = { workspace = true } + +[dev-dependencies] +crossbeam-channel = "0.5.6" +countmap = "0.2.0" diff --git a/services/src/block_listenser.rs b/services/src/block_listenser.rs index 392a4ccb..2b911b2b 100644 --- a/services/src/block_listenser.rs +++ b/services/src/block_listenser.rs @@ -1,6 +1,6 @@ use std::{ sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicU64, Ordering}, Arc, }, time::Duration, @@ -34,6 +34,7 @@ use solana_lite_rpc_core::{ }, subscription_handler::{SubscptionHanderSink, SubscriptionHandler}, tx_store::{TxProps, TxStore}, + AnyhowJoinHandle, }; lazy_static::lazy_static! { @@ -237,7 +238,11 @@ impl BlockListener { // TODO insert if not exists leader_id into accountaddrs // fetch cluster time from rpc - let block_time = self.rpc_client.get_block_time(slot).await?; + let block_time = self + .rpc_client + .get_block_time(slot) + .await + .context("failed to get block time")?; // fetch local time from blockstore let block_info = self @@ -271,7 +276,6 @@ impl BlockListener { commitment_config: CommitmentConfig, notifier: Option, estimated_slot: Arc, - exit_signal: Arc, ) -> anyhow::Result<()> { let (slot_retry_queue_sx, mut slot_retry_queue_rx) = tokio::sync::mpsc::unbounded_channel(); let (block_schedule_queue_sx, block_schedule_queue_rx) = async_channel::unbounded::(); @@ -279,18 +283,13 @@ impl BlockListener { // task to fetch blocks // let this = self.clone(); - let exit_signal_l = exit_signal.clone(); let slot_indexer_tasks = (0..8).map(move |_| { let this = this.clone(); let notifier = notifier.clone(); let slot_retry_queue_sx = slot_retry_queue_sx.clone(); let block_schedule_queue_rx = block_schedule_queue_rx.clone(); - let exit_signal_l = exit_signal_l.clone(); - let task: JoinHandle> = tokio::spawn(async move { + let task: AnyhowJoinHandle = tokio::spawn(async move { loop { - if exit_signal_l.load(Ordering::Relaxed) { - break; - } match block_schedule_queue_rx.recv().await { Ok(slot) => { if commitment_config.is_finalized() { @@ -322,7 +321,6 @@ impl BlockListener { } } } - bail!("Block Slot channel closed") }); task @@ -334,13 +332,8 @@ impl BlockListener { let slot_retry_task: JoinHandle> = { let block_schedule_queue_sx = block_schedule_queue_sx.clone(); let recent_slot = recent_slot.clone(); - let exit_signal_l = exit_signal.clone(); tokio::spawn(async move { while let Some((slot, instant)) = slot_retry_queue_rx.recv().await { - if exit_signal_l.load(Ordering::Relaxed) { - break; - } - BLOCKS_IN_RETRY_QUEUE.dec(); let recent_slot = recent_slot.load(std::sync::atomic::Ordering::Relaxed); // if slot is too old ignore @@ -365,7 +358,7 @@ impl BlockListener { BLOCKS_IN_CONFIRMED_QUEUE.inc(); } } - + error!("Slot retry task exit"); bail!("Slot retry task exit") }) }; @@ -390,9 +383,6 @@ impl BlockListener { continue; } - if exit_signal.load(Ordering::Relaxed) { - break; - } // filter already processed slots let new_block_slots: Vec = (last_latest_slot..new_slot).collect(); // context for lock @@ -414,45 +404,48 @@ impl BlockListener { last_latest_slot = new_slot; recent_slot.store(last_latest_slot, std::sync::atomic::Ordering::Relaxed); } - Ok(()) }); tokio::select! { res = get_slot_task => { - anyhow::bail!("Get slot task exited unexpectedly {res:?}") + error!("Get slot task exited unexpectedly {res:?}"); + bail!("Get slot task exited unexpectedly {res:?}") } res = slot_retry_task => { - anyhow::bail!("Slot retry task exited unexpectedly {res:?}") + error!("Slot retry task exited unexpectedly {res:?}"); + bail!("Slot retry task exited unexpectedly {res:?}") }, res = futures::future::try_join_all(slot_indexer_tasks) => { - anyhow::bail!("Slot indexer exited unexpectedly {res:?}") + error!("Slot indexer exited unexpectedly {res:?}"); + bail!("Slot indexer exited unexpectedly {res:?}") }, } } // continuosly poll processed blocks and feed into blockstore - pub fn listen_processed(self, exit_signal: Arc) -> JoinHandle> { - let block_processor = self.block_processor; - + pub fn listen_processed(self) -> AnyhowJoinHandle { tokio::spawn(async move { info!("processed block listner started"); - loop { - if exit_signal.load(Ordering::Relaxed) { - break; - } + let mut errors = 0; - if let Err(err) = block_processor + while errors <= 10 { + if let Err(err) = self + .block_processor .poll_latest_block(CommitmentConfig::processed()) .await { + errors += 1; error!("Error fetching latest processed block {err:?}"); + } else { + errors = 0; } // sleep tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; } - Ok(()) + error!("{errors} consecutive errors while polling processed blocks"); + bail!("{errors} consecutive errors while polling processed blocks") }) } diff --git a/services/src/cleaner.rs b/services/src/cleaner.rs index 265a8001..cab9f3af 100644 --- a/services/src/cleaner.rs +++ b/services/src/cleaner.rs @@ -1,13 +1,10 @@ -use std::sync::atomic::AtomicBool; -use std::sync::Arc; use std::time::Duration; -use crate::block_listenser::BlockListener; -use crate::tx_sender::TxSender; +use crate::{block_listenser::BlockListener, tx_sender::TxSender}; use log::info; -use prometheus::core::GenericGauge; -use prometheus::{opts, register_int_gauge}; +use prometheus::{core::GenericGauge, opts, register_int_gauge}; use solana_lite_rpc_core::block_store::BlockStore; +use solana_sdk::commitment_config::CommitmentConfig; use tokio::task::JoinHandle; lazy_static::lazy_static! { @@ -35,8 +32,12 @@ impl Cleaner { } } - pub fn clean_tx_sender(&self, ttl_duration: Duration) { - self.tx_sender.cleanup(ttl_duration); + pub async fn clean_tx_sender(&self) { + let (_, blockhash_finalized) = self + .block_store + .get_latest_block(CommitmentConfig::finalized()) + .await; + self.tx_sender.cleanup(blockhash_finalized.block_height); } /// Clean Signature Subscribers from Block Listeners @@ -44,32 +45,24 @@ impl Cleaner { self.block_listenser.clean(ttl_duration); } - pub async fn clean_block_store(&self, ttl_duration: Duration) { - self.block_store.clean(ttl_duration).await; + pub async fn clean_block_store(&self) { + self.block_store.clean().await; BLOCKS_IN_BLOCKSTORE.set(self.block_store.number_of_blocks_in_store() as i64); } - pub fn start( - self, - ttl_duration: Duration, - exit_signal: Arc, - ) -> JoinHandle> { + pub fn start(self, ttl_duration: Duration) -> JoinHandle> { let mut ttl = tokio::time::interval(ttl_duration); tokio::spawn(async move { info!("Cleaning memory"); loop { - if exit_signal.load(std::sync::atomic::Ordering::Relaxed) { - break; - } ttl.tick().await; - self.clean_tx_sender(ttl_duration); + self.clean_tx_sender().await; self.clean_block_listeners(ttl_duration); - self.clean_block_store(ttl_duration).await; + self.clean_block_store().await; } - Ok(()) }) } } diff --git a/services/src/prometheus_sync.rs b/services/src/prometheus_sync.rs index c8836caa..3a8a9973 100644 --- a/services/src/prometheus_sync.rs +++ b/services/src/prometheus_sync.rs @@ -2,10 +2,10 @@ use std::time::Duration; use log::error; use prometheus::{Encoder, TextEncoder}; +use solana_lite_rpc_core::AnyhowJoinHandle; use tokio::{ io::AsyncWriteExt, net::{TcpListener, TcpStream, ToSocketAddrs}, - task::JoinHandle, }; pub struct PrometheusSync; @@ -19,7 +19,7 @@ impl PrometheusSync { ) } - async fn handle_stream(&self, stream: &mut TcpStream) -> anyhow::Result<()> { + async fn handle_stream(stream: &mut TcpStream) -> anyhow::Result<()> { let mut metrics_buffer = Vec::new(); let encoder = TextEncoder::new(); @@ -39,8 +39,7 @@ impl PrometheusSync { Ok(()) } - pub fn sync(self, addr: impl ToSocketAddrs + Send + 'static) -> JoinHandle> { - #[allow(unreachable_code)] + pub fn sync(addr: impl ToSocketAddrs + Send + 'static) -> AnyhowJoinHandle { tokio::spawn(async move { let listener = TcpListener::bind(addr).await?; @@ -51,10 +50,8 @@ impl PrometheusSync { continue; }; - let _ = self.handle_stream(&mut stream).await; + let _ = Self::handle_stream(&mut stream).await; } - - Ok(()) }) } } diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 27a7bd88..68a9b5f5 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -1,10 +1,13 @@ use dashmap::DashMap; -use log::{error, info, trace, warn}; +use log::{error, trace}; use prometheus::{core::GenericGauge, opts, register_int_gauge}; -use quinn::{Connection, Endpoint}; +use quinn::Endpoint; use solana_lite_rpc_core::{ - quic_connection_utils::QuicConnectionUtils, rotating_queue::RotatingQueue, - structures::identity_stakes::IdentityStakes, tx_store::TxStore, + quic_connection::QuicConnectionPool, + quic_connection_utils::{QuicConnectionParameters, QuicConnectionUtils}, + rotating_queue::RotatingQueue, + structures::identity_stakes::IdentityStakes, + tx_store::TxStore, }; use solana_sdk::pubkey::Pubkey; use solana_streamer::nonblocking::quic::compute_max_allowed_uni_streams; @@ -15,18 +18,8 @@ use std::{ atomic::{AtomicBool, AtomicU64, Ordering}, Arc, }, - time::Duration, }; -use anyhow::bail; -use itertools::Itertools; -use solana_client::client_error::reqwest::header::WARNING; -use solana_sdk::transaction::VersionedTransaction; -use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; -use tokio::time::timeout; -use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; - -pub const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); -pub const CONNECTION_RETRY_COUNT: usize = 10; +use tokio::sync::{broadcast::Receiver, broadcast::Sender}; lazy_static::lazy_static! { static ref NB_QUIC_CONNECTIONS: GenericGauge = @@ -39,34 +32,34 @@ lazy_static::lazy_static! { register_int_gauge!(opts!("literpc_quic_tasks", "Number of connections to keep asked by tpu service")).unwrap(); } -pub struct ActiveConnection { - endpoint: Endpoint, +#[derive(Clone)] +struct ActiveConnection { + endpoints: RotatingQueue, identity: Pubkey, tpu_address: SocketAddr, exit_signal: Arc, txs_sent_store: TxStore, + connection_parameters: QuicConnectionParameters, } impl ActiveConnection { pub fn new( - endpoint: Endpoint, + endpoints: RotatingQueue, tpu_address: SocketAddr, identity: Pubkey, txs_sent_store: TxStore, + connection_parameters: QuicConnectionParameters, ) -> Self { Self { - endpoint, + endpoints, tpu_address, identity, exit_signal: Arc::new(AtomicBool::new(false)), txs_sent_store, + connection_parameters, } } - fn on_connect() { - NB_QUIC_CONNECTIONS.inc(); - } - fn check_for_confirmation(txs_sent_store: &TxStore, signature: String) -> bool { match txs_sent_store.get(&signature) { Some(props) => props.status.is_some(), @@ -76,29 +69,37 @@ impl ActiveConnection { #[allow(clippy::too_many_arguments)] async fn listen( + &self, transaction_reciever: Receiver<(String, Vec)>, exit_oneshot_channel: tokio::sync::mpsc::Receiver<()>, - endpoint: Endpoint, - tpu_address: SocketAddr, - exit_signal: Arc, - identity: Pubkey, + addr: SocketAddr, identity_stakes: IdentityStakes, txs_sent_store: TxStore, ) { NB_QUIC_ACTIVE_CONNECTIONS.inc(); let mut transaction_reciever = transaction_reciever; let mut exit_oneshot_channel = exit_oneshot_channel; + let identity = self.identity; let max_uni_stream_connections: u64 = compute_max_allowed_uni_streams( identity_stakes.peer_type, identity_stakes.stakes, identity_stakes.total_stakes, ) as u64; - let number_of_transactions_per_unistream = 5; + let number_of_transactions_per_unistream = self + .connection_parameters + .number_of_transactions_per_unistream; + let max_number_of_connections = self.connection_parameters.max_number_of_connections; let task_counter: Arc = Arc::new(AtomicU64::new(0)); - let mut connection: Option>> = None; - let last_stable_id: Arc = Arc::new(AtomicU64::new(0)); + let exit_signal = self.exit_signal.clone(); + let connection_pool = QuicConnectionPool::new( + identity, + self.endpoints.clone(), + addr, + self.connection_parameters, + exit_signal.clone(), + ); loop { // exit signal set @@ -145,68 +146,20 @@ impl ActiveConnection { } } - if connection.is_none() { - // initial connection - let conn = QuicConnectionUtils::connect( - identity, - false, - endpoint.clone(), - tpu_address, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - exit_signal.clone(), - Self::on_connect).await; - - if let Some(conn) = conn { - // could connect - connection = Some(Arc::new(RwLock::new(conn))); - } else { - break; - } + // queue getting full and a connection poll is getting slower + // add more connections to the pool + if connection_pool.len() < max_number_of_connections { + connection_pool.add_connection().await; + NB_QUIC_CONNECTIONS.inc(); } let task_counter = task_counter.clone(); - let endpoint = endpoint.clone(); - let exit_signal = exit_signal.clone(); - let connection = connection.clone(); - let last_stable_id = last_stable_id.clone(); + let connection_pool = connection_pool.clone(); tokio::spawn(async move { task_counter.fetch_add(1, Ordering::Relaxed); NB_QUIC_TASKS.inc(); - let connection = connection.unwrap(); - - if true { - // TODO split to new service - // SOS - info!("Sending copy of transaction batch of {} to tpu with identity {} to quic proxy", - txs.len(), identity); - Self::send_copy_of_txs_to_quicproxy( - &txs, endpoint.clone(), - // proxy address - "127.0.0.1:11111".parse().unwrap(), - tpu_address, - identity.clone()).await.unwrap(); - } - - - if false { - QuicConnectionUtils::send_transaction_batch( - connection, - txs, - identity, - endpoint, - tpu_address, - exit_signal, - last_stable_id, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - || { - // do nothing as we are using the same connection - } - ).await; - } - + connection_pool.send_transaction_batch(txs).await; NB_QUIC_TASKS.dec(); task_counter.fetch_sub(1, Ordering::Relaxed); }); @@ -221,76 +174,20 @@ impl ActiveConnection { NB_QUIC_ACTIVE_CONNECTIONS.dec(); } - async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, endpoint: Endpoint, - proxy_address: SocketAddr, tpu_target_address: SocketAddr, - identity: Pubkey) -> anyhow::Result<()> { - - info!("sending vecvec: {}", raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(",")); - - let raw_tx_batch_copy = raw_tx_batch.clone(); - - let mut txs = vec![]; - - for raw_tx in raw_tx_batch_copy { - let tx = match bincode::deserialize::(&raw_tx) { - Ok(tx) => tx, - Err(err) => { - bail!(err.to_string()); - } - }; - txs.push(tx); - } - - let forwarding_request = TpuForwardingRequest::new(tpu_target_address, identity, txs); - - let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); - - let send_result = timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)); - - match send_result.await { - Ok(..) => { - info!("Successfully sent data to quic proxy"); - } - Err(e) => { - warn!("Failed to send data to quic proxy: {:?}", e); - } - } - Ok(()) - } - - async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { - info!("sending {} bytes to proxy", proxy_request_raw.len()); - - let mut connecting = endpoint.connect(proxy_address, "localhost")?; - let connection = timeout(Duration::from_millis(500), connecting).await??; - let mut send = connection.open_uni().await?; - - send.write_all(proxy_request_raw).await?; - - send.finish().await?; - - Ok(()) - } - pub fn start_listening( &self, transaction_reciever: Receiver<(String, Vec)>, exit_oneshot_channel: tokio::sync::mpsc::Receiver<()>, identity_stakes: IdentityStakes, ) { - let endpoint = self.endpoint.clone(); - let tpu_address = self.tpu_address; - let exit_signal = self.exit_signal.clone(); - let identity = self.identity; + let addr = self.tpu_address; let txs_sent_store = self.txs_sent_store.clone(); + let this = self.clone(); tokio::spawn(async move { - Self::listen( + this.listen( transaction_reciever, exit_oneshot_channel, - endpoint, - tpu_address, - exit_signal, - identity, + addr, identity_stakes, txs_sent_store, ) @@ -310,12 +207,17 @@ pub struct TpuConnectionManager { } impl TpuConnectionManager { - pub fn new(certificate: rustls::Certificate, key: rustls::PrivateKey, fanout: usize) -> Self { - let number_of_clients = if fanout > 5 { fanout / 4 } else { 1 }; + pub async fn new( + certificate: rustls::Certificate, + key: rustls::PrivateKey, + fanout: usize, + ) -> Self { + let number_of_clients = fanout * 2; Self { endpoints: RotatingQueue::new(number_of_clients, || { QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()) - }), + }) + .await, identity_to_active_connection: Arc::new(DashMap::new()), } } @@ -326,17 +228,18 @@ impl TpuConnectionManager { connections_to_keep: HashMap, identity_stakes: IdentityStakes, txs_sent_store: TxStore, + connection_parameters: QuicConnectionParameters, ) { NB_CONNECTIONS_TO_KEEP.set(connections_to_keep.len() as i64); for (identity, socket_addr) in &connections_to_keep { if self.identity_to_active_connection.get(identity).is_none() { trace!("added a connection for {}, {}", identity, socket_addr); - let endpoint = self.endpoints.get(); let active_connection = ActiveConnection::new( - endpoint, + self.endpoints.clone(), *socket_addr, *identity, txs_sent_store.clone(), + connection_parameters, ); // using mpsc as a oneshot channel/ because with one shot channel we cannot reuse the reciever let (sx, rx) = tokio::sync::mpsc::channel(1); diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 8ddaafc2..7b925033 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -1,11 +1,12 @@ -use anyhow::bail; +use anyhow::{bail, Context}; use log::{error, info}; use prometheus::{core::GenericGauge, opts, register_int_gauge}; use solana_client::nonblocking::rpc_client::RpcClient; use solana_lite_rpc_core::{ - leader_schedule::LeaderSchedule, solana_utils::SolanaUtils, - structures::identity_stakes::IdentityStakes, tx_store::TxStore, AnyhowJoinHandle, + leader_schedule::LeaderSchedule, quic_connection_utils::QuicConnectionParameters, + solana_utils::SolanaUtils, structures::identity_stakes::IdentityStakes, tx_store::TxStore, + AnyhowJoinHandle, }; use super::tpu_connection_manager::TpuConnectionManager; @@ -17,7 +18,7 @@ use std::{ net::{IpAddr, Ipv4Addr}, str::FromStr, sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicU64, Ordering}, Arc, }, }; @@ -26,12 +27,6 @@ use tokio::{ time::{Duration, Instant}, }; -const CACHE_NEXT_SLOT_LEADERS_PUBKEY_SIZE: usize = 1024; // Save pubkey and contact info of next 1024 leaders in the queue -const CLUSTERINFO_REFRESH_TIME: u64 = 60 * 60; // stakes every 1hrs -const LEADER_SCHEDULE_UPDATE_INTERVAL: u64 = 10; // update leader schedule every 10s -const MAXIMUM_TRANSACTIONS_IN_QUEUE: usize = 200_000; -const MAX_NB_ERRORS: usize = 10; - lazy_static::lazy_static! { static ref NB_CLUSTER_NODES: GenericGauge = register_int_gauge!(opts!("literpc_nb_cluster_nodes", "Number of cluster nodes in saved")).unwrap(); @@ -46,64 +41,60 @@ lazy_static::lazy_static! { register_int_gauge!(opts!("literpc_estimated_slot", "Estimated slot seen by last rpc")).unwrap(); } -/// service (singleton) +#[derive(Clone, Copy)] +pub struct TpuServiceConfig { + pub fanout_slots: u64, + pub number_of_leaders_to_cache: usize, + pub clusterinfo_refresh_time: Duration, + pub leader_schedule_update_frequency: Duration, + pub maximum_transaction_in_queue: usize, + pub maximum_number_of_errors: usize, + pub quic_connection_params: QuicConnectionParameters, +} + #[derive(Clone)] pub struct TpuService { - /// out current_slot: Arc, - /// out estimated_slot: Arc, - /// in - fanout_slots: u64, - /// in rpc_client: Arc, - /// in - rpc_ws_address: String, - /// in broadcast_sender: Arc)>>, - /// in tpu_connection_manager: Arc, - /// in - identity: Arc, - /// out identity_stakes: Arc>, - /// in txs_sent_store: TxStore, - /// out leader_schedule: Arc, + config: TpuServiceConfig, + identity: Pubkey, } impl TpuService { pub async fn new( + config: TpuServiceConfig, + identity: Arc, current_slot: Slot, - fanout_slots: u64, - validator_identity: Arc, rpc_client: Arc, - rpc_ws_address: String, txs_sent_store: TxStore, ) -> anyhow::Result { - let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); + let (sender, _) = tokio::sync::broadcast::channel(config.maximum_transaction_in_queue); let (certificate, key) = new_self_signed_tls_certificate( - validator_identity.as_ref(), + identity.as_ref(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) - .expect("Failed to initialize QUIC connection certificates"); + .expect("Failed to initialize QUIC client certificates"); let tpu_connection_manager = - TpuConnectionManager::new(certificate, key, fanout_slots as usize); + TpuConnectionManager::new(certificate, key, config.fanout_slots as usize).await; Ok(Self { current_slot: Arc::new(AtomicU64::new(current_slot)), estimated_slot: Arc::new(AtomicU64::new(current_slot)), - leader_schedule: Arc::new(LeaderSchedule::new(CACHE_NEXT_SLOT_LEADERS_PUBKEY_SIZE)), - fanout_slots, + leader_schedule: Arc::new(LeaderSchedule::new(config.number_of_leaders_to_cache)), rpc_client, - rpc_ws_address, broadcast_sender: Arc::new(sender), tpu_connection_manager: Arc::new(tpu_connection_manager), - identity: validator_identity, identity_stakes: Arc::new(RwLock::new(IdentityStakes::default())), txs_sent_store, + identity: identity.pubkey(), + config, }) } @@ -112,11 +103,8 @@ impl TpuService { // update stakes for the identity { let mut lock = self.identity_stakes.write().await; - *lock = SolanaUtils::get_stakes_for_identity( - self.rpc_client.clone(), - self.identity.pubkey(), - ) - .await?; + *lock = SolanaUtils::get_stakes_for_identity(self.rpc_client.clone(), self.identity) + .await?; } Ok(()) } @@ -148,12 +136,12 @@ impl TpuService { current_slot }; - let fanout = self.fanout_slots; + let fanout = self.config.fanout_slots; let last_slot = estimated_slot + fanout; let next_leaders = self.leader_schedule.get_leaders(load_slot, last_slot).await; let connections_to_keep = next_leaders - .iter() + .into_iter() .filter(|x| x.tpu.is_some()) .map(|x| { let mut addr = x.tpu.unwrap(); @@ -171,79 +159,68 @@ impl TpuService { connections_to_keep, *identity_stakes, self.txs_sent_store.clone(), + self.config.quic_connection_params, ) .await; } - fn check_exit_signal(exit_signal: &Arc) -> bool { - exit_signal.load(Ordering::Relaxed) - } - - async fn update_current_slot( + fn update_current_slot( &self, update_notifier: tokio::sync::mpsc::UnboundedSender, - exit_signal: Arc, - ) { + ) -> AnyhowJoinHandle { let current_slot = self.current_slot.clone(); - let update_slot = |slot: u64| { + let rpc_client = self.rpc_client.clone(); + + let update_slot = move |slot: u64| { if slot > current_slot.load(Ordering::Relaxed) { current_slot.store(slot, Ordering::Relaxed); CURRENT_SLOT.set(slot as i64); let _ = update_notifier.send(slot); } }; - let mut nb_errror = 0; - loop { - if Self::check_exit_signal(&exit_signal) { - break; - } - // always loop update the current slots as it is central to working of TPU - if let Err(e) = - SolanaUtils::poll_slots(self.rpc_client.clone(), &self.rpc_ws_address, update_slot) - .await - { - nb_errror += 1; - log::info!("Got error while polling slot {}", e); - if nb_errror > MAX_NB_ERRORS { - error!( - "Reached max amount of errors to fetch latest slot, exiting poll slot loop" - ); - break; - } - } else { - nb_errror = 0; + let max_nb_errors = self.config.maximum_number_of_errors; + tokio::spawn(async move { + let mut nb_error = 0; + + while nb_error < max_nb_errors { + // always loop update the current slots as it is central to working of TPU + let Err(err) = SolanaUtils::poll_slots(&rpc_client, &update_slot).await else { + nb_error = 0; + continue; + }; + + nb_error += 1; + log::info!("Got error while polling slot {}", err); } - } + + error!("Reached max amount of errors to fetch latest slot, exiting poll slot loop"); + bail!("Reached max amount of errors to fetch latest slot, exiting poll slot loop") + }) } - pub async fn start(&self, exit_signal: Arc) -> anyhow::Result<()> { + pub async fn start(&self) -> anyhow::Result<()> { + // setup self.leader_schedule .load_cluster_info(self.rpc_client.clone()) - .await?; + .await + .context("failed to load initial cluster info")?; self.update_current_stakes().await?; self.update_leader_schedule().await?; self.update_quic_connections().await; let this = self.clone(); - let exit_signal_l = exit_signal.clone(); - let jh_update_leaders = tokio::spawn(async move { + let update_leader_schedule_service = tokio::spawn(async move { let mut last_cluster_info_update = Instant::now(); - let leader_schedule_update_interval = - Duration::from_secs(LEADER_SCHEDULE_UPDATE_INTERVAL); - let cluster_info_update_interval = Duration::from_secs(CLUSTERINFO_REFRESH_TIME); + let leader_schedule_update_interval = this.config.leader_schedule_update_frequency; + let cluster_info_update_interval = this.config.clusterinfo_refresh_time; loop { - if Self::check_exit_signal(&exit_signal_l) { - break; - } tokio::time::sleep(leader_schedule_update_interval).await; - if Self::check_exit_signal(&exit_signal_l) { - break; - } - info!("update leader schedule and cluster nodes"); + if this.update_leader_schedule().await.is_err() { error!("unable to update leader shedule"); } + if last_cluster_info_update.elapsed() > cluster_info_update_interval { if this.update_current_stakes().await.is_err() { error!("unable to update cluster infos"); @@ -254,47 +231,41 @@ impl TpuService { } }); - let this = self.clone(); let (slot_sender, slot_reciever) = tokio::sync::mpsc::unbounded_channel::(); - let exit_signal_l = exit_signal.clone(); - let slot_sub_task: AnyhowJoinHandle = tokio::spawn(async move { - this.update_current_slot(slot_sender, exit_signal_l).await; - Ok(()) - }); - let estimated_slot = self.estimated_slot.clone(); - let current_slot = self.current_slot.clone(); + // Service to poll current slot from upstream rpc + let slot_poll_service = self.update_current_slot(slot_sender); + + // Service to estimate slots let this = self.clone(); - let exit_signal_l = exit_signal.clone(); - let estimated_slot_calculation = tokio::spawn(async move { + let estimated_slot_service = tokio::spawn(async move { let mut slot_update_notifier = slot_reciever; loop { - if Self::check_exit_signal(&exit_signal_l) { - break; - } - if SolanaUtils::slot_estimator( &mut slot_update_notifier, - current_slot.clone(), - estimated_slot.clone(), + this.current_slot.clone(), + this.estimated_slot.clone(), ) .await { - ESTIMATED_SLOT.set(estimated_slot.load(Ordering::Relaxed) as i64); + ESTIMATED_SLOT.set(this.estimated_slot.load(Ordering::Relaxed) as i64); this.update_quic_connections().await; } } }); tokio::select! { - res = jh_update_leaders => { - bail!("Leader update service exited unexpectedly {res:?}"); + res = update_leader_schedule_service => { + error!("Leader update Service {res:?}"); + bail!("Leader update Service {res:?}"); }, - res = slot_sub_task => { - bail!("Leader update service exited unexpectedly {res:?}"); + res = slot_poll_service => { + error!("Slot Poll Service {res:?}"); + bail!("Slot Poll Service {res:?}"); }, - res = estimated_slot_calculation => { - bail!("Estimated slot calculation service exited unexpectedly {res:?}"); + res = estimated_slot_service => { + error!("Estimated slot Service {res:?}"); + bail!("Estimated slot Service {res:?}"); }, } } diff --git a/services/src/transaction_replayer.rs b/services/src/transaction_replayer.rs index 2bcd5e43..bae3a3cc 100644 --- a/services/src/transaction_replayer.rs +++ b/services/src/transaction_replayer.rs @@ -1,14 +1,11 @@ use crate::tpu_utils::tpu_service::TpuService; +use anyhow::bail; use log::error; use prometheus::{core::GenericGauge, opts, register_int_gauge}; -use solana_lite_rpc_core::tx_store::TxStore; -use std::{ - sync::{atomic::AtomicBool, Arc}, - time::Duration, -}; +use solana_lite_rpc_core::{tx_store::TxStore, AnyhowJoinHandle}; +use std::time::Duration; use tokio::{ sync::mpsc::{UnboundedReceiver, UnboundedSender}, - task::JoinHandle, time::Instant, }; @@ -45,56 +42,44 @@ impl TransactionReplayer { pub fn start_service( &self, sender: UnboundedSender, - reciever: UnboundedReceiver, - exit_signal: Arc, - ) -> JoinHandle> { + mut reciever: UnboundedReceiver, + ) -> AnyhowJoinHandle { let tpu_service = self.tpu_service.clone(); let tx_store = self.tx_store.clone(); let retry_after = self.retry_after; + tokio::spawn(async move { - let mut reciever = reciever; - loop { - if exit_signal.load(std::sync::atomic::Ordering::Relaxed) { - break; + while let Some(mut tx_replay) = reciever.recv().await { + MESSAGES_IN_REPLAY_QUEUE.dec(); + if Instant::now() < tx_replay.replay_at { + tokio::time::sleep_until(tx_replay.replay_at).await; } - let tx = reciever.recv().await; - match tx { - Some(mut tx_replay) => { - MESSAGES_IN_REPLAY_QUEUE.dec(); - if Instant::now() < tx_replay.replay_at { - tokio::time::sleep_until(tx_replay.replay_at).await; - } - if let Some(tx) = tx_store.get(&tx_replay.signature) { - if tx.status.is_some() { - // transaction has been confirmed / no retry needed - continue; - } - } else { - // transaction timed out - continue; - } - // ignore reset error - let _ = tpu_service - .send_transaction(tx_replay.signature.clone(), tx_replay.tx.clone()); - - if tx_replay.replay_count < tx_replay.max_replay { - tx_replay.replay_count += 1; - tx_replay.replay_at = Instant::now() + retry_after; - if let Err(e) = sender.send(tx_replay) { - error!("error while scheduling replay ({})", e); - continue; - } else { - MESSAGES_IN_REPLAY_QUEUE.inc(); - } - } + if let Some(tx) = tx_store.get(&tx_replay.signature) { + if tx.status.is_some() { + // transaction has been confirmed / no retry needed + continue; } - None => { - error!("transaction replay channel broken"); - break; + } else { + // transaction timed out + continue; + } + // ignore reset error + let _ = + tpu_service.send_transaction(tx_replay.signature.clone(), tx_replay.tx.clone()); + + if tx_replay.replay_count < tx_replay.max_replay { + tx_replay.replay_count += 1; + tx_replay.replay_at = Instant::now() + retry_after; + if let Err(e) = sender.send(tx_replay) { + error!("error while scheduling replay ({})", e); + continue; + } else { + MESSAGES_IN_REPLAY_QUEUE.inc(); } } } - Ok(()) + error!("transaction replay channel broken"); + bail!("transaction replay channel broken"); }) } } diff --git a/services/src/transaction_service.rs b/services/src/transaction_service.rs index b21a2673..961c7b48 100644 --- a/services/src/transaction_service.rs +++ b/services/src/transaction_service.rs @@ -1,31 +1,25 @@ // This class will manage the lifecycle for a transaction // It will send, replay if necessary and confirm by listening to blocks -use std::{ - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - time::Duration, -}; +use std::time::Duration; use crate::{ block_listenser::BlockListener, cleaner::Cleaner, tpu_utils::tpu_service::TpuService, transaction_replayer::{TransactionReplay, TransactionReplayer, MESSAGES_IN_REPLAY_QUEUE}, - tx_sender::{TxSender, WireTransaction}, + tx_sender::{TransactionInfo, TxSender}, }; use anyhow::bail; use solana_lite_rpc_core::{ block_store::{BlockInformation, BlockStore}, notifications::NotificationSender, + AnyhowJoinHandle, }; use solana_rpc_client::rpc_client::SerializableTransaction; use solana_sdk::{commitment_config::CommitmentConfig, transaction::VersionedTransaction}; use tokio::{ sync::mpsc::{self, Sender, UnboundedSender}, - task::JoinHandle, time::Instant, }; @@ -55,90 +49,80 @@ impl TransactionServiceBuilder { } } - pub async fn start( - &self, + pub fn start( + self, notifier: Option, block_store: BlockStore, max_retries: usize, - clean_interval: Duration, - ) -> (TransactionService, JoinHandle) { + ) -> (TransactionService, AnyhowJoinHandle) { let (transaction_channel, tx_recv) = mpsc::channel(self.max_nb_txs_in_queue); let (replay_channel, replay_reciever) = tokio::sync::mpsc::unbounded_channel(); - let tx_sender = self.tx_sender.clone(); - let block_listner = self.block_listner.clone(); - let tx_replayer = self.tx_replayer.clone(); - let tpu_service = self.tpu_service.clone(); - let replay_channel_task = replay_channel.clone(); - let exit_signal = Arc::new(AtomicBool::new(false)); - let exit_signal_t = exit_signal.clone(); - let block_store_t = block_store.clone(); - let jh_services: JoinHandle = tokio::spawn(async move { - let tpu_service_fx = tpu_service.start(exit_signal_t.clone()); - - let tx_sender_jh = - tx_sender - .clone() - .execute(tx_recv, notifier.clone(), exit_signal_t.clone()); - - let replay_service = tx_replayer.start_service( - replay_channel_task, - replay_reciever, - exit_signal_t.clone(), - ); - - let finalized_block_listener = block_listner.clone().listen( - CommitmentConfig::finalized(), - notifier.clone(), - tpu_service.get_estimated_slot_holder(), - exit_signal_t.clone(), - ); - - let confirmed_block_listener = block_listner.clone().listen( - CommitmentConfig::confirmed(), - None, - tpu_service.get_estimated_slot_holder(), - exit_signal_t.clone(), - ); - - let processed_block_listener = block_listner - .clone() - .listen_processed(exit_signal_t.clone()); - - let cleaner = Cleaner::new(tx_sender.clone(), block_listner.clone(), block_store_t) - .start(clean_interval, exit_signal_t); - - tokio::select! { - res = tpu_service_fx => { - format!("{res:?}") - }, - res = tx_sender_jh => { - format!("{res:?}") - }, - - res = finalized_block_listener => { - format!("{res:?}") - }, - res = confirmed_block_listener => { - format!("{res:?}") - }, - res = processed_block_listener => { - format!("{res:?}") - }, - res = replay_service => { - format!("{res:?}") - }, - res = cleaner => { - format!("{res:?}") - }, - } - }); + let jh_services: AnyhowJoinHandle = { + let tx_sender = self.tx_sender.clone(); + let block_listner = self.block_listner.clone(); + let tx_replayer = self.tx_replayer.clone(); + let tpu_service = self.tpu_service.clone(); + let replay_channel_task = replay_channel.clone(); + let block_store_t = block_store.clone(); + + tokio::spawn(async move { + let tpu_service_fx = tpu_service.start(); + + let tx_sender_jh = tx_sender.clone().execute(tx_recv, notifier.clone()); + + let replay_service = + tx_replayer.start_service(replay_channel_task, replay_reciever); + + let finalized_block_listener = block_listner.clone().listen( + CommitmentConfig::finalized(), + notifier.clone(), + tpu_service.get_estimated_slot_holder(), + ); + + let confirmed_block_listener = block_listner.clone().listen( + CommitmentConfig::confirmed(), + None, + tpu_service.get_estimated_slot_holder(), + ); + + let processed_block_listener = block_listner.clone().listen_processed(); + + // transactions get invalid in around 1 mins, because the block hash expires in 150 blocks so 150 * 400ms = 60s + // Setting it to two to give some margin of error / as not all the blocks are filled. + let cleaner = Cleaner::new(tx_sender.clone(), block_listner.clone(), block_store_t) + .start(Duration::from_secs(120)); + + tokio::select! { + res = tpu_service_fx => { + bail!("Tpu Service {res:?}") + }, + res = tx_sender_jh => { + bail!("Tx Sender {res:?}") + }, + res = finalized_block_listener => { + bail!("Finalized Block Listener {res:?}") + }, + res = confirmed_block_listener => { + bail!("Confirmed Block Listener {res:?}") + }, + res = processed_block_listener => { + bail!("Processed Block Listener {res:?}") + }, + res = replay_service => { + bail!("Replay Service {res:?}") + }, + res = cleaner => { + bail!("Cleaner {res:?}") + }, + } + }) + }; ( TransactionService { transaction_channel, replay_channel, - exit_signal, block_store, max_retries, replay_after: self.tx_replayer.retry_after, @@ -150,9 +134,8 @@ impl TransactionServiceBuilder { #[derive(Clone)] pub struct TransactionService { - pub transaction_channel: Sender<(String, WireTransaction, u64)>, + pub transaction_channel: Sender, pub replay_channel: UnboundedSender, - pub exit_signal: Arc, pub block_store: BlockStore, pub max_retries: usize, pub replay_after: Duration, @@ -172,7 +155,7 @@ impl TransactionService { }; let signature = tx.signatures[0]; - let Some(BlockInformation { slot, .. }) = self + let Some(BlockInformation { slot, last_valid_blockheight, .. }) = self .block_store .get_block_info(&tx.get_recent_blockhash().to_string()) else { @@ -183,7 +166,12 @@ impl TransactionService { let max_replay = max_retries.map_or(self.max_retries, |x| x as usize); if let Err(e) = self .transaction_channel - .send((signature.to_string(), raw_tx, slot)) + .send(TransactionInfo { + signature: signature.to_string(), + last_valid_block_height: last_valid_blockheight, + slot, + transaction: raw_tx, + }) .await { bail!( @@ -208,8 +196,4 @@ impl TransactionService { } Ok(signature.to_string()) } - - pub fn stop(&self) { - self.exit_signal.store(true, Ordering::Relaxed) - } } diff --git a/services/src/tx_sender.rs b/services/src/tx_sender.rs index 7418158f..9b2a6a02 100644 --- a/services/src/tx_sender.rs +++ b/services/src/tx_sender.rs @@ -1,7 +1,4 @@ -use std::{ - sync::{atomic::AtomicBool, Arc}, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use anyhow::bail; use chrono::Utc; @@ -11,6 +8,7 @@ use prometheus::{ core::GenericGauge, histogram_opts, opts, register_histogram, register_int_counter, register_int_gauge, Histogram, IntCounter, }; +use solana_sdk::slot_history::Slot; use tokio::{sync::mpsc::Receiver, task::JoinHandle}; use crate::tpu_utils::tpu_service::TpuService; @@ -36,6 +34,14 @@ lazy_static::lazy_static! { } pub type WireTransaction = Vec; + +#[derive(Clone, Debug)] +pub struct TransactionInfo { + pub signature: String, + pub slot: Slot, + pub transaction: WireTransaction, + pub last_valid_block_height: u64, +} // making 250 as sleep time will effectively make lite rpc send // (1000/250) * 5 * 512 = 10240 tps const INTERVAL_PER_BATCH_IN_MS: u64 = 50; @@ -61,13 +67,10 @@ impl TxSender { /// retry enqued_tx(s) async fn forward_txs( &self, - sigs_and_slots: Vec<(String, u64)>, - txs: Vec, + transaction_infos: Vec, notifier: Option, ) { - assert_eq!(sigs_and_slots.len(), txs.len()); - - if sigs_and_slots.is_empty() { + if transaction_infos.is_empty() { return; } @@ -77,18 +80,30 @@ impl TxSender { let tpu_client = self.tpu_service.clone(); let txs_sent = self.txs_sent_store.clone(); - for (sig, _) in &sigs_and_slots { - trace!("sending transaction {}", sig); - txs_sent.insert(sig.to_owned(), TxProps::default()); + for transaction_info in &transaction_infos { + trace!("sending transaction {}", transaction_info.signature); + txs_sent.insert( + transaction_info.signature.clone(), + TxProps { + status: None, + last_valid_blockheight: transaction_info.last_valid_block_height, + }, + ); } let forwarded_slot = tpu_client.get_estimated_slot(); let forwarded_local_time = Utc::now(); let mut quic_responses = vec![]; - for (tx, (signature, _)) in txs.iter().zip(sigs_and_slots.clone()) { - txs_sent.insert(signature.to_owned(), TxProps::default()); - let quic_response = match tpu_client.send_transaction(signature.clone(), tx.clone()) { + for transaction_info in transaction_infos.iter() { + txs_sent.insert( + transaction_info.signature.clone(), + TxProps::new(transaction_info.last_valid_block_height), + ); + let quic_response = match tpu_client.send_transaction( + transaction_info.signature.clone(), + transaction_info.transaction.clone(), + ) { Ok(_) => { TXS_SENT.inc_by(1); 1 @@ -102,12 +117,12 @@ impl TxSender { quic_responses.push(quic_response); } if let Some(notifier) = ¬ifier { - let notification_msgs = sigs_and_slots + let notification_msgs = transaction_infos .iter() .enumerate() - .map(|(index, (sig, recent_slot))| TransactionNotification { - signature: sig.clone(), - recent_slot: *recent_slot, + .map(|(index, transaction_info)| TransactionNotification { + signature: transaction_info.signature.clone(), + recent_slot: transaction_info.slot, forwarded_slot, forwarded_local_time, processed_slot: None, @@ -123,49 +138,47 @@ impl TxSender { trace!( "It took {} ms to send a batch of {} transaction(s)", start.elapsed().as_millis(), - sigs_and_slots.len() + transaction_infos.len() ); } /// retry and confirm transactions every 2ms (avg time to confirm tx) pub fn execute( self, - mut recv: Receiver<(String, WireTransaction, u64)>, + mut recv: Receiver, notifier: Option, - exit_signal: Arc, ) -> JoinHandle> { tokio::spawn(async move { - let tx_sender = self.clone(); loop { - let mut sigs_and_slots = Vec::with_capacity(MAX_BATCH_SIZE_IN_PER_INTERVAL); - let mut txs = Vec::with_capacity(MAX_BATCH_SIZE_IN_PER_INTERVAL); + let mut transaction_infos = Vec::with_capacity(MAX_BATCH_SIZE_IN_PER_INTERVAL); let mut timeout_interval = INTERVAL_PER_BATCH_IN_MS; - if exit_signal.load(std::sync::atomic::Ordering::Relaxed) { - break; - } // In solana there in sig verify stage rate is limited to 2000 txs in 50ms // taking this as reference - while txs.len() <= MAX_BATCH_SIZE_IN_PER_INTERVAL { + while transaction_infos.len() <= MAX_BATCH_SIZE_IN_PER_INTERVAL { let instance = tokio::time::Instant::now(); match tokio::time::timeout(Duration::from_millis(timeout_interval), recv.recv()) .await { Ok(value) => match value { - Some((sig, tx, slot)) => { + Some(transaction_info) => { TXS_IN_CHANNEL.dec(); - if self.txs_sent_store.contains_key(&sig) { - // duplicate transaction + + // duplicate transaction + if self + .txs_sent_store + .contains_key(&transaction_info.signature) + { continue; } - sigs_and_slots.push((sig, slot)); - txs.push(tx); + transaction_infos.push(transaction_info); // update the timeout inteval timeout_interval = timeout_interval .saturating_sub(instance.elapsed().as_millis() as u64) .max(1); } None => { + log::error!("Channel Disconnected"); bail!("Channel Disconnected"); } }, @@ -175,25 +188,21 @@ impl TxSender { } } - assert_eq!(sigs_and_slots.len(), txs.len()); - - if sigs_and_slots.is_empty() { + if transaction_infos.is_empty() { continue; } - TX_BATCH_SIZES.set(txs.len() as i64); - tx_sender - .forward_txs(sigs_and_slots, txs, notifier.clone()) - .await; + TX_BATCH_SIZES.set(transaction_infos.len() as i64); + + self.forward_txs(transaction_infos, notifier.clone()).await; } - Ok(()) }) } - pub fn cleanup(&self, ttl_duration: Duration) { + pub fn cleanup(&self, current_finalized_blochash: u64) { let length_before = self.txs_sent_store.len(); self.txs_sent_store.retain(|_k, v| { - let retain = v.sent_at.elapsed() < ttl_duration; + let retain = v.last_valid_blockheight >= current_finalized_blochash; if !retain && v.status.is_none() { TX_TIMED_OUT.inc(); } diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs new file mode 100644 index 00000000..28f25a11 --- /dev/null +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -0,0 +1,444 @@ +use anyhow::bail; +use countmap::CountMap; +use crossbeam_channel::{Receiver, RecvError, RecvTimeoutError, Sender}; +use futures::future::join_all; +use log::{debug, error, info, trace, warn}; +use quinn::TokioRuntime; +use serde::de::Unexpected::Option; +use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; +use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; +use solana_lite_rpc_core::tx_store::empty_tx_store; +use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; +use solana_rpc_client::rpc_client::SerializableTransaction; +use solana_sdk::hash::Hash; +use solana_sdk::instruction::Instruction; +use solana_sdk::message::Message; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::{Keypair, Signature, Signer}; +use solana_sdk::signer::keypair; +use solana_sdk::transaction::{Transaction, VersionedTransaction}; +use solana_streamer::nonblocking::quic::ConnectionPeerType; +use solana_streamer::packet::PacketBatch; +use solana_streamer::quic::StreamStats; +use solana_streamer::streamer::StakedNodes; +use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; +use std::ops::Deref; +use std::path::Path; +use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, RwLock}; +use std::thread; +use std::time::{Duration, Instant}; +use tokio::runtime::{Builder, Runtime}; +use tokio::sync::broadcast; +use tokio::sync::broadcast::error::SendError; +use tokio::task::JoinHandle; +use tokio::time::{interval, sleep}; +use tokio::{join, spawn, task}; +use tracing_subscriber::{filter::LevelFilter, fmt}; + +// note: logging will be auto-adjusted +const SAMPLE_TX_COUNT: u32 = 20; + +const MAXIMUM_TRANSACTIONS_IN_QUEUE: usize = 200_000; +const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; // like solana repo + +#[test] // note: tokio runtimes get created as part of the integration test +pub fn wireup_and_send_txs_via_channel() { + configure_logging(); + + // value from solana - see quic streamer - see quic.rs -> rt() + const NUM_QUIC_STREAMER_WORKER_THREADS: usize = 1; + let runtime_quic1 = Builder::new_multi_thread() + .worker_threads(1) + .thread_name("quic-server") + .enable_all() + .build() + .expect("failed to build tokio runtime for testing quic server"); + + // lite-rpc + let runtime_literpc = tokio::runtime::Builder::new_multi_thread() + // see lite-rpc -> main.rs + .worker_threads(16) // note: this value has changed with the "deadlock fix" - TODO experiment with it + .enable_all() + .build() + .expect("failed to build tokio runtime for lite-rpc-tpu-client"); + + let literpc_validator_identity = Arc::new(Keypair::new()); + let udp_listen_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let listen_addr = udp_listen_socket.local_addr().unwrap(); + + let (inbound_packets_sender, inbound_packets_receiver) = crossbeam_channel::unbounded(); + + runtime_quic1.block_on(async { + /// setup solana Quic streamer + // see log "Start quic server on UdpSocket { addr: 127.0.0.1:xxxxx, fd: 10 }" + let staked_nodes = StakedNodes { + total_stake: 100, + max_stake: 40, + min_stake: 0, + ip_stake_map: Default::default(), + pubkey_stake_map: + // literpc_validator_identity.as_ref() + if STAKE_CONNECTION { + let mut map = HashMap::new(); + map.insert(literpc_validator_identity.pubkey(),30); + map + } else { HashMap::default() } + }; + + let _solana_quic_streamer = SolanaQuicStreamer::new_start_listening( + udp_listen_socket, + inbound_packets_sender, + staked_nodes, + ); + }); + + runtime_literpc.block_on(async { + tokio::spawn(start_literpc_client( + listen_addr, + literpc_validator_identity, + )); + }); + + let packet_consumer_jh = thread::spawn(move || { + info!("start pulling packets..."); + let mut packet_count = 0; + let time_to_first = Instant::now(); + let mut latest_tx = Instant::now(); + let timer = Instant::now(); + // second half + let mut timer2 = None; + let mut packet_count2 = 0; + let mut count_map: CountMap = CountMap::with_capacity(SAMPLE_TX_COUNT as usize); + const WARMUP_TX_COUNT: u32 = SAMPLE_TX_COUNT / 2; + while packet_count < SAMPLE_TX_COUNT { + if latest_tx.elapsed() > Duration::from_secs(5) { + warn!("abort after timeout waiting for packet from quic streamer"); + break; + } + + let packet_batch = match inbound_packets_receiver + .recv_timeout(Duration::from_millis(500)) + { + Ok(batch) => batch, + Err(_) => { + debug!("consumer thread did not receive packets on inbound channel recently - continue polling"); + continue; + } + }; + + // reset timer + latest_tx = Instant::now(); + + if packet_count == 0 { + info!( + "time to first packet {}ms", + time_to_first.elapsed().as_millis() + ); + } + + packet_count = packet_count + packet_batch.len() as u32; + if timer2.is_some() { + packet_count2 = packet_count2 + packet_batch.len() as u32; + } + + for packet in packet_batch.iter() { + let tx = packet + .deserialize_slice::(..) + .unwrap(); + trace!( + "read transaction from quic streaming server: {:?}", + tx.get_signature() + ); + count_map.insert_or_increment(*tx.get_signature()); + // for ix in tx.message.instructions() { + // info!("instruction: {:?}", ix.data); + // } + } + + if packet_count == WARMUP_TX_COUNT { + timer2 = Some(Instant::now()); + } + // info!("received packets so far: {}", packet_count); + if packet_count == SAMPLE_TX_COUNT { + break; + } + } // -- while not all packets received - by count + + let total_duration = timer.elapsed(); + let half_duration = timer2 + .map(|t| t.elapsed()) + .unwrap_or(Duration::from_secs(3333)); + + // throughput_50 is second half of transactions - should iron out startup effects + info!( + "consumed {} packets in {}us - throughput {:.2} tps, throughput_50 {:.2} tps, ", + packet_count, + total_duration.as_micros(), + packet_count as f64 / total_duration.as_secs_f64(), + packet_count2 as f64 / half_duration.as_secs_f64(), + ); + + info!("got all expected packets - shutting down tokio runtime with lite-rpc client"); + + assert_eq!( + count_map.len() as u32, + SAMPLE_TX_COUNT, + "count_map size should be equal to SAMPLE_TX_COUNT" + ); + // note: this assumption will not hold as soon as test is configured to do fanout + assert!( + count_map.values().all(|cnt| *cnt == 1), + "all transactions should be unique" + ); + + runtime_literpc.shutdown_timeout(Duration::from_millis(1000)); + }); + + // shutdown streamer + // solana_quic_streamer.shutdown().await; + + packet_consumer_jh.join().unwrap(); +} + +fn configure_logging() { + let env_filter = if SAMPLE_TX_COUNT < 100 { + "trace,rustls=info,quinn_proto=debug" + } else { + "debug,quinn_proto=info,rustls=info,solana_streamer=debug" + }; + tracing_subscriber::fmt::fmt() + // .with_max_level(LevelFilter::DEBUG) + .with_env_filter(env_filter) + .init(); +} + +const STAKE_CONNECTION: bool = true; + +const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameters { + connection_timeout: Duration::from_secs(1), + connection_retry_count: 10, + finalize_timeout: Duration::from_millis(200), + max_number_of_connections: 8, + unistream_timeout: Duration::from_millis(500), + write_timeout: Duration::from_secs(1), + number_of_transactions_per_unistream: 10, +}; + +async fn start_literpc_client( + streamer_listen_addrs: SocketAddr, + literpc_validator_identity: Arc, +) -> anyhow::Result<()> { + info!("Start lite-rpc test client ..."); + + let fanout_slots = 4; + + // (String, Vec) (signature, transaction) + let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); + let broadcast_sender = Arc::new(sender); + let (certificate, key) = new_self_signed_tls_certificate( + literpc_validator_identity.as_ref(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC connection certificates"); + + let tpu_connection_manager = + TpuConnectionManager::new(certificate, key, fanout_slots as usize).await; + + // this effectively controls how many connections we will have + let mut connections_to_keep: HashMap = HashMap::new(); + let addr1 = UdpSocket::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + connections_to_keep.insert( + Pubkey::from_str("1111111jepwNWbYG87sgwnBbUJnQHrPiUJzMpqJXZ")?, + addr1, + ); + + let addr2 = UdpSocket::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + connections_to_keep.insert( + Pubkey::from_str("1111111k4AYMctpyJakWNvGcte6tR8BLyZw54R8qu")?, + addr2, + ); + + // this is the real streamer + connections_to_keep.insert(literpc_validator_identity.pubkey(), streamer_listen_addrs); + + // get information about the optional validator identity stake + // populated from get_stakes_for_identity() + let identity_stakes = IdentityStakes { + peer_type: ConnectionPeerType::Staked, + stakes: if STAKE_CONNECTION { 30 } else { 0 }, // stake of lite-rpc + min_stakes: 0, + max_stakes: 40, + total_stakes: 100, + }; + + // solana_streamer::nonblocking::quic: Peer type: Staked, stake 30, total stake 0, max streams 128 receive_window Ok(12320) from peer 127.0.0.1:8000 + + tpu_connection_manager + .update_connections( + broadcast_sender.clone(), + connections_to_keep, + identity_stakes, + // note: tx_store is useless in this scenario as it is never changed; it's only used to check for duplicates + empty_tx_store().clone(), + QUIC_CONNECTION_PARAMS, + ) + .await; + + // TODO this is a race + sleep(Duration::from_millis(1500)).await; + + for i in 0..SAMPLE_TX_COUNT { + let raw_sample_tx = build_raw_sample_tx(i); + debug!( + "broadcast transaction {} to {} receivers: {}", + raw_sample_tx.0, + broadcast_sender.receiver_count(), + format!("hi {}", i) + ); + + broadcast_sender.send(raw_sample_tx)?; + } + + sleep(Duration::from_secs(30)).await; + + Ok(()) +} + +#[tokio::test] +// taken from solana -> test_nonblocking_quic_client_multiple_writes +async fn solana_quic_streamer_start() { + let (sender, _receiver) = crossbeam_channel::unbounded(); + let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); + // will create random free port + let sock = UdpSocket::bind("127.0.0.1:0").unwrap(); + let exit = Arc::new(AtomicBool::new(false)); + // keypair to derive the server tls certificate + let keypair = Keypair::new(); + // gossip_host is used in the server certificate + let gossip_host = "127.0.0.1".parse().unwrap(); + let stats = Arc::new(StreamStats::default()); + let (_, t) = solana_streamer::nonblocking::quic::spawn_server( + sock.try_clone().unwrap(), + &keypair, + gossip_host, + sender, + exit.clone(), + 1, + staked_nodes, + 10, + 10, + stats.clone(), + 1000, + ) + .unwrap(); + + let addr = sock.local_addr().unwrap().ip(); + let port = sock.local_addr().unwrap().port(); + let tpu_addr = SocketAddr::new(addr, port); + + // sleep(Duration::from_millis(500)).await; + + exit.store(true, Ordering::Relaxed); + t.await.unwrap(); + + stats.report(); +} + +struct SolanaQuicStreamer { + sock: UdpSocket, + exit: Arc, + join_handler: JoinHandle<()>, + stats: Arc, +} + +impl SolanaQuicStreamer { + pub fn get_socket_addr(&self) -> SocketAddr { + self.sock.local_addr().unwrap() + } +} + +impl SolanaQuicStreamer { + pub async fn shutdown(self) { + self.exit.store(true, Ordering::Relaxed); + self.join_handler.await.unwrap(); + self.stats.report(); + } +} + +impl SolanaQuicStreamer { + fn new_start_listening( + udp_socket: UdpSocket, + sender: Sender, + staked_nodes: StakedNodes, + ) -> Self { + let staked_nodes = Arc::new(RwLock::new(staked_nodes)); + let exit = Arc::new(AtomicBool::new(false)); + // keypair to derive the server tls certificate + let keypair = Keypair::new(); + // gossip_host is used in the server certificate + let gossip_host = "127.0.0.1".parse().unwrap(); + let stats = Arc::new(StreamStats::default()); + let (_, jh) = solana_streamer::nonblocking::quic::spawn_server( + udp_socket.try_clone().unwrap(), + &keypair, + gossip_host, + sender, + exit.clone(), + MAX_QUIC_CONNECTIONS_PER_PEER, + staked_nodes.clone(), + 10, + 10, + stats.clone(), + 1000, + ) + .unwrap(); + + let addr = udp_socket.local_addr().unwrap().ip(); + let port = udp_socket.local_addr().unwrap().port(); + let tpu_addr = SocketAddr::new(addr, port); + + Self { + sock: udp_socket, + exit, + join_handler: jh, + stats, + } + } +} + +const MEMO_PROGRAM_ID: &str = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"; + +pub fn build_raw_sample_tx(i: u32) -> (String, Vec) { + let payer_keypair = Keypair::from_base58_string( + "rKiJ7H5UUp3JR18kNyTF1XPuwPKHEM7gMLWHZPWP5djrW1vSjfwjhvJrevxF9MPmUmN9gJMLHZdLMgc9ao78eKr", + ); + + let tx = build_sample_tx(&payer_keypair, i); + + let raw_tx = bincode::serialize::(&tx).expect("failed to serialize tx"); + + (tx.get_signature().to_string(), raw_tx) +} + +fn build_sample_tx(payer_keypair: &Keypair, i: u32) -> VersionedTransaction { + let blockhash = Hash::default(); + create_memo_tx(format!("hi {}", i).as_bytes(), payer_keypair, blockhash).into() +} + +fn create_memo_tx(msg: &[u8], payer: &Keypair, blockhash: Hash) -> Transaction { + let memo = Pubkey::from_str(MEMO_PROGRAM_ID).unwrap(); + + let instruction = Instruction::new_with_bytes(memo, msg, vec![]); + let message = Message::new(&[instruction], Some(&payer.pubkey())); + Transaction::new(&[payer], message, blockhash) +} From 9e064001f8ce5b3aaf73622ffaaccb6bc1c7027a Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 20 Jul 2023 14:08:38 +0200 Subject: [PATCH 019/128] extract integration test --- Cargo.lock | 40 +++++++++++++++++++ Cargo.toml | 1 + core/src/solana_utils.rs | 35 ++++++++++++++++ services/src/transaction_service.rs | 2 +- ...literpc_tpu_quic_server_integrationtest.rs | 2 +- 5 files changed, 78 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22f5d894..523c0e7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4074,6 +4074,46 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "solana-lite-rpc-quic-forward-proxy-integration-test" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-channel", + "async-trait", + "base64 0.21.0", + "bincode", + "bs58", + "bytes", + "chrono", + "clap 4.2.4", + "countmap", + "crossbeam-channel", + "dashmap", + "dotenv", + "futures", + "itertools", + "lazy_static", + "log", + "native-tls", + "prometheus", + "quinn", + "rcgen 0.9.3", + "rustls 0.20.8", + "serde", + "serde_json", + "solana-lite-rpc-core", + "solana-lite-rpc-services", + "solana-net-utils", + "solana-sdk", + "solana-streamer", + "solana-transaction-status", + "spl-memo", + "thiserror", + "tokio", + "tracing-subscriber", +] + [[package]] name = "solana-lite-rpc-services" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index db88042b..daa148af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "services", "lite-rpc", "quic-forward-proxy", + "quic-forward-proxy-integration-test", "bench" ] diff --git a/core/src/solana_utils.rs b/core/src/solana_utils.rs index 21bdc860..2221988b 100644 --- a/core/src/solana_utils.rs +++ b/core/src/solana_utils.rs @@ -12,6 +12,11 @@ use std::{ }, time::Duration, }; +use serde::Serialize; +use solana_sdk::hash::Hash; +use solana_sdk::message::{LegacyMessage, v0}; +use solana_sdk::signature::Signature; +use solana_sdk::transaction::{Transaction, uses_durable_nonce, VersionedTransaction}; use tokio::sync::mpsc::UnboundedReceiver; const AVERAGE_SLOT_CHANGE_TIME_IN_MILLIS: u64 = 400; @@ -119,3 +124,33 @@ impl SolanaUtils { } } } + +/// Trait used to add support for versioned transactions to RPC APIs while +/// retaining backwards compatibility +pub trait SerializableTransaction: Serialize { + fn get_signature(&self) -> &Signature; + fn get_recent_blockhash(&self) -> &Hash; + fn uses_durable_nonce(&self) -> bool; +} +impl SerializableTransaction for Transaction { + fn get_signature(&self) -> &Signature { + &self.signatures[0] + } + fn get_recent_blockhash(&self) -> &Hash { + &self.message.recent_blockhash + } + fn uses_durable_nonce(&self) -> bool { + uses_durable_nonce(self).is_some() + } +} +impl SerializableTransaction for VersionedTransaction { + fn get_signature(&self) -> &Signature { + &self.signatures[0] + } + fn get_recent_blockhash(&self) -> &Hash { + self.message.recent_blockhash() + } + fn uses_durable_nonce(&self) -> bool { + self.uses_durable_nonce() + } +} diff --git a/services/src/transaction_service.rs b/services/src/transaction_service.rs index 961c7b48..d89bcb45 100644 --- a/services/src/transaction_service.rs +++ b/services/src/transaction_service.rs @@ -16,7 +16,7 @@ use solana_lite_rpc_core::{ notifications::NotificationSender, AnyhowJoinHandle, }; -use solana_rpc_client::rpc_client::SerializableTransaction; +use solana_lite_rpc_core::solana_utils::SerializableTransaction; use solana_sdk::{commitment_config::CommitmentConfig, transaction::VersionedTransaction}; use tokio::{ sync::mpsc::{self, Sender, UnboundedSender}, diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index 28f25a11..ce3cbc1d 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -8,8 +8,8 @@ use serde::de::Unexpected::Option; use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; use solana_lite_rpc_core::tx_store::empty_tx_store; +use solana_lite_rpc_core::solana_utils::SerializableTransaction; use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; -use solana_rpc_client::rpc_client::SerializableTransaction; use solana_sdk::hash::Hash; use solana_sdk::instruction::Instruction; use solana_sdk::message::Message; From 04147bf4991f72ca8b36a854e3b26367d226cb9d Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 20 Jul 2023 14:55:47 +0200 Subject: [PATCH 020/128] add tpu connection path --- services/src/tpu_utils/mod.rs | 1 + .../src/tpu_utils/tpu_connection_manager.rs | 33 +++++++++++++++++-- services/src/tpu_utils/tpu_connection_path.rs | 19 +++++++++++ services/src/tpu_utils/tpu_service.rs | 5 ++- 4 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 services/src/tpu_utils/tpu_connection_path.rs diff --git a/services/src/tpu_utils/mod.rs b/services/src/tpu_utils/mod.rs index 11b79705..a282c27f 100644 --- a/services/src/tpu_utils/mod.rs +++ b/services/src/tpu_utils/mod.rs @@ -1,2 +1,3 @@ pub mod tpu_connection_manager; pub mod tpu_service; +pub mod tpu_connection_path; diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 68a9b5f5..fcd0b485 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -1,5 +1,5 @@ use dashmap::DashMap; -use log::{error, trace}; +use log::{error, info, trace}; use prometheus::{core::GenericGauge, opts, register_int_gauge}; use quinn::Endpoint; use solana_lite_rpc_core::{ @@ -20,6 +20,8 @@ use std::{ }, }; use tokio::sync::{broadcast::Receiver, broadcast::Sender}; +use crate::tpu_utils::tpu_connection_path::TpuConnectionPath; +use crate::tpu_utils::tpu_connection_path::TpuConnectionPath::QuicForwardProxy; lazy_static::lazy_static! { static ref NB_QUIC_CONNECTIONS: GenericGauge = @@ -40,6 +42,7 @@ struct ActiveConnection { exit_signal: Arc, txs_sent_store: TxStore, connection_parameters: QuicConnectionParameters, + tpu_connection_path: TpuConnectionPath, } impl ActiveConnection { @@ -49,6 +52,7 @@ impl ActiveConnection { identity: Pubkey, txs_sent_store: TxStore, connection_parameters: QuicConnectionParameters, + tpu_connection_path: TpuConnectionPath, ) -> Self { Self { endpoints, @@ -57,6 +61,7 @@ impl ActiveConnection { exit_signal: Arc::new(AtomicBool::new(false)), txs_sent_store, connection_parameters, + tpu_connection_path } } @@ -159,7 +164,27 @@ impl ActiveConnection { tokio::spawn(async move { task_counter.fetch_add(1, Ordering::Relaxed); NB_QUIC_TASKS.inc(); - connection_pool.send_transaction_batch(txs).await; + + // TODO split to new service + + connection_pool.send_transaction_batch(txs).await; + + // match self.tpu_connection_path { + // QuicDirect => { + // connection_pool.send_transaction_batch(txs).await; + // }, + // QuicForwardProxy { forward_proxy_address } => { + // info!("Sending copy of transaction batch of {} to tpu with identity {} to quic proxy", + // txs.len(), identity); + // Self::send_copy_of_txs_to_quicproxy( + // &txs, endpoint.clone(), + // // proxy address + // forward_proxy_address, + // tpu_address, + // identity.clone()).await.unwrap(); + // } + // } + NB_QUIC_TASKS.dec(); task_counter.fetch_sub(1, Ordering::Relaxed); }); @@ -204,6 +229,7 @@ struct ActiveConnectionWithExitChannel { pub struct TpuConnectionManager { endpoints: RotatingQueue, identity_to_active_connection: Arc>>, + tpu_connection_path: TpuConnectionPath, } impl TpuConnectionManager { @@ -211,6 +237,7 @@ impl TpuConnectionManager { certificate: rustls::Certificate, key: rustls::PrivateKey, fanout: usize, + tpu_connection_path: TpuConnectionPath, ) -> Self { let number_of_clients = fanout * 2; Self { @@ -219,6 +246,7 @@ impl TpuConnectionManager { }) .await, identity_to_active_connection: Arc::new(DashMap::new()), + tpu_connection_path: tpu_connection_path.clone(), } } @@ -240,6 +268,7 @@ impl TpuConnectionManager { *identity, txs_sent_store.clone(), connection_parameters, + self.tpu_connection_path, ); // using mpsc as a oneshot channel/ because with one shot channel we cannot reuse the reciever let (sx, rx) = tokio::sync::mpsc::channel(1); diff --git a/services/src/tpu_utils/tpu_connection_path.rs b/services/src/tpu_utils/tpu_connection_path.rs new file mode 100644 index 00000000..965cc176 --- /dev/null +++ b/services/src/tpu_utils/tpu_connection_path.rs @@ -0,0 +1,19 @@ +use std::fmt::Display; +use std::net::SocketAddr; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum TpuConnectionPath { + QuicDirect, + QuicForwardProxy { forward_proxy_address: SocketAddr }, +} + +impl Display for TpuConnectionPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TpuConnectionPath::QuicDirect => write!(f, "Direct QUIC connection to TPU"), + TpuConnectionPath::QuicForwardProxy { forward_proxy_address } => { + write!(f, "QUIC Forward Proxy on {}", forward_proxy_address) + } + } + } +} diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 7b925033..3f1c4c98 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -26,6 +26,7 @@ use tokio::{ sync::RwLock, time::{Duration, Instant}, }; +use crate::tpu_utils::tpu_connection_path::TpuConnectionPath; lazy_static::lazy_static! { static ref NB_CLUSTER_NODES: GenericGauge = @@ -50,6 +51,7 @@ pub struct TpuServiceConfig { pub maximum_transaction_in_queue: usize, pub maximum_number_of_errors: usize, pub quic_connection_params: QuicConnectionParameters, + pub tpu_connection_path: TpuConnectionPath, } #[derive(Clone)] @@ -82,7 +84,8 @@ impl TpuService { .expect("Failed to initialize QUIC client certificates"); let tpu_connection_manager = - TpuConnectionManager::new(certificate, key, config.fanout_slots as usize).await; + TpuConnectionManager::new(certificate, key, + config.fanout_slots as usize, config.tpu_connection_path).await; Ok(Self { current_slot: Arc::new(AtomicU64::new(current_slot)), From 9c1ce6b9420ac3943001a27be299dcbc4c3007ac Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 20 Jul 2023 17:54:26 +0200 Subject: [PATCH 021/128] wire up quic proxy connection path --- .../Cargo.toml | 50 ++ .../tests/quic_proxy_tpu_integrationtest.rs | 458 ++++++++++++++++++ quic-forward-proxy/src/active_connection.rs | 95 +--- quic-forward-proxy/src/lib.rs | 12 + .../src/proxy_request_format.rs | 2 +- services/Cargo.toml | 1 + services/src/tpu_utils/mod.rs | 5 +- .../quic_proxy_connection_manager.rs | 206 ++++++++ .../src/tpu_utils/tpu_connection_manager.rs | 10 +- services/src/tpu_utils/tpu_connection_path.rs | 16 +- services/src/tpu_utils/tpu_service.rs | 67 ++- 11 files changed, 813 insertions(+), 109 deletions(-) create mode 100644 quic-forward-proxy-integration-test/Cargo.toml create mode 100644 quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs create mode 100644 quic-forward-proxy/src/lib.rs create mode 100644 services/src/tpu_utils/quic_proxy_connection_manager.rs diff --git a/quic-forward-proxy-integration-test/Cargo.toml b/quic-forward-proxy-integration-test/Cargo.toml new file mode 100644 index 00000000..bc5618fa --- /dev/null +++ b/quic-forward-proxy-integration-test/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "solana-lite-rpc-quic-forward-proxy-integration-test" +version = "0.1.0" +edition = "2021" +description = "Integration test for quic proxy " +rust-version = "1.67.1" +#default-run = "lite-rpc" +repository = "https://github.com/blockworks-foundation/lite-rpc" +license = "AGPL" +publish = false + +[dependencies] +solana-lite-rpc-core = { workspace = true } +solana-lite-rpc-services = { workspace = true } +solana-lite-rpc-quic-forward-proxy = { path = "../quic-forward-proxy" } +solana-sdk = { workspace = true } +solana-streamer = { workspace = true } +solana-transaction-status = { workspace = true } +solana-net-utils = { workspace = true } +rustls = { workspace = true, features = ["dangerous_configuration"]} +serde = { workspace = true } +serde_json = { workspace = true } +bincode = { workspace = true } +bs58 = { workspace = true } +base64 = { workspace = true } +thiserror = { workspace = true } +bytes = { workspace = true } +anyhow = { workspace = true } +log = { workspace = true } +clap = { workspace = true } +dashmap = { workspace = true } +itertools = { workspace = true } +tracing-subscriber = { workspace = true, features = ["std", "env-filter"] } +native-tls = { workspace = true } +prometheus = { workspace = true } +lazy_static = { workspace = true } +dotenv = { workspace = true } +async-channel = { workspace = true } +quinn = { workspace = true } +async-trait = { workspace = true } +chrono = { workspace = true } +tokio = { version = "1.28.2", features = ["full", "fs"]} +futures = { workspace = true } +rcgen = "0.9.3" +spl-memo = "3.0.1" + +[dev-dependencies] +crossbeam-channel = "0.5.6" +countmap = "0.2.0" + diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs new file mode 100644 index 00000000..94781d06 --- /dev/null +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -0,0 +1,458 @@ +use anyhow::bail; +use countmap::CountMap; +use crossbeam_channel::{Receiver, RecvError, RecvTimeoutError, Sender}; +use futures::future::join_all; +use log::{debug, error, info, trace, warn}; +use quinn::TokioRuntime; +use serde::de::Unexpected::Option; +use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; +use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; +use solana_lite_rpc_core::tx_store::empty_tx_store; +use solana_lite_rpc_core::solana_utils::SerializableTransaction; +use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; +use solana_sdk::hash::Hash; +use solana_sdk::instruction::Instruction; +use solana_sdk::message::Message; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::{Keypair, Signature, Signer}; +use solana_sdk::signer::keypair; +use solana_sdk::transaction::{Transaction, VersionedTransaction}; +use solana_streamer::nonblocking::quic::ConnectionPeerType; +use solana_streamer::packet::PacketBatch; +use solana_streamer::quic::StreamStats; +use solana_streamer::streamer::StakedNodes; +use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; +use std::ops::Deref; +use std::path::Path; +use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, RwLock}; +use std::thread; +use std::time::{Duration, Instant}; +use tokio::runtime::{Builder, Runtime}; +use tokio::sync::broadcast; +use tokio::sync::broadcast::error::SendError; +use tokio::task::JoinHandle; +use tokio::time::{interval, sleep}; +use tokio::{join, spawn, task}; +use tracing_subscriber::{filter::LevelFilter, fmt}; +use solana_lite_rpc_quic_forward_proxy::cli::get_identity_keypair; +use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; +use solana_lite_rpc_quic_forward_proxy::tls_config_provicer::SelfSignedTlsConfigProvider; +use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; +use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; +use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath::QuicForwardProxyPath; + +// note: logging will be auto-adjusted +const SAMPLE_TX_COUNT: u32 = 20; + +const MAXIMUM_TRANSACTIONS_IN_QUEUE: usize = 200_000; +const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; // like solana repo + +#[test] // note: tokio runtimes get created as part of the integration test +pub fn quic_proxy_and_solana_streamer() { + configure_logging(); + + // value from solana - see quic streamer - see quic.rs -> rt() + const NUM_QUIC_STREAMER_WORKER_THREADS: usize = 1; + let runtime_quic1 = Builder::new_multi_thread() + .worker_threads(1) + .thread_name("quic-server") + .enable_all() + .build() + .expect("failed to build tokio runtime for testing quic server"); + + // lite-rpc + let runtime_literpc = tokio::runtime::Builder::new_multi_thread() + // see lite-rpc -> main.rs + .worker_threads(16) // note: this value has changed with the "deadlock fix" - TODO experiment with it + .enable_all() + .build() + .expect("failed to build tokio runtime for lite-rpc-tpu-client"); + + // quic-forward-proxy + let runtime_quic_proxy = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to build tokio runtime for quic-forward-proxy"); + + + let literpc_validator_identity = Arc::new(Keypair::new()); + let udp_listen_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + let listen_addr = udp_listen_socket.local_addr().unwrap(); + + let proxy_listen_addr = UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap(); + + + let (inbound_packets_sender, inbound_packets_receiver) = crossbeam_channel::unbounded(); + + runtime_quic1.block_on(async { + /// setup solana Quic streamer + // see log "Start quic server on UdpSocket { addr: 127.0.0.1:xxxxx, fd: 10 }" + let staked_nodes = StakedNodes { + total_stake: 100, + max_stake: 40, + min_stake: 0, + ip_stake_map: Default::default(), + pubkey_stake_map: + // literpc_validator_identity.as_ref() + if STAKE_CONNECTION { + let mut map = HashMap::new(); + map.insert(literpc_validator_identity.pubkey(),30); + map + } else { HashMap::default() } + }; + + let _solana_quic_streamer = SolanaQuicStreamer::new_start_listening( + udp_listen_socket, + inbound_packets_sender, + staked_nodes, + ); + }); + + runtime_quic_proxy.block_on(async { + tokio::spawn(start_quic_proxy( + proxy_listen_addr, + )); + }); + + runtime_literpc.block_on(async { + tokio::spawn(start_literpc_client( + listen_addr, + literpc_validator_identity, + TpuConnectionPath::QuicForwardProxyPath { + forward_proxy_address: proxy_listen_addr, + }, + )); + }); + + let packet_consumer_jh = thread::spawn(move || { + info!("start pulling packets..."); + let mut packet_count = 0; + let time_to_first = Instant::now(); + let mut latest_tx = Instant::now(); + let timer = Instant::now(); + // second half + let mut timer2 = None; + let mut packet_count2 = 0; + let mut count_map: CountMap = CountMap::with_capacity(SAMPLE_TX_COUNT as usize); + const WARMUP_TX_COUNT: u32 = SAMPLE_TX_COUNT / 2; + while packet_count < SAMPLE_TX_COUNT { + if latest_tx.elapsed() > Duration::from_secs(5) { + warn!("abort after timeout waiting for packet from quic streamer"); + break; + } + + let packet_batch = match inbound_packets_receiver + .recv_timeout(Duration::from_millis(500)) + { + Ok(batch) => batch, + Err(_) => { + debug!("consumer thread did not receive packets on inbound channel recently - continue polling"); + continue; + } + }; + + // reset timer + latest_tx = Instant::now(); + + if packet_count == 0 { + info!( + "time to first packet {}ms", + time_to_first.elapsed().as_millis() + ); + } + + packet_count = packet_count + packet_batch.len() as u32; + if timer2.is_some() { + packet_count2 = packet_count2 + packet_batch.len() as u32; + } + + for packet in packet_batch.iter() { + let tx = packet + .deserialize_slice::(..) + .unwrap(); + trace!( + "read transaction from quic streaming server: {:?}", + tx.get_signature() + ); + count_map.insert_or_increment(*tx.get_signature()); + // for ix in tx.message.instructions() { + // info!("instruction: {:?}", ix.data); + // } + } + + if packet_count == WARMUP_TX_COUNT { + timer2 = Some(Instant::now()); + } + // info!("received packets so far: {}", packet_count); + if packet_count == SAMPLE_TX_COUNT { + break; + } + } // -- while not all packets received - by count + + let total_duration = timer.elapsed(); + let half_duration = timer2 + .map(|t| t.elapsed()) + .unwrap_or(Duration::from_secs(3333)); + + // throughput_50 is second half of transactions - should iron out startup effects + info!( + "consumed {} packets in {}us - throughput {:.2} tps, throughput_50 {:.2} tps, ", + packet_count, + total_duration.as_micros(), + packet_count as f64 / total_duration.as_secs_f64(), + packet_count2 as f64 / half_duration.as_secs_f64(), + ); + + info!("got all expected packets - shutting down tokio runtime with lite-rpc client"); + + assert_eq!( + count_map.len() as u32, + SAMPLE_TX_COUNT, + "count_map size should be equal to SAMPLE_TX_COUNT" + ); + // note: this assumption will not hold as soon as test is configured to do fanout + assert!( + count_map.values().all(|cnt| *cnt == 1), + "all transactions should be unique" + ); + + runtime_literpc.shutdown_timeout(Duration::from_millis(1000)); + }); + + // shutdown streamer + // solana_quic_streamer.shutdown().await; + + packet_consumer_jh.join().unwrap(); +} + +fn configure_logging() { + let env_filter = if SAMPLE_TX_COUNT < 100 { + "trace,rustls=info,quinn_proto=debug" + } else { + "debug,quinn_proto=info,rustls=info,solana_streamer=debug" + }; + tracing_subscriber::fmt::fmt() + // .with_max_level(LevelFilter::DEBUG) + .with_env_filter(env_filter) + .init(); +} + +const STAKE_CONNECTION: bool = true; + +const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameters { + connection_timeout: Duration::from_secs(1), + connection_retry_count: 10, + finalize_timeout: Duration::from_millis(200), + max_number_of_connections: 8, + unistream_timeout: Duration::from_millis(500), + write_timeout: Duration::from_secs(1), + number_of_transactions_per_unistream: 10, +}; + +async fn start_literpc_client( + streamer_listen_addrs: SocketAddr, + literpc_validator_identity: Arc, + tpu_connection_path: TpuConnectionPath, +) -> anyhow::Result<()> { + info!("Start lite-rpc test client using {} ...", tpu_connection_path); + + let fanout_slots = 4; + + // (String, Vec) (signature, transaction) + let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); + let broadcast_sender = Arc::new(sender); + let (certificate, key) = new_self_signed_tls_certificate( + literpc_validator_identity.as_ref(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC connection certificates"); + + // let tpu_connection_manager = + // TpuConnectionManager::new(certificate, key, fanout_slots as usize).await; + + let forward_proxy_address = match tpu_connection_path { + TpuConnectionPath::QuicForwardProxyPath { forward_proxy_address } => forward_proxy_address, + _ => panic!("Expected TpuConnectionPath::Quic"), + }; + + let quic_proxy_connection_manager = + QuicProxyConnectionManager::new(certificate, key, literpc_validator_identity.clone(), forward_proxy_address).await; + + // this effectively controls how many connections we will have + let mut connections_to_keep: HashMap = HashMap::new(); + let addr1 = UdpSocket::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + connections_to_keep.insert( + Pubkey::from_str("1111111jepwNWbYG87sgwnBbUJnQHrPiUJzMpqJXZ")?, + addr1, + ); + + let addr2 = UdpSocket::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + connections_to_keep.insert( + Pubkey::from_str("1111111k4AYMctpyJakWNvGcte6tR8BLyZw54R8qu")?, + addr2, + ); + + // this is the real streamer + connections_to_keep.insert(literpc_validator_identity.pubkey(), streamer_listen_addrs); + + // get information about the optional validator identity stake + // populated from get_stakes_for_identity() + let identity_stakes = IdentityStakes { + peer_type: ConnectionPeerType::Staked, + stakes: if STAKE_CONNECTION { 30 } else { 0 }, // stake of lite-rpc + min_stakes: 0, + max_stakes: 40, + total_stakes: 100, + }; + + // solana_streamer::nonblocking::quic: Peer type: Staked, stake 30, total stake 0, max streams 128 receive_window Ok(12320) from peer 127.0.0.1:8000 + // + // tpu_connection_manager + // .update_connections( + // broadcast_sender.clone(), + // connections_to_keep, + // identity_stakes, + // // note: tx_store is useless in this scenario as it is never changed; it's only used to check for duplicates + // empty_tx_store().clone(), + // QUIC_CONNECTION_PARAMS, + // ) + // .await; + + quic_proxy_connection_manager.update_connection(broadcast_sender.clone(), connections_to_keep).await; + + // TODO this is a race + sleep(Duration::from_millis(1500)).await; + + for i in 0..SAMPLE_TX_COUNT { + let raw_sample_tx = build_raw_sample_tx(i); + debug!( + "broadcast transaction {} to {} receivers: {}", + raw_sample_tx.0, + broadcast_sender.receiver_count(), + format!("hi {}", i) + ); + + broadcast_sender.send(raw_sample_tx)?; + } + + sleep(Duration::from_secs(30)).await; + + Ok(()) +} + +async fn start_quic_proxy(proxy_listen_addr: SocketAddr) -> anyhow::Result<()> { + + let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); + let random_unstaked_validator_identity = Arc::new(Keypair::new()); + + let tls_config = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); + let proxy_service = QuicForwardProxy::new(proxy_listen_addr, &tls_config, random_unstaked_validator_identity) + .await? + .start_services(); + + tokio::select! { + _ = proxy_service => { + panic!("Proxy service stopped unexpectedly") + }, + } +} + + +struct SolanaQuicStreamer { + sock: UdpSocket, + exit: Arc, + join_handler: JoinHandle<()>, + stats: Arc, +} + +impl SolanaQuicStreamer { + pub fn get_socket_addr(&self) -> SocketAddr { + self.sock.local_addr().unwrap() + } +} + +impl SolanaQuicStreamer { + pub async fn shutdown(self) { + self.exit.store(true, Ordering::Relaxed); + self.join_handler.await.unwrap(); + self.stats.report(); + } +} + +impl SolanaQuicStreamer { + fn new_start_listening( + udp_socket: UdpSocket, + sender: Sender, + staked_nodes: StakedNodes, + ) -> Self { + let staked_nodes = Arc::new(RwLock::new(staked_nodes)); + let exit = Arc::new(AtomicBool::new(false)); + // keypair to derive the server tls certificate + let keypair = Keypair::new(); + // gossip_host is used in the server certificate + let gossip_host = "127.0.0.1".parse().unwrap(); + let stats = Arc::new(StreamStats::default()); + let (_, jh) = solana_streamer::nonblocking::quic::spawn_server( + udp_socket.try_clone().unwrap(), + &keypair, + gossip_host, + sender, + exit.clone(), + MAX_QUIC_CONNECTIONS_PER_PEER, + staked_nodes.clone(), + 10, + 10, + stats.clone(), + 1000, + ) + .unwrap(); + + let addr = udp_socket.local_addr().unwrap().ip(); + let port = udp_socket.local_addr().unwrap().port(); + let tpu_addr = SocketAddr::new(addr, port); + + Self { + sock: udp_socket, + exit, + join_handler: jh, + stats, + } + } +} + +const MEMO_PROGRAM_ID: &str = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"; + +pub fn build_raw_sample_tx(i: u32) -> (String, Vec) { + let payer_keypair = Keypair::from_base58_string( + "rKiJ7H5UUp3JR18kNyTF1XPuwPKHEM7gMLWHZPWP5djrW1vSjfwjhvJrevxF9MPmUmN9gJMLHZdLMgc9ao78eKr", + ); + + let tx = build_sample_tx(&payer_keypair, i); + + let raw_tx = bincode::serialize::(&tx).expect("failed to serialize tx"); + + (tx.get_signature().to_string(), raw_tx) +} + +fn build_sample_tx(payer_keypair: &Keypair, i: u32) -> VersionedTransaction { + let blockhash = Hash::default(); + create_memo_tx(format!("hi {}", i).as_bytes(), payer_keypair, blockhash).into() +} + +fn create_memo_tx(msg: &[u8], payer: &Keypair, blockhash: Hash) -> Transaction { + let memo = Pubkey::from_str(MEMO_PROGRAM_ID).unwrap(); + + let instruction = Instruction::new_with_bytes(memo, msg, vec![]); + let message = Message::new(&[instruction], Some(&payer.pubkey())); + Transaction::new(&[payer], message, blockhash) +} diff --git a/quic-forward-proxy/src/active_connection.rs b/quic-forward-proxy/src/active_connection.rs index 9e8e7450..f13d474d 100644 --- a/quic-forward-proxy/src/active_connection.rs +++ b/quic-forward-proxy/src/active_connection.rs @@ -169,36 +169,20 @@ impl ActiveConnection { NB_QUIC_TASKS.inc(); let connection = connection.unwrap(); - if true { - // TODO split to new service - // SOS - info!("Sending copy of transaction batch of {} to tpu with identity {} to quic proxy", - txs.len(), identity); - Self::send_copy_of_txs_to_quicproxy( - &txs, endpoint.clone(), - // proxy address - "127.0.0.1:11111".parse().unwrap(), - tpu_address, - identity.clone()).await.unwrap(); - } - - - if false { - QuicConnectionUtils::send_transaction_batch( - connection, - txs, - identity, - endpoint, - tpu_address, - exit_signal, - last_stable_id, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - || { - // do nothing as we are using the same connection - } - ).await; - } + QuicConnectionUtils::send_transaction_batch( + connection, + txs, + identity, + endpoint, + tpu_address, + exit_signal, + last_stable_id, + QUIC_CONNECTION_TIMEOUT, + CONNECTION_RETRY_COUNT, + || { + // do nothing as we are using the same connection + } + ).await; NB_QUIC_TASKS.dec(); task_counter.fetch_sub(1, Ordering::Relaxed); @@ -214,57 +198,6 @@ impl ActiveConnection { NB_QUIC_ACTIVE_CONNECTIONS.dec(); } - async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, endpoint: Endpoint, - proxy_address: SocketAddr, tpu_target_address: SocketAddr, - identity: Pubkey) -> anyhow::Result<()> { - - info!("sending vecvec: {}", raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(",")); - - let raw_tx_batch_copy = raw_tx_batch.clone(); - - let mut txs = vec![]; - - for raw_tx in raw_tx_batch_copy { - let tx = match bincode::deserialize::(&raw_tx) { - Ok(tx) => tx, - Err(err) => { - bail!(err.to_string()); - } - }; - txs.push(tx); - } - - let forwarding_request = TpuForwardingRequest::new(tpu_target_address, identity, txs); - - let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); - - let send_result = timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)); - - match send_result.await { - Ok(..) => { - info!("Successfully sent data to quic proxy"); - } - Err(e) => { - warn!("Failed to send data to quic proxy: {:?}", e); - } - } - Ok(()) - } - - async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { - info!("sending {} bytes to proxy", proxy_request_raw.len()); - - let mut connecting = endpoint.connect(proxy_address, "localhost")?; - let connection = timeout(Duration::from_millis(500), connecting).await??; - let mut send = connection.open_uni().await?; - - send.write_all(proxy_request_raw).await?; - - send.finish().await?; - - Ok(()) - } - pub fn start_listening( &self, transaction_reciever: Receiver<(String, Vec)>, diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs new file mode 100644 index 00000000..bca2f999 --- /dev/null +++ b/quic-forward-proxy/src/lib.rs @@ -0,0 +1,12 @@ +pub mod quic_util; +pub mod tls_config_provicer; +pub mod proxy; +pub mod proxy_request_format; +pub mod tpu_quic_connection; +pub mod active_connection; +pub mod cli; +pub mod test_client; +mod util; +mod tx_store; +mod identity_stakes; +mod quic_connection_utils; diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 4675870d..969d68a0 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -16,7 +16,7 @@ const FORMAT_VERSION1: u16 = 2301; pub struct TpuForwardingRequest { format_version: u16, tpu_socket_addr: SocketAddr, // TODO is that correct - identity_tpunode: Pubkey, + identity_tpunode: Pubkey, // note: this is only used fro transactions: Vec, } diff --git a/services/Cargo.toml b/services/Cargo.toml index 7ef7b40d..c17c4e08 100644 --- a/services/Cargo.toml +++ b/services/Cargo.toml @@ -28,6 +28,7 @@ futures = { workspace = true } bytes = { workspace = true } anyhow = { workspace = true } itertools = { workspace = true } +async-trait = { workspace = true } log = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["std", "env-filter"] } diff --git a/services/src/tpu_utils/mod.rs b/services/src/tpu_utils/mod.rs index a282c27f..437f7a1b 100644 --- a/services/src/tpu_utils/mod.rs +++ b/services/src/tpu_utils/mod.rs @@ -1,3 +1,6 @@ -pub mod tpu_connection_manager; pub mod tpu_service; + pub mod tpu_connection_path; +pub mod tpu_connection_manager; +pub mod quic_proxy_connection_manager; + diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs new file mode 100644 index 00000000..206e4838 --- /dev/null +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -0,0 +1,206 @@ +use std::cell::Cell; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use std::sync::atomic::Ordering::Relaxed; +use std::thread; +use std::time::Duration; +use anyhow::bail; +use async_trait::async_trait; +use itertools::Itertools; +use log::{debug, error, info, warn}; +use quinn::Endpoint; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signer::Signer; +use solana_sdk::signature::Keypair; +use solana_sdk::transaction::VersionedTransaction; +use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; +use tokio::time::timeout; +use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; +use solana_lite_rpc_core::quic_connection_utils::{QuicConnectionParameters, QuicConnectionUtils}; +use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; +use solana_lite_rpc_core::tx_store::TxStore; + +#[derive(Clone, Copy, Debug)] +pub struct TpuNode { + pub tpu_identity: Pubkey, + pub tpu_address: SocketAddr, +} + +pub struct QuicProxyConnectionManager { + endpoint: Endpoint, + validator_identity: Arc, + simple_thread_started: AtomicBool, + proxy_addr: SocketAddr, + current_tpu_nodes: Arc>> +} + +impl QuicProxyConnectionManager { + pub async fn new( + certificate: rustls::Certificate, + key: rustls::PrivateKey, + validator_identity: Arc, + proxy_addr: SocketAddr, + ) -> Self { + let endpoint = QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()); + + Self { + endpoint, + validator_identity, + simple_thread_started: AtomicBool::from(false), + proxy_addr, + current_tpu_nodes: Arc::new(RwLock::new(vec![])), + } + } + + pub async fn update_connection( + &self, + transaction_sender: Arc)>>, + // for duration of this slot these tpu nodes will receive the transactions + connections_to_keep: HashMap, + ) { + debug!("reconfigure quic proxy connection (# of tpu nodes: {})", connections_to_keep.len()); + + + { + let mut lock = self.current_tpu_nodes.write().await; + + let list_of_nodes = connections_to_keep.iter().map(|(identity, tpu_address)| { + TpuNode { + tpu_identity: identity.clone(), + tpu_address: tpu_address.clone(), + } + }).collect_vec(); + + *lock = list_of_nodes; + } + + + if self.simple_thread_started.load(Relaxed) { + // already started + return; + } + + info!("Starting very simple proxy thread"); + + let mut transaction_receiver = transaction_sender.subscribe(); + + // TODO + + tokio::spawn(Self::read_transactions_and_broadcast( + transaction_receiver, + self.current_tpu_nodes.clone(), + self.proxy_addr, + self.endpoint.clone(), + )); + + } + + // blocks and loops until exit signal (TODO) + async fn read_transactions_and_broadcast( + mut transaction_receiver: Receiver<(String, Vec)>, + current_tpu_nodes: Arc>>, + proxy_addr: SocketAddr, + endpoint: Endpoint, + ) { + loop { + // TODO exit signal ??? + + tokio::select! { + // TODO add timeout + tx = transaction_receiver.recv() => { + // exit signal??? + + + let the_tx: Vec = match tx { + Ok((sig, tx)) => { + // if Self::check_for_confirmation(&txs_sent_store, sig) { + // // transaction is already confirmed/ no need to send + // continue; + // } + tx + }, + Err(e) => { + error!( + "Broadcast channel error on recv error {}", e); + continue; + } + }; + + // TODO read all txs from channel (see "let mut txs = vec![first_tx];") + let txs = vec![the_tx]; + + let tpu_fanout_nodes = current_tpu_nodes.read().await; + + info!("Sending copy of transaction batch of {} to {} tpu nodes via quic proxy", + txs.len(), tpu_fanout_nodes.len()); + + for target_tpu_node in &*tpu_fanout_nodes { + Self::send_copy_of_txs_to_quicproxy( + &txs, endpoint.clone(), + proxy_addr, + target_tpu_node.tpu_address, + target_tpu_node.tpu_identity).await.unwrap(); + } + + }, + }; + } + } + + async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, endpoint: Endpoint, + proxy_address: SocketAddr, tpu_target_address: SocketAddr, + target_tpu_identity: Pubkey) -> anyhow::Result<()> { + + info!("sending vecvec: {}", raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(",")); + + let raw_tx_batch_copy = raw_tx_batch.clone(); + + let mut txs = vec![]; + + for raw_tx in raw_tx_batch_copy { + let tx = match bincode::deserialize::(&raw_tx) { + Ok(tx) => tx, + Err(err) => { + bail!(err.to_string()); + } + }; + txs.push(tx); + } + + let forwarding_request = TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, txs); + + let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); + + let send_result = timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)); + + match send_result.await { + Ok(..) => { + info!("Successfully sent data to quic proxy"); + } + Err(e) => { + warn!("Failed to send data to quic proxy: {:?}", e); + } + } + Ok(()) + } + + + async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { + info!("sending {} bytes to proxy", proxy_request_raw.len()); + + let mut connecting = endpoint.connect(proxy_address, "localhost")?; + let connection = timeout(Duration::from_millis(500), connecting).await??; + let mut send = connection.open_uni().await?; + + send.write_all(proxy_request_raw).await?; + + send.finish().await?; + + Ok(()) + } + +} + + diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index fcd0b485..70fc9bae 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -21,7 +21,7 @@ use std::{ }; use tokio::sync::{broadcast::Receiver, broadcast::Sender}; use crate::tpu_utils::tpu_connection_path::TpuConnectionPath; -use crate::tpu_utils::tpu_connection_path::TpuConnectionPath::QuicForwardProxy; +use crate::tpu_utils::tpu_connection_path::TpuConnectionPath::QuicForwardProxyPath; lazy_static::lazy_static! { static ref NB_QUIC_CONNECTIONS: GenericGauge = @@ -42,7 +42,6 @@ struct ActiveConnection { exit_signal: Arc, txs_sent_store: TxStore, connection_parameters: QuicConnectionParameters, - tpu_connection_path: TpuConnectionPath, } impl ActiveConnection { @@ -52,7 +51,6 @@ impl ActiveConnection { identity: Pubkey, txs_sent_store: TxStore, connection_parameters: QuicConnectionParameters, - tpu_connection_path: TpuConnectionPath, ) -> Self { Self { endpoints, @@ -61,7 +59,6 @@ impl ActiveConnection { exit_signal: Arc::new(AtomicBool::new(false)), txs_sent_store, connection_parameters, - tpu_connection_path } } @@ -229,15 +226,14 @@ struct ActiveConnectionWithExitChannel { pub struct TpuConnectionManager { endpoints: RotatingQueue, identity_to_active_connection: Arc>>, - tpu_connection_path: TpuConnectionPath, } + impl TpuConnectionManager { pub async fn new( certificate: rustls::Certificate, key: rustls::PrivateKey, fanout: usize, - tpu_connection_path: TpuConnectionPath, ) -> Self { let number_of_clients = fanout * 2; Self { @@ -246,7 +242,6 @@ impl TpuConnectionManager { }) .await, identity_to_active_connection: Arc::new(DashMap::new()), - tpu_connection_path: tpu_connection_path.clone(), } } @@ -268,7 +263,6 @@ impl TpuConnectionManager { *identity, txs_sent_store.clone(), connection_parameters, - self.tpu_connection_path, ); // using mpsc as a oneshot channel/ because with one shot channel we cannot reuse the reciever let (sx, rx) = tokio::sync::mpsc::channel(1); diff --git a/services/src/tpu_utils/tpu_connection_path.rs b/services/src/tpu_utils/tpu_connection_path.rs index 965cc176..33b5bd1b 100644 --- a/services/src/tpu_utils/tpu_connection_path.rs +++ b/services/src/tpu_utils/tpu_connection_path.rs @@ -1,17 +1,25 @@ +use std::collections::HashMap; use std::fmt::Display; use std::net::SocketAddr; +use std::sync::Arc; +use async_trait::async_trait; +use solana_sdk::pubkey::Pubkey; +use tokio::sync::{broadcast::Receiver, broadcast::Sender}; +use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; +use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; +use solana_lite_rpc_core::tx_store::TxStore; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum TpuConnectionPath { - QuicDirect, - QuicForwardProxy { forward_proxy_address: SocketAddr }, + QuicDirectPath, + QuicForwardProxyPath { forward_proxy_address: SocketAddr }, } impl Display for TpuConnectionPath { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - TpuConnectionPath::QuicDirect => write!(f, "Direct QUIC connection to TPU"), - TpuConnectionPath::QuicForwardProxy { forward_proxy_address } => { + TpuConnectionPath::QuicDirectPath => write!(f, "Direct QUIC connection to TPU"), + TpuConnectionPath::QuicForwardProxyPath { forward_proxy_address } => { write!(f, "QUIC Forward Proxy on {}", forward_proxy_address) } } diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 3f1c4c98..431704ce 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -26,7 +26,9 @@ use tokio::{ sync::RwLock, time::{Duration, Instant}, }; +use crate::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; use crate::tpu_utils::tpu_connection_path::TpuConnectionPath; +use crate::tpu_utils::tpu_service::ConnectionManager::{DirectTpu, QuicProxy}; lazy_static::lazy_static! { static ref NB_CLUSTER_NODES: GenericGauge = @@ -60,7 +62,7 @@ pub struct TpuService { estimated_slot: Arc, rpc_client: Arc, broadcast_sender: Arc)>>, - tpu_connection_manager: Arc, + connection_manager: ConnectionManager, identity_stakes: Arc>, txs_sent_store: TxStore, leader_schedule: Arc, @@ -68,6 +70,12 @@ pub struct TpuService { identity: Pubkey, } +#[derive(Clone)] +enum ConnectionManager { + DirectTpu { tpu_connection_manager: Arc }, + QuicProxy { quic_proxy_connection_manager: Arc }, +} + impl TpuService { pub async fn new( config: TpuServiceConfig, @@ -83,9 +91,25 @@ impl TpuService { ) .expect("Failed to initialize QUIC client certificates"); - let tpu_connection_manager = - TpuConnectionManager::new(certificate, key, - config.fanout_slots as usize, config.tpu_connection_path).await; + let connection_manager = + match config.tpu_connection_path { + TpuConnectionPath::QuicDirectPath => { + let tpu_connection_manager = + TpuConnectionManager::new(certificate, key, + config.fanout_slots as usize).await; + DirectTpu { + tpu_connection_manager: Arc::new(tpu_connection_manager), + } + } + TpuConnectionPath::QuicForwardProxyPath { forward_proxy_address } => { + let quic_proxy_connection_manager = + QuicProxyConnectionManager::new(certificate, key, identity.clone(), forward_proxy_address).await; + + QuicProxy { + quic_proxy_connection_manager: Arc::new(quic_proxy_connection_manager), + } + } + }; Ok(Self { current_slot: Arc::new(AtomicU64::new(current_slot)), @@ -93,7 +117,7 @@ impl TpuService { leader_schedule: Arc::new(LeaderSchedule::new(config.number_of_leaders_to_cache)), rpc_client, broadcast_sender: Arc::new(sender), - tpu_connection_manager: Arc::new(tpu_connection_manager), + connection_manager, identity_stakes: Arc::new(RwLock::new(IdentityStakes::default())), txs_sent_store, identity: identity.pubkey(), @@ -128,6 +152,7 @@ impl TpuService { Ok(()) } + // update/reconfigure connections on slot change async fn update_quic_connections(&self) { let estimated_slot = self.estimated_slot.load(Ordering::Relaxed); let current_slot = self.current_slot.load(Ordering::Relaxed); @@ -156,15 +181,29 @@ impl TpuService { let identity_stakes = self.identity_stakes.read().await; - self.tpu_connection_manager - .update_connections( - self.broadcast_sender.clone(), - connections_to_keep, - *identity_stakes, - self.txs_sent_store.clone(), - self.config.quic_connection_params, - ) - .await; + match &self.connection_manager { + DirectTpu { + tpu_connection_manager, + } => { + tpu_connection_manager + .update_connections( + self.broadcast_sender.clone(), + connections_to_keep, + *identity_stakes, + self.txs_sent_store.clone(), + self.config.quic_connection_params, + ) + .await; + }, + QuicProxy { quic_proxy_connection_manager } => { + // TODO maybe we need more data (see .update_connections) + quic_proxy_connection_manager.update_connection( + self.broadcast_sender.clone(), + connections_to_keep, + ).await; + } + } + } fn update_current_slot( From c4093b58dbe12a64b05be10098f564de8c2f0ded Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 20 Jul 2023 18:07:50 +0200 Subject: [PATCH 022/128] test works - send tx via test quic proxy --- core/src/quic_connection_utils.rs | 2 +- .../quic_proxy_connection_manager.rs | 56 ++++++++++++++++--- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index 055d2aa1..8dd54314 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -196,7 +196,7 @@ impl QuicConnectionUtils { } } -struct SkipServerVerification; +pub struct SkipServerVerification; impl SkipServerVerification { pub fn new() -> Arc { diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 206e4838..12363c40 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -1,16 +1,16 @@ use std::cell::Cell; use std::collections::HashMap; -use std::net::SocketAddr; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use std::sync::atomic::Ordering::Relaxed; use std::thread; use std::time::Duration; -use anyhow::bail; +use anyhow::{bail, Context}; use async_trait::async_trait; use itertools::Itertools; use log::{debug, error, info, warn}; -use quinn::Endpoint; +use quinn::{ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig}; use solana_sdk::pubkey::Pubkey; use solana_sdk::signer::Signer; use solana_sdk::signature::Keypair; @@ -18,7 +18,7 @@ use solana_sdk::transaction::VersionedTransaction; use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; use tokio::time::timeout; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -use solana_lite_rpc_core::quic_connection_utils::{QuicConnectionParameters, QuicConnectionUtils}; +use solana_lite_rpc_core::quic_connection_utils::{QuicConnectionParameters, QuicConnectionUtils, SkipServerVerification}; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; use solana_lite_rpc_core::tx_store::TxStore; @@ -43,7 +43,7 @@ impl QuicProxyConnectionManager { validator_identity: Arc, proxy_addr: SocketAddr, ) -> Self { - let endpoint = QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()); + let endpoint = Self::create_proxy_client_endpoint(certificate.clone(), key.clone()); Self { endpoint, @@ -97,6 +97,42 @@ impl QuicProxyConnectionManager { } + fn create_proxy_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { + + const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; + + let mut endpoint = { + let client_socket = + solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::UNSPECIFIED), (8000, 10000)) + .expect("create_endpoint bind_in_range") + .1; + let config = EndpointConfig::default(); + quinn::Endpoint::new(config, None, client_socket, TokioRuntime) + .expect("create_endpoint quinn::Endpoint::new") + }; + + let mut crypto = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(SkipServerVerification::new()) + .with_single_cert(vec![certificate], key) + .expect("Failed to set QUIC client certificates"); + + crypto.enable_early_data = true; + crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; + + let mut config = ClientConfig::new(Arc::new(crypto)); + let mut transport_config = TransportConfig::default(); + + let timeout = IdleTimeout::try_from(Duration::from_secs(1)).unwrap(); + transport_config.max_idle_timeout(Some(timeout)); + transport_config.keep_alive_interval(Some(Duration::from_millis(500))); + config.transport_config(Arc::new(transport_config)); + + endpoint.set_default_client_config(config); + + endpoint + } + // blocks and loops until exit signal (TODO) async fn read_transactions_and_broadcast( mut transaction_receiver: Receiver<(String, Vec)>, @@ -173,14 +209,16 @@ impl QuicProxyConnectionManager { let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); - let send_result = timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)); + let send_result = + timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)) + .await.context("Timeout sending data to quic proxy")?; - match send_result.await { - Ok(..) => { + match send_result { + Ok(()) => { info!("Successfully sent data to quic proxy"); } Err(e) => { - warn!("Failed to send data to quic proxy: {:?}", e); + bail!("Failed to send data to quic proxy: {:?}", e); } } Ok(()) From 4c603e4e853cf15d211031af457e6f1af9cb4058 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 20 Jul 2023 18:23:57 +0200 Subject: [PATCH 023/128] use quic proxy in integration test --- .../tests/quic_proxy_tpu_integrationtest.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 94781d06..64edfe0c 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -122,9 +122,7 @@ pub fn quic_proxy_and_solana_streamer() { tokio::spawn(start_literpc_client( listen_addr, literpc_validator_identity, - TpuConnectionPath::QuicForwardProxyPath { - forward_proxy_address: proxy_listen_addr, - }, + proxy_listen_addr, )); }); @@ -256,9 +254,9 @@ const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameter async fn start_literpc_client( streamer_listen_addrs: SocketAddr, literpc_validator_identity: Arc, - tpu_connection_path: TpuConnectionPath, + forward_proxy_address: SocketAddr, ) -> anyhow::Result<()> { - info!("Start lite-rpc test client using {} ...", tpu_connection_path); + info!("Start lite-rpc test client using quic proxy at {} ...", forward_proxy_address); let fanout_slots = 4; @@ -274,11 +272,6 @@ async fn start_literpc_client( // let tpu_connection_manager = // TpuConnectionManager::new(certificate, key, fanout_slots as usize).await; - let forward_proxy_address = match tpu_connection_path { - TpuConnectionPath::QuicForwardProxyPath { forward_proxy_address } => forward_proxy_address, - _ => panic!("Expected TpuConnectionPath::Quic"), - }; - let quic_proxy_connection_manager = QuicProxyConnectionManager::new(certificate, key, literpc_validator_identity.clone(), forward_proxy_address).await; From 5d89c6304af58bf74565ad5e0a02e3947cc734d2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 21 Jul 2023 10:05:11 +0200 Subject: [PATCH 024/128] cleanup imports --- services/src/tpu_utils/quic_proxy_connection_manager.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 12363c40..a2de3ec8 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -8,6 +8,7 @@ use std::thread; use std::time::Duration; use anyhow::{bail, Context}; use async_trait::async_trait; +use futures::FutureExt; use itertools::Itertools; use log::{debug, error, info, warn}; use quinn::{ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig}; From a2a52be0317d65020bd23193e753ca6958ec66d2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 21 Jul 2023 10:05:48 +0200 Subject: [PATCH 025/128] integration test now supports proxy and non-proxy mode --- .../tests/quic_proxy_tpu_integrationtest.rs | 136 +++++++++++++++--- 1 file changed, 114 insertions(+), 22 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 64edfe0c..f89076c3 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -36,8 +36,6 @@ use tokio::sync::broadcast; use tokio::sync::broadcast::error::SendError; use tokio::task::JoinHandle; use tokio::time::{interval, sleep}; -use tokio::{join, spawn, task}; -use tracing_subscriber::{filter::LevelFilter, fmt}; use solana_lite_rpc_quic_forward_proxy::cli::get_identity_keypair; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_config_provicer::SelfSignedTlsConfigProvider; @@ -47,10 +45,23 @@ use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath: // note: logging will be auto-adjusted const SAMPLE_TX_COUNT: u32 = 20; +const STAKE_CONNECTION: bool = true; +const PROXY_MODE: bool = true; const MAXIMUM_TRANSACTIONS_IN_QUEUE: usize = 200_000; const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; // like solana repo + +const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameters { + connection_timeout: Duration::from_secs(1), + connection_retry_count: 10, + finalize_timeout: Duration::from_millis(200), + max_number_of_connections: 8, + unistream_timeout: Duration::from_millis(500), + write_timeout: Duration::from_secs(1), + number_of_transactions_per_unistream: 10, +}; + #[test] // note: tokio runtimes get created as part of the integration test pub fn quic_proxy_and_solana_streamer() { configure_logging(); @@ -119,11 +130,19 @@ pub fn quic_proxy_and_solana_streamer() { }); runtime_literpc.block_on(async { - tokio::spawn(start_literpc_client( - listen_addr, - literpc_validator_identity, - proxy_listen_addr, - )); + if PROXY_MODE { + tokio::spawn(start_literpc_client_proxy_mode( + listen_addr, + literpc_validator_identity, + proxy_listen_addr, + )); + } else { + tokio::spawn(start_literpc_client_direct_mode( + listen_addr, + literpc_validator_identity, + // proxy_listen_addr, + )); + } }); let packet_consumer_jh = thread::spawn(move || { @@ -205,8 +224,6 @@ pub fn quic_proxy_and_solana_streamer() { packet_count2 as f64 / half_duration.as_secs_f64(), ); - info!("got all expected packets - shutting down tokio runtime with lite-rpc client"); - assert_eq!( count_map.len() as u32, SAMPLE_TX_COUNT, @@ -229,9 +246,9 @@ pub fn quic_proxy_and_solana_streamer() { fn configure_logging() { let env_filter = if SAMPLE_TX_COUNT < 100 { - "trace,rustls=info,quinn_proto=debug" + "debug, rustls=info, quinn_proto=debug, solana_streamer=debug, solana_lite_rpc_quic_forward_proxy=trace " } else { - "debug,quinn_proto=info,rustls=info,solana_streamer=debug" + "debug, rustls=info, quinn_proto=info, solana_streamer=debug, solana_lite_rpc_quic_forward_proxy=debug " }; tracing_subscriber::fmt::fmt() // .with_max_level(LevelFilter::DEBUG) @@ -239,19 +256,94 @@ fn configure_logging() { .init(); } -const STAKE_CONNECTION: bool = true; +// no quic proxy +async fn start_literpc_client_direct_mode( + streamer_listen_addrs: SocketAddr, + literpc_validator_identity: Arc, +) -> anyhow::Result<()> { + info!("Start lite-rpc test client in direct-mode..."); -const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameters { - connection_timeout: Duration::from_secs(1), - connection_retry_count: 10, - finalize_timeout: Duration::from_millis(200), - max_number_of_connections: 8, - unistream_timeout: Duration::from_millis(500), - write_timeout: Duration::from_secs(1), - number_of_transactions_per_unistream: 10, -}; + let fanout_slots = 4; + + // (String, Vec) (signature, transaction) + let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); + let broadcast_sender = Arc::new(sender); + let (certificate, key) = new_self_signed_tls_certificate( + literpc_validator_identity.as_ref(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC connection certificates"); + + let tpu_connection_manager = + TpuConnectionManager::new(certificate, key, fanout_slots as usize).await; + + // this effectively controls how many connections we will have + let mut connections_to_keep: HashMap = HashMap::new(); + let addr1 = UdpSocket::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + connections_to_keep.insert( + Pubkey::from_str("1111111jepwNWbYG87sgwnBbUJnQHrPiUJzMpqJXZ")?, + addr1, + ); + + let addr2 = UdpSocket::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + connections_to_keep.insert( + Pubkey::from_str("1111111k4AYMctpyJakWNvGcte6tR8BLyZw54R8qu")?, + addr2, + ); + + // this is the real streamer + connections_to_keep.insert(literpc_validator_identity.pubkey(), streamer_listen_addrs); + + // get information about the optional validator identity stake + // populated from get_stakes_for_identity() + let identity_stakes = IdentityStakes { + peer_type: ConnectionPeerType::Staked, + stakes: if STAKE_CONNECTION { 30 } else { 0 }, // stake of lite-rpc + min_stakes: 0, + max_stakes: 40, + total_stakes: 100, + }; + + // solana_streamer::nonblocking::quic: Peer type: Staked, stake 30, total stake 0, max streams 128 receive_window Ok(12320) from peer 127.0.0.1:8000 + + tpu_connection_manager + .update_connections( + broadcast_sender.clone(), + connections_to_keep, + identity_stakes, + // note: tx_store is useless in this scenario as it is never changed; it's only used to check for duplicates + empty_tx_store().clone(), + QUIC_CONNECTION_PARAMS, + ) + .await; + + // TODO this is a race + sleep(Duration::from_millis(1500)).await; + + for i in 0..SAMPLE_TX_COUNT { + let raw_sample_tx = build_raw_sample_tx(i); + debug!( + "broadcast transaction {} to {} receivers: {}", + raw_sample_tx.0, + broadcast_sender.receiver_count(), + format!("hi {}", i) + ); + + broadcast_sender.send(raw_sample_tx)?; + } + + sleep(Duration::from_secs(30)).await; + + Ok(()) +} -async fn start_literpc_client( +async fn start_literpc_client_proxy_mode( streamer_listen_addrs: SocketAddr, literpc_validator_identity: Arc, forward_proxy_address: SocketAddr, From dd03d33c6a317bd8af479b3e055f6acdc6e60f85 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 21 Jul 2023 10:06:00 +0200 Subject: [PATCH 026/128] remove misleading log --- services/tests/literpc_tpu_quic_server_integrationtest.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index ce3cbc1d..eaed37c9 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -182,8 +182,6 @@ pub fn wireup_and_send_txs_via_channel() { packet_count2 as f64 / half_duration.as_secs_f64(), ); - info!("got all expected packets - shutting down tokio runtime with lite-rpc client"); - assert_eq!( count_map.len() as u32, SAMPLE_TX_COUNT, From 4c7f01de171185217236dc930cec10b2bf29a027 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 21 Jul 2023 17:46:41 +0200 Subject: [PATCH 027/128] fix env format --- .../tests/quic_proxy_tpu_integrationtest.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index f89076c3..e0dad30a 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -246,9 +246,9 @@ pub fn quic_proxy_and_solana_streamer() { fn configure_logging() { let env_filter = if SAMPLE_TX_COUNT < 100 { - "debug, rustls=info, quinn_proto=debug, solana_streamer=debug, solana_lite_rpc_quic_forward_proxy=trace " + "debug,rustls=info,quinn_proto=debug,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=trace" } else { - "debug, rustls=info, quinn_proto=info, solana_streamer=debug, solana_lite_rpc_quic_forward_proxy=debug " + "debug,rustls=info,quinn_proto=info,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=debug" }; tracing_subscriber::fmt::fmt() // .with_max_level(LevelFilter::DEBUG) From 21bef36a0ab7f26c964b94cbd4d9b1977d3eb4ad Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 21 Jul 2023 17:57:23 +0200 Subject: [PATCH 028/128] port refactored integration test to proxy branch --- .../tests/quic_proxy_tpu_integrationtest.rs | 287 ++++++++++++++---- ...literpc_tpu_quic_server_integrationtest.rs | 160 ++++++---- 2 files changed, 331 insertions(+), 116 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index e0dad30a..52513479 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -4,12 +4,11 @@ use crossbeam_channel::{Receiver, RecvError, RecvTimeoutError, Sender}; use futures::future::join_all; use log::{debug, error, info, trace, warn}; use quinn::TokioRuntime; -use serde::de::Unexpected::Option; use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; use solana_lite_rpc_core::tx_store::empty_tx_store; -use solana_lite_rpc_core::solana_utils::SerializableTransaction; use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; +use solana_lite_rpc_core::solana_utils::SerializableTransaction; use solana_sdk::hash::Hash; use solana_sdk::instruction::Instruction; use solana_sdk::message::Message; @@ -25,9 +24,10 @@ use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; use std::ops::Deref; +use std::option::Option; use std::path::Path; use std::str::FromStr; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; use std::thread; use std::time::{Duration, Instant}; @@ -36,36 +36,90 @@ use tokio::sync::broadcast; use tokio::sync::broadcast::error::SendError; use tokio::task::JoinHandle; use tokio::time::{interval, sleep}; -use solana_lite_rpc_quic_forward_proxy::cli::get_identity_keypair; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{filter::LevelFilter, fmt}; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_config_provicer::SelfSignedTlsConfigProvider; use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; -use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; -use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath::QuicForwardProxyPath; -// note: logging will be auto-adjusted -const SAMPLE_TX_COUNT: u32 = 20; -const STAKE_CONNECTION: bool = true; -const PROXY_MODE: bool = true; +#[derive(Copy, Clone, Debug)] +struct TestCaseParams { + sample_tx_count: u32, + stake_connection: bool, + proxy_mode: bool, +} const MAXIMUM_TRANSACTIONS_IN_QUEUE: usize = 200_000; const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; // like solana repo - const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameters { - connection_timeout: Duration::from_secs(1), + connection_timeout: Duration::from_secs(2), connection_retry_count: 10, - finalize_timeout: Duration::from_millis(200), + finalize_timeout: Duration::from_secs(2), max_number_of_connections: 8, - unistream_timeout: Duration::from_millis(500), - write_timeout: Duration::from_secs(1), + unistream_timeout: Duration::from_secs(2), + write_timeout: Duration::from_secs(2), number_of_transactions_per_unistream: 10, }; -#[test] // note: tokio runtimes get created as part of the integration test -pub fn quic_proxy_and_solana_streamer() { - configure_logging(); +#[test] +pub fn small_tx_batch_staked() { + configure_logging(true); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 20, + stake_connection: true, + proxy_mode: false, + }); +} + +#[test] +pub fn small_tx_batch_staked_proxy() { + configure_logging(true); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 20, + stake_connection: true, + proxy_mode: true, + }); +} + +#[test] +pub fn small_tx_batch_unstaked() { + configure_logging(true); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 20, + stake_connection: false, + proxy_mode: false, + }); +} + +#[test] +pub fn many_transactions() { + configure_logging(false); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 10000, + stake_connection: true, + proxy_mode: false, + }); +} + +#[ignore] +#[test] +pub fn too_many_transactions() { + configure_logging(false); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 100000, + stake_connection: false, + proxy_mode: false, + }); +} +// note: this not a tokio test as runtimes get created as part of the integration test +fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { // value from solana - see quic streamer - see quic.rs -> rt() const NUM_QUIC_STREAMER_WORKER_THREADS: usize = 1; let runtime_quic1 = Builder::new_multi_thread() @@ -76,9 +130,9 @@ pub fn quic_proxy_and_solana_streamer() { .expect("failed to build tokio runtime for testing quic server"); // lite-rpc - let runtime_literpc = tokio::runtime::Builder::new_multi_thread() + let runtime_literpc = Builder::new_multi_thread() // see lite-rpc -> main.rs - .worker_threads(16) // note: this value has changed with the "deadlock fix" - TODO experiment with it + .worker_threads(16) // also works with 1 .enable_all() .build() .expect("failed to build tokio runtime for lite-rpc-tpu-client"); @@ -89,14 +143,12 @@ pub fn quic_proxy_and_solana_streamer() { .build() .expect("failed to build tokio runtime for quic-forward-proxy"); - let literpc_validator_identity = Arc::new(Keypair::new()); let udp_listen_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); let listen_addr = udp_listen_socket.local_addr().unwrap(); let proxy_listen_addr = UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap(); - let (inbound_packets_sender, inbound_packets_receiver) = crossbeam_channel::unbounded(); runtime_quic1.block_on(async { @@ -107,13 +159,13 @@ pub fn quic_proxy_and_solana_streamer() { max_stake: 40, min_stake: 0, ip_stake_map: Default::default(), - pubkey_stake_map: - // literpc_validator_identity.as_ref() - if STAKE_CONNECTION { - let mut map = HashMap::new(); - map.insert(literpc_validator_identity.pubkey(),30); - map - } else { HashMap::default() } + pubkey_stake_map: if test_case_params.stake_connection { + let mut map = HashMap::new(); + map.insert(literpc_validator_identity.pubkey(), 30); + map + } else { + HashMap::default() + }, }; let _solana_quic_streamer = SolanaQuicStreamer::new_start_listening( @@ -129,15 +181,18 @@ pub fn quic_proxy_and_solana_streamer() { )); }); + runtime_literpc.block_on(async { - if PROXY_MODE { + if test_case_params.proxy_mode { tokio::spawn(start_literpc_client_proxy_mode( + test_case_params.clone(), listen_addr, literpc_validator_identity, proxy_listen_addr, )); } else { tokio::spawn(start_literpc_client_direct_mode( + test_case_params.clone(), listen_addr, literpc_validator_identity, // proxy_listen_addr, @@ -154,9 +209,10 @@ pub fn quic_proxy_and_solana_streamer() { // second half let mut timer2 = None; let mut packet_count2 = 0; - let mut count_map: CountMap = CountMap::with_capacity(SAMPLE_TX_COUNT as usize); - const WARMUP_TX_COUNT: u32 = SAMPLE_TX_COUNT / 2; - while packet_count < SAMPLE_TX_COUNT { + let mut count_map: CountMap = + CountMap::with_capacity(test_case_params.sample_tx_count as usize); + let warmup_tx_count: u32 = test_case_params.sample_tx_count / 2; + while packet_count < test_case_params.sample_tx_count { if latest_tx.elapsed() > Duration::from_secs(5) { warn!("abort after timeout waiting for packet from quic streamer"); break; @@ -196,16 +252,12 @@ pub fn quic_proxy_and_solana_streamer() { tx.get_signature() ); count_map.insert_or_increment(*tx.get_signature()); - // for ix in tx.message.instructions() { - // info!("instruction: {:?}", ix.data); - // } } - if packet_count == WARMUP_TX_COUNT { + if packet_count == warmup_tx_count { timer2 = Some(Instant::now()); } - // info!("received packets so far: {}", packet_count); - if packet_count == SAMPLE_TX_COUNT { + if packet_count == test_case_params.sample_tx_count { break; } } // -- while not all packets received - by count @@ -224,10 +276,12 @@ pub fn quic_proxy_and_solana_streamer() { packet_count2 as f64 / half_duration.as_secs_f64(), ); + info!("got all expected packets - shutting down tokio runtime with lite-rpc client"); + assert_eq!( count_map.len() as u32, - SAMPLE_TX_COUNT, - "count_map size should be equal to SAMPLE_TX_COUNT" + test_case_params.sample_tx_count, + "count_map size should be equal to sample_tx_count" ); // note: this assumption will not hold as soon as test is configured to do fanout assert!( @@ -238,26 +292,150 @@ pub fn quic_proxy_and_solana_streamer() { runtime_literpc.shutdown_timeout(Duration::from_millis(1000)); }); - // shutdown streamer - // solana_quic_streamer.shutdown().await; - packet_consumer_jh.join().unwrap(); } -fn configure_logging() { - let env_filter = if SAMPLE_TX_COUNT < 100 { +fn configure_logging(verbose: bool) { + let env_filter = if verbose { "debug,rustls=info,quinn_proto=debug,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=trace" } else { "debug,rustls=info,quinn_proto=info,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=debug" }; - tracing_subscriber::fmt::fmt() - // .with_max_level(LevelFilter::DEBUG) + let result = tracing_subscriber::fmt::fmt() .with_env_filter(env_filter) - .init(); + .try_init(); + if result.is_err() { + println!("Logging already initialized - ignore"); + } } +async fn start_literpc_client( + test_case_params: TestCaseParams, + streamer_listen_addrs: SocketAddr, + literpc_validator_identity: Arc, +) -> anyhow::Result<()> { + info!("Start lite-rpc test client ..."); + + let fanout_slots = 4; + + // (String, Vec) (signature, transaction) + let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); + let broadcast_sender = Arc::new(sender); + let (certificate, key) = new_self_signed_tls_certificate( + literpc_validator_identity.as_ref(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC connection certificates"); + + let tpu_connection_manager = + TpuConnectionManager::new(certificate, key, fanout_slots as usize).await; + + // this effectively controls how many connections we will have + let mut connections_to_keep: HashMap = HashMap::new(); + let addr1 = UdpSocket::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + connections_to_keep.insert( + Pubkey::from_str("1111111jepwNWbYG87sgwnBbUJnQHrPiUJzMpqJXZ")?, + addr1, + ); + + let addr2 = UdpSocket::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + connections_to_keep.insert( + Pubkey::from_str("1111111k4AYMctpyJakWNvGcte6tR8BLyZw54R8qu")?, + addr2, + ); + + // this is the real streamer + connections_to_keep.insert(literpc_validator_identity.pubkey(), streamer_listen_addrs); + + // get information about the optional validator identity stake + // populated from get_stakes_for_identity() + let identity_stakes = IdentityStakes { + peer_type: ConnectionPeerType::Staked, + stakes: if test_case_params.stake_connection { + 30 + } else { + 0 + }, // stake of lite-rpc + min_stakes: 0, + max_stakes: 40, + total_stakes: 100, + }; + + // solana_streamer::nonblocking::quic: Peer type: Staked, stake 30, total stake 0, max streams 128 receive_window Ok(12320) from peer 127.0.0.1:8000 + + tpu_connection_manager + .update_connections( + broadcast_sender.clone(), + connections_to_keep, + identity_stakes, + // note: tx_store is useless in this scenario as it is never changed; it's only used to check for duplicates + empty_tx_store().clone(), + QUIC_CONNECTION_PARAMS, + ) + .await; + + for i in 0..test_case_params.sample_tx_count { + let raw_sample_tx = build_raw_sample_tx(i); + broadcast_sender.send(raw_sample_tx)?; + } + + // we need that to keep the tokio runtime dedicated to lite-rpc up long enough + sleep(Duration::from_secs(30)).await; + + // reaching this point means there is problem with test setup and the consumer threed + panic!("should never reach this point") +} + +#[tokio::test] +// taken from solana -> test_nonblocking_quic_client_multiple_writes +async fn solana_quic_streamer_start() { + let (sender, _receiver) = crossbeam_channel::unbounded(); + let staked_nodes = Arc::new(RwLock::new(StakedNodes::default())); + // will create random free port + let sock = UdpSocket::bind("127.0.0.1:0").unwrap(); + let exit = Arc::new(AtomicBool::new(false)); + // keypair to derive the server tls certificate + let keypair = Keypair::new(); + // gossip_host is used in the server certificate + let gossip_host = "127.0.0.1".parse().unwrap(); + let stats = Arc::new(StreamStats::default()); + let (_, t) = solana_streamer::nonblocking::quic::spawn_server( + sock.try_clone().unwrap(), + &keypair, + gossip_host, + sender, + exit.clone(), + 1, + staked_nodes, + 10, + 10, + stats.clone(), + 1000, + ) + .unwrap(); + + let addr = sock.local_addr().unwrap().ip(); + let port = sock.local_addr().unwrap().port(); + let tpu_addr = SocketAddr::new(addr, port); + + // sleep(Duration::from_millis(500)).await; + + exit.store(true, Ordering::Relaxed); + t.await.unwrap(); + + stats.report(); +} + + // no quic proxy async fn start_literpc_client_direct_mode( + test_case_params: TestCaseParams, streamer_listen_addrs: SocketAddr, literpc_validator_identity: Arc, ) -> anyhow::Result<()> { @@ -304,7 +482,7 @@ async fn start_literpc_client_direct_mode( // populated from get_stakes_for_identity() let identity_stakes = IdentityStakes { peer_type: ConnectionPeerType::Staked, - stakes: if STAKE_CONNECTION { 30 } else { 0 }, // stake of lite-rpc + stakes: if test_case_params.stake_connection { 30 } else { 0 }, // stake of lite-rpc min_stakes: 0, max_stakes: 40, total_stakes: 100, @@ -326,7 +504,7 @@ async fn start_literpc_client_direct_mode( // TODO this is a race sleep(Duration::from_millis(1500)).await; - for i in 0..SAMPLE_TX_COUNT { + for i in 0..test_case_params.sample_tx_count { let raw_sample_tx = build_raw_sample_tx(i); debug!( "broadcast transaction {} to {} receivers: {}", @@ -344,6 +522,7 @@ async fn start_literpc_client_direct_mode( } async fn start_literpc_client_proxy_mode( + test_case_params: TestCaseParams, streamer_listen_addrs: SocketAddr, literpc_validator_identity: Arc, forward_proxy_address: SocketAddr, @@ -359,7 +538,7 @@ async fn start_literpc_client_proxy_mode( literpc_validator_identity.as_ref(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) - .expect("Failed to initialize QUIC connection certificates"); + .expect("Failed to initialize QUIC connection certificates"); // let tpu_connection_manager = // TpuConnectionManager::new(certificate, key, fanout_slots as usize).await; @@ -394,7 +573,7 @@ async fn start_literpc_client_proxy_mode( // populated from get_stakes_for_identity() let identity_stakes = IdentityStakes { peer_type: ConnectionPeerType::Staked, - stakes: if STAKE_CONNECTION { 30 } else { 0 }, // stake of lite-rpc + stakes: if test_case_params.stake_connection { 30 } else { 0 }, // stake of lite-rpc min_stakes: 0, max_stakes: 40, total_stakes: 100, @@ -418,7 +597,7 @@ async fn start_literpc_client_proxy_mode( // TODO this is a race sleep(Duration::from_millis(1500)).await; - for i in 0..SAMPLE_TX_COUNT { + for i in 0..test_case_params.sample_tx_count { let raw_sample_tx = build_raw_sample_tx(i); debug!( "broadcast transaction {} to {} receivers: {}", diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index eaed37c9..4bf8fff1 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -4,12 +4,11 @@ use crossbeam_channel::{Receiver, RecvError, RecvTimeoutError, Sender}; use futures::future::join_all; use log::{debug, error, info, trace, warn}; use quinn::TokioRuntime; -use serde::de::Unexpected::Option; use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; use solana_lite_rpc_core::tx_store::empty_tx_store; -use solana_lite_rpc_core::solana_utils::SerializableTransaction; use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; +use solana_rpc_client::rpc_client::SerializableTransaction; use solana_sdk::hash::Hash; use solana_sdk::instruction::Instruction; use solana_sdk::message::Message; @@ -25,9 +24,10 @@ use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; use std::ops::Deref; +use std::option::Option; use std::path::Path; use std::str::FromStr; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::{Arc, RwLock}; use std::thread; use std::time::{Duration, Instant}; @@ -36,19 +36,71 @@ use tokio::sync::broadcast; use tokio::sync::broadcast::error::SendError; use tokio::task::JoinHandle; use tokio::time::{interval, sleep}; -use tokio::{join, spawn, task}; +use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{filter::LevelFilter, fmt}; -// note: logging will be auto-adjusted -const SAMPLE_TX_COUNT: u32 = 20; +#[derive(Copy, Clone, Debug)] +struct TestCaseParams { + sample_tx_count: u32, + stake_connection: bool, +} const MAXIMUM_TRANSACTIONS_IN_QUEUE: usize = 200_000; const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; // like solana repo -#[test] // note: tokio runtimes get created as part of the integration test -pub fn wireup_and_send_txs_via_channel() { - configure_logging(); +const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameters { + connection_timeout: Duration::from_secs(2), + connection_retry_count: 10, + finalize_timeout: Duration::from_secs(2), + max_number_of_connections: 8, + unistream_timeout: Duration::from_secs(2), + write_timeout: Duration::from_secs(2), + number_of_transactions_per_unistream: 10, +}; + +#[test] +pub fn small_tx_batch_staked() { + configure_logging(true); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 20, + stake_connection: true, + }); +} + +#[test] +pub fn small_tx_batch_unstaked() { + configure_logging(true); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 20, + stake_connection: false, + }); +} + +#[test] +pub fn many_transactions() { + configure_logging(false); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 10000, + stake_connection: true, + }); +} +#[ignore] +#[test] +pub fn too_many_transactions() { + configure_logging(false); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 100000, + stake_connection: false, + }); +} + +// note: this not a tokio test as runtimes get created as part of the integration test +fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { // value from solana - see quic streamer - see quic.rs -> rt() const NUM_QUIC_STREAMER_WORKER_THREADS: usize = 1; let runtime_quic1 = Builder::new_multi_thread() @@ -59,9 +111,9 @@ pub fn wireup_and_send_txs_via_channel() { .expect("failed to build tokio runtime for testing quic server"); // lite-rpc - let runtime_literpc = tokio::runtime::Builder::new_multi_thread() + let runtime_literpc = Builder::new_multi_thread() // see lite-rpc -> main.rs - .worker_threads(16) // note: this value has changed with the "deadlock fix" - TODO experiment with it + .worker_threads(16) // also works with 1 .enable_all() .build() .expect("failed to build tokio runtime for lite-rpc-tpu-client"); @@ -80,13 +132,13 @@ pub fn wireup_and_send_txs_via_channel() { max_stake: 40, min_stake: 0, ip_stake_map: Default::default(), - pubkey_stake_map: - // literpc_validator_identity.as_ref() - if STAKE_CONNECTION { - let mut map = HashMap::new(); - map.insert(literpc_validator_identity.pubkey(),30); - map - } else { HashMap::default() } + pubkey_stake_map: if test_case_params.stake_connection { + let mut map = HashMap::new(); + map.insert(literpc_validator_identity.pubkey(), 30); + map + } else { + HashMap::default() + }, }; let _solana_quic_streamer = SolanaQuicStreamer::new_start_listening( @@ -98,6 +150,7 @@ pub fn wireup_and_send_txs_via_channel() { runtime_literpc.block_on(async { tokio::spawn(start_literpc_client( + test_case_params.clone(), listen_addr, literpc_validator_identity, )); @@ -112,9 +165,10 @@ pub fn wireup_and_send_txs_via_channel() { // second half let mut timer2 = None; let mut packet_count2 = 0; - let mut count_map: CountMap = CountMap::with_capacity(SAMPLE_TX_COUNT as usize); - const WARMUP_TX_COUNT: u32 = SAMPLE_TX_COUNT / 2; - while packet_count < SAMPLE_TX_COUNT { + let mut count_map: CountMap = + CountMap::with_capacity(test_case_params.sample_tx_count as usize); + let warmup_tx_count: u32 = test_case_params.sample_tx_count / 2; + while packet_count < test_case_params.sample_tx_count { if latest_tx.elapsed() > Duration::from_secs(5) { warn!("abort after timeout waiting for packet from quic streamer"); break; @@ -154,16 +208,12 @@ pub fn wireup_and_send_txs_via_channel() { tx.get_signature() ); count_map.insert_or_increment(*tx.get_signature()); - // for ix in tx.message.instructions() { - // info!("instruction: {:?}", ix.data); - // } } - if packet_count == WARMUP_TX_COUNT { + if packet_count == warmup_tx_count { timer2 = Some(Instant::now()); } - // info!("received packets so far: {}", packet_count); - if packet_count == SAMPLE_TX_COUNT { + if packet_count == test_case_params.sample_tx_count { break; } } // -- while not all packets received - by count @@ -182,10 +232,12 @@ pub fn wireup_and_send_txs_via_channel() { packet_count2 as f64 / half_duration.as_secs_f64(), ); + info!("got all expected packets - shutting down tokio runtime with lite-rpc client"); + assert_eq!( count_map.len() as u32, - SAMPLE_TX_COUNT, - "count_map size should be equal to SAMPLE_TX_COUNT" + test_case_params.sample_tx_count, + "count_map size should be equal to sample_tx_count" ); // note: this assumption will not hold as soon as test is configured to do fanout assert!( @@ -196,37 +248,25 @@ pub fn wireup_and_send_txs_via_channel() { runtime_literpc.shutdown_timeout(Duration::from_millis(1000)); }); - // shutdown streamer - // solana_quic_streamer.shutdown().await; - packet_consumer_jh.join().unwrap(); } -fn configure_logging() { - let env_filter = if SAMPLE_TX_COUNT < 100 { +fn configure_logging(verbose: bool) { + let env_filter = if verbose { "trace,rustls=info,quinn_proto=debug" } else { "debug,quinn_proto=info,rustls=info,solana_streamer=debug" }; - tracing_subscriber::fmt::fmt() - // .with_max_level(LevelFilter::DEBUG) + let result = tracing_subscriber::fmt::fmt() .with_env_filter(env_filter) - .init(); + .try_init(); + if result.is_err() { + println!("Logging already initialized - ignore"); + } } -const STAKE_CONNECTION: bool = true; - -const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameters { - connection_timeout: Duration::from_secs(1), - connection_retry_count: 10, - finalize_timeout: Duration::from_millis(200), - max_number_of_connections: 8, - unistream_timeout: Duration::from_millis(500), - write_timeout: Duration::from_secs(1), - number_of_transactions_per_unistream: 10, -}; - async fn start_literpc_client( + test_case_params: TestCaseParams, streamer_listen_addrs: SocketAddr, literpc_validator_identity: Arc, ) -> anyhow::Result<()> { @@ -273,7 +313,11 @@ async fn start_literpc_client( // populated from get_stakes_for_identity() let identity_stakes = IdentityStakes { peer_type: ConnectionPeerType::Staked, - stakes: if STAKE_CONNECTION { 30 } else { 0 }, // stake of lite-rpc + stakes: if test_case_params.stake_connection { + 30 + } else { + 0 + }, // stake of lite-rpc min_stakes: 0, max_stakes: 40, total_stakes: 100, @@ -292,24 +336,16 @@ async fn start_literpc_client( ) .await; - // TODO this is a race - sleep(Duration::from_millis(1500)).await; - - for i in 0..SAMPLE_TX_COUNT { + for i in 0..test_case_params.sample_tx_count { let raw_sample_tx = build_raw_sample_tx(i); - debug!( - "broadcast transaction {} to {} receivers: {}", - raw_sample_tx.0, - broadcast_sender.receiver_count(), - format!("hi {}", i) - ); - broadcast_sender.send(raw_sample_tx)?; } + // we need that to keep the tokio runtime dedicated to lite-rpc up long enough sleep(Duration::from_secs(30)).await; - Ok(()) + // reaching this point means there is problem with test setup and the consumer threed + panic!("should never reach this point") } #[tokio::test] From 9ab08d241b18dc8d8d13022ba4f7db2e80ae808b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 21 Jul 2023 18:08:32 +0200 Subject: [PATCH 029/128] fix compile issues --- lite-rpc/src/bridge.rs | 5 +++- lite-rpc/src/main.rs | 5 ++-- .../tests/quic_proxy_tpu_integrationtest.rs | 24 ++++++++++++++++++- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index e0370c2e..e01b41fa 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -38,10 +38,12 @@ use solana_sdk::{ }; use solana_transaction_status::TransactionStatus; use std::{ops::Deref, str::FromStr, sync::Arc, time::Duration}; +use std::convert::identity; use tokio::{ net::ToSocketAddrs, sync::mpsc::{self, Sender}, }; +use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; lazy_static::lazy_static! { static ref RPC_SEND_TX: IntCounter = @@ -106,11 +108,12 @@ impl LiteBridge { write_timeout: Duration::from_secs(1), number_of_transactions_per_unistream: 8, }, + tpu_connection_path: TpuConnectionPath::QuicDirectPath, }; let tpu_service = TpuService::new( tpu_config, - Arc::new(identity), + validator_identity, current_slot, rpc_client.clone(), tx_store.clone(), diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 3635197d..2bd0e644 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -9,6 +9,7 @@ use lite_rpc::{bridge::LiteBridge, cli::Args}; use solana_sdk::signature::Keypair; use std::env; +use std::sync::Arc; use crate::rpc_tester::RpcTester; @@ -49,7 +50,7 @@ pub async fn start_lite_rpc(args: Args) -> anyhow::Result<()> { transaction_retry_after_secs, } = args; - let identity = get_identity_keypair(&identity_keypair).await; + let validator_identity = Arc::new(get_identity_keypair(&identity_keypair).await); let retry_after = Duration::from_secs(transaction_retry_after_secs); @@ -57,7 +58,7 @@ pub async fn start_lite_rpc(args: Args) -> anyhow::Result<()> { rpc_addr, ws_addr, fanout_size, - identity, + validator_identity, retry_after, maximum_retries_per_tx, ) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 52513479..aee41888 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -96,7 +96,29 @@ pub fn small_tx_batch_unstaked() { } #[test] -pub fn many_transactions() { +pub fn with_100_transactions() { + configure_logging(false); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 100, + stake_connection: true, + proxy_mode: false, + }); +} + +#[test] +pub fn with_1000_transactions() { + configure_logging(false); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 1000, + stake_connection: true, + proxy_mode: false, + }); +} + +#[test] +pub fn with_10000_transactions() { configure_logging(false); wireup_and_send_txs_via_channel(TestCaseParams { From 0d1c95c32771e7ef22e8ea875546631dd0091c75 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 24 Jul 2023 10:45:17 +0200 Subject: [PATCH 030/128] move proxy send code --- quic-forward-proxy/src/active_connection.rs | 10 +++++---- quic-forward-proxy/src/proxy.rs | 23 ++++++++++++--------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/quic-forward-proxy/src/active_connection.rs b/quic-forward-proxy/src/active_connection.rs index f13d474d..73570e9b 100644 --- a/quic-forward-proxy/src/active_connection.rs +++ b/quic-forward-proxy/src/active_connection.rs @@ -61,10 +61,12 @@ impl ActiveConnection { } fn check_for_confirmation(txs_sent_store: &TxStore, signature: String) -> bool { - match txs_sent_store.get(&signature) { - Some(props) => props.status.is_some(), - None => false, - } + // TODO build a smarter duplication check + false + // match txs_sent_store.get(&signature) { + // Some(props) => props.status.is_some(), + // None => false, + // } } #[allow(clippy::too_many_arguments)] diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index fc5ea9ff..bac47a07 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -56,19 +56,18 @@ impl QuicForwardProxy { let quic_proxy: AnyhowJoinHandle = tokio::spawn(async move { info!("TPU Quic Proxy server start on {}", endpoint.local_addr()?); - let identity_keypair = Keypair::new(); // TODO - while let Some(conn) = endpoint.accept().await { - trace!("connection incoming"); - - let active_tpu_connection = - TpuQuicConnection::new_with_validator_identity(self.validator_identity.as_ref()); - let fut = handle_connection(conn, active_tpu_connection, exit_signal.clone(), self.validator_identity.clone()); + let exit_signal = exit_signal.clone(); + let validator_identity_copy = self.validator_identity.clone(); tokio::spawn(async move { - if let Err(e) = fut.await { - error!("connection failed: {reason}", reason = e.to_string()) + match setup_connection(conn, exit_signal, validator_identity_copy).await { + Ok(()) => {} + Err(err) => { + error!("setup connection failed: {reason}", reason = err); + } } + }); } @@ -85,9 +84,13 @@ impl QuicForwardProxy { } -async fn handle_connection(connecting: Connecting, active_tpu_connection: TpuQuicConnection, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { +async fn setup_connection(connecting: Connecting, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { let connection = connecting.await?; debug!("inbound connection established, remote {connection}", connection = connection.remote_address()); + + let active_tpu_connection = + TpuQuicConnection::new_with_validator_identity(validator_identity.as_ref()); + loop { let maybe_stream = connection.accept_uni().await; let mut recv_stream = match maybe_stream { From 067c1a10f71f5a725496628a2665bf2399e5ac9d Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 24 Jul 2023 11:14:04 +0200 Subject: [PATCH 031/128] experiment with tokio tracing --- quic-forward-proxy/Cargo.toml | 1 + quic-forward-proxy/src/proxy.rs | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 200a56c6..f36261d5 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -27,6 +27,7 @@ log = { workspace = true } clap = { workspace = true } dashmap = { workspace = true } itertools = { workspace = true } +tracing = "0.1.37" tracing-subscriber = { workspace = true } native-tls = { workspace = true } prometheus = { workspace = true } diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index bac47a07..8c828394 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -4,7 +4,8 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::thread::sleep; use std::time::Duration; -use anyhow::{anyhow, bail}; +use tracing::{debug_span, instrument, Instrument, span}; +use anyhow::{anyhow, bail, Context}; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig, VarInt}; @@ -56,12 +57,15 @@ impl QuicForwardProxy { let quic_proxy: AnyhowJoinHandle = tokio::spawn(async move { info!("TPU Quic Proxy server start on {}", endpoint.local_addr()?); - while let Some(conn) = endpoint.accept().await { + while let Some(connecting) = endpoint.accept().await { let exit_signal = exit_signal.clone(); let validator_identity_copy = self.validator_identity.clone(); tokio::spawn(async move { - match setup_connection(conn, exit_signal, validator_identity_copy).await { + + let connection = connecting.await.context("accept connection").unwrap(); + match setup_connection(connection, exit_signal, validator_identity_copy) + .instrument(debug_span!("inbound_conn")).await { Ok(()) => {} Err(err) => { error!("setup connection failed: {reason}", reason = err); @@ -84,9 +88,8 @@ impl QuicForwardProxy { } -async fn setup_connection(connecting: Connecting, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { - let connection = connecting.await?; - debug!("inbound connection established, remote {connection}", connection = connection.remote_address()); +async fn setup_connection(connection: Connection, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { + debug!("inbound connection established, remote {}", connection.remote_address()); let active_tpu_connection = TpuQuicConnection::new_with_validator_identity(validator_identity.as_ref()); @@ -125,7 +128,7 @@ async fn setup_connection(connecting: Connecting, exit_signal: Arc, active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await; - }); + }).instrument(debug_span!("send_txs_to_tpu")).await.unwrap(); } // -- loop } From 19f7922e1f3d775fc543cf7573a7438a64804f89 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 24 Jul 2023 15:52:56 +0200 Subject: [PATCH 032/128] build dumb lazy-one-connection per tpu allocation strategy --- core/src/proxy_request_format.rs | 4 +- .../tests/quic_proxy_tpu_integrationtest.rs | 25 +- quic-forward-proxy/src/active_connection.rs | 228 ------------------ quic-forward-proxy/src/lib.rs | 3 +- quic-forward-proxy/src/main.rs | 2 +- quic-forward-proxy/src/proxy.rs | 41 +++- .../src/proxy_request_format.rs | 5 +- .../src/quic_connection_utils.rs | 141 +++-------- ..._quic_connection.rs => tpu_quic_client.rs} | 76 +++--- 9 files changed, 132 insertions(+), 393 deletions(-) delete mode 100644 quic-forward-proxy/src/active_connection.rs rename quic-forward-proxy/src/{tpu_quic_connection.rs => tpu_quic_client.rs} (59%) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index 4675870d..e888ae2c 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -22,8 +22,8 @@ pub struct TpuForwardingRequest { impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "TpuForwardingRequest for tpu target {} with identity {}", - &self.get_tpu_socket_addr(), &self.get_identity_tpunode()) + write!(f, "TpuForwardingRequest for tpu target {} with identity {}: payload {} tx", + &self.get_tpu_socket_addr(), &self.get_identity_tpunode(), &self.get_transactions().len()) } } diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index aee41888..0b20d655 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -38,6 +38,7 @@ use tokio::task::JoinHandle; use tokio::time::{interval, sleep}; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{filter::LevelFilter, fmt}; +use tracing_subscriber::fmt::format::FmtSpan; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_config_provicer::SelfSignedTlsConfigProvider; use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; @@ -117,6 +118,20 @@ pub fn with_1000_transactions() { }); } +#[test] +pub fn bench_proxy() { + configure_logging(true); + + // consumed 1000 packets in 2059004us - throughput 485.67 tps, throughput_50 6704.05 tps + + wireup_and_send_txs_via_channel(TestCaseParams { + // sample_tx_count: 1000, // this is the goal -- ATM test runs too long + sample_tx_count: 200, + stake_connection: true, + proxy_mode: true, + }); +} + #[test] pub fn with_10000_transactions() { configure_logging(false); @@ -319,12 +334,18 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { fn configure_logging(verbose: bool) { let env_filter = if verbose { - "debug,rustls=info,quinn_proto=debug,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=trace" + "debug,rustls=info,quinn=info,quinn_proto=debug,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=trace" + } else { + "debug,rustls=info,quinn=info,quinn_proto=info,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=debug" + }; + let span_mode = if verbose { + FmtSpan::FULL } else { - "debug,rustls=info,quinn_proto=info,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=debug" + FmtSpan::NONE }; let result = tracing_subscriber::fmt::fmt() .with_env_filter(env_filter) + .with_span_events(span_mode) .try_init(); if result.is_err() { println!("Logging already initialized - ignore"); diff --git a/quic-forward-proxy/src/active_connection.rs b/quic-forward-proxy/src/active_connection.rs deleted file mode 100644 index 73570e9b..00000000 --- a/quic-forward-proxy/src/active_connection.rs +++ /dev/null @@ -1,228 +0,0 @@ -use std::net::SocketAddr; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::time::Duration; -use anyhow::bail; -use log::{error, info, warn}; -use prometheus::{opts, register_int_gauge}; -use prometheus::core::GenericGauge; -use quinn::{Connection, Endpoint}; -use solana_sdk::pubkey::Pubkey; -use solana_sdk::transaction::VersionedTransaction; -use solana_streamer::nonblocking::quic::compute_max_allowed_uni_streams; -use tokio::sync::broadcast::Receiver; -use tokio::sync::RwLock; -use tokio::time::timeout; -use crate::identity_stakes::IdentityStakes; -use crate::proxy_request_format::TpuForwardingRequest; -use crate::quic_connection_utils::QuicConnectionUtils; -use crate::tpu_quic_connection::{CONNECTION_RETRY_COUNT, QUIC_CONNECTION_TIMEOUT}; -use crate::tx_store::TxStore; -use itertools::Itertools; - -lazy_static::lazy_static! { - // TODO rename / cleanup - static ref NB_QUIC_CONNECTIONS: GenericGauge = - register_int_gauge!(opts!("literpc_nb_active_quic_connections", "Number of quic connections open")).unwrap(); - static ref NB_QUIC_ACTIVE_CONNECTIONS: GenericGauge = - register_int_gauge!(opts!("literpc_nb_active_connections", "Number quic tasks that are running")).unwrap(); - static ref NB_CONNECTIONS_TO_KEEP: GenericGauge = - register_int_gauge!(opts!("literpc_connections_to_keep", "Number of connections to keep asked by tpu service")).unwrap(); - static ref NB_QUIC_TASKS: GenericGauge = - register_int_gauge!(opts!("literpc_quic_tasks", "Number of connections to keep asked by tpu service")).unwrap(); -} - -pub struct ActiveConnection { - endpoint: Endpoint, - identity: Pubkey, - tpu_address: SocketAddr, - exit_signal: Arc, - txs_sent_store: TxStore, -} - -impl ActiveConnection { - pub fn new( - endpoint: Endpoint, - tpu_address: SocketAddr, - identity: Pubkey, - txs_sent_store: TxStore, - ) -> Self { - Self { - endpoint, - tpu_address, - identity, - exit_signal: Arc::new(AtomicBool::new(false)), - txs_sent_store, - } - } - - fn on_connect() { - NB_QUIC_CONNECTIONS.inc(); - } - - fn check_for_confirmation(txs_sent_store: &TxStore, signature: String) -> bool { - // TODO build a smarter duplication check - false - // match txs_sent_store.get(&signature) { - // Some(props) => props.status.is_some(), - // None => false, - // } - } - - #[allow(clippy::too_many_arguments)] - async fn listen( - transaction_reciever: Receiver<(String, Vec)>, - exit_oneshot_channel: tokio::sync::mpsc::Receiver<()>, - endpoint: Endpoint, - tpu_address: SocketAddr, - exit_signal: Arc, - identity: Pubkey, - identity_stakes: IdentityStakes, - txs_sent_store: TxStore, - ) { - NB_QUIC_ACTIVE_CONNECTIONS.inc(); - let mut transaction_reciever = transaction_reciever; - let mut exit_oneshot_channel = exit_oneshot_channel; - - let max_uni_stream_connections: u64 = compute_max_allowed_uni_streams( - identity_stakes.peer_type, - identity_stakes.stakes, - identity_stakes.total_stakes, - ) as u64; - let number_of_transactions_per_unistream = 5; - - let task_counter: Arc = Arc::new(AtomicU64::new(0)); - let mut connection: Option>> = None; - let last_stable_id: Arc = Arc::new(AtomicU64::new(0)); - - loop { - // exit signal set - if exit_signal.load(Ordering::Relaxed) { - break; - } - - if task_counter.load(Ordering::Relaxed) >= max_uni_stream_connections { - tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; - continue; - } - - tokio::select! { - tx = transaction_reciever.recv() => { - // exit signal set - if exit_signal.load(Ordering::Relaxed) { - break; - } - - let first_tx: Vec = match tx { - Ok((sig, tx)) => { - if Self::check_for_confirmation(&txs_sent_store, sig) { - // transaction is already confirmed/ no need to send - continue; - } - tx - }, - Err(e) => { - error!( - "Broadcast channel error on recv for {} error {}", - identity, e - ); - continue; - } - }; - - let mut txs = vec![first_tx]; - for _ in 1..number_of_transactions_per_unistream { - if let Ok((signature, tx)) = transaction_reciever.try_recv() { - if Self::check_for_confirmation(&txs_sent_store, signature) { - continue; - } - txs.push(tx); - } - } - - if connection.is_none() { - // initial connection - let conn = QuicConnectionUtils::connect( - identity, - false, - endpoint.clone(), - tpu_address, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - exit_signal.clone(), - Self::on_connect).await; - - if let Some(conn) = conn { - // could connect - connection = Some(Arc::new(RwLock::new(conn))); - } else { - break; - } - } - - let task_counter = task_counter.clone(); - let endpoint = endpoint.clone(); - let exit_signal = exit_signal.clone(); - let connection = connection.clone(); - let last_stable_id = last_stable_id.clone(); - - tokio::spawn(async move { - task_counter.fetch_add(1, Ordering::Relaxed); - NB_QUIC_TASKS.inc(); - let connection = connection.unwrap(); - - QuicConnectionUtils::send_transaction_batch( - connection, - txs, - identity, - endpoint, - tpu_address, - exit_signal, - last_stable_id, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - || { - // do nothing as we are using the same connection - } - ).await; - - NB_QUIC_TASKS.dec(); - task_counter.fetch_sub(1, Ordering::Relaxed); - }); - }, - _ = exit_oneshot_channel.recv() => { - break; - } - }; - } - drop(transaction_reciever); - NB_QUIC_CONNECTIONS.dec(); - NB_QUIC_ACTIVE_CONNECTIONS.dec(); - } - - pub fn start_listening( - &self, - transaction_reciever: Receiver<(String, Vec)>, - exit_oneshot_channel: tokio::sync::mpsc::Receiver<()>, - identity_stakes: IdentityStakes, - ) { - let endpoint = self.endpoint.clone(); - let tpu_address = self.tpu_address; - let exit_signal = self.exit_signal.clone(); - let identity = self.identity; - let txs_sent_store = self.txs_sent_store.clone(); - tokio::spawn(async move { - Self::listen( - transaction_reciever, - exit_oneshot_channel, - endpoint, - tpu_address, - exit_signal, - identity, - identity_stakes, - txs_sent_store, - ) - .await; - }); - } -} diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index bca2f999..2aa1d6b6 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -2,8 +2,7 @@ pub mod quic_util; pub mod tls_config_provicer; pub mod proxy; pub mod proxy_request_format; -pub mod tpu_quic_connection; -pub mod active_connection; +pub mod tpu_quic_client; pub mod cli; pub mod test_client; mod util; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 80983b04..0d840af0 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -14,7 +14,7 @@ pub mod quic_util; pub mod tls_config_provicer; pub mod proxy; pub mod proxy_request_format; -pub mod tpu_quic_connection; +pub mod tpu_quic_client; pub mod active_connection; pub mod cli; pub mod test_client; diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 8c828394..a6f45efa 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -6,6 +6,7 @@ use std::thread::sleep; use std::time::Duration; use tracing::{debug_span, instrument, Instrument, span}; use anyhow::{anyhow, bail, Context}; +use dashmap::DashMap; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig, VarInt}; @@ -21,14 +22,15 @@ use tokio::net::ToSocketAddrs; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; use crate::proxy_request_format::TpuForwardingRequest; -use crate::tpu_quic_connection::TpuQuicConnection; +use crate::quic_connection_utils::QuicConnectionUtils; +use crate::tpu_quic_client::{TpuQuicClient}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::AnyhowJoinHandle; - pub struct QuicForwardProxy { endpoint: Endpoint, validator_identity: Arc, + tpu_quic_client: TpuQuicClient, } impl QuicForwardProxy { @@ -44,7 +46,10 @@ impl QuicForwardProxy { info!("tpu forward proxy listening on {}", endpoint.local_addr()?); info!("staking from validator identity {}", validator_identity.pubkey()); - Ok(Self {endpoint, validator_identity }) + let tpu_quic_client = + TpuQuicClient::new_with_validator_identity(validator_identity.as_ref()).await; + + Ok(Self { endpoint, validator_identity, tpu_quic_client }) } @@ -57,15 +62,17 @@ impl QuicForwardProxy { let quic_proxy: AnyhowJoinHandle = tokio::spawn(async move { info!("TPU Quic Proxy server start on {}", endpoint.local_addr()?); + while let Some(connecting) = endpoint.accept().await { let exit_signal = exit_signal.clone(); let validator_identity_copy = self.validator_identity.clone(); + let tpu_quic_client = self.tpu_quic_client.clone(); tokio::spawn(async move { let connection = connecting.await.context("accept connection").unwrap(); - match setup_connection(connection, exit_signal, validator_identity_copy) - .instrument(debug_span!("inbound_conn")).await { + match accept_client_connection(connection, tpu_quic_client, exit_signal, validator_identity_copy) + .await { Ok(()) => {} Err(err) => { error!("setup connection failed: {reason}", reason = err); @@ -88,14 +95,17 @@ impl QuicForwardProxy { } -async fn setup_connection(connection: Connection, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { - debug!("inbound connection established, remote {}", connection.remote_address()); +// TODO use interface abstraction for connection_per_tpunode +#[tracing::instrument(skip_all, level = "debug")] +async fn accept_client_connection(client_connection: Connection, tpu_quic_client: TpuQuicClient, + exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { + debug!("inbound connection established, client {}", client_connection.remote_address()); - let active_tpu_connection = - TpuQuicConnection::new_with_validator_identity(validator_identity.as_ref()); + // let active_tpu_connection = + // TpuQuicClient::new_with_validator_identity(validator_identity.as_ref()).await; loop { - let maybe_stream = connection.accept_uni().await; + let maybe_stream = client_connection.accept_uni().await; let mut recv_stream = match maybe_stream { Err(quinn::ConnectionError::ApplicationClosed(reason)) => { debug!("connection closed by peer - reason: {:?}", reason); @@ -111,10 +121,11 @@ async fn setup_connection(connection: Connection, exit_signal: Arc, } Ok(s) => s, }; - let active_tpu_connection_copy = active_tpu_connection.clone(); let exit_signal_copy = exit_signal.clone(); let validator_identity_copy = validator_identity.clone(); + let tpu_quic_client_copy = tpu_quic_client.clone(); tokio::spawn(async move { + let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const .unwrap(); debug!("read proxy_request {} bytes", raw_request.len()); @@ -126,7 +137,13 @@ async fn setup_connection(connection: Connection, exit_signal: Arc, let tpu_address = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transactions(); - active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await; + let tpu_connection = tpu_quic_client_copy.get_or_create_connection(tpu_address).await; + + info!("send transaction batch of size {} to address {}", txs.len(), tpu_address); + tpu_quic_client_copy.send_txs_to_tpu(tpu_connection, &txs, exit_signal_copy).await; + + + // active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await; }).instrument(debug_span!("send_txs_to_tpu")).await.unwrap(); diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 969d68a0..2339173e 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -17,13 +17,14 @@ pub struct TpuForwardingRequest { format_version: u16, tpu_socket_addr: SocketAddr, // TODO is that correct identity_tpunode: Pubkey, // note: this is only used fro + // TODO consider not deserializing transactions in proxy transactions: Vec, } impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "TpuForwardingRequest for tpu target {} with identity {}", - &self.get_tpu_socket_addr(), &self.get_identity_tpunode()) + write!(f, "TpuForwardingRequest for tpu target {} with identity {}: payload {} tx", + &self.get_tpu_socket_addr(), &self.get_identity_tpunode(), &self.get_transactions().len()) } } diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs index 73b58271..be4b2a57 100644 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -1,8 +1,5 @@ -use log::{info, trace, warn}; -use quinn::{ - ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, - TokioRuntime, TransportConfig, -}; +use log::{error, info, trace, warn}; +use quinn::{ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, WriteError}; use solana_sdk::pubkey::Pubkey; use std::{ collections::VecDeque, @@ -15,6 +12,7 @@ use std::{ }; use anyhow::bail; use tokio::{sync::RwLock, time::timeout}; +use tokio::time::error::Elapsed; const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; @@ -127,30 +125,26 @@ impl QuicConnectionUtils { } pub async fn write_all( - mut send_stream: SendStream, + send_stream: &mut SendStream, tx: &Vec, - identity: Pubkey, - last_stable_id: Arc, - connection_stable_id: u64, connection_timeout: Duration, - ) -> bool { + ) { let write_timeout_res = timeout(connection_timeout, send_stream.write_all(tx.as_slice())).await; match write_timeout_res { Ok(write_res) => { if let Err(e) = write_res { trace!( - "Error while writing transaction for {}, error {}", - identity, + "Error while writing transaction for TBD, error {}", + // identity, // TODO add more context e ); - // retry - last_stable_id.store(connection_stable_id, Ordering::Relaxed); - return true; + return; } } Err(_) => { - warn!("timeout while writing transaction for {}", identity); + warn!("timeout while writing transaction for TBD"); // TODO add more context + panic!("TODO handle timeout"); // FIXME } } @@ -158,33 +152,31 @@ impl QuicConnectionUtils { match finish_timeout_res { Ok(finish_res) => { if let Err(e) = finish_res { - last_stable_id.store(connection_stable_id, Ordering::Relaxed); + // last_stable_id.store(connection_stable_id, Ordering::Relaxed); trace!( - "Error while writing transaction for {}, error {}", - identity, + "Error while writing transaction for TBD, error {}", + // identity, e ); - return true; + return; } } Err(_) => { - warn!("timeout while finishing transaction for {}", identity); + warn!("timeout while finishing transaction for TBD"); // TODO + panic!("TODO handle timeout"); // FIXME } } - false } pub async fn open_unistream( connection: Connection, - last_stable_id: Arc, connection_timeout: Duration, ) -> (Option, bool) { match timeout(connection_timeout, connection.open_uni()).await { Ok(Ok(unistream)) => (Some(unistream), false), Ok(Err(_)) => { // reset connection for next retry - last_stable_id.store(connection.stable_id() as u64, Ordering::Relaxed); (None, true) } // timeout @@ -194,94 +186,41 @@ impl QuicConnectionUtils { #[allow(clippy::too_many_arguments)] pub async fn send_transaction_batch( - connection: Arc>, + connection: Connection, txs: Vec>, - identity: Pubkey, - endpoint: Endpoint, - tpu_address: SocketAddr, exit_signal: Arc, - last_stable_id: Arc, connection_timeout: Duration, - connection_retry_count: usize, - on_connect: fn(), ) { - info!("send transaction batch of size {} to address {}", txs.len(), tpu_address); - let mut queue = VecDeque::new(); - for tx in txs { - queue.push_back(tx); - } - for _ in 0..connection_retry_count { - if queue.is_empty() || exit_signal.load(Ordering::Relaxed) { - // return + let (mut stream, _retry_conn) = + Self::open_unistream(connection.clone(), connection_timeout) + .await; + if let Some(ref mut send_stream) = stream { + if exit_signal.load(Ordering::Relaxed) { return; } - // get new connection reset if necessary - let conn = { - let last_stable_id = last_stable_id.load(Ordering::Relaxed) as usize; - let conn = connection.read().await; - if conn.stable_id() == last_stable_id { - let current_stable_id = conn.stable_id(); - // problematic connection - drop(conn); - let mut conn = connection.write().await; - // check may be already written by another thread - if conn.stable_id() != current_stable_id { - conn.clone() - } else { - let new_conn = Self::connect( - identity, - true, - endpoint.clone(), - tpu_address, - connection_timeout, - connection_retry_count, - exit_signal.clone(), - on_connect, - ) - .await; - if let Some(new_conn) = new_conn { - *conn = new_conn; - conn.clone() - } else { - // could not connect - return; + + for tx in txs { + let write_timeout_res = + timeout(connection_timeout, send_stream.write_all(tx.as_slice())).await; + match write_timeout_res { + Ok(no_timeout) => { + match no_timeout { + Ok(()) => {} + Err(write_error) => { + error!("Error writing transaction to stream: {}", write_error); + } } } - } else { - conn.clone() - } - }; - let mut retry = false; - while !queue.is_empty() { - let tx = queue.pop_front().unwrap(); - let (stream, retry_conn) = - Self::open_unistream(conn.clone(), last_stable_id.clone(), connection_timeout) - .await; - if let Some(send_stream) = stream { - if exit_signal.load(Ordering::Relaxed) { - return; + Err(elapsed) => { + warn!("timeout sending transactions") } - - retry = Self::write_all( - send_stream, - &tx, - identity, - last_stable_id.clone(), - conn.stable_id() as u64, - connection_timeout, - ) - .await; - } else { - retry = retry_conn; - } - if retry { - queue.push_back(tx); - break; } } - if !retry { - break; - } + // TODO wrap in timeout + stream.unwrap().finish().await.unwrap(); + + } else { + panic!("no retry handling"); // FIXME } } } diff --git a/quic-forward-proxy/src/tpu_quic_connection.rs b/quic-forward-proxy/src/tpu_quic_client.rs similarity index 59% rename from quic-forward-proxy/src/tpu_quic_connection.rs rename to quic-forward-proxy/src/tpu_quic_client.rs index 69ef3182..8c19f429 100644 --- a/quic-forward-proxy/src/tpu_quic_connection.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -1,9 +1,11 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::time::Duration; use anyhow::{anyhow, bail}; +use dashmap::DashMap; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig, VarInt}; @@ -21,20 +23,23 @@ use tokio::sync::RwLock; use crate::quic_connection_utils::QuicConnectionUtils; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; -pub const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); +const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); pub const CONNECTION_RETRY_COUNT: usize = 10; /// stable connect to TPU to send transactions - optimized for proxy use case -#[derive(Clone)] -pub struct TpuQuicConnection { +#[derive(Debug, Clone)] +pub struct TpuQuicClient { endpoint: Endpoint, + // naive single non-recoverable connection - TODO moke it smarter + connection_per_tpunode: DashMap, } -impl TpuQuicConnection { +impl TpuQuicClient { /// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU // note: ATM the provided identity might or might not be a valid validator keypair - pub fn new_with_validator_identity(validator_identity: &Keypair) -> TpuQuicConnection { + pub async fn new_with_validator_identity(validator_identity: &Keypair) -> TpuQuicClient { + info!("Setup TPU Quic stable connection ..."); let (certificate, key) = new_self_signed_tls_certificate( validator_identity, IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), @@ -43,60 +48,45 @@ impl TpuQuicConnection { let endpoint_outbound = QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()); - let active_tpu_connection = TpuQuicConnection { + let active_tpu_connection = TpuQuicClient { endpoint: endpoint_outbound.clone(), + connection_per_tpunode: DashMap::new(), }; active_tpu_connection } - pub async fn send_txs_to_tpu(&self, - exit_signal: Arc, - validator_identity: Arc, - tpu_identity: Pubkey, - tpu_address: SocketAddr, - txs: &Vec) { + #[tracing::instrument(skip(self), level = "debug")] + pub async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> Connection { + if let Some(conn) = self.connection_per_tpunode.get(&tpu_address) { + return conn.clone(); + } - let last_stable_id: Arc = Arc::new(AtomicU64::new(0)); + debug!("Creating new QUIC connection to {} and put it into the connection map", tpu_address); + let connection = + // TODO try 0rff + QuicConnectionUtils::make_connection( + self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) + .await.unwrap(); - let txs_raw = serialize_to_vecvec(&txs); - info!("received vecvec: {}", txs_raw.iter().map(|tx| tx.len().to_string()).into_iter().join(",")); + self.connection_per_tpunode.insert(tpu_address, connection.clone()); - let connection = - Arc::new(RwLock::new( - QuicConnectionUtils::connect( - tpu_identity, - false, - self.endpoint.clone(), - tpu_address, - QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - exit_signal.clone(), - || { - // do nothing - }, - ).await.unwrap())); + return connection; + } + pub async fn send_txs_to_tpu(&self, + connection: Connection, + txs: &Vec, + exit_signal: Arc, + ) { QuicConnectionUtils::send_transaction_batch( - connection.clone(), - txs_raw, - tpu_identity, - self.endpoint.clone(), - tpu_address, + connection, + serialize_to_vecvec(txs), exit_signal.clone(), - last_stable_id, QUIC_CONNECTION_TIMEOUT, - CONNECTION_RETRY_COUNT, - || { - // do nothing - } ).await; - { - let conn = connection.clone(); - conn.write().await.close(0u32.into(), b"done"); - } } } From fd2cf5a0eb7ba9cde1e7c71318f4344c43ee4e9f Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 24 Jul 2023 15:59:09 +0200 Subject: [PATCH 033/128] fix dashmap arc issue --- quic-forward-proxy/src/tpu_quic_client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index 8c19f429..88db04bc 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -31,7 +31,7 @@ pub const CONNECTION_RETRY_COUNT: usize = 10; pub struct TpuQuicClient { endpoint: Endpoint, // naive single non-recoverable connection - TODO moke it smarter - connection_per_tpunode: DashMap, + connection_per_tpunode: Arc>, } impl TpuQuicClient { @@ -50,7 +50,7 @@ impl TpuQuicClient { let active_tpu_connection = TpuQuicClient { endpoint: endpoint_outbound.clone(), - connection_per_tpunode: DashMap::new(), + connection_per_tpunode: Arc::new(DashMap::new()), }; active_tpu_connection @@ -62,7 +62,6 @@ impl TpuQuicClient { return conn.clone(); } - debug!("Creating new QUIC connection to {} and put it into the connection map", tpu_address); let connection = // TODO try 0rff QuicConnectionUtils::make_connection( @@ -72,6 +71,7 @@ impl TpuQuicClient { self.connection_per_tpunode.insert(tpu_address, connection.clone()); + debug!("Created new Quic connection to TPU node {}, total connections is now {}", tpu_address, self.connection_per_tpunode.len()); return connection; } From a2207da37a1095a7d7212245aac8674a62a5df89 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 25 Jul 2023 09:04:09 +0200 Subject: [PATCH 034/128] wip works --- quic-forward-proxy/src/main.rs | 1 - quic-forward-proxy/src/proxy.rs | 6 +- .../src/quic_connection_utils.rs | 2 + quic-forward-proxy/src/tpu_quic_client.rs | 23 +++++-- .../quic_proxy_connection_manager.rs | 63 ++++++++++++------- .../src/tpu_utils/tpu_connection_manager.rs | 1 + 6 files changed, 67 insertions(+), 29 deletions(-) diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 0d840af0..7d4a5947 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -15,7 +15,6 @@ pub mod tls_config_provicer; pub mod proxy; pub mod proxy_request_format; pub mod tpu_quic_client; -pub mod active_connection; pub mod cli; pub mod test_client; mod util; diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index a6f45efa..0a7444f0 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -124,6 +124,7 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client let exit_signal_copy = exit_signal.clone(); let validator_identity_copy = validator_identity.clone(); let tpu_quic_client_copy = tpu_quic_client.clone(); + tokio::spawn(async move { let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const @@ -137,15 +138,16 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client let tpu_address = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transactions(); + // TODO join get_or_create_connection future and read_to_end let tpu_connection = tpu_quic_client_copy.get_or_create_connection(tpu_address).await; info!("send transaction batch of size {} to address {}", txs.len(), tpu_address); tpu_quic_client_copy.send_txs_to_tpu(tpu_connection, &txs, exit_signal_copy).await; - // active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await; - }).instrument(debug_span!("send_txs_to_tpu")).await.unwrap(); + }) + .await.unwrap(); } // -- loop } diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs index be4b2a57..32883a55 100644 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -13,6 +13,7 @@ use std::{ use anyhow::bail; use tokio::{sync::RwLock, time::timeout}; use tokio::time::error::Elapsed; +use tracing::instrument; const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; @@ -185,6 +186,7 @@ impl QuicConnectionUtils { } #[allow(clippy::too_many_arguments)] + #[tracing::instrument(skip_all, level = "debug")] pub async fn send_transaction_batch( connection: Connection, txs: Vec>, diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index 88db04bc..57b3a5f4 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -1,7 +1,9 @@ +use std::collections::HashMap; +use std::io::Write; use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::Path; use std::str::FromStr; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::sync::atomic::{AtomicBool, AtomicU64}; use std::time::Duration; use anyhow::{anyhow, bail}; @@ -31,6 +33,7 @@ pub const CONNECTION_RETRY_COUNT: usize = 10; pub struct TpuQuicClient { endpoint: Endpoint, // naive single non-recoverable connection - TODO moke it smarter + // TODO consider using DashMap again connection_per_tpunode: Arc>, } @@ -58,8 +61,18 @@ impl TpuQuicClient { #[tracing::instrument(skip(self), level = "debug")] pub async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> Connection { - if let Some(conn) = self.connection_per_tpunode.get(&tpu_address) { - return conn.clone(); + info!("looking up {}", tpu_address); + // TODO try 0rff + // QuicConnectionUtils::make_connection( + // self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) + // .await.unwrap() + + + { + if let Some(conn) = self.connection_per_tpunode.get(&tpu_address) { + debug!("reusing connection {:?}", conn); + return conn.clone(); + } } let connection = @@ -68,8 +81,8 @@ impl TpuQuicClient { self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) .await.unwrap(); - - self.connection_per_tpunode.insert(tpu_address, connection.clone()); + let old_value = self.connection_per_tpunode.insert(tpu_address, connection.clone()); + assert!(old_value.is_none(), "no prev value must be overridden"); debug!("Created new Quic connection to TPU node {}, total connections is now {}", tpu_address, self.connection_per_tpunode.len()); return connection; diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index a2de3ec8..2deceb44 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use futures::FutureExt; use itertools::Itertools; use log::{debug, error, info, warn}; -use quinn::{ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig}; +use quinn::{ClientConfig, Connection, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig}; use solana_sdk::pubkey::Pubkey; use solana_sdk::signer::Signer; use solana_sdk::signature::Keypair; @@ -65,8 +65,6 @@ impl QuicProxyConnectionManager { { - let mut lock = self.current_tpu_nodes.write().await; - let list_of_nodes = connections_to_keep.iter().map(|(identity, tpu_address)| { TpuNode { tpu_identity: identity.clone(), @@ -74,6 +72,7 @@ impl QuicProxyConnectionManager { } }).collect_vec(); + let mut lock = self.current_tpu_nodes.write().await; *lock = list_of_nodes; } @@ -82,6 +81,7 @@ impl QuicProxyConnectionManager { // already started return; } + self.simple_thread_started.store(true, Relaxed); info!("Starting very simple proxy thread"); @@ -141,6 +141,10 @@ impl QuicProxyConnectionManager { proxy_addr: SocketAddr, endpoint: Endpoint, ) { + + let mut connection = endpoint.connect(proxy_addr, "localhost").unwrap() + .await.unwrap(); + loop { // TODO exit signal ??? @@ -150,7 +154,7 @@ impl QuicProxyConnectionManager { // exit signal??? - let the_tx: Vec = match tx { + let first_tx: Vec = match tx { Ok((sig, tx)) => { // if Self::check_for_confirmation(&txs_sent_store, sig) { // // transaction is already confirmed/ no need to send @@ -165,20 +169,31 @@ impl QuicProxyConnectionManager { } }; - // TODO read all txs from channel (see "let mut txs = vec![first_tx];") - let txs = vec![the_tx]; + let number_of_transactions_per_unistream = 8; // TODO read from QuicConnectionParameters - let tpu_fanout_nodes = current_tpu_nodes.read().await; + let mut txs = vec![first_tx]; + // TODO comment in + // for _ in 1..number_of_transactions_per_unistream { + // if let Ok((signature, tx)) = transaction_receiver.try_recv() { + // // if Self::check_for_confirmation(&txs_sent_store, signature) { + // // continue; + // // } + // txs.push(tx); + // } + // } - info!("Sending copy of transaction batch of {} to {} tpu nodes via quic proxy", + let tpu_fanout_nodes = current_tpu_nodes.read().await.clone(); + + info!("Sending copy of transaction batch of {} txs to {} tpu nodes via quic proxy", txs.len(), tpu_fanout_nodes.len()); - for target_tpu_node in &*tpu_fanout_nodes { + for target_tpu_node in tpu_fanout_nodes { Self::send_copy_of_txs_to_quicproxy( &txs, endpoint.clone(), - proxy_addr, + proxy_addr, target_tpu_node.tpu_address, - target_tpu_node.tpu_identity).await.unwrap(); + target_tpu_node.tpu_identity) + .await.unwrap(); } }, @@ -190,7 +205,11 @@ impl QuicProxyConnectionManager { proxy_address: SocketAddr, tpu_target_address: SocketAddr, target_tpu_identity: Pubkey) -> anyhow::Result<()> { - info!("sending vecvec: {}", raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(",")); + info!("sending vecvec {} to quic proxy for TPU node {}", + raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(","), tpu_target_address); + + // TODO add timeout + // let mut send_stream = timeout(Duration::from_millis(500), connection.open_uni()).await??; let raw_tx_batch_copy = raw_tx_batch.clone(); @@ -207,6 +226,7 @@ impl QuicProxyConnectionManager { } let forwarding_request = TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, txs); + debug!("forwarding_request: {}", forwarding_request); let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); @@ -222,23 +242,24 @@ impl QuicProxyConnectionManager { bail!("Failed to send data to quic proxy: {:?}", e); } } + Ok(()) } + async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { + info!("sending {} bytes to proxy", proxy_request_raw.len()); - async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { - info!("sending {} bytes to proxy", proxy_request_raw.len()); + let mut connecting = endpoint.connect(proxy_address, "localhost")?; + let connection = timeout(Duration::from_millis(500), connecting).await??; + let mut send = connection.open_uni().await?; - let mut connecting = endpoint.connect(proxy_address, "localhost")?; - let connection = timeout(Duration::from_millis(500), connecting).await??; - let mut send = connection.open_uni().await?; + send.write_all(proxy_request_raw).await?; - send.write_all(proxy_request_raw).await?; + send.finish().await?; - send.finish().await?; + Ok(()) + } - Ok(()) - } } diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 70fc9bae..8f665121 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -237,6 +237,7 @@ impl TpuConnectionManager { ) -> Self { let number_of_clients = fanout * 2; Self { + // TODO endpoints: RotatingQueue::new(number_of_clients, || { QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()) }) From c9c6b7b98e0eb488a15608c5771a878c758c450f Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 25 Jul 2023 09:50:21 +0200 Subject: [PATCH 035/128] use stream per client-proxy connection --- quic-forward-proxy/src/proxy.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 0a7444f0..84ef4dee 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -9,7 +9,7 @@ use anyhow::{anyhow, bail, Context}; use dashmap::DashMap; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; -use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig, VarInt}; +use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig, TransportConfig, VarInt}; use rcgen::generate_simple_self_signed; use rustls::{Certificate, PrivateKey}; use rustls::server::ResolvesServerCert; @@ -41,6 +41,10 @@ impl QuicForwardProxy { let server_tls_config = tls_config.get_server_tls_crypto_config(); let mut quinn_server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); + let mut transport_config = TransportConfig::default(); + transport_config.max_idle_timeout(None); + transport_config.keep_alive_interval(Some(Duration::from_millis(500))); + quinn_server_config.transport_config(Arc::new(transport_config)); let endpoint = Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap(); info!("tpu forward proxy listening on {}", endpoint.local_addr()?); @@ -146,8 +150,7 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client // active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await; - }) - .await.unwrap(); + }); } // -- loop } From 391a42ec22f121a4f7cf9249d9bf0f01aed69acf Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 25 Jul 2023 12:00:39 +0200 Subject: [PATCH 036/128] wip --- quic-forward-proxy/src/proxy.rs | 6 +++--- .../src/quic_connection_utils.rs | 4 +++- .../tpu_utils/quic_proxy_connection_manager.rs | 18 ++++++++++-------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 84ef4dee..41a88d95 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -112,11 +112,11 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client let maybe_stream = client_connection.accept_uni().await; let mut recv_stream = match maybe_stream { Err(quinn::ConnectionError::ApplicationClosed(reason)) => { - debug!("connection closed by peer - reason: {:?}", reason); + debug!("connection closed by client - reason: {:?}", reason); if reason.error_code != VarInt::from_u32(0) { - return Err(anyhow!("connection closed by peer with unexpected reason: {:?}", reason)); + return Err(anyhow!("connection closed by client with unexpected reason: {:?}", reason)); } - debug!("connection gracefully closed by peer"); + debug!("connection gracefully closed by client"); return Ok(()); }, Err(e) => { diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs index 32883a55..a3019e05 100644 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -214,9 +214,11 @@ impl QuicConnectionUtils { } } Err(elapsed) => { - warn!("timeout sending transactions") + warn!("timeout sending transactions"); } } + + } // TODO wrap in timeout stream.unwrap().finish().await.unwrap(); diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 2deceb44..c24e6a3e 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -12,6 +12,7 @@ use futures::FutureExt; use itertools::Itertools; use log::{debug, error, info, warn}; use quinn::{ClientConfig, Connection, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig}; +use solana_sdk::packet::PACKET_DATA_SIZE; use solana_sdk::pubkey::Pubkey; use solana_sdk::signer::Signer; use solana_sdk::signature::Keypair; @@ -173,14 +174,15 @@ impl QuicProxyConnectionManager { let mut txs = vec![first_tx]; // TODO comment in - // for _ in 1..number_of_transactions_per_unistream { - // if let Ok((signature, tx)) = transaction_receiver.try_recv() { - // // if Self::check_for_confirmation(&txs_sent_store, signature) { - // // continue; - // // } - // txs.push(tx); - // } - // } + let foo = PACKET_DATA_SIZE; + for _ in 1..number_of_transactions_per_unistream { + if let Ok((signature, tx)) = transaction_receiver.try_recv() { + // if Self::check_for_confirmation(&txs_sent_store, signature) { + // continue; + // } + txs.push(tx); + } + } let tpu_fanout_nodes = current_tpu_nodes.read().await.clone(); From 5a4546c8078a0d2b3fac952ab437168b00011de6 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 25 Jul 2023 12:54:47 +0200 Subject: [PATCH 037/128] implement chunk+parallel sending --- .../src/quic_connection_utils.rs | 53 ++++++++++++++++++- quic-forward-proxy/src/tpu_quic_client.rs | 23 +++++--- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs index a3019e05..5db42364 100644 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -1,4 +1,4 @@ -use log::{error, info, trace, warn}; +use log::{debug, error, info, trace, warn}; use quinn::{ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, WriteError}; use solana_sdk::pubkey::Pubkey; use std::{ @@ -11,6 +11,8 @@ use std::{ time::Duration, }; use anyhow::bail; +use futures::future::join_all; +use itertools::Itertools; use tokio::{sync::RwLock, time::timeout}; use tokio::time::error::Elapsed; use tracing::instrument; @@ -185,9 +187,10 @@ impl QuicConnectionUtils { } } + #[allow(clippy::too_many_arguments)] #[tracing::instrument(skip_all, level = "debug")] - pub async fn send_transaction_batch( + pub async fn send_transaction_batch_serial( connection: Connection, txs: Vec>, exit_signal: Arc, @@ -227,6 +230,52 @@ impl QuicConnectionUtils { panic!("no retry handling"); // FIXME } } + + // open streams in parallel + // one stream is used for one transaction + // number of parallel streams that connect to TPU must be limited by caller (should be 8) + #[allow(clippy::too_many_arguments)] + #[tracing::instrument(skip_all, level = "debug")] + pub async fn send_transaction_batch_parallel( + connection: Connection, + txs: Vec>, + exit_signal: Arc, + connection_timeout: Duration, + ) { + assert_ne!(txs.len(), 0, "no transactions to send"); + debug!("Opening {} parallel quic streams", txs.len()); + + let all_send_fns = (0..txs.len()).map(|i| Self::send_tx_to_new_stream(&txs[i], connection.clone(), connection_timeout)).collect_vec(); + + join_all(all_send_fns).await; + } + + + async fn send_tx_to_new_stream(tx: &Vec, connection: Connection, connection_timeout: Duration) { + let mut send_stream = Self::open_unistream(connection.clone(), connection_timeout) + .await.0 + .unwrap(); + + let write_timeout_res = + timeout(connection_timeout, send_stream.write_all(tx.as_slice())).await; + match write_timeout_res { + Ok(no_timeout) => { + match no_timeout { + Ok(()) => {} + Err(write_error) => { + error!("Error writing transaction to stream: {}", write_error); + } + } + } + Err(elapsed) => { + warn!("timeout sending transactions"); + } + } + + // TODO wrap in timeout + send_stream.finish().await.unwrap(); + + } } pub struct SkipServerVerification; diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index 57b3a5f4..60f868ec 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -28,6 +28,10 @@ use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProv const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); pub const CONNECTION_RETRY_COUNT: usize = 10; +pub const MAX_TRANSACTIONS_PER_BATCH: usize = 10; +pub const MAX_BYTES_PER_BATCH: usize = 10; +const MAX_PARALLEL_STREAMS: usize = 6; + /// stable connect to TPU to send transactions - optimized for proxy use case #[derive(Debug, Clone)] pub struct TpuQuicClient { @@ -93,12 +97,19 @@ impl TpuQuicClient { txs: &Vec, exit_signal: Arc, ) { - QuicConnectionUtils::send_transaction_batch( - connection, - serialize_to_vecvec(txs), - exit_signal.clone(), - QUIC_CONNECTION_TIMEOUT, - ).await; + + for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { + let vecvec = chunk.iter().map(|tx| { + let tx_raw = bincode::serialize(tx).unwrap(); + tx_raw + }).collect_vec(); + QuicConnectionUtils::send_transaction_batch_parallel( + connection.clone(), + vecvec, + exit_signal.clone(), + QUIC_CONNECTION_TIMEOUT, + ).await; + } } From 4f09278cd2398f15bfacfac1c58906db49d577cb Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 25 Jul 2023 13:54:34 +0200 Subject: [PATCH 038/128] works --- quic-forward-proxy/Cargo.toml | 1 + quic-forward-proxy/src/quic_connection_utils.rs | 2 +- services/src/tpu_utils/tpu_connection_manager.rs | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index f36261d5..34dd1e5d 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -36,6 +36,7 @@ dotenv = { workspace = true } async-channel = { workspace = true } quinn = { workspace = true } async-trait = { workspace = true } +futures = { workspace = true } chrono = { workspace = true } tokio = { version = "1.28.2", features = ["full", "fs"]} rcgen = "0.9.3" diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs index 5db42364..2b3cb17c 100644 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -272,7 +272,7 @@ impl QuicConnectionUtils { } } - // TODO wrap in timeout + // TODO wrap in small timeout send_stream.finish().await.unwrap(); } diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 8f665121..82def708 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -88,6 +88,8 @@ impl ActiveConnection { identity_stakes.stakes, identity_stakes.total_stakes, ) as u64; + // TODO remove + println!("max_uni_stream_connections {}", max_uni_stream_connections); let number_of_transactions_per_unistream = self .connection_parameters .number_of_transactions_per_unistream; @@ -95,7 +97,7 @@ impl ActiveConnection { let task_counter: Arc = Arc::new(AtomicU64::new(0)); let exit_signal = self.exit_signal.clone(); - let connection_pool = QuicConnectionPool::new( + let connection_pool = QuicCd onnectionPool::new( identity, self.endpoints.clone(), addr, From 7b33ec7d328f799b7424cf42e5da5ea52bb54314 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 26 Jul 2023 11:10:12 +0200 Subject: [PATCH 039/128] set quic client+server transport config * lite-rpc -> proxy * proxy -> TPU --- quic-forward-proxy/src/proxy.rs | 21 ++++++++++++++----- .../src/quic_connection_utils.rs | 21 +++++++++++-------- .../quic_proxy_connection_manager.rs | 11 +++++++--- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 41a88d95..e912c065 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -23,10 +23,14 @@ use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; use crate::proxy_request_format::TpuForwardingRequest; use crate::quic_connection_utils::QuicConnectionUtils; -use crate::tpu_quic_client::{TpuQuicClient}; +use crate::tpu_quic_client::{SingleTPUConnectionManager, TpuQuicClient}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::AnyhowJoinHandle; +// TODO tweak this value - solana server sets 256 +// setting this to "1" did not make a difference! +const MAX_CONCURRENT_UNI_STREAMS: u32 = 24; + pub struct QuicForwardProxy { endpoint: Endpoint, validator_identity: Arc, @@ -39,12 +43,19 @@ impl QuicForwardProxy { tls_config: &SelfSignedTlsConfigProvider, validator_identity: Arc) -> anyhow::Result { let server_tls_config = tls_config.get_server_tls_crypto_config(); - let mut quinn_server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); - let mut transport_config = TransportConfig::default(); - transport_config.max_idle_timeout(None); + + // note: this config must be aligned with lite-rpc's client config + let transport_config = Arc::get_mut(&mut quinn_server_config.transport).unwrap(); + // TODO experiment with this value + transport_config.max_concurrent_uni_streams(VarInt::from_u32(MAX_CONCURRENT_UNI_STREAMS)); + // no bidi streams used + transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); + let timeout = Duration::from_secs(10).try_into().unwrap(); + transport_config.max_idle_timeout(Some(timeout)); transport_config.keep_alive_interval(Some(Duration::from_millis(500))); - quinn_server_config.transport_config(Arc::new(transport_config)); + transport_config.stream_receive_window((PACKET_DATA_SIZE as u32).into()); + transport_config.receive_window((PACKET_DATA_SIZE as u32 * MAX_CONCURRENT_UNI_STREAMS).into()); let endpoint = Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap(); info!("tpu forward proxy listening on {}", endpoint.local_addr()?); diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs index 2b3cb17c..abc77ca4 100644 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -1,5 +1,5 @@ use log::{debug, error, info, trace, warn}; -use quinn::{ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, WriteError}; +use quinn::{ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, VarInt, WriteError}; use solana_sdk::pubkey::Pubkey; use std::{ collections::VecDeque, @@ -13,6 +13,7 @@ use std::{ use anyhow::bail; use futures::future::join_all; use itertools::Itertools; +use solana_sdk::quic::QUIC_MAX_TIMEOUT_MS; use tokio::{sync::RwLock, time::timeout}; use tokio::time::error::Elapsed; use tracing::instrument; @@ -22,7 +23,8 @@ const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; pub struct QuicConnectionUtils {} impl QuicConnectionUtils { - pub fn create_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { + // TODO move to a more specific place + pub fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { let mut endpoint = { let client_socket = solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::UNSPECIFIED), (8000, 10000)) @@ -40,18 +42,19 @@ impl QuicConnectionUtils { .expect("Failed to set QUIC client certificates"); crypto.enable_early_data = true; - // FIXME TEMP HACK TO ALLOW PROXY PROTOCOL - const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; - crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec(), ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; + crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; let mut config = ClientConfig::new(Arc::new(crypto)); - let mut transport_config = TransportConfig::default(); - // TODO check timing - let timeout = IdleTimeout::try_from(Duration::from_secs(5)).unwrap(); + // note: this should be aligned with solana quic server's endpoint config + let mut transport_config = TransportConfig::default(); + // no remotely-initiated streams required + transport_config.max_concurrent_uni_streams(VarInt::from_u32(0)); + transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); + let timeout = IdleTimeout::try_from(Duration::from_millis(QUIC_MAX_TIMEOUT_MS as u64)).unwrap(); transport_config.max_idle_timeout(Some(timeout)); - transport_config.keep_alive_interval(Some(Duration::from_millis(500))); + transport_config.keep_alive_interval(None); config.transport_config(Arc::new(transport_config)); endpoint.set_default_client_config(config); diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index c24e6a3e..52dd7f48 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use futures::FutureExt; use itertools::Itertools; use log::{debug, error, info, warn}; -use quinn::{ClientConfig, Connection, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig}; +use quinn::{ClientConfig, Connection, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt}; use solana_sdk::packet::PACKET_DATA_SIZE; use solana_sdk::pubkey::Pubkey; use solana_sdk::signer::Signer; @@ -123,13 +123,18 @@ impl QuicProxyConnectionManager { crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; let mut config = ClientConfig::new(Arc::new(crypto)); - let mut transport_config = TransportConfig::default(); + // note: this config must be aligned with quic-proxy's server config + let mut transport_config = TransportConfig::default(); let timeout = IdleTimeout::try_from(Duration::from_secs(1)).unwrap(); + // no remotely-initiated streams required + transport_config.max_concurrent_uni_streams(VarInt::from_u32(0)); + transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); + let timeout = Duration::from_secs(10).try_into().unwrap(); transport_config.max_idle_timeout(Some(timeout)); transport_config.keep_alive_interval(Some(Duration::from_millis(500))); - config.transport_config(Arc::new(transport_config)); + config.transport_config(Arc::new(transport_config)); endpoint.set_default_client_config(config); endpoint From 400048ccba64597cc19cdf0e2373e2b5abed3f29 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 26 Jul 2023 11:10:57 +0200 Subject: [PATCH 040/128] integrate send_transaction_batch --- .../tests/quic_proxy_tpu_integrationtest.rs | 2 +- quic-forward-proxy/README.md | 8 +- quic-forward-proxy/src/proxy.rs | 6 +- .../src/quic_connection_utils.rs | 84 ++++++- quic-forward-proxy/src/tpu_quic_client.rs | 216 ++++++++++++++---- .../quic_proxy_connection_manager.rs | 7 +- .../src/tpu_utils/tpu_connection_manager.rs | 2 +- 7 files changed, 272 insertions(+), 53 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 0b20d655..b521749a 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -339,7 +339,7 @@ fn configure_logging(verbose: bool) { "debug,rustls=info,quinn=info,quinn_proto=info,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=debug" }; let span_mode = if verbose { - FmtSpan::FULL + FmtSpan::CLOSE } else { FmtSpan::NONE }; diff --git a/quic-forward-proxy/README.md b/quic-forward-proxy/README.md index 4f37a6df..262f024e 100644 --- a/quic-forward-proxy/README.md +++ b/quic-forward-proxy/README.md @@ -44,4 +44,10 @@ cd bench; cargo run -- --tx-count=10 [2023-06-26T15:16:18.430839000Z DEBUG solana_streamer::nonblocking::quic] Peer type: Staked, stake 999999997717120, total stake 999999997717120, max streams 2048 receive_window Ok(12320) from peer 127.0.0.1:8058 [2023-06-26T15:16:18.430850000Z DEBUG solana_streamer::nonblocking::quic] quic new connection 127.0.0.1:8058 streams: 0 connections: 1 [2023-06-26T15:16:18.430854000Z DEBUG solana_streamer::nonblocking::quic] stream error: ApplicationClosed(ApplicationClose { error_code: 0, reason: b"done" }) -``` \ No newline at end of file +``` + + +QUIC/QUINN Endpoint and Connection specifics +--------------------------- +* keep-alive and idle timeout: both values must be aligned AND they must be configured on both endpoints (see [docs](https://docs.rs/quinn/latest/quinn/struct.TransportConfig.html#method.keep_alive_interval)) +* tune or disable __max_concurrent_uni_streams__ respectively diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index e912c065..58a2c26a 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -14,7 +14,9 @@ use rcgen::generate_simple_self_signed; use rustls::{Certificate, PrivateKey}; use rustls::server::ResolvesServerCert; use serde::{Deserialize, Serialize}; +use solana_sdk::packet::PACKET_DATA_SIZE; use solana_sdk::pubkey::Pubkey; +use solana_sdk::quic::QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS; use solana_sdk::signature::Keypair; use solana_sdk::signer::Signer; use solana_sdk::transaction::VersionedTransaction; @@ -153,11 +155,9 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client let tpu_address = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transactions(); - // TODO join get_or_create_connection future and read_to_end - let tpu_connection = tpu_quic_client_copy.get_or_create_connection(tpu_address).await; info!("send transaction batch of size {} to address {}", txs.len(), tpu_address); - tpu_quic_client_copy.send_txs_to_tpu(tpu_connection, &txs, exit_signal_copy).await; + tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &txs, exit_signal_copy).await; // active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await; diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs index abc77ca4..ecb840cb 100644 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -22,6 +22,23 @@ const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; pub struct QuicConnectionUtils {} +pub enum QuicConnectionError { + TimeOut, + ConnectionError { retry: bool }, +} + +// TODO check whot we need from this +#[derive(Clone, Copy)] +pub struct QuicConnectionParameters { + // pub connection_timeout: Duration, + pub unistream_timeout: Duration, + pub write_timeout: Duration, + pub finalize_timeout: Duration, + pub connection_retry_count: usize, + // pub max_number_of_connections: usize, + // pub number_of_transactions_per_unistream: usize, +} + impl QuicConnectionUtils { // TODO move to a more specific place pub fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { @@ -131,6 +148,56 @@ impl QuicConnectionUtils { } pub async fn write_all( + mut send_stream: SendStream, + tx: &Vec, + // identity: Pubkey, + connection_params: QuicConnectionParameters, + ) -> Result<(), QuicConnectionError> { + let write_timeout_res = timeout( + connection_params.write_timeout, + send_stream.write_all(tx.as_slice()), + ) + .await; + match write_timeout_res { + Ok(write_res) => { + if let Err(e) = write_res { + trace!( + "Error while writing transaction for {}, error {}", + "identity", + e + ); + return Err(QuicConnectionError::ConnectionError { retry: true }); + } + } + Err(_) => { + warn!("timeout while writing transaction for {}", "identity"); + return Err(QuicConnectionError::TimeOut); + } + } + + let finish_timeout_res = + timeout(connection_params.finalize_timeout, send_stream.finish()).await; + match finish_timeout_res { + Ok(finish_res) => { + if let Err(e) = finish_res { + trace!( + "Error while finishing transaction for {}, error {}", + "identity", + e + ); + return Err(QuicConnectionError::ConnectionError { retry: false }); + } + } + Err(_) => { + warn!("timeout while finishing transaction for {}", "identity"); + return Err(QuicConnectionError::TimeOut); + } + } + + Ok(()) + } + + pub async fn write_all_simple( send_stream: &mut SendStream, tx: &Vec, connection_timeout: Duration, @@ -178,6 +245,17 @@ impl QuicConnectionUtils { pub async fn open_unistream( connection: Connection, connection_timeout: Duration, + ) -> Result { + match timeout(connection_timeout, connection.open_uni()).await { + Ok(Ok(unistream)) => Ok(unistream), + Ok(Err(_)) => Err(QuicConnectionError::ConnectionError { retry: true }), + Err(_) => Err(QuicConnectionError::TimeOut), + } + } + + pub async fn open_unistream_simple( + connection: Connection, + connection_timeout: Duration, ) -> (Option, bool) { match timeout(connection_timeout, connection.open_uni()).await { Ok(Ok(unistream)) => (Some(unistream), false), @@ -200,7 +278,7 @@ impl QuicConnectionUtils { connection_timeout: Duration, ) { let (mut stream, _retry_conn) = - Self::open_unistream(connection.clone(), connection_timeout) + Self::open_unistream_simple(connection.clone(), connection_timeout) .await; if let Some(ref mut send_stream) = stream { if exit_signal.load(Ordering::Relaxed) { @@ -255,7 +333,7 @@ impl QuicConnectionUtils { async fn send_tx_to_new_stream(tx: &Vec, connection: Connection, connection_timeout: Duration) { - let mut send_stream = Self::open_unistream(connection.clone(), connection_timeout) + let mut send_stream = Self::open_unistream_simple(connection.clone(), connection_timeout) .await.0 .unwrap(); @@ -276,7 +354,7 @@ impl QuicConnectionUtils { } // TODO wrap in small timeout - send_stream.finish().await.unwrap(); + let _ = timeout(Duration::from_millis(200), send_stream.finish()).await; } } diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index 60f868ec..9a35f027 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -1,12 +1,13 @@ -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::io::Write; use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::Path; use std::str::FromStr; use std::sync::{Arc, Mutex}; -use std::sync::atomic::{AtomicBool, AtomicU64}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::Duration; use anyhow::{anyhow, bail}; +use async_trait::async_trait; use dashmap::DashMap; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; @@ -22,7 +23,7 @@ use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; -use crate::quic_connection_utils::QuicConnectionUtils; +use crate::quic_connection_utils::{QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); @@ -39,6 +40,62 @@ pub struct TpuQuicClient { // naive single non-recoverable connection - TODO moke it smarter // TODO consider using DashMap again connection_per_tpunode: Arc>, + last_stable_id: Arc, +} + +/// per TPU connection manager +#[async_trait] +pub trait SingleTPUConnectionManager { + async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> anyhow::Result; + fn update_last_stable_id(&self, stable_id: u64); +} + +pub type SingleTPUConnectionManagerWrapper = dyn SingleTPUConnectionManager + Sync + Send; + +#[async_trait] +impl SingleTPUConnectionManager for TpuQuicClient { + + #[tracing::instrument(skip(self), level = "debug")] + // TODO improve error handling + async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> anyhow::Result { + // TODO try 0rff + // QuicConnectionUtils::make_connection( + // self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) + // .await.unwrap() + + + { + if let Some(conn) = self.connection_per_tpunode.get(&tpu_address) { + debug!("reusing connection {} for tpu {}; last_stable_id is {}", + conn.stable_id(), tpu_address, self.last_stable_id.load(Ordering::Relaxed)); + return Ok(conn.clone()); + } + } + + let connection = + // TODO try 0rff + match QuicConnectionUtils::make_connection_0rtt( + self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) + .await { + Ok(conn) => conn, + Err(err) => { + warn!("Failed to open Quic connection to TPU {}: {}", tpu_address, err); + return Err(anyhow!("Failed to create Quic connection to TPU {}: {}", tpu_address, err)); + }, + }; + + let old_value = self.connection_per_tpunode.insert(tpu_address, connection.clone()); + assert!(old_value.is_none(), "no prev value must be overridden"); + + debug!("Created new Quic connection {} to TPU node {}, total connections is now {}", + connection.stable_id(), tpu_address, self.connection_per_tpunode.len()); + return Ok(connection); + } + + fn update_last_stable_id(&self, stable_id: u64) { + self.last_stable_id.store(stable_id, Ordering::Relaxed); + } + } impl TpuQuicClient { @@ -53,64 +110,141 @@ impl TpuQuicClient { ) .expect("Failed to initialize QUIC connection certificates"); - let endpoint_outbound = QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()); + let endpoint_outbound = QuicConnectionUtils::create_tpu_client_endpoint(certificate.clone(), key.clone()); let active_tpu_connection = TpuQuicClient { endpoint: endpoint_outbound.clone(), connection_per_tpunode: Arc::new(DashMap::new()), + last_stable_id: Arc::new(AtomicU64::new(0)), }; active_tpu_connection } - #[tracing::instrument(skip(self), level = "debug")] - pub async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> Connection { - info!("looking up {}", tpu_address); - // TODO try 0rff - // QuicConnectionUtils::make_connection( - // self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) - // .await.unwrap() + pub async fn send_txs_to_tpu(&self, + tpu_address: SocketAddr, + txs: &Vec, + exit_signal: Arc, + ) { + if true { + // throughput_50 493.70 tps + // throughput_50 769.43 tps (with finish timeout) + // TODO join get_or_create_connection future and read_to_end + // TODO add error handling + let tpu_connection = self.get_or_create_connection(tpu_address).await.unwrap(); - { - if let Some(conn) = self.connection_per_tpunode.get(&tpu_address) { - debug!("reusing connection {:?}", conn); - return conn.clone(); + for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { + let vecvec = chunk.iter().map(|tx| { + let tx_raw = bincode::serialize(tx).unwrap(); + tx_raw + }).collect_vec(); + QuicConnectionUtils::send_transaction_batch_parallel( + tpu_connection.clone(), + vecvec, + exit_signal.clone(), + QUIC_CONNECTION_TIMEOUT, + ).await; } - } + } else { + // throughput_50 676.65 tps + let connection_params = QuicConnectionParameters { + connection_retry_count: 10, + finalize_timeout: Duration::from_millis(200), + unistream_timeout: Duration::from_millis(500), + write_timeout: Duration::from_secs(1), + }; - let connection = - // TODO try 0rff - QuicConnectionUtils::make_connection( - self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) - .await.unwrap(); + let connection_manager = self as &SingleTPUConnectionManagerWrapper; - let old_value = self.connection_per_tpunode.insert(tpu_address, connection.clone()); - assert!(old_value.is_none(), "no prev value must be overridden"); + Self::send_transaction_batch(serialize_to_vecvec(&txs), tpu_address, exit_signal, connection_params, connection_manager).await; + + } - debug!("Created new Quic connection to TPU node {}, total connections is now {}", tpu_address, self.connection_per_tpunode.len()); - return connection; } - pub async fn send_txs_to_tpu(&self, - connection: Connection, - txs: &Vec, - exit_signal: Arc, + pub async fn send_transaction_batch(txs: Vec>, + tpu_address: SocketAddr, + exit_signal: Arc, + // _timeout_counters: Arc, + // last_stable_id: Arc, + connection_params: QuicConnectionParameters, + connection_manager: &SingleTPUConnectionManagerWrapper, ) { - - for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { - let vecvec = chunk.iter().map(|tx| { - let tx_raw = bincode::serialize(tx).unwrap(); - tx_raw - }).collect_vec(); - QuicConnectionUtils::send_transaction_batch_parallel( - connection.clone(), - vecvec, - exit_signal.clone(), - QUIC_CONNECTION_TIMEOUT, - ).await; + let mut queue = VecDeque::new(); + for tx in txs { + queue.push_back(tx); } + info!("send_transaction_batch: queue size is {}", queue.len()); + let connection_retry_count = connection_params.connection_retry_count; + for _ in 0..connection_retry_count { + if queue.is_empty() || exit_signal.load(Ordering::Relaxed) { + // return + return; + } + let mut do_retry = false; + while !queue.is_empty() { + let tx = queue.pop_front().unwrap(); + // remove Option + let connection = connection_manager.get_or_create_connection(tpu_address).await; + + if exit_signal.load(Ordering::Relaxed) { + return; + } + + if let Ok(connection) = connection { + let current_stable_id = connection.stable_id() as u64; + match QuicConnectionUtils::open_unistream( + connection, + connection_params.unistream_timeout, + ) + .await + { + Ok(send_stream) => { + match QuicConnectionUtils::write_all( + send_stream, + &tx, + connection_params, + ) + .await + { + Ok(()) => { + // do nothing + } + Err(QuicConnectionError::ConnectionError { retry }) => { + do_retry = retry; + } + Err(QuicConnectionError::TimeOut) => { + // timeout_counters.fetch_add(1, Ordering::Relaxed); + } + } + } + Err(QuicConnectionError::ConnectionError { retry }) => { + do_retry = retry; + } + Err(QuicConnectionError::TimeOut) => { + // timeout_counters.fetch_add(1, Ordering::Relaxed); + } + } + if do_retry { + connection_manager.update_last_stable_id(current_stable_id); + + queue.push_back(tx); + break; + } + } else { + warn!( + "Could not establish connection with {}", + "identity" + ); + break; + } + } + if !do_retry { + break; + } + } } } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 52dd7f48..cf1f0e60 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -253,6 +253,7 @@ impl QuicProxyConnectionManager { Ok(()) } + // TODO optimize connection async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { info!("sending {} bytes to proxy", proxy_request_raw.len()); @@ -260,11 +261,11 @@ impl QuicProxyConnectionManager { let connection = timeout(Duration::from_millis(500), connecting).await??; let mut send = connection.open_uni().await?; - send.write_all(proxy_request_raw).await?; + send.write_all(proxy_request_raw).await?; - send.finish().await?; + send.finish().await?; - Ok(()) + Ok(()) } diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 82def708..1fba7461 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -97,7 +97,7 @@ impl ActiveConnection { let task_counter: Arc = Arc::new(AtomicU64::new(0)); let exit_signal = self.exit_signal.clone(); - let connection_pool = QuicCd onnectionPool::new( + let connection_pool = QuicConnectionPool::new( identity, self.endpoints.clone(), addr, From 01020580524a9938b55b17b36dcad7b015aa8257 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 26 Jul 2023 13:22:56 +0200 Subject: [PATCH 041/128] connection stats --- core/src/quic_connection_utils.rs | 12 ++++ quic-forward-proxy/src/proxy.rs | 47 ++++++------ .../src/quic_connection_utils.rs | 16 ++++- quic-forward-proxy/src/tpu_quic_client.rs | 71 +++++++++++++------ .../quic_proxy_connection_manager.rs | 4 +- 5 files changed, 107 insertions(+), 43 deletions(-) diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index 8dd54314..05e2eb50 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -217,3 +217,15 @@ impl rustls::client::ServerCertVerifier for SkipServerVerification { Ok(rustls::client::ServerCertVerified::assertion()) } } + +// connection for sending proxy request: FrameStats { +// ACK: 2, CONNECTION_CLOSE: 0, CRYPTO: 3, DATA_BLOCKED: 0, DATAGRAM: 0, HANDSHAKE_DONE: 1, +// MAX_DATA: 0, MAX_STREAM_DATA: 1, MAX_STREAMS_BIDI: 0, MAX_STREAMS_UNI: 0, NEW_CONNECTION_ID: 4, +// NEW_TOKEN: 0, PATH_CHALLENGE: 0, PATH_RESPONSE: 0, PING: 0, RESET_STREAM: 0, RETIRE_CONNECTION_ID: 1, +// STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } +// rtt=1.08178ms +pub fn connection_stats(connection: &Connection) -> String { + format!("stable_id {}, rtt={:?}, stats {:?}", + connection.stable_id(), connection.stats().path.rtt, connection.stats().frame_rx) +} + diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 58a2c26a..04b513dd 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -24,7 +24,7 @@ use tokio::net::ToSocketAddrs; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; use crate::proxy_request_format::TpuForwardingRequest; -use crate::quic_connection_utils::QuicConnectionUtils; +use crate::quic_connection_utils::{connection_stats, QuicConnectionUtils}; use crate::tpu_quic_client::{SingleTPUConnectionManager, TpuQuicClient}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::AnyhowJoinHandle; @@ -123,7 +123,7 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client loop { let maybe_stream = client_connection.accept_uni().await; - let mut recv_stream = match maybe_stream { + let result = match maybe_stream { Err(quinn::ConnectionError::ApplicationClosed(reason)) => { debug!("connection closed by client - reason: {:?}", reason); if reason.error_code != VarInt::from_u32(0) { @@ -136,32 +136,39 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client error!("failed to accept stream: {}", e); return Err(anyhow::Error::msg("error accepting stream")); } - Ok(s) => s, - }; - let exit_signal_copy = exit_signal.clone(); - let validator_identity_copy = validator_identity.clone(); - let tpu_quic_client_copy = tpu_quic_client.clone(); + Ok(recv_stream) => { + let exit_signal_copy = exit_signal.clone(); + let validator_identity_copy = validator_identity.clone(); + let tpu_quic_client_copy = tpu_quic_client.clone(); - tokio::spawn(async move { + tokio::spawn(async move { - let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const - .unwrap(); - debug!("read proxy_request {} bytes", raw_request.len()); + let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const + .unwrap(); + trace!("read proxy_request {} bytes", raw_request.len()); - let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); + let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); - debug!("proxy request details: {}", proxy_request); - let tpu_identity = proxy_request.get_identity_tpunode(); - let tpu_address = proxy_request.get_tpu_socket_addr(); - let txs = proxy_request.get_transactions(); + trace!("proxy request details: {}", proxy_request); + let tpu_identity = proxy_request.get_identity_tpunode(); + let tpu_address = proxy_request.get_tpu_socket_addr(); + let txs = proxy_request.get_transactions(); + debug!("send transaction batch of size {} to address {}", txs.len(), tpu_address); + tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &txs, exit_signal_copy).await; - info!("send transaction batch of size {} to address {}", txs.len(), tpu_address); - tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &txs, exit_signal_copy).await; + debug!("connection stats (proxy inbound): {}", connection_stats(&client_connection)); - // active_tpu_connection_copy.send_txs_to_tpu(exit_signal_copy, validator_identity_copy, tpu_identity, tpu_address, &txs).await; + }); - }); + Ok(()) + }, + }; // -- result + + if let Err(e) = result { + return Err(e); + } + return Ok(()); } // -- loop } diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs index ecb840cb..c2a3f3d3 100644 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -243,7 +243,7 @@ impl QuicConnectionUtils { } pub async fn open_unistream( - connection: Connection, + connection: &Connection, connection_timeout: Duration, ) -> Result { match timeout(connection_timeout, connection.open_uni()).await { @@ -329,6 +329,8 @@ impl QuicConnectionUtils { let all_send_fns = (0..txs.len()).map(|i| Self::send_tx_to_new_stream(&txs[i], connection.clone(), connection_timeout)).collect_vec(); join_all(all_send_fns).await; + + debug!("connection stats (proxy send tx parallel): {}", connection_stats(&connection)); } @@ -380,3 +382,15 @@ impl rustls::client::ServerCertVerifier for SkipServerVerification { Ok(rustls::client::ServerCertVerified::assertion()) } } + +// stable_id 140266619216912, rtt=2.156683ms, +// stats FrameStats { ACK: 3, CONNECTION_CLOSE: 0, CRYPTO: 3, +// DATA_BLOCKED: 0, DATAGRAM: 0, HANDSHAKE_DONE: 1, MAX_DATA: 0, +// MAX_STREAM_DATA: 1, MAX_STREAMS_BIDI: 0, MAX_STREAMS_UNI: 0, NEW_CONNECTION_ID: 4, +// NEW_TOKEN: 0, PATH_CHALLENGE: 0, PATH_RESPONSE: 0, PING: 0, RESET_STREAM: 0, +// RETIRE_CONNECTION_ID: 1, STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, +// STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } +pub fn connection_stats(connection: &Connection) -> String { + format!("stable_id {} stats {:?}, rtt={:?}", + connection.stable_id(), connection.stats().frame_rx, connection.stats().path.rtt) +} \ No newline at end of file diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index 9a35f027..d6299688 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use dashmap::DashMap; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; -use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig, VarInt}; +use quinn::{Connecting, Connection, ConnectionError, Endpoint, SendStream, ServerConfig, VarInt}; use rcgen::generate_simple_self_signed; use rustls::{Certificate, PrivateKey}; use rustls::server::ResolvesServerCert; @@ -23,7 +23,7 @@ use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; -use crate::quic_connection_utils::{QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils}; +use crate::quic_connection_utils::{connection_stats, QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); @@ -46,6 +46,7 @@ pub struct TpuQuicClient { /// per TPU connection manager #[async_trait] pub trait SingleTPUConnectionManager { + // async fn refresh_connection(&self, connection: &Connection) -> Connection; async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> anyhow::Result; fn update_last_stable_id(&self, stable_id: u64); } @@ -55,8 +56,28 @@ pub type SingleTPUConnectionManagerWrapper = dyn SingleTPUConnectionManager + Sy #[async_trait] impl SingleTPUConnectionManager for TpuQuicClient { + // make sure the connection is usable for a resonable time + // never returns the "same" instances but a clone + // async fn refresh_connection(&self, connection: &Connection) -> Connection { + // let reverse_lookup = self.connection_per_tpunode.into_read_only().values().find(|conn| { + // conn.stable_id() == connection.stable_id() + // }); + // + // match reverse_lookup { + // Some(existing_conn) => { + // return existing_conn.clone(); + // } + // None => { + // TpuQuicClient::create_new(&self, reverse_lookup).await.unwrap()) + // } + // } + // + // // TODO implement + // connection.clone() + // } + #[tracing::instrument(skip(self), level = "debug")] - // TODO improve error handling + // TODO improve error handling; might need to signal if connection was reset async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> anyhow::Result { // TODO try 0rff // QuicConnectionUtils::make_connection( @@ -72,23 +93,11 @@ impl SingleTPUConnectionManager for TpuQuicClient { } } - let connection = - // TODO try 0rff - match QuicConnectionUtils::make_connection_0rtt( - self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) - .await { - Ok(conn) => conn, - Err(err) => { - warn!("Failed to open Quic connection to TPU {}: {}", tpu_address, err); - return Err(anyhow!("Failed to create Quic connection to TPU {}: {}", tpu_address, err)); - }, - }; - - let old_value = self.connection_per_tpunode.insert(tpu_address, connection.clone()); - assert!(old_value.is_none(), "no prev value must be overridden"); + let connection = match self.create_new(tpu_address).await { + Ok(value) => value, + Err(err) => return Err(err), + }; - debug!("Created new Quic connection {} to TPU node {}, total connections is now {}", - connection.stable_id(), tpu_address, self.connection_per_tpunode.len()); return Ok(connection); } @@ -186,7 +195,6 @@ impl TpuQuicClient { let mut do_retry = false; while !queue.is_empty() { let tx = queue.pop_front().unwrap(); - // remove Option let connection = connection_manager.get_or_create_connection(tpu_address).await; if exit_signal.load(Ordering::Relaxed) { @@ -196,7 +204,7 @@ impl TpuQuicClient { if let Ok(connection) = connection { let current_stable_id = connection.stable_id() as u64; match QuicConnectionUtils::open_unistream( - connection, + &connection, connection_params.unistream_timeout, ) .await @@ -211,6 +219,7 @@ impl TpuQuicClient { { Ok(()) => { // do nothing + debug!("connection stats (proxy send tx batch): {}", connection_stats(&connection)); } Err(QuicConnectionError::ConnectionError { retry }) => { do_retry = retry; @@ -247,6 +256,26 @@ impl TpuQuicClient { } } + pub(crate) async fn create_new(&self, tpu_address: SocketAddr) -> anyhow::Result { + let connection = + // TODO try 0rff + match QuicConnectionUtils::make_connection_0rtt( + self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) + .await { + Ok(conn) => conn, + Err(err) => { + warn!("Failed to open Quic connection to TPU {}: {}", tpu_address, err); + return Err(anyhow!("Failed to create Quic connection to TPU {}: {}", tpu_address, err)); + }, + }; + + let old_value = self.connection_per_tpunode.insert(tpu_address, connection.clone()); + assert!(old_value.is_none(), "no prev value must be overridden"); + + debug!("Created new Quic connection {} to TPU node {}, total connections is now {}", + connection.stable_id(), tpu_address, self.connection_per_tpunode.len()); + Ok(connection) + } } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index cf1f0e60..8a1e1511 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -19,8 +19,9 @@ use solana_sdk::signature::Keypair; use solana_sdk::transaction::VersionedTransaction; use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; use tokio::time::timeout; +use tracing::field::debug; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -use solana_lite_rpc_core::quic_connection_utils::{QuicConnectionParameters, QuicConnectionUtils, SkipServerVerification}; +use solana_lite_rpc_core::quic_connection_utils::{connection_stats, QuicConnectionParameters, QuicConnectionUtils, SkipServerVerification}; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; use solana_lite_rpc_core::tx_store::TxStore; @@ -265,6 +266,7 @@ impl QuicProxyConnectionManager { send.finish().await?; + debug!("connection stats (lite-rpc to proxy): {}", connection_stats(&connection)); Ok(()) } From f2dbdd8deaece3ee8f92be0ec3982488790c4572 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 26 Jul 2023 23:33:49 +0200 Subject: [PATCH 042/128] add cli option --- lite-rpc/src/bridge.rs | 3 +- lite-rpc/src/cli.rs | 2 + lite-rpc/src/main.rs | 16 ++++++++ quic-forward-proxy/Cargo.toml | 2 +- quic-forward-proxy/src/proxy.rs | 7 ++-- quic-forward-proxy/src/tpu_quic_client.rs | 40 ++++++++----------- .../quic_proxy_connection_manager.rs | 1 + .../src/tpu_utils/tpu_connection_manager.rs | 2 - 8 files changed, 41 insertions(+), 32 deletions(-) diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index e01b41fa..d2434eda 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -83,6 +83,7 @@ impl LiteBridge { validator_identity: Arc, retry_after: Duration, max_retries: usize, + tpu_connection_path: TpuConnectionPath, ) -> anyhow::Result { let rpc_client = Arc::new(RpcClient::new(rpc_url.clone())); let current_slot = rpc_client @@ -108,7 +109,7 @@ impl LiteBridge { write_timeout: Duration::from_secs(1), number_of_transactions_per_unistream: 8, }, - tpu_connection_path: TpuConnectionPath::QuicDirectPath, + tpu_connection_path, }; let tpu_service = TpuService::new( diff --git a/lite-rpc/src/cli.rs b/lite-rpc/src/cli.rs index fb6c7f81..5fc91bda 100644 --- a/lite-rpc/src/cli.rs +++ b/lite-rpc/src/cli.rs @@ -29,4 +29,6 @@ pub struct Args { pub maximum_retries_per_tx: usize, #[arg(long, default_value_t = DEFAULT_RETRY_TIMEOUT)] pub transaction_retry_after_secs: u64, + #[arg(long)] + pub experimental_quic_proxy_addr: Option, } diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 2bd0e644..3f5e0b99 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -10,6 +10,8 @@ use lite_rpc::{bridge::LiteBridge, cli::Args}; use solana_sdk::signature::Keypair; use std::env; use std::sync::Arc; +use clap::builder::TypedValueParser; +use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; use crate::rpc_tester::RpcTester; @@ -48,12 +50,15 @@ pub async fn start_lite_rpc(args: Args) -> anyhow::Result<()> { identity_keypair, maximum_retries_per_tx, transaction_retry_after_secs, + experimental_quic_proxy_addr, } = args; let validator_identity = Arc::new(get_identity_keypair(&identity_keypair).await); let retry_after = Duration::from_secs(transaction_retry_after_secs); + let tpu_connection_path = configure_tpu_connection_path(experimental_quic_proxy_addr); + LiteBridge::new( rpc_addr, ws_addr, @@ -61,6 +66,7 @@ pub async fn start_lite_rpc(args: Args) -> anyhow::Result<()> { validator_identity, retry_after, maximum_retries_per_tx, + tpu_connection_path ) .await .context("Error building LiteBridge")? @@ -73,6 +79,16 @@ pub async fn start_lite_rpc(args: Args) -> anyhow::Result<()> { .await } +fn configure_tpu_connection_path(experimental_quic_proxy_addr: Option) -> TpuConnectionPath { + match experimental_quic_proxy_addr { + None => TpuConnectionPath::QuicDirectPath, + Some(prox_address) => TpuConnectionPath::QuicForwardProxyPath { + // e.g. "127.0.0.1:11111" + forward_proxy_address: prox_address.parse().unwrap() + }, + } +} + fn get_args() -> Args { let mut args = Args::parse(); diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 34dd1e5d..18e5a264 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -27,7 +27,7 @@ log = { workspace = true } clap = { workspace = true } dashmap = { workspace = true } itertools = { workspace = true } -tracing = "0.1.37" +tracing = { workspace = true } tracing-subscriber = { workspace = true } native-tls = { workspace = true } prometheus = { workspace = true } diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 04b513dd..52cd15de 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -60,8 +60,7 @@ impl QuicForwardProxy { transport_config.receive_window((PACKET_DATA_SIZE as u32 * MAX_CONCURRENT_UNI_STREAMS).into()); let endpoint = Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap(); - info!("tpu forward proxy listening on {}", endpoint.local_addr()?); - info!("staking from validator identity {}", validator_identity.pubkey()); + info!("Quic proxy uses validator identity {}", validator_identity.pubkey()); let tpu_quic_client = TpuQuicClient::new_with_validator_identity(validator_identity.as_ref()).await; @@ -77,7 +76,7 @@ impl QuicForwardProxy { let endpoint = self.endpoint.clone(); let quic_proxy: AnyhowJoinHandle = tokio::spawn(async move { - info!("TPU Quic Proxy server start on {}", endpoint.local_addr()?); + info!("TPU Quic Proxy server listening on {}", endpoint.local_addr()?); while let Some(connecting) = endpoint.accept().await { @@ -87,7 +86,7 @@ impl QuicForwardProxy { let tpu_quic_client = self.tpu_quic_client.clone(); tokio::spawn(async move { - let connection = connecting.await.context("accept connection").unwrap(); + let connection = connecting.await.context("handshake").unwrap(); match accept_client_connection(connection, tpu_quic_client, exit_signal, validator_identity_copy) .await { Ok(()) => {} diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index d6299688..ceda5066 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -93,11 +93,23 @@ impl SingleTPUConnectionManager for TpuQuicClient { } } - let connection = match self.create_new(tpu_address).await { - Ok(value) => value, - Err(err) => return Err(err), - }; + let connection = + // TODO try 0rff + match QuicConnectionUtils::make_connection_0rtt( + self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) + .await { + Ok(conn) => conn, + Err(err) => { + warn!("Failed to open Quic connection to TPU {}: {}", tpu_address, err); + return Err(anyhow!("Failed to create Quic connection to TPU {}: {}", tpu_address, err)); + }, + }; + let old_value = self.connection_per_tpunode.insert(tpu_address, connection.clone()); + assert!(old_value.is_none(), "no prev value must be overridden"); + + debug!("Created new Quic connection {} to TPU node {}, total connections is now {}", + connection.stable_id(), tpu_address, self.connection_per_tpunode.len()); return Ok(connection); } @@ -256,26 +268,6 @@ impl TpuQuicClient { } } - pub(crate) async fn create_new(&self, tpu_address: SocketAddr) -> anyhow::Result { - let connection = - // TODO try 0rff - match QuicConnectionUtils::make_connection_0rtt( - self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) - .await { - Ok(conn) => conn, - Err(err) => { - warn!("Failed to open Quic connection to TPU {}: {}", tpu_address, err); - return Err(anyhow!("Failed to create Quic connection to TPU {}: {}", tpu_address, err)); - }, - }; - - let old_value = self.connection_per_tpunode.insert(tpu_address, connection.clone()); - assert!(old_value.is_none(), "no prev value must be overridden"); - - debug!("Created new Quic connection {} to TPU node {}, total connections is now {}", - connection.stable_id(), tpu_address, self.connection_per_tpunode.len()); - Ok(connection) - } } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 8a1e1511..e9005273 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -46,6 +46,7 @@ impl QuicProxyConnectionManager { validator_identity: Arc, proxy_addr: SocketAddr, ) -> Self { + info!("Configure Quic proxy connection manager to {}", proxy_addr); let endpoint = Self::create_proxy_client_endpoint(certificate.clone(), key.clone()); Self { diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 1fba7461..8f665121 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -88,8 +88,6 @@ impl ActiveConnection { identity_stakes.stakes, identity_stakes.total_stakes, ) as u64; - // TODO remove - println!("max_uni_stream_connections {}", max_uni_stream_connections); let number_of_transactions_per_unistream = self .connection_parameters .number_of_transactions_per_unistream; From 3bebfa2c3b17c3a4d2625f2f032027e0797cabf5 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 27 Jul 2023 00:06:20 +0200 Subject: [PATCH 043/128] replace DashMap --- quic-forward-proxy/Cargo.toml | 1 - quic-forward-proxy/src/tpu_quic_client.rs | 13 +++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 18e5a264..a95fc1b9 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -25,7 +25,6 @@ bytes = { workspace = true } anyhow = { workspace = true } log = { workspace = true } clap = { workspace = true } -dashmap = { workspace = true } itertools = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index ceda5066..f6f4088d 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -33,13 +33,13 @@ pub const MAX_TRANSACTIONS_PER_BATCH: usize = 10; pub const MAX_BYTES_PER_BATCH: usize = 10; const MAX_PARALLEL_STREAMS: usize = 6; -/// stable connect to TPU to send transactions - optimized for proxy use case +/// maintain many quic connection and streams to one TPU to send transactions - optimized for proxy use case #[derive(Debug, Clone)] pub struct TpuQuicClient { endpoint: Endpoint, // naive single non-recoverable connection - TODO moke it smarter // TODO consider using DashMap again - connection_per_tpunode: Arc>, + connection_per_tpunode: Arc>>, last_stable_id: Arc, } @@ -86,7 +86,7 @@ impl SingleTPUConnectionManager for TpuQuicClient { { - if let Some(conn) = self.connection_per_tpunode.get(&tpu_address) { + if let Some(conn) = self.connection_per_tpunode.read().await.get(&tpu_address) { debug!("reusing connection {} for tpu {}; last_stable_id is {}", conn.stable_id(), tpu_address, self.last_stable_id.load(Ordering::Relaxed)); return Ok(conn.clone()); @@ -105,11 +105,12 @@ impl SingleTPUConnectionManager for TpuQuicClient { }, }; - let old_value = self.connection_per_tpunode.insert(tpu_address, connection.clone()); + let mut lock = self.connection_per_tpunode.write().await; + let old_value = lock.insert(tpu_address, connection.clone()); assert!(old_value.is_none(), "no prev value must be overridden"); debug!("Created new Quic connection {} to TPU node {}, total connections is now {}", - connection.stable_id(), tpu_address, self.connection_per_tpunode.len()); + connection.stable_id(), tpu_address, lock.len()); return Ok(connection); } @@ -135,7 +136,7 @@ impl TpuQuicClient { let active_tpu_connection = TpuQuicClient { endpoint: endpoint_outbound.clone(), - connection_per_tpunode: Arc::new(DashMap::new()), + connection_per_tpunode: Arc::new(RwLock::new(HashMap::new())), last_stable_id: Arc::new(AtomicU64::new(0)), }; From d378209195b0cb9fc8af10f9fd64a94497576b36 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 27 Jul 2023 11:20:03 +0200 Subject: [PATCH 044/128] restore DashMap dependency --- quic-forward-proxy/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index a95fc1b9..18e5a264 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -25,6 +25,7 @@ bytes = { workspace = true } anyhow = { workspace = true } log = { workspace = true } clap = { workspace = true } +dashmap = { workspace = true } itertools = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } From 23ffbc50abb50e37bf189b5602b342239df1b6ca Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 27 Jul 2023 11:27:31 +0200 Subject: [PATCH 045/128] refactor --- core/src/proxy_request_format.rs | 2 +- quic-forward-proxy/src/proxy.rs | 50 +++++++++---------- .../src/proxy_request_format.rs | 4 +- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index e888ae2c..cfc7d5ef 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -15,7 +15,7 @@ const FORMAT_VERSION1: u16 = 2301; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { format_version: u16, - tpu_socket_addr: SocketAddr, // TODO is that correct + tpu_socket_addr: SocketAddr, // TODO is that correct, maybe it should be V4; maybe we also need to provide a list identity_tpunode: Pubkey, transactions: Vec, } diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 52cd15de..b6e9f1d2 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -1,3 +1,4 @@ +use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::Path; use std::sync::Arc; @@ -5,7 +6,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64}; use std::thread::sleep; use std::time::Duration; use tracing::{debug_span, instrument, Instrument, span}; -use anyhow::{anyhow, bail, Context}; +use anyhow::{anyhow, bail, Context, Error}; use dashmap::DashMap; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; @@ -75,31 +76,7 @@ impl QuicForwardProxy { let exit_signal = Arc::new(AtomicBool::new(false)); let endpoint = self.endpoint.clone(); - let quic_proxy: AnyhowJoinHandle = tokio::spawn(async move { - info!("TPU Quic Proxy server listening on {}", endpoint.local_addr()?); - - - while let Some(connecting) = endpoint.accept().await { - - let exit_signal = exit_signal.clone(); - let validator_identity_copy = self.validator_identity.clone(); - let tpu_quic_client = self.tpu_quic_client.clone(); - tokio::spawn(async move { - - let connection = connecting.await.context("handshake").unwrap(); - match accept_client_connection(connection, tpu_quic_client, exit_signal, validator_identity_copy) - .await { - Ok(()) => {} - Err(err) => { - error!("setup connection failed: {reason}", reason = err); - } - } - - }); - } - - bail!("TPU Quic Proxy server stopped"); - }); + let quic_proxy: AnyhowJoinHandle = tokio::spawn(self.listen(exit_signal, endpoint)); tokio::select! { res = quic_proxy => { @@ -108,6 +85,27 @@ impl QuicForwardProxy { } } + async fn listen(mut self, exit_signal: Arc, endpoint: Endpoint) -> anyhow::Result<()> { + info!("TPU Quic Proxy server listening on {}", endpoint.local_addr()?); + + while let Some(connecting) = endpoint.accept().await { + let exit_signal = exit_signal.clone(); + let validator_identity_copy = self.validator_identity.clone(); + let tpu_quic_client = self.tpu_quic_client.clone(); + tokio::spawn(async move { + let connection = connecting.await.context("handshake").unwrap(); + match accept_client_connection(connection, tpu_quic_client, exit_signal, validator_identity_copy) + .await { + Ok(()) => {} + Err(err) => { + error!("setup connection failed: {reason}", reason = err); + } + } + }); + } + + bail!("TPU Quic Proxy server stopped"); + } } diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 2339173e..d926b71b 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -10,7 +10,7 @@ use solana_sdk::transaction::VersionedTransaction; /// lite-rpc to proxy wire format /// compat info: non-public format ATM /// initial version -const FORMAT_VERSION1: u16 = 2301; +pub const FORMAT_VERSION1: u16 = 2301; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { @@ -50,7 +50,7 @@ impl TpuForwardingRequest { .context("deserialize proxy request") .unwrap(); - assert_eq!(request.format_version, 2301); + assert_eq!(request.format_version, FORMAT_VERSION1); request } From d2d28bb474f3b3080aca348c2d98ca4f38bb2fa3 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 27 Jul 2023 16:13:39 +0200 Subject: [PATCH 046/128] add tx forwarder --- core/src/quic_connection_utils.rs | 1 + quic-forward-proxy/src/proxy.rs | 120 ++++++++++++++++++++-- quic-forward-proxy/src/tpu_quic_client.rs | 55 +++++++++- 3 files changed, 165 insertions(+), 11 deletions(-) diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index 05e2eb50..4c79c48f 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -225,6 +225,7 @@ impl rustls::client::ServerCertVerifier for SkipServerVerification { // STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } // rtt=1.08178ms pub fn connection_stats(connection: &Connection) -> String { + // see https://www.rfc-editor.org/rfc/rfc9000.html#name-frame-types-and-formats format!("stable_id {}, rtt={:?}, stats {:?}", connection.stable_id(), connection.stats().path.rtt, connection.stats().frame_rx) } diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index b6e9f1d2..7062d6dd 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::Path; @@ -23,10 +24,15 @@ use solana_sdk::signer::Signer; use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use tokio::sync::mpsc::{channel, Receiver, Sender}; use tokio::sync::RwLock; +use tokio::task::JoinHandle; +use tokio::time::error::Elapsed; +use tokio::time::timeout; +use tracing::field::debug; use crate::proxy_request_format::TpuForwardingRequest; use crate::quic_connection_utils::{connection_stats, QuicConnectionUtils}; -use crate::tpu_quic_client::{SingleTPUConnectionManager, TpuQuicClient}; +use crate::tpu_quic_client::{send_txs_to_tpu_static, SingleTPUConnectionManager, TpuQuicClient}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::AnyhowJoinHandle; @@ -40,6 +46,13 @@ pub struct QuicForwardProxy { tpu_quic_client: TpuQuicClient, } +/// internal structure with transactions and target TPU +#[derive(Debug)] +struct ForwardPacket { + pub transactions: Vec, + pub tpu_address: SocketAddr, +} + impl QuicForwardProxy { pub async fn new( proxy_listener_addr: SocketAddr, @@ -75,26 +88,36 @@ impl QuicForwardProxy { ) -> anyhow::Result<()> { let exit_signal = Arc::new(AtomicBool::new(false)); + let tpu_quic_client_copy = self.tpu_quic_client.clone(); let endpoint = self.endpoint.clone(); - let quic_proxy: AnyhowJoinHandle = tokio::spawn(self.listen(exit_signal, endpoint)); + let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); + + let quic_proxy: AnyhowJoinHandle = tokio::spawn(self.listen(exit_signal.clone(), endpoint, forwarder_channel)); + + let forwarder: AnyhowJoinHandle = tokio::spawn(tx_forwarder(tpu_quic_client_copy, forward_receiver, exit_signal.clone())); tokio::select! { res = quic_proxy => { bail!("TPU Quic Proxy server exited unexpectedly {res:?}"); }, + res = forwarder => { + bail!("TPU Quic Tx forwarder exited unexpectedly {res:?}"); + }, } } - async fn listen(mut self, exit_signal: Arc, endpoint: Endpoint) -> anyhow::Result<()> { + async fn listen(self, exit_signal: Arc, endpoint: Endpoint, forwarder_channel: Sender) -> anyhow::Result<()> { info!("TPU Quic Proxy server listening on {}", endpoint.local_addr()?); while let Some(connecting) = endpoint.accept().await { let exit_signal = exit_signal.clone(); let validator_identity_copy = self.validator_identity.clone(); let tpu_quic_client = self.tpu_quic_client.clone(); + let forwarder_channel_copy = forwarder_channel.clone(); tokio::spawn(async move { let connection = connecting.await.context("handshake").unwrap(); - match accept_client_connection(connection, tpu_quic_client, exit_signal, validator_identity_copy) + match accept_client_connection(connection, forwarder_channel_copy, + tpu_quic_client, exit_signal, validator_identity_copy) .await { Ok(()) => {} Err(err) => { @@ -111,7 +134,8 @@ impl QuicForwardProxy { // TODO use interface abstraction for connection_per_tpunode #[tracing::instrument(skip_all, level = "debug")] -async fn accept_client_connection(client_connection: Connection, tpu_quic_client: TpuQuicClient, +async fn accept_client_connection(client_connection: Connection, forwarder_channel: Sender, + tpu_quic_client: TpuQuicClient, exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { debug!("inbound connection established, client {}", client_connection.remote_address()); @@ -138,6 +162,7 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client let validator_identity_copy = validator_identity.clone(); let tpu_quic_client_copy = tpu_quic_client.clone(); + let forwarder_channel_copy = forwarder_channel.clone(); tokio::spawn(async move { let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const @@ -151,10 +176,11 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client let tpu_address = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transactions(); - debug!("send transaction batch of size {} to address {}", txs.len(), tpu_address); - tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &txs, exit_signal_copy).await; + debug!("enqueue transaction batch of size {} to address {}", txs.len(), tpu_address); + // tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &txs, exit_signal_copy).await; + forwarder_channel_copy.send(ForwardPacket { transactions: txs, tpu_address }).await.unwrap(); - debug!("connection stats (proxy inbound): {}", connection_stats(&client_connection)); + // debug!("connection stats (proxy inbound): {}", connection_stats(&client_connection)); }); @@ -166,6 +192,82 @@ async fn accept_client_connection(client_connection: Connection, tpu_quic_client return Err(e); } - return Ok(()); } // -- loop } + +// takes transactions from upstream clients and forwards them to the TPU +async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: Receiver, exit_signal: Arc) -> anyhow::Result<()> { + info!("TPU Quic forwarder started"); + + let mut agents: HashMap> = HashMap::new(); + + let tpu_quic_client_copy = tpu_quic_client.clone(); + loop { + // TODO add exit + + let forward_packet = transaction_channel.recv().await.expect("channel closed unexpectedly"); + let tpu_address = forward_packet.tpu_address; + + if !agents.contains_key(&tpu_address) { + let (sender, mut receiver) = channel::(10000); + // TODO cleanup agent after a while of iactivity + agents.insert(tpu_address, sender); + + let tpu_quic_client_copy = tpu_quic_client.clone(); + let exit_signal = exit_signal.clone(); + tokio::spawn(async move { + debug!("Start Quic forwarder agent for TPU {}", tpu_address); + // TODO pass+check the tpu_address + // TODO connect + // TODO consume queue + // TODO exit signal + + loop { + let maybe_connection = tpu_quic_client_copy.create_connection(tpu_address).await; + if maybe_connection.is_err() { + // TODO implement retries etc + error!("failed to connect to TPU {} - giving up, dropping unprocessed elements in channel", tpu_address); + return; + } + + let connection = maybe_connection.unwrap(); + + let exit_signal = exit_signal.clone(); + while let Some(packet) = receiver.recv().await { + assert_eq!(packet.tpu_address, tpu_address, "routing error"); + + debug!("forwarding transaction batch of size {} to address {}", packet.transactions.len(), packet.tpu_address); + + // TODo move send_txs_to_tpu_static to tpu_quic_client + timeout(Duration::from_millis(500), + send_txs_to_tpu_static(connection.clone(), tpu_address, &packet.transactions, exit_signal.clone())).await + .expect("timeout sending data to TPU node") + + } + + } + + }); + + } // -- new agent + + let agent_channel = agents.get(&tpu_address).unwrap(); + agent_channel.send(forward_packet).await.unwrap(); + + + + // check if the tpu has already a task+queue running, if not start one, sort+queue packets by tpu address + // maintain the health of a TPU connection, debounce errors; if failing, drop the respective messages + + // let exit_signal_copy = exit_signal.clone(); + // debug!("send transaction batch of size {} to address {}", forward_packet.transactions.len(), forward_packet.tpu_address); + // // TODO: this will block/timeout if the TPU is not available + // timeout(Duration::from_millis(500), + // tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &forward_packet.transactions, exit_signal_copy)).await; + // tpu_quic_client_copy.send_txs_to_tpu(forward_packet.tpu_address, &forward_packet.transactions, exit_signal_copy).await; + + } + + bail!("TPU Quic forward service stopped"); +} + diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index f6f4088d..9c85d715 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::Duration; -use anyhow::{anyhow, bail}; +use anyhow::{anyhow, bail, Error}; use async_trait::async_trait; use dashmap::DashMap; use itertools::{any, Itertools}; @@ -43,6 +43,24 @@ pub struct TpuQuicClient { last_stable_id: Arc, } +impl TpuQuicClient { + pub async fn create_connection(&self, tpu_address: SocketAddr) -> anyhow::Result { + let connection = + // TODO try 0rff + match QuicConnectionUtils::make_connection_0rtt( + self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) + .await { + Ok(conn) => conn, + Err(err) => { + warn!("Failed to open Quic connection to TPU {}: {}", tpu_address, err); + return Err(anyhow!("Failed to create Quic connection to TPU {}: {}", tpu_address, err)); + }, + }; + + Ok(connection) + } +} + /// per TPU connection manager #[async_trait] pub trait SingleTPUConnectionManager { @@ -143,13 +161,15 @@ impl TpuQuicClient { active_tpu_connection } + #[tracing::instrument(skip_all, level = "debug")] pub async fn send_txs_to_tpu(&self, tpu_address: SocketAddr, txs: &Vec, exit_signal: Arc, ) { - if true { + if false { + // note: this impl does not deal with connection errors // throughput_50 493.70 tps // throughput_50 769.43 tps (with finish timeout) // TODO join get_or_create_connection future and read_to_end @@ -179,6 +199,7 @@ impl TpuQuicClient { let connection_manager = self as &SingleTPUConnectionManagerWrapper; + // TODO connection_params should be part of connection_manager Self::send_transaction_batch(serialize_to_vecvec(&txs), tpu_address, exit_signal, connection_params, connection_manager).await; } @@ -278,3 +299,33 @@ fn serialize_to_vecvec(transactions: &Vec) -> Vec> tx_raw }).collect_vec() } + + +// send potentially large amount of transactions to a single TPU +pub async fn send_txs_to_tpu_static( + tpu_connection: Connection, + tpu_address: SocketAddr, + txs: &Vec, + exit_signal: Arc, +) { + + // note: this impl does not deal with connection errors + // throughput_50 493.70 tps + // throughput_50 769.43 tps (with finish timeout) + // TODO join get_or_create_connection future and read_to_end + // TODO add error handling + + for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { + let vecvec = chunk.iter().map(|tx| { + let tx_raw = bincode::serialize(tx).unwrap(); + tx_raw + }).collect_vec(); + QuicConnectionUtils::send_transaction_batch_parallel( + tpu_connection.clone(), + vecvec, + exit_signal.clone(), + QUIC_CONNECTION_TIMEOUT, + ).await; + } + +} \ No newline at end of file From 7abf1351e8dd60775e0c5c48b804b782cb2544d3 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 28 Jul 2023 14:38:55 +0200 Subject: [PATCH 047/128] add quinn auto reconnect --- .../src/quinn_auto_reconnect.rs | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 quic-forward-proxy/src/quinn_auto_reconnect.rs diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs new file mode 100644 index 00000000..4d639fff --- /dev/null +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -0,0 +1,118 @@ +use std::cell::RefCell; +use std::fmt; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicU32, Ordering}; +use tracing::{debug, info}; +use quinn::{Connection, Endpoint}; +use tokio::sync::{RwLock, RwLockWriteGuard}; + +pub struct AutoReconnect { + endpoint: Endpoint, + // note: no read lock is used ATM + current: RwLock>, + target_address: SocketAddr, + reconnect_count: AtomicU32, +} + +impl AutoReconnect { + pub fn new(endpoint: Endpoint, target_address: SocketAddr) -> Self { + Self { + endpoint, + current: RwLock::new(None), + target_address, + reconnect_count: AtomicU32::new(0), + } + } + + pub async fn roundtrip(&self, payload: Vec) -> anyhow::Result> { + // TOOD do smart error handling + reconnect + // self.refresh().await.open_bi().await.unwrap() + let (mut send_stream, recv_stream) = self.refresh().await.open_bi().await?; + send_stream.write_all(payload.as_slice()).await?; + send_stream.finish().await?; + + let answer = recv_stream.read_to_end(64 * 1024).await?; + + Ok(answer) + } + + pub async fn send(&self, payload: Vec) -> anyhow::Result<()> { + // TOOD do smart error handling + reconnect + let mut send_stream = self.refresh().await.open_uni().await?; + send_stream.write_all(payload.as_slice()).await?; + send_stream.finish().await?; + + + Ok(()) + } + + + pub async fn refresh(&self) -> Connection { + { + let lock = self.current.read().await; + let maybe_conn: &Option = &*lock; + if maybe_conn.as_ref().filter(|conn| conn.close_reason().is_none()).is_some() { + // let reuse = lock.unwrap().clone(); + let reuse = maybe_conn.as_ref().unwrap(); + debug!("Reuse connection {}", reuse.stable_id()); + return reuse.clone(); + } + } + let mut lock = self.current.write().await; + match &*lock { + Some(current) => { + + if current.close_reason().is_some() { + info!("Connection is closed for reason: {:?}", current.close_reason()); + // TODO log + + let new_connection = self.create_connection().await; + *lock = Some(new_connection.clone()); + // let old_conn = lock.replace(new_connection.clone()); + self.reconnect_count.fetch_add(1, Ordering::SeqCst); + + // debug!("Replace closed connection {} with {} (retry {})", + // old_conn.map(|c| c.stable_id().to_string()).unwrap_or("none".to_string()), + // new_connection.stable_id(), + // self.reconnect_count.load(Ordering::SeqCst)); + // TODO log old vs new stable_id + + + return new_connection.clone(); + } else { + debug!("Reuse connection {} with write-lock", current.stable_id()); + return current.clone(); + } + + } + None => { + let new_connection = self.create_connection().await; + + // let old_conn = lock.replace(new_connection.clone()); + // assert!(old_conn.is_none(), "old connection should be None"); + *lock = Some(new_connection.clone()); + // let old_conn = foo.replace(Some(new_connection.clone())); + // TODO log old vs new stable_id + debug!("Create initial connection {}", new_connection.stable_id()); + + return new_connection.clone(); + } + } + } + + async fn create_connection(&self) -> Connection { + let connection = + self.endpoint.connect(self.target_address, "localhost").expect("handshake"); + + connection.await.expect("connection") + } +} + +impl fmt::Display for AutoReconnect { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Connection to {}", + self.target_address, + ) + } +} + From 8ea7428afda813cba3de729e8642a20096add5c7 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 28 Jul 2023 14:39:59 +0200 Subject: [PATCH 048/128] inline proxy outbound quinn handling --- quic-forward-proxy/src/lib.rs | 1 + quic-forward-proxy/src/main.rs | 1 + quic-forward-proxy/src/proxy.rs | 28 ++++++++++---------- quic-forward-proxy/src/tpu_quic_client.rs | 31 +++++++++++++++-------- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index 2aa1d6b6..7bda23d9 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -9,3 +9,4 @@ mod util; mod tx_store; mod identity_stakes; mod quic_connection_utils; +mod quinn_auto_reconnect; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 7d4a5947..04ed9930 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -21,6 +21,7 @@ mod util; mod tx_store; mod identity_stakes; mod quic_connection_utils; +mod quinn_auto_reconnect; #[tokio::main(flavor = "multi_thread", worker_threads = 16)] diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 7062d6dd..4651c5d9 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -4,6 +4,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64}; +use std::thread; use std::thread::sleep; use std::time::Duration; use tracing::{debug_span, instrument, Instrument, span}; @@ -11,7 +12,7 @@ use anyhow::{anyhow, bail, Context, Error}; use dashmap::DashMap; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; -use quinn::{Connecting, Connection, Endpoint, SendStream, ServerConfig, TransportConfig, VarInt}; +use quinn::{Connecting, Connection, ConnectionError, Endpoint, SendStream, ServerConfig, TransportConfig, VarInt}; use rcgen::generate_simple_self_signed; use rustls::{Certificate, PrivateKey}; use rustls::server::ResolvesServerCert; @@ -25,13 +26,14 @@ use solana_sdk::transaction::VersionedTransaction; use tokio::net::ToSocketAddrs; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::mpsc::{channel, Receiver, Sender}; -use tokio::sync::RwLock; +use tokio::sync::{RwLock, RwLockReadGuard}; use tokio::task::JoinHandle; use tokio::time::error::Elapsed; use tokio::time::timeout; use tracing::field::debug; use crate::proxy_request_format::TpuForwardingRequest; use crate::quic_connection_utils::{connection_stats, QuicConnectionUtils}; +use crate::quinn_auto_reconnect::AutoReconnect; use crate::tpu_quic_client::{send_txs_to_tpu_static, SingleTPUConnectionManager, TpuQuicClient}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::AnyhowJoinHandle; @@ -222,26 +224,23 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R // TODO consume queue // TODO exit signal + let auto_connection = AutoReconnect::new(tpu_quic_client_copy.get_endpoint(), tpu_address); + // let mut connection = tpu_quic_client_copy.create_connection(tpu_address).await.expect("handshake"); loop { - let maybe_connection = tpu_quic_client_copy.create_connection(tpu_address).await; - if maybe_connection.is_err() { - // TODO implement retries etc - error!("failed to connect to TPU {} - giving up, dropping unprocessed elements in channel", tpu_address); - return; - } - - let connection = maybe_connection.unwrap(); let exit_signal = exit_signal.clone(); - while let Some(packet) = receiver.recv().await { + loop { + let packet = receiver.recv().await.unwrap(); assert_eq!(packet.tpu_address, tpu_address, "routing error"); debug!("forwarding transaction batch of size {} to address {}", packet.transactions.len(), packet.tpu_address); // TODo move send_txs_to_tpu_static to tpu_quic_client - timeout(Duration::from_millis(500), - send_txs_to_tpu_static(connection.clone(), tpu_address, &packet.transactions, exit_signal.clone())).await - .expect("timeout sending data to TPU node") + let result = timeout(Duration::from_millis(500), + send_txs_to_tpu_static(&auto_connection, &packet.transactions)).await; + // .expect("timeout sending data to TPU node"); + + debug!("send_txs_to_tpu_static result {:?} - loop over errors", result); } @@ -271,3 +270,4 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R bail!("TPU Quic forward service stopped"); } + diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index 9c85d715..e51acf76 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -9,6 +9,7 @@ use std::time::Duration; use anyhow::{anyhow, bail, Error}; use async_trait::async_trait; use dashmap::DashMap; +use futures::future::join_all; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; use quinn::{Connecting, Connection, ConnectionError, Endpoint, SendStream, ServerConfig, VarInt}; @@ -24,6 +25,7 @@ use tokio::net::ToSocketAddrs; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; use crate::quic_connection_utils::{connection_stats, QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils}; +use crate::quinn_auto_reconnect::AutoReconnect; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); @@ -44,6 +46,12 @@ pub struct TpuQuicClient { } impl TpuQuicClient { + + // note: this is a dirty workaround to expose enpoint to autoconnect class + pub fn get_endpoint(&self) -> Endpoint { + self.endpoint.clone() + } + pub async fn create_connection(&self, tpu_address: SocketAddr) -> anyhow::Result { let connection = // TODO try 0rff @@ -302,11 +310,10 @@ fn serialize_to_vecvec(transactions: &Vec) -> Vec> // send potentially large amount of transactions to a single TPU +#[tracing::instrument(skip_all, level = "debug")] pub async fn send_txs_to_tpu_static( - tpu_connection: Connection, - tpu_address: SocketAddr, + auto_connection: &AutoReconnect, txs: &Vec, - exit_signal: Arc, ) { // note: this impl does not deal with connection errors @@ -316,16 +323,18 @@ pub async fn send_txs_to_tpu_static( // TODO add error handling for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { - let vecvec = chunk.iter().map(|tx| { + let all_send_fns = chunk.iter().map(|tx| { let tx_raw = bincode::serialize(tx).unwrap(); tx_raw - }).collect_vec(); - QuicConnectionUtils::send_transaction_batch_parallel( - tpu_connection.clone(), - vecvec, - exit_signal.clone(), - QUIC_CONNECTION_TIMEOUT, - ).await; + }) + .map(|tx_raw| { + auto_connection.send(tx_raw) // ignores error + }); + + // let all_send_fns = (0..txs.len()).map(|i| auto_connection.roundtrip(vecvec.get(i))).collect_vec(); + + join_all(all_send_fns).await; + } } \ No newline at end of file From 7cae76bb3dd2b00767ef47f41244c91d8660ae22 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 28 Jul 2023 14:47:38 +0200 Subject: [PATCH 049/128] drain queues --- quic-forward-proxy/src/proxy.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 4651c5d9..761f9401 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -233,11 +233,22 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R let packet = receiver.recv().await.unwrap(); assert_eq!(packet.tpu_address, tpu_address, "routing error"); - debug!("forwarding transaction batch of size {} to address {}", packet.transactions.len(), packet.tpu_address); + let mut transactions_batch = packet.transactions; + + let mut batch_size = 1; + while let Ok(more) = receiver.try_recv() { + transactions_batch.extend(more.transactions); + batch_size += 1; + } + if batch_size > 1 { + debug!("encountered batch of size {}", batch_size); + } + + debug!("forwarding transaction batch of size {} to address {}", transactions_batch.len(), packet.tpu_address); // TODo move send_txs_to_tpu_static to tpu_quic_client let result = timeout(Duration::from_millis(500), - send_txs_to_tpu_static(&auto_connection, &packet.transactions)).await; + send_txs_to_tpu_static(&auto_connection, &transactions_batch)).await; // .expect("timeout sending data to TPU node"); debug!("send_txs_to_tpu_static result {:?} - loop over errors", result); @@ -251,8 +262,17 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R } // -- new agent let agent_channel = agents.get(&tpu_address).unwrap(); + agent_channel.send(forward_packet).await.unwrap(); + let mut batch_size = 1; + while let Ok(more) = transaction_channel.try_recv() { + agent_channel.send(more).await.unwrap(); + batch_size += 1; + } + if batch_size > 1 { + debug!("encountered batch of size {}", batch_size); + } // check if the tpu has already a task+queue running, if not start one, sort+queue packets by tpu address From 8cacfe6aa3766d95cc95a79d972864def92fde48 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 28 Jul 2023 14:52:22 +0200 Subject: [PATCH 050/128] add exit signal --- .../src/tpu_utils/quic_proxy_connection_manager.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index e9005273..c398da99 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -90,13 +90,15 @@ impl QuicProxyConnectionManager { let mut transaction_receiver = transaction_sender.subscribe(); - // TODO + // TODO use it + let exit_signal = Arc::new(AtomicBool::new(false)); tokio::spawn(Self::read_transactions_and_broadcast( transaction_receiver, self.current_tpu_nodes.clone(), self.proxy_addr, self.endpoint.clone(), + exit_signal, )); } @@ -148,18 +150,21 @@ impl QuicProxyConnectionManager { current_tpu_nodes: Arc>>, proxy_addr: SocketAddr, endpoint: Endpoint, + exit_signal: Arc, ) { let mut connection = endpoint.connect(proxy_addr, "localhost").unwrap() .await.unwrap(); loop { - // TODO exit signal ??? + // exit signal set + if exit_signal.load(Ordering::Relaxed) { + break; + } tokio::select! { // TODO add timeout tx = transaction_receiver.recv() => { - // exit signal??? let first_tx: Vec = match tx { From d5df190bee7d8e14e8d48366a11a493da7eb81cd Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 28 Jul 2023 14:56:58 +0200 Subject: [PATCH 051/128] add auto reconnect to proxy client --- services/src/tpu_utils/mod.rs | 1 + .../quic_proxy_connection_manager.rs | 19 +-- .../src/tpu_utils/quinn_auto_reconnect.rs | 118 ++++++++++++++++++ 3 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 services/src/tpu_utils/quinn_auto_reconnect.rs diff --git a/services/src/tpu_utils/mod.rs b/services/src/tpu_utils/mod.rs index 437f7a1b..b9ebedd1 100644 --- a/services/src/tpu_utils/mod.rs +++ b/services/src/tpu_utils/mod.rs @@ -3,4 +3,5 @@ pub mod tpu_service; pub mod tpu_connection_path; pub mod tpu_connection_manager; pub mod quic_proxy_connection_manager; +pub mod quinn_auto_reconnect; diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index c398da99..ed4e4570 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -24,6 +24,7 @@ use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; use solana_lite_rpc_core::quic_connection_utils::{connection_stats, QuicConnectionParameters, QuicConnectionUtils, SkipServerVerification}; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; use solana_lite_rpc_core::tx_store::TxStore; +use crate::tpu_utils::quinn_auto_reconnect::AutoReconnect; #[derive(Clone, Copy, Debug)] pub struct TpuNode { @@ -153,8 +154,10 @@ impl QuicProxyConnectionManager { exit_signal: Arc, ) { - let mut connection = endpoint.connect(proxy_addr, "localhost").unwrap() - .await.unwrap(); + // let mut connection = endpoint.connect(proxy_addr, "localhost").unwrap() + // .await.unwrap(); + + let auto_connection = AutoReconnect::new(endpoint, proxy_addr); loop { // exit signal set @@ -203,7 +206,7 @@ impl QuicProxyConnectionManager { for target_tpu_node in tpu_fanout_nodes { Self::send_copy_of_txs_to_quicproxy( - &txs, endpoint.clone(), + &txs, &auto_connection, proxy_addr, target_tpu_node.tpu_address, target_tpu_node.tpu_identity) @@ -215,7 +218,7 @@ impl QuicProxyConnectionManager { } } - async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, endpoint: Endpoint, + async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, auto_connection: &AutoReconnect, proxy_address: SocketAddr, tpu_target_address: SocketAddr, target_tpu_identity: Pubkey) -> anyhow::Result<()> { @@ -244,9 +247,11 @@ impl QuicProxyConnectionManager { let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); - let send_result = - timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)) - .await.context("Timeout sending data to quic proxy")?; + let send_result = auto_connection.send(proxy_request_raw).await; + + // let send_result = + // timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)) + // .await.context("Timeout sending data to quic proxy")?; match send_result { Ok(()) => { diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs new file mode 100644 index 00000000..4d639fff --- /dev/null +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -0,0 +1,118 @@ +use std::cell::RefCell; +use std::fmt; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicU32, Ordering}; +use tracing::{debug, info}; +use quinn::{Connection, Endpoint}; +use tokio::sync::{RwLock, RwLockWriteGuard}; + +pub struct AutoReconnect { + endpoint: Endpoint, + // note: no read lock is used ATM + current: RwLock>, + target_address: SocketAddr, + reconnect_count: AtomicU32, +} + +impl AutoReconnect { + pub fn new(endpoint: Endpoint, target_address: SocketAddr) -> Self { + Self { + endpoint, + current: RwLock::new(None), + target_address, + reconnect_count: AtomicU32::new(0), + } + } + + pub async fn roundtrip(&self, payload: Vec) -> anyhow::Result> { + // TOOD do smart error handling + reconnect + // self.refresh().await.open_bi().await.unwrap() + let (mut send_stream, recv_stream) = self.refresh().await.open_bi().await?; + send_stream.write_all(payload.as_slice()).await?; + send_stream.finish().await?; + + let answer = recv_stream.read_to_end(64 * 1024).await?; + + Ok(answer) + } + + pub async fn send(&self, payload: Vec) -> anyhow::Result<()> { + // TOOD do smart error handling + reconnect + let mut send_stream = self.refresh().await.open_uni().await?; + send_stream.write_all(payload.as_slice()).await?; + send_stream.finish().await?; + + + Ok(()) + } + + + pub async fn refresh(&self) -> Connection { + { + let lock = self.current.read().await; + let maybe_conn: &Option = &*lock; + if maybe_conn.as_ref().filter(|conn| conn.close_reason().is_none()).is_some() { + // let reuse = lock.unwrap().clone(); + let reuse = maybe_conn.as_ref().unwrap(); + debug!("Reuse connection {}", reuse.stable_id()); + return reuse.clone(); + } + } + let mut lock = self.current.write().await; + match &*lock { + Some(current) => { + + if current.close_reason().is_some() { + info!("Connection is closed for reason: {:?}", current.close_reason()); + // TODO log + + let new_connection = self.create_connection().await; + *lock = Some(new_connection.clone()); + // let old_conn = lock.replace(new_connection.clone()); + self.reconnect_count.fetch_add(1, Ordering::SeqCst); + + // debug!("Replace closed connection {} with {} (retry {})", + // old_conn.map(|c| c.stable_id().to_string()).unwrap_or("none".to_string()), + // new_connection.stable_id(), + // self.reconnect_count.load(Ordering::SeqCst)); + // TODO log old vs new stable_id + + + return new_connection.clone(); + } else { + debug!("Reuse connection {} with write-lock", current.stable_id()); + return current.clone(); + } + + } + None => { + let new_connection = self.create_connection().await; + + // let old_conn = lock.replace(new_connection.clone()); + // assert!(old_conn.is_none(), "old connection should be None"); + *lock = Some(new_connection.clone()); + // let old_conn = foo.replace(Some(new_connection.clone())); + // TODO log old vs new stable_id + debug!("Create initial connection {}", new_connection.stable_id()); + + return new_connection.clone(); + } + } + } + + async fn create_connection(&self) -> Connection { + let connection = + self.endpoint.connect(self.target_address, "localhost").expect("handshake"); + + connection.await.expect("connection") + } +} + +impl fmt::Display for AutoReconnect { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Connection to {}", + self.target_address, + ) + } +} + From e8f2befbee7d1ee53b32aadc7a9c090b79793433 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 28 Jul 2023 15:30:55 +0200 Subject: [PATCH 052/128] reduce logging; add parallel streams on client side --- .../tests/quic_proxy_tpu_integrationtest.rs | 31 ++++++++++--- .../quic_proxy_connection_manager.rs | 43 +++++++++++-------- .../src/tpu_utils/quinn_auto_reconnect.rs | 3 +- 3 files changed, 49 insertions(+), 28 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index b521749a..d348ce87 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -34,7 +34,7 @@ use std::time::{Duration, Instant}; use tokio::runtime::{Builder, Runtime}; use tokio::sync::broadcast; use tokio::sync::broadcast::error::SendError; -use tokio::task::JoinHandle; +use tokio::task::{JoinHandle, yield_now}; use tokio::time::{interval, sleep}; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{filter::LevelFilter, fmt}; @@ -126,7 +126,7 @@ pub fn bench_proxy() { wireup_and_send_txs_via_channel(TestCaseParams { // sample_tx_count: 1000, // this is the goal -- ATM test runs too long - sample_tx_count: 200, + sample_tx_count: 1000, stake_connection: true, proxy_mode: true, }); @@ -143,6 +143,17 @@ pub fn with_10000_transactions() { }); } +#[test] +pub fn with_10000_transactions_proxy() { + configure_logging(false); + + wireup_and_send_txs_via_channel(TestCaseParams { + sample_tx_count: 10000, + stake_connection: true, + proxy_mode: true, + }); +} + #[ignore] #[test] pub fn too_many_transactions() { @@ -151,7 +162,7 @@ pub fn too_many_transactions() { wireup_and_send_txs_via_channel(TestCaseParams { sample_tx_count: 100000, stake_connection: false, - proxy_mode: false, + proxy_mode: true, }); } @@ -250,7 +261,7 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { CountMap::with_capacity(test_case_params.sample_tx_count as usize); let warmup_tx_count: u32 = test_case_params.sample_tx_count / 2; while packet_count < test_case_params.sample_tx_count { - if latest_tx.elapsed() > Duration::from_secs(5) { + if latest_tx.elapsed() > Duration::from_secs(25) { warn!("abort after timeout waiting for packet from quic streamer"); break; } @@ -336,7 +347,7 @@ fn configure_logging(verbose: bool) { let env_filter = if verbose { "debug,rustls=info,quinn=info,quinn_proto=debug,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=trace" } else { - "debug,rustls=info,quinn=info,quinn_proto=info,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=debug" + "info,rustls=info,quinn=info,quinn_proto=info,solana_streamer=info,solana_lite_rpc_quic_forward_proxy=info" }; let span_mode = if verbose { FmtSpan::CLOSE @@ -426,6 +437,9 @@ async fn start_literpc_client( for i in 0..test_case_params.sample_tx_count { let raw_sample_tx = build_raw_sample_tx(i); broadcast_sender.send(raw_sample_tx)?; + if (i+1) % 1000 == 0 { + yield_now().await; + } } // we need that to keep the tokio runtime dedicated to lite-rpc up long enough @@ -549,7 +563,7 @@ async fn start_literpc_client_direct_mode( for i in 0..test_case_params.sample_tx_count { let raw_sample_tx = build_raw_sample_tx(i); - debug!( + trace!( "broadcast transaction {} to {} receivers: {}", raw_sample_tx.0, broadcast_sender.receiver_count(), @@ -642,7 +656,7 @@ async fn start_literpc_client_proxy_mode( for i in 0..test_case_params.sample_tx_count { let raw_sample_tx = build_raw_sample_tx(i); - debug!( + trace!( "broadcast transaction {} to {} receivers: {}", raw_sample_tx.0, broadcast_sender.receiver_count(), @@ -650,6 +664,9 @@ async fn start_literpc_client_proxy_mode( ); broadcast_sender.send(raw_sample_tx)?; + if (i+1) % 1000 == 0 { + yield_now().await; + } } sleep(Duration::from_secs(30)).await; diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index ed4e4570..ae8b9194 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -10,7 +10,7 @@ use anyhow::{bail, Context}; use async_trait::async_trait; use futures::FutureExt; use itertools::Itertools; -use log::{debug, error, info, warn}; +use log::{debug, error, info, trace, warn}; use quinn::{ClientConfig, Connection, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt}; use solana_sdk::packet::PACKET_DATA_SIZE; use solana_sdk::pubkey::Pubkey; @@ -40,6 +40,8 @@ pub struct QuicProxyConnectionManager { current_tpu_nodes: Arc>> } +const PARALLEL_STREAMS_TO_PROXY: usize = 4; + impl QuicProxyConnectionManager { pub async fn new( certificate: rustls::Certificate, @@ -201,7 +203,7 @@ impl QuicProxyConnectionManager { let tpu_fanout_nodes = current_tpu_nodes.read().await.clone(); - info!("Sending copy of transaction batch of {} txs to {} tpu nodes via quic proxy", + trace!("Sending copy of transaction batch of {} txs to {} tpu nodes via quic proxy", txs.len(), tpu_fanout_nodes.len()); for target_tpu_node in tpu_fanout_nodes { @@ -222,9 +224,6 @@ impl QuicProxyConnectionManager { proxy_address: SocketAddr, tpu_target_address: SocketAddr, target_tpu_identity: Pubkey) -> anyhow::Result<()> { - info!("sending vecvec {} to quic proxy for TPU node {}", - raw_tx_batch.iter().map(|tx| tx.len()).into_iter().join(","), tpu_target_address); - // TODO add timeout // let mut send_stream = timeout(Duration::from_millis(500), connection.open_uni()).await??; @@ -242,25 +241,31 @@ impl QuicProxyConnectionManager { txs.push(tx); } - let forwarding_request = TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, txs); - debug!("forwarding_request: {}", forwarding_request); - let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); + for chunk in txs.chunks(PARALLEL_STREAMS_TO_PROXY) { - let send_result = auto_connection.send(proxy_request_raw).await; + let forwarding_request = TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, chunk.into()); + debug!("forwarding_request: {}", forwarding_request); - // let send_result = - // timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)) - // .await.context("Timeout sending data to quic proxy")?; + let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); - match send_result { - Ok(()) => { - info!("Successfully sent data to quic proxy"); - } - Err(e) => { - bail!("Failed to send data to quic proxy: {:?}", e); + let send_result = auto_connection.send(proxy_request_raw).await; + + // let send_result = + // timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)) + // .await.context("Timeout sending data to quic proxy")?; + + match send_result { + Ok(()) => { + debug!("Successfully sent {} txs to quic proxy", txs.len()); + } + Err(e) => { + bail!("Failed to send data to quic proxy: {:?}", e); + } } - } + + } // -- one chunk + Ok(()) } diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 4d639fff..56ab85c9 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -40,8 +40,7 @@ impl AutoReconnect { // TOOD do smart error handling + reconnect let mut send_stream = self.refresh().await.open_uni().await?; send_stream.write_all(payload.as_slice()).await?; - send_stream.finish().await?; - + let _ = send_stream.finish().await; Ok(()) } From 385d94bc025bd4dacfda3c35f416a078653481d5 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 28 Jul 2023 22:23:57 +0200 Subject: [PATCH 053/128] use fanout --- quic-forward-proxy/Cargo.toml | 2 + quic-forward-proxy/src/proxy.rs | 84 +++++++++++-------- quic-forward-proxy/src/tpu_quic_client.rs | 2 +- .../quic_proxy_connection_manager.rs | 5 +- .../src/tpu_utils/quinn_auto_reconnect.rs | 29 +++---- 5 files changed, 67 insertions(+), 55 deletions(-) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 18e5a264..9dc99be8 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -41,4 +41,6 @@ chrono = { workspace = true } tokio = { version = "1.28.2", features = ["full", "fs"]} rcgen = "0.9.3" spl-memo = "3.0.1" +# tokio channel fanout +fan = "0.1.3" diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 761f9401..5d2e5798 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -10,6 +10,8 @@ use std::time::Duration; use tracing::{debug_span, instrument, Instrument, span}; use anyhow::{anyhow, bail, Context, Error}; use dashmap::DashMap; +use fan::tokio::mpsc::FanOut; +use futures::sink::Fanout; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; use quinn::{Connecting, Connection, ConnectionError, Endpoint, SendStream, ServerConfig, TransportConfig, VarInt}; @@ -201,7 +203,7 @@ async fn accept_client_connection(client_connection: Connection, forwarder_chann async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: Receiver, exit_signal: Arc) -> anyhow::Result<()> { info!("TPU Quic forwarder started"); - let mut agents: HashMap> = HashMap::new(); + let mut agents: HashMap> = HashMap::new(); let tpu_quic_client_copy = tpu_quic_client.clone(); loop { @@ -211,53 +213,65 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R let tpu_address = forward_packet.tpu_address; if !agents.contains_key(&tpu_address) { - let (sender, mut receiver) = channel::(10000); // TODO cleanup agent after a while of iactivity - agents.insert(tpu_address, sender); - let tpu_quic_client_copy = tpu_quic_client.clone(); - let exit_signal = exit_signal.clone(); - tokio::spawn(async move { - debug!("Start Quic forwarder agent for TPU {}", tpu_address); - // TODO pass+check the tpu_address - // TODO connect - // TODO consume queue - // TODO exit signal - - let auto_connection = AutoReconnect::new(tpu_quic_client_copy.get_endpoint(), tpu_address); - // let mut connection = tpu_quic_client_copy.create_connection(tpu_address).await.expect("handshake"); - loop { - - let exit_signal = exit_signal.clone(); + let mut senders = Vec::new(); + for i in 0..4 { + let (sender, mut receiver) = channel::(100000); + senders.push(sender); + let endpoint = tpu_quic_client.get_endpoint().clone(); + let exit_signal = exit_signal.clone(); + tokio::spawn(async move { + debug!("Start Quic forwarder agent for TPU {}", tpu_address); + // TODO pass+check the tpu_address + // TODO connect + // TODO consume queue + // TODO exit signal + + let auto_connection = AutoReconnect::new(endpoint, tpu_address); + // let mut connection = tpu_quic_client_copy.create_connection(tpu_address).await.expect("handshake"); loop { - let packet = receiver.recv().await.unwrap(); - assert_eq!(packet.tpu_address, tpu_address, "routing error"); - let mut transactions_batch = packet.transactions; + let exit_signal = exit_signal.clone(); + loop { + let packet = receiver.recv().await.unwrap(); + assert_eq!(packet.tpu_address, tpu_address, "routing error"); - let mut batch_size = 1; - while let Ok(more) = receiver.try_recv() { - transactions_batch.extend(more.transactions); - batch_size += 1; - } - if batch_size > 1 { - debug!("encountered batch of size {}", batch_size); - } + let mut transactions_batch = packet.transactions; + + let mut batch_size = 1; + while let Ok(more) = receiver.try_recv() { + transactions_batch.extend(more.transactions); + batch_size += 1; + } + if batch_size > 1 { + debug!("encountered batch of size {}", batch_size); + } - debug!("forwarding transaction batch of size {} to address {}", transactions_batch.len(), packet.tpu_address); + debug!("forwarding transaction batch of size {} to address {}", transactions_batch.len(), packet.tpu_address); - // TODo move send_txs_to_tpu_static to tpu_quic_client - let result = timeout(Duration::from_millis(500), - send_txs_to_tpu_static(&auto_connection, &transactions_batch)).await; + // TODo move send_txs_to_tpu_static to tpu_quic_client + let result = timeout(Duration::from_millis(500), + send_txs_to_tpu_static(&auto_connection, &transactions_batch)).await; // .expect("timeout sending data to TPU node"); - debug!("send_txs_to_tpu_static result {:?} - loop over errors", result); + if result.is_err() { + warn!("send_txs_to_tpu_static result {:?} - loop over errors", result); + } else { + debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); + } + + } } - } + }); - }); + } + + let fanout = FanOut::new(senders); + + agents.insert(tpu_address, fanout); } // -- new agent diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index e51acf76..a7032f63 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -151,7 +151,7 @@ impl TpuQuicClient { /// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU // note: ATM the provided identity might or might not be a valid validator keypair pub async fn new_with_validator_identity(validator_identity: &Keypair) -> TpuQuicClient { - info!("Setup TPU Quic stable connection ..."); + info!("Setup TPU Quic stable connection with validator identity {} ...", bs58::encode(validator_identity.pubkey()).into_string()); let (certificate, key) = new_self_signed_tls_certificate( validator_identity, IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index ae8b9194..676eb1e0 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -40,7 +40,8 @@ pub struct QuicProxyConnectionManager { current_tpu_nodes: Arc>> } -const PARALLEL_STREAMS_TO_PROXY: usize = 4; +// TODO consolidate with number_of_transactions_per_unistream +const CHUNK_SIZE_PER_STREAM: usize = 20; impl QuicProxyConnectionManager { pub async fn new( @@ -242,7 +243,7 @@ impl QuicProxyConnectionManager { } - for chunk in txs.chunks(PARALLEL_STREAMS_TO_PROXY) { + for chunk in txs.chunks(CHUNK_SIZE_PER_STREAM) { let forwarding_request = TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, chunk.into()); debug!("forwarding_request: {}", forwarding_request); diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 56ab85c9..bce87d1f 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -2,6 +2,7 @@ use std::cell::RefCell; use std::fmt; use std::net::SocketAddr; use std::sync::atomic::{AtomicU32, Ordering}; +use log::{trace, warn}; use tracing::{debug, info}; use quinn::{Connection, Endpoint}; use tokio::sync::{RwLock, RwLockWriteGuard}; @@ -53,46 +54,40 @@ impl AutoReconnect { if maybe_conn.as_ref().filter(|conn| conn.close_reason().is_none()).is_some() { // let reuse = lock.unwrap().clone(); let reuse = maybe_conn.as_ref().unwrap(); - debug!("Reuse connection {}", reuse.stable_id()); + trace!("Reuse connection {}", reuse.stable_id()); return reuse.clone(); } } let mut lock = self.current.write().await; + match &*lock { Some(current) => { if current.close_reason().is_some() { - info!("Connection is closed for reason: {:?}", current.close_reason()); - // TODO log - + warn!("Connection i s closed for reason: {:?}", current.close_reason()); let new_connection = self.create_connection().await; + let prev_stable_id = current.stable_id(); *lock = Some(new_connection.clone()); // let old_conn = lock.replace(new_connection.clone()); self.reconnect_count.fetch_add(1, Ordering::SeqCst); - - // debug!("Replace closed connection {} with {} (retry {})", - // old_conn.map(|c| c.stable_id().to_string()).unwrap_or("none".to_string()), - // new_connection.stable_id(), - // self.reconnect_count.load(Ordering::SeqCst)); + debug!("Replace closed connection {} with {} (retry {})", + prev_stable_id, + new_connection.stable_id(), + self.reconnect_count.load(Ordering::SeqCst)); // TODO log old vs new stable_id - return new_connection.clone(); } else { - debug!("Reuse connection {} with write-lock", current.stable_id()); + // TODO check log if that ever happens + warn!("Reuse connection {} with write-lock", current.stable_id()); return current.clone(); } } None => { let new_connection = self.create_connection().await; - - // let old_conn = lock.replace(new_connection.clone()); - // assert!(old_conn.is_none(), "old connection should be None"); *lock = Some(new_connection.clone()); - // let old_conn = foo.replace(Some(new_connection.clone())); - // TODO log old vs new stable_id - debug!("Create initial connection {}", new_connection.stable_id()); + trace!("Create initial connection {}", new_connection.stable_id()); return new_connection.clone(); } From 816f978ee6cdf9e95b1be9ad2c0705c8862d178f Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 28 Jul 2023 23:04:53 +0200 Subject: [PATCH 054/128] make validator id optional from cli --- .../tests/quic_proxy_tpu_integrationtest.rs | 9 +++++---- quic-forward-proxy/src/cli.rs | 10 +++++----- quic-forward-proxy/src/lib.rs | 1 + quic-forward-proxy/src/main.rs | 6 ++++-- quic-forward-proxy/src/proxy.rs | 20 ++++++++----------- quic-forward-proxy/src/tpu_quic_client.rs | 9 +++++---- .../quic_proxy_connection_manager.rs | 1 + 7 files changed, 29 insertions(+), 27 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index d348ce87..cff864d6 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -41,6 +41,7 @@ use tracing_subscriber::{filter::LevelFilter, fmt}; use tracing_subscriber::fmt::format::FmtSpan; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_config_provicer::SelfSignedTlsConfigProvider; +use solana_lite_rpc_quic_forward_proxy::ValidatorIdentity; use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; #[derive(Copy, Clone, Debug)] @@ -366,7 +367,7 @@ fn configure_logging(verbose: bool) { async fn start_literpc_client( test_case_params: TestCaseParams, streamer_listen_addrs: SocketAddr, - literpc_validator_identity: Arc, + literpc_validator_identity: ValidatorIdentity, ) -> anyhow::Result<()> { info!("Start lite-rpc test client ..."); @@ -376,7 +377,7 @@ async fn start_literpc_client( let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); let broadcast_sender = Arc::new(sender); let (certificate, key) = new_self_signed_tls_certificate( - literpc_validator_identity.as_ref(), + literpc_validator_identity.get_keypair_for_tls().as_ref(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) .expect("Failed to initialize QUIC connection certificates"); @@ -405,7 +406,7 @@ async fn start_literpc_client( ); // this is the real streamer - connections_to_keep.insert(literpc_validator_identity.pubkey(), streamer_listen_addrs); + connections_to_keep.insert(literpc_validator_identity.get_pubkey(), streamer_listen_addrs); // get information about the optional validator identity stake // populated from get_stakes_for_identity() @@ -677,7 +678,7 @@ async fn start_literpc_client_proxy_mode( async fn start_quic_proxy(proxy_listen_addr: SocketAddr) -> anyhow::Result<()> { let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); - let random_unstaked_validator_identity = Arc::new(Keypair::new()); + let random_unstaked_validator_identity = ValidatorIdentity::new(None); let tls_config = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let proxy_service = QuicForwardProxy::new(proxy_listen_addr, &tls_config, random_unstaked_validator_identity) diff --git a/quic-forward-proxy/src/cli.rs b/quic-forward-proxy/src/cli.rs index 7042f291..78775b17 100644 --- a/quic-forward-proxy/src/cli.rs +++ b/quic-forward-proxy/src/cli.rs @@ -10,25 +10,25 @@ pub struct Args { } // note this is duplicated from lite-rpc module -pub async fn get_identity_keypair(identity_from_cli: &String) -> Keypair { +pub async fn get_identity_keypair(identity_from_cli: &String) -> Option { if let Ok(identity_env_var) = env::var("IDENTITY") { if let Ok(identity_bytes) = serde_json::from_str::>(identity_env_var.as_str()) { - Keypair::from_bytes(identity_bytes.as_slice()).unwrap() + Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap()) } else { // must be a file let identity_file = tokio::fs::read_to_string(identity_env_var.as_str()) .await .expect("Cannot find the identity file provided"); let identity_bytes: Vec = serde_json::from_str(&identity_file).unwrap(); - Keypair::from_bytes(identity_bytes.as_slice()).unwrap() + Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap()) } } else if identity_from_cli.is_empty() { - Keypair::new() + None } else { let identity_file = tokio::fs::read_to_string(identity_from_cli.as_str()) .await .expect("Cannot find the identity file provided"); let identity_bytes: Vec = serde_json::from_str(&identity_file).unwrap(); - Keypair::from_bytes(identity_bytes.as_slice()).unwrap() + Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap()) } } diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index 7bda23d9..83e58fe1 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -10,3 +10,4 @@ mod tx_store; mod identity_stakes; mod quic_connection_utils; mod quinn_auto_reconnect; +mod validator_identity; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 04ed9930..af679826 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -8,6 +8,7 @@ use crate::cli::{Args, get_identity_keypair}; use crate::proxy::QuicForwardProxy; use crate::test_client::quic_test_client::QuicTestClient; pub use tls_config_provicer::SelfSignedTlsConfigProvider; +use crate::validator_identity::ValidatorIdentity; pub mod quic_util; @@ -22,6 +23,7 @@ mod tx_store; mod identity_stakes; mod quic_connection_utils; mod quinn_auto_reconnect; +mod validator_identity; #[tokio::main(flavor = "multi_thread", worker_threads = 16)] @@ -37,8 +39,8 @@ pub async fn main() -> anyhow::Result<()> { // TODO build args struct dedicyted to proxy let proxy_listener_addr = "127.0.0.1:11111".parse().unwrap(); let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); - let validator_identity = Arc::new(get_identity_keypair(&identity_keypair).await); - + let validator_identity = + ValidatorIdentity::new(get_identity_keypair(&identity_keypair).await); let tls_config = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let main_services = QuicForwardProxy::new(proxy_listener_addr, &tls_config, validator_identity) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 5d2e5798..0a28276a 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -39,6 +39,7 @@ use crate::quinn_auto_reconnect::AutoReconnect; use crate::tpu_quic_client::{send_txs_to_tpu_static, SingleTPUConnectionManager, TpuQuicClient}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::AnyhowJoinHandle; +use crate::validator_identity::ValidatorIdentity; // TODO tweak this value - solana server sets 256 // setting this to "1" did not make a difference! @@ -46,7 +47,7 @@ const MAX_CONCURRENT_UNI_STREAMS: u32 = 24; pub struct QuicForwardProxy { endpoint: Endpoint, - validator_identity: Arc, + // validator_identity: ValidatorIdentity, tpu_quic_client: TpuQuicClient, } @@ -61,7 +62,7 @@ impl QuicForwardProxy { pub async fn new( proxy_listener_addr: SocketAddr, tls_config: &SelfSignedTlsConfigProvider, - validator_identity: Arc) -> anyhow::Result { + validator_identity: ValidatorIdentity) -> anyhow::Result { let server_tls_config = tls_config.get_server_tls_crypto_config(); let mut quinn_server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); @@ -78,12 +79,12 @@ impl QuicForwardProxy { transport_config.receive_window((PACKET_DATA_SIZE as u32 * MAX_CONCURRENT_UNI_STREAMS).into()); let endpoint = Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap(); - info!("Quic proxy uses validator identity {}", validator_identity.pubkey()); + info!("Quic proxy uses validator identity {}", validator_identity); let tpu_quic_client = - TpuQuicClient::new_with_validator_identity(validator_identity.as_ref()).await; + TpuQuicClient::new_with_validator_identity(validator_identity).await; - Ok(Self { endpoint, validator_identity, tpu_quic_client }) + Ok(Self { endpoint, tpu_quic_client }) } @@ -115,13 +116,12 @@ impl QuicForwardProxy { while let Some(connecting) = endpoint.accept().await { let exit_signal = exit_signal.clone(); - let validator_identity_copy = self.validator_identity.clone(); let tpu_quic_client = self.tpu_quic_client.clone(); let forwarder_channel_copy = forwarder_channel.clone(); tokio::spawn(async move { let connection = connecting.await.context("handshake").unwrap(); match accept_client_connection(connection, forwarder_channel_copy, - tpu_quic_client, exit_signal, validator_identity_copy) + tpu_quic_client, exit_signal) .await { Ok(()) => {} Err(err) => { @@ -140,12 +140,9 @@ impl QuicForwardProxy { #[tracing::instrument(skip_all, level = "debug")] async fn accept_client_connection(client_connection: Connection, forwarder_channel: Sender, tpu_quic_client: TpuQuicClient, - exit_signal: Arc, validator_identity: Arc) -> anyhow::Result<()> { + exit_signal: Arc) -> anyhow::Result<()> { debug!("inbound connection established, client {}", client_connection.remote_address()); - // let active_tpu_connection = - // TpuQuicClient::new_with_validator_identity(validator_identity.as_ref()).await; - loop { let maybe_stream = client_connection.accept_uni().await; let result = match maybe_stream { @@ -163,7 +160,6 @@ async fn accept_client_connection(client_connection: Connection, forwarder_chann } Ok(recv_stream) => { let exit_signal_copy = exit_signal.clone(); - let validator_identity_copy = validator_identity.clone(); let tpu_quic_client_copy = tpu_quic_client.clone(); let forwarder_channel_copy = forwarder_channel.clone(); diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index a7032f63..ff54981c 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -13,7 +13,7 @@ use futures::future::join_all; use itertools::{any, Itertools}; use log::{debug, error, info, trace, warn}; use quinn::{Connecting, Connection, ConnectionError, Endpoint, SendStream, ServerConfig, VarInt}; -use rcgen::generate_simple_self_signed; +use rcgen::{generate_simple_self_signed, KeyPair}; use rustls::{Certificate, PrivateKey}; use rustls::server::ResolvesServerCert; use serde::{Deserialize, Serialize}; @@ -27,6 +27,7 @@ use tokio::sync::RwLock; use crate::quic_connection_utils::{connection_stats, QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils}; use crate::quinn_auto_reconnect::AutoReconnect; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; +use crate::validator_identity::ValidatorIdentity; const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); pub const CONNECTION_RETRY_COUNT: usize = 10; @@ -150,10 +151,10 @@ impl TpuQuicClient { /// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU // note: ATM the provided identity might or might not be a valid validator keypair - pub async fn new_with_validator_identity(validator_identity: &Keypair) -> TpuQuicClient { - info!("Setup TPU Quic stable connection with validator identity {} ...", bs58::encode(validator_identity.pubkey()).into_string()); + pub async fn new_with_validator_identity(validator_identity: ValidatorIdentity) -> TpuQuicClient { + info!("Setup TPU Quic stable connection with validator identity {} ...", validator_identity); let (certificate, key) = new_self_signed_tls_certificate( - validator_identity, + &validator_identity.get_keypair_for_tls(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) .expect("Failed to initialize QUIC connection certificates"); diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 676eb1e0..41940985 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -34,6 +34,7 @@ pub struct TpuNode { pub struct QuicProxyConnectionManager { endpoint: Endpoint, + // TODO remove validator_identity: Arc, simple_thread_started: AtomicBool, proxy_addr: SocketAddr, From c13f0f5e7c64f2b9eafaa3a67b81ef94d379d604 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 28 Jul 2023 23:06:41 +0200 Subject: [PATCH 055/128] add validator identity --- quic-forward-proxy/src/validator_identity.rs | 45 ++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 quic-forward-proxy/src/validator_identity.rs diff --git a/quic-forward-proxy/src/validator_identity.rs b/quic-forward-proxy/src/validator_identity.rs new file mode 100644 index 00000000..196864c7 --- /dev/null +++ b/quic-forward-proxy/src/validator_identity.rs @@ -0,0 +1,45 @@ +use std::fmt::Display; +use std::sync::Arc; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::Keypair; +use solana_sdk::signer::Signer; + +#[derive(Clone)] +pub struct ValidatorIdentity { + keypair: Option>, + dummy_keypair: Arc, +} + +impl ValidatorIdentity { + pub fn new(keypair: Option) -> Self { + let dummy_keypair = Keypair::new(); + ValidatorIdentity { + keypair: keypair.map(|kp| Arc::new(kp)), + dummy_keypair: Arc::new(dummy_keypair), + } + } + + pub fn get_keypair_for_tls(&self) -> Arc { + match &self.keypair { + Some(keypair) => keypair.clone(), + None => self.dummy_keypair.clone(), + } + } + + pub fn get_pubkey(&self) -> Pubkey { + let keypair = match &self.keypair { + Some(keypair) => keypair.clone(), + None => self.dummy_keypair.clone(), + }; + keypair.pubkey() + } +} + +impl Display for ValidatorIdentity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.keypair { + Some(keypair) => write!(f, "{}", keypair.pubkey().to_string()), + None => write!(f, "no keypair"), + } + } +} From 88b6935319fb71b1d56160892954d04468cfeee6 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Sat, 29 Jul 2023 00:06:02 +0200 Subject: [PATCH 056/128] fix wrong agent-tpu assignment --- quic-forward-proxy/src/proxy.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 0a28276a..5def09dd 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -201,7 +201,7 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R let mut agents: HashMap> = HashMap::new(); - let tpu_quic_client_copy = tpu_quic_client.clone(); + let endpoint = tpu_quic_client.get_endpoint().clone(); loop { // TODO add exit @@ -215,8 +215,8 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R for i in 0..4 { let (sender, mut receiver) = channel::(100000); senders.push(sender); - let endpoint = tpu_quic_client.get_endpoint().clone(); let exit_signal = exit_signal.clone(); + let endpoint_copy = endpoint.clone(); tokio::spawn(async move { debug!("Start Quic forwarder agent for TPU {}", tpu_address); // TODO pass+check the tpu_address @@ -224,7 +224,7 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R // TODO consume queue // TODO exit signal - let auto_connection = AutoReconnect::new(endpoint, tpu_address); + let auto_connection = AutoReconnect::new(endpoint_copy, tpu_address); // let mut connection = tpu_quic_client_copy.create_connection(tpu_address).await.expect("handshake"); loop { @@ -275,14 +275,14 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R agent_channel.send(forward_packet).await.unwrap(); - let mut batch_size = 1; - while let Ok(more) = transaction_channel.try_recv() { - agent_channel.send(more).await.unwrap(); - batch_size += 1; - } - if batch_size > 1 { - debug!("encountered batch of size {}", batch_size); - } + // let mut batch_size = 1; + // while let Ok(more) = transaction_channel.try_recv() { + // agent_channel.send(more).await.unwrap(); + // batch_size += 1; + // } + // if batch_size > 1 { + // debug!("encountered batch of size {}", batch_size); + // } // check if the tpu has already a task+queue running, if not start one, sort+queue packets by tpu address @@ -297,7 +297,7 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R } - bail!("TPU Quic forward service stopped"); + panic!("not reachable"); } From 1d4e626dc876666a5aa17d49fd227a19f1cbaec7 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Sat, 29 Jul 2023 00:14:49 +0200 Subject: [PATCH 057/128] make proxy address configurable --- quic-forward-proxy/README.md | 4 ++-- quic-forward-proxy/src/cli.rs | 2 ++ quic-forward-proxy/src/main.rs | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/quic-forward-proxy/README.md b/quic-forward-proxy/README.md index 262f024e..d2f4df2a 100644 --- a/quic-forward-proxy/README.md +++ b/quic-forward-proxy/README.md @@ -24,11 +24,11 @@ RUST_LOG="error,solana_streamer::nonblocking::quic=debug" solana-test-validator ``` 3. run quic proxy ```bash -RUST_LOG=debug cargo run --bin solana-lite-rpc-quic-forward-proxy -- --identity-keypair /pathto-test-ledger/validator-keypair.json +RUST_LOG=debug cargo run --bin solana-lite-rpc-quic-forward-proxy -- --proxy-rpc-addr 0.0.0.0:11111 --identity-keypair /pathto-test-ledger/validator-keypair.json ``` 2. run lite-rpc ```bash -RUST_LOG=debug cargo run --bin lite-rpc +RUST_LOG=debug cargo run --bin lite-rpc -- --experimental-quic-proxy-addr 127.0.0.1:11111 ``` 3. run rust bench tool in _lite-rpc_ ```bash diff --git a/quic-forward-proxy/src/cli.rs b/quic-forward-proxy/src/cli.rs index 78775b17..fafdfed0 100644 --- a/quic-forward-proxy/src/cli.rs +++ b/quic-forward-proxy/src/cli.rs @@ -7,6 +7,8 @@ use solana_sdk::signature::Keypair; pub struct Args { #[arg(short = 'k', long, default_value_t = String::new())] pub identity_keypair: String, + #[arg(short = 'l', long)] + pub proxy_rpc_addr: String, } // note this is duplicated from lite-rpc module diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index af679826..bf22d146 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -32,12 +32,13 @@ pub async fn main() -> anyhow::Result<()> { let Args { identity_keypair, + proxy_rpc_addr, } = Args::parse(); dotenv().ok(); // TODO build args struct dedicyted to proxy - let proxy_listener_addr = "127.0.0.1:11111".parse().unwrap(); + let proxy_listener_addr = proxy_rpc_addr.parse().unwrap(); let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let validator_identity = ValidatorIdentity::new(get_identity_keypair(&identity_keypair).await); From 87b8c077d09a48dd5e09ed136cbc883863577aff Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Sat, 29 Jul 2023 11:18:20 +0200 Subject: [PATCH 058/128] cargo.lock --- Cargo.lock | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 523c0e7b..23d512f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1406,6 +1406,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fan" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73353b3e9d86ba3d1f2cb930cc04e951ffce0a011877dea8db3f4f9c7ced502f" +dependencies = [ + "futures", + "tokio", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -4054,6 +4064,8 @@ dependencies = [ "clap 4.2.4", "dashmap", "dotenv", + "fan", + "futures", "itertools", "lazy_static", "log", @@ -4071,6 +4083,7 @@ dependencies = [ "spl-memo", "thiserror", "tokio", + "tracing", "tracing-subscriber", ] @@ -4103,6 +4116,7 @@ dependencies = [ "serde", "serde_json", "solana-lite-rpc-core", + "solana-lite-rpc-quic-forward-proxy", "solana-lite-rpc-services", "solana-net-utils", "solana-sdk", @@ -4120,6 +4134,7 @@ version = "0.2.2" dependencies = [ "anyhow", "async-channel", + "async-trait", "base64 0.21.0", "bincode", "bs58", From 94686f31b63d4180a4e655ce4eaf42ef80dbeb3d Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Sat, 29 Jul 2023 11:24:10 +0200 Subject: [PATCH 059/128] fix compile ValidatorIdentity --- quic-forward-proxy/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index 83e58fe1..66a086c1 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -5,9 +5,9 @@ pub mod proxy_request_format; pub mod tpu_quic_client; pub mod cli; pub mod test_client; +pub mod validator_identity; mod util; mod tx_store; mod identity_stakes; mod quic_connection_utils; mod quinn_auto_reconnect; -mod validator_identity; From a27b146ec96222eef3dc4d3c0aa918642e5c1d5b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Sat, 29 Jul 2023 11:24:34 +0200 Subject: [PATCH 060/128] fix compile ValidatorIdentity --- .../tests/quic_proxy_tpu_integrationtest.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index cff864d6..5bd49847 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -41,7 +41,7 @@ use tracing_subscriber::{filter::LevelFilter, fmt}; use tracing_subscriber::fmt::format::FmtSpan; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_config_provicer::SelfSignedTlsConfigProvider; -use solana_lite_rpc_quic_forward_proxy::ValidatorIdentity; +use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; #[derive(Copy, Clone, Debug)] @@ -367,7 +367,7 @@ fn configure_logging(verbose: bool) { async fn start_literpc_client( test_case_params: TestCaseParams, streamer_listen_addrs: SocketAddr, - literpc_validator_identity: ValidatorIdentity, + literpc_validator_identity: Arc, ) -> anyhow::Result<()> { info!("Start lite-rpc test client ..."); @@ -377,7 +377,7 @@ async fn start_literpc_client( let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); let broadcast_sender = Arc::new(sender); let (certificate, key) = new_self_signed_tls_certificate( - literpc_validator_identity.get_keypair_for_tls().as_ref(), + literpc_validator_identity.as_ref(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) .expect("Failed to initialize QUIC connection certificates"); @@ -406,7 +406,7 @@ async fn start_literpc_client( ); // this is the real streamer - connections_to_keep.insert(literpc_validator_identity.get_pubkey(), streamer_listen_addrs); + connections_to_keep.insert(literpc_validator_identity.pubkey(), streamer_listen_addrs); // get information about the optional validator identity stake // populated from get_stakes_for_identity() From 20c27102024cacf183e12d0ef83cad88ff7778e2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Sat, 29 Jul 2023 11:26:21 +0200 Subject: [PATCH 061/128] cargo fix --- core/src/proxy_request_format.rs | 2 +- core/src/solana_utils.rs | 2 +- lite-rpc/src/bridge.rs | 2 +- .../tests/quic_proxy_tpu_integrationtest.rs | 40 +++++------ quic-forward-proxy/src/main.rs | 8 +-- quic-forward-proxy/src/proxy.rs | 72 +++++++++---------- .../src/proxy_request_format.rs | 2 +- .../src/quic_connection_utils.rs | 21 +++--- .../src/quinn_auto_reconnect.rs | 4 +- .../src/test_client/quic_test_client.rs | 10 +-- .../src/test_client/sample_data_factory.rs | 2 +- quic-forward-proxy/src/tpu_quic_client.rs | 36 +++++----- .../tests/proxy_request_format.rs | 16 ++--- .../quic_proxy_connection_manager.rs | 38 +++++----- .../src/tpu_utils/quinn_auto_reconnect.rs | 6 +- .../src/tpu_utils/tpu_connection_manager.rs | 6 +- services/src/tpu_utils/tpu_connection_path.rs | 16 ++--- ...literpc_tpu_quic_server_integrationtest.rs | 34 ++++----- 18 files changed, 158 insertions(+), 159 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index cfc7d5ef..a03a8e49 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -1,6 +1,6 @@ use std::fmt; use std::fmt::Display; -use std::net::{SocketAddr, SocketAddrV4}; +use std::net::{SocketAddr}; use anyhow::Context; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; diff --git a/core/src/solana_utils.rs b/core/src/solana_utils.rs index 2221988b..8b9a5b27 100644 --- a/core/src/solana_utils.rs +++ b/core/src/solana_utils.rs @@ -14,7 +14,7 @@ use std::{ }; use serde::Serialize; use solana_sdk::hash::Hash; -use solana_sdk::message::{LegacyMessage, v0}; + use solana_sdk::signature::Signature; use solana_sdk::transaction::{Transaction, uses_durable_nonce, VersionedTransaction}; use tokio::sync::mpsc::UnboundedReceiver; diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index d2434eda..fa3c4ba2 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -38,7 +38,7 @@ use solana_sdk::{ }; use solana_transaction_status::TransactionStatus; use std::{ops::Deref, str::FromStr, sync::Arc, time::Duration}; -use std::convert::identity; + use tokio::{ net::ToSocketAddrs, sync::mpsc::{self, Sender}, diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 5bd49847..7cd7deca 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -1,9 +1,9 @@ -use anyhow::bail; + use countmap::CountMap; -use crossbeam_channel::{Receiver, RecvError, RecvTimeoutError, Sender}; -use futures::future::join_all; -use log::{debug, error, info, trace, warn}; -use quinn::TokioRuntime; +use crossbeam_channel::{Sender}; + +use log::{debug, info, trace, warn}; + use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; use solana_lite_rpc_core::tx_store::empty_tx_store; @@ -14,7 +14,7 @@ use solana_sdk::instruction::Instruction; use solana_sdk::message::Message; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, Signature, Signer}; -use solana_sdk::signer::keypair; + use solana_sdk::transaction::{Transaction, VersionedTransaction}; use solana_streamer::nonblocking::quic::ConnectionPeerType; use solana_streamer::packet::PacketBatch; @@ -23,21 +23,21 @@ use solana_streamer::streamer::StakedNodes; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; -use std::ops::Deref; -use std::option::Option; -use std::path::Path; + + + use std::str::FromStr; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; use std::thread; use std::time::{Duration, Instant}; -use tokio::runtime::{Builder, Runtime}; -use tokio::sync::broadcast; -use tokio::sync::broadcast::error::SendError; +use tokio::runtime::{Builder}; + + use tokio::task::{JoinHandle, yield_now}; -use tokio::time::{interval, sleep}; +use tokio::time::{sleep}; use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::{filter::LevelFilter, fmt}; + use tracing_subscriber::fmt::format::FmtSpan; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_config_provicer::SelfSignedTlsConfigProvider; @@ -480,7 +480,7 @@ async fn solana_quic_streamer_start() { let addr = sock.local_addr().unwrap().ip(); let port = sock.local_addr().unwrap().port(); - let tpu_addr = SocketAddr::new(addr, port); + let _tpu_addr = SocketAddr::new(addr, port); // sleep(Duration::from_millis(500)).await; @@ -587,7 +587,7 @@ async fn start_literpc_client_proxy_mode( ) -> anyhow::Result<()> { info!("Start lite-rpc test client using quic proxy at {} ...", forward_proxy_address); - let fanout_slots = 4; + let _fanout_slots = 4; // (String, Vec) (signature, transaction) let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); @@ -629,7 +629,7 @@ async fn start_literpc_client_proxy_mode( // get information about the optional validator identity stake // populated from get_stakes_for_identity() - let identity_stakes = IdentityStakes { + let _identity_stakes = IdentityStakes { peer_type: ConnectionPeerType::Staked, stakes: if test_case_params.stake_connection { 30 } else { 0 }, // stake of lite-rpc min_stakes: 0, @@ -677,7 +677,7 @@ async fn start_literpc_client_proxy_mode( async fn start_quic_proxy(proxy_listen_addr: SocketAddr) -> anyhow::Result<()> { - let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); + let _tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let random_unstaked_validator_identity = ValidatorIdentity::new(None); let tls_config = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); @@ -744,7 +744,7 @@ impl SolanaQuicStreamer { let addr = udp_socket.local_addr().unwrap().ip(); let port = udp_socket.local_addr().unwrap().port(); - let tpu_addr = SocketAddr::new(addr, port); + let _tpu_addr = SocketAddr::new(addr, port); Self { sock: udp_socket, diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index bf22d146..e448f894 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -1,12 +1,12 @@ -use std::net::{IpAddr, SocketAddr}; -use std::sync::Arc; + + use anyhow::bail; use clap::Parser; use dotenv::dotenv; use log::info; use crate::cli::{Args, get_identity_keypair}; use crate::proxy::QuicForwardProxy; -use crate::test_client::quic_test_client::QuicTestClient; + pub use tls_config_provicer::SelfSignedTlsConfigProvider; use crate::validator_identity::ValidatorIdentity; @@ -39,7 +39,7 @@ pub async fn main() -> anyhow::Result<()> { // TODO build args struct dedicyted to proxy let proxy_listener_addr = proxy_rpc_addr.parse().unwrap(); - let tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); + let _tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let validator_identity = ValidatorIdentity::new(get_identity_keypair(&identity_keypair).await); diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 5def09dd..9f9047e0 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -1,42 +1,42 @@ use std::collections::HashMap; -use std::future::Future; -use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; -use std::path::Path; + +use std::net::{SocketAddr}; + use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU64}; -use std::thread; -use std::thread::sleep; +use std::sync::atomic::{AtomicBool}; + + use std::time::Duration; -use tracing::{debug_span, instrument, Instrument, span}; -use anyhow::{anyhow, bail, Context, Error}; -use dashmap::DashMap; + +use anyhow::{anyhow, bail, Context}; + use fan::tokio::mpsc::FanOut; -use futures::sink::Fanout; -use itertools::{any, Itertools}; + + use log::{debug, error, info, trace, warn}; -use quinn::{Connecting, Connection, ConnectionError, Endpoint, SendStream, ServerConfig, TransportConfig, VarInt}; -use rcgen::generate_simple_self_signed; -use rustls::{Certificate, PrivateKey}; -use rustls::server::ResolvesServerCert; -use serde::{Deserialize, Serialize}; +use quinn::{Connection, Endpoint, ServerConfig, VarInt}; + + + + use solana_sdk::packet::PACKET_DATA_SIZE; -use solana_sdk::pubkey::Pubkey; -use solana_sdk::quic::QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS; -use solana_sdk::signature::Keypair; -use solana_sdk::signer::Signer; + + + + use solana_sdk::transaction::VersionedTransaction; -use tokio::net::ToSocketAddrs; -use solana_streamer::tls_certificates::new_self_signed_tls_certificate; + + use tokio::sync::mpsc::{channel, Receiver, Sender}; -use tokio::sync::{RwLock, RwLockReadGuard}; -use tokio::task::JoinHandle; -use tokio::time::error::Elapsed; + + + use tokio::time::timeout; -use tracing::field::debug; + use crate::proxy_request_format::TpuForwardingRequest; -use crate::quic_connection_utils::{connection_stats, QuicConnectionUtils}; + use crate::quinn_auto_reconnect::AutoReconnect; -use crate::tpu_quic_client::{send_txs_to_tpu_static, SingleTPUConnectionManager, TpuQuicClient}; +use crate::tpu_quic_client::{send_txs_to_tpu_static, TpuQuicClient}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::AnyhowJoinHandle; use crate::validator_identity::ValidatorIdentity; @@ -89,7 +89,7 @@ impl QuicForwardProxy { } pub async fn start_services( - mut self, + self, ) -> anyhow::Result<()> { let exit_signal = Arc::new(AtomicBool::new(false)); @@ -159,8 +159,8 @@ async fn accept_client_connection(client_connection: Connection, forwarder_chann return Err(anyhow::Error::msg("error accepting stream")); } Ok(recv_stream) => { - let exit_signal_copy = exit_signal.clone(); - let tpu_quic_client_copy = tpu_quic_client.clone(); + let _exit_signal_copy = exit_signal.clone(); + let _tpu_quic_client_copy = tpu_quic_client.clone(); let forwarder_channel_copy = forwarder_channel.clone(); tokio::spawn(async move { @@ -172,7 +172,7 @@ async fn accept_client_connection(client_connection: Connection, forwarder_chann let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); trace!("proxy request details: {}", proxy_request); - let tpu_identity = proxy_request.get_identity_tpunode(); + let _tpu_identity = proxy_request.get_identity_tpunode(); let tpu_address = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transactions(); @@ -212,7 +212,7 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R // TODO cleanup agent after a while of iactivity let mut senders = Vec::new(); - for i in 0..4 { + for _i in 0..4 { let (sender, mut receiver) = channel::(100000); senders.push(sender); let exit_signal = exit_signal.clone(); @@ -228,7 +228,7 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R // let mut connection = tpu_quic_client_copy.create_connection(tpu_address).await.expect("handshake"); loop { - let exit_signal = exit_signal.clone(); + let _exit_signal = exit_signal.clone(); loop { let packet = receiver.recv().await.unwrap(); assert_eq!(packet.tpu_address, tpu_address, "routing error"); @@ -295,9 +295,9 @@ async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: R // tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &forward_packet.transactions, exit_signal_copy)).await; // tpu_quic_client_copy.send_txs_to_tpu(forward_packet.tpu_address, &forward_packet.transactions, exit_signal_copy).await; - } + } // -- loop over transactions from ustream channels - panic!("not reachable"); + // not reachable } diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index d926b71b..7ae100ec 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -1,6 +1,6 @@ use std::fmt; use std::fmt::Display; -use std::net::{SocketAddr, SocketAddrV4}; +use std::net::{SocketAddr}; use anyhow::Context; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs index c2a3f3d3..7f4b9aa6 100644 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ b/quic-forward-proxy/src/quic_connection_utils.rs @@ -1,22 +1,21 @@ -use log::{debug, error, info, trace, warn}; -use quinn::{ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, VarInt, WriteError}; +use log::{debug, error, trace, warn}; +use quinn::{ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, VarInt}; use solana_sdk::pubkey::Pubkey; use std::{ - collections::VecDeque, net::{IpAddr, Ipv4Addr, SocketAddr}, sync::{ - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, Ordering}, Arc, }, time::Duration, }; -use anyhow::bail; + use futures::future::join_all; use itertools::Itertools; use solana_sdk::quic::QUIC_MAX_TIMEOUT_MS; -use tokio::{sync::RwLock, time::timeout}; -use tokio::time::error::Elapsed; -use tracing::instrument; +use tokio::{time::timeout}; + + const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; @@ -297,7 +296,7 @@ impl QuicConnectionUtils { } } } - Err(elapsed) => { + Err(_elapsed) => { warn!("timeout sending transactions"); } } @@ -320,7 +319,7 @@ impl QuicConnectionUtils { pub async fn send_transaction_batch_parallel( connection: Connection, txs: Vec>, - exit_signal: Arc, + _exit_signal: Arc, connection_timeout: Duration, ) { assert_ne!(txs.len(), 0, "no transactions to send"); @@ -350,7 +349,7 @@ impl QuicConnectionUtils { } } } - Err(elapsed) => { + Err(_elapsed) => { warn!("timeout sending transactions"); } } diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 4d639fff..58a0d4ca 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -1,10 +1,10 @@ -use std::cell::RefCell; + use std::fmt; use std::net::SocketAddr; use std::sync::atomic::{AtomicU32, Ordering}; use tracing::{debug, info}; use quinn::{Connection, Endpoint}; -use tokio::sync::{RwLock, RwLockWriteGuard}; +use tokio::sync::{RwLock}; pub struct AutoReconnect { endpoint: Endpoint, diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index e9648696..56a4fd99 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -1,9 +1,9 @@ -use std::net::{SocketAddr, SocketAddrV4}; +use std::net::{SocketAddr}; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; use anyhow::bail; -use bytes::BufMut; + use log::{info, trace}; use quinn::{Endpoint, VarInt}; use rustls::ClientConfig; @@ -14,7 +14,7 @@ use crate::proxy_request_format::TpuForwardingRequest; use crate::quic_connection_utils::SkipServerVerification; use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; use crate::tls_config_provicer::ProxyTlsConfigProvider; -use crate::test_client::sample_data_factory::build_raw_sample_tx; + use crate::util::AnyhowJoinHandle; pub struct QuicTestClient { @@ -36,7 +36,7 @@ impl QuicTestClient { // connect to a server pub async fn start_services( - mut self, + self, ) -> anyhow::Result<()> { let endpoint_copy = self.endpoint.clone(); let test_client_service: AnyhowJoinHandle = tokio::spawn(async move { @@ -50,7 +50,7 @@ impl QuicTestClient { let connecting = endpoint_copy.connect(self.proxy_addr, "localhost").unwrap(); let connection = tokio::time::timeout(connection_timeout, connecting).await??; - for si in 0..5 { + for _si in 0..5 { let mut send = connection.open_uni().await?; let raw = build_memo_tx_raw(); diff --git a/quic-forward-proxy/src/test_client/sample_data_factory.rs b/quic-forward-proxy/src/test_client/sample_data_factory.rs index a5597ede..4542e9ea 100644 --- a/quic-forward-proxy/src/test_client/sample_data_factory.rs +++ b/quic-forward-proxy/src/test_client/sample_data_factory.rs @@ -4,7 +4,7 @@ use solana_sdk::hash::Hash; use solana_sdk::instruction::Instruction; use solana_sdk::message::Message; use solana_sdk::pubkey::Pubkey; -use solana_sdk::signature::{Keypair, keypair, Signature, Signer}; +use solana_sdk::signature::{Keypair, keypair, Signer}; use solana_sdk::transaction::{Transaction, VersionedTransaction}; const MEMO_PROGRAM_ID: &str = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"; diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/tpu_quic_client.rs index ff54981c..d43a31cf 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/tpu_quic_client.rs @@ -1,32 +1,32 @@ use std::collections::{HashMap, VecDeque}; use std::io::Write; -use std::net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4}; -use std::path::Path; -use std::str::FromStr; -use std::sync::{Arc, Mutex}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + +use std::sync::{Arc}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::time::Duration; -use anyhow::{anyhow, bail, Error}; +use anyhow::{anyhow}; use async_trait::async_trait; -use dashmap::DashMap; + use futures::future::join_all; -use itertools::{any, Itertools}; -use log::{debug, error, info, trace, warn}; -use quinn::{Connecting, Connection, ConnectionError, Endpoint, SendStream, ServerConfig, VarInt}; -use rcgen::{generate_simple_self_signed, KeyPair}; -use rustls::{Certificate, PrivateKey}; -use rustls::server::ResolvesServerCert; -use serde::{Deserialize, Serialize}; -use solana_sdk::pubkey::Pubkey; -use solana_sdk::signature::Keypair; -use solana_sdk::signer::Signer; +use itertools::{Itertools}; +use log::{debug, info, warn}; +use quinn::{Connection, Endpoint}; + + + + + + + use solana_sdk::transaction::VersionedTransaction; -use tokio::net::ToSocketAddrs; + use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; use crate::quic_connection_utils::{connection_stats, QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils}; use crate::quinn_auto_reconnect::AutoReconnect; -use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; + use crate::validator_identity::ValidatorIdentity; const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index 926f4415..c9e7ac0f 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -1,14 +1,14 @@ -use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; + use std::str::FromStr; -use anyhow::Context; -use bincode::DefaultOptions; -use log::info; -use serde::Serialize; -use solana_sdk::hash::{Hash, Hasher}; + + + + + use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, Signer}; -use solana_sdk::transaction::{Transaction, VersionedTransaction}; -use spl_memo::solana_program::message::VersionedMessage; +use solana_sdk::transaction::{Transaction}; + use solana_lite_rpc_quic_forward_proxy::proxy_request_format::TpuForwardingRequest; #[test] diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 41940985..5d469e97 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -1,29 +1,29 @@ -use std::cell::Cell; + use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::Ordering::Relaxed; -use std::thread; + use std::time::Duration; -use anyhow::{bail, Context}; -use async_trait::async_trait; +use anyhow::{bail}; + use futures::FutureExt; use itertools::Itertools; -use log::{debug, error, info, trace, warn}; -use quinn::{ClientConfig, Connection, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt}; +use log::{debug, error, info, trace}; +use quinn::{ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt}; use solana_sdk::packet::PACKET_DATA_SIZE; use solana_sdk::pubkey::Pubkey; -use solana_sdk::signer::Signer; + use solana_sdk::signature::Keypair; use solana_sdk::transaction::VersionedTransaction; use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; use tokio::time::timeout; -use tracing::field::debug; + use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -use solana_lite_rpc_core::quic_connection_utils::{connection_stats, QuicConnectionParameters, QuicConnectionUtils, SkipServerVerification}; -use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; -use solana_lite_rpc_core::tx_store::TxStore; +use solana_lite_rpc_core::quic_connection_utils::{connection_stats, SkipServerVerification}; + + use crate::tpu_utils::quinn_auto_reconnect::AutoReconnect; #[derive(Clone, Copy, Debug)] @@ -93,7 +93,7 @@ impl QuicProxyConnectionManager { info!("Starting very simple proxy thread"); - let mut transaction_receiver = transaction_sender.subscribe(); + let transaction_receiver = transaction_sender.subscribe(); // TODO use it let exit_signal = Arc::new(AtomicBool::new(false)); @@ -135,7 +135,7 @@ impl QuicProxyConnectionManager { // note: this config must be aligned with quic-proxy's server config let mut transport_config = TransportConfig::default(); - let timeout = IdleTimeout::try_from(Duration::from_secs(1)).unwrap(); + let _timeout = IdleTimeout::try_from(Duration::from_secs(1)).unwrap(); // no remotely-initiated streams required transport_config.max_concurrent_uni_streams(VarInt::from_u32(0)); transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); @@ -175,7 +175,7 @@ impl QuicProxyConnectionManager { let first_tx: Vec = match tx { - Ok((sig, tx)) => { + Ok((_sig, tx)) => { // if Self::check_for_confirmation(&txs_sent_store, sig) { // // transaction is already confirmed/ no need to send // continue; @@ -193,9 +193,9 @@ impl QuicProxyConnectionManager { let mut txs = vec![first_tx]; // TODO comment in - let foo = PACKET_DATA_SIZE; + let _foo = PACKET_DATA_SIZE; for _ in 1..number_of_transactions_per_unistream { - if let Ok((signature, tx)) = transaction_receiver.try_recv() { + if let Ok((_signature, tx)) = transaction_receiver.try_recv() { // if Self::check_for_confirmation(&txs_sent_store, signature) { // continue; // } @@ -223,7 +223,7 @@ impl QuicProxyConnectionManager { } async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, auto_connection: &AutoReconnect, - proxy_address: SocketAddr, tpu_target_address: SocketAddr, + _proxy_address: SocketAddr, tpu_target_address: SocketAddr, target_tpu_identity: Pubkey) -> anyhow::Result<()> { // TODO add timeout @@ -276,7 +276,7 @@ impl QuicProxyConnectionManager { async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { info!("sending {} bytes to proxy", proxy_request_raw.len()); - let mut connecting = endpoint.connect(proxy_address, "localhost")?; + let connecting = endpoint.connect(proxy_address, "localhost")?; let connection = timeout(Duration::from_millis(500), connecting).await??; let mut send = connection.open_uni().await?; diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index bce87d1f..13c4d074 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -1,11 +1,11 @@ -use std::cell::RefCell; + use std::fmt; use std::net::SocketAddr; use std::sync::atomic::{AtomicU32, Ordering}; use log::{trace, warn}; -use tracing::{debug, info}; +use tracing::{debug}; use quinn::{Connection, Endpoint}; -use tokio::sync::{RwLock, RwLockWriteGuard}; +use tokio::sync::{RwLock}; pub struct AutoReconnect { endpoint: Endpoint, diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 8f665121..7de1e9a8 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -1,5 +1,5 @@ use dashmap::DashMap; -use log::{error, info, trace}; +use log::{error, trace}; use prometheus::{core::GenericGauge, opts, register_int_gauge}; use quinn::Endpoint; use solana_lite_rpc_core::{ @@ -20,8 +20,8 @@ use std::{ }, }; use tokio::sync::{broadcast::Receiver, broadcast::Sender}; -use crate::tpu_utils::tpu_connection_path::TpuConnectionPath; -use crate::tpu_utils::tpu_connection_path::TpuConnectionPath::QuicForwardProxyPath; + + lazy_static::lazy_static! { static ref NB_QUIC_CONNECTIONS: GenericGauge = diff --git a/services/src/tpu_utils/tpu_connection_path.rs b/services/src/tpu_utils/tpu_connection_path.rs index 33b5bd1b..f42bfe11 100644 --- a/services/src/tpu_utils/tpu_connection_path.rs +++ b/services/src/tpu_utils/tpu_connection_path.rs @@ -1,13 +1,13 @@ -use std::collections::HashMap; + use std::fmt::Display; use std::net::SocketAddr; -use std::sync::Arc; -use async_trait::async_trait; -use solana_sdk::pubkey::Pubkey; -use tokio::sync::{broadcast::Receiver, broadcast::Sender}; -use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; -use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; -use solana_lite_rpc_core::tx_store::TxStore; + + + + + + + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum TpuConnectionPath { diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index 4bf8fff1..66c585b3 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -1,9 +1,9 @@ -use anyhow::bail; + use countmap::CountMap; -use crossbeam_channel::{Receiver, RecvError, RecvTimeoutError, Sender}; -use futures::future::join_all; -use log::{debug, error, info, trace, warn}; -use quinn::TokioRuntime; +use crossbeam_channel::{Sender}; + +use log::{debug, info, trace, warn}; + use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; use solana_lite_rpc_core::tx_store::empty_tx_store; @@ -14,7 +14,7 @@ use solana_sdk::instruction::Instruction; use solana_sdk::message::Message; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, Signature, Signer}; -use solana_sdk::signer::keypair; + use solana_sdk::transaction::{Transaction, VersionedTransaction}; use solana_streamer::nonblocking::quic::ConnectionPeerType; use solana_streamer::packet::PacketBatch; @@ -23,21 +23,21 @@ use solana_streamer::streamer::StakedNodes; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; -use std::ops::Deref; -use std::option::Option; -use std::path::Path; + + + use std::str::FromStr; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; use std::thread; use std::time::{Duration, Instant}; -use tokio::runtime::{Builder, Runtime}; -use tokio::sync::broadcast; -use tokio::sync::broadcast::error::SendError; +use tokio::runtime::{Builder}; + + use tokio::task::JoinHandle; -use tokio::time::{interval, sleep}; +use tokio::time::{sleep}; use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::{filter::LevelFilter, fmt}; + #[derive(Copy, Clone, Debug)] struct TestCaseParams { @@ -378,7 +378,7 @@ async fn solana_quic_streamer_start() { let addr = sock.local_addr().unwrap().ip(); let port = sock.local_addr().unwrap().port(); - let tpu_addr = SocketAddr::new(addr, port); + let _tpu_addr = SocketAddr::new(addr, port); // sleep(Duration::from_millis(500)).await; @@ -439,7 +439,7 @@ impl SolanaQuicStreamer { let addr = udp_socket.local_addr().unwrap().ip(); let port = udp_socket.local_addr().unwrap().port(); - let tpu_addr = SocketAddr::new(addr, port); + let _tpu_addr = SocketAddr::new(addr, port); Self { sock: udp_socket, From aea5657135a39399dffe2f4f8cdc7bcc5846ec86 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 09:58:17 +0200 Subject: [PATCH 062/128] works: split listener+forwarder --- .../tests/quic_proxy_tpu_integrationtest.rs | 6 +- quic-forward-proxy/src/.DS_Store | Bin 0 -> 6148 bytes quic-forward-proxy/src/cli.rs | 2 +- quic-forward-proxy/src/identity_stakes.rs | 22 -- quic-forward-proxy/src/inbound/mod.rs | 1 + .../src/inbound/proxy_listener.rs | 152 ++++++++++ quic-forward-proxy/src/lib.rs | 6 +- quic-forward-proxy/src/main.rs | 10 +- quic-forward-proxy/src/outbound/mod.rs | 3 + .../src/{ => outbound}/tpu_quic_client.rs | 7 +- quic-forward-proxy/src/outbound/tx_forward.rs | 137 +++++++++ .../src/{ => outbound}/validator_identity.rs | 0 quic-forward-proxy/src/proxy.rs | 272 ++---------------- quic-forward-proxy/src/share/mod.rs | 9 + 14 files changed, 346 insertions(+), 281 deletions(-) create mode 100644 quic-forward-proxy/src/.DS_Store delete mode 100644 quic-forward-proxy/src/identity_stakes.rs create mode 100644 quic-forward-proxy/src/inbound/mod.rs create mode 100644 quic-forward-proxy/src/inbound/proxy_listener.rs create mode 100644 quic-forward-proxy/src/outbound/mod.rs rename quic-forward-proxy/src/{ => outbound}/tpu_quic_client.rs (98%) create mode 100644 quic-forward-proxy/src/outbound/tx_forward.rs rename quic-forward-proxy/src/{ => outbound}/validator_identity.rs (100%) create mode 100644 quic-forward-proxy/src/share/mod.rs diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 7cd7deca..98245dcd 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -39,9 +39,9 @@ use tokio::time::{sleep}; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::fmt::format::FmtSpan; +use solana_lite_rpc_quic_forward_proxy::outbound::validator_identity::ValidatorIdentity; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_config_provicer::SelfSignedTlsConfigProvider; -use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; #[derive(Copy, Clone, Debug)] @@ -680,8 +680,8 @@ async fn start_quic_proxy(proxy_listen_addr: SocketAddr) -> anyhow::Result<()> { let _tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let random_unstaked_validator_identity = ValidatorIdentity::new(None); - let tls_config = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); - let proxy_service = QuicForwardProxy::new(proxy_listen_addr, &tls_config, random_unstaked_validator_identity) + let tls_config = Arc::new(SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost()); + let proxy_service = QuicForwardProxy::new(proxy_listen_addr, tls_config, random_unstaked_validator_identity) .await? .start_services(); diff --git a/quic-forward-proxy/src/.DS_Store b/quic-forward-proxy/src/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..9d671b19042201cf3b55274a81eebe23bcb5a2bf GIT binary patch literal 6148 zcmeHKF-`+P474E}c|XpwmjmS9*a5t~jrM~- zzT!yS%izcI=JJV(N&zV#1*Cu!kOF@xzyb?f-zI950#ZNZ9*7nv0JZw0;pVJRF? literal 0 HcmV?d00001 diff --git a/quic-forward-proxy/src/cli.rs b/quic-forward-proxy/src/cli.rs index fafdfed0..08bad1d9 100644 --- a/quic-forward-proxy/src/cli.rs +++ b/quic-forward-proxy/src/cli.rs @@ -8,7 +8,7 @@ pub struct Args { #[arg(short = 'k', long, default_value_t = String::new())] pub identity_keypair: String, #[arg(short = 'l', long)] - pub proxy_rpc_addr: String, + pub proxy_listen_addr: String, } // note this is duplicated from lite-rpc module diff --git a/quic-forward-proxy/src/identity_stakes.rs b/quic-forward-proxy/src/identity_stakes.rs deleted file mode 100644 index 2b1194cd..00000000 --- a/quic-forward-proxy/src/identity_stakes.rs +++ /dev/null @@ -1,22 +0,0 @@ -use solana_streamer::nonblocking::quic::ConnectionPeerType; - -#[derive(Debug, Copy, Clone)] -pub struct IdentityStakes { - pub peer_type: ConnectionPeerType, - pub stakes: u64, - pub total_stakes: u64, - pub min_stakes: u64, - pub max_stakes: u64, -} - -impl Default for IdentityStakes { - fn default() -> Self { - IdentityStakes { - peer_type: ConnectionPeerType::Unstaked, - stakes: 0, - total_stakes: 0, - max_stakes: 0, - min_stakes: 0, - } - } -} diff --git a/quic-forward-proxy/src/inbound/mod.rs b/quic-forward-proxy/src/inbound/mod.rs new file mode 100644 index 00000000..545cb4c9 --- /dev/null +++ b/quic-forward-proxy/src/inbound/mod.rs @@ -0,0 +1 @@ +pub(crate) mod proxy_listener; \ No newline at end of file diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs new file mode 100644 index 00000000..3186ad72 --- /dev/null +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -0,0 +1,152 @@ +use std::net::SocketAddr; + +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + + +use std::time::Duration; + +use anyhow::{anyhow, bail, Context}; + +use log::{debug, error, info, trace}; +use quinn::{Connection, Endpoint, ServerConfig, VarInt}; + +use solana_sdk::packet::PACKET_DATA_SIZE; + + +use tokio::sync::mpsc::Sender; +use crate::inbound::proxy_listener; +use crate::outbound::tx_forward::tx_forwarder; +use crate::outbound::validator_identity::ValidatorIdentity; + + +use crate::proxy_request_format::TpuForwardingRequest; +use crate::share::ForwardPacket; + +use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; +use crate::util::AnyhowJoinHandle; + +// TODO tweak this value - solana server sets 256 +// setting this to "1" did not make a difference! +const MAX_CONCURRENT_UNI_STREAMS: u32 = 24; + + +pub struct ProxyListener { + tls_config: Arc, + proxy_listener_addr: SocketAddr, +} + +impl ProxyListener { + + pub fn new( + proxy_listener_addr: SocketAddr, + tls_config: Arc) -> Self { + Self { proxy_listener_addr, tls_config } + } + + pub async fn listen(&self, exit_signal: Arc, forwarder_channel: Sender) -> anyhow::Result<()> { + info!("TPU Quic Proxy server listening on {}", self.proxy_listener_addr); + + let endpoint = Self::new_proxy_listen_endpoint(&self.tls_config, self.proxy_listener_addr).await; + + while let Some(connecting) = endpoint.accept().await { + let exit_signal = exit_signal.clone(); + let forwarder_channel_copy = forwarder_channel.clone(); + tokio::spawn(async move { + let connection = connecting.await.context("handshake").unwrap(); + match Self::accept_client_connection(connection, forwarder_channel_copy, + exit_signal) + .await { + Ok(()) => {} + Err(err) => { + error!("setup connection failed: {reason}", reason = err); + } + } + }); + } + + bail!("TPU Quic Proxy server stopped"); + } + + + async fn new_proxy_listen_endpoint(tls_config: &SelfSignedTlsConfigProvider, proxy_listener_addr: SocketAddr) -> Endpoint { + + let server_tls_config = tls_config.get_server_tls_crypto_config(); + let mut quinn_server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); + + // note: this config must be aligned with lite-rpc's client config + let transport_config = Arc::get_mut(&mut quinn_server_config.transport).unwrap(); + // TODO experiment with this value + transport_config.max_concurrent_uni_streams(VarInt::from_u32(MAX_CONCURRENT_UNI_STREAMS)); + // no bidi streams used + transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); + let timeout = Duration::from_secs(10).try_into().unwrap(); + transport_config.max_idle_timeout(Some(timeout)); + transport_config.keep_alive_interval(Some(Duration::from_millis(500))); + transport_config.stream_receive_window((PACKET_DATA_SIZE as u32).into()); + transport_config.receive_window((PACKET_DATA_SIZE as u32 * MAX_CONCURRENT_UNI_STREAMS).into()); + + Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap() + } + + + // TODO use interface abstraction for connection_per_tpunode + #[tracing::instrument(skip_all, level = "debug")] + async fn accept_client_connection(client_connection: Connection, forwarder_channel: Sender, + exit_signal: Arc) -> anyhow::Result<()> { + debug!("inbound connection established, client {}", client_connection.remote_address()); + + loop { + let maybe_stream = client_connection.accept_uni().await; + let result = match maybe_stream { + Err(quinn::ConnectionError::ApplicationClosed(reason)) => { + debug!("connection closed by client - reason: {:?}", reason); + if reason.error_code != VarInt::from_u32(0) { + return Err(anyhow!("connection closed by client with unexpected reason: {:?}", reason)); + } + debug!("connection gracefully closed by client"); + return Ok(()); + }, + Err(e) => { + error!("failed to accept stream: {}", e); + return Err(anyhow::Error::msg("error accepting stream")); + } + Ok(recv_stream) => { + let _exit_signal_copy = exit_signal.clone(); + + let forwarder_channel_copy = forwarder_channel.clone(); + tokio::spawn(async move { + + let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const + .unwrap(); + trace!("read proxy_request {} bytes", raw_request.len()); + + let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); + + trace!("proxy request details: {}", proxy_request); + let _tpu_identity = proxy_request.get_identity_tpunode(); + let tpu_address = proxy_request.get_tpu_socket_addr(); + let txs = proxy_request.get_transactions(); + + debug!("enqueue transaction batch of size {} to address {}", txs.len(), tpu_address); + // tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &txs, exit_signal_copy).await; + forwarder_channel_copy.send(ForwardPacket { transactions: txs, tpu_address }).await.unwrap(); + + // debug!("connection stats (proxy inbound): {}", connection_stats(&client_connection)); + + }); + + Ok(()) + }, + }; // -- result + + if let Err(e) = result { + return Err(e); + } + + } // -- loop + } + +} + + diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index 66a086c1..2f97f182 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -2,12 +2,12 @@ pub mod quic_util; pub mod tls_config_provicer; pub mod proxy; pub mod proxy_request_format; -pub mod tpu_quic_client; pub mod cli; pub mod test_client; -pub mod validator_identity; mod util; mod tx_store; -mod identity_stakes; mod quic_connection_utils; mod quinn_auto_reconnect; +pub mod outbound; +mod inbound; +mod share; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index e448f894..20b81724 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -8,22 +8,22 @@ use crate::cli::{Args, get_identity_keypair}; use crate::proxy::QuicForwardProxy; pub use tls_config_provicer::SelfSignedTlsConfigProvider; -use crate::validator_identity::ValidatorIdentity; +use crate::outbound::validator_identity::ValidatorIdentity; pub mod quic_util; pub mod tls_config_provicer; pub mod proxy; pub mod proxy_request_format; -pub mod tpu_quic_client; pub mod cli; pub mod test_client; mod util; mod tx_store; -mod identity_stakes; mod quic_connection_utils; mod quinn_auto_reconnect; -mod validator_identity; +mod outbound; +mod inbound; +mod share; #[tokio::main(flavor = "multi_thread", worker_threads = 16)] @@ -32,7 +32,7 @@ pub async fn main() -> anyhow::Result<()> { let Args { identity_keypair, - proxy_rpc_addr, + proxy_listen_addr: proxy_rpc_addr, } = Args::parse(); dotenv().ok(); diff --git a/quic-forward-proxy/src/outbound/mod.rs b/quic-forward-proxy/src/outbound/mod.rs new file mode 100644 index 00000000..6894a6d0 --- /dev/null +++ b/quic-forward-proxy/src/outbound/mod.rs @@ -0,0 +1,3 @@ +mod tpu_quic_client; +pub mod validator_identity; +pub mod tx_forward; \ No newline at end of file diff --git a/quic-forward-proxy/src/tpu_quic_client.rs b/quic-forward-proxy/src/outbound/tpu_quic_client.rs similarity index 98% rename from quic-forward-proxy/src/tpu_quic_client.rs rename to quic-forward-proxy/src/outbound/tpu_quic_client.rs index d43a31cf..f93eed22 100644 --- a/quic-forward-proxy/src/tpu_quic_client.rs +++ b/quic-forward-proxy/src/outbound/tpu_quic_client.rs @@ -24,11 +24,10 @@ use solana_sdk::transaction::VersionedTransaction; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::RwLock; +use crate::outbound::validator_identity::ValidatorIdentity; use crate::quic_connection_utils::{connection_stats, QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils}; use crate::quinn_auto_reconnect::AutoReconnect; -use crate::validator_identity::ValidatorIdentity; - const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); pub const CONNECTION_RETRY_COUNT: usize = 10; @@ -151,7 +150,7 @@ impl TpuQuicClient { /// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU // note: ATM the provided identity might or might not be a valid validator keypair - pub async fn new_with_validator_identity(validator_identity: ValidatorIdentity) -> TpuQuicClient { + pub async fn new_with_validator_identity_delme(validator_identity: ValidatorIdentity) -> TpuQuicClient { info!("Setup TPU Quic stable connection with validator identity {} ...", validator_identity); let (certificate, key) = new_self_signed_tls_certificate( &validator_identity.get_keypair_for_tls(), @@ -338,4 +337,4 @@ pub async fn send_txs_to_tpu_static( } -} \ No newline at end of file +} diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs new file mode 100644 index 00000000..ff3762ed --- /dev/null +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -0,0 +1,137 @@ +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use log::{debug, info, warn}; +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use fan::tokio::mpsc::FanOut; +use std::time::Duration; +use quinn::Endpoint; +use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use tokio::sync::mpsc::{channel, Receiver}; +use tokio::time::timeout; +use crate::outbound::tpu_quic_client::{send_txs_to_tpu_static, TpuQuicClient}; +use crate::outbound::validator_identity::ValidatorIdentity; +use crate::quic_connection_utils::QuicConnectionUtils; +use crate::quinn_auto_reconnect::AutoReconnect; +use crate::share::ForwardPacket; + +// takes transactions from upstream clients and forwards them to the TPU +pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction_channel: Receiver, exit_signal: Arc) -> anyhow::Result<()> { + info!("TPU Quic forwarder started"); + + let endpoint = new_endpoint_with_validator_identity(validator_identity).await; + + let mut agents: HashMap> = HashMap::new(); + + loop { + // TODO add exit + + let forward_packet = transaction_channel.recv().await.expect("channel closed unexpectedly"); + let tpu_address = forward_packet.tpu_address; + + if !agents.contains_key(&tpu_address) { + // TODO cleanup agent after a while of iactivity + + let mut senders = Vec::new(); + for _i in 0..4 { + let (sender, mut receiver) = channel::(100000); + senders.push(sender); + let exit_signal = exit_signal.clone(); + let endpoint_copy = endpoint.clone(); + tokio::spawn(async move { + debug!("Start Quic forwarder agent for TPU {}", tpu_address); + // TODO pass+check the tpu_address + // TODO connect + // TODO consume queue + // TODO exit signal + + let auto_connection = AutoReconnect::new(endpoint_copy, tpu_address); + // let mut connection = tpu_quic_client_copy.create_connection(tpu_address).await.expect("handshake"); + loop { + + let _exit_signal = exit_signal.clone(); + loop { + let packet = receiver.recv().await.unwrap(); + assert_eq!(packet.tpu_address, tpu_address, "routing error"); + + let mut transactions_batch = packet.transactions; + + let mut batch_size = 1; + while let Ok(more) = receiver.try_recv() { + transactions_batch.extend(more.transactions); + batch_size += 1; + } + if batch_size > 1 { + debug!("encountered batch of size {}", batch_size); + } + + debug!("forwarding transaction batch of size {} to address {}", transactions_batch.len(), packet.tpu_address); + + // TODo move send_txs_to_tpu_static to tpu_quic_client + let result = timeout(Duration::from_millis(500), + send_txs_to_tpu_static(&auto_connection, &transactions_batch)).await; + // .expect("timeout sending data to TPU node"); + + if result.is_err() { + warn!("send_txs_to_tpu_static result {:?} - loop over errors", result); + } else { + debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); + } + + } + + } + + }); + + } + + let fanout = FanOut::new(senders); + + agents.insert(tpu_address, fanout); + + } // -- new agent + + let agent_channel = agents.get(&tpu_address).unwrap(); + + agent_channel.send(forward_packet).await.unwrap(); + + // let mut batch_size = 1; + // while let Ok(more) = transaction_channel.try_recv() { + // agent_channel.send(more).await.unwrap(); + // batch_size += 1; + // } + // if batch_size > 1 { + // debug!("encountered batch of size {}", batch_size); + // } + + + // check if the tpu has already a task+queue running, if not start one, sort+queue packets by tpu address + // maintain the health of a TPU connection, debounce errors; if failing, drop the respective messages + + // let exit_signal_copy = exit_signal.clone(); + // debug!("send transaction batch of size {} to address {}", forward_packet.transactions.len(), forward_packet.tpu_address); + // // TODO: this will block/timeout if the TPU is not available + // timeout(Duration::from_millis(500), + // tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &forward_packet.transactions, exit_signal_copy)).await; + // tpu_quic_client_copy.send_txs_to_tpu(forward_packet.tpu_address, &forward_packet.transactions, exit_signal_copy).await; + + } // -- loop over transactions from ustream channels + + // not reachable +} + +/// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU +// note: ATM the provided identity might or might not be a valid validator keypair +async fn new_endpoint_with_validator_identity(validator_identity: ValidatorIdentity) -> Endpoint { + info!("Setup TPU Quic stable connection with validator identity {} ...", validator_identity); + let (certificate, key) = new_self_signed_tls_certificate( + &validator_identity.get_keypair_for_tls(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC connection certificates"); + + let endpoint_outbound = QuicConnectionUtils::create_tpu_client_endpoint(certificate.clone(), key.clone()); + + endpoint_outbound +} \ No newline at end of file diff --git a/quic-forward-proxy/src/validator_identity.rs b/quic-forward-proxy/src/outbound/validator_identity.rs similarity index 100% rename from quic-forward-proxy/src/validator_identity.rs rename to quic-forward-proxy/src/outbound/validator_identity.rs diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 9f9047e0..b9bae14a 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -1,19 +1,14 @@ -use std::collections::HashMap; - -use std::net::{SocketAddr}; +use std::net::SocketAddr; use std::sync::Arc; -use std::sync::atomic::{AtomicBool}; +use std::sync::atomic::AtomicBool; use std::time::Duration; use anyhow::{anyhow, bail, Context}; -use fan::tokio::mpsc::FanOut; - - -use log::{debug, error, info, trace, warn}; +use log::{debug, error, info, trace}; use quinn::{Connection, Endpoint, ServerConfig, VarInt}; @@ -22,69 +17,36 @@ use quinn::{Connection, Endpoint, ServerConfig, VarInt}; use solana_sdk::packet::PACKET_DATA_SIZE; +use tokio::sync::mpsc::Sender; +use crate::inbound::proxy_listener; +use crate::outbound::tx_forward::tx_forwarder; +use crate::outbound::validator_identity::ValidatorIdentity; -use solana_sdk::transaction::VersionedTransaction; - - -use tokio::sync::mpsc::{channel, Receiver, Sender}; - - - -use tokio::time::timeout; - use crate::proxy_request_format::TpuForwardingRequest; +use crate::share::ForwardPacket; -use crate::quinn_auto_reconnect::AutoReconnect; -use crate::tpu_quic_client::{send_txs_to_tpu_static, TpuQuicClient}; use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::AnyhowJoinHandle; -use crate::validator_identity::ValidatorIdentity; -// TODO tweak this value - solana server sets 256 -// setting this to "1" did not make a difference! -const MAX_CONCURRENT_UNI_STREAMS: u32 = 24; -pub struct QuicForwardProxy { - endpoint: Endpoint, - // validator_identity: ValidatorIdentity, - tpu_quic_client: TpuQuicClient, -} -/// internal structure with transactions and target TPU -#[derive(Debug)] -struct ForwardPacket { - pub transactions: Vec, - pub tpu_address: SocketAddr, +pub struct QuicForwardProxy { + // endpoint: Endpoint, + validator_identity: ValidatorIdentity, + tls_config: Arc, + pub proxy_listener_addr: SocketAddr, } impl QuicForwardProxy { pub async fn new( proxy_listener_addr: SocketAddr, - tls_config: &SelfSignedTlsConfigProvider, + tls_config: Arc, validator_identity: ValidatorIdentity) -> anyhow::Result { - let server_tls_config = tls_config.get_server_tls_crypto_config(); - let mut quinn_server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); - - // note: this config must be aligned with lite-rpc's client config - let transport_config = Arc::get_mut(&mut quinn_server_config.transport).unwrap(); - // TODO experiment with this value - transport_config.max_concurrent_uni_streams(VarInt::from_u32(MAX_CONCURRENT_UNI_STREAMS)); - // no bidi streams used - transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); - let timeout = Duration::from_secs(10).try_into().unwrap(); - transport_config.max_idle_timeout(Some(timeout)); - transport_config.keep_alive_interval(Some(Duration::from_millis(500))); - transport_config.stream_receive_window((PACKET_DATA_SIZE as u32).into()); - transport_config.receive_window((PACKET_DATA_SIZE as u32 * MAX_CONCURRENT_UNI_STREAMS).into()); - - let endpoint = Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap(); - info!("Quic proxy uses validator identity {}", validator_identity); - let tpu_quic_client = - TpuQuicClient::new_with_validator_identity(validator_identity).await; + info!("Quic proxy uses validator identity {}", validator_identity); - Ok(Self { endpoint, tpu_quic_client }) + Ok(Self { proxy_listener_addr, validator_identity, tls_config }) } @@ -93,13 +55,22 @@ impl QuicForwardProxy { ) -> anyhow::Result<()> { let exit_signal = Arc::new(AtomicBool::new(false)); - let tpu_quic_client_copy = self.tpu_quic_client.clone(); - let endpoint = self.endpoint.clone(); let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); - let quic_proxy: AnyhowJoinHandle = tokio::spawn(self.listen(exit_signal.clone(), endpoint, forwarder_channel)); + let proxy_listener = proxy_listener::ProxyListener::new( + self.proxy_listener_addr, + self.tls_config); + + let exit_signal_clone = exit_signal.clone(); + let quic_proxy = tokio::spawn(async move { + + proxy_listener.listen(exit_signal_clone.clone(), forwarder_channel).await; + }); - let forwarder: AnyhowJoinHandle = tokio::spawn(tx_forwarder(tpu_quic_client_copy, forward_receiver, exit_signal.clone())); + let validator_identity = self.validator_identity.clone(); + let exit_signal_clone = exit_signal.clone(); + let forwarder: AnyhowJoinHandle = tokio::spawn(tx_forwarder(validator_identity, + forward_receiver, exit_signal_clone)); tokio::select! { res = quic_proxy => { @@ -111,193 +82,8 @@ impl QuicForwardProxy { } } - async fn listen(self, exit_signal: Arc, endpoint: Endpoint, forwarder_channel: Sender) -> anyhow::Result<()> { - info!("TPU Quic Proxy server listening on {}", endpoint.local_addr()?); - - while let Some(connecting) = endpoint.accept().await { - let exit_signal = exit_signal.clone(); - let tpu_quic_client = self.tpu_quic_client.clone(); - let forwarder_channel_copy = forwarder_channel.clone(); - tokio::spawn(async move { - let connection = connecting.await.context("handshake").unwrap(); - match accept_client_connection(connection, forwarder_channel_copy, - tpu_quic_client, exit_signal) - .await { - Ok(()) => {} - Err(err) => { - error!("setup connection failed: {reason}", reason = err); - } - } - }); - } - - bail!("TPU Quic Proxy server stopped"); - } -} - - -// TODO use interface abstraction for connection_per_tpunode -#[tracing::instrument(skip_all, level = "debug")] -async fn accept_client_connection(client_connection: Connection, forwarder_channel: Sender, - tpu_quic_client: TpuQuicClient, - exit_signal: Arc) -> anyhow::Result<()> { - debug!("inbound connection established, client {}", client_connection.remote_address()); - - loop { - let maybe_stream = client_connection.accept_uni().await; - let result = match maybe_stream { - Err(quinn::ConnectionError::ApplicationClosed(reason)) => { - debug!("connection closed by client - reason: {:?}", reason); - if reason.error_code != VarInt::from_u32(0) { - return Err(anyhow!("connection closed by client with unexpected reason: {:?}", reason)); - } - debug!("connection gracefully closed by client"); - return Ok(()); - }, - Err(e) => { - error!("failed to accept stream: {}", e); - return Err(anyhow::Error::msg("error accepting stream")); - } - Ok(recv_stream) => { - let _exit_signal_copy = exit_signal.clone(); - let _tpu_quic_client_copy = tpu_quic_client.clone(); - - let forwarder_channel_copy = forwarder_channel.clone(); - tokio::spawn(async move { - - let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const - .unwrap(); - trace!("read proxy_request {} bytes", raw_request.len()); - - let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); - - trace!("proxy request details: {}", proxy_request); - let _tpu_identity = proxy_request.get_identity_tpunode(); - let tpu_address = proxy_request.get_tpu_socket_addr(); - let txs = proxy_request.get_transactions(); - - debug!("enqueue transaction batch of size {} to address {}", txs.len(), tpu_address); - // tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &txs, exit_signal_copy).await; - forwarder_channel_copy.send(ForwardPacket { transactions: txs, tpu_address }).await.unwrap(); - - // debug!("connection stats (proxy inbound): {}", connection_stats(&client_connection)); - - }); - - Ok(()) - }, - }; // -- result - - if let Err(e) = result { - return Err(e); - } - - } // -- loop } -// takes transactions from upstream clients and forwards them to the TPU -async fn tx_forwarder(tpu_quic_client: TpuQuicClient, mut transaction_channel: Receiver, exit_signal: Arc) -> anyhow::Result<()> { - info!("TPU Quic forwarder started"); - - let mut agents: HashMap> = HashMap::new(); - - let endpoint = tpu_quic_client.get_endpoint().clone(); - loop { - // TODO add exit - - let forward_packet = transaction_channel.recv().await.expect("channel closed unexpectedly"); - let tpu_address = forward_packet.tpu_address; - - if !agents.contains_key(&tpu_address) { - // TODO cleanup agent after a while of iactivity - - let mut senders = Vec::new(); - for _i in 0..4 { - let (sender, mut receiver) = channel::(100000); - senders.push(sender); - let exit_signal = exit_signal.clone(); - let endpoint_copy = endpoint.clone(); - tokio::spawn(async move { - debug!("Start Quic forwarder agent for TPU {}", tpu_address); - // TODO pass+check the tpu_address - // TODO connect - // TODO consume queue - // TODO exit signal - - let auto_connection = AutoReconnect::new(endpoint_copy, tpu_address); - // let mut connection = tpu_quic_client_copy.create_connection(tpu_address).await.expect("handshake"); - loop { - - let _exit_signal = exit_signal.clone(); - loop { - let packet = receiver.recv().await.unwrap(); - assert_eq!(packet.tpu_address, tpu_address, "routing error"); - - let mut transactions_batch = packet.transactions; - - let mut batch_size = 1; - while let Ok(more) = receiver.try_recv() { - transactions_batch.extend(more.transactions); - batch_size += 1; - } - if batch_size > 1 { - debug!("encountered batch of size {}", batch_size); - } - - debug!("forwarding transaction batch of size {} to address {}", transactions_batch.len(), packet.tpu_address); - - // TODo move send_txs_to_tpu_static to tpu_quic_client - let result = timeout(Duration::from_millis(500), - send_txs_to_tpu_static(&auto_connection, &transactions_batch)).await; - // .expect("timeout sending data to TPU node"); - - if result.is_err() { - warn!("send_txs_to_tpu_static result {:?} - loop over errors", result); - } else { - debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); - } - } - - } - - }); - - } - - let fanout = FanOut::new(senders); - - agents.insert(tpu_address, fanout); - - } // -- new agent - - let agent_channel = agents.get(&tpu_address).unwrap(); - - agent_channel.send(forward_packet).await.unwrap(); - - // let mut batch_size = 1; - // while let Ok(more) = transaction_channel.try_recv() { - // agent_channel.send(more).await.unwrap(); - // batch_size += 1; - // } - // if batch_size > 1 { - // debug!("encountered batch of size {}", batch_size); - // } - - - // check if the tpu has already a task+queue running, if not start one, sort+queue packets by tpu address - // maintain the health of a TPU connection, debounce errors; if failing, drop the respective messages - - // let exit_signal_copy = exit_signal.clone(); - // debug!("send transaction batch of size {} to address {}", forward_packet.transactions.len(), forward_packet.tpu_address); - // // TODO: this will block/timeout if the TPU is not available - // timeout(Duration::from_millis(500), - // tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &forward_packet.transactions, exit_signal_copy)).await; - // tpu_quic_client_copy.send_txs_to_tpu(forward_packet.tpu_address, &forward_packet.transactions, exit_signal_copy).await; - - } // -- loop over transactions from ustream channels - - // not reachable -} diff --git a/quic-forward-proxy/src/share/mod.rs b/quic-forward-proxy/src/share/mod.rs new file mode 100644 index 00000000..6b3f4c43 --- /dev/null +++ b/quic-forward-proxy/src/share/mod.rs @@ -0,0 +1,9 @@ +use solana_sdk::transaction::VersionedTransaction; +use std::net::SocketAddr; + +/// internal structure with transactions and target TPU +#[derive(Debug)] +pub struct ForwardPacket { + pub transactions: Vec, + pub tpu_address: SocketAddr, +} From 0e19b85d551bf0da6cb4bbece9334576626f238f Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 10:16:36 +0200 Subject: [PATCH 063/128] clenaup proxy.rs --- .../src/inbound/proxy_listener.rs | 34 +++++-------------- quic-forward-proxy/src/proxy.rs | 5 +-- quic-forward-proxy/src/util.rs | 12 +++++++ 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 3186ad72..17e8c110 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -1,30 +1,16 @@ use std::net::SocketAddr; - use std::sync::Arc; use std::sync::atomic::AtomicBool; - - use std::time::Duration; - use anyhow::{anyhow, bail, Context}; - use log::{debug, error, info, trace}; use quinn::{Connection, Endpoint, ServerConfig, VarInt}; - use solana_sdk::packet::PACKET_DATA_SIZE; - - use tokio::sync::mpsc::Sender; -use crate::inbound::proxy_listener; -use crate::outbound::tx_forward::tx_forwarder; -use crate::outbound::validator_identity::ValidatorIdentity; - - use crate::proxy_request_format::TpuForwardingRequest; use crate::share::ForwardPacket; - use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; -use crate::util::AnyhowJoinHandle; +use crate::util::FALLBACK_TIMEOUT; // TODO tweak this value - solana server sets 256 // setting this to "1" did not make a difference! @@ -47,7 +33,7 @@ impl ProxyListener { pub async fn listen(&self, exit_signal: Arc, forwarder_channel: Sender) -> anyhow::Result<()> { info!("TPU Quic Proxy server listening on {}", self.proxy_listener_addr); - let endpoint = Self::new_proxy_listen_endpoint(&self.tls_config, self.proxy_listener_addr).await; + let endpoint = Self::new_proxy_listen_server_endpoint(&self.tls_config, self.proxy_listener_addr).await; while let Some(connecting) = endpoint.accept().await { let exit_signal = exit_signal.clone(); @@ -69,7 +55,7 @@ impl ProxyListener { } - async fn new_proxy_listen_endpoint(tls_config: &SelfSignedTlsConfigProvider, proxy_listener_addr: SocketAddr) -> Endpoint { + async fn new_proxy_listen_server_endpoint(tls_config: &SelfSignedTlsConfigProvider, proxy_listener_addr: SocketAddr) -> Endpoint { let server_tls_config = tls_config.get_server_tls_crypto_config(); let mut quinn_server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); @@ -112,14 +98,11 @@ impl ProxyListener { return Err(anyhow::Error::msg("error accepting stream")); } Ok(recv_stream) => { - let _exit_signal_copy = exit_signal.clone(); - let forwarder_channel_copy = forwarder_channel.clone(); tokio::spawn(async move { - let raw_request = recv_stream.read_to_end(10_000_000).await // TODO extract to const + let raw_request = recv_stream.read_to_end(10_000_000).await .unwrap(); - trace!("read proxy_request {} bytes", raw_request.len()); let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); @@ -129,10 +112,11 @@ impl ProxyListener { let txs = proxy_request.get_transactions(); debug!("enqueue transaction batch of size {} to address {}", txs.len(), tpu_address); - // tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &txs, exit_signal_copy).await; - forwarder_channel_copy.send(ForwardPacket { transactions: txs, tpu_address }).await.unwrap(); - - // debug!("connection stats (proxy inbound): {}", connection_stats(&client_connection)); + forwarder_channel_copy.send_timeout(ForwardPacket { transactions: txs, tpu_address }, + FALLBACK_TIMEOUT) + .await + .context("sending internal packet from proxy to forwarder") + .unwrap(); }); diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index b9bae14a..515b4fe1 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -55,7 +55,7 @@ impl QuicForwardProxy { ) -> anyhow::Result<()> { let exit_signal = Arc::new(AtomicBool::new(false)); - let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); + let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(100_000); let proxy_listener = proxy_listener::ProxyListener::new( self.proxy_listener_addr, @@ -64,7 +64,8 @@ impl QuicForwardProxy { let exit_signal_clone = exit_signal.clone(); let quic_proxy = tokio::spawn(async move { - proxy_listener.listen(exit_signal_clone.clone(), forwarder_channel).await; + proxy_listener.listen(exit_signal_clone.clone(), forwarder_channel).await + .expect("proxy listen service"); }); let validator_identity = self.validator_identity.clone(); diff --git a/quic-forward-proxy/src/util.rs b/quic-forward-proxy/src/util.rs index 4b6b70fd..423a1f32 100644 --- a/quic-forward-proxy/src/util.rs +++ b/quic-forward-proxy/src/util.rs @@ -1,2 +1,14 @@ +use std::future::Future; +use std::time::Duration; +use futures::TryFutureExt; +use tokio::time::{Timeout, timeout}; pub type AnyhowJoinHandle = tokio::task::JoinHandle>; +pub const FALLBACK_TIMEOUT: Duration = Duration::from_secs(5); + +pub fn timeout_fallback(future: F) -> Timeout + where + F: Future, +{ + tokio::time::timeout(FALLBACK_TIMEOUT, future) +} From 0b718bc93460b6df6dc2b11dd24ab6f710626ec2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 10:48:52 +0200 Subject: [PATCH 064/128] cleanup mod dependencies --- .../src/inbound/proxy_listener.rs | 4 +- quic-forward-proxy/src/lib.rs | 17 +- quic-forward-proxy/src/main.rs | 16 +- quic-forward-proxy/src/outbound/mod.rs | 4 +- .../src/outbound/tpu_quic_client.rs | 340 ------------------ quic-forward-proxy/src/outbound/tx_forward.rs | 58 ++- quic-forward-proxy/src/proxy.rs | 23 +- .../src/{share => shared}/mod.rs | 0 .../src/test_client/quic_test_client.rs | 2 +- ...fig_provicer.rs => tls_config_provider.rs} | 0 .../src/{outbound => }/validator_identity.rs | 0 .../tests/proxy_request_format.rs | 6 - 12 files changed, 78 insertions(+), 392 deletions(-) delete mode 100644 quic-forward-proxy/src/outbound/tpu_quic_client.rs rename quic-forward-proxy/src/{share => shared}/mod.rs (100%) rename quic-forward-proxy/src/{tls_config_provicer.rs => tls_config_provider.rs} (100%) rename quic-forward-proxy/src/{outbound => }/validator_identity.rs (100%) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 17e8c110..02ba3066 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -8,8 +8,8 @@ use quinn::{Connection, Endpoint, ServerConfig, VarInt}; use solana_sdk::packet::PACKET_DATA_SIZE; use tokio::sync::mpsc::Sender; use crate::proxy_request_format::TpuForwardingRequest; -use crate::share::ForwardPacket; -use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; +use crate::shared::ForwardPacket; +use crate::tls_config_provider::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::FALLBACK_TIMEOUT; // TODO tweak this value - solana server sets 256 diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index 2f97f182..1520dffd 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -1,13 +1,16 @@ -pub mod quic_util; -pub mod tls_config_provicer; +// lib definition is only required for 'quic-forward-proxy-integration-test' to work + +mod quic_util; +pub mod tls_config_provider; pub mod proxy; -pub mod proxy_request_format; -pub mod cli; -pub mod test_client; +pub mod validator_identity; +mod proxy_request_format; +mod cli; +mod test_client; mod util; mod tx_store; mod quic_connection_utils; mod quinn_auto_reconnect; -pub mod outbound; +mod outbound; mod inbound; -mod share; +mod shared; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 20b81724..2b7827df 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -1,5 +1,4 @@ - - +use std::sync::Arc; use anyhow::bail; use clap::Parser; use dotenv::dotenv; @@ -7,12 +6,12 @@ use log::info; use crate::cli::{Args, get_identity_keypair}; use crate::proxy::QuicForwardProxy; -pub use tls_config_provicer::SelfSignedTlsConfigProvider; -use crate::outbound::validator_identity::ValidatorIdentity; +pub use tls_config_provider::SelfSignedTlsConfigProvider; +use crate::validator_identity::ValidatorIdentity; pub mod quic_util; -pub mod tls_config_provicer; +pub mod tls_config_provider; pub mod proxy; pub mod proxy_request_format; pub mod cli; @@ -23,7 +22,8 @@ mod quic_connection_utils; mod quinn_auto_reconnect; mod outbound; mod inbound; -mod share; +mod shared; +mod validator_identity; #[tokio::main(flavor = "multi_thread", worker_threads = 16)] @@ -43,8 +43,8 @@ pub async fn main() -> anyhow::Result<()> { let validator_identity = ValidatorIdentity::new(get_identity_keypair(&identity_keypair).await); - let tls_config = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); - let main_services = QuicForwardProxy::new(proxy_listener_addr, &tls_config, validator_identity) + let tls_config = Arc::new(SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost()); + let main_services = QuicForwardProxy::new(proxy_listener_addr, tls_config, validator_identity) .await? .start_services(); diff --git a/quic-forward-proxy/src/outbound/mod.rs b/quic-forward-proxy/src/outbound/mod.rs index 6894a6d0..30b80e1d 100644 --- a/quic-forward-proxy/src/outbound/mod.rs +++ b/quic-forward-proxy/src/outbound/mod.rs @@ -1,3 +1 @@ -mod tpu_quic_client; -pub mod validator_identity; -pub mod tx_forward; \ No newline at end of file +pub mod tx_forward; diff --git a/quic-forward-proxy/src/outbound/tpu_quic_client.rs b/quic-forward-proxy/src/outbound/tpu_quic_client.rs deleted file mode 100644 index f93eed22..00000000 --- a/quic-forward-proxy/src/outbound/tpu_quic_client.rs +++ /dev/null @@ -1,340 +0,0 @@ -use std::collections::{HashMap, VecDeque}; -use std::io::Write; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; - - -use std::sync::{Arc}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::time::Duration; -use anyhow::{anyhow}; -use async_trait::async_trait; - -use futures::future::join_all; -use itertools::{Itertools}; -use log::{debug, info, warn}; -use quinn::{Connection, Endpoint}; - - - - - - - -use solana_sdk::transaction::VersionedTransaction; - -use solana_streamer::tls_certificates::new_self_signed_tls_certificate; -use tokio::sync::RwLock; -use crate::outbound::validator_identity::ValidatorIdentity; -use crate::quic_connection_utils::{connection_stats, QuicConnectionError, QuicConnectionParameters, QuicConnectionUtils}; -use crate::quinn_auto_reconnect::AutoReconnect; - -const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); -pub const CONNECTION_RETRY_COUNT: usize = 10; - -pub const MAX_TRANSACTIONS_PER_BATCH: usize = 10; -pub const MAX_BYTES_PER_BATCH: usize = 10; -const MAX_PARALLEL_STREAMS: usize = 6; - -/// maintain many quic connection and streams to one TPU to send transactions - optimized for proxy use case -#[derive(Debug, Clone)] -pub struct TpuQuicClient { - endpoint: Endpoint, - // naive single non-recoverable connection - TODO moke it smarter - // TODO consider using DashMap again - connection_per_tpunode: Arc>>, - last_stable_id: Arc, -} - -impl TpuQuicClient { - - // note: this is a dirty workaround to expose enpoint to autoconnect class - pub fn get_endpoint(&self) -> Endpoint { - self.endpoint.clone() - } - - pub async fn create_connection(&self, tpu_address: SocketAddr) -> anyhow::Result { - let connection = - // TODO try 0rff - match QuicConnectionUtils::make_connection_0rtt( - self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) - .await { - Ok(conn) => conn, - Err(err) => { - warn!("Failed to open Quic connection to TPU {}: {}", tpu_address, err); - return Err(anyhow!("Failed to create Quic connection to TPU {}: {}", tpu_address, err)); - }, - }; - - Ok(connection) - } -} - -/// per TPU connection manager -#[async_trait] -pub trait SingleTPUConnectionManager { - // async fn refresh_connection(&self, connection: &Connection) -> Connection; - async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> anyhow::Result; - fn update_last_stable_id(&self, stable_id: u64); -} - -pub type SingleTPUConnectionManagerWrapper = dyn SingleTPUConnectionManager + Sync + Send; - -#[async_trait] -impl SingleTPUConnectionManager for TpuQuicClient { - - // make sure the connection is usable for a resonable time - // never returns the "same" instances but a clone - // async fn refresh_connection(&self, connection: &Connection) -> Connection { - // let reverse_lookup = self.connection_per_tpunode.into_read_only().values().find(|conn| { - // conn.stable_id() == connection.stable_id() - // }); - // - // match reverse_lookup { - // Some(existing_conn) => { - // return existing_conn.clone(); - // } - // None => { - // TpuQuicClient::create_new(&self, reverse_lookup).await.unwrap()) - // } - // } - // - // // TODO implement - // connection.clone() - // } - - #[tracing::instrument(skip(self), level = "debug")] - // TODO improve error handling; might need to signal if connection was reset - async fn get_or_create_connection(&self, tpu_address: SocketAddr) -> anyhow::Result { - // TODO try 0rff - // QuicConnectionUtils::make_connection( - // self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) - // .await.unwrap() - - - { - if let Some(conn) = self.connection_per_tpunode.read().await.get(&tpu_address) { - debug!("reusing connection {} for tpu {}; last_stable_id is {}", - conn.stable_id(), tpu_address, self.last_stable_id.load(Ordering::Relaxed)); - return Ok(conn.clone()); - } - } - - let connection = - // TODO try 0rff - match QuicConnectionUtils::make_connection_0rtt( - self.endpoint.clone(), tpu_address, QUIC_CONNECTION_TIMEOUT) - .await { - Ok(conn) => conn, - Err(err) => { - warn!("Failed to open Quic connection to TPU {}: {}", tpu_address, err); - return Err(anyhow!("Failed to create Quic connection to TPU {}: {}", tpu_address, err)); - }, - }; - - let mut lock = self.connection_per_tpunode.write().await; - let old_value = lock.insert(tpu_address, connection.clone()); - assert!(old_value.is_none(), "no prev value must be overridden"); - - debug!("Created new Quic connection {} to TPU node {}, total connections is now {}", - connection.stable_id(), tpu_address, lock.len()); - return Ok(connection); - } - - fn update_last_stable_id(&self, stable_id: u64) { - self.last_stable_id.store(stable_id, Ordering::Relaxed); - } - -} - -impl TpuQuicClient { - - /// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU - // note: ATM the provided identity might or might not be a valid validator keypair - pub async fn new_with_validator_identity_delme(validator_identity: ValidatorIdentity) -> TpuQuicClient { - info!("Setup TPU Quic stable connection with validator identity {} ...", validator_identity); - let (certificate, key) = new_self_signed_tls_certificate( - &validator_identity.get_keypair_for_tls(), - IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), - ) - .expect("Failed to initialize QUIC connection certificates"); - - let endpoint_outbound = QuicConnectionUtils::create_tpu_client_endpoint(certificate.clone(), key.clone()); - - let active_tpu_connection = TpuQuicClient { - endpoint: endpoint_outbound.clone(), - connection_per_tpunode: Arc::new(RwLock::new(HashMap::new())), - last_stable_id: Arc::new(AtomicU64::new(0)), - }; - - active_tpu_connection - } - - #[tracing::instrument(skip_all, level = "debug")] - pub async fn send_txs_to_tpu(&self, - tpu_address: SocketAddr, - txs: &Vec, - exit_signal: Arc, - ) { - - if false { - // note: this impl does not deal with connection errors - // throughput_50 493.70 tps - // throughput_50 769.43 tps (with finish timeout) - // TODO join get_or_create_connection future and read_to_end - // TODO add error handling - let tpu_connection = self.get_or_create_connection(tpu_address).await.unwrap(); - - for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { - let vecvec = chunk.iter().map(|tx| { - let tx_raw = bincode::serialize(tx).unwrap(); - tx_raw - }).collect_vec(); - QuicConnectionUtils::send_transaction_batch_parallel( - tpu_connection.clone(), - vecvec, - exit_signal.clone(), - QUIC_CONNECTION_TIMEOUT, - ).await; - } - } else { - // throughput_50 676.65 tps - let connection_params = QuicConnectionParameters { - connection_retry_count: 10, - finalize_timeout: Duration::from_millis(200), - unistream_timeout: Duration::from_millis(500), - write_timeout: Duration::from_secs(1), - }; - - let connection_manager = self as &SingleTPUConnectionManagerWrapper; - - // TODO connection_params should be part of connection_manager - Self::send_transaction_batch(serialize_to_vecvec(&txs), tpu_address, exit_signal, connection_params, connection_manager).await; - - } - - } - - pub async fn send_transaction_batch(txs: Vec>, - tpu_address: SocketAddr, - exit_signal: Arc, - // _timeout_counters: Arc, - // last_stable_id: Arc, - connection_params: QuicConnectionParameters, - connection_manager: &SingleTPUConnectionManagerWrapper, - ) { - let mut queue = VecDeque::new(); - for tx in txs { - queue.push_back(tx); - } - info!("send_transaction_batch: queue size is {}", queue.len()); - let connection_retry_count = connection_params.connection_retry_count; - for _ in 0..connection_retry_count { - if queue.is_empty() || exit_signal.load(Ordering::Relaxed) { - // return - return; - } - - let mut do_retry = false; - while !queue.is_empty() { - let tx = queue.pop_front().unwrap(); - let connection = connection_manager.get_or_create_connection(tpu_address).await; - - if exit_signal.load(Ordering::Relaxed) { - return; - } - - if let Ok(connection) = connection { - let current_stable_id = connection.stable_id() as u64; - match QuicConnectionUtils::open_unistream( - &connection, - connection_params.unistream_timeout, - ) - .await - { - Ok(send_stream) => { - match QuicConnectionUtils::write_all( - send_stream, - &tx, - connection_params, - ) - .await - { - Ok(()) => { - // do nothing - debug!("connection stats (proxy send tx batch): {}", connection_stats(&connection)); - } - Err(QuicConnectionError::ConnectionError { retry }) => { - do_retry = retry; - } - Err(QuicConnectionError::TimeOut) => { - // timeout_counters.fetch_add(1, Ordering::Relaxed); - } - } - } - Err(QuicConnectionError::ConnectionError { retry }) => { - do_retry = retry; - } - Err(QuicConnectionError::TimeOut) => { - // timeout_counters.fetch_add(1, Ordering::Relaxed); - } - } - if do_retry { - connection_manager.update_last_stable_id(current_stable_id); - - queue.push_back(tx); - break; - } - } else { - warn!( - "Could not establish connection with {}", - "identity" - ); - break; - } - } - if !do_retry { - break; - } - } - } - -} - - -fn serialize_to_vecvec(transactions: &Vec) -> Vec> { - transactions.iter().map(|tx| { - let tx_raw = bincode::serialize(tx).unwrap(); - tx_raw - }).collect_vec() -} - - -// send potentially large amount of transactions to a single TPU -#[tracing::instrument(skip_all, level = "debug")] -pub async fn send_txs_to_tpu_static( - auto_connection: &AutoReconnect, - txs: &Vec, -) { - - // note: this impl does not deal with connection errors - // throughput_50 493.70 tps - // throughput_50 769.43 tps (with finish timeout) - // TODO join get_or_create_connection future and read_to_end - // TODO add error handling - - for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { - let all_send_fns = chunk.iter().map(|tx| { - let tx_raw = bincode::serialize(tx).unwrap(); - tx_raw - }) - .map(|tx_raw| { - auto_connection.send(tx_raw) // ignores error - }); - - // let all_send_fns = (0..txs.len()).map(|i| auto_connection.roundtrip(vecvec.get(i))).collect_vec(); - - join_all(all_send_fns).await; - - } - -} diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index ff3762ed..3a017b81 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -5,15 +5,17 @@ use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use fan::tokio::mpsc::FanOut; use std::time::Duration; +use futures::future::join_all; +use itertools::Itertools; use quinn::Endpoint; +use solana_sdk::transaction::VersionedTransaction; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::mpsc::{channel, Receiver}; use tokio::time::timeout; -use crate::outbound::tpu_quic_client::{send_txs_to_tpu_static, TpuQuicClient}; -use crate::outbound::validator_identity::ValidatorIdentity; use crate::quic_connection_utils::QuicConnectionUtils; use crate::quinn_auto_reconnect::AutoReconnect; -use crate::share::ForwardPacket; +use crate::shared::ForwardPacket; +use crate::validator_identity::ValidatorIdentity; // takes transactions from upstream clients and forwards them to the TPU pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction_channel: Receiver, exit_signal: Arc) -> anyhow::Result<()> { @@ -134,4 +136,52 @@ async fn new_endpoint_with_validator_identity(validator_identity: ValidatorIdent let endpoint_outbound = QuicConnectionUtils::create_tpu_client_endpoint(certificate.clone(), key.clone()); endpoint_outbound -} \ No newline at end of file +} + + +const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); +pub const CONNECTION_RETRY_COUNT: usize = 10; + +pub const MAX_TRANSACTIONS_PER_BATCH: usize = 10; +pub const MAX_BYTES_PER_BATCH: usize = 10; +const MAX_PARALLEL_STREAMS: usize = 6; + + + +fn serialize_to_vecvec(transactions: &Vec) -> Vec> { + transactions.iter().map(|tx| { + let tx_raw = bincode::serialize(tx).unwrap(); + tx_raw + }).collect_vec() +} + + +// send potentially large amount of transactions to a single TPU +#[tracing::instrument(skip_all, level = "debug")] +async fn send_txs_to_tpu_static( + auto_connection: &AutoReconnect, + txs: &Vec, +) { + + // note: this impl does not deal with connection errors + // throughput_50 493.70 tps + // throughput_50 769.43 tps (with finish timeout) + // TODO join get_or_create_connection future and read_to_end + // TODO add error handling + + for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { + let all_send_fns = chunk.iter().map(|tx| { + let tx_raw = bincode::serialize(tx).unwrap(); + tx_raw + }) + .map(|tx_raw| { + auto_connection.send(tx_raw) // ignores error + }); + + // let all_send_fns = (0..txs.len()).map(|i| auto_connection.roundtrip(vecvec.get(i))).collect_vec(); + + join_all(all_send_fns).await; + + } + +} diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 515b4fe1..431d0fba 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -2,33 +2,14 @@ use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::AtomicBool; - - -use std::time::Duration; - use anyhow::{anyhow, bail, Context}; use log::{debug, error, info, trace}; -use quinn::{Connection, Endpoint, ServerConfig, VarInt}; - - - - -use solana_sdk::packet::PACKET_DATA_SIZE; - - -use tokio::sync::mpsc::Sender; use crate::inbound::proxy_listener; use crate::outbound::tx_forward::tx_forwarder; -use crate::outbound::validator_identity::ValidatorIdentity; - - -use crate::proxy_request_format::TpuForwardingRequest; -use crate::share::ForwardPacket; - -use crate::tls_config_provicer::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; +use crate::tls_config_provider::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; use crate::util::AnyhowJoinHandle; - +use crate::validator_identity::ValidatorIdentity; pub struct QuicForwardProxy { diff --git a/quic-forward-proxy/src/share/mod.rs b/quic-forward-proxy/src/shared/mod.rs similarity index 100% rename from quic-forward-proxy/src/share/mod.rs rename to quic-forward-proxy/src/shared/mod.rs diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index 56a4fd99..fbd01a57 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -13,7 +13,7 @@ use tokio::io::AsyncWriteExt; use crate::proxy_request_format::TpuForwardingRequest; use crate::quic_connection_utils::SkipServerVerification; use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; -use crate::tls_config_provicer::ProxyTlsConfigProvider; +use crate::tls_config_provider::ProxyTlsConfigProvider; use crate::util::AnyhowJoinHandle; diff --git a/quic-forward-proxy/src/tls_config_provicer.rs b/quic-forward-proxy/src/tls_config_provider.rs similarity index 100% rename from quic-forward-proxy/src/tls_config_provicer.rs rename to quic-forward-proxy/src/tls_config_provider.rs diff --git a/quic-forward-proxy/src/outbound/validator_identity.rs b/quic-forward-proxy/src/validator_identity.rs similarity index 100% rename from quic-forward-proxy/src/outbound/validator_identity.rs rename to quic-forward-proxy/src/validator_identity.rs diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index c9e7ac0f..a7622782 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -1,14 +1,8 @@ use std::str::FromStr; - - - - - use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, Signer}; use solana_sdk::transaction::{Transaction}; - use solana_lite_rpc_quic_forward_proxy::proxy_request_format::TpuForwardingRequest; #[test] From 40bdae5066b3bcb540131ae28263ebc80cef79ab Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 11:09:49 +0200 Subject: [PATCH 065/128] cleanup unused quic code --- quic-forward-proxy/src/lib.rs | 4 +- quic-forward-proxy/src/main.rs | 4 +- quic-forward-proxy/src/outbound/tx_forward.rs | 2 +- .../src/quic_connection_utils.rs | 395 ------------------ .../src/test_client/quic_test_client.rs | 2 +- quic-forward-proxy/src/tls_config_provider.rs | 2 +- .../src/tpu_quic_connection_utils.rs | 115 +++++ 7 files changed, 122 insertions(+), 402 deletions(-) delete mode 100644 quic-forward-proxy/src/quic_connection_utils.rs create mode 100644 quic-forward-proxy/src/tpu_quic_connection_utils.rs diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index 1520dffd..c40e4f0d 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -4,12 +4,12 @@ mod quic_util; pub mod tls_config_provider; pub mod proxy; pub mod validator_identity; -mod proxy_request_format; +pub mod proxy_request_format; mod cli; mod test_client; mod util; mod tx_store; -mod quic_connection_utils; +mod tpu_quic_connection_utils; mod quinn_auto_reconnect; mod outbound; mod inbound; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 2b7827df..654c180a 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -18,7 +18,7 @@ pub mod cli; pub mod test_client; mod util; mod tx_store; -mod quic_connection_utils; +mod tpu_quic_connection_utils; mod quinn_auto_reconnect; mod outbound; mod inbound; @@ -37,7 +37,7 @@ pub async fn main() -> anyhow::Result<()> { dotenv().ok(); - // TODO build args struct dedicyted to proxy + // TODO build args struct dedicated to proxy let proxy_listener_addr = proxy_rpc_addr.parse().unwrap(); let _tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let validator_identity = diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 3a017b81..af137c73 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -12,7 +12,7 @@ use solana_sdk::transaction::VersionedTransaction; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::mpsc::{channel, Receiver}; use tokio::time::timeout; -use crate::quic_connection_utils::QuicConnectionUtils; +use crate::tpu_quic_connection_utils::QuicConnectionUtils; use crate::quinn_auto_reconnect::AutoReconnect; use crate::shared::ForwardPacket; use crate::validator_identity::ValidatorIdentity; diff --git a/quic-forward-proxy/src/quic_connection_utils.rs b/quic-forward-proxy/src/quic_connection_utils.rs deleted file mode 100644 index 7f4b9aa6..00000000 --- a/quic-forward-proxy/src/quic_connection_utils.rs +++ /dev/null @@ -1,395 +0,0 @@ -use log::{debug, error, trace, warn}; -use quinn::{ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, VarInt}; -use solana_sdk::pubkey::Pubkey; -use std::{ - net::{IpAddr, Ipv4Addr, SocketAddr}, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - time::Duration, -}; - -use futures::future::join_all; -use itertools::Itertools; -use solana_sdk::quic::QUIC_MAX_TIMEOUT_MS; -use tokio::{time::timeout}; - - - -const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; - -pub struct QuicConnectionUtils {} - -pub enum QuicConnectionError { - TimeOut, - ConnectionError { retry: bool }, -} - -// TODO check whot we need from this -#[derive(Clone, Copy)] -pub struct QuicConnectionParameters { - // pub connection_timeout: Duration, - pub unistream_timeout: Duration, - pub write_timeout: Duration, - pub finalize_timeout: Duration, - pub connection_retry_count: usize, - // pub max_number_of_connections: usize, - // pub number_of_transactions_per_unistream: usize, -} - -impl QuicConnectionUtils { - // TODO move to a more specific place - pub fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { - let mut endpoint = { - let client_socket = - solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::UNSPECIFIED), (8000, 10000)) - .expect("create_endpoint bind_in_range") - .1; - let config = EndpointConfig::default(); - quinn::Endpoint::new(config, None, client_socket, TokioRuntime) - .expect("create_endpoint quinn::Endpoint::new") - }; - - let mut crypto = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(SkipServerVerification::new()) - .with_single_cert(vec![certificate], key) - .expect("Failed to set QUIC client certificates"); - - crypto.enable_early_data = true; - - crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; - - let mut config = ClientConfig::new(Arc::new(crypto)); - - // note: this should be aligned with solana quic server's endpoint config - let mut transport_config = TransportConfig::default(); - // no remotely-initiated streams required - transport_config.max_concurrent_uni_streams(VarInt::from_u32(0)); - transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); - let timeout = IdleTimeout::try_from(Duration::from_millis(QUIC_MAX_TIMEOUT_MS as u64)).unwrap(); - transport_config.max_idle_timeout(Some(timeout)); - transport_config.keep_alive_interval(None); - config.transport_config(Arc::new(transport_config)); - - endpoint.set_default_client_config(config); - - endpoint - } - - pub async fn make_connection( - endpoint: Endpoint, - addr: SocketAddr, - connection_timeout: Duration, - ) -> anyhow::Result { - let connecting = endpoint.connect(addr, "connect")?; - let res = timeout(connection_timeout, connecting).await??; - Ok(res) - } - - pub async fn make_connection_0rtt( - endpoint: Endpoint, - addr: SocketAddr, - connection_timeout: Duration, - ) -> anyhow::Result { - let connecting = endpoint.connect(addr, "connect")?; - let connection = match connecting.into_0rtt() { - Ok((connection, zero_rtt)) => { - if (timeout(connection_timeout, zero_rtt).await).is_ok() { - connection - } else { - return Err(ConnectionError::TimedOut.into()); - } - } - Err(connecting) => { - if let Ok(connecting_result) = timeout(connection_timeout, connecting).await { - connecting_result? - } else { - return Err(ConnectionError::TimedOut.into()); - } - } - }; - Ok(connection) - } - - #[allow(clippy::too_many_arguments)] - pub async fn connect( - identity: Pubkey, - already_connected: bool, - endpoint: Endpoint, - tpu_address: SocketAddr, - connection_timeout: Duration, - connection_retry_count: usize, - exit_signal: Arc, - on_connect: fn(), - ) -> Option { - for _ in 0..connection_retry_count { - let conn = if already_connected { - Self::make_connection_0rtt(endpoint.clone(), tpu_address, connection_timeout).await - } else { - Self::make_connection(endpoint.clone(), tpu_address, connection_timeout).await - }; - match conn { - Ok(conn) => { - on_connect(); - return Some(conn); - } - Err(e) => { - warn!("Could not connect to tpu {}/{}, error: {}", tpu_address, identity, e); - if exit_signal.load(Ordering::Relaxed) { - break; - } - } - } - } - None - } - - pub async fn write_all( - mut send_stream: SendStream, - tx: &Vec, - // identity: Pubkey, - connection_params: QuicConnectionParameters, - ) -> Result<(), QuicConnectionError> { - let write_timeout_res = timeout( - connection_params.write_timeout, - send_stream.write_all(tx.as_slice()), - ) - .await; - match write_timeout_res { - Ok(write_res) => { - if let Err(e) = write_res { - trace!( - "Error while writing transaction for {}, error {}", - "identity", - e - ); - return Err(QuicConnectionError::ConnectionError { retry: true }); - } - } - Err(_) => { - warn!("timeout while writing transaction for {}", "identity"); - return Err(QuicConnectionError::TimeOut); - } - } - - let finish_timeout_res = - timeout(connection_params.finalize_timeout, send_stream.finish()).await; - match finish_timeout_res { - Ok(finish_res) => { - if let Err(e) = finish_res { - trace!( - "Error while finishing transaction for {}, error {}", - "identity", - e - ); - return Err(QuicConnectionError::ConnectionError { retry: false }); - } - } - Err(_) => { - warn!("timeout while finishing transaction for {}", "identity"); - return Err(QuicConnectionError::TimeOut); - } - } - - Ok(()) - } - - pub async fn write_all_simple( - send_stream: &mut SendStream, - tx: &Vec, - connection_timeout: Duration, - ) { - let write_timeout_res = - timeout(connection_timeout, send_stream.write_all(tx.as_slice())).await; - match write_timeout_res { - Ok(write_res) => { - if let Err(e) = write_res { - trace!( - "Error while writing transaction for TBD, error {}", - // identity, // TODO add more context - e - ); - return; - } - } - Err(_) => { - warn!("timeout while writing transaction for TBD"); // TODO add more context - panic!("TODO handle timeout"); // FIXME - } - } - - let finish_timeout_res = timeout(connection_timeout, send_stream.finish()).await; - match finish_timeout_res { - Ok(finish_res) => { - if let Err(e) = finish_res { - // last_stable_id.store(connection_stable_id, Ordering::Relaxed); - trace!( - "Error while writing transaction for TBD, error {}", - // identity, - e - ); - return; - } - } - Err(_) => { - warn!("timeout while finishing transaction for TBD"); // TODO - panic!("TODO handle timeout"); // FIXME - } - } - - } - - pub async fn open_unistream( - connection: &Connection, - connection_timeout: Duration, - ) -> Result { - match timeout(connection_timeout, connection.open_uni()).await { - Ok(Ok(unistream)) => Ok(unistream), - Ok(Err(_)) => Err(QuicConnectionError::ConnectionError { retry: true }), - Err(_) => Err(QuicConnectionError::TimeOut), - } - } - - pub async fn open_unistream_simple( - connection: Connection, - connection_timeout: Duration, - ) -> (Option, bool) { - match timeout(connection_timeout, connection.open_uni()).await { - Ok(Ok(unistream)) => (Some(unistream), false), - Ok(Err(_)) => { - // reset connection for next retry - (None, true) - } - // timeout - Err(_) => (None, false), - } - } - - - #[allow(clippy::too_many_arguments)] - #[tracing::instrument(skip_all, level = "debug")] - pub async fn send_transaction_batch_serial( - connection: Connection, - txs: Vec>, - exit_signal: Arc, - connection_timeout: Duration, - ) { - let (mut stream, _retry_conn) = - Self::open_unistream_simple(connection.clone(), connection_timeout) - .await; - if let Some(ref mut send_stream) = stream { - if exit_signal.load(Ordering::Relaxed) { - return; - } - - for tx in txs { - let write_timeout_res = - timeout(connection_timeout, send_stream.write_all(tx.as_slice())).await; - match write_timeout_res { - Ok(no_timeout) => { - match no_timeout { - Ok(()) => {} - Err(write_error) => { - error!("Error writing transaction to stream: {}", write_error); - } - } - } - Err(_elapsed) => { - warn!("timeout sending transactions"); - } - } - - - } - // TODO wrap in timeout - stream.unwrap().finish().await.unwrap(); - - } else { - panic!("no retry handling"); // FIXME - } - } - - // open streams in parallel - // one stream is used for one transaction - // number of parallel streams that connect to TPU must be limited by caller (should be 8) - #[allow(clippy::too_many_arguments)] - #[tracing::instrument(skip_all, level = "debug")] - pub async fn send_transaction_batch_parallel( - connection: Connection, - txs: Vec>, - _exit_signal: Arc, - connection_timeout: Duration, - ) { - assert_ne!(txs.len(), 0, "no transactions to send"); - debug!("Opening {} parallel quic streams", txs.len()); - - let all_send_fns = (0..txs.len()).map(|i| Self::send_tx_to_new_stream(&txs[i], connection.clone(), connection_timeout)).collect_vec(); - - join_all(all_send_fns).await; - - debug!("connection stats (proxy send tx parallel): {}", connection_stats(&connection)); - } - - - async fn send_tx_to_new_stream(tx: &Vec, connection: Connection, connection_timeout: Duration) { - let mut send_stream = Self::open_unistream_simple(connection.clone(), connection_timeout) - .await.0 - .unwrap(); - - let write_timeout_res = - timeout(connection_timeout, send_stream.write_all(tx.as_slice())).await; - match write_timeout_res { - Ok(no_timeout) => { - match no_timeout { - Ok(()) => {} - Err(write_error) => { - error!("Error writing transaction to stream: {}", write_error); - } - } - } - Err(_elapsed) => { - warn!("timeout sending transactions"); - } - } - - // TODO wrap in small timeout - let _ = timeout(Duration::from_millis(200), send_stream.finish()).await; - - } -} - -pub struct SkipServerVerification; - -impl SkipServerVerification { - pub fn new() -> Arc { - Arc::new(Self) - } -} - -impl rustls::client::ServerCertVerifier for SkipServerVerification { - fn verify_server_cert( - &self, - _end_entity: &rustls::Certificate, - _intermediates: &[rustls::Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, - _ocsp_response: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) - } -} - -// stable_id 140266619216912, rtt=2.156683ms, -// stats FrameStats { ACK: 3, CONNECTION_CLOSE: 0, CRYPTO: 3, -// DATA_BLOCKED: 0, DATAGRAM: 0, HANDSHAKE_DONE: 1, MAX_DATA: 0, -// MAX_STREAM_DATA: 1, MAX_STREAMS_BIDI: 0, MAX_STREAMS_UNI: 0, NEW_CONNECTION_ID: 4, -// NEW_TOKEN: 0, PATH_CHALLENGE: 0, PATH_RESPONSE: 0, PING: 0, RESET_STREAM: 0, -// RETIRE_CONNECTION_ID: 1, STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, -// STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } -pub fn connection_stats(connection: &Connection) -> String { - format!("stable_id {} stats {:?}, rtt={:?}", - connection.stable_id(), connection.stats().frame_rx, connection.stats().path.rtt) -} \ No newline at end of file diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index fbd01a57..672d9ab5 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -11,7 +11,7 @@ use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::{Transaction, VersionedTransaction}; use tokio::io::AsyncWriteExt; use crate::proxy_request_format::TpuForwardingRequest; -use crate::quic_connection_utils::SkipServerVerification; +use crate::tpu_quic_connection_utils::SkipServerVerification; use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; use crate::tls_config_provider::ProxyTlsConfigProvider; diff --git a/quic-forward-proxy/src/tls_config_provider.rs b/quic-forward-proxy/src/tls_config_provider.rs index 30c219ca..4b8b3177 100644 --- a/quic-forward-proxy/src/tls_config_provider.rs +++ b/quic-forward-proxy/src/tls_config_provider.rs @@ -1,7 +1,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use rcgen::generate_simple_self_signed; use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; -use crate::quic_connection_utils::SkipServerVerification; +use crate::tpu_quic_connection_utils::SkipServerVerification; use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; // TODO integrate with tpu_service + quic_connection_utils diff --git a/quic-forward-proxy/src/tpu_quic_connection_utils.rs b/quic-forward-proxy/src/tpu_quic_connection_utils.rs new file mode 100644 index 00000000..9443cf3c --- /dev/null +++ b/quic-forward-proxy/src/tpu_quic_connection_utils.rs @@ -0,0 +1,115 @@ +use log::{debug, error, trace, warn}; +use quinn::{ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, VarInt}; +use solana_sdk::pubkey::Pubkey; +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; + +use futures::future::join_all; +use itertools::Itertools; +use solana_sdk::quic::QUIC_MAX_TIMEOUT_MS; +use tokio::{time::timeout}; + + + +const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; + +pub struct QuicConnectionUtils {} + +pub enum QuicConnectionError { + TimeOut, + ConnectionError { retry: bool }, +} + +// TODO check whot we need from this +#[derive(Clone, Copy)] +pub struct QuicConnectionParameters { + // pub connection_timeout: Duration, + pub unistream_timeout: Duration, + pub write_timeout: Duration, + pub finalize_timeout: Duration, + pub connection_retry_count: usize, + // pub max_number_of_connections: usize, + // pub number_of_transactions_per_unistream: usize, +} + +impl QuicConnectionUtils { + // TODO move to a more specific place + pub fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { + let mut endpoint = { + let client_socket = + solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::UNSPECIFIED), (8000, 10000)) + .expect("create_endpoint bind_in_range") + .1; + let config = EndpointConfig::default(); + quinn::Endpoint::new(config, None, client_socket, TokioRuntime) + .expect("create_endpoint quinn::Endpoint::new") + }; + + let mut crypto = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(SkipServerVerification::new()) + .with_single_cert(vec![certificate], key) + .expect("Failed to set QUIC client certificates"); + + crypto.enable_early_data = true; + + crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; + + let mut config = ClientConfig::new(Arc::new(crypto)); + + // note: this should be aligned with solana quic server's endpoint config + let mut transport_config = TransportConfig::default(); + // no remotely-initiated streams required + transport_config.max_concurrent_uni_streams(VarInt::from_u32(0)); + transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); + let timeout = IdleTimeout::try_from(Duration::from_millis(QUIC_MAX_TIMEOUT_MS as u64)).unwrap(); + transport_config.max_idle_timeout(Some(timeout)); + transport_config.keep_alive_interval(None); + config.transport_config(Arc::new(transport_config)); + + endpoint.set_default_client_config(config); + + endpoint + } + +} + +pub struct SkipServerVerification; + +impl SkipServerVerification { + pub fn new() -> Arc { + Arc::new(Self) + } +} + +impl rustls::client::ServerCertVerifier for SkipServerVerification { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &rustls::ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: std::time::SystemTime, + ) -> Result { + Ok(rustls::client::ServerCertVerified::assertion()) + } +} + +// stable_id 140266619216912, rtt=2.156683ms, +// stats FrameStats { ACK: 3, CONNECTION_CLOSE: 0, CRYPTO: 3, +// DATA_BLOCKED: 0, DATAGRAM: 0, HANDSHAKE_DONE: 1, MAX_DATA: 0, +// MAX_STREAM_DATA: 1, MAX_STREAMS_BIDI: 0, MAX_STREAMS_UNI: 0, NEW_CONNECTION_ID: 4, +// NEW_TOKEN: 0, PATH_CHALLENGE: 0, PATH_RESPONSE: 0, PING: 0, RESET_STREAM: 0, +// RETIRE_CONNECTION_ID: 1, STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, +// STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } +pub fn connection_stats(connection: &Connection) -> String { + format!("stable_id {} stats {:?}, rtt={:?}", + connection.stable_id(), connection.stats().frame_rx, connection.stats().path.rtt) +} \ No newline at end of file From ff803fd4ef28755534e528d6da1c1ef9c5cbf042 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 11:14:19 +0200 Subject: [PATCH 066/128] minor rename --- quic-forward-proxy/src/tls_config_provider.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quic-forward-proxy/src/tls_config_provider.rs b/quic-forward-proxy/src/tls_config_provider.rs index 4b8b3177..cef10100 100644 --- a/quic-forward-proxy/src/tls_config_provider.rs +++ b/quic-forward-proxy/src/tls_config_provider.rs @@ -44,7 +44,7 @@ impl SelfSignedTlsConfigProvider { hostnames, certificate, private_key, - client_crypto: Self::build_client_crypto(), + client_crypto: Self::build_client_crypto_insecure(), server_crypto: server_crypto, } } @@ -55,7 +55,7 @@ impl SelfSignedTlsConfigProvider { (Certificate(cert.serialize_der().unwrap()), PrivateKey(key)) } - fn build_client_crypto() -> ClientConfig { + fn build_client_crypto_insecure() -> ClientConfig { let mut client_crypto = rustls::ClientConfig::builder() .with_safe_defaults() // .with_root_certificates(roots) From 9bfc9ac33ef1dc18d869e29093684637a46f85a8 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 11:30:53 +0200 Subject: [PATCH 067/128] split tls client/server/self signed --- .../tests/quic_proxy_tpu_integrationtest.rs | 4 ++-- .../src/inbound/proxy_listener.rs | 3 ++- quic-forward-proxy/src/lib.rs | 4 +++- quic-forward-proxy/src/main.rs | 6 ++++-- quic-forward-proxy/src/proxy.rs | 2 +- .../src/test_client/quic_test_client.rs | 6 +++--- .../src/tls_config_provider_client.rs | 13 ++++++++++++ .../src/tls_config_provider_server.rs | 13 ++++++++++++ ...r.rs => tls_self_signed_pair_generator.rs} | 20 ++++++++----------- 9 files changed, 49 insertions(+), 22 deletions(-) create mode 100644 quic-forward-proxy/src/tls_config_provider_client.rs create mode 100644 quic-forward-proxy/src/tls_config_provider_server.rs rename quic-forward-proxy/src/{tls_config_provider.rs => tls_self_signed_pair_generator.rs} (93%) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 98245dcd..3f4a7a51 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -39,9 +39,9 @@ use tokio::time::{sleep}; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::fmt::format::FmtSpan; -use solana_lite_rpc_quic_forward_proxy::outbound::validator_identity::ValidatorIdentity; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; -use solana_lite_rpc_quic_forward_proxy::tls_config_provicer::SelfSignedTlsConfigProvider; +use solana_lite_rpc_quic_forward_proxy::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; +use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; #[derive(Copy, Clone, Debug)] diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 02ba3066..12720f11 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -9,7 +9,8 @@ use solana_sdk::packet::PACKET_DATA_SIZE; use tokio::sync::mpsc::Sender; use crate::proxy_request_format::TpuForwardingRequest; use crate::shared::ForwardPacket; -use crate::tls_config_provider::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; +use crate::tls_config_provider_server::ProxyTlsConfigProvider; +use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use crate::util::FALLBACK_TIMEOUT; // TODO tweak this value - solana server sets 256 diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index c40e4f0d..d03d5099 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -1,7 +1,9 @@ // lib definition is only required for 'quic-forward-proxy-integration-test' to work mod quic_util; -pub mod tls_config_provider; +pub mod tls_config_provider_client; +pub mod tls_config_provider_server; +pub mod tls_self_signed_pair_generator; pub mod proxy; pub mod validator_identity; pub mod proxy_request_format; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 654c180a..0817f533 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -5,13 +5,15 @@ use dotenv::dotenv; use log::info; use crate::cli::{Args, get_identity_keypair}; use crate::proxy::QuicForwardProxy; +use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; -pub use tls_config_provider::SelfSignedTlsConfigProvider; use crate::validator_identity::ValidatorIdentity; pub mod quic_util; -pub mod tls_config_provider; +pub mod tls_config_provider_client; +pub mod tls_config_provider_server; +pub mod tls_self_signed_pair_generator; pub mod proxy; pub mod proxy_request_format; pub mod cli; diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 431d0fba..c826315a 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -7,7 +7,7 @@ use anyhow::{anyhow, bail, Context}; use log::{debug, error, info, trace}; use crate::inbound::proxy_listener; use crate::outbound::tx_forward::tx_forwarder; -use crate::tls_config_provider::{ProxyTlsConfigProvider, SelfSignedTlsConfigProvider}; +use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use crate::util::AnyhowJoinHandle; use crate::validator_identity::ValidatorIdentity; diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index 672d9ab5..c6204dcf 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -13,8 +13,8 @@ use tokio::io::AsyncWriteExt; use crate::proxy_request_format::TpuForwardingRequest; use crate::tpu_quic_connection_utils::SkipServerVerification; use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; -use crate::tls_config_provider::ProxyTlsConfigProvider; - +use crate::tls_config_provider_server::ProxyTlsConfigProvider; +use crate::tls_config_provider_client::TpuCLientTlsConfigProvider; use crate::util::AnyhowJoinHandle; pub struct QuicTestClient { @@ -25,7 +25,7 @@ pub struct QuicTestClient { impl QuicTestClient { pub async fn new_with_endpoint( proxy_addr: SocketAddr, - tls_config: &impl ProxyTlsConfigProvider + tls_config: &impl TpuCLientTlsConfigProvider ) -> anyhow::Result { let client_crypto = tls_config.get_client_tls_crypto_config(); let mut endpoint = quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())?; diff --git a/quic-forward-proxy/src/tls_config_provider_client.rs b/quic-forward-proxy/src/tls_config_provider_client.rs new file mode 100644 index 00000000..7d7240e0 --- /dev/null +++ b/quic-forward-proxy/src/tls_config_provider_client.rs @@ -0,0 +1,13 @@ +use std::sync::atomic::{AtomicU32, Ordering}; +use rcgen::generate_simple_self_signed; +use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; +use crate::tpu_quic_connection_utils::SkipServerVerification; +use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; + +// TODO integrate with tpu_service + quic_connection_utils + +pub trait TpuCLientTlsConfigProvider { + + fn get_client_tls_crypto_config(&self) -> ClientConfig; + +} diff --git a/quic-forward-proxy/src/tls_config_provider_server.rs b/quic-forward-proxy/src/tls_config_provider_server.rs new file mode 100644 index 00000000..4e38c270 --- /dev/null +++ b/quic-forward-proxy/src/tls_config_provider_server.rs @@ -0,0 +1,13 @@ +use std::sync::atomic::{AtomicU32, Ordering}; +use rcgen::generate_simple_self_signed; +use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; +use crate::tpu_quic_connection_utils::SkipServerVerification; +use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; + +// TODO integrate with tpu_service + quic_connection_utils + +pub trait ProxyTlsConfigProvider { + + fn get_server_tls_crypto_config(&self) -> ServerConfig; + +} diff --git a/quic-forward-proxy/src/tls_config_provider.rs b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs similarity index 93% rename from quic-forward-proxy/src/tls_config_provider.rs rename to quic-forward-proxy/src/tls_self_signed_pair_generator.rs index cef10100..c50236ce 100644 --- a/quic-forward-proxy/src/tls_config_provider.rs +++ b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs @@ -3,24 +3,19 @@ use rcgen::generate_simple_self_signed; use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; use crate::tpu_quic_connection_utils::SkipServerVerification; use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; +use crate::tls_config_provider_client::TpuCLientTlsConfigProvider; +use crate::tls_config_provider_server::ProxyTlsConfigProvider; -// TODO integrate with tpu_service + quic_connection_utils - -pub trait ProxyTlsConfigProvider { - - fn get_client_tls_crypto_config(&self) -> ClientConfig; - fn get_server_tls_crypto_config(&self) -> ServerConfig; - +impl ProxyTlsConfigProvider for SelfSignedTlsConfigProvider { + fn get_server_tls_crypto_config(&self) -> ServerConfig { + self.server_crypto.clone() + } } -impl ProxyTlsConfigProvider for SelfSignedTlsConfigProvider { +impl TpuCLientTlsConfigProvider for SelfSignedTlsConfigProvider { fn get_client_tls_crypto_config(&self) -> ClientConfig { self.client_crypto.clone() } - - fn get_server_tls_crypto_config(&self) -> ServerConfig { - self.server_crypto.clone() - } } pub struct SelfSignedTlsConfigProvider { @@ -85,3 +80,4 @@ impl SelfSignedTlsConfigProvider { } + From 949646c5171ca2cc98083ace8c9d74e261a39ae1 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 11:41:58 +0200 Subject: [PATCH 068/128] quic util cleanup --- quic-forward-proxy/src/lib.rs | 1 - quic-forward-proxy/src/main.rs | 1 - quic-forward-proxy/src/outbound/tx_forward.rs | 46 ++++++- quic-forward-proxy/src/quic_util.rs | 36 ++++++ .../src/test_client/quic_test_client.rs | 4 +- .../src/tls_config_provider_client.rs | 4 - .../src/tls_config_provider_server.rs | 6 +- .../src/tls_self_signed_pair_generator.rs | 3 +- .../src/tpu_quic_connection_utils.rs | 115 ------------------ 9 files changed, 81 insertions(+), 135 deletions(-) delete mode 100644 quic-forward-proxy/src/tpu_quic_connection_utils.rs diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index d03d5099..844a36a5 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -11,7 +11,6 @@ mod cli; mod test_client; mod util; mod tx_store; -mod tpu_quic_connection_utils; mod quinn_auto_reconnect; mod outbound; mod inbound; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 0817f533..7cff67fc 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -20,7 +20,6 @@ pub mod cli; pub mod test_client; mod util; mod tx_store; -mod tpu_quic_connection_utils; mod quinn_auto_reconnect; mod outbound; mod inbound; diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index af137c73..5416a3dc 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -7,12 +7,14 @@ use fan::tokio::mpsc::FanOut; use std::time::Duration; use futures::future::join_all; use itertools::Itertools; -use quinn::Endpoint; +use quinn::{ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt}; +use solana_sdk::quic::QUIC_MAX_TIMEOUT_MS; use solana_sdk::transaction::VersionedTransaction; +use solana_streamer::nonblocking::quic::ALPN_TPU_PROTOCOL_ID; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::mpsc::{channel, Receiver}; use tokio::time::timeout; -use crate::tpu_quic_connection_utils::QuicConnectionUtils; +use crate::quic_util::SkipServerVerification; use crate::quinn_auto_reconnect::AutoReconnect; use crate::shared::ForwardPacket; use crate::validator_identity::ValidatorIdentity; @@ -133,7 +135,7 @@ async fn new_endpoint_with_validator_identity(validator_identity: ValidatorIdent ) .expect("Failed to initialize QUIC connection certificates"); - let endpoint_outbound = QuicConnectionUtils::create_tpu_client_endpoint(certificate.clone(), key.clone()); + let endpoint_outbound = create_tpu_client_endpoint(certificate.clone(), key.clone()); endpoint_outbound } @@ -146,7 +148,43 @@ pub const MAX_TRANSACTIONS_PER_BATCH: usize = 10; pub const MAX_BYTES_PER_BATCH: usize = 10; const MAX_PARALLEL_STREAMS: usize = 6; - +fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { + let mut endpoint = { + let client_socket = + solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::UNSPECIFIED), (8000, 10000)) + .expect("create_endpoint bind_in_range") + .1; + let config = EndpointConfig::default(); + quinn::Endpoint::new(config, None, client_socket, TokioRuntime) + .expect("create_endpoint quinn::Endpoint::new") + }; + + let mut crypto = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(SkipServerVerification::new()) + .with_single_cert(vec![certificate], key) + .expect("Failed to set QUIC client certificates"); + + crypto.enable_early_data = true; + + crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; + + let mut config = ClientConfig::new(Arc::new(crypto)); + + // note: this should be aligned with solana quic server's endpoint config + let mut transport_config = TransportConfig::default(); + // no remotely-initiated streams required + transport_config.max_concurrent_uni_streams(VarInt::from_u32(0)); + transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); + let timeout = IdleTimeout::try_from(Duration::from_millis(QUIC_MAX_TIMEOUT_MS as u64)).unwrap(); + transport_config.max_idle_timeout(Some(timeout)); + transport_config.keep_alive_interval(None); + config.transport_config(Arc::new(transport_config)); + + endpoint.set_default_client_config(config); + + endpoint +} fn serialize_to_vecvec(transactions: &Vec) -> Vec> { transactions.iter().map(|tx| { diff --git a/quic-forward-proxy/src/quic_util.rs b/quic-forward-proxy/src/quic_util.rs index 21f61221..63eb0399 100644 --- a/quic-forward-proxy/src/quic_util.rs +++ b/quic-forward-proxy/src/quic_util.rs @@ -1,3 +1,39 @@ +use std::sync::Arc; +use quinn::Connection; pub const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; + +pub struct SkipServerVerification; + +impl SkipServerVerification { + pub fn new() -> Arc { + Arc::new(Self) + } +} + +impl rustls::client::ServerCertVerifier for SkipServerVerification { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &rustls::ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: std::time::SystemTime, + ) -> Result { + Ok(rustls::client::ServerCertVerified::assertion()) + } +} + +// stable_id 140266619216912, rtt=2.156683ms, +// stats FrameStats { ACK: 3, CONNECTION_CLOSE: 0, CRYPTO: 3, +// DATA_BLOCKED: 0, DATAGRAM: 0, HANDSHAKE_DONE: 1, MAX_DATA: 0, +// MAX_STREAM_DATA: 1, MAX_STREAMS_BIDI: 0, MAX_STREAMS_UNI: 0, NEW_CONNECTION_ID: 4, +// NEW_TOKEN: 0, PATH_CHALLENGE: 0, PATH_RESPONSE: 0, PING: 0, RESET_STREAM: 0, +// RETIRE_CONNECTION_ID: 1, STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, +// STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } +pub fn connection_stats(connection: &Connection) -> String { + format!("stable_id {} stats {:?}, rtt={:?}", + connection.stable_id(), connection.stats().frame_rx, connection.stats().path.rtt) +} diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index c6204dcf..a3b4ebb7 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -11,9 +11,7 @@ use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::{Transaction, VersionedTransaction}; use tokio::io::AsyncWriteExt; use crate::proxy_request_format::TpuForwardingRequest; -use crate::tpu_quic_connection_utils::SkipServerVerification; -use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; -use crate::tls_config_provider_server::ProxyTlsConfigProvider; +use crate::quic_util::{ALPN_TPU_FORWARDPROXY_PROTOCOL_ID, SkipServerVerification}; use crate::tls_config_provider_client::TpuCLientTlsConfigProvider; use crate::util::AnyhowJoinHandle; diff --git a/quic-forward-proxy/src/tls_config_provider_client.rs b/quic-forward-proxy/src/tls_config_provider_client.rs index 7d7240e0..94ed1e22 100644 --- a/quic-forward-proxy/src/tls_config_provider_client.rs +++ b/quic-forward-proxy/src/tls_config_provider_client.rs @@ -1,8 +1,4 @@ -use std::sync::atomic::{AtomicU32, Ordering}; -use rcgen::generate_simple_self_signed; use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; -use crate::tpu_quic_connection_utils::SkipServerVerification; -use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; // TODO integrate with tpu_service + quic_connection_utils diff --git a/quic-forward-proxy/src/tls_config_provider_server.rs b/quic-forward-proxy/src/tls_config_provider_server.rs index 4e38c270..9417ad79 100644 --- a/quic-forward-proxy/src/tls_config_provider_server.rs +++ b/quic-forward-proxy/src/tls_config_provider_server.rs @@ -1,8 +1,4 @@ -use std::sync::atomic::{AtomicU32, Ordering}; -use rcgen::generate_simple_self_signed; -use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; -use crate::tpu_quic_connection_utils::SkipServerVerification; -use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; +use rustls::ServerConfig; // TODO integrate with tpu_service + quic_connection_utils diff --git a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs index c50236ce..df0dbe6f 100644 --- a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs +++ b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs @@ -1,8 +1,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use rcgen::generate_simple_self_signed; use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; -use crate::tpu_quic_connection_utils::SkipServerVerification; -use crate::quic_util::ALPN_TPU_FORWARDPROXY_PROTOCOL_ID; +use crate::quic_util::{ALPN_TPU_FORWARDPROXY_PROTOCOL_ID, SkipServerVerification}; use crate::tls_config_provider_client::TpuCLientTlsConfigProvider; use crate::tls_config_provider_server::ProxyTlsConfigProvider; diff --git a/quic-forward-proxy/src/tpu_quic_connection_utils.rs b/quic-forward-proxy/src/tpu_quic_connection_utils.rs deleted file mode 100644 index 9443cf3c..00000000 --- a/quic-forward-proxy/src/tpu_quic_connection_utils.rs +++ /dev/null @@ -1,115 +0,0 @@ -use log::{debug, error, trace, warn}; -use quinn::{ClientConfig, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendStream, TokioRuntime, TransportConfig, VarInt}; -use solana_sdk::pubkey::Pubkey; -use std::{ - net::{IpAddr, Ipv4Addr, SocketAddr}, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - time::Duration, -}; - -use futures::future::join_all; -use itertools::Itertools; -use solana_sdk::quic::QUIC_MAX_TIMEOUT_MS; -use tokio::{time::timeout}; - - - -const ALPN_TPU_PROTOCOL_ID: &[u8] = b"solana-tpu"; - -pub struct QuicConnectionUtils {} - -pub enum QuicConnectionError { - TimeOut, - ConnectionError { retry: bool }, -} - -// TODO check whot we need from this -#[derive(Clone, Copy)] -pub struct QuicConnectionParameters { - // pub connection_timeout: Duration, - pub unistream_timeout: Duration, - pub write_timeout: Duration, - pub finalize_timeout: Duration, - pub connection_retry_count: usize, - // pub max_number_of_connections: usize, - // pub number_of_transactions_per_unistream: usize, -} - -impl QuicConnectionUtils { - // TODO move to a more specific place - pub fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { - let mut endpoint = { - let client_socket = - solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::UNSPECIFIED), (8000, 10000)) - .expect("create_endpoint bind_in_range") - .1; - let config = EndpointConfig::default(); - quinn::Endpoint::new(config, None, client_socket, TokioRuntime) - .expect("create_endpoint quinn::Endpoint::new") - }; - - let mut crypto = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(SkipServerVerification::new()) - .with_single_cert(vec![certificate], key) - .expect("Failed to set QUIC client certificates"); - - crypto.enable_early_data = true; - - crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; - - let mut config = ClientConfig::new(Arc::new(crypto)); - - // note: this should be aligned with solana quic server's endpoint config - let mut transport_config = TransportConfig::default(); - // no remotely-initiated streams required - transport_config.max_concurrent_uni_streams(VarInt::from_u32(0)); - transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); - let timeout = IdleTimeout::try_from(Duration::from_millis(QUIC_MAX_TIMEOUT_MS as u64)).unwrap(); - transport_config.max_idle_timeout(Some(timeout)); - transport_config.keep_alive_interval(None); - config.transport_config(Arc::new(transport_config)); - - endpoint.set_default_client_config(config); - - endpoint - } - -} - -pub struct SkipServerVerification; - -impl SkipServerVerification { - pub fn new() -> Arc { - Arc::new(Self) - } -} - -impl rustls::client::ServerCertVerifier for SkipServerVerification { - fn verify_server_cert( - &self, - _end_entity: &rustls::Certificate, - _intermediates: &[rustls::Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, - _ocsp_response: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) - } -} - -// stable_id 140266619216912, rtt=2.156683ms, -// stats FrameStats { ACK: 3, CONNECTION_CLOSE: 0, CRYPTO: 3, -// DATA_BLOCKED: 0, DATAGRAM: 0, HANDSHAKE_DONE: 1, MAX_DATA: 0, -// MAX_STREAM_DATA: 1, MAX_STREAMS_BIDI: 0, MAX_STREAMS_UNI: 0, NEW_CONNECTION_ID: 4, -// NEW_TOKEN: 0, PATH_CHALLENGE: 0, PATH_RESPONSE: 0, PING: 0, RESET_STREAM: 0, -// RETIRE_CONNECTION_ID: 1, STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, -// STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } -pub fn connection_stats(connection: &Connection) -> String { - format!("stable_id {} stats {:?}, rtt={:?}", - connection.stable_id(), connection.stats().frame_rx, connection.stats().path.rtt) -} \ No newline at end of file From 86f25724eb09fd1c233a14b25f34e4d1c8684b5a Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 13:27:03 +0200 Subject: [PATCH 069/128] minor cleanup --- quic-forward-proxy/src/lib.rs | 1 - quic-forward-proxy/src/main.rs | 1 - quic-forward-proxy/src/outbound/tx_forward.rs | 120 ++++++------------ quic-forward-proxy/src/tx_store.rs | 27 ---- 4 files changed, 38 insertions(+), 111 deletions(-) delete mode 100644 quic-forward-proxy/src/tx_store.rs diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index 844a36a5..b423cf26 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -10,7 +10,6 @@ pub mod proxy_request_format; mod cli; mod test_client; mod util; -mod tx_store; mod quinn_auto_reconnect; mod outbound; mod inbound; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 7cff67fc..33d03760 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -19,7 +19,6 @@ pub mod proxy_request_format; pub mod cli; pub mod test_client; mod util; -mod tx_store; mod quinn_auto_reconnect; mod outbound; mod inbound; diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 5416a3dc..97e87469 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -1,10 +1,11 @@ use std::sync::Arc; -use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicBool, Ordering}; use log::{debug, info, warn}; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use fan::tokio::mpsc::FanOut; use std::time::Duration; +use anyhow::{bail, Context}; use futures::future::join_all; use itertools::Itertools; use quinn::{ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt}; @@ -13,12 +14,21 @@ use solana_sdk::transaction::VersionedTransaction; use solana_streamer::nonblocking::quic::ALPN_TPU_PROTOCOL_ID; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::mpsc::{channel, Receiver}; -use tokio::time::timeout; use crate::quic_util::SkipServerVerification; use crate::quinn_auto_reconnect::AutoReconnect; use crate::shared::ForwardPacket; +use crate::util::timeout_fallback; use crate::validator_identity::ValidatorIdentity; + +const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); +pub const CONNECTION_RETRY_COUNT: usize = 10; + +pub const MAX_TRANSACTIONS_PER_BATCH: usize = 10; +pub const MAX_BYTES_PER_BATCH: usize = 10; +const MAX_PARALLEL_STREAMS: usize = 6; +pub const PARALLEL_TPU_CONNECTION_COUNT: usize = 4; + // takes transactions from upstream clients and forwards them to the TPU pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction_channel: Receiver, exit_signal: Arc) -> anyhow::Result<()> { info!("TPU Quic forwarder started"); @@ -28,60 +38,50 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction let mut agents: HashMap> = HashMap::new(); loop { - // TODO add exit + if exit_signal.load(Ordering::Relaxed) { + bail!("exit signal received"); + } let forward_packet = transaction_channel.recv().await.expect("channel closed unexpectedly"); + // TODO drain the queue with .try_recv() and batch the transactions let tpu_address = forward_packet.tpu_address; if !agents.contains_key(&tpu_address) { // TODO cleanup agent after a while of iactivity let mut senders = Vec::new(); - for _i in 0..4 { - let (sender, mut receiver) = channel::(100000); + for connection_idx in 1..PARALLEL_TPU_CONNECTION_COUNT { + let (sender, mut receiver) = channel::(100_000); senders.push(sender); let exit_signal = exit_signal.clone(); let endpoint_copy = endpoint.clone(); tokio::spawn(async move { - debug!("Start Quic forwarder agent for TPU {}", tpu_address); - // TODO pass+check the tpu_address - // TODO connect - // TODO consume queue - // TODO exit signal + debug!("Start Quic forwarder agent #{} for TPU {}", connection_idx, tpu_address); let auto_connection = AutoReconnect::new(endpoint_copy, tpu_address); - // let mut connection = tpu_quic_client_copy.create_connection(tpu_address).await.expect("handshake"); - loop { - - let _exit_signal = exit_signal.clone(); - loop { - let packet = receiver.recv().await.unwrap(); - assert_eq!(packet.tpu_address, tpu_address, "routing error"); - let mut transactions_batch = packet.transactions; + let exit_signal_copy = exit_signal.clone(); + loop { + let packet = receiver.recv().await.unwrap(); + assert_eq!(packet.tpu_address, tpu_address, "routing error"); - let mut batch_size = 1; - while let Ok(more) = receiver.try_recv() { - transactions_batch.extend(more.transactions); - batch_size += 1; - } - if batch_size > 1 { - debug!("encountered batch of size {}", batch_size); - } + let mut transactions_batch = packet.transactions; - debug!("forwarding transaction batch of size {} to address {}", transactions_batch.len(), packet.tpu_address); + let mut batch_size = 1; + while let Ok(more) = receiver.try_recv() { + transactions_batch.extend(more.transactions); + batch_size += 1; + } - // TODo move send_txs_to_tpu_static to tpu_quic_client - let result = timeout(Duration::from_millis(500), - send_txs_to_tpu_static(&auto_connection, &transactions_batch)).await; - // .expect("timeout sending data to TPU node"); + debug!("forwarding transaction batch of size {} to address {}", transactions_batch.len(), packet.tpu_address); - if result.is_err() { - warn!("send_txs_to_tpu_static result {:?} - loop over errors", result); - } else { - debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); - } + let result = timeout_fallback(send_tx_batch_to_tpu(&auto_connection, &transactions_batch)).await + .context("send txs to tpu"); + if result.is_err() { + warn!("got send_txs_to_tpu_static error {:?} - loop over errors", result); + } else { + debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); } } @@ -100,26 +100,6 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction agent_channel.send(forward_packet).await.unwrap(); - // let mut batch_size = 1; - // while let Ok(more) = transaction_channel.try_recv() { - // agent_channel.send(more).await.unwrap(); - // batch_size += 1; - // } - // if batch_size > 1 { - // debug!("encountered batch of size {}", batch_size); - // } - - - // check if the tpu has already a task+queue running, if not start one, sort+queue packets by tpu address - // maintain the health of a TPU connection, debounce errors; if failing, drop the respective messages - - // let exit_signal_copy = exit_signal.clone(); - // debug!("send transaction batch of size {} to address {}", forward_packet.transactions.len(), forward_packet.tpu_address); - // // TODO: this will block/timeout if the TPU is not available - // timeout(Duration::from_millis(500), - // tpu_quic_client_copy.send_txs_to_tpu(tpu_address, &forward_packet.transactions, exit_signal_copy)).await; - // tpu_quic_client_copy.send_txs_to_tpu(forward_packet.tpu_address, &forward_packet.transactions, exit_signal_copy).await; - } // -- loop over transactions from ustream channels // not reachable @@ -140,14 +120,6 @@ async fn new_endpoint_with_validator_identity(validator_identity: ValidatorIdent endpoint_outbound } - -const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); -pub const CONNECTION_RETRY_COUNT: usize = 10; - -pub const MAX_TRANSACTIONS_PER_BATCH: usize = 10; -pub const MAX_BYTES_PER_BATCH: usize = 10; -const MAX_PARALLEL_STREAMS: usize = 6; - fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { let mut endpoint = { let client_socket = @@ -179,6 +151,7 @@ fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::Pri let timeout = IdleTimeout::try_from(Duration::from_millis(QUIC_MAX_TIMEOUT_MS as u64)).unwrap(); transport_config.max_idle_timeout(Some(timeout)); transport_config.keep_alive_interval(None); + config.transport_config(Arc::new(transport_config)); endpoint.set_default_client_config(config); @@ -186,27 +159,12 @@ fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::Pri endpoint } -fn serialize_to_vecvec(transactions: &Vec) -> Vec> { - transactions.iter().map(|tx| { - let tx_raw = bincode::serialize(tx).unwrap(); - tx_raw - }).collect_vec() -} - - // send potentially large amount of transactions to a single TPU #[tracing::instrument(skip_all, level = "debug")] -async fn send_txs_to_tpu_static( +async fn send_tx_batch_to_tpu( auto_connection: &AutoReconnect, txs: &Vec, ) { - - // note: this impl does not deal with connection errors - // throughput_50 493.70 tps - // throughput_50 769.43 tps (with finish timeout) - // TODO join get_or_create_connection future and read_to_end - // TODO add error handling - for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { let all_send_fns = chunk.iter().map(|tx| { let tx_raw = bincode::serialize(tx).unwrap(); @@ -216,8 +174,6 @@ async fn send_txs_to_tpu_static( auto_connection.send(tx_raw) // ignores error }); - // let all_send_fns = (0..txs.len()).map(|i| auto_connection.roundtrip(vecvec.get(i))).collect_vec(); - join_all(all_send_fns).await; } diff --git a/quic-forward-proxy/src/tx_store.rs b/quic-forward-proxy/src/tx_store.rs deleted file mode 100644 index 3300b104..00000000 --- a/quic-forward-proxy/src/tx_store.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::sync::Arc; - -use dashmap::DashMap; -use solana_transaction_status::TransactionStatus; -use tokio::time::Instant; -/// Transaction Properties - -pub struct TxProps { - pub status: Option, - /// Time at which transaction was forwarded - pub sent_at: Instant, -} - -impl Default for TxProps { - fn default() -> Self { - Self { - status: Default::default(), - sent_at: Instant::now(), - } - } -} - -pub type TxStore = Arc>; - -pub fn empty_tx_store() -> TxStore { - Arc::new(DashMap::new()) -} From 62a2f76c93ece5dd8b81c0d7a51f00ea4da8fd65 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 13:39:41 +0200 Subject: [PATCH 070/128] add exit signal --- quic-forward-proxy/src/outbound/tx_forward.rs | 9 ++++++--- quic-forward-proxy/src/quinn_auto_reconnect.rs | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 97e87469..6472f232 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -57,12 +57,14 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction let endpoint_copy = endpoint.clone(); tokio::spawn(async move { debug!("Start Quic forwarder agent #{} for TPU {}", connection_idx, tpu_address); + if exit_signal.load(Ordering::Relaxed) { + return; + } let auto_connection = AutoReconnect::new(endpoint_copy, tpu_address); let exit_signal_copy = exit_signal.clone(); - loop { - let packet = receiver.recv().await.unwrap(); + while let Some(packet) = receiver.recv().await { assert_eq!(packet.tpu_address, tpu_address, "routing error"); let mut transactions_batch = packet.transactions; @@ -84,8 +86,9 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); } - } + } // -- while all packtes from channel + info!("Quic forwarder agent #{} for TPU {} exited", connection_idx, tpu_address); }); } diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 58a0d4ca..0419aa82 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -31,7 +31,7 @@ impl AutoReconnect { send_stream.write_all(payload.as_slice()).await?; send_stream.finish().await?; - let answer = recv_stream.read_to_end(64 * 1024).await?; + let answer = recv_stream.reread_to_end(64 * 1024).await?; Ok(answer) } From 4844844e21629c3a4095384e78a45fd70341c9d1 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 14:22:56 +0200 Subject: [PATCH 071/128] add error context --- quic-forward-proxy/README.md | 2 +- .../src/inbound/proxy_listener.rs | 10 +-- quic-forward-proxy/src/outbound/tx_forward.rs | 9 +-- .../src/quinn_auto_reconnect.rs | 62 +++++++------------ 4 files changed, 35 insertions(+), 48 deletions(-) diff --git a/quic-forward-proxy/README.md b/quic-forward-proxy/README.md index d2f4df2a..319d7323 100644 --- a/quic-forward-proxy/README.md +++ b/quic-forward-proxy/README.md @@ -24,7 +24,7 @@ RUST_LOG="error,solana_streamer::nonblocking::quic=debug" solana-test-validator ``` 3. run quic proxy ```bash -RUST_LOG=debug cargo run --bin solana-lite-rpc-quic-forward-proxy -- --proxy-rpc-addr 0.0.0.0:11111 --identity-keypair /pathto-test-ledger/validator-keypair.json +RUST_LOG=debug cargo run --bin solana-lite-rpc-quic-forward-proxy -- --proxy-listen-addr 0.0.0.0:11111 --identity-keypair /pathto-test-ledger/validator-keypair.json ``` 2. run lite-rpc ```bash diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 12720f11..07c94c96 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -41,12 +41,14 @@ impl ProxyListener { let forwarder_channel_copy = forwarder_channel.clone(); tokio::spawn(async move { let connection = connecting.await.context("handshake").unwrap(); - match Self::accept_client_connection(connection, forwarder_channel_copy, - exit_signal) + match Self::accept_client_connection( + connection, forwarder_channel_copy, exit_signal) .await { - Ok(()) => {} + Ok(()) => { + debug!("connection handles correctly"); + } Err(err) => { - error!("setup connection failed: {reason}", reason = err); + error!("failed to accect connection from client: {reason} - skip", reason = err); } } }); diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 6472f232..be43ac64 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -78,7 +78,7 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction debug!("forwarding transaction batch of size {} to address {}", transactions_batch.len(), packet.tpu_address); let result = timeout_fallback(send_tx_batch_to_tpu(&auto_connection, &transactions_batch)).await - .context("send txs to tpu"); + .context(format!("send txs to tpu node {}", auto_connection.target_address)); if result.is_err() { warn!("got send_txs_to_tpu_static error {:?} - loop over errors", result); @@ -101,9 +101,10 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction let agent_channel = agents.get(&tpu_address).unwrap(); - agent_channel.send(forward_packet).await.unwrap(); + timeout_fallback(agent_channel.send(forward_packet)).await + .context("send to agent channel")??; - } // -- loop over transactions from ustream channels + } // -- loop over transactions from upstream channels // not reachable } @@ -174,7 +175,7 @@ async fn send_tx_batch_to_tpu( tx_raw }) .map(|tx_raw| { - auto_connection.send(tx_raw) // ignores error + auto_connection.send_uni(tx_raw) // ignores error }); join_all(all_send_fns).await; diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 0419aa82..d7b06d47 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -2,15 +2,18 @@ use std::fmt; use std::net::SocketAddr; use std::sync::atomic::{AtomicU32, Ordering}; +use anyhow::Context; +use log::warn; use tracing::{debug, info}; use quinn::{Connection, Endpoint}; use tokio::sync::{RwLock}; +use crate::util::timeout_fallback; pub struct AutoReconnect { + // endoint should be configures with keep-alive and idle timeout endpoint: Endpoint, - // note: no read lock is used ATM current: RwLock>, - target_address: SocketAddr, + pub target_address: SocketAddr, reconnect_count: AtomicU32, } @@ -24,25 +27,12 @@ impl AutoReconnect { } } - pub async fn roundtrip(&self, payload: Vec) -> anyhow::Result> { + pub async fn send_uni(&self, payload: Vec) -> anyhow::Result<()> { // TOOD do smart error handling + reconnect - // self.refresh().await.open_bi().await.unwrap() - let (mut send_stream, recv_stream) = self.refresh().await.open_bi().await?; + let mut send_stream = timeout_fallback(self.refresh().await.open_uni()).await + .context("open uni stream for sending")??; send_stream.write_all(payload.as_slice()).await?; send_stream.finish().await?; - - let answer = recv_stream.reread_to_end(64 * 1024).await?; - - Ok(answer) - } - - pub async fn send(&self, payload: Vec) -> anyhow::Result<()> { - // TOOD do smart error handling + reconnect - let mut send_stream = self.refresh().await.open_uni().await?; - send_stream.write_all(payload.as_slice()).await?; - send_stream.finish().await?; - - Ok(()) } @@ -50,52 +40,46 @@ impl AutoReconnect { pub async fn refresh(&self) -> Connection { { let lock = self.current.read().await; - let maybe_conn: &Option = &*lock; - if maybe_conn.as_ref().filter(|conn| conn.close_reason().is_none()).is_some() { - // let reuse = lock.unwrap().clone(); - let reuse = maybe_conn.as_ref().unwrap(); + let maybe_conn = lock.as_ref(); + if maybe_conn.filter(|conn| conn.close_reason().is_none()).is_some() { + let reuse = maybe_conn.unwrap(); debug!("Reuse connection {}", reuse.stable_id()); return reuse.clone(); } } let mut lock = self.current.write().await; - match &*lock { + let maybe_conn = lock.as_ref(); + return match maybe_conn { Some(current) => { - if current.close_reason().is_some() { - info!("Connection is closed for reason: {:?}", current.close_reason()); - // TODO log + let old_stable_id = current.stable_id(); + warn!("Connection {} is closed for reason: {:?}", old_stable_id, current.close_reason()); let new_connection = self.create_connection().await; *lock = Some(new_connection.clone()); // let old_conn = lock.replace(new_connection.clone()); self.reconnect_count.fetch_add(1, Ordering::SeqCst); - // debug!("Replace closed connection {} with {} (retry {})", - // old_conn.map(|c| c.stable_id().to_string()).unwrap_or("none".to_string()), - // new_connection.stable_id(), - // self.reconnect_count.load(Ordering::SeqCst)); - // TODO log old vs new stable_id - + debug!("Replace closed connection {} with {} (retry {})", + old_stable_id, + new_connection.stable_id(), + self.reconnect_count.load(Ordering::SeqCst)); - return new_connection.clone(); + new_connection.clone() } else { debug!("Reuse connection {} with write-lock", current.stable_id()); - return current.clone(); + current.clone() } - } None => { let new_connection = self.create_connection().await; - // let old_conn = lock.replace(new_connection.clone()); - // assert!(old_conn.is_none(), "old connection should be None"); + assert!(lock.is_none(), "old connection must be None"); *lock = Some(new_connection.clone()); // let old_conn = foo.replace(Some(new_connection.clone())); - // TODO log old vs new stable_id debug!("Create initial connection {}", new_connection.stable_id()); - return new_connection.clone(); + new_connection.clone() } } } From 035ce6e3d52a8f82ae067759695908d45a54760b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 14:26:21 +0200 Subject: [PATCH 072/128] fmt+clippy --- quic-forward-proxy/src/cli.rs | 2 +- quic-forward-proxy/src/inbound/mod.rs | 2 +- .../src/inbound/proxy_listener.rs | 118 +++++++++++------- quic-forward-proxy/src/lib.rs | 18 +-- quic-forward-proxy/src/main.rs | 31 ++--- quic-forward-proxy/src/outbound/tx_forward.rs | 112 ++++++++++------- quic-forward-proxy/src/proxy.rs | 44 ++++--- .../src/proxy_request_format.rs | 30 +++-- quic-forward-proxy/src/quic_util.rs | 11 +- .../src/quinn_auto_reconnect.rs | 47 ++++--- .../src/test_client/quic_test_client.rs | 35 +++--- .../src/test_client/sample_data_factory.rs | 19 ++- .../src/tls_config_provider_client.rs | 4 +- .../src/tls_config_provider_server.rs | 2 - .../src/tls_self_signed_pair_generator.rs | 17 +-- quic-forward-proxy/src/util.rs | 8 +- quic-forward-proxy/src/validator_identity.rs | 4 +- .../tests/proxy_request_format.rs | 20 ++- 18 files changed, 289 insertions(+), 235 deletions(-) diff --git a/quic-forward-proxy/src/cli.rs b/quic-forward-proxy/src/cli.rs index 08bad1d9..cfdd1608 100644 --- a/quic-forward-proxy/src/cli.rs +++ b/quic-forward-proxy/src/cli.rs @@ -1,6 +1,6 @@ -use std::env; use clap::Parser; use solana_sdk::signature::Keypair; +use std::env; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] diff --git a/quic-forward-proxy/src/inbound/mod.rs b/quic-forward-proxy/src/inbound/mod.rs index 545cb4c9..33c20f48 100644 --- a/quic-forward-proxy/src/inbound/mod.rs +++ b/quic-forward-proxy/src/inbound/mod.rs @@ -1 +1 @@ -pub(crate) mod proxy_listener; \ No newline at end of file +pub(crate) mod proxy_listener; diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 07c94c96..858a6444 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -1,40 +1,51 @@ -use std::net::SocketAddr; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; -use std::time::Duration; -use anyhow::{anyhow, bail, Context}; -use log::{debug, error, info, trace}; -use quinn::{Connection, Endpoint, ServerConfig, VarInt}; -use solana_sdk::packet::PACKET_DATA_SIZE; -use tokio::sync::mpsc::Sender; use crate::proxy_request_format::TpuForwardingRequest; use crate::shared::ForwardPacket; use crate::tls_config_provider_server::ProxyTlsConfigProvider; use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use crate::util::FALLBACK_TIMEOUT; +use anyhow::{anyhow, bail, Context}; +use log::{debug, error, info, trace}; +use quinn::{Connection, Endpoint, ServerConfig, VarInt}; +use solana_sdk::packet::PACKET_DATA_SIZE; +use std::net::SocketAddr; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc::Sender; // TODO tweak this value - solana server sets 256 // setting this to "1" did not make a difference! const MAX_CONCURRENT_UNI_STREAMS: u32 = 24; - pub struct ProxyListener { tls_config: Arc, proxy_listener_addr: SocketAddr, } impl ProxyListener { - pub fn new( proxy_listener_addr: SocketAddr, - tls_config: Arc) -> Self { - Self { proxy_listener_addr, tls_config } + tls_config: Arc, + ) -> Self { + Self { + proxy_listener_addr, + tls_config, + } } - pub async fn listen(&self, exit_signal: Arc, forwarder_channel: Sender) -> anyhow::Result<()> { - info!("TPU Quic Proxy server listening on {}", self.proxy_listener_addr); + pub async fn listen( + &self, + exit_signal: Arc, + forwarder_channel: Sender, + ) -> anyhow::Result<()> { + info!( + "TPU Quic Proxy server listening on {}", + self.proxy_listener_addr + ); - let endpoint = Self::new_proxy_listen_server_endpoint(&self.tls_config, self.proxy_listener_addr).await; + let endpoint = + Self::new_proxy_listen_server_endpoint(&self.tls_config, self.proxy_listener_addr) + .await; while let Some(connecting) = endpoint.accept().await { let exit_signal = exit_signal.clone(); @@ -42,13 +53,20 @@ impl ProxyListener { tokio::spawn(async move { let connection = connecting.await.context("handshake").unwrap(); match Self::accept_client_connection( - connection, forwarder_channel_copy, exit_signal) - .await { + connection, + forwarder_channel_copy, + exit_signal, + ) + .await + { Ok(()) => { debug!("connection handles correctly"); } Err(err) => { - error!("failed to accect connection from client: {reason} - skip", reason = err); + error!( + "failed to accect connection from client: {reason} - skip", + reason = err + ); } } }); @@ -57,9 +75,10 @@ impl ProxyListener { bail!("TPU Quic Proxy server stopped"); } - - async fn new_proxy_listen_server_endpoint(tls_config: &SelfSignedTlsConfigProvider, proxy_listener_addr: SocketAddr) -> Endpoint { - + async fn new_proxy_listen_server_endpoint( + tls_config: &SelfSignedTlsConfigProvider, + proxy_listener_addr: SocketAddr, + ) -> Endpoint { let server_tls_config = tls_config.get_server_tls_crypto_config(); let mut quinn_server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); @@ -73,17 +92,23 @@ impl ProxyListener { transport_config.max_idle_timeout(Some(timeout)); transport_config.keep_alive_interval(Some(Duration::from_millis(500))); transport_config.stream_receive_window((PACKET_DATA_SIZE as u32).into()); - transport_config.receive_window((PACKET_DATA_SIZE as u32 * MAX_CONCURRENT_UNI_STREAMS).into()); + transport_config + .receive_window((PACKET_DATA_SIZE as u32 * MAX_CONCURRENT_UNI_STREAMS).into()); Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap() } - // TODO use interface abstraction for connection_per_tpunode #[tracing::instrument(skip_all, level = "debug")] - async fn accept_client_connection(client_connection: Connection, forwarder_channel: Sender, - exit_signal: Arc) -> anyhow::Result<()> { - debug!("inbound connection established, client {}", client_connection.remote_address()); + async fn accept_client_connection( + client_connection: Connection, + forwarder_channel: Sender, + _exit_signal: Arc, + ) -> anyhow::Result<()> { + debug!( + "inbound connection established, client {}", + client_connection.remote_address() + ); loop { let maybe_stream = client_connection.accept_uni().await; @@ -91,11 +116,14 @@ impl ProxyListener { Err(quinn::ConnectionError::ApplicationClosed(reason)) => { debug!("connection closed by client - reason: {:?}", reason); if reason.error_code != VarInt::from_u32(0) { - return Err(anyhow!("connection closed by client with unexpected reason: {:?}", reason)); + return Err(anyhow!( + "connection closed by client with unexpected reason: {:?}", + reason + )); } debug!("connection gracefully closed by client"); return Ok(()); - }, + } Err(e) => { error!("failed to accept stream: {}", e); return Err(anyhow::Error::msg("error accepting stream")); @@ -103,37 +131,41 @@ impl ProxyListener { Ok(recv_stream) => { let forwarder_channel_copy = forwarder_channel.clone(); tokio::spawn(async move { + let raw_request = recv_stream.read_to_end(10_000_000).await.unwrap(); - let raw_request = recv_stream.read_to_end(10_000_000).await - .unwrap(); - - let proxy_request = TpuForwardingRequest::deserialize_from_raw_request(&raw_request); + let proxy_request = + TpuForwardingRequest::deserialize_from_raw_request(&raw_request); trace!("proxy request details: {}", proxy_request); let _tpu_identity = proxy_request.get_identity_tpunode(); let tpu_address = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transactions(); - debug!("enqueue transaction batch of size {} to address {}", txs.len(), tpu_address); - forwarder_channel_copy.send_timeout(ForwardPacket { transactions: txs, tpu_address }, - FALLBACK_TIMEOUT) - .await + debug!( + "enqueue transaction batch of size {} to address {}", + txs.len(), + tpu_address + ); + forwarder_channel_copy + .send_timeout( + ForwardPacket { + transactions: txs, + tpu_address, + }, + FALLBACK_TIMEOUT, + ) + .await .context("sending internal packet from proxy to forwarder") .unwrap(); - }); Ok(()) - }, + } }; // -- result if let Err(e) = result { return Err(e); } - } // -- loop } - } - - diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index b423cf26..dc88528b 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -1,16 +1,16 @@ // lib definition is only required for 'quic-forward-proxy-integration-test' to work +mod cli; +mod inbound; +mod outbound; +pub mod proxy; +pub mod proxy_request_format; mod quic_util; +mod quinn_auto_reconnect; +mod shared; +mod test_client; pub mod tls_config_provider_client; pub mod tls_config_provider_server; pub mod tls_self_signed_pair_generator; -pub mod proxy; -pub mod validator_identity; -pub mod proxy_request_format; -mod cli; -mod test_client; mod util; -mod quinn_auto_reconnect; -mod outbound; -mod inbound; -mod shared; +pub mod validator_identity; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 33d03760..96829e0e 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -1,31 +1,29 @@ -use std::sync::Arc; +use crate::cli::{get_identity_keypair, Args}; +use crate::proxy::QuicForwardProxy; +use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use anyhow::bail; use clap::Parser; use dotenv::dotenv; use log::info; -use crate::cli::{Args, get_identity_keypair}; -use crate::proxy::QuicForwardProxy; -use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; +use std::sync::Arc; use crate::validator_identity::ValidatorIdentity; - +pub mod cli; +mod inbound; +mod outbound; +pub mod proxy; +pub mod proxy_request_format; pub mod quic_util; +mod quinn_auto_reconnect; +mod shared; +pub mod test_client; pub mod tls_config_provider_client; pub mod tls_config_provider_server; pub mod tls_self_signed_pair_generator; -pub mod proxy; -pub mod proxy_request_format; -pub mod cli; -pub mod test_client; mod util; -mod quinn_auto_reconnect; -mod outbound; -mod inbound; -mod shared; mod validator_identity; - #[tokio::main(flavor = "multi_thread", worker_threads = 16)] pub async fn main() -> anyhow::Result<()> { tracing_subscriber::fmt::init(); @@ -40,8 +38,7 @@ pub async fn main() -> anyhow::Result<()> { // TODO build args struct dedicated to proxy let proxy_listener_addr = proxy_rpc_addr.parse().unwrap(); let _tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); - let validator_identity = - ValidatorIdentity::new(get_identity_keypair(&identity_keypair).await); + let validator_identity = ValidatorIdentity::new(get_identity_keypair(&identity_keypair).await); let tls_config = Arc::new(SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost()); let main_services = QuicForwardProxy::new(proxy_listener_addr, tls_config, validator_identity) @@ -54,7 +51,6 @@ pub async fn main() -> anyhow::Result<()> { // .await? // .start_services(); - let ctrl_c_signal = tokio::signal::ctrl_c(); tokio::select! { @@ -70,5 +66,4 @@ pub async fn main() -> anyhow::Result<()> { Ok(()) } } - } diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index be43ac64..b26a30e4 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -1,25 +1,26 @@ -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use log::{debug, info, warn}; -use std::collections::HashMap; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use fan::tokio::mpsc::FanOut; -use std::time::Duration; +use crate::quic_util::SkipServerVerification; +use crate::quinn_auto_reconnect::AutoReconnect; +use crate::shared::ForwardPacket; +use crate::util::timeout_fallback; +use crate::validator_identity::ValidatorIdentity; use anyhow::{bail, Context}; +use fan::tokio::mpsc::FanOut; use futures::future::join_all; use itertools::Itertools; -use quinn::{ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt}; +use log::{debug, info, warn}; +use quinn::{ + ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt, +}; use solana_sdk::quic::QUIC_MAX_TIMEOUT_MS; use solana_sdk::transaction::VersionedTransaction; use solana_streamer::nonblocking::quic::ALPN_TPU_PROTOCOL_ID; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; use tokio::sync::mpsc::{channel, Receiver}; -use crate::quic_util::SkipServerVerification; -use crate::quinn_auto_reconnect::AutoReconnect; -use crate::shared::ForwardPacket; -use crate::util::timeout_fallback; -use crate::validator_identity::ValidatorIdentity; - const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); pub const CONNECTION_RETRY_COUNT: usize = 10; @@ -30,7 +31,11 @@ const MAX_PARALLEL_STREAMS: usize = 6; pub const PARALLEL_TPU_CONNECTION_COUNT: usize = 4; // takes transactions from upstream clients and forwards them to the TPU -pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction_channel: Receiver, exit_signal: Arc) -> anyhow::Result<()> { +pub async fn tx_forwarder( + validator_identity: ValidatorIdentity, + mut transaction_channel: Receiver, + exit_signal: Arc, +) -> anyhow::Result<()> { info!("TPU Quic forwarder started"); let endpoint = new_endpoint_with_validator_identity(validator_identity).await; @@ -42,7 +47,10 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction bail!("exit signal received"); } - let forward_packet = transaction_channel.recv().await.expect("channel closed unexpectedly"); + let forward_packet = transaction_channel + .recv() + .await + .expect("channel closed unexpectedly"); // TODO drain the queue with .try_recv() and batch the transactions let tpu_address = forward_packet.tpu_address; @@ -56,14 +64,17 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction let exit_signal = exit_signal.clone(); let endpoint_copy = endpoint.clone(); tokio::spawn(async move { - debug!("Start Quic forwarder agent #{} for TPU {}", connection_idx, tpu_address); + debug!( + "Start Quic forwarder agent #{} for TPU {}", + connection_idx, tpu_address + ); if exit_signal.load(Ordering::Relaxed) { return; } let auto_connection = AutoReconnect::new(endpoint_copy, tpu_address); - let exit_signal_copy = exit_signal.clone(); + let _exit_signal_copy = exit_signal.clone(); while let Some(packet) = receiver.recv().await { assert_eq!(packet.tpu_address, tpu_address, "routing error"); @@ -75,35 +86,49 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction batch_size += 1; } - debug!("forwarding transaction batch of size {} to address {}", transactions_batch.len(), packet.tpu_address); - - let result = timeout_fallback(send_tx_batch_to_tpu(&auto_connection, &transactions_batch)).await - .context(format!("send txs to tpu node {}", auto_connection.target_address)); + debug!( + "forwarding transaction batch of size {} to address {}", + transactions_batch.len(), + packet.tpu_address + ); + + let result = timeout_fallback(send_tx_batch_to_tpu( + &auto_connection, + &transactions_batch, + )) + .await + .context(format!( + "send txs to tpu node {}", + auto_connection.target_address + )); if result.is_err() { - warn!("got send_txs_to_tpu_static error {:?} - loop over errors", result); + warn!( + "got send_txs_to_tpu_static error {:?} - loop over errors", + result + ); } else { debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); } - } // -- while all packtes from channel - info!("Quic forwarder agent #{} for TPU {} exited", connection_idx, tpu_address); + info!( + "Quic forwarder agent #{} for TPU {} exited", + connection_idx, tpu_address + ); }); - } let fanout = FanOut::new(senders); agents.insert(tpu_address, fanout); - } // -- new agent let agent_channel = agents.get(&tpu_address).unwrap(); - timeout_fallback(agent_channel.send(forward_packet)).await + timeout_fallback(agent_channel.send(forward_packet)) + .await .context("send to agent channel")??; - } // -- loop over transactions from upstream channels // not reachable @@ -112,19 +137,25 @@ pub async fn tx_forwarder(validator_identity: ValidatorIdentity, mut transaction /// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU // note: ATM the provided identity might or might not be a valid validator keypair async fn new_endpoint_with_validator_identity(validator_identity: ValidatorIdentity) -> Endpoint { - info!("Setup TPU Quic stable connection with validator identity {} ...", validator_identity); + info!( + "Setup TPU Quic stable connection with validator identity {} ...", + validator_identity + ); let (certificate, key) = new_self_signed_tls_certificate( &validator_identity.get_keypair_for_tls(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) - .expect("Failed to initialize QUIC connection certificates"); + .expect("Failed to initialize QUIC connection certificates"); let endpoint_outbound = create_tpu_client_endpoint(certificate.clone(), key.clone()); endpoint_outbound } -fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { +fn create_tpu_client_endpoint( + certificate: rustls::Certificate, + key: rustls::PrivateKey, +) -> Endpoint { let mut endpoint = { let client_socket = solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::UNSPECIFIED), (8000, 10000)) @@ -165,21 +196,18 @@ fn create_tpu_client_endpoint(certificate: rustls::Certificate, key: rustls::Pri // send potentially large amount of transactions to a single TPU #[tracing::instrument(skip_all, level = "debug")] -async fn send_tx_batch_to_tpu( - auto_connection: &AutoReconnect, - txs: &Vec, -) { +async fn send_tx_batch_to_tpu(auto_connection: &AutoReconnect, txs: &Vec) { for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { - let all_send_fns = chunk.iter().map(|tx| { - let tx_raw = bincode::serialize(tx).unwrap(); - tx_raw - }) + let all_send_fns = chunk + .iter() + .map(|tx| { + let tx_raw = bincode::serialize(tx).unwrap(); + tx_raw + }) .map(|tx_raw| { auto_connection.send_uni(tx_raw) // ignores error }); join_all(all_send_fns).await; - } - } diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index c826315a..a8796f81 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -1,16 +1,15 @@ use std::net::SocketAddr; -use std::sync::Arc; +use anyhow::bail; use std::sync::atomic::AtomicBool; -use anyhow::{anyhow, bail, Context}; +use std::sync::Arc; -use log::{debug, error, info, trace}; use crate::inbound::proxy_listener; use crate::outbound::tx_forward::tx_forwarder; use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use crate::util::AnyhowJoinHandle; use crate::validator_identity::ValidatorIdentity; - +use log::info; pub struct QuicForwardProxy { // endpoint: Endpoint, @@ -23,36 +22,40 @@ impl QuicForwardProxy { pub async fn new( proxy_listener_addr: SocketAddr, tls_config: Arc, - validator_identity: ValidatorIdentity) -> anyhow::Result { - + validator_identity: ValidatorIdentity, + ) -> anyhow::Result { info!("Quic proxy uses validator identity {}", validator_identity); - Ok(Self { proxy_listener_addr, validator_identity, tls_config }) - + Ok(Self { + proxy_listener_addr, + validator_identity, + tls_config, + }) } - pub async fn start_services( - self, - ) -> anyhow::Result<()> { + pub async fn start_services(self) -> anyhow::Result<()> { let exit_signal = Arc::new(AtomicBool::new(false)); let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(100_000); - let proxy_listener = proxy_listener::ProxyListener::new( - self.proxy_listener_addr, - self.tls_config); + let proxy_listener = + proxy_listener::ProxyListener::new(self.proxy_listener_addr, self.tls_config); let exit_signal_clone = exit_signal.clone(); let quic_proxy = tokio::spawn(async move { - - proxy_listener.listen(exit_signal_clone.clone(), forwarder_channel).await + proxy_listener + .listen(exit_signal_clone.clone(), forwarder_channel) + .await .expect("proxy listen service"); }); let validator_identity = self.validator_identity.clone(); let exit_signal_clone = exit_signal.clone(); - let forwarder: AnyhowJoinHandle = tokio::spawn(tx_forwarder(validator_identity, - forward_receiver, exit_signal_clone)); + let forwarder: AnyhowJoinHandle = tokio::spawn(tx_forwarder( + validator_identity, + forward_receiver, + exit_signal_clone, + )); tokio::select! { res = quic_proxy => { @@ -63,9 +66,4 @@ impl QuicForwardProxy { }, } } - } - - - - diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 7ae100ec..bc503ffc 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -1,10 +1,10 @@ -use std::fmt; -use std::fmt::Display; -use std::net::{SocketAddr}; use anyhow::Context; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::VersionedTransaction; +use std::fmt; +use std::fmt::Display; +use std::net::SocketAddr; /// /// lite-rpc to proxy wire format @@ -16,21 +16,29 @@ pub const FORMAT_VERSION1: u16 = 2301; pub struct TpuForwardingRequest { format_version: u16, tpu_socket_addr: SocketAddr, // TODO is that correct - identity_tpunode: Pubkey, // note: this is only used fro + identity_tpunode: Pubkey, // note: this is only used fro // TODO consider not deserializing transactions in proxy transactions: Vec, } impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "TpuForwardingRequest for tpu target {} with identity {}: payload {} tx", - &self.get_tpu_socket_addr(), &self.get_identity_tpunode(), &self.get_transactions().len()) + write!( + f, + "TpuForwardingRequest for tpu target {} with identity {}: payload {} tx", + &self.get_tpu_socket_addr(), + &self.get_identity_tpunode(), + &self.get_transactions().len() + ) } } impl TpuForwardingRequest { - pub fn new(tpu_socket_addr: SocketAddr, identity_tpunode: Pubkey, - transactions: Vec) -> Self { + pub fn new( + tpu_socket_addr: SocketAddr, + identity_tpunode: Pubkey, + transactions: Vec, + ) -> Self { TpuForwardingRequest { format_version: FORMAT_VERSION1, tpu_socket_addr, @@ -39,8 +47,7 @@ impl TpuForwardingRequest { } } - pub fn serialize_wire_format( - &self) -> Vec { + pub fn serialize_wire_format(&self) -> Vec { bincode::serialize(&self).expect("Expect to serialize transactions") } @@ -67,6 +74,3 @@ impl TpuForwardingRequest { self.transactions.clone() } } - - - diff --git a/quic-forward-proxy/src/quic_util.rs b/quic-forward-proxy/src/quic_util.rs index 63eb0399..b2d28886 100644 --- a/quic-forward-proxy/src/quic_util.rs +++ b/quic-forward-proxy/src/quic_util.rs @@ -1,9 +1,8 @@ -use std::sync::Arc; use quinn::Connection; +use std::sync::Arc; pub const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; - pub struct SkipServerVerification; impl SkipServerVerification { @@ -34,6 +33,10 @@ impl rustls::client::ServerCertVerifier for SkipServerVerification { // RETIRE_CONNECTION_ID: 1, STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, // STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } pub fn connection_stats(connection: &Connection) -> String { - format!("stable_id {} stats {:?}, rtt={:?}", - connection.stable_id(), connection.stats().frame_rx, connection.stats().path.rtt) + format!( + "stable_id {} stats {:?}, rtt={:?}", + connection.stable_id(), + connection.stats().frame_rx, + connection.stats().path.rtt + ) } diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index d7b06d47..5b04642c 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -1,13 +1,12 @@ - -use std::fmt; -use std::net::SocketAddr; -use std::sync::atomic::{AtomicU32, Ordering}; +use crate::util::timeout_fallback; use anyhow::Context; use log::warn; -use tracing::{debug, info}; use quinn::{Connection, Endpoint}; -use tokio::sync::{RwLock}; -use crate::util::timeout_fallback; +use std::fmt; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicU32, Ordering}; +use tokio::sync::RwLock; +use tracing::debug; pub struct AutoReconnect { // endoint should be configures with keep-alive and idle timeout @@ -29,19 +28,22 @@ impl AutoReconnect { pub async fn send_uni(&self, payload: Vec) -> anyhow::Result<()> { // TOOD do smart error handling + reconnect - let mut send_stream = timeout_fallback(self.refresh().await.open_uni()).await + let mut send_stream = timeout_fallback(self.refresh().await.open_uni()) + .await .context("open uni stream for sending")??; send_stream.write_all(payload.as_slice()).await?; send_stream.finish().await?; Ok(()) } - pub async fn refresh(&self) -> Connection { { let lock = self.current.read().await; let maybe_conn = lock.as_ref(); - if maybe_conn.filter(|conn| conn.close_reason().is_none()).is_some() { + if maybe_conn + .filter(|conn| conn.close_reason().is_none()) + .is_some() + { let reuse = maybe_conn.unwrap(); debug!("Reuse connection {}", reuse.stable_id()); return reuse.clone(); @@ -53,17 +55,23 @@ impl AutoReconnect { Some(current) => { if current.close_reason().is_some() { let old_stable_id = current.stable_id(); - warn!("Connection {} is closed for reason: {:?}", old_stable_id, current.close_reason()); + warn!( + "Connection {} is closed for reason: {:?}", + old_stable_id, + current.close_reason() + ); let new_connection = self.create_connection().await; *lock = Some(new_connection.clone()); // let old_conn = lock.replace(new_connection.clone()); self.reconnect_count.fetch_add(1, Ordering::SeqCst); - debug!("Replace closed connection {} with {} (retry {})", + debug!( + "Replace closed connection {} with {} (retry {})", old_stable_id, new_connection.stable_id(), - self.reconnect_count.load(Ordering::SeqCst)); + self.reconnect_count.load(Ordering::SeqCst) + ); new_connection.clone() } else { @@ -81,12 +89,14 @@ impl AutoReconnect { new_connection.clone() } - } + }; } async fn create_connection(&self) -> Connection { - let connection = - self.endpoint.connect(self.target_address, "localhost").expect("handshake"); + let connection = self + .endpoint + .connect(self.target_address, "localhost") + .expect("handshake"); connection.await.expect("connection") } @@ -94,9 +104,6 @@ impl AutoReconnect { impl fmt::Display for AutoReconnect { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Connection to {}", - self.target_address, - ) + write!(f, "Connection to {}", self.target_address,) } } - diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs index a3b4ebb7..ef4d704a 100644 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ b/quic-forward-proxy/src/test_client/quic_test_client.rs @@ -1,19 +1,19 @@ -use std::net::{SocketAddr}; +use anyhow::bail; +use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use anyhow::bail; +use crate::proxy_request_format::TpuForwardingRequest; +use crate::quic_util::{SkipServerVerification, ALPN_TPU_FORWARDPROXY_PROTOCOL_ID}; +use crate::tls_config_provider_client::TpuCLientTlsConfigProvider; +use crate::util::AnyhowJoinHandle; use log::{info, trace}; use quinn::{Endpoint, VarInt}; use rustls::ClientConfig; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::{Transaction, VersionedTransaction}; use tokio::io::AsyncWriteExt; -use crate::proxy_request_format::TpuForwardingRequest; -use crate::quic_util::{ALPN_TPU_FORWARDPROXY_PROTOCOL_ID, SkipServerVerification}; -use crate::tls_config_provider_client::TpuCLientTlsConfigProvider; -use crate::util::AnyhowJoinHandle; pub struct QuicTestClient { pub endpoint: Endpoint, @@ -23,19 +23,20 @@ pub struct QuicTestClient { impl QuicTestClient { pub async fn new_with_endpoint( proxy_addr: SocketAddr, - tls_config: &impl TpuCLientTlsConfigProvider + tls_config: &impl TpuCLientTlsConfigProvider, ) -> anyhow::Result { let client_crypto = tls_config.get_client_tls_crypto_config(); let mut endpoint = quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())?; endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(client_crypto))); - Ok(Self { proxy_addr, endpoint }) + Ok(Self { + proxy_addr, + endpoint, + }) } // connect to a server - pub async fn start_services( - self, - ) -> anyhow::Result<()> { + pub async fn start_services(self) -> anyhow::Result<()> { let endpoint_copy = self.endpoint.clone(); let test_client_service: AnyhowJoinHandle = tokio::spawn(async move { info!("Sample Quic Client starting ..."); @@ -64,8 +65,6 @@ impl QuicTestClient { ticker.tick().await; } - - Ok(()) }); @@ -75,7 +74,6 @@ impl QuicTestClient { }, } } - } fn build_tls_config() -> ClientConfig { @@ -94,7 +92,6 @@ fn build_tls_config() -> ClientConfig { return client_crypto; } - fn build_memo_tx_raw() -> Vec { let payer_pubkey = Pubkey::new_unique(); let signer_pubkey = Pubkey::new_unique(); @@ -107,19 +104,19 @@ fn build_memo_tx_raw() -> Vec { // FIXME hardcoded to local test-validator "127.0.0.1:1027".parse().unwrap(), Pubkey::from_str("EPLzGRhibYmZ7qysF9BiPmSTRaL8GiLhrQdFTfL8h2fy").unwrap(), - vec![tx.into()]); + vec![tx.into()], + ); println!("wire_data: {:02X?}", wire_data); wire_data } - fn serialize_tpu_forwarding_request( tpu_socket_addr: SocketAddr, tpu_identity: Pubkey, - transactions: Vec) -> Vec { - + transactions: Vec, +) -> Vec { let request = TpuForwardingRequest::new(tpu_socket_addr, tpu_identity, transactions); bincode::serialize(&request).expect("Expect to serialize transactions") diff --git a/quic-forward-proxy/src/test_client/sample_data_factory.rs b/quic-forward-proxy/src/test_client/sample_data_factory.rs index 4542e9ea..ad7ad57a 100644 --- a/quic-forward-proxy/src/test_client/sample_data_factory.rs +++ b/quic-forward-proxy/src/test_client/sample_data_factory.rs @@ -1,21 +1,18 @@ -use std::path::Path; -use std::str::FromStr; use solana_sdk::hash::Hash; use solana_sdk::instruction::Instruction; use solana_sdk::message::Message; use solana_sdk::pubkey::Pubkey; -use solana_sdk::signature::{Keypair, keypair, Signer}; +use solana_sdk::signature::{keypair, Keypair, Signer}; use solana_sdk::transaction::{Transaction, VersionedTransaction}; +use std::path::Path; +use std::str::FromStr; const MEMO_PROGRAM_ID: &str = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"; - - pub fn build_raw_sample_tx() -> Vec { - - let payer_keypair = keypair::read_keypair_file( - Path::new("/Users/stefan/mango/solana-wallet/solana-testnet-stefantest.json") - ).unwrap(); - + let payer_keypair = keypair::read_keypair_file(Path::new( + "/Users/stefan/mango/solana-wallet/solana-testnet-stefantest.json", + )) + .unwrap(); let tx = build_sample_tx(&payer_keypair); @@ -37,5 +34,3 @@ pub fn create_memo_tx(msg: &[u8], payer: &Keypair, blockhash: Hash) -> Transacti let message = Message::new(&[instruction], Some(&payer.pubkey())); Transaction::new(&[payer], message, blockhash) } - - diff --git a/quic-forward-proxy/src/tls_config_provider_client.rs b/quic-forward-proxy/src/tls_config_provider_client.rs index 94ed1e22..7e0516ae 100644 --- a/quic-forward-proxy/src/tls_config_provider_client.rs +++ b/quic-forward-proxy/src/tls_config_provider_client.rs @@ -1,9 +1,7 @@ -use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; +use rustls::ClientConfig; // TODO integrate with tpu_service + quic_connection_utils pub trait TpuCLientTlsConfigProvider { - fn get_client_tls_crypto_config(&self) -> ClientConfig; - } diff --git a/quic-forward-proxy/src/tls_config_provider_server.rs b/quic-forward-proxy/src/tls_config_provider_server.rs index 9417ad79..159e6505 100644 --- a/quic-forward-proxy/src/tls_config_provider_server.rs +++ b/quic-forward-proxy/src/tls_config_provider_server.rs @@ -3,7 +3,5 @@ use rustls::ServerConfig; // TODO integrate with tpu_service + quic_connection_utils pub trait ProxyTlsConfigProvider { - fn get_server_tls_crypto_config(&self) -> ServerConfig; - } diff --git a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs index df0dbe6f..796cc154 100644 --- a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs +++ b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs @@ -1,9 +1,9 @@ -use std::sync::atomic::{AtomicU32, Ordering}; -use rcgen::generate_simple_self_signed; -use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; -use crate::quic_util::{ALPN_TPU_FORWARDPROXY_PROTOCOL_ID, SkipServerVerification}; +use crate::quic_util::{SkipServerVerification, ALPN_TPU_FORWARDPROXY_PROTOCOL_ID}; use crate::tls_config_provider_client::TpuCLientTlsConfigProvider; use crate::tls_config_provider_server::ProxyTlsConfigProvider; +use rcgen::generate_simple_self_signed; +use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; +use std::sync::atomic::{AtomicU32, Ordering}; impl ProxyTlsConfigProvider for SelfSignedTlsConfigProvider { fn get_server_tls_crypto_config(&self) -> ServerConfig { @@ -30,7 +30,11 @@ const INSTANCES: AtomicU32 = AtomicU32::new(0); impl SelfSignedTlsConfigProvider { pub fn new_singleton_self_signed_localhost() -> Self { // note: this check could be relaxed when you know what you are doing! - assert_eq!(INSTANCES.fetch_add(1, Ordering::Relaxed), 0, "should be a singleton"); + assert_eq!( + INSTANCES.fetch_add(1, Ordering::Relaxed), + 0, + "should be a singleton" + ); let hostnames = vec!["localhost".to_string()]; let (certificate, private_key) = Self::gen_tls_certificate_and_key(hostnames.clone()); let server_crypto = Self::build_server_crypto(certificate.clone(), private_key.clone()); @@ -76,7 +80,4 @@ impl SelfSignedTlsConfigProvider { pub fn get_client_tls_crypto_config(&self) -> &ClientConfig { &self.client_crypto } - } - - diff --git a/quic-forward-proxy/src/util.rs b/quic-forward-proxy/src/util.rs index 423a1f32..72cb2b2e 100644 --- a/quic-forward-proxy/src/util.rs +++ b/quic-forward-proxy/src/util.rs @@ -1,14 +1,14 @@ use std::future::Future; use std::time::Duration; -use futures::TryFutureExt; -use tokio::time::{Timeout, timeout}; + +use tokio::time::Timeout; pub type AnyhowJoinHandle = tokio::task::JoinHandle>; pub const FALLBACK_TIMEOUT: Duration = Duration::from_secs(5); pub fn timeout_fallback(future: F) -> Timeout - where - F: Future, +where + F: Future, { tokio::time::timeout(FALLBACK_TIMEOUT, future) } diff --git a/quic-forward-proxy/src/validator_identity.rs b/quic-forward-proxy/src/validator_identity.rs index 196864c7..3397b432 100644 --- a/quic-forward-proxy/src/validator_identity.rs +++ b/quic-forward-proxy/src/validator_identity.rs @@ -1,8 +1,8 @@ -use std::fmt::Display; -use std::sync::Arc; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::Keypair; use solana_sdk::signer::Signer; +use std::fmt::Display; +use std::sync::Arc; #[derive(Clone)] pub struct ValidatorIdentity { diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index a7622782..46694dd3 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -1,14 +1,14 @@ - -use std::str::FromStr; +use solana_lite_rpc_quic_forward_proxy::proxy_request_format::TpuForwardingRequest; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, Signer}; -use solana_sdk::transaction::{Transaction}; -use solana_lite_rpc_quic_forward_proxy::proxy_request_format::TpuForwardingRequest; +use solana_sdk::transaction::Transaction; +use std::str::FromStr; #[test] fn roundtrip() { - - let payer = Keypair::from_base58_string("rKiJ7H5UUp3JR18kNyTF1XPuwPKHEM7gMLWHZPWP5djrW1vSjfwjhvJrevxF9MPmUmN9gJMLHZdLMgc9ao78eKr"); + let payer = Keypair::from_base58_string( + "rKiJ7H5UUp3JR18kNyTF1XPuwPKHEM7gMLWHZPWP5djrW1vSjfwjhvJrevxF9MPmUmN9gJMLHZdLMgc9ao78eKr", + ); let payer_pubkey = payer.pubkey(); let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&payer_pubkey]); @@ -18,8 +18,9 @@ fn roundtrip() { let wire_data = TpuForwardingRequest::new( "127.0.0.1:5454".parse().unwrap(), Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), - vec![tx.into()] - ).serialize_wire_format(); + vec![tx.into()], + ) + .serialize_wire_format(); println!("wire_data: {:02X?}", wire_data); @@ -27,7 +28,4 @@ fn roundtrip() { assert_eq!(request.get_tpu_socket_addr().is_ipv4(), true); assert_eq!(request.get_transactions().len(), 1); - } - - From 3551c47450262845c60038c04f29271618d7ffdf Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 14:44:29 +0200 Subject: [PATCH 073/128] fix typo --- services/src/tpu_utils/quinn_auto_reconnect.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 13c4d074..9675d2da 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -64,7 +64,7 @@ impl AutoReconnect { Some(current) => { if current.close_reason().is_some() { - warn!("Connection i s closed for reason: {:?}", current.close_reason()); + warn!("Connection is closed for reason: {:?}", current.close_reason()); let new_connection = self.create_connection().await; let prev_stable_id = current.stable_id(); *lock = Some(new_connection.clone()); From a76c02d54155d2a138351a37e509075ab411b7e6 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 14:49:01 +0200 Subject: [PATCH 074/128] clean service integration --- .../src/tpu_utils/tpu_connection_manager.rs | 26 +------------------ services/src/tpu_utils/tpu_connection_path.rs | 7 ----- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 7de1e9a8..68a9b5f5 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -21,8 +21,6 @@ use std::{ }; use tokio::sync::{broadcast::Receiver, broadcast::Sender}; - - lazy_static::lazy_static! { static ref NB_QUIC_CONNECTIONS: GenericGauge = register_int_gauge!(opts!("literpc_nb_active_quic_connections", "Number of quic connections open")).unwrap(); @@ -161,27 +159,7 @@ impl ActiveConnection { tokio::spawn(async move { task_counter.fetch_add(1, Ordering::Relaxed); NB_QUIC_TASKS.inc(); - - // TODO split to new service - - connection_pool.send_transaction_batch(txs).await; - - // match self.tpu_connection_path { - // QuicDirect => { - // connection_pool.send_transaction_batch(txs).await; - // }, - // QuicForwardProxy { forward_proxy_address } => { - // info!("Sending copy of transaction batch of {} to tpu with identity {} to quic proxy", - // txs.len(), identity); - // Self::send_copy_of_txs_to_quicproxy( - // &txs, endpoint.clone(), - // // proxy address - // forward_proxy_address, - // tpu_address, - // identity.clone()).await.unwrap(); - // } - // } - + connection_pool.send_transaction_batch(txs).await; NB_QUIC_TASKS.dec(); task_counter.fetch_sub(1, Ordering::Relaxed); }); @@ -228,7 +206,6 @@ pub struct TpuConnectionManager { identity_to_active_connection: Arc>>, } - impl TpuConnectionManager { pub async fn new( certificate: rustls::Certificate, @@ -237,7 +214,6 @@ impl TpuConnectionManager { ) -> Self { let number_of_clients = fanout * 2; Self { - // TODO endpoints: RotatingQueue::new(number_of_clients, || { QuicConnectionUtils::create_endpoint(certificate.clone(), key.clone()) }) diff --git a/services/src/tpu_utils/tpu_connection_path.rs b/services/src/tpu_utils/tpu_connection_path.rs index f42bfe11..048f7c88 100644 --- a/services/src/tpu_utils/tpu_connection_path.rs +++ b/services/src/tpu_utils/tpu_connection_path.rs @@ -2,13 +2,6 @@ use std::fmt::Display; use std::net::SocketAddr; - - - - - - - #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum TpuConnectionPath { QuicDirectPath, From d03f73df95c816df5af5ceae308ff62697dd7b06 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 31 Jul 2023 14:53:19 +0200 Subject: [PATCH 075/128] align services/ --- services/src/tpu_utils/mod.rs | 1 + .../quic_proxy_connection_manager.rs | 2 +- .../src/tpu_utils/quinn_auto_reconnect.rs | 99 ++++++++++--------- 3 files changed, 53 insertions(+), 49 deletions(-) diff --git a/services/src/tpu_utils/mod.rs b/services/src/tpu_utils/mod.rs index b9ebedd1..24e94707 100644 --- a/services/src/tpu_utils/mod.rs +++ b/services/src/tpu_utils/mod.rs @@ -5,3 +5,4 @@ pub mod tpu_connection_manager; pub mod quic_proxy_connection_manager; pub mod quinn_auto_reconnect; + diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 5d469e97..a71af7e1 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -251,7 +251,7 @@ impl QuicProxyConnectionManager { let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); - let send_result = auto_connection.send(proxy_request_raw).await; + let send_result = auto_connection.send_uni(proxy_request_raw).await; // let send_result = // timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)) diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 9675d2da..8fb4bf2b 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -1,17 +1,19 @@ - +use anyhow::Context; +use log::warn; +use quinn::{Connection, Endpoint}; use std::fmt; use std::net::SocketAddr; use std::sync::atomic::{AtomicU32, Ordering}; -use log::{trace, warn}; -use tracing::{debug}; -use quinn::{Connection, Endpoint}; -use tokio::sync::{RwLock}; +use std::time::Duration; +use tokio::sync::RwLock; +use tokio::time::timeout; +use tracing::debug; pub struct AutoReconnect { + // endoint should be configures with keep-alive and idle timeout endpoint: Endpoint, - // note: no read lock is used ATM current: RwLock>, - target_address: SocketAddr, + pub target_address: SocketAddr, reconnect_count: AtomicU32, } @@ -25,24 +27,15 @@ impl AutoReconnect { } } - pub async fn roundtrip(&self, payload: Vec) -> anyhow::Result> { + pub async fn send_uni(&self, payload: Vec) -> anyhow::Result<()> { // TOOD do smart error handling + reconnect - // self.refresh().await.open_bi().await.unwrap() - let (mut send_stream, recv_stream) = self.refresh().await.open_bi().await?; + let mut send_stream = timeout( + Duration::from_secs(4), self.refresh() + .await.open_uni()) + .await + .context("open uni stream for sending")??; send_stream.write_all(payload.as_slice()).await?; send_stream.finish().await?; - - let answer = recv_stream.read_to_end(64 * 1024).await?; - - Ok(answer) - } - - pub async fn send(&self, payload: Vec) -> anyhow::Result<()> { - // TOOD do smart error handling + reconnect - let mut send_stream = self.refresh().await.open_uni().await?; - send_stream.write_all(payload.as_slice()).await?; - let _ = send_stream.finish().await; - Ok(()) } @@ -50,53 +43,65 @@ impl AutoReconnect { pub async fn refresh(&self) -> Connection { { let lock = self.current.read().await; - let maybe_conn: &Option = &*lock; - if maybe_conn.as_ref().filter(|conn| conn.close_reason().is_none()).is_some() { - // let reuse = lock.unwrap().clone(); - let reuse = maybe_conn.as_ref().unwrap(); - trace!("Reuse connection {}", reuse.stable_id()); + let maybe_conn = lock.as_ref(); + if maybe_conn + .filter(|conn| conn.close_reason().is_none()) + .is_some() + { + let reuse = maybe_conn.unwrap(); + debug!("Reuse connection {}", reuse.stable_id()); return reuse.clone(); } } let mut lock = self.current.write().await; - - match &*lock { + let maybe_conn = lock.as_ref(); + return match maybe_conn { Some(current) => { - if current.close_reason().is_some() { - warn!("Connection is closed for reason: {:?}", current.close_reason()); + let old_stable_id = current.stable_id(); + warn!( + "Connection {} is closed for reason: {:?}", + old_stable_id, + current.close_reason() + ); + let new_connection = self.create_connection().await; let prev_stable_id = current.stable_id(); *lock = Some(new_connection.clone()); // let old_conn = lock.replace(new_connection.clone()); self.reconnect_count.fetch_add(1, Ordering::SeqCst); - debug!("Replace closed connection {} with {} (retry {})", - prev_stable_id, + + debug!( + "Replace closed connection {} with {} (retry {})", + old_stable_id, new_connection.stable_id(), - self.reconnect_count.load(Ordering::SeqCst)); - // TODO log old vs new stable_id + self.reconnect_count.load(Ordering::SeqCst) + ); - return new_connection.clone(); + new_connection.clone() } else { - // TODO check log if that ever happens - warn!("Reuse connection {} with write-lock", current.stable_id()); - return current.clone(); + debug!("Reuse connection {} with write-lock", current.stable_id()); + current.clone() } - } None => { let new_connection = self.create_connection().await; + + assert!(lock.is_none(), "old connection must be None"); *lock = Some(new_connection.clone()); - trace!("Create initial connection {}", new_connection.stable_id()); + // let old_conn = foo.replace(Some(new_connection.clone())); + debug!("Create initial connection {}", new_connection.stable_id()); - return new_connection.clone(); + new_connection.clone() } - } + }; } async fn create_connection(&self) -> Connection { - let connection = - self.endpoint.connect(self.target_address, "localhost").expect("handshake"); + let connection = self + .endpoint + .connect(self.target_address, "localhost") + .expect("handshake"); connection.await.expect("connection") } @@ -104,9 +109,7 @@ impl AutoReconnect { impl fmt::Display for AutoReconnect { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Connection to {}", - self.target_address, - ) + write!(f, "Connection to {}", self.target_address,) } } From 89ad1bc136faad68ce890cbfed8b1966c57c67be Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 13:46:54 +0200 Subject: [PATCH 076/128] comment on selfsigned cert logic --- quic-forward-proxy/src/outbound/tx_forward.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index b26a30e4..e438f72f 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -141,6 +141,7 @@ async fn new_endpoint_with_validator_identity(validator_identity: ValidatorIdent "Setup TPU Quic stable connection with validator identity {} ...", validator_identity ); + // the counterpart of this function is get_remote_pubkey+get_pubkey_from_tls_certificate let (certificate, key) = new_self_signed_tls_certificate( &validator_identity.get_keypair_for_tls(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), From 2f97cb83858b61c755f4e7eb87a29085d5dd5e38 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 13:57:00 +0200 Subject: [PATCH 077/128] cleanup --- .../src/inbound/proxy_listener.rs | 6 +- quic-forward-proxy/src/lib.rs | 1 - quic-forward-proxy/src/main.rs | 11 +- quic-forward-proxy/src/outbound/tx_forward.rs | 1 - quic-forward-proxy/src/test_client/mod.rs | 2 - .../src/test_client/quic_test_client.rs | 123 ------------------ .../src/test_client/sample_data_factory.rs | 36 ----- 7 files changed, 4 insertions(+), 176 deletions(-) delete mode 100644 quic-forward-proxy/src/test_client/mod.rs delete mode 100644 quic-forward-proxy/src/test_client/quic_test_client.rs delete mode 100644 quic-forward-proxy/src/test_client/sample_data_factory.rs diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 858a6444..18b2a75d 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -13,8 +13,8 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::Sender; -// TODO tweak this value - solana server sets 256 -// setting this to "1" did not make a difference! +// note: setting this to "1" did not make a difference! +// solana server sets this to 256 const MAX_CONCURRENT_UNI_STREAMS: u32 = 24; pub struct ProxyListener { @@ -84,7 +84,6 @@ impl ProxyListener { // note: this config must be aligned with lite-rpc's client config let transport_config = Arc::get_mut(&mut quinn_server_config.transport).unwrap(); - // TODO experiment with this value transport_config.max_concurrent_uni_streams(VarInt::from_u32(MAX_CONCURRENT_UNI_STREAMS)); // no bidi streams used transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); @@ -98,7 +97,6 @@ impl ProxyListener { Endpoint::server(quinn_server_config, proxy_listener_addr).unwrap() } - // TODO use interface abstraction for connection_per_tpunode #[tracing::instrument(skip_all, level = "debug")] async fn accept_client_connection( client_connection: Connection, diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index dc88528b..ffc51a52 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -8,7 +8,6 @@ pub mod proxy_request_format; mod quic_util; mod quinn_auto_reconnect; mod shared; -mod test_client; pub mod tls_config_provider_client; pub mod tls_config_provider_server; pub mod tls_self_signed_pair_generator; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 96829e0e..2a929016 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -17,7 +17,6 @@ pub mod proxy_request_format; pub mod quic_util; mod quinn_auto_reconnect; mod shared; -pub mod test_client; pub mod tls_config_provider_client; pub mod tls_config_provider_server; pub mod tls_self_signed_pair_generator; @@ -30,13 +29,13 @@ pub async fn main() -> anyhow::Result<()> { let Args { identity_keypair, - proxy_listen_addr: proxy_rpc_addr, + proxy_listen_addr: proxy_listen_addr, } = Args::parse(); dotenv().ok(); // TODO build args struct dedicated to proxy - let proxy_listener_addr = proxy_rpc_addr.parse().unwrap(); + let proxy_listener_addr = proxy_listen_addr.parse().unwrap(); let _tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let validator_identity = ValidatorIdentity::new(get_identity_keypair(&identity_keypair).await); @@ -45,12 +44,6 @@ pub async fn main() -> anyhow::Result<()> { .await? .start_services(); - // let proxy_addr = "127.0.0.1:11111".parse().unwrap(); - // let test_client = QuicTestClient::new_with_endpoint( - // proxy_addr, &tls_configuration) - // .await? - // .start_services(); - let ctrl_c_signal = tokio::signal::ctrl_c(); tokio::select! { diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index e438f72f..f436d1f4 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -51,7 +51,6 @@ pub async fn tx_forwarder( .recv() .await .expect("channel closed unexpectedly"); - // TODO drain the queue with .try_recv() and batch the transactions let tpu_address = forward_packet.tpu_address; if !agents.contains_key(&tpu_address) { diff --git a/quic-forward-proxy/src/test_client/mod.rs b/quic-forward-proxy/src/test_client/mod.rs deleted file mode 100644 index c86b5c98..00000000 --- a/quic-forward-proxy/src/test_client/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod quic_test_client; -mod sample_data_factory; diff --git a/quic-forward-proxy/src/test_client/quic_test_client.rs b/quic-forward-proxy/src/test_client/quic_test_client.rs deleted file mode 100644 index ef4d704a..00000000 --- a/quic-forward-proxy/src/test_client/quic_test_client.rs +++ /dev/null @@ -1,123 +0,0 @@ -use anyhow::bail; -use std::net::SocketAddr; -use std::str::FromStr; -use std::sync::Arc; -use std::time::Duration; - -use crate::proxy_request_format::TpuForwardingRequest; -use crate::quic_util::{SkipServerVerification, ALPN_TPU_FORWARDPROXY_PROTOCOL_ID}; -use crate::tls_config_provider_client::TpuCLientTlsConfigProvider; -use crate::util::AnyhowJoinHandle; -use log::{info, trace}; -use quinn::{Endpoint, VarInt}; -use rustls::ClientConfig; -use solana_sdk::pubkey::Pubkey; -use solana_sdk::transaction::{Transaction, VersionedTransaction}; -use tokio::io::AsyncWriteExt; - -pub struct QuicTestClient { - pub endpoint: Endpoint, - pub proxy_addr: SocketAddr, -} - -impl QuicTestClient { - pub async fn new_with_endpoint( - proxy_addr: SocketAddr, - tls_config: &impl TpuCLientTlsConfigProvider, - ) -> anyhow::Result { - let client_crypto = tls_config.get_client_tls_crypto_config(); - let mut endpoint = quinn::Endpoint::client("0.0.0.0:0".parse().unwrap())?; - endpoint.set_default_client_config(quinn::ClientConfig::new(Arc::new(client_crypto))); - - Ok(Self { - proxy_addr, - endpoint, - }) - } - - // connect to a server - pub async fn start_services(self) -> anyhow::Result<()> { - let endpoint_copy = self.endpoint.clone(); - let test_client_service: AnyhowJoinHandle = tokio::spawn(async move { - info!("Sample Quic Client starting ..."); - - let mut ticker = tokio::time::interval(Duration::from_secs(3)); - // TODO exit signal - loop { - // create new connection everytime - let connection_timeout = Duration::from_secs(5); - let connecting = endpoint_copy.connect(self.proxy_addr, "localhost").unwrap(); - let connection = tokio::time::timeout(connection_timeout, connecting).await??; - - for _si in 0..5 { - let mut send = connection.open_uni().await?; - - let raw = build_memo_tx_raw(); - trace!("raw: {:02X?}", raw); - // send.write_all(format!("SAMPLE DATA on stream {}", si).as_bytes()).await?; - send.write_all(&raw).await?; - - // shutdown stream - send.finish().await?; - } - - connection.close(VarInt::from_u32(0), b"done"); - ticker.tick().await; - } - - Ok(()) - }); - - tokio::select! { - res = test_client_service => { - bail!("Sample client service exited unexpectedly {res:?}"); - }, - } - } -} - -fn build_tls_config() -> ClientConfig { - // FIXME configured insecure https://quinn-rs.github.io/quinn/quinn/certificate.html - let mut _roots = rustls::RootCertStore::empty(); - // TODO add certs - - let mut client_crypto = rustls::ClientConfig::builder() - .with_safe_defaults() - // .with_root_certificates(roots) - .with_custom_certificate_verifier(SkipServerVerification::new()) - .with_no_client_auth(); - client_crypto.enable_early_data = true; - client_crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; - - return client_crypto; -} - -fn build_memo_tx_raw() -> Vec { - let payer_pubkey = Pubkey::new_unique(); - let signer_pubkey = Pubkey::new_unique(); - - let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&signer_pubkey]); - - let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); - - let wire_data = serialize_tpu_forwarding_request( - // FIXME hardcoded to local test-validator - "127.0.0.1:1027".parse().unwrap(), - Pubkey::from_str("EPLzGRhibYmZ7qysF9BiPmSTRaL8GiLhrQdFTfL8h2fy").unwrap(), - vec![tx.into()], - ); - - println!("wire_data: {:02X?}", wire_data); - - wire_data -} - -fn serialize_tpu_forwarding_request( - tpu_socket_addr: SocketAddr, - tpu_identity: Pubkey, - transactions: Vec, -) -> Vec { - let request = TpuForwardingRequest::new(tpu_socket_addr, tpu_identity, transactions); - - bincode::serialize(&request).expect("Expect to serialize transactions") -} diff --git a/quic-forward-proxy/src/test_client/sample_data_factory.rs b/quic-forward-proxy/src/test_client/sample_data_factory.rs deleted file mode 100644 index ad7ad57a..00000000 --- a/quic-forward-proxy/src/test_client/sample_data_factory.rs +++ /dev/null @@ -1,36 +0,0 @@ -use solana_sdk::hash::Hash; -use solana_sdk::instruction::Instruction; -use solana_sdk::message::Message; -use solana_sdk::pubkey::Pubkey; -use solana_sdk::signature::{keypair, Keypair, Signer}; -use solana_sdk::transaction::{Transaction, VersionedTransaction}; -use std::path::Path; -use std::str::FromStr; -const MEMO_PROGRAM_ID: &str = "MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr"; - -pub fn build_raw_sample_tx() -> Vec { - let payer_keypair = keypair::read_keypair_file(Path::new( - "/Users/stefan/mango/solana-wallet/solana-testnet-stefantest.json", - )) - .unwrap(); - - let tx = build_sample_tx(&payer_keypair); - - let raw_tx = bincode::serialize::(&tx).expect("failed to serialize tx"); - - raw_tx -} - -fn build_sample_tx(payer_keypair: &Keypair) -> VersionedTransaction { - let blockhash = Hash::default(); - create_memo_tx(b"hi", payer_keypair, blockhash).into() -} - -// from bench helpers -pub fn create_memo_tx(msg: &[u8], payer: &Keypair, blockhash: Hash) -> Transaction { - let memo = Pubkey::from_str(MEMO_PROGRAM_ID).unwrap(); - - let instruction = Instruction::new_with_bytes(memo, msg, vec![]); - let message = Message::new(&[instruction], Some(&payer.pubkey())); - Transaction::new(&[payer], message, blockhash) -} From b41d05e6132836cc817388611690e2302b26e277 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 14:15:39 +0200 Subject: [PATCH 078/128] cleanup minor todos --- core/src/proxy_request_format.rs | 5 +-- .../tests/quic_proxy_tpu_integrationtest.rs | 19 +++------- quic-forward-proxy/src/main.rs | 3 -- .../src/proxy_request_format.rs | 2 +- .../src/tls_config_provider_client.rs | 4 +- .../src/tls_config_provider_server.rs | 2 - .../src/tls_self_signed_pair_generator.rs | 4 +- .../quic_proxy_connection_manager.rs | 38 +++---------------- services/src/tpu_utils/tpu_service.rs | 4 +- 9 files changed, 19 insertions(+), 62 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index a03a8e49..53ddc8a3 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -10,12 +10,12 @@ use solana_sdk::transaction::VersionedTransaction; /// lite-rpc to proxy wire format /// compat info: non-public format ATM /// initial version -const FORMAT_VERSION1: u16 = 2301; +const FORMAT_VERSION1: u16 = 2302; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { format_version: u16, - tpu_socket_addr: SocketAddr, // TODO is that correct, maybe it should be V4; maybe we also need to provide a list + tpu_socket_addr: SocketAddr, identity_tpunode: Pubkey, transactions: Vec, } @@ -43,7 +43,6 @@ impl TpuForwardingRequest { bincode::serialize(&self).expect("Expect to serialize transactions") } - // TODO reame pub fn deserialize_from_raw_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { let request = bincode::deserialize::(&raw_proxy_request) .context("deserialize proxy request") diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 3f4a7a51..9ed289f3 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -559,9 +559,6 @@ async fn start_literpc_client_direct_mode( ) .await; - // TODO this is a race - sleep(Duration::from_millis(1500)).await; - for i in 0..test_case_params.sample_tx_count { let raw_sample_tx = build_raw_sample_tx(i); trace!( @@ -582,27 +579,22 @@ async fn start_literpc_client_direct_mode( async fn start_literpc_client_proxy_mode( test_case_params: TestCaseParams, streamer_listen_addrs: SocketAddr, - literpc_validator_identity: Arc, + validator_identity: Arc, forward_proxy_address: SocketAddr, ) -> anyhow::Result<()> { info!("Start lite-rpc test client using quic proxy at {} ...", forward_proxy_address); - let _fanout_slots = 4; - // (String, Vec) (signature, transaction) let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); let broadcast_sender = Arc::new(sender); let (certificate, key) = new_self_signed_tls_certificate( - literpc_validator_identity.as_ref(), + validator_identity.as_ref(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) .expect("Failed to initialize QUIC connection certificates"); - // let tpu_connection_manager = - // TpuConnectionManager::new(certificate, key, fanout_slots as usize).await; - let quic_proxy_connection_manager = - QuicProxyConnectionManager::new(certificate, key, literpc_validator_identity.clone(), forward_proxy_address).await; + QuicProxyConnectionManager::new(certificate, key, forward_proxy_address).await; // this effectively controls how many connections we will have let mut connections_to_keep: HashMap = HashMap::new(); @@ -625,7 +617,7 @@ async fn start_literpc_client_proxy_mode( ); // this is the real streamer - connections_to_keep.insert(literpc_validator_identity.pubkey(), streamer_listen_addrs); + connections_to_keep.insert(validator_identity.pubkey(), streamer_listen_addrs); // get information about the optional validator identity stake // populated from get_stakes_for_identity() @@ -650,7 +642,8 @@ async fn start_literpc_client_proxy_mode( // ) // .await; - quic_proxy_connection_manager.update_connection(broadcast_sender.clone(), connections_to_keep).await; + quic_proxy_connection_manager.update_connection( + broadcast_sender.clone(), connections_to_keep, QUIC_CONNECTION_PARAMS).await; // TODO this is a race sleep(Duration::from_millis(1500)).await; diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 2a929016..163add9c 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -31,12 +31,9 @@ pub async fn main() -> anyhow::Result<()> { identity_keypair, proxy_listen_addr: proxy_listen_addr, } = Args::parse(); - dotenv().ok(); - // TODO build args struct dedicated to proxy let proxy_listener_addr = proxy_listen_addr.parse().unwrap(); - let _tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let validator_identity = ValidatorIdentity::new(get_identity_keypair(&identity_keypair).await); let tls_config = Arc::new(SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost()); diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index bc503ffc..8345175e 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -10,7 +10,7 @@ use std::net::SocketAddr; /// lite-rpc to proxy wire format /// compat info: non-public format ATM /// initial version -pub const FORMAT_VERSION1: u16 = 2301; +pub const FORMAT_VERSION1: u16 = 2302; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { diff --git a/quic-forward-proxy/src/tls_config_provider_client.rs b/quic-forward-proxy/src/tls_config_provider_client.rs index 7e0516ae..32c3e9e7 100644 --- a/quic-forward-proxy/src/tls_config_provider_client.rs +++ b/quic-forward-proxy/src/tls_config_provider_client.rs @@ -1,7 +1,5 @@ use rustls::ClientConfig; -// TODO integrate with tpu_service + quic_connection_utils - -pub trait TpuCLientTlsConfigProvider { +pub trait TpuClientTlsConfigProvider { fn get_client_tls_crypto_config(&self) -> ClientConfig; } diff --git a/quic-forward-proxy/src/tls_config_provider_server.rs b/quic-forward-proxy/src/tls_config_provider_server.rs index 159e6505..b1fb2a28 100644 --- a/quic-forward-proxy/src/tls_config_provider_server.rs +++ b/quic-forward-proxy/src/tls_config_provider_server.rs @@ -1,7 +1,5 @@ use rustls::ServerConfig; -// TODO integrate with tpu_service + quic_connection_utils - pub trait ProxyTlsConfigProvider { fn get_server_tls_crypto_config(&self) -> ServerConfig; } diff --git a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs index 796cc154..3abae350 100644 --- a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs +++ b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs @@ -1,5 +1,5 @@ use crate::quic_util::{SkipServerVerification, ALPN_TPU_FORWARDPROXY_PROTOCOL_ID}; -use crate::tls_config_provider_client::TpuCLientTlsConfigProvider; +use crate::tls_config_provider_client::TpuClientTlsConfigProvider; use crate::tls_config_provider_server::ProxyTlsConfigProvider; use rcgen::generate_simple_self_signed; use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; @@ -11,7 +11,7 @@ impl ProxyTlsConfigProvider for SelfSignedTlsConfigProvider { } } -impl TpuCLientTlsConfigProvider for SelfSignedTlsConfigProvider { +impl TpuClientTlsConfigProvider for SelfSignedTlsConfigProvider { fn get_client_tls_crypto_config(&self) -> ClientConfig { self.client_crypto.clone() } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index a71af7e1..d86acd22 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -21,7 +21,7 @@ use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; use tokio::time::timeout; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -use solana_lite_rpc_core::quic_connection_utils::{connection_stats, SkipServerVerification}; +use solana_lite_rpc_core::quic_connection_utils::{connection_stats, QuicConnectionParameters, SkipServerVerification}; use crate::tpu_utils::quinn_auto_reconnect::AutoReconnect; @@ -34,21 +34,17 @@ pub struct TpuNode { pub struct QuicProxyConnectionManager { endpoint: Endpoint, - // TODO remove - validator_identity: Arc, simple_thread_started: AtomicBool, proxy_addr: SocketAddr, current_tpu_nodes: Arc>> } -// TODO consolidate with number_of_transactions_per_unistream const CHUNK_SIZE_PER_STREAM: usize = 20; impl QuicProxyConnectionManager { pub async fn new( certificate: rustls::Certificate, key: rustls::PrivateKey, - validator_identity: Arc, proxy_addr: SocketAddr, ) -> Self { info!("Configure Quic proxy connection manager to {}", proxy_addr); @@ -56,7 +52,6 @@ impl QuicProxyConnectionManager { Self { endpoint, - validator_identity, simple_thread_started: AtomicBool::from(false), proxy_addr, current_tpu_nodes: Arc::new(RwLock::new(vec![])), @@ -68,10 +63,10 @@ impl QuicProxyConnectionManager { transaction_sender: Arc)>>, // for duration of this slot these tpu nodes will receive the transactions connections_to_keep: HashMap, + connection_parameters: QuicConnectionParameters, ) { debug!("reconfigure quic proxy connection (# of tpu nodes: {})", connections_to_keep.len()); - { let list_of_nodes = connections_to_keep.iter().map(|(identity, tpu_address)| { TpuNode { @@ -95,7 +90,6 @@ impl QuicProxyConnectionManager { let transaction_receiver = transaction_sender.subscribe(); - // TODO use it let exit_signal = Arc::new(AtomicBool::new(false)); tokio::spawn(Self::read_transactions_and_broadcast( @@ -104,6 +98,7 @@ impl QuicProxyConnectionManager { self.proxy_addr, self.endpoint.clone(), exit_signal, + connection_parameters, )); } @@ -149,18 +144,15 @@ impl QuicProxyConnectionManager { endpoint } - // blocks and loops until exit signal (TODO) async fn read_transactions_and_broadcast( mut transaction_receiver: Receiver<(String, Vec)>, current_tpu_nodes: Arc>>, proxy_addr: SocketAddr, endpoint: Endpoint, exit_signal: Arc, + connection_parameters: QuicConnectionParameters, ) { - // let mut connection = endpoint.connect(proxy_addr, "localhost").unwrap() - // .await.unwrap(); - let auto_connection = AutoReconnect::new(endpoint, proxy_addr); loop { @@ -173,13 +165,8 @@ impl QuicProxyConnectionManager { // TODO add timeout tx = transaction_receiver.recv() => { - let first_tx: Vec = match tx { Ok((_sig, tx)) => { - // if Self::check_for_confirmation(&txs_sent_store, sig) { - // // transaction is already confirmed/ no need to send - // continue; - // } tx }, Err(e) => { @@ -189,16 +176,9 @@ impl QuicProxyConnectionManager { } }; - let number_of_transactions_per_unistream = 8; // TODO read from QuicConnectionParameters - let mut txs = vec![first_tx]; - // TODO comment in - let _foo = PACKET_DATA_SIZE; - for _ in 1..number_of_transactions_per_unistream { + for _ in 1..connection_parameters.number_of_transactions_per_unistream { if let Ok((_signature, tx)) = transaction_receiver.try_recv() { - // if Self::check_for_confirmation(&txs_sent_store, signature) { - // continue; - // } txs.push(tx); } } @@ -226,9 +206,6 @@ impl QuicProxyConnectionManager { _proxy_address: SocketAddr, tpu_target_address: SocketAddr, target_tpu_identity: Pubkey) -> anyhow::Result<()> { - // TODO add timeout - // let mut send_stream = timeout(Duration::from_millis(500), connection.open_uni()).await??; - let raw_tx_batch_copy = raw_tx_batch.clone(); let mut txs = vec![]; @@ -253,10 +230,6 @@ impl QuicProxyConnectionManager { let send_result = auto_connection.send_uni(proxy_request_raw).await; - // let send_result = - // timeout(Duration::from_millis(3500), Self::send_proxy_request(endpoint, proxy_address, &proxy_request_raw)) - // .await.context("Timeout sending data to quic proxy")?; - match send_result { Ok(()) => { debug!("Successfully sent {} txs to quic proxy", txs.len()); @@ -272,7 +245,6 @@ impl QuicProxyConnectionManager { Ok(()) } - // TODO optimize connection async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { info!("sending {} bytes to proxy", proxy_request_raw.len()); diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 5dffa2e2..27d2e882 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -103,7 +103,7 @@ impl TpuService { } TpuConnectionPath::QuicForwardProxyPath { forward_proxy_address } => { let quic_proxy_connection_manager = - QuicProxyConnectionManager::new(certificate, key, identity.clone(), forward_proxy_address).await; + QuicProxyConnectionManager::new(certificate, key, forward_proxy_address).await; QuicProxy { quic_proxy_connection_manager: Arc::new(quic_proxy_connection_manager), @@ -196,10 +196,10 @@ impl TpuService { .await; }, QuicProxy { quic_proxy_connection_manager } => { - // TODO maybe we need more data (see .update_connections) quic_proxy_connection_manager.update_connection( self.broadcast_sender.clone(), connections_to_keep, + self.config.quic_connection_params, ).await; } } From 92a977fb984e1bc9447367820019302810f6a9b9 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 14:26:13 +0200 Subject: [PATCH 079/128] clippy --- quic-forward-proxy/src/outbound/tx_forward.rs | 9 +++------ quic-forward-proxy/src/proxy_request_format.rs | 2 +- quic-forward-proxy/src/quinn_auto_reconnect.rs | 8 ++++---- .../src/tls_self_signed_pair_generator.rs | 13 +++---------- quic-forward-proxy/src/validator_identity.rs | 4 ++-- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index f436d1f4..b6977c65 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -147,9 +147,7 @@ async fn new_endpoint_with_validator_identity(validator_identity: ValidatorIdent ) .expect("Failed to initialize QUIC connection certificates"); - let endpoint_outbound = create_tpu_client_endpoint(certificate.clone(), key.clone()); - - endpoint_outbound + create_tpu_client_endpoint(certificate, key) } fn create_tpu_client_endpoint( @@ -196,13 +194,12 @@ fn create_tpu_client_endpoint( // send potentially large amount of transactions to a single TPU #[tracing::instrument(skip_all, level = "debug")] -async fn send_tx_batch_to_tpu(auto_connection: &AutoReconnect, txs: &Vec) { +async fn send_tx_batch_to_tpu(auto_connection: &AutoReconnect, txs: &[VersionedTransaction]) { for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { let all_send_fns = chunk .iter() .map(|tx| { - let tx_raw = bincode::serialize(tx).unwrap(); - tx_raw + bincode::serialize(tx).unwrap() }) .map(|tx_raw| { auto_connection.send_uni(tx_raw) // ignores error diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 8345175e..b0b7918a 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -53,7 +53,7 @@ impl TpuForwardingRequest { // TODO reame pub fn deserialize_from_raw_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { - let request = bincode::deserialize::(&raw_proxy_request) + let request = bincode::deserialize::(raw_proxy_request) .context("deserialize proxy request") .unwrap(); diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 5b04642c..e3f62759 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -51,7 +51,7 @@ impl AutoReconnect { } let mut lock = self.current.write().await; let maybe_conn = lock.as_ref(); - return match maybe_conn { + match maybe_conn { Some(current) => { if current.close_reason().is_some() { let old_stable_id = current.stable_id(); @@ -73,7 +73,7 @@ impl AutoReconnect { self.reconnect_count.load(Ordering::SeqCst) ); - new_connection.clone() + new_connection } else { debug!("Reuse connection {} with write-lock", current.stable_id()); current.clone() @@ -87,9 +87,9 @@ impl AutoReconnect { // let old_conn = foo.replace(Some(new_connection.clone())); debug!("Create initial connection {}", new_connection.stable_id()); - new_connection.clone() + new_connection } - }; + } } async fn create_connection(&self) -> Connection { diff --git a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs index 3abae350..2a6ccbd8 100644 --- a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs +++ b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs @@ -25,16 +25,9 @@ pub struct SelfSignedTlsConfigProvider { server_crypto: ServerConfig, } -const INSTANCES: AtomicU32 = AtomicU32::new(0); - impl SelfSignedTlsConfigProvider { pub fn new_singleton_self_signed_localhost() -> Self { // note: this check could be relaxed when you know what you are doing! - assert_eq!( - INSTANCES.fetch_add(1, Ordering::Relaxed), - 0, - "should be a singleton" - ); let hostnames = vec!["localhost".to_string()]; let (certificate, private_key) = Self::gen_tls_certificate_and_key(hostnames.clone()); let server_crypto = Self::build_server_crypto(certificate.clone(), private_key.clone()); @@ -43,7 +36,7 @@ impl SelfSignedTlsConfigProvider { certificate, private_key, client_crypto: Self::build_client_crypto_insecure(), - server_crypto: server_crypto, + server_crypto, } } @@ -61,7 +54,7 @@ impl SelfSignedTlsConfigProvider { .with_no_client_auth(); client_crypto.enable_early_data = true; client_crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; - return client_crypto; + client_crypto } fn build_server_crypto(server_cert: Certificate, server_key: PrivateKey) -> ServerConfig { @@ -74,7 +67,7 @@ impl SelfSignedTlsConfigProvider { .with_single_cert(vec![server_cert], server_key) .unwrap(); server_crypto.alpn_protocols = vec![ALPN_TPU_FORWARDPROXY_PROTOCOL_ID.to_vec()]; - return server_crypto; + server_crypto } pub fn get_client_tls_crypto_config(&self) -> &ClientConfig { diff --git a/quic-forward-proxy/src/validator_identity.rs b/quic-forward-proxy/src/validator_identity.rs index 3397b432..df4f659f 100644 --- a/quic-forward-proxy/src/validator_identity.rs +++ b/quic-forward-proxy/src/validator_identity.rs @@ -14,7 +14,7 @@ impl ValidatorIdentity { pub fn new(keypair: Option) -> Self { let dummy_keypair = Keypair::new(); ValidatorIdentity { - keypair: keypair.map(|kp| Arc::new(kp)), + keypair: keypair.map(Arc::new), dummy_keypair: Arc::new(dummy_keypair), } } @@ -38,7 +38,7 @@ impl ValidatorIdentity { impl Display for ValidatorIdentity { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self.keypair { - Some(keypair) => write!(f, "{}", keypair.pubkey().to_string()), + Some(keypair) => write!(f, "{}", keypair.pubkey()), None => write!(f, "no keypair"), } } From 12fb60267feb346379a8cacad1547e8543dcbaf2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 14:55:11 +0200 Subject: [PATCH 080/128] fix all clippy errors --- quic-forward-proxy/src/cli.rs | 26 ----------------- .../src/inbound/proxy_listener.rs | 11 ++++---- quic-forward-proxy/src/main.rs | 5 ++-- quic-forward-proxy/src/outbound/tx_forward.rs | 28 ++++++------------- .../src/proxy_request_format.rs | 2 +- .../src/quinn_auto_reconnect.rs | 23 +++++++++++++++ .../src/tls_self_signed_pair_generator.rs | 11 ++------ quic-forward-proxy/src/util.rs | 28 +++++++++++++++++++ .../tests/proxy_request_format.rs | 2 +- 9 files changed, 72 insertions(+), 64 deletions(-) diff --git a/quic-forward-proxy/src/cli.rs b/quic-forward-proxy/src/cli.rs index cfdd1608..77ba3a7d 100644 --- a/quic-forward-proxy/src/cli.rs +++ b/quic-forward-proxy/src/cli.rs @@ -1,6 +1,4 @@ use clap::Parser; -use solana_sdk::signature::Keypair; -use std::env; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -10,27 +8,3 @@ pub struct Args { #[arg(short = 'l', long)] pub proxy_listen_addr: String, } - -// note this is duplicated from lite-rpc module -pub async fn get_identity_keypair(identity_from_cli: &String) -> Option { - if let Ok(identity_env_var) = env::var("IDENTITY") { - if let Ok(identity_bytes) = serde_json::from_str::>(identity_env_var.as_str()) { - Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap()) - } else { - // must be a file - let identity_file = tokio::fs::read_to_string(identity_env_var.as_str()) - .await - .expect("Cannot find the identity file provided"); - let identity_bytes: Vec = serde_json::from_str(&identity_file).unwrap(); - Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap()) - } - } else if identity_from_cli.is_empty() { - None - } else { - let identity_file = tokio::fs::read_to_string(identity_from_cli.as_str()) - .await - .expect("Cannot find the identity file provided"); - let identity_bytes: Vec = serde_json::from_str(&identity_file).unwrap(); - Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap()) - } -} diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 18b2a75d..2bf509fe 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -12,6 +12,7 @@ use std::sync::atomic::AtomicBool; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::Sender; +use crate::quic_util::connection_stats; // note: setting this to "1" did not make a difference! // solana server sets this to 256 @@ -110,7 +111,7 @@ impl ProxyListener { loop { let maybe_stream = client_connection.accept_uni().await; - let result = match maybe_stream { + match maybe_stream { Err(quinn::ConnectionError::ApplicationClosed(reason)) => { debug!("connection closed by client - reason: {:?}", reason); if reason.error_code != VarInt::from_u32(0) { @@ -124,7 +125,7 @@ impl ProxyListener { } Err(e) => { error!("failed to accept stream: {}", e); - return Err(anyhow::Error::msg("error accepting stream")); + bail!("error accepting stream"); } Ok(recv_stream) => { let forwarder_channel_copy = forwarder_channel.clone(); @@ -157,13 +158,11 @@ impl ProxyListener { .unwrap(); }); - Ok(()) + debug!("Inbound connection stats: {}", connection_stats(&client_connection)); + } }; // -- result - if let Err(e) = result { - return Err(e); - } } // -- loop } } diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 163add9c..97e467bd 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -1,4 +1,4 @@ -use crate::cli::{get_identity_keypair, Args}; +use crate::cli::Args; use crate::proxy::QuicForwardProxy; use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use anyhow::bail; @@ -6,6 +6,7 @@ use clap::Parser; use dotenv::dotenv; use log::info; use std::sync::Arc; +use crate::util::get_identity_keypair; use crate::validator_identity::ValidatorIdentity; @@ -29,7 +30,7 @@ pub async fn main() -> anyhow::Result<()> { let Args { identity_keypair, - proxy_listen_addr: proxy_listen_addr, + proxy_listen_addr, } = Args::parse(); dotenv().ok(); diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index b6977c65..78beaba2 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -6,7 +6,6 @@ use crate::validator_identity::ValidatorIdentity; use anyhow::{bail, Context}; use fan::tokio::mpsc::FanOut; use futures::future::join_all; -use itertools::Itertools; use log::{debug, info, warn}; use quinn::{ ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt, @@ -22,11 +21,6 @@ use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::{channel, Receiver}; -const QUIC_CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); -pub const CONNECTION_RETRY_COUNT: usize = 10; - -pub const MAX_TRANSACTIONS_PER_BATCH: usize = 10; -pub const MAX_BYTES_PER_BATCH: usize = 10; const MAX_PARALLEL_STREAMS: usize = 6; pub const PARALLEL_TPU_CONNECTION_COUNT: usize = 4; @@ -53,8 +47,7 @@ pub async fn tx_forwarder( .expect("channel closed unexpectedly"); let tpu_address = forward_packet.tpu_address; - if !agents.contains_key(&tpu_address) { - // TODO cleanup agent after a while of iactivity + agents.entry(tpu_address).or_insert_with(|| { let mut senders = Vec::new(); for connection_idx in 1..PARALLEL_TPU_CONNECTION_COUNT { @@ -79,10 +72,8 @@ pub async fn tx_forwarder( let mut transactions_batch = packet.transactions; - let mut batch_size = 1; while let Ok(more) = receiver.try_recv() { transactions_batch.extend(more.transactions); - batch_size += 1; } debug!( @@ -95,11 +86,11 @@ pub async fn tx_forwarder( &auto_connection, &transactions_batch, )) - .await - .context(format!( - "send txs to tpu node {}", - auto_connection.target_address - )); + .await + .context(format!( + "send txs to tpu node {}", + auto_connection.target_address + )); if result.is_err() { warn!( @@ -108,6 +99,7 @@ pub async fn tx_forwarder( ); } else { debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); + debug!("Outbound connection stats: {}", &auto_connection.connection_stats().await); } } // -- while all packtes from channel @@ -118,10 +110,8 @@ pub async fn tx_forwarder( }); } - let fanout = FanOut::new(senders); - - agents.insert(tpu_address, fanout); - } // -- new agent + FanOut::new(senders) + }); // -- new agent let agent_channel = agents.get(&tpu_address).unwrap(); diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index b0b7918a..9b8b5761 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -52,7 +52,7 @@ impl TpuForwardingRequest { } // TODO reame - pub fn deserialize_from_raw_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { + pub fn deserialize_from_raw_request(raw_proxy_request: &[u8]) -> TpuForwardingRequest { let request = bincode::deserialize::(raw_proxy_request) .context("deserialize proxy request") .unwrap(); diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index e3f62759..26e69dd7 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -100,6 +100,29 @@ impl AutoReconnect { connection.await.expect("connection") } + + + // stable_id 140266619216912, rtt=2.156683ms, + // stats FrameStats { ACK: 3, CONNECTION_CLOSE: 0, CRYPTO: 3, + // DATA_BLOCKED: 0, DATAGRAM: 0, HANDSHAKE_DONE: 1, MAX_DATA: 0, + // MAX_STREAM_DATA: 1, MAX_STREAMS_BIDI: 0, MAX_STREAMS_UNI: 0, NEW_CONNECTION_ID: 4, + // NEW_TOKEN: 0, PATH_CHALLENGE: 0, PATH_RESPONSE: 0, PING: 0, RESET_STREAM: 0, + // RETIRE_CONNECTION_ID: 1, STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, + // STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } + pub async fn connection_stats(&self) -> String { + let lock = self.current.read().await; + let maybe_conn = lock.as_ref(); + match maybe_conn { + Some(connection) => format!( + "stable_id {} stats {:?}, rtt={:?}", + connection.stable_id(), + connection.stats().frame_rx, + connection.stats().path.rtt + ), + None => "n/a".to_string(), + } + } + } impl fmt::Display for AutoReconnect { diff --git a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs index 2a6ccbd8..79052b38 100644 --- a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs +++ b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs @@ -3,7 +3,6 @@ use crate::tls_config_provider_client::TpuClientTlsConfigProvider; use crate::tls_config_provider_server::ProxyTlsConfigProvider; use rcgen::generate_simple_self_signed; use rustls::{Certificate, ClientConfig, PrivateKey, ServerConfig}; -use std::sync::atomic::{AtomicU32, Ordering}; impl ProxyTlsConfigProvider for SelfSignedTlsConfigProvider { fn get_server_tls_crypto_config(&self) -> ServerConfig { @@ -18,9 +17,6 @@ impl TpuClientTlsConfigProvider for SelfSignedTlsConfigProvider { } pub struct SelfSignedTlsConfigProvider { - hostnames: Vec, - certificate: Certificate, - private_key: PrivateKey, client_crypto: ClientConfig, server_crypto: ServerConfig, } @@ -29,12 +25,9 @@ impl SelfSignedTlsConfigProvider { pub fn new_singleton_self_signed_localhost() -> Self { // note: this check could be relaxed when you know what you are doing! let hostnames = vec!["localhost".to_string()]; - let (certificate, private_key) = Self::gen_tls_certificate_and_key(hostnames.clone()); - let server_crypto = Self::build_server_crypto(certificate.clone(), private_key.clone()); + let (certificate, private_key) = Self::gen_tls_certificate_and_key(hostnames); + let server_crypto = Self::build_server_crypto(certificate, private_key); Self { - hostnames, - certificate, - private_key, client_crypto: Self::build_client_crypto_insecure(), server_crypto, } diff --git a/quic-forward-proxy/src/util.rs b/quic-forward-proxy/src/util.rs index 72cb2b2e..87ded89a 100644 --- a/quic-forward-proxy/src/util.rs +++ b/quic-forward-proxy/src/util.rs @@ -1,5 +1,8 @@ +#![allow(dead_code)] +use std::env; use std::future::Future; use std::time::Duration; +use solana_sdk::signature::Keypair; use tokio::time::Timeout; @@ -12,3 +15,28 @@ where { tokio::time::timeout(FALLBACK_TIMEOUT, future) } + + +// note this is duplicated from lite-rpc module +pub async fn get_identity_keypair(identity_from_cli: &String) -> Option { + if let Ok(identity_env_var) = env::var("IDENTITY") { + if let Ok(identity_bytes) = serde_json::from_str::>(identity_env_var.as_str()) { + Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap()) + } else { + // must be a file + let identity_file = tokio::fs::read_to_string(identity_env_var.as_str()) + .await + .expect("Cannot find the identity file provided"); + let identity_bytes: Vec = serde_json::from_str(&identity_file).unwrap(); + Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap()) + } + } else if identity_from_cli.is_empty() { + None + } else { + let identity_file = tokio::fs::read_to_string(identity_from_cli.as_str()) + .await + .expect("Cannot find the identity file provided"); + let identity_bytes: Vec = serde_json::from_str(&identity_file).unwrap(); + Some(Keypair::from_bytes(identity_bytes.as_slice()).unwrap()) + } +} diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index 46694dd3..bb752be8 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -26,6 +26,6 @@ fn roundtrip() { let request = TpuForwardingRequest::deserialize_from_raw_request(&wire_data); - assert_eq!(request.get_tpu_socket_addr().is_ipv4(), true); + assert!(request.get_tpu_socket_addr().is_ipv4()); assert_eq!(request.get_transactions().len(), 1); } From 92e0972f412de8a71a47f78573cfaec2bd95f4cb Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 14:55:31 +0200 Subject: [PATCH 081/128] cargo fmt --- .../src/inbound/proxy_listener.rs | 9 +++++---- quic-forward-proxy/src/main.rs | 2 +- quic-forward-proxy/src/outbound/tx_forward.rs | 20 +++++++++---------- .../src/quinn_auto_reconnect.rs | 12 +++++------ quic-forward-proxy/src/util.rs | 3 +-- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 2bf509fe..e514e335 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -1,4 +1,5 @@ use crate::proxy_request_format::TpuForwardingRequest; +use crate::quic_util::connection_stats; use crate::shared::ForwardPacket; use crate::tls_config_provider_server::ProxyTlsConfigProvider; use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; @@ -12,7 +13,6 @@ use std::sync::atomic::AtomicBool; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc::Sender; -use crate::quic_util::connection_stats; // note: setting this to "1" did not make a difference! // solana server sets this to 256 @@ -158,11 +158,12 @@ impl ProxyListener { .unwrap(); }); - debug!("Inbound connection stats: {}", connection_stats(&client_connection)); - + debug!( + "Inbound connection stats: {}", + connection_stats(&client_connection) + ); } }; // -- result - } // -- loop } } diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 97e467bd..e59ad8f1 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -1,12 +1,12 @@ use crate::cli::Args; use crate::proxy::QuicForwardProxy; use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; +use crate::util::get_identity_keypair; use anyhow::bail; use clap::Parser; use dotenv::dotenv; use log::info; use std::sync::Arc; -use crate::util::get_identity_keypair; use crate::validator_identity::ValidatorIdentity; diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 78beaba2..6e535f81 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -48,7 +48,6 @@ pub async fn tx_forwarder( let tpu_address = forward_packet.tpu_address; agents.entry(tpu_address).or_insert_with(|| { - let mut senders = Vec::new(); for connection_idx in 1..PARALLEL_TPU_CONNECTION_COUNT { let (sender, mut receiver) = channel::(100_000); @@ -86,11 +85,11 @@ pub async fn tx_forwarder( &auto_connection, &transactions_batch, )) - .await - .context(format!( - "send txs to tpu node {}", - auto_connection.target_address - )); + .await + .context(format!( + "send txs to tpu node {}", + auto_connection.target_address + )); if result.is_err() { warn!( @@ -99,7 +98,10 @@ pub async fn tx_forwarder( ); } else { debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); - debug!("Outbound connection stats: {}", &auto_connection.connection_stats().await); + debug!( + "Outbound connection stats: {}", + &auto_connection.connection_stats().await + ); } } // -- while all packtes from channel @@ -188,9 +190,7 @@ async fn send_tx_batch_to_tpu(auto_connection: &AutoReconnect, txs: &[VersionedT for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { let all_send_fns = chunk .iter() - .map(|tx| { - bincode::serialize(tx).unwrap() - }) + .map(|tx| bincode::serialize(tx).unwrap()) .map(|tx_raw| { auto_connection.send_uni(tx_raw) // ignores error }); diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 26e69dd7..9e7a7d66 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -101,7 +101,6 @@ impl AutoReconnect { connection.await.expect("connection") } - // stable_id 140266619216912, rtt=2.156683ms, // stats FrameStats { ACK: 3, CONNECTION_CLOSE: 0, CRYPTO: 3, // DATA_BLOCKED: 0, DATAGRAM: 0, HANDSHAKE_DONE: 1, MAX_DATA: 0, @@ -114,15 +113,14 @@ impl AutoReconnect { let maybe_conn = lock.as_ref(); match maybe_conn { Some(connection) => format!( - "stable_id {} stats {:?}, rtt={:?}", - connection.stable_id(), - connection.stats().frame_rx, - connection.stats().path.rtt - ), + "stable_id {} stats {:?}, rtt={:?}", + connection.stable_id(), + connection.stats().frame_rx, + connection.stats().path.rtt + ), None => "n/a".to_string(), } } - } impl fmt::Display for AutoReconnect { diff --git a/quic-forward-proxy/src/util.rs b/quic-forward-proxy/src/util.rs index 87ded89a..d1f85e25 100644 --- a/quic-forward-proxy/src/util.rs +++ b/quic-forward-proxy/src/util.rs @@ -1,8 +1,8 @@ #![allow(dead_code)] +use solana_sdk::signature::Keypair; use std::env; use std::future::Future; use std::time::Duration; -use solana_sdk::signature::Keypair; use tokio::time::Timeout; @@ -16,7 +16,6 @@ where tokio::time::timeout(FALLBACK_TIMEOUT, future) } - // note this is duplicated from lite-rpc module pub async fn get_identity_keypair(identity_from_cli: &String) -> Option { if let Ok(identity_env_var) = env::var("IDENTITY") { From f00d174c30376d41b65b936f9b3d99e1a75be51c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 15:09:50 +0200 Subject: [PATCH 082/128] simplified validator identity --- quic-forward-proxy/src/validator_identity.rs | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/quic-forward-proxy/src/validator_identity.rs b/quic-forward-proxy/src/validator_identity.rs index df4f659f..c04362dd 100644 --- a/quic-forward-proxy/src/validator_identity.rs +++ b/quic-forward-proxy/src/validator_identity.rs @@ -6,40 +6,40 @@ use std::sync::Arc; #[derive(Clone)] pub struct ValidatorIdentity { - keypair: Option>, - dummy_keypair: Arc, + keypair: Arc, + is_dummy_keypair: bool, } impl ValidatorIdentity { pub fn new(keypair: Option) -> Self { - let dummy_keypair = Keypair::new(); - ValidatorIdentity { - keypair: keypair.map(Arc::new), - dummy_keypair: Arc::new(dummy_keypair), + match keypair { + Some(keypair) => ValidatorIdentity { + keypair: Arc::new(keypair), + is_dummy_keypair: false, + }, + None => ValidatorIdentity { + keypair: Arc::new(Keypair::new()), + is_dummy_keypair: true, + }, } } pub fn get_keypair_for_tls(&self) -> Arc { - match &self.keypair { - Some(keypair) => keypair.clone(), - None => self.dummy_keypair.clone(), - } + self.keypair.clone() } pub fn get_pubkey(&self) -> Pubkey { - let keypair = match &self.keypair { - Some(keypair) => keypair.clone(), - None => self.dummy_keypair.clone(), - }; + let keypair = self.keypair.clone(); keypair.pubkey() } } impl Display for ValidatorIdentity { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self.keypair { - Some(keypair) => write!(f, "{}", keypair.pubkey()), - None => write!(f, "no keypair"), + if self.is_dummy_keypair { + write!(f, "no keypair") + } else { + write!(f, "{}", self.keypair.pubkey()) } } } From a8863cd74d6956b60888cd6a6e2bf0a27b61b9d9 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 15:22:59 +0200 Subject: [PATCH 083/128] code format lite-rpc --- core/src/lib.rs | 2 +- core/src/proxy_request_format.rs | 32 ++-- core/src/quic_connection_utils.rs | 9 +- core/src/solana_utils.rs | 6 +- lite-rpc/src/bridge.rs | 2 +- lite-rpc/src/main.rs | 14 +- services/src/tpu_utils/mod.rs | 6 +- .../quic_proxy_connection_manager.rs | 158 ++++++++---------- .../src/tpu_utils/quinn_auto_reconnect.rs | 19 +-- services/src/tpu_utils/tpu_connection_path.rs | 5 +- services/src/tpu_utils/tpu_service.rs | 64 +++---- services/src/transaction_service.rs | 2 +- ...literpc_tpu_quic_server_integrationtest.rs | 1 - 13 files changed, 157 insertions(+), 163 deletions(-) diff --git a/core/src/lib.rs b/core/src/lib.rs index 06060784..97da6f71 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -2,6 +2,7 @@ pub mod block_processor; pub mod block_store; pub mod leader_schedule; pub mod notifications; +pub mod proxy_request_format; pub mod quic_connection; pub mod quic_connection_utils; pub mod rotating_queue; @@ -10,6 +11,5 @@ pub mod structures; pub mod subscription_handler; pub mod subscription_sink; pub mod tx_store; -pub mod proxy_request_format; pub type AnyhowJoinHandle = tokio::task::JoinHandle>; diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index 53ddc8a3..8970becd 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -1,10 +1,10 @@ -use std::fmt; -use std::fmt::Display; -use std::net::{SocketAddr}; use anyhow::Context; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::VersionedTransaction; +use std::fmt; +use std::fmt::Display; +use std::net::SocketAddr; /// /// lite-rpc to proxy wire format @@ -22,14 +22,22 @@ pub struct TpuForwardingRequest { impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "TpuForwardingRequest for tpu target {} with identity {}: payload {} tx", - &self.get_tpu_socket_addr(), &self.get_identity_tpunode(), &self.get_transactions().len()) + write!( + f, + "TpuForwardingRequest for tpu target {} with identity {}: payload {} tx", + &self.get_tpu_socket_addr(), + &self.get_identity_tpunode(), + &self.get_transactions().len() + ) } } impl TpuForwardingRequest { - pub fn new(tpu_socket_addr: SocketAddr, identity_tpunode: Pubkey, - transactions: Vec) -> Self { + pub fn new( + tpu_socket_addr: SocketAddr, + identity_tpunode: Pubkey, + transactions: Vec, + ) -> Self { TpuForwardingRequest { format_version: FORMAT_VERSION1, tpu_socket_addr, @@ -38,13 +46,12 @@ impl TpuForwardingRequest { } } - pub fn serialize_wire_format( - &self) -> Vec { + pub fn serialize_wire_format(&self) -> Vec { bincode::serialize(&self).expect("Expect to serialize transactions") } - pub fn deserialize_from_raw_request(raw_proxy_request: &Vec) -> TpuForwardingRequest { - let request = bincode::deserialize::(&raw_proxy_request) + pub fn deserialize_from_raw_request(raw_proxy_request: &[u8]) -> TpuForwardingRequest { + let request = bincode::deserialize::(raw_proxy_request) .context("deserialize proxy request") .unwrap(); @@ -65,6 +72,3 @@ impl TpuForwardingRequest { self.transactions.clone() } } - - - diff --git a/core/src/quic_connection_utils.rs b/core/src/quic_connection_utils.rs index 4c79c48f..c4907e4c 100644 --- a/core/src/quic_connection_utils.rs +++ b/core/src/quic_connection_utils.rs @@ -226,7 +226,10 @@ impl rustls::client::ServerCertVerifier for SkipServerVerification { // rtt=1.08178ms pub fn connection_stats(connection: &Connection) -> String { // see https://www.rfc-editor.org/rfc/rfc9000.html#name-frame-types-and-formats - format!("stable_id {}, rtt={:?}, stats {:?}", - connection.stable_id(), connection.stats().path.rtt, connection.stats().frame_rx) + format!( + "stable_id {}, rtt={:?}, stats {:?}", + connection.stable_id(), + connection.stats().path.rtt, + connection.stats().frame_rx + ) } - diff --git a/core/src/solana_utils.rs b/core/src/solana_utils.rs index 8b9a5b27..d875e90c 100644 --- a/core/src/solana_utils.rs +++ b/core/src/solana_utils.rs @@ -1,7 +1,9 @@ use crate::structures::identity_stakes::IdentityStakes; use anyhow::Context; use log::info; +use serde::Serialize; use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use solana_sdk::hash::Hash; use solana_sdk::pubkey::Pubkey; use solana_streamer::nonblocking::quic::ConnectionPeerType; use std::{ @@ -12,11 +14,9 @@ use std::{ }, time::Duration, }; -use serde::Serialize; -use solana_sdk::hash::Hash; use solana_sdk::signature::Signature; -use solana_sdk::transaction::{Transaction, uses_durable_nonce, VersionedTransaction}; +use solana_sdk::transaction::{uses_durable_nonce, Transaction, VersionedTransaction}; use tokio::sync::mpsc::UnboundedReceiver; const AVERAGE_SLOT_CHANGE_TIME_IN_MILLIS: u64 = 400; diff --git a/lite-rpc/src/bridge.rs b/lite-rpc/src/bridge.rs index fa3c4ba2..6fc2f51a 100644 --- a/lite-rpc/src/bridge.rs +++ b/lite-rpc/src/bridge.rs @@ -39,11 +39,11 @@ use solana_sdk::{ use solana_transaction_status::TransactionStatus; use std::{ops::Deref, str::FromStr, sync::Arc, time::Duration}; +use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; use tokio::{ net::ToSocketAddrs, sync::mpsc::{self, Sender}, }; -use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; lazy_static::lazy_static! { static ref RPC_SEND_TX: IntCounter = diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index fd0d415c..ab1e9182 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -7,11 +7,11 @@ use clap::Parser; use dotenv::dotenv; use lite_rpc::{bridge::LiteBridge, cli::Args}; +use clap::builder::TypedValueParser; +use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; use solana_sdk::signature::Keypair; use std::env; use std::sync::Arc; -use clap::builder::TypedValueParser; -use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; use crate::rpc_tester::RpcTester; @@ -66,7 +66,7 @@ pub async fn start_lite_rpc(args: Args) -> anyhow::Result<()> { validator_identity, retry_after, maximum_retries_per_tx, - tpu_connection_path + tpu_connection_path, ) .await .context("Error building LiteBridge")? @@ -123,12 +123,14 @@ pub async fn main() -> anyhow::Result<()> { } } -fn configure_tpu_connection_path(experimental_quic_proxy_addr: Option) -> TpuConnectionPath { +fn configure_tpu_connection_path( + experimental_quic_proxy_addr: Option, +) -> TpuConnectionPath { match experimental_quic_proxy_addr { None => TpuConnectionPath::QuicDirectPath, Some(prox_address) => TpuConnectionPath::QuicForwardProxyPath { // e.g. "127.0.0.1:11111" - forward_proxy_address: prox_address.parse().unwrap() + forward_proxy_address: prox_address.parse().unwrap(), }, } -} \ No newline at end of file +} diff --git a/services/src/tpu_utils/mod.rs b/services/src/tpu_utils/mod.rs index 24e94707..c32b8deb 100644 --- a/services/src/tpu_utils/mod.rs +++ b/services/src/tpu_utils/mod.rs @@ -1,8 +1,6 @@ pub mod tpu_service; -pub mod tpu_connection_path; -pub mod tpu_connection_manager; pub mod quic_proxy_connection_manager; pub mod quinn_auto_reconnect; - - +pub mod tpu_connection_manager; +pub mod tpu_connection_path; diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index d86acd22..8bcfa89b 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -1,28 +1,27 @@ - use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::Ordering::Relaxed; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use anyhow::bail; use std::time::Duration; -use anyhow::{bail}; -use futures::FutureExt; use itertools::Itertools; use log::{debug, error, info, trace}; -use quinn::{ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt}; -use solana_sdk::packet::PACKET_DATA_SIZE; +use quinn::{ + ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt, +}; use solana_sdk::pubkey::Pubkey; -use solana_sdk::signature::Keypair; use solana_sdk::transaction::VersionedTransaction; use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; use tokio::time::timeout; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; -use solana_lite_rpc_core::quic_connection_utils::{connection_stats, QuicConnectionParameters, SkipServerVerification}; - +use solana_lite_rpc_core::quic_connection_utils::{ + connection_stats, QuicConnectionParameters, SkipServerVerification, +}; use crate::tpu_utils::quinn_auto_reconnect::AutoReconnect; @@ -36,7 +35,7 @@ pub struct QuicProxyConnectionManager { endpoint: Endpoint, simple_thread_started: AtomicBool, proxy_addr: SocketAddr, - current_tpu_nodes: Arc>> + current_tpu_nodes: Arc>>, } const CHUNK_SIZE_PER_STREAM: usize = 20; @@ -48,7 +47,7 @@ impl QuicProxyConnectionManager { proxy_addr: SocketAddr, ) -> Self { info!("Configure Quic proxy connection manager to {}", proxy_addr); - let endpoint = Self::create_proxy_client_endpoint(certificate.clone(), key.clone()); + let endpoint = Self::create_proxy_client_endpoint(certificate, key); Self { endpoint, @@ -65,21 +64,24 @@ impl QuicProxyConnectionManager { connections_to_keep: HashMap, connection_parameters: QuicConnectionParameters, ) { - debug!("reconfigure quic proxy connection (# of tpu nodes: {})", connections_to_keep.len()); + debug!( + "reconfigure quic proxy connection (# of tpu nodes: {})", + connections_to_keep.len() + ); { - let list_of_nodes = connections_to_keep.iter().map(|(identity, tpu_address)| { - TpuNode { - tpu_identity: identity.clone(), - tpu_address: tpu_address.clone(), - } - }).collect_vec(); + let list_of_nodes = connections_to_keep + .iter() + .map(|(identity, tpu_address)| TpuNode { + tpu_identity: *identity, + tpu_address: *tpu_address, + }) + .collect_vec(); let mut lock = self.current_tpu_nodes.write().await; *lock = list_of_nodes; } - if self.simple_thread_started.load(Relaxed) { // already started return; @@ -100,11 +102,12 @@ impl QuicProxyConnectionManager { exit_signal, connection_parameters, )); - } - fn create_proxy_client_endpoint(certificate: rustls::Certificate, key: rustls::PrivateKey) -> Endpoint { - + fn create_proxy_client_endpoint( + certificate: rustls::Certificate, + key: rustls::PrivateKey, + ) -> Endpoint { const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; let mut endpoint = { @@ -152,7 +155,6 @@ impl QuicProxyConnectionManager { exit_signal: Arc, connection_parameters: QuicConnectionParameters, ) { - let auto_connection = AutoReconnect::new(endpoint, proxy_addr); loop { @@ -162,56 +164,58 @@ impl QuicProxyConnectionManager { } tokio::select! { - // TODO add timeout - tx = transaction_receiver.recv() => { - - let first_tx: Vec = match tx { - Ok((_sig, tx)) => { - tx - }, - Err(e) => { - error!( - "Broadcast channel error on recv error {}", e); - continue; - } - }; - - let mut txs = vec![first_tx]; - for _ in 1..connection_parameters.number_of_transactions_per_unistream { - if let Ok((_signature, tx)) = transaction_receiver.try_recv() { - txs.push(tx); - } + // TODO add timeout + tx = transaction_receiver.recv() => { + + let first_tx: Vec = match tx { + Ok((_sig, tx)) => { + tx + }, + Err(e) => { + error!( + "Broadcast channel error on recv error {}", e); + continue; } + }; - let tpu_fanout_nodes = current_tpu_nodes.read().await.clone(); + let mut txs = vec![first_tx]; + for _ in 1..connection_parameters.number_of_transactions_per_unistream { + if let Ok((_signature, tx)) = transaction_receiver.try_recv() { + txs.push(tx); + } + } - trace!("Sending copy of transaction batch of {} txs to {} tpu nodes via quic proxy", - txs.len(), tpu_fanout_nodes.len()); + let tpu_fanout_nodes = current_tpu_nodes.read().await.clone(); - for target_tpu_node in tpu_fanout_nodes { - Self::send_copy_of_txs_to_quicproxy( - &txs, &auto_connection, - proxy_addr, - target_tpu_node.tpu_address, - target_tpu_node.tpu_identity) - .await.unwrap(); - } + trace!("Sending copy of transaction batch of {} txs to {} tpu nodes via quic proxy", + txs.len(), tpu_fanout_nodes.len()); - }, - }; + for target_tpu_node in tpu_fanout_nodes { + Self::send_copy_of_txs_to_quicproxy( + &txs, &auto_connection, + proxy_addr, + target_tpu_node.tpu_address, + target_tpu_node.tpu_identity) + .await.unwrap(); + } + + }, + }; } } - async fn send_copy_of_txs_to_quicproxy(raw_tx_batch: &Vec>, auto_connection: &AutoReconnect, - _proxy_address: SocketAddr, tpu_target_address: SocketAddr, - target_tpu_identity: Pubkey) -> anyhow::Result<()> { - - let raw_tx_batch_copy = raw_tx_batch.clone(); + async fn send_copy_of_txs_to_quicproxy( + raw_tx_batch: &[Vec], + auto_connection: &AutoReconnect, + _proxy_address: SocketAddr, + tpu_target_address: SocketAddr, + target_tpu_identity: Pubkey, + ) -> anyhow::Result<()> { let mut txs = vec![]; - for raw_tx in raw_tx_batch_copy { - let tx = match bincode::deserialize::(&raw_tx) { + for raw_tx in raw_tx_batch { + let tx = match bincode::deserialize::(raw_tx) { Ok(tx) => tx, Err(err) => { bail!(err.to_string()); @@ -220,13 +224,13 @@ impl QuicProxyConnectionManager { txs.push(tx); } - for chunk in txs.chunks(CHUNK_SIZE_PER_STREAM) { - - let forwarding_request = TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, chunk.into()); + let forwarding_request = + TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, chunk.into()); debug!("forwarding_request: {}", forwarding_request); - let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); + let proxy_request_raw = + bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); let send_result = auto_connection.send_uni(proxy_request_raw).await; @@ -238,29 +242,9 @@ impl QuicProxyConnectionManager { bail!("Failed to send data to quic proxy: {:?}", e); } } - } // -- one chunk - Ok(()) } - async fn send_proxy_request(endpoint: Endpoint, proxy_address: SocketAddr, proxy_request_raw: &Vec) -> anyhow::Result<()> { - info!("sending {} bytes to proxy", proxy_request_raw.len()); - - let connecting = endpoint.connect(proxy_address, "localhost")?; - let connection = timeout(Duration::from_millis(500), connecting).await??; - let mut send = connection.open_uni().await?; - - send.write_all(proxy_request_raw).await?; - - send.finish().await?; - - debug!("connection stats (lite-rpc to proxy): {}", connection_stats(&connection)); - Ok(()) - } - - } - - diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 8fb4bf2b..ee5a3b09 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -29,17 +29,14 @@ impl AutoReconnect { pub async fn send_uni(&self, payload: Vec) -> anyhow::Result<()> { // TOOD do smart error handling + reconnect - let mut send_stream = timeout( - Duration::from_secs(4), self.refresh() - .await.open_uni()) - .await - .context("open uni stream for sending")??; + let mut send_stream = timeout(Duration::from_secs(4), self.refresh().await.open_uni()) + .await + .context("open uni stream for sending")??; send_stream.write_all(payload.as_slice()).await?; send_stream.finish().await?; Ok(()) } - pub async fn refresh(&self) -> Connection { { let lock = self.current.read().await; @@ -55,7 +52,7 @@ impl AutoReconnect { } let mut lock = self.current.write().await; let maybe_conn = lock.as_ref(); - return match maybe_conn { + match maybe_conn { Some(current) => { if current.close_reason().is_some() { let old_stable_id = current.stable_id(); @@ -66,7 +63,6 @@ impl AutoReconnect { ); let new_connection = self.create_connection().await; - let prev_stable_id = current.stable_id(); *lock = Some(new_connection.clone()); // let old_conn = lock.replace(new_connection.clone()); self.reconnect_count.fetch_add(1, Ordering::SeqCst); @@ -78,7 +74,7 @@ impl AutoReconnect { self.reconnect_count.load(Ordering::SeqCst) ); - new_connection.clone() + new_connection } else { debug!("Reuse connection {} with write-lock", current.stable_id()); current.clone() @@ -92,9 +88,9 @@ impl AutoReconnect { // let old_conn = foo.replace(Some(new_connection.clone())); debug!("Create initial connection {}", new_connection.stable_id()); - new_connection.clone() + new_connection } - }; + } } async fn create_connection(&self) -> Connection { @@ -112,4 +108,3 @@ impl fmt::Display for AutoReconnect { write!(f, "Connection to {}", self.target_address,) } } - diff --git a/services/src/tpu_utils/tpu_connection_path.rs b/services/src/tpu_utils/tpu_connection_path.rs index 048f7c88..a71108d5 100644 --- a/services/src/tpu_utils/tpu_connection_path.rs +++ b/services/src/tpu_utils/tpu_connection_path.rs @@ -1,4 +1,3 @@ - use std::fmt::Display; use std::net::SocketAddr; @@ -12,7 +11,9 @@ impl Display for TpuConnectionPath { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { TpuConnectionPath::QuicDirectPath => write!(f, "Direct QUIC connection to TPU"), - TpuConnectionPath::QuicForwardProxyPath { forward_proxy_address } => { + TpuConnectionPath::QuicForwardProxyPath { + forward_proxy_address, + } => { write!(f, "QUIC Forward Proxy on {}", forward_proxy_address) } } diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 27d2e882..984c3101 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -10,6 +10,9 @@ use solana_lite_rpc_core::{ }; use super::tpu_connection_manager::TpuConnectionManager; +use crate::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; +use crate::tpu_utils::tpu_connection_path::TpuConnectionPath; +use crate::tpu_utils::tpu_service::ConnectionManager::{DirectTpu, QuicProxy}; use solana_sdk::{ pubkey::Pubkey, quic::QUIC_PORT_OFFSET, signature::Keypair, signer::Signer, slot_history::Slot, }; @@ -26,9 +29,6 @@ use tokio::{ sync::RwLock, time::{Duration, Instant}, }; -use crate::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; -use crate::tpu_utils::tpu_connection_path::TpuConnectionPath; -use crate::tpu_utils::tpu_service::ConnectionManager::{DirectTpu, QuicProxy}; lazy_static::lazy_static! { static ref NB_CLUSTER_NODES: GenericGauge = @@ -72,8 +72,12 @@ pub struct TpuService { #[derive(Clone)] enum ConnectionManager { - DirectTpu { tpu_connection_manager: Arc }, - QuicProxy { quic_proxy_connection_manager: Arc }, + DirectTpu { + tpu_connection_manager: Arc, + }, + QuicProxy { + quic_proxy_connection_manager: Arc, + }, } impl TpuService { @@ -91,25 +95,25 @@ impl TpuService { ) .expect("Failed to initialize QUIC client certificates"); - let connection_manager = - match config.tpu_connection_path { - TpuConnectionPath::QuicDirectPath => { - let tpu_connection_manager = - TpuConnectionManager::new(certificate, key, - config.fanout_slots as usize).await; - DirectTpu { - tpu_connection_manager: Arc::new(tpu_connection_manager), - } + let connection_manager = match config.tpu_connection_path { + TpuConnectionPath::QuicDirectPath => { + let tpu_connection_manager = + TpuConnectionManager::new(certificate, key, config.fanout_slots as usize).await; + DirectTpu { + tpu_connection_manager: Arc::new(tpu_connection_manager), } - TpuConnectionPath::QuicForwardProxyPath { forward_proxy_address } => { - let quic_proxy_connection_manager = - QuicProxyConnectionManager::new(certificate, key, forward_proxy_address).await; + } + TpuConnectionPath::QuicForwardProxyPath { + forward_proxy_address, + } => { + let quic_proxy_connection_manager = + QuicProxyConnectionManager::new(certificate, key, forward_proxy_address).await; - QuicProxy { - quic_proxy_connection_manager: Arc::new(quic_proxy_connection_manager), - } + QuicProxy { + quic_proxy_connection_manager: Arc::new(quic_proxy_connection_manager), } - }; + } + }; Ok(Self { current_slot: Arc::new(AtomicU64::new(current_slot)), @@ -194,13 +198,17 @@ impl TpuService { self.config.quic_connection_params, ) .await; - }, - QuicProxy { quic_proxy_connection_manager } => { - quic_proxy_connection_manager.update_connection( - self.broadcast_sender.clone(), - connections_to_keep, - self.config.quic_connection_params, - ).await; + } + QuicProxy { + quic_proxy_connection_manager, + } => { + quic_proxy_connection_manager + .update_connection( + self.broadcast_sender.clone(), + connections_to_keep, + self.config.quic_connection_params, + ) + .await; } } } diff --git a/services/src/transaction_service.rs b/services/src/transaction_service.rs index d89bcb45..8cacf59e 100644 --- a/services/src/transaction_service.rs +++ b/services/src/transaction_service.rs @@ -11,12 +11,12 @@ use crate::{ tx_sender::{TransactionInfo, TxSender}, }; use anyhow::bail; +use solana_lite_rpc_core::solana_utils::SerializableTransaction; use solana_lite_rpc_core::{ block_store::{BlockInformation, BlockStore}, notifications::NotificationSender, AnyhowJoinHandle, }; -use solana_lite_rpc_core::solana_utils::SerializableTransaction; use solana_sdk::{commitment_config::CommitmentConfig, transaction::VersionedTransaction}; use tokio::{ sync::mpsc::{self, Sender, UnboundedSender}, diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index a652e8cc..e10700dc 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -33,7 +33,6 @@ use tokio::task::JoinHandle; use tokio::time::sleep; use tracing_subscriber::util::SubscriberInitExt; - #[derive(Copy, Clone, Debug)] struct TestCaseParams { sample_tx_count: u32, From e1be1d0812a412b5116b2eb2a7447be510e15ad0 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 15:23:09 +0200 Subject: [PATCH 084/128] fmt --- .../tests/quic_proxy_tpu_integrationtest.rs | 73 +++++++++++-------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 9ed289f3..2fd05318 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -1,14 +1,13 @@ - use countmap::CountMap; -use crossbeam_channel::{Sender}; +use crossbeam_channel::Sender; use log::{debug, info, trace, warn}; use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; +use solana_lite_rpc_core::solana_utils::SerializableTransaction; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; use solana_lite_rpc_core::tx_store::empty_tx_store; use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; -use solana_lite_rpc_core::solana_utils::SerializableTransaction; use solana_sdk::hash::Hash; use solana_sdk::instruction::Instruction; use solana_sdk::message::Message; @@ -24,25 +23,22 @@ use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; - - use std::str::FromStr; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; use std::thread; use std::time::{Duration, Instant}; -use tokio::runtime::{Builder}; +use tokio::runtime::Builder; - -use tokio::task::{JoinHandle, yield_now}; -use tokio::time::{sleep}; +use tokio::task::{yield_now, JoinHandle}; +use tokio::time::sleep; use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::fmt::format::FmtSpan; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; +use tracing_subscriber::fmt::format::FmtSpan; #[derive(Copy, Clone, Debug)] struct TestCaseParams { @@ -196,7 +192,10 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { let udp_listen_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); let listen_addr = udp_listen_socket.local_addr().unwrap(); - let proxy_listen_addr = UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap(); + let proxy_listen_addr = UdpSocket::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); let (inbound_packets_sender, inbound_packets_receiver) = crossbeam_channel::unbounded(); @@ -225,12 +224,9 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { }); runtime_quic_proxy.block_on(async { - tokio::spawn(start_quic_proxy( - proxy_listen_addr, - )); + tokio::spawn(start_quic_proxy(proxy_listen_addr)); }); - runtime_literpc.block_on(async { if test_case_params.proxy_mode { tokio::spawn(start_literpc_client_proxy_mode( @@ -438,7 +434,7 @@ async fn start_literpc_client( for i in 0..test_case_params.sample_tx_count { let raw_sample_tx = build_raw_sample_tx(i); broadcast_sender.send(raw_sample_tx)?; - if (i+1) % 1000 == 0 { + if (i + 1) % 1000 == 0 { yield_now().await; } } @@ -490,7 +486,6 @@ async fn solana_quic_streamer_start() { stats.report(); } - // no quic proxy async fn start_literpc_client_direct_mode( test_case_params: TestCaseParams, @@ -508,7 +503,7 @@ async fn start_literpc_client_direct_mode( literpc_validator_identity.as_ref(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) - .expect("Failed to initialize QUIC connection certificates"); + .expect("Failed to initialize QUIC connection certificates"); let tpu_connection_manager = TpuConnectionManager::new(certificate, key, fanout_slots as usize).await; @@ -540,7 +535,11 @@ async fn start_literpc_client_direct_mode( // populated from get_stakes_for_identity() let identity_stakes = IdentityStakes { peer_type: ConnectionPeerType::Staked, - stakes: if test_case_params.stake_connection { 30 } else { 0 }, // stake of lite-rpc + stakes: if test_case_params.stake_connection { + 30 + } else { + 0 + }, // stake of lite-rpc min_stakes: 0, max_stakes: 40, total_stakes: 100, @@ -582,7 +581,10 @@ async fn start_literpc_client_proxy_mode( validator_identity: Arc, forward_proxy_address: SocketAddr, ) -> anyhow::Result<()> { - info!("Start lite-rpc test client using quic proxy at {} ...", forward_proxy_address); + info!( + "Start lite-rpc test client using quic proxy at {} ...", + forward_proxy_address + ); // (String, Vec) (signature, transaction) let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); @@ -591,7 +593,7 @@ async fn start_literpc_client_proxy_mode( validator_identity.as_ref(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), ) - .expect("Failed to initialize QUIC connection certificates"); + .expect("Failed to initialize QUIC connection certificates"); let quic_proxy_connection_manager = QuicProxyConnectionManager::new(certificate, key, forward_proxy_address).await; @@ -623,7 +625,11 @@ async fn start_literpc_client_proxy_mode( // populated from get_stakes_for_identity() let _identity_stakes = IdentityStakes { peer_type: ConnectionPeerType::Staked, - stakes: if test_case_params.stake_connection { 30 } else { 0 }, // stake of lite-rpc + stakes: if test_case_params.stake_connection { + 30 + } else { + 0 + }, // stake of lite-rpc min_stakes: 0, max_stakes: 40, total_stakes: 100, @@ -642,8 +648,13 @@ async fn start_literpc_client_proxy_mode( // ) // .await; - quic_proxy_connection_manager.update_connection( - broadcast_sender.clone(), connections_to_keep, QUIC_CONNECTION_PARAMS).await; + quic_proxy_connection_manager + .update_connection( + broadcast_sender.clone(), + connections_to_keep, + QUIC_CONNECTION_PARAMS, + ) + .await; // TODO this is a race sleep(Duration::from_millis(1500)).await; @@ -658,7 +669,7 @@ async fn start_literpc_client_proxy_mode( ); broadcast_sender.send(raw_sample_tx)?; - if (i+1) % 1000 == 0 { + if (i + 1) % 1000 == 0 { yield_now().await; } } @@ -669,14 +680,17 @@ async fn start_literpc_client_proxy_mode( } async fn start_quic_proxy(proxy_listen_addr: SocketAddr) -> anyhow::Result<()> { - let _tls_configuration = SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost(); let random_unstaked_validator_identity = ValidatorIdentity::new(None); let tls_config = Arc::new(SelfSignedTlsConfigProvider::new_singleton_self_signed_localhost()); - let proxy_service = QuicForwardProxy::new(proxy_listen_addr, tls_config, random_unstaked_validator_identity) - .await? - .start_services(); + let proxy_service = QuicForwardProxy::new( + proxy_listen_addr, + tls_config, + random_unstaked_validator_identity, + ) + .await? + .start_services(); tokio::select! { _ = proxy_service => { @@ -685,7 +699,6 @@ async fn start_quic_proxy(proxy_listen_addr: SocketAddr) -> anyhow::Result<()> { } } - struct SolanaQuicStreamer { sock: UdpSocket, exit: Arc, From ca7927dca34dc724ba86255550e074952bc3e053 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 15:51:23 +0200 Subject: [PATCH 085/128] fmt again --- services/src/tpu_utils/quic_proxy_connection_manager.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 8bcfa89b..76a81aa8 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -211,7 +211,6 @@ impl QuicProxyConnectionManager { tpu_target_address: SocketAddr, target_tpu_identity: Pubkey, ) -> anyhow::Result<()> { - let mut txs = vec![]; for raw_tx in raw_tx_batch { @@ -246,5 +245,4 @@ impl QuicProxyConnectionManager { Ok(()) } - } From bd3fc4826769e2cd61392374843604cecc50e3fe Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 2 Aug 2023 16:34:38 +0200 Subject: [PATCH 086/128] fmt+clippy --- lite-rpc/src/main.rs | 1 - .../tests/quic_proxy_tpu_integrationtest.rs | 116 ++---------------- .../quic_proxy_connection_manager.rs | 3 +- ...literpc_tpu_quic_server_integrationtest.rs | 1 - 4 files changed, 8 insertions(+), 113 deletions(-) diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index ab1e9182..d1eeb113 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -7,7 +7,6 @@ use clap::Parser; use dotenv::dotenv; use lite_rpc::{bridge::LiteBridge, cli::Args}; -use clap::builder::TypedValueParser; use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; use solana_sdk::signature::Keypair; use std::env; diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 2fd05318..554268d8 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -32,7 +32,6 @@ use tokio::runtime::Builder; use tokio::task::{yield_now, JoinHandle}; use tokio::time::sleep; -use tracing_subscriber::util::SubscriberInitExt; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; @@ -166,7 +165,6 @@ pub fn too_many_transactions() { // note: this not a tokio test as runtimes get created as part of the integration test fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { // value from solana - see quic streamer - see quic.rs -> rt() - const NUM_QUIC_STREAMER_WORKER_THREADS: usize = 1; let runtime_quic1 = Builder::new_multi_thread() .worker_threads(1) .thread_name("quic-server") @@ -200,7 +198,7 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { let (inbound_packets_sender, inbound_packets_receiver) = crossbeam_channel::unbounded(); runtime_quic1.block_on(async { - /// setup solana Quic streamer + // setup solana Quic streamer // see log "Start quic server on UdpSocket { addr: 127.0.0.1:xxxxx, fd: 10 }" let staked_nodes = StakedNodes { total_stake: 100, @@ -230,17 +228,16 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { runtime_literpc.block_on(async { if test_case_params.proxy_mode { tokio::spawn(start_literpc_client_proxy_mode( - test_case_params.clone(), + test_case_params, listen_addr, literpc_validator_identity, proxy_listen_addr, )); } else { tokio::spawn(start_literpc_client_direct_mode( - test_case_params.clone(), + test_case_params, listen_addr, literpc_validator_identity, - // proxy_listen_addr, )); } }); @@ -283,9 +280,9 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { ); } - packet_count = packet_count + packet_batch.len() as u32; + packet_count += packet_batch.len() as u32; if timer2.is_some() { - packet_count2 = packet_count2 + packet_batch.len() as u32; + packet_count2 += packet_batch.len() as u32; } for packet in packet_batch.iter() { @@ -360,92 +357,6 @@ fn configure_logging(verbose: bool) { } } -async fn start_literpc_client( - test_case_params: TestCaseParams, - streamer_listen_addrs: SocketAddr, - literpc_validator_identity: Arc, -) -> anyhow::Result<()> { - info!("Start lite-rpc test client ..."); - - let fanout_slots = 4; - - // (String, Vec) (signature, transaction) - let (sender, _) = tokio::sync::broadcast::channel(MAXIMUM_TRANSACTIONS_IN_QUEUE); - let broadcast_sender = Arc::new(sender); - let (certificate, key) = new_self_signed_tls_certificate( - literpc_validator_identity.as_ref(), - IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), - ) - .expect("Failed to initialize QUIC connection certificates"); - - let tpu_connection_manager = - TpuConnectionManager::new(certificate, key, fanout_slots as usize).await; - - // this effectively controls how many connections we will have - let mut connections_to_keep: HashMap = HashMap::new(); - let addr1 = UdpSocket::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap(); - connections_to_keep.insert( - Pubkey::from_str("1111111jepwNWbYG87sgwnBbUJnQHrPiUJzMpqJXZ")?, - addr1, - ); - - let addr2 = UdpSocket::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap(); - connections_to_keep.insert( - Pubkey::from_str("1111111k4AYMctpyJakWNvGcte6tR8BLyZw54R8qu")?, - addr2, - ); - - // this is the real streamer - connections_to_keep.insert(literpc_validator_identity.pubkey(), streamer_listen_addrs); - - // get information about the optional validator identity stake - // populated from get_stakes_for_identity() - let identity_stakes = IdentityStakes { - peer_type: ConnectionPeerType::Staked, - stakes: if test_case_params.stake_connection { - 30 - } else { - 0 - }, // stake of lite-rpc - min_stakes: 0, - max_stakes: 40, - total_stakes: 100, - }; - - // solana_streamer::nonblocking::quic: Peer type: Staked, stake 30, total stake 0, max streams 128 receive_window Ok(12320) from peer 127.0.0.1:8000 - - tpu_connection_manager - .update_connections( - broadcast_sender.clone(), - connections_to_keep, - identity_stakes, - // note: tx_store is useless in this scenario as it is never changed; it's only used to check for duplicates - empty_tx_store().clone(), - QUIC_CONNECTION_PARAMS, - ) - .await; - - for i in 0..test_case_params.sample_tx_count { - let raw_sample_tx = build_raw_sample_tx(i); - broadcast_sender.send(raw_sample_tx)?; - if (i + 1) % 1000 == 0 { - yield_now().await; - } - } - - // we need that to keep the tokio runtime dedicated to lite-rpc up long enough - sleep(Duration::from_secs(30)).await; - - // reaching this point means there is problem with test setup and the consumer threed - panic!("should never reach this point") -} - #[tokio::test] // taken from solana -> test_nonblocking_quic_client_multiple_writes async fn solana_quic_streamer_start() { @@ -699,6 +610,7 @@ async fn start_quic_proxy(proxy_listen_addr: SocketAddr) -> anyhow::Result<()> { } } +#[allow(dead_code)] struct SolanaQuicStreamer { sock: UdpSocket, exit: Arc, @@ -706,20 +618,6 @@ struct SolanaQuicStreamer { stats: Arc, } -impl SolanaQuicStreamer { - pub fn get_socket_addr(&self) -> SocketAddr { - self.sock.local_addr().unwrap() - } -} - -impl SolanaQuicStreamer { - pub async fn shutdown(self) { - self.exit.store(true, Ordering::Relaxed); - self.join_handler.await.unwrap(); - self.stats.report(); - } -} - impl SolanaQuicStreamer { fn new_start_listening( udp_socket: UdpSocket, @@ -740,7 +638,7 @@ impl SolanaQuicStreamer { sender, exit.clone(), MAX_QUIC_CONNECTIONS_PER_PEER, - staked_nodes.clone(), + staked_nodes, 10, 10, stats.clone(), diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 76a81aa8..14cf9d97 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -16,11 +16,10 @@ use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::VersionedTransaction; use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; -use tokio::time::timeout; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; use solana_lite_rpc_core::quic_connection_utils::{ - connection_stats, QuicConnectionParameters, SkipServerVerification, + QuicConnectionParameters, SkipServerVerification, }; use crate::tpu_utils::quinn_auto_reconnect::AutoReconnect; diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index e10700dc..076f5122 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -31,7 +31,6 @@ use std::time::{Duration, Instant}; use tokio::runtime::Builder; use tokio::task::JoinHandle; use tokio::time::sleep; -use tracing_subscriber::util::SubscriberInitExt; #[derive(Copy, Clone, Debug)] struct TestCaseParams { From 9cb24e67884af09a30b9676e6790067d20b755e5 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 3 Aug 2023 09:08:38 +0200 Subject: [PATCH 087/128] log forward channel buffering --- quic-forward-proxy/src/inbound/proxy_listener.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index e514e335..5d01d2db 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -145,6 +145,10 @@ impl ProxyListener { txs.len(), tpu_address ); + if forwarder_channel_copy.capacity() < forwarder_channel_copy.max_capacity() { + debug!("forward channel buffered: capacity {} of {}", + forwarder_channel_copy.capacity(), forwarder_channel_copy.max_capacity()); + } forwarder_channel_copy .send_timeout( ForwardPacket { From d7e1539e0680199e4becc571f3cade1cc4a69e2a Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 3 Aug 2023 10:54:31 +0200 Subject: [PATCH 088/128] fix sender getting dropped in .listen --- quic-forward-proxy/src/inbound/proxy_listener.rs | 2 +- quic-forward-proxy/src/proxy.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 5d01d2db..f726b095 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -37,7 +37,7 @@ impl ProxyListener { pub async fn listen( &self, exit_signal: Arc, - forwarder_channel: Sender, + forwarder_channel: &Sender, ) -> anyhow::Result<()> { info!( "TPU Quic Proxy server listening on {}", diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index a8796f81..ea790ffc 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -44,7 +44,7 @@ impl QuicForwardProxy { let exit_signal_clone = exit_signal.clone(); let quic_proxy = tokio::spawn(async move { proxy_listener - .listen(exit_signal_clone.clone(), forwarder_channel) + .listen(exit_signal_clone.clone(), &forwarder_channel) .await .expect("proxy listen service"); }); From 2651469bb79388e67ee42ddec0729f46c48127b8 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 7 Aug 2023 18:40:55 +0200 Subject: [PATCH 089/128] remove DS_Store --- quic-forward-proxy/src/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 quic-forward-proxy/src/.DS_Store diff --git a/quic-forward-proxy/src/.DS_Store b/quic-forward-proxy/src/.DS_Store deleted file mode 100644 index 9d671b19042201cf3b55274a81eebe23bcb5a2bf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKF-`+P474E}c|XpwmjmS9*a5t~jrM~- zzT!yS%izcI=JJV(N&zV#1*Cu!kOF@xzyb?f-zI950#ZNZ9*7nv0JZw0;pVJRF? From bcd8d29abf4238e71b8891a1fc0822754bab47d2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 7 Aug 2023 22:18:33 +0200 Subject: [PATCH 090/128] fix fanout to n agents using broadcast channel --- quic-forward-proxy/Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 9dc99be8..18e5a264 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -41,6 +41,4 @@ chrono = { workspace = true } tokio = { version = "1.28.2", features = ["full", "fs"]} rcgen = "0.9.3" spl-memo = "3.0.1" -# tokio channel fanout -fan = "0.1.3" From 6938a0aa869f309f02fdfea50642c6768b89ddf1 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 7 Aug 2023 22:19:14 +0200 Subject: [PATCH 091/128] clippy-fmt --- .../tests/quic_proxy_tpu_integrationtest.rs | 25 ++++-- .../src/inbound/proxy_listener.rs | 10 ++- quic-forward-proxy/src/outbound/tx_forward.rs | 77 ++++++++++++------- .../quic_proxy_connection_manager.rs | 6 +- 4 files changed, 79 insertions(+), 39 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 554268d8..c4e9e5d7 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -1,7 +1,7 @@ use countmap::CountMap; use crossbeam_channel::Sender; -use log::{debug, info, trace, warn}; +use log::{debug, error, info, trace, warn}; use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; use solana_lite_rpc_core::solana_utils::SerializableTransaction; @@ -121,7 +121,6 @@ pub fn bench_proxy() { // consumed 1000 packets in 2059004us - throughput 485.67 tps, throughput_50 6704.05 tps wireup_and_send_txs_via_channel(TestCaseParams { - // sample_tx_count: 1000, // this is the goal -- ATM test runs too long sample_tx_count: 1000, stake_connection: true, proxy_mode: true, @@ -164,6 +163,21 @@ pub fn too_many_transactions() { // note: this not a tokio test as runtimes get created as part of the integration test fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { + let default_panic = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + default_panic(panic_info); + if let Some(location) = panic_info.location() { + error!( + "panic occurred in file '{}' at line {}", + location.file(), + location.line(), + ); + } else { + error!("panic occurred but can't get location information..."); + } + // std::process::exit(1); + })); + // value from solana - see quic streamer - see quic.rs -> rt() let runtime_quic1 = Builder::new_multi_thread() .worker_threads(1) @@ -254,7 +268,7 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { let mut count_map: CountMap = CountMap::with_capacity(test_case_params.sample_tx_count as usize); let warmup_tx_count: u32 = test_case_params.sample_tx_count / 2; - while packet_count < test_case_params.sample_tx_count { + while (count_map.len() as u32) < test_case_params.sample_tx_count { if latest_tx.elapsed() > Duration::from_secs(25) { warn!("abort after timeout waiting for packet from quic streamer"); break; @@ -299,9 +313,6 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { if packet_count == warmup_tx_count { timer2 = Some(Instant::now()); } - if packet_count == test_case_params.sample_tx_count { - break; - } } // -- while not all packets received - by count let total_duration = timer.elapsed(); @@ -327,7 +338,7 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { ); // note: this assumption will not hold as soon as test is configured to do fanout assert!( - count_map.values().all(|cnt| *cnt == 1), + count_map.values().all(|cnt| *cnt >= 1), "all transactions should be unique" ); diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index f726b095..e458d7cd 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -145,9 +145,13 @@ impl ProxyListener { txs.len(), tpu_address ); - if forwarder_channel_copy.capacity() < forwarder_channel_copy.max_capacity() { - debug!("forward channel buffered: capacity {} of {}", - forwarder_channel_copy.capacity(), forwarder_channel_copy.max_capacity()); + if forwarder_channel_copy.capacity() < forwarder_channel_copy.max_capacity() + { + debug!( + "forward channel buffered: capacity {} of {}", + forwarder_channel_copy.capacity(), + forwarder_channel_copy.max_capacity() + ); } forwarder_channel_copy .send_timeout( diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 6e535f81..4e9e7f4a 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -4,7 +4,6 @@ use crate::shared::ForwardPacket; use crate::util::timeout_fallback; use crate::validator_identity::ValidatorIdentity; use anyhow::{bail, Context}; -use fan::tokio::mpsc::FanOut; use futures::future::join_all; use log::{debug, info, warn}; use quinn::{ @@ -19,7 +18,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use tokio::sync::mpsc::{channel, Receiver}; +use tokio::sync::mpsc::Receiver; const MAX_PARALLEL_STREAMS: usize = 6; pub const PARALLEL_TPU_CONNECTION_COUNT: usize = 4; @@ -34,45 +33,64 @@ pub async fn tx_forwarder( let endpoint = new_endpoint_with_validator_identity(validator_identity).await; - let mut agents: HashMap> = HashMap::new(); + let (broadcast_in, _) = tokio::sync::broadcast::channel::>(1000); + + let mut agents: HashMap>> = HashMap::new(); loop { if exit_signal.load(Ordering::Relaxed) { bail!("exit signal received"); } - let forward_packet = transaction_channel - .recv() - .await - .expect("channel closed unexpectedly"); + let forward_packet = Arc::new( + transaction_channel + .recv() + .await + .expect("channel closed unexpectedly"), + ); let tpu_address = forward_packet.tpu_address; agents.entry(tpu_address).or_insert_with(|| { - let mut senders = Vec::new(); + let mut agent_exit_signals = Vec::new(); for connection_idx in 1..PARALLEL_TPU_CONNECTION_COUNT { - let (sender, mut receiver) = channel::(100_000); - senders.push(sender); - let exit_signal = exit_signal.clone(); + let global_exit_signal = exit_signal.clone(); + let agent_exit_signal = Arc::new(AtomicBool::new(false)); let endpoint_copy = endpoint.clone(); + let agent_exit_signal_copy = agent_exit_signal.clone(); + // by subscribing we expect to get a copy of each packet + let mut per_connection_receiver = broadcast_in.subscribe(); tokio::spawn(async move { debug!( "Start Quic forwarder agent #{} for TPU {}", connection_idx, tpu_address ); - if exit_signal.load(Ordering::Relaxed) { + if global_exit_signal.load(Ordering::Relaxed) { + warn!("Caught global exit signal - stopping agent thread"); + return; + } + if agent_exit_signal_copy.load(Ordering::Relaxed) { + warn!("Caught exit signal for this agent - stopping agent thread"); return; } + // get a copy of the packet from broadcast channel let auto_connection = AutoReconnect::new(endpoint_copy, tpu_address); - let _exit_signal_copy = exit_signal.clone(); - while let Some(packet) = receiver.recv().await { - assert_eq!(packet.tpu_address, tpu_address, "routing error"); + // TODO check exit signal (using select! or maybe replace with oneshot) + let _exit_signal_copy = global_exit_signal.clone(); + while let Ok(packet) = per_connection_receiver.recv().await { + if packet.tpu_address != tpu_address { + continue; + } - let mut transactions_batch = packet.transactions; + let mut transactions_batch: Vec = + packet.transactions.clone(); - while let Ok(more) = receiver.try_recv() { - transactions_batch.extend(more.transactions); + while let Ok(more) = per_connection_receiver.try_recv() { + if more.tpu_address != tpu_address { + continue; + } + transactions_batch.extend(more.transactions.clone()); } debug!( @@ -105,21 +123,28 @@ pub async fn tx_forwarder( } } // -- while all packtes from channel - info!( + warn!( "Quic forwarder agent #{} for TPU {} exited", connection_idx, tpu_address ); - }); - } + }); // -- spawned thread for one connection to one TPU + agent_exit_signals.push(agent_exit_signal); + } // -- for parallel connections to one TPU - FanOut::new(senders) + // FanOut::new(senders) + agent_exit_signals }); // -- new agent - let agent_channel = agents.get(&tpu_address).unwrap(); + let _agent_channel = agents.get(&tpu_address).unwrap(); + + if broadcast_in.len() > 5 { + debug!("tx-forward queue len: {}", broadcast_in.len()) + } - timeout_fallback(agent_channel.send(forward_packet)) - .await - .context("send to agent channel")??; + // TODO use agent_exit signal to clean them up + broadcast_in + .send(forward_packet) + .expect("send must succeed"); } // -- loop over transactions from upstream channels // not reachable diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 14cf9d97..89cd3054 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -172,8 +172,8 @@ impl QuicProxyConnectionManager { }, Err(e) => { error!( - "Broadcast channel error on recv error {}", e); - continue; + "Broadcast channel error (close) on recv: {} - aborting", e); + return; } }; @@ -192,7 +192,7 @@ impl QuicProxyConnectionManager { for target_tpu_node in tpu_fanout_nodes { Self::send_copy_of_txs_to_quicproxy( &txs, &auto_connection, - proxy_addr, + proxy_addr, target_tpu_node.tpu_address, target_tpu_node.tpu_identity) .await.unwrap(); From 485bb399a8ede3d17d43edc9b3ae633740bb9cb2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 10:27:12 +0200 Subject: [PATCH 092/128] add sharding to spread work across connections --- quic-forward-proxy/src/outbound/mod.rs | 1 + quic-forward-proxy/src/outbound/tx_forward.rs | 12 +++++++++++- quic-forward-proxy/src/shared/mod.rs | 11 +++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/quic-forward-proxy/src/outbound/mod.rs b/quic-forward-proxy/src/outbound/mod.rs index 30b80e1d..968674c2 100644 --- a/quic-forward-proxy/src/outbound/mod.rs +++ b/quic-forward-proxy/src/outbound/mod.rs @@ -1 +1,2 @@ +mod sharder; pub mod tx_forward; diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 4e9e7f4a..1a13ceb0 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -1,3 +1,4 @@ +use crate::outbound::sharder::Sharder; use crate::quic_util::SkipServerVerification; use crate::quinn_auto_reconnect::AutoReconnect; use crate::shared::ForwardPacket; @@ -52,12 +53,15 @@ pub async fn tx_forwarder( agents.entry(tpu_address).or_insert_with(|| { let mut agent_exit_signals = Vec::new(); - for connection_idx in 1..PARALLEL_TPU_CONNECTION_COUNT { + for connection_idx in 0..PARALLEL_TPU_CONNECTION_COUNT { + let sharder = + Sharder::new(connection_idx as u32, PARALLEL_TPU_CONNECTION_COUNT as u32); let global_exit_signal = exit_signal.clone(); let agent_exit_signal = Arc::new(AtomicBool::new(false)); let endpoint_copy = endpoint.clone(); let agent_exit_signal_copy = agent_exit_signal.clone(); // by subscribing we expect to get a copy of each packet + let mut per_connection_receiver = broadcast_in.subscribe(); tokio::spawn(async move { debug!( @@ -82,6 +86,9 @@ pub async fn tx_forwarder( if packet.tpu_address != tpu_address { continue; } + if !sharder.matching(packet.shard_hash()) { + continue; + } let mut transactions_batch: Vec = packet.transactions.clone(); @@ -90,6 +97,9 @@ pub async fn tx_forwarder( if more.tpu_address != tpu_address { continue; } + if !sharder.matching(more.shard_hash()) { + continue; + } transactions_batch.extend(more.transactions.clone()); } diff --git a/quic-forward-proxy/src/shared/mod.rs b/quic-forward-proxy/src/shared/mod.rs index 6b3f4c43..63557f81 100644 --- a/quic-forward-proxy/src/shared/mod.rs +++ b/quic-forward-proxy/src/shared/mod.rs @@ -1,4 +1,6 @@ use solana_sdk::transaction::VersionedTransaction; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; use std::net::SocketAddr; /// internal structure with transactions and target TPU @@ -7,3 +9,12 @@ pub struct ForwardPacket { pub transactions: Vec, pub tpu_address: SocketAddr, } + +impl ForwardPacket { + pub fn shard_hash(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + // note: assumes that there are transactions with >=0 signatures + self.transactions[0].signatures[0].hash(&mut hasher); + hasher.finish() + } +} From 3bbe99fecda218bf3c332053f7eb79dde9ca1a3d Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 11:19:01 +0200 Subject: [PATCH 093/128] make connection failures more explicit --- .../src/quinn_auto_reconnect.rs | 153 ++++++++++++------ .../src/tpu_utils/quinn_auto_reconnect.rs | 1 + 2 files changed, 105 insertions(+), 49 deletions(-) diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 9e7a7d66..01e552f2 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -1,17 +1,23 @@ use crate::util::timeout_fallback; -use anyhow::Context; -use log::warn; -use quinn::{Connection, Endpoint}; +use anyhow::{bail, Context}; +use log::{info, warn}; +use quinn::{Connection, ConnectionError, Endpoint}; use std::fmt; use std::net::SocketAddr; use std::sync::atomic::{AtomicU32, Ordering}; use tokio::sync::RwLock; use tracing::debug; +enum ConnectionState { + NotConnected, + Connection(Connection), + PermanentError, +} + pub struct AutoReconnect { // endoint should be configures with keep-alive and idle timeout endpoint: Endpoint, - current: RwLock>, + current: RwLock, pub target_address: SocketAddr, reconnect_count: AtomicU32, } @@ -20,15 +26,14 @@ impl AutoReconnect { pub fn new(endpoint: Endpoint, target_address: SocketAddr) -> Self { Self { endpoint, - current: RwLock::new(None), + current: RwLock::new(ConnectionState::NotConnected), target_address, reconnect_count: AtomicU32::new(0), } } pub async fn send_uni(&self, payload: Vec) -> anyhow::Result<()> { - // TOOD do smart error handling + reconnect - let mut send_stream = timeout_fallback(self.refresh().await.open_uni()) + let mut send_stream = timeout_fallback(self.refresh_and_get().await?.open_uni()) .await .context("open uni stream for sending")??; send_stream.write_all(payload.as_slice()).await?; @@ -36,23 +41,31 @@ impl AutoReconnect { Ok(()) } - pub async fn refresh(&self) -> Connection { + pub async fn refresh_and_get(&self) -> anyhow::Result { + self.refresh().await; + + let lock = self.current.read().await; + match &*lock { + ConnectionState::NotConnected => bail!("not connected"), + ConnectionState::Connection(conn) => Ok(conn.clone()), + ConnectionState::PermanentError => bail!("permanent error"), + } + } + + pub async fn refresh(&self) { { + // first check for existing connection using a cheap read-lock let lock = self.current.read().await; - let maybe_conn = lock.as_ref(); - if maybe_conn - .filter(|conn| conn.close_reason().is_none()) - .is_some() - { - let reuse = maybe_conn.unwrap(); - debug!("Reuse connection {}", reuse.stable_id()); - return reuse.clone(); + if let ConnectionState::Connection(conn) = &*lock { + if conn.close_reason().is_none() { + debug!("Reuse connection {}", conn.stable_id()); + return; + } } } let mut lock = self.current.write().await; - let maybe_conn = lock.as_ref(); - match maybe_conn { - Some(current) => { + match &*lock { + ConnectionState::Connection(current) => { if current.close_reason().is_some() { let old_stable_id = current.stable_id(); warn!( @@ -61,44 +74,86 @@ impl AutoReconnect { current.close_reason() ); - let new_connection = self.create_connection().await; - *lock = Some(new_connection.clone()); - // let old_conn = lock.replace(new_connection.clone()); - self.reconnect_count.fetch_add(1, Ordering::SeqCst); + match self.create_connection().await { + Some(new_connection) => { + *lock = ConnectionState::Connection(new_connection.clone()); + let reconnect_count = + self.reconnect_count.fetch_add(1, Ordering::SeqCst); - debug!( - "Replace closed connection {} with {} (retry {})", - old_stable_id, - new_connection.stable_id(), - self.reconnect_count.load(Ordering::SeqCst) - ); - - new_connection + if reconnect_count < 10 { + info!( + "Replace closed connection {} with {} (retry {})", + old_stable_id, + new_connection.stable_id(), + reconnect_count + ); + } else { + *lock = ConnectionState::PermanentError; + warn!( + "Too many reconnect attempts {} with {} (retry {})", + old_stable_id, + new_connection.stable_id(), + reconnect_count + ); + } + } + None => { + warn!( + "Reconnect to {} failed for connection {}", + self.target_address, old_stable_id + ); + *lock = ConnectionState::PermanentError; + } + }; } else { debug!("Reuse connection {} with write-lock", current.stable_id()); - current.clone() } } - None => { - let new_connection = self.create_connection().await; - - assert!(lock.is_none(), "old connection must be None"); - *lock = Some(new_connection.clone()); - // let old_conn = foo.replace(Some(new_connection.clone())); - debug!("Create initial connection {}", new_connection.stable_id()); + ConnectionState::NotConnected => { + match self.create_connection().await { + Some(new_connection) => { + *lock = ConnectionState::Connection(new_connection.clone()); + self.reconnect_count.fetch_add(1, Ordering::SeqCst); - new_connection + info!( + "Create initial connection {} to {}", + new_connection.stable_id(), + self.target_address + ); + } + None => { + warn!( + "Initial connection to {} failed permanently", + self.target_address + ); + *lock = ConnectionState::PermanentError; + } + }; + } + ConnectionState::PermanentError => { + // no nothing + debug!("Not using connection with permanent error"); } } } - async fn create_connection(&self) -> Connection { + async fn create_connection(&self) -> Option { let connection = self .endpoint .connect(self.target_address, "localhost") .expect("handshake"); - connection.await.expect("connection") + match connection.await { + Ok(conn) => Some(conn), + Err(ConnectionError::TimedOut) => None, + // maybe we should also treat TransportError explicitly + Err(unexpected_error) => { + panic!( + "Connection to {} failed with unexpected error: {}", + self.target_address, unexpected_error + ); + } + } } // stable_id 140266619216912, rtt=2.156683ms, @@ -110,15 +165,15 @@ impl AutoReconnect { // STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } pub async fn connection_stats(&self) -> String { let lock = self.current.read().await; - let maybe_conn = lock.as_ref(); - match maybe_conn { - Some(connection) => format!( + match &*lock { + ConnectionState::Connection(conn) => format!( "stable_id {} stats {:?}, rtt={:?}", - connection.stable_id(), - connection.stats().frame_rx, - connection.stats().path.rtt + conn.stable_id(), + conn.stats().frame_rx, + conn.stats().path.rtt ), - None => "n/a".to_string(), + ConnectionState::NotConnected => "n/c".to_string(), + ConnectionState::PermanentError => "n/a (permanent)".to_string(), } } } diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index ee5a3b09..e8dd13b0 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -9,6 +9,7 @@ use tokio::sync::RwLock; use tokio::time::timeout; use tracing::debug; +/// copy of quic-proxy AutoReconnect - used that for reference pub struct AutoReconnect { // endoint should be configures with keep-alive and idle timeout endpoint: Endpoint, From dd5ad1b875df646a9c10a9c400977549ea2e6750 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 11:19:01 +0200 Subject: [PATCH 094/128] make connection failures more explicit --- .../tests/quic_proxy_tpu_integrationtest.rs | 1 + quic-forward-proxy/src/outbound/sharder.rs | 32 ++++ .../src/quinn_auto_reconnect.rs | 153 ++++++++++++------ .../src/tpu_utils/quinn_auto_reconnect.rs | 1 + ...literpc_tpu_quic_server_integrationtest.rs | 5 +- 5 files changed, 139 insertions(+), 53 deletions(-) create mode 100644 quic-forward-proxy/src/outbound/sharder.rs diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index c4e9e5d7..d0c1c84d 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -268,6 +268,7 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { let mut count_map: CountMap = CountMap::with_capacity(test_case_params.sample_tx_count as usize); let warmup_tx_count: u32 = test_case_params.sample_tx_count / 2; + while (count_map.len() as u32) < test_case_params.sample_tx_count { if latest_tx.elapsed() > Duration::from_secs(25) { warn!("abort after timeout waiting for packet from quic streamer"); diff --git a/quic-forward-proxy/src/outbound/sharder.rs b/quic-forward-proxy/src/outbound/sharder.rs new file mode 100644 index 00000000..fa55d8c6 --- /dev/null +++ b/quic-forward-proxy/src/outbound/sharder.rs @@ -0,0 +1,32 @@ +pub struct Sharder { + n_shards: u32, + pos: u32, +} + +impl Sharder { + pub fn new(pos: u32, n_shards: u32) -> Self { + assert!(n_shards > 0); + assert!(pos < n_shards, "out of range"); + + Self { n_shards, pos } + } + + pub fn matching(&self, hash: u64) -> bool { + (hash % self.n_shards as u64) as u32 == self.pos + } +} + +#[cfg(test)] +mod tests { + use crate::outbound::sharder::Sharder; + + #[test] + fn shard() { + let sharder = Sharder::new(3, 10); + + assert!(sharder.matching(13)); + assert!(sharder.matching(23)); + assert!(sharder.matching(33)); + assert!(!sharder.matching(31)); + } +} diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 9e7a7d66..01e552f2 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -1,17 +1,23 @@ use crate::util::timeout_fallback; -use anyhow::Context; -use log::warn; -use quinn::{Connection, Endpoint}; +use anyhow::{bail, Context}; +use log::{info, warn}; +use quinn::{Connection, ConnectionError, Endpoint}; use std::fmt; use std::net::SocketAddr; use std::sync::atomic::{AtomicU32, Ordering}; use tokio::sync::RwLock; use tracing::debug; +enum ConnectionState { + NotConnected, + Connection(Connection), + PermanentError, +} + pub struct AutoReconnect { // endoint should be configures with keep-alive and idle timeout endpoint: Endpoint, - current: RwLock>, + current: RwLock, pub target_address: SocketAddr, reconnect_count: AtomicU32, } @@ -20,15 +26,14 @@ impl AutoReconnect { pub fn new(endpoint: Endpoint, target_address: SocketAddr) -> Self { Self { endpoint, - current: RwLock::new(None), + current: RwLock::new(ConnectionState::NotConnected), target_address, reconnect_count: AtomicU32::new(0), } } pub async fn send_uni(&self, payload: Vec) -> anyhow::Result<()> { - // TOOD do smart error handling + reconnect - let mut send_stream = timeout_fallback(self.refresh().await.open_uni()) + let mut send_stream = timeout_fallback(self.refresh_and_get().await?.open_uni()) .await .context("open uni stream for sending")??; send_stream.write_all(payload.as_slice()).await?; @@ -36,23 +41,31 @@ impl AutoReconnect { Ok(()) } - pub async fn refresh(&self) -> Connection { + pub async fn refresh_and_get(&self) -> anyhow::Result { + self.refresh().await; + + let lock = self.current.read().await; + match &*lock { + ConnectionState::NotConnected => bail!("not connected"), + ConnectionState::Connection(conn) => Ok(conn.clone()), + ConnectionState::PermanentError => bail!("permanent error"), + } + } + + pub async fn refresh(&self) { { + // first check for existing connection using a cheap read-lock let lock = self.current.read().await; - let maybe_conn = lock.as_ref(); - if maybe_conn - .filter(|conn| conn.close_reason().is_none()) - .is_some() - { - let reuse = maybe_conn.unwrap(); - debug!("Reuse connection {}", reuse.stable_id()); - return reuse.clone(); + if let ConnectionState::Connection(conn) = &*lock { + if conn.close_reason().is_none() { + debug!("Reuse connection {}", conn.stable_id()); + return; + } } } let mut lock = self.current.write().await; - let maybe_conn = lock.as_ref(); - match maybe_conn { - Some(current) => { + match &*lock { + ConnectionState::Connection(current) => { if current.close_reason().is_some() { let old_stable_id = current.stable_id(); warn!( @@ -61,44 +74,86 @@ impl AutoReconnect { current.close_reason() ); - let new_connection = self.create_connection().await; - *lock = Some(new_connection.clone()); - // let old_conn = lock.replace(new_connection.clone()); - self.reconnect_count.fetch_add(1, Ordering::SeqCst); + match self.create_connection().await { + Some(new_connection) => { + *lock = ConnectionState::Connection(new_connection.clone()); + let reconnect_count = + self.reconnect_count.fetch_add(1, Ordering::SeqCst); - debug!( - "Replace closed connection {} with {} (retry {})", - old_stable_id, - new_connection.stable_id(), - self.reconnect_count.load(Ordering::SeqCst) - ); - - new_connection + if reconnect_count < 10 { + info!( + "Replace closed connection {} with {} (retry {})", + old_stable_id, + new_connection.stable_id(), + reconnect_count + ); + } else { + *lock = ConnectionState::PermanentError; + warn!( + "Too many reconnect attempts {} with {} (retry {})", + old_stable_id, + new_connection.stable_id(), + reconnect_count + ); + } + } + None => { + warn!( + "Reconnect to {} failed for connection {}", + self.target_address, old_stable_id + ); + *lock = ConnectionState::PermanentError; + } + }; } else { debug!("Reuse connection {} with write-lock", current.stable_id()); - current.clone() } } - None => { - let new_connection = self.create_connection().await; - - assert!(lock.is_none(), "old connection must be None"); - *lock = Some(new_connection.clone()); - // let old_conn = foo.replace(Some(new_connection.clone())); - debug!("Create initial connection {}", new_connection.stable_id()); + ConnectionState::NotConnected => { + match self.create_connection().await { + Some(new_connection) => { + *lock = ConnectionState::Connection(new_connection.clone()); + self.reconnect_count.fetch_add(1, Ordering::SeqCst); - new_connection + info!( + "Create initial connection {} to {}", + new_connection.stable_id(), + self.target_address + ); + } + None => { + warn!( + "Initial connection to {} failed permanently", + self.target_address + ); + *lock = ConnectionState::PermanentError; + } + }; + } + ConnectionState::PermanentError => { + // no nothing + debug!("Not using connection with permanent error"); } } } - async fn create_connection(&self) -> Connection { + async fn create_connection(&self) -> Option { let connection = self .endpoint .connect(self.target_address, "localhost") .expect("handshake"); - connection.await.expect("connection") + match connection.await { + Ok(conn) => Some(conn), + Err(ConnectionError::TimedOut) => None, + // maybe we should also treat TransportError explicitly + Err(unexpected_error) => { + panic!( + "Connection to {} failed with unexpected error: {}", + self.target_address, unexpected_error + ); + } + } } // stable_id 140266619216912, rtt=2.156683ms, @@ -110,15 +165,15 @@ impl AutoReconnect { // STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } pub async fn connection_stats(&self) -> String { let lock = self.current.read().await; - let maybe_conn = lock.as_ref(); - match maybe_conn { - Some(connection) => format!( + match &*lock { + ConnectionState::Connection(conn) => format!( "stable_id {} stats {:?}, rtt={:?}", - connection.stable_id(), - connection.stats().frame_rx, - connection.stats().path.rtt + conn.stable_id(), + conn.stats().frame_rx, + conn.stats().path.rtt ), - None => "n/a".to_string(), + ConnectionState::NotConnected => "n/c".to_string(), + ConnectionState::PermanentError => "n/a (permanent)".to_string(), } } } diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index ee5a3b09..e8dd13b0 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -9,6 +9,7 @@ use tokio::sync::RwLock; use tokio::time::timeout; use tracing::debug; +/// copy of quic-proxy AutoReconnect - used that for reference pub struct AutoReconnect { // endoint should be configures with keep-alive and idle timeout endpoint: Endpoint, diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index 076f5122..3da561a2 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -71,8 +71,6 @@ pub fn small_tx_batch_unstaked() { }); } -// fails on ci -#[ignore] #[test] pub fn many_transactions() { configure_logging(false); @@ -162,7 +160,7 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { let mut count_map: CountMap = CountMap::with_capacity(test_case_params.sample_tx_count as usize); let warmup_tx_count: u32 = test_case_params.sample_tx_count / 2; - while packet_count < test_case_params.sample_tx_count { + while (count_map.len() as u32) < test_case_params.sample_tx_count { if latest_tx.elapsed() > Duration::from_secs(5) { warn!("abort after timeout waiting for packet from quic streamer"); break; @@ -233,7 +231,6 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { test_case_params.sample_tx_count, "count_map size should be equal to sample_tx_count" ); - // note: this assumption will not hold as soon as test is configured to do fanout assert!( count_map.values().all(|cnt| *cnt == 1), "all transactions should be unique" From 3902dcca33ab3e6302a2c47c8f043b8997d0be5e Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 11:50:02 +0200 Subject: [PATCH 095/128] improve loggin --- .github/workflows/test.yml | 2 +- .../tests/quic_proxy_tpu_integrationtest.rs | 7 ++++++- .../literpc_tpu_quic_server_integrationtest.rs | 18 +++++++++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 74159726..b714cccf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: run: cargo clippy --all-targets -- -D warnings - name: Run Tests - run: cargo test + run: RUST_LOG=info cargo test - name: Install node deps run: yarn diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index d0c1c84d..2d7ffcd7 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -32,6 +32,7 @@ use tokio::runtime::Builder; use tokio::task::{yield_now, JoinHandle}; use tokio::time::sleep; +use tracing_subscriber::EnvFilter; use solana_lite_rpc_quic_forward_proxy::proxy::QuicForwardProxy; use solana_lite_rpc_quic_forward_proxy::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; @@ -360,8 +361,12 @@ fn configure_logging(verbose: bool) { } else { FmtSpan::NONE }; + // EnvFilter::try_from_default_env().unwrap_or(env_filter) + let filter = + EnvFilter::try_from_default_env().unwrap_or(EnvFilter::from_str(env_filter).unwrap()); + let result = tracing_subscriber::fmt::fmt() - .with_env_filter(env_filter) + .with_env_filter(filter) .with_span_events(span_mode) .try_init(); if result.is_err() { diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index 3da561a2..40142493 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -31,6 +31,8 @@ use std::time::{Duration, Instant}; use tokio::runtime::Builder; use tokio::task::JoinHandle; use tokio::time::sleep; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::EnvFilter; #[derive(Copy, Clone, Debug)] struct TestCaseParams { @@ -244,12 +246,22 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { fn configure_logging(verbose: bool) { let env_filter = if verbose { - "trace,rustls=info,quinn_proto=debug" + "debug,rustls=info,quinn=info,quinn_proto=debug,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=trace" } else { - "debug,quinn_proto=info,rustls=info,solana_streamer=debug" + "info,rustls=info,quinn=info,quinn_proto=info,solana_streamer=info,solana_lite_rpc_quic_forward_proxy=info" }; + let span_mode = if verbose { + FmtSpan::CLOSE + } else { + FmtSpan::NONE + }; + // EnvFilter::try_from_default_env().unwrap_or(env_filter) + let filter = + EnvFilter::try_from_default_env().unwrap_or(EnvFilter::from_str(env_filter).unwrap()); + let result = tracing_subscriber::fmt::fmt() - .with_env_filter(env_filter) + .with_env_filter(filter) + .with_span_events(span_mode) .try_init(); if result.is_err() { println!("Logging already initialized - ignore"); From 905cb2f9ad51aab546e88947650a85926a0b16e1 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 12:11:05 +0200 Subject: [PATCH 096/128] docs on autoconnect --- quic-forward-proxy/src/quinn_auto_reconnect.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 01e552f2..7194e400 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -8,6 +8,15 @@ use std::sync::atomic::{AtomicU32, Ordering}; use tokio::sync::RwLock; use tracing::debug; +/// connection manager with automatic reconnect; designated for connection to Solana TPU nodes +/// +/// assumptions: +/// * connection to TPU node is reliable +/// * manager and TPU nodes run both in data centers with fast internet connectivity +/// * ping times vary between 50ms and 400ms depending on the location +/// * TPU address might be wrong which then is a permanent problem +/// * the ActiveConnection instance gets renewed on leader schedule change + enum ConnectionState { NotConnected, Connection(Connection), From 9e05d2bd690c13a328f99ecb5f030e637eb7ec18 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 12:14:31 +0200 Subject: [PATCH 097/128] refactored out configure_panic_hook --- .../tests/quic_proxy_tpu_integrationtest.rs | 32 +++++++++++-------- ...literpc_tpu_quic_server_integrationtest.rs | 20 +++++++++++- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 2d7ffcd7..bb8ff556 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -164,20 +164,7 @@ pub fn too_many_transactions() { // note: this not a tokio test as runtimes get created as part of the integration test fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { - let default_panic = std::panic::take_hook(); - std::panic::set_hook(Box::new(move |panic_info| { - default_panic(panic_info); - if let Some(location) = panic_info.location() { - error!( - "panic occurred in file '{}' at line {}", - location.file(), - location.line(), - ); - } else { - error!("panic occurred but can't get location information..."); - } - // std::process::exit(1); - })); + configure_panic_hook(); // value from solana - see quic streamer - see quic.rs -> rt() let runtime_quic1 = Builder::new_multi_thread() @@ -350,6 +337,23 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { packet_consumer_jh.join().unwrap(); } +fn configure_panic_hook() { + let default_panic = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + default_panic(panic_info); + if let Some(location) = panic_info.location() { + error!( + "panic occurred in file '{}' at line {}", + location.file(), + location.line(), + ); + } else { + error!("panic occurred but can't get location information..."); + } + // std::process::exit(1); + })); +} + fn configure_logging(verbose: bool) { let env_filter = if verbose { "debug,rustls=info,quinn=info,quinn_proto=debug,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=trace" diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index 40142493..663e1b82 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -1,7 +1,7 @@ use countmap::CountMap; use crossbeam_channel::Sender; -use log::{debug, info, trace, warn}; +use log::{debug, error, info, trace, warn}; use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; use solana_lite_rpc_core::structures::identity_stakes::IdentityStakes; @@ -96,6 +96,7 @@ pub fn too_many_transactions() { // note: this not a tokio test as runtimes get created as part of the integration test fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { + configure_panic_hook(); // value from solana - see quic streamer - see quic.rs -> rt() const NUM_QUIC_STREAMER_WORKER_THREADS: usize = 1; let runtime_quic1 = Builder::new_multi_thread() @@ -244,6 +245,23 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { packet_consumer_jh.join().unwrap(); } +fn configure_panic_hook() { + let default_panic = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |panic_info| { + default_panic(panic_info); + if let Some(location) = panic_info.location() { + error!( + "panic occurred in file '{}' at line {}", + location.file(), + location.line(), + ); + } else { + error!("panic occurred but can't get location information..."); + } + // std::process::exit(1); + })); +} + fn configure_logging(verbose: bool) { let env_filter = if verbose { "debug,rustls=info,quinn=info,quinn_proto=debug,solana_streamer=debug,solana_lite_rpc_quic_forward_proxy=trace" From 6a1d4c66c51feec0dd4b4136e2d3285022684517 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 12:16:04 +0200 Subject: [PATCH 098/128] comment on configure_panic_hook --- .../tests/quic_proxy_tpu_integrationtest.rs | 2 +- services/tests/literpc_tpu_quic_server_integrationtest.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index bb8ff556..d1d0c853 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -350,7 +350,7 @@ fn configure_panic_hook() { } else { error!("panic occurred but can't get location information..."); } - // std::process::exit(1); + // note: we do not exit the process to allow proper test execution })); } diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index 663e1b82..b133dec2 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -258,7 +258,7 @@ fn configure_panic_hook() { } else { error!("panic occurred but can't get location information..."); } - // std::process::exit(1); + // note: we do not exit the process to allow proper test execution })); } From 946536f42317c64b821bbaa03656d7a9e3311399 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 13:13:27 +0200 Subject: [PATCH 099/128] use TryFrom+TryInto for (de-)serialization if proxy_request_format --- core/src/proxy_request_format.rs | 38 ++++++++++++------- .../src/inbound/proxy_listener.rs | 2 +- .../src/proxy_request_format.rs | 37 ++++++++++-------- .../tests/proxy_request_format.rs | 14 +++++-- 4 files changed, 58 insertions(+), 33 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index 8970becd..70f51c7b 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -46,20 +46,6 @@ impl TpuForwardingRequest { } } - pub fn serialize_wire_format(&self) -> Vec { - bincode::serialize(&self).expect("Expect to serialize transactions") - } - - pub fn deserialize_from_raw_request(raw_proxy_request: &[u8]) -> TpuForwardingRequest { - let request = bincode::deserialize::(raw_proxy_request) - .context("deserialize proxy request") - .unwrap(); - - assert_eq!(request.format_version, 2301); - - request - } - pub fn get_tpu_socket_addr(&self) -> SocketAddr { self.tpu_socket_addr } @@ -72,3 +58,27 @@ impl TpuForwardingRequest { self.transactions.clone() } } + +impl TryInto> for TpuForwardingRequest { + type Error = anyhow::Error; + + fn try_into(self) -> Result, Self::Error> { + bincode::serialize(&self) + .context("serialize proxy request") + .map_err(anyhow::Error::from) + } +} + +impl TryFrom<&[u8]> for TpuForwardingRequest { + type Error = anyhow::Error; + + fn try_from(value: &[u8]) -> Result { + let request = bincode::deserialize::(value) + .context("deserialize proxy request") + .map_err(anyhow::Error::from); + if let Ok(ref req) = request { + assert_eq!(req.format_version, FORMAT_VERSION1); + } + request + } +} diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index e458d7cd..e5966478 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -133,7 +133,7 @@ impl ProxyListener { let raw_request = recv_stream.read_to_end(10_000_000).await.unwrap(); let proxy_request = - TpuForwardingRequest::deserialize_from_raw_request(&raw_request); + TpuForwardingRequest::try_from(raw_request.as_slice()).unwrap(); trace!("proxy request details: {}", proxy_request); let _tpu_identity = proxy_request.get_identity_tpunode(); diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 9b8b5761..ec4dfcd1 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -47,21 +47,6 @@ impl TpuForwardingRequest { } } - pub fn serialize_wire_format(&self) -> Vec { - bincode::serialize(&self).expect("Expect to serialize transactions") - } - - // TODO reame - pub fn deserialize_from_raw_request(raw_proxy_request: &[u8]) -> TpuForwardingRequest { - let request = bincode::deserialize::(raw_proxy_request) - .context("deserialize proxy request") - .unwrap(); - - assert_eq!(request.format_version, FORMAT_VERSION1); - - request - } - pub fn get_tpu_socket_addr(&self) -> SocketAddr { self.tpu_socket_addr } @@ -74,3 +59,25 @@ impl TpuForwardingRequest { self.transactions.clone() } } + +impl TryInto> for TpuForwardingRequest { + type Error = anyhow::Error; + + fn try_into(self) -> Result, Self::Error> { + bincode::serialize(&self).map_err(anyhow::Error::from) + } +} + +impl TryFrom<&[u8]> for TpuForwardingRequest { + type Error = anyhow::Error; + + fn try_from(value: &[u8]) -> Result { + let request = bincode::deserialize::(value) + .context("deserialize proxy request") + .map_err(anyhow::Error::from); + if let Ok(ref req) = request { + assert_eq!(req.format_version, FORMAT_VERSION1); + } + request + } +} diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index bb752be8..2256a6f5 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -15,17 +15,25 @@ fn roundtrip() { let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); - let wire_data = TpuForwardingRequest::new( + let wire_data: Vec = TpuForwardingRequest::new( "127.0.0.1:5454".parse().unwrap(), Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), vec![tx.into()], ) - .serialize_wire_format(); + .try_into() + .unwrap(); println!("wire_data: {:02X?}", wire_data); - let request = TpuForwardingRequest::deserialize_from_raw_request(&wire_data); + let request = TpuForwardingRequest::try_from(wire_data.as_slice()).unwrap(); assert!(request.get_tpu_socket_addr().is_ipv4()); assert_eq!(request.get_transactions().len(), 1); } + +#[test] +fn deserialize_error() { + let value: &[u8] = &[1, 2, 3, 4]; + let result = TpuForwardingRequest::try_from(value); + assert_eq!(result.unwrap_err().to_string(), "deserialize proxy request"); +} From 415835912a339a56425bbddc09646480fe0f2690 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 14:19:34 +0200 Subject: [PATCH 100/128] return transactions as slice --- core/src/proxy_request_format.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index 70f51c7b..c6b281f1 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -54,8 +54,8 @@ impl TpuForwardingRequest { self.identity_tpunode } - pub fn get_transactions(&self) -> Vec { - self.transactions.clone() + pub fn get_transactions(&self) -> &[VersionedTransaction] { + self.transactions.as_slice() } } From b19a7a5907df7c37d14c02e2b99eabf951e14c0c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 15:30:42 +0200 Subject: [PATCH 101/128] Revert "use TryFrom+TryInto for (de-)serialization if proxy_request_format" This reverts commit 946536f42317c64b821bbaa03656d7a9e3311399. --- core/src/proxy_request_format.rs | 38 +++++++------------ .../src/inbound/proxy_listener.rs | 2 +- .../src/proxy_request_format.rs | 37 ++++++++---------- .../tests/proxy_request_format.rs | 14 ++----- 4 files changed, 33 insertions(+), 58 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index c6b281f1..e45ba847 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -46,6 +46,20 @@ impl TpuForwardingRequest { } } + pub fn serialize_wire_format(&self) -> Vec { + bincode::serialize(&self).expect("Expect to serialize transactions") + } + + pub fn deserialize_from_raw_request(raw_proxy_request: &[u8]) -> TpuForwardingRequest { + let request = bincode::deserialize::(raw_proxy_request) + .context("deserialize proxy request") + .unwrap(); + + assert_eq!(request.format_version, 2301); + + request + } + pub fn get_tpu_socket_addr(&self) -> SocketAddr { self.tpu_socket_addr } @@ -58,27 +72,3 @@ impl TpuForwardingRequest { self.transactions.as_slice() } } - -impl TryInto> for TpuForwardingRequest { - type Error = anyhow::Error; - - fn try_into(self) -> Result, Self::Error> { - bincode::serialize(&self) - .context("serialize proxy request") - .map_err(anyhow::Error::from) - } -} - -impl TryFrom<&[u8]> for TpuForwardingRequest { - type Error = anyhow::Error; - - fn try_from(value: &[u8]) -> Result { - let request = bincode::deserialize::(value) - .context("deserialize proxy request") - .map_err(anyhow::Error::from); - if let Ok(ref req) = request { - assert_eq!(req.format_version, FORMAT_VERSION1); - } - request - } -} diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index e5966478..e458d7cd 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -133,7 +133,7 @@ impl ProxyListener { let raw_request = recv_stream.read_to_end(10_000_000).await.unwrap(); let proxy_request = - TpuForwardingRequest::try_from(raw_request.as_slice()).unwrap(); + TpuForwardingRequest::deserialize_from_raw_request(&raw_request); trace!("proxy request details: {}", proxy_request); let _tpu_identity = proxy_request.get_identity_tpunode(); diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index ec4dfcd1..9b8b5761 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -47,6 +47,21 @@ impl TpuForwardingRequest { } } + pub fn serialize_wire_format(&self) -> Vec { + bincode::serialize(&self).expect("Expect to serialize transactions") + } + + // TODO reame + pub fn deserialize_from_raw_request(raw_proxy_request: &[u8]) -> TpuForwardingRequest { + let request = bincode::deserialize::(raw_proxy_request) + .context("deserialize proxy request") + .unwrap(); + + assert_eq!(request.format_version, FORMAT_VERSION1); + + request + } + pub fn get_tpu_socket_addr(&self) -> SocketAddr { self.tpu_socket_addr } @@ -59,25 +74,3 @@ impl TpuForwardingRequest { self.transactions.clone() } } - -impl TryInto> for TpuForwardingRequest { - type Error = anyhow::Error; - - fn try_into(self) -> Result, Self::Error> { - bincode::serialize(&self).map_err(anyhow::Error::from) - } -} - -impl TryFrom<&[u8]> for TpuForwardingRequest { - type Error = anyhow::Error; - - fn try_from(value: &[u8]) -> Result { - let request = bincode::deserialize::(value) - .context("deserialize proxy request") - .map_err(anyhow::Error::from); - if let Ok(ref req) = request { - assert_eq!(req.format_version, FORMAT_VERSION1); - } - request - } -} diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index 2256a6f5..bb752be8 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -15,25 +15,17 @@ fn roundtrip() { let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); - let wire_data: Vec = TpuForwardingRequest::new( + let wire_data = TpuForwardingRequest::new( "127.0.0.1:5454".parse().unwrap(), Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), vec![tx.into()], ) - .try_into() - .unwrap(); + .serialize_wire_format(); println!("wire_data: {:02X?}", wire_data); - let request = TpuForwardingRequest::try_from(wire_data.as_slice()).unwrap(); + let request = TpuForwardingRequest::deserialize_from_raw_request(&wire_data); assert!(request.get_tpu_socket_addr().is_ipv4()); assert_eq!(request.get_transactions().len(), 1); } - -#[test] -fn deserialize_error() { - let value: &[u8] = &[1, 2, 3, 4]; - let result = TpuForwardingRequest::try_from(value); - assert_eq!(result.unwrap_err().to_string(), "deserialize proxy request"); -} From 47629018de17030b9a92af5b80497defa33ea88f Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 15:41:29 +0200 Subject: [PATCH 102/128] rename request wire format --- .../src/inbound/proxy_listener.rs | 3 ++- .../src/proxy_request_format.rs | 21 ++++++++++++------- .../tests/proxy_request_format.rs | 12 +++++++++-- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index e458d7cd..1e0d2d29 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -133,7 +133,8 @@ impl ProxyListener { let raw_request = recv_stream.read_to_end(10_000_000).await.unwrap(); let proxy_request = - TpuForwardingRequest::deserialize_from_raw_request(&raw_request); + TpuForwardingRequest::try_deserialize_from_wire_format(&raw_request) + .unwrap(); trace!("proxy request details: {}", proxy_request); let _tpu_identity = proxy_request.get_identity_tpunode(); diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 9b8b5761..cd44d3aa 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -47,19 +47,24 @@ impl TpuForwardingRequest { } } - pub fn serialize_wire_format(&self) -> Vec { - bincode::serialize(&self).expect("Expect to serialize transactions") + pub fn try_serialize_wire_format(&self) -> anyhow::Result> { + bincode::serialize(&self) + .context("serialize proxy request") + .map_err(anyhow::Error::from) } - // TODO reame - pub fn deserialize_from_raw_request(raw_proxy_request: &[u8]) -> TpuForwardingRequest { - let request = bincode::deserialize::(raw_proxy_request) - .context("deserialize proxy request") - .unwrap(); + pub fn try_deserialize_from_wire_format( + raw_proxy_request: &[u8], + ) -> anyhow::Result { + let request = bincode::deserialize::(raw_proxy_request); - assert_eq!(request.format_version, FORMAT_VERSION1); + if let Ok(ref req) = request { + assert_eq!(req.format_version, FORMAT_VERSION1); + } request + .context("deserialize proxy request") + .map_err(anyhow::Error::from) } pub fn get_tpu_socket_addr(&self) -> SocketAddr { diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index bb752be8..d76b7ac3 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -20,12 +20,20 @@ fn roundtrip() { Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), vec![tx.into()], ) - .serialize_wire_format(); + .try_serialize_wire_format() + .unwrap(); println!("wire_data: {:02X?}", wire_data); - let request = TpuForwardingRequest::deserialize_from_raw_request(&wire_data); + let request = TpuForwardingRequest::try_deserialize_from_wire_format(&wire_data).unwrap(); assert!(request.get_tpu_socket_addr().is_ipv4()); assert_eq!(request.get_transactions().len(), 1); } + +#[test] +fn deserialize_error() { + let value: &[u8] = &[1, 2, 3, 4]; + let result = TpuForwardingRequest::try_deserialize_from_wire_format(value); + assert_eq!(result.unwrap_err().to_string(), "deserialize proxy request"); +} From 8a1b74b569b82532474355820faaeb292d051b10 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 15:57:09 +0200 Subject: [PATCH 103/128] wrap forward packet --- quic-forward-proxy/src/inbound/proxy_listener.rs | 5 +---- quic-forward-proxy/src/outbound/tx_forward.rs | 4 ++-- quic-forward-proxy/src/proxy_request_format.rs | 8 ++++---- quic-forward-proxy/src/shared/mod.rs | 16 ++++++++++++++-- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 1e0d2d29..db736bbe 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -156,10 +156,7 @@ impl ProxyListener { } forwarder_channel_copy .send_timeout( - ForwardPacket { - transactions: txs, - tpu_address, - }, + ForwardPacket::new(txs, tpu_address), FALLBACK_TIMEOUT, ) .await diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 1a13ceb0..b80ada9b 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -86,7 +86,7 @@ pub async fn tx_forwarder( if packet.tpu_address != tpu_address { continue; } - if !sharder.matching(packet.shard_hash()) { + if !sharder.matching(packet.shard_hash) { continue; } @@ -97,7 +97,7 @@ pub async fn tx_forwarder( if more.tpu_address != tpu_address { continue; } - if !sharder.matching(more.shard_hash()) { + if !sharder.matching(more.shard_hash) { continue; } transactions_batch.extend(more.transactions.clone()); diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index cd44d3aa..d4ee26c4 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -15,8 +15,8 @@ pub const FORMAT_VERSION1: u16 = 2302; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { format_version: u16, - tpu_socket_addr: SocketAddr, // TODO is that correct - identity_tpunode: Pubkey, // note: this is only used fro + tpu_socket_addr: SocketAddr, + identity_tpunode: Pubkey, // note: this is only used for debugging // TODO consider not deserializing transactions in proxy transactions: Vec, } @@ -75,7 +75,7 @@ impl TpuForwardingRequest { self.identity_tpunode } - pub fn get_transactions(&self) -> Vec { - self.transactions.clone() + pub fn get_transactions(&self) -> &[VersionedTransaction] { + self.transactions.as_slice() } } diff --git a/quic-forward-proxy/src/shared/mod.rs b/quic-forward-proxy/src/shared/mod.rs index 63557f81..281097b0 100644 --- a/quic-forward-proxy/src/shared/mod.rs +++ b/quic-forward-proxy/src/shared/mod.rs @@ -8,13 +8,25 @@ use std::net::SocketAddr; pub struct ForwardPacket { pub transactions: Vec, pub tpu_address: SocketAddr, + pub shard_hash: u64, } impl ForwardPacket { - pub fn shard_hash(&self) -> u64 { + + pub fn new(transactions: &[VersionedTransaction], tpu_address: SocketAddr) -> Self { + assert!(!transactions.is_empty(), "no transactions"); + let hash = Self::shard_hash(&transactions); + Self { + transactions: transactions.to_vec(), + tpu_address, + shard_hash: hash, + } + } + + fn shard_hash(transactions: &[VersionedTransaction]) -> u64 { let mut hasher = DefaultHasher::new(); // note: assumes that there are transactions with >=0 signatures - self.transactions[0].signatures[0].hash(&mut hasher); + transactions[0].signatures[0].hash(&mut hasher); hasher.finish() } } From 3667c6155277a8e156e954f814eb3f654456dc83 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 16:36:24 +0200 Subject: [PATCH 104/128] avoid de-serialization of wire format --- core/Cargo.toml | 3 +- core/src/proxy_request_format.rs | 24 +++++++------ .../src/inbound/proxy_listener.rs | 4 +-- quic-forward-proxy/src/outbound/tx_forward.rs | 15 +++----- .../src/proxy_request_format.rs | 36 ++++++++++++++----- .../src/quinn_auto_reconnect.rs | 2 +- quic-forward-proxy/src/shared/mod.rs | 18 ++-------- .../tests/proxy_request_format.rs | 2 +- 8 files changed, 56 insertions(+), 48 deletions(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index 12f5ee78..ea901a40 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -32,4 +32,5 @@ dashmap = { workspace = true } quinn = { workspace = true } chrono = { workspace = true } rustls = { workspace = true } -async-trait = { workspace = true } \ No newline at end of file +async-trait = { workspace = true } +itertools = { workspace = true } \ No newline at end of file diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index e45ba847..ecf00398 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -1,6 +1,8 @@ use anyhow::Context; +use itertools::Itertools; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::Signature; use solana_sdk::transaction::VersionedTransaction; use std::fmt; use std::fmt::Display; @@ -10,24 +12,26 @@ use std::net::SocketAddr; /// lite-rpc to proxy wire format /// compat info: non-public format ATM /// initial version -const FORMAT_VERSION1: u16 = 2302; +const FORMAT_VERSION1: u16 = 2400; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TxData(Signature, Vec); #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { format_version: u16, tpu_socket_addr: SocketAddr, - identity_tpunode: Pubkey, - transactions: Vec, + identity_tpunode: Pubkey, // note: this is only used for debugging + transactions: Vec, } impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "TpuForwardingRequest for tpu target {} with identity {}: payload {} tx", + "TpuForwardingRequest for tpu target {} with identity {}", &self.get_tpu_socket_addr(), &self.get_identity_tpunode(), - &self.get_transactions().len() ) } } @@ -42,10 +46,14 @@ impl TpuForwardingRequest { format_version: FORMAT_VERSION1, tpu_socket_addr, identity_tpunode, - transactions, + transactions: transactions.iter().map(Self::serialize).collect_vec(), } } + fn serialize(tx: &VersionedTransaction) -> TxData { + TxData(tx.signatures[0], bincode::serialize(&tx).unwrap()) + } + pub fn serialize_wire_format(&self) -> Vec { bincode::serialize(&self).expect("Expect to serialize transactions") } @@ -67,8 +75,4 @@ impl TpuForwardingRequest { pub fn get_identity_tpunode(&self) -> Pubkey { self.identity_tpunode } - - pub fn get_transactions(&self) -> &[VersionedTransaction] { - self.transactions.as_slice() - } } diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index db736bbe..948b436d 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -139,7 +139,7 @@ impl ProxyListener { trace!("proxy request details: {}", proxy_request); let _tpu_identity = proxy_request.get_identity_tpunode(); let tpu_address = proxy_request.get_tpu_socket_addr(); - let txs = proxy_request.get_transactions(); + let txs = proxy_request.get_transaction_bytes(); debug!( "enqueue transaction batch of size {} to address {}", @@ -156,7 +156,7 @@ impl ProxyListener { } forwarder_channel_copy .send_timeout( - ForwardPacket::new(txs, tpu_address), + ForwardPacket::new(txs, tpu_address, proxy_request.get_hash()), FALLBACK_TIMEOUT, ) .await diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index b80ada9b..c51b9c31 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -11,7 +11,6 @@ use quinn::{ ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt, }; use solana_sdk::quic::QUIC_MAX_TIMEOUT_MS; -use solana_sdk::transaction::VersionedTransaction; use solana_streamer::nonblocking::quic::ALPN_TPU_PROTOCOL_ID; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::collections::HashMap; @@ -90,8 +89,7 @@ pub async fn tx_forwarder( continue; } - let mut transactions_batch: Vec = - packet.transactions.clone(); + let mut transactions_batch: Vec> = packet.transactions.clone(); while let Ok(more) = per_connection_receiver.try_recv() { if more.tpu_address != tpu_address { @@ -221,14 +219,11 @@ fn create_tpu_client_endpoint( // send potentially large amount of transactions to a single TPU #[tracing::instrument(skip_all, level = "debug")] -async fn send_tx_batch_to_tpu(auto_connection: &AutoReconnect, txs: &[VersionedTransaction]) { +async fn send_tx_batch_to_tpu(auto_connection: &AutoReconnect, txs: &[Vec]) { for chunk in txs.chunks(MAX_PARALLEL_STREAMS) { - let all_send_fns = chunk - .iter() - .map(|tx| bincode::serialize(tx).unwrap()) - .map(|tx_raw| { - auto_connection.send_uni(tx_raw) // ignores error - }); + let all_send_fns = chunk.iter().map(|tx_raw| { + auto_connection.send_uni(tx_raw) // ignores error + }); join_all(all_send_fns).await; } diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index d4ee26c4..48807a6d 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -1,24 +1,30 @@ use anyhow::Context; +use itertools::Itertools; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::Signature; use solana_sdk::transaction::VersionedTransaction; +use std::collections::hash_map::DefaultHasher; use std::fmt; use std::fmt::Display; +use std::hash::{Hash, Hasher}; use std::net::SocketAddr; /// /// lite-rpc to proxy wire format /// compat info: non-public format ATM /// initial version -pub const FORMAT_VERSION1: u16 = 2302; +pub const FORMAT_VERSION1: u16 = 2400; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TxData(Signature, Vec); #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { format_version: u16, tpu_socket_addr: SocketAddr, - identity_tpunode: Pubkey, // note: this is only used for debugging - // TODO consider not deserializing transactions in proxy - transactions: Vec, + identity_tpunode: Pubkey, // note: this is only used for debugging + transactions: Vec, } impl Display for TpuForwardingRequest { @@ -28,7 +34,7 @@ impl Display for TpuForwardingRequest { "TpuForwardingRequest for tpu target {} with identity {}: payload {} tx", &self.get_tpu_socket_addr(), &self.get_identity_tpunode(), - &self.get_transactions().len() + &self.get_transaction_bytes().len() ) } } @@ -43,10 +49,14 @@ impl TpuForwardingRequest { format_version: FORMAT_VERSION1, tpu_socket_addr, identity_tpunode, - transactions, + transactions: transactions.into_iter().map(Self::serialize).collect_vec(), } } + fn serialize(tx: VersionedTransaction) -> TxData { + TxData(tx.signatures[0], bincode::serialize(&tx).unwrap()) + } + pub fn try_serialize_wire_format(&self) -> anyhow::Result> { bincode::serialize(&self) .context("serialize proxy request") @@ -75,7 +85,17 @@ impl TpuForwardingRequest { self.identity_tpunode } - pub fn get_transactions(&self) -> &[VersionedTransaction] { - self.transactions.as_slice() + pub fn get_transaction_bytes(&self) -> Vec> { + self.transactions + .iter() + .map(|tx| tx.1.clone()) + .collect_vec() + } + + pub fn get_hash(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + // note: assumes that there are transactions with >=0 signatures + self.transactions[0].0.hash(&mut hasher); + hasher.finish() } } diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 7194e400..8d3d2d0b 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -41,7 +41,7 @@ impl AutoReconnect { } } - pub async fn send_uni(&self, payload: Vec) -> anyhow::Result<()> { + pub async fn send_uni(&self, payload: &Vec) -> anyhow::Result<()> { let mut send_stream = timeout_fallback(self.refresh_and_get().await?.open_uni()) .await .context("open uni stream for sending")??; diff --git a/quic-forward-proxy/src/shared/mod.rs b/quic-forward-proxy/src/shared/mod.rs index 281097b0..4d3d40e9 100644 --- a/quic-forward-proxy/src/shared/mod.rs +++ b/quic-forward-proxy/src/shared/mod.rs @@ -1,32 +1,20 @@ -use solana_sdk::transaction::VersionedTransaction; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; use std::net::SocketAddr; /// internal structure with transactions and target TPU #[derive(Debug)] pub struct ForwardPacket { - pub transactions: Vec, + pub transactions: Vec>, pub tpu_address: SocketAddr, pub shard_hash: u64, } impl ForwardPacket { - - pub fn new(transactions: &[VersionedTransaction], tpu_address: SocketAddr) -> Self { + pub fn new(transactions: Vec>, tpu_address: SocketAddr, hash: u64) -> Self { assert!(!transactions.is_empty(), "no transactions"); - let hash = Self::shard_hash(&transactions); Self { - transactions: transactions.to_vec(), + transactions, tpu_address, shard_hash: hash, } } - - fn shard_hash(transactions: &[VersionedTransaction]) -> u64 { - let mut hasher = DefaultHasher::new(); - // note: assumes that there are transactions with >=0 signatures - transactions[0].signatures[0].hash(&mut hasher); - hasher.finish() - } } diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index d76b7ac3..768c4e8d 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -28,7 +28,7 @@ fn roundtrip() { let request = TpuForwardingRequest::try_deserialize_from_wire_format(&wire_data).unwrap(); assert!(request.get_tpu_socket_addr().is_ipv4()); - assert_eq!(request.get_transactions().len(), 1); + assert_eq!(request.get_transaction_bytes().len(), 1); } #[test] From d776ed9dca49c38ae9598f32c96cfc1420ca171b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 8 Aug 2023 16:38:44 +0200 Subject: [PATCH 105/128] auto-reconnect: more context in logs --- quic-forward-proxy/src/quinn_auto_reconnect.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 8d3d2d0b..d2f6b391 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -67,7 +67,7 @@ impl AutoReconnect { let lock = self.current.read().await; if let ConnectionState::Connection(conn) = &*lock { if conn.close_reason().is_none() { - debug!("Reuse connection {}", conn.stable_id()); + debug!("Reuse connection {} to {}", conn.stable_id(), self.target_address); return; } } @@ -78,8 +78,9 @@ impl AutoReconnect { if current.close_reason().is_some() { let old_stable_id = current.stable_id(); warn!( - "Connection {} is closed for reason: {:?}", + "Connection {} to {} is closed for reason: {:?}", old_stable_id, + self.target_address, current.close_reason() ); @@ -115,7 +116,7 @@ impl AutoReconnect { } }; } else { - debug!("Reuse connection {} with write-lock", current.stable_id()); + debug!("Reuse connection {} to {} with write-lock", current.stable_id(), self.target_address); } } ConnectionState::NotConnected => { @@ -141,7 +142,7 @@ impl AutoReconnect { } ConnectionState::PermanentError => { // no nothing - debug!("Not using connection with permanent error"); + debug!("Not using connection to {} with permanent error", self.target_address); } } } From 47c95d549e24182158846bb13b7b17baf9206df7 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 10:44:52 +0200 Subject: [PATCH 106/128] agent cleanup --- quic-forward-proxy/src/outbound/tx_forward.rs | 106 ++++++++++++++---- 1 file changed, 85 insertions(+), 21 deletions(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index c51b9c31..2a8a5c5a 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -15,14 +15,30 @@ use solana_streamer::nonblocking::quic::ALPN_TPU_PROTOCOL_ID; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; +use itertools::Itertools; use tokio::sync::mpsc::Receiver; const MAX_PARALLEL_STREAMS: usize = 6; pub const PARALLEL_TPU_CONNECTION_COUNT: usize = 4; +struct AgentHandle { + pub tpu_address: SocketAddr, + pub agent_exit_signal: Arc, + pub created_at: Instant, + // relative to start + pub last_used_ms: AtomicU64, +} + +impl AgentHandle { + pub fn touch(&self) { + let last_used_ms = Instant::now().duration_since(self.created_at).as_millis() as u64; + self.last_used_ms.store(last_used_ms, Ordering::Relaxed); + } +} + // takes transactions from upstream clients and forwards them to the TPU pub async fn tx_forwarder( validator_identity: ValidatorIdentity, @@ -35,7 +51,7 @@ pub async fn tx_forwarder( let (broadcast_in, _) = tokio::sync::broadcast::channel::>(1000); - let mut agents: HashMap>> = HashMap::new(); + let mut agents: HashMap = HashMap::new(); loop { if exit_signal.load(Ordering::Relaxed) { @@ -51,37 +67,48 @@ pub async fn tx_forwarder( let tpu_address = forward_packet.tpu_address; agents.entry(tpu_address).or_insert_with(|| { - let mut agent_exit_signals = Vec::new(); + let agent_exit_signal = Arc::new(AtomicBool::new(false)); + for connection_idx in 0..PARALLEL_TPU_CONNECTION_COUNT { let sharder = Sharder::new(connection_idx as u32, PARALLEL_TPU_CONNECTION_COUNT as u32); let global_exit_signal = exit_signal.clone(); - let agent_exit_signal = Arc::new(AtomicBool::new(false)); let endpoint_copy = endpoint.clone(); let agent_exit_signal_copy = agent_exit_signal.clone(); - // by subscribing we expect to get a copy of each packet - let mut per_connection_receiver = broadcast_in.subscribe(); tokio::spawn(async move { debug!( "Start Quic forwarder agent #{} for TPU {}", connection_idx, tpu_address ); - if global_exit_signal.load(Ordering::Relaxed) { - warn!("Caught global exit signal - stopping agent thread"); - return; - } - if agent_exit_signal_copy.load(Ordering::Relaxed) { - warn!("Caught exit signal for this agent - stopping agent thread"); - return; - } - // get a copy of the packet from broadcast channel let auto_connection = AutoReconnect::new(endpoint_copy, tpu_address); // TODO check exit signal (using select! or maybe replace with oneshot) let _exit_signal_copy = global_exit_signal.clone(); - while let Ok(packet) = per_connection_receiver.recv().await { + 'tx_channel_loop: loop { + let timeout_result = timeout_fallback(per_connection_receiver.recv()).await; + + if global_exit_signal.load(Ordering::Relaxed) { + warn!("Caught global exit signal - stopping agent thread"); + return; + } + if agent_exit_signal_copy.load(Ordering::Relaxed) { + warn!("Caught exit signal for this agent ({} #{}) - stopping agent thread", tpu_address, connection_idx); + return; + } + + if let Err(_elapsed) = timeout_result { + continue; + } + let maybe_packet = timeout_result.unwrap(); + + if let Err(_recv_error) = maybe_packet { + break 'tx_channel_loop; + } + + let packet = maybe_packet.unwrap(); + if packet.tpu_address != tpu_address { continue; } @@ -136,14 +163,22 @@ pub async fn tx_forwarder( connection_idx, tpu_address ); }); // -- spawned thread for one connection to one TPU - agent_exit_signals.push(agent_exit_signal); } // -- for parallel connections to one TPU - // FanOut::new(senders) - agent_exit_signals + let now = Instant::now(); + AgentHandle { + tpu_address, + agent_exit_signal, + created_at: now, + last_used_ms: AtomicU64::new(0), + } }); // -- new agent - let _agent_channel = agents.get(&tpu_address).unwrap(); + let agent = agents.get(&tpu_address).unwrap(); + agent.touch(); + + // TODO only call from time to time + cleanup_agents(&mut agents, &tpu_address); if broadcast_in.len() > 5 { debug!("tx-forward queue len: {}", broadcast_in.len()) @@ -158,6 +193,35 @@ pub async fn tx_forwarder( // not reachable } +// ms +const AGENT_SHUTDOWN_IDLE: u64 = 5_000; + +fn cleanup_agents(agents: &mut HashMap, current_tpu_address: &SocketAddr) { + let mut to_shutdown = Vec::new(); + for (tpu_address, handle) in &*agents { + if tpu_address == current_tpu_address { + continue; + } + + let last_used_ms = handle.last_used_ms.load(Ordering::Relaxed); + + if last_used_ms > AGENT_SHUTDOWN_IDLE { + to_shutdown.push(tpu_address.to_owned()) + } + } + + for tpu_address in to_shutdown.iter() { + if let Some(removed_agent) = agents.remove(&tpu_address) { + if let Ok(_) = removed_agent.agent_exit_signal.compare_exchange( + false, true, Ordering::Relaxed, Ordering::Relaxed) { + let last_used_ms = removed_agent.last_used_ms.load(Ordering::Relaxed); + debug!("Agent for tpu node {} was IDLE for {}ms - sending exit signal", removed_agent.tpu_address, last_used_ms); + } + } + } + +} + /// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU // note: ATM the provided identity might or might not be a valid validator keypair async fn new_endpoint_with_validator_identity(validator_identity: ValidatorIdentity) -> Endpoint { From 4ca4e035c20af5a3164526203ef52df593a64abd Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 13:29:11 +0200 Subject: [PATCH 107/128] add shutdown for agent --- quic-forward-proxy/src/outbound/debouncer.rs | 80 ++++++++ quic-forward-proxy/src/outbound/mod.rs | 1 + quic-forward-proxy/src/outbound/tx_forward.rs | 52 +++--- .../src/quinn_auto_reconnect.rs | 31 +++- .../quic_proxy_connection_manager.rs | 2 +- .../src/tpu_utils/quinn_auto_reconnect.rs | 173 +++++++++++++----- 6 files changed, 261 insertions(+), 78 deletions(-) create mode 100644 quic-forward-proxy/src/outbound/debouncer.rs diff --git a/quic-forward-proxy/src/outbound/debouncer.rs b/quic-forward-proxy/src/outbound/debouncer.rs new file mode 100644 index 00000000..58ec567b --- /dev/null +++ b/quic-forward-proxy/src/outbound/debouncer.rs @@ -0,0 +1,80 @@ +use log::trace; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +pub struct Debouncer { + cooldown_ms: i64, + last: AtomicI64, +} + +impl Debouncer { + pub fn new(cooldown: Duration) -> Self { + Self { + cooldown_ms: cooldown.as_millis() as i64, + last: AtomicI64::new(-1), + } + } + pub fn can_fire(&self) -> bool { + let now = SystemTime::now(); + let epoch_now = now.duration_since(UNIX_EPOCH).unwrap().as_millis() as i64; + + let results = self + .last + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |last| { + let elapsed = epoch_now - last; + if elapsed > self.cooldown_ms { + trace!("trigger it!"); + Some(epoch_now) + } else { + trace!("have to wait - not yet .. (elapsed {})", elapsed); + None + } + }); // -- compare+swap + + results.is_ok() + } +} + +#[cfg(test)] +mod tests { + use crate::outbound::debouncer::Debouncer; + use std::sync::Arc; + use std::thread; + use std::thread::sleep; + use std::time::Duration; + + #[test] + fn fire() { + let debouncer = Debouncer::new(Duration::from_millis(500)); + + assert!(debouncer.can_fire()); + assert!(!debouncer.can_fire()); + sleep(Duration::from_millis(200)); + assert!(!debouncer.can_fire()); + sleep(Duration::from_millis(400)); + assert!(debouncer.can_fire()); + } + + #[test] + fn threading() { + let debouncer = Debouncer::new(Duration::from_millis(500)); + + thread::spawn(move || { + debouncer.can_fire(); + }); + } + + #[test] + fn shared() { + let debouncer = Arc::new(Debouncer::new(Duration::from_millis(500))); + + let debouncer_copy = debouncer.clone(); + thread::spawn(move || { + debouncer_copy.can_fire(); + }); + + thread::spawn(move || { + debouncer.can_fire(); + }); + } +} diff --git a/quic-forward-proxy/src/outbound/mod.rs b/quic-forward-proxy/src/outbound/mod.rs index 968674c2..ea89826f 100644 --- a/quic-forward-proxy/src/outbound/mod.rs +++ b/quic-forward-proxy/src/outbound/mod.rs @@ -1,2 +1,3 @@ +mod debouncer; mod sharder; pub mod tx_forward; diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 2a8a5c5a..60e7d7dd 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -1,3 +1,4 @@ +use crate::outbound::debouncer::Debouncer; use crate::outbound::sharder::Sharder; use crate::quic_util::SkipServerVerification; use crate::quinn_auto_reconnect::AutoReconnect; @@ -18,24 +19,24 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; -use itertools::Itertools; use tokio::sync::mpsc::Receiver; const MAX_PARALLEL_STREAMS: usize = 6; pub const PARALLEL_TPU_CONNECTION_COUNT: usize = 4; +const AGENT_SHUTDOWN_IDLE: u64 = 2500; // ms; should be 4x400ms+buffer struct AgentHandle { pub tpu_address: SocketAddr, pub agent_exit_signal: Arc, pub created_at: Instant, // relative to start - pub last_used_ms: AtomicU64, + pub age_ms: AtomicU64, } impl AgentHandle { pub fn touch(&self) { - let last_used_ms = Instant::now().duration_since(self.created_at).as_millis() as u64; - self.last_used_ms.store(last_used_ms, Ordering::Relaxed); + let age_ms = Instant::now().duration_since(self.created_at).as_millis() as u64; + self.age_ms.store(age_ms, Ordering::Relaxed); } } @@ -52,6 +53,7 @@ pub async fn tx_forwarder( let (broadcast_in, _) = tokio::sync::broadcast::channel::>(1000); let mut agents: HashMap = HashMap::new(); + let agent_shutdown_debouncer = Debouncer::new(Duration::from_millis(200)); loop { if exit_signal.load(Ordering::Relaxed) { @@ -99,7 +101,7 @@ pub async fn tx_forwarder( } if let Err(_elapsed) = timeout_result { - continue; + continue 'tx_channel_loop; } let maybe_packet = timeout_result.unwrap(); @@ -110,20 +112,20 @@ pub async fn tx_forwarder( let packet = maybe_packet.unwrap(); if packet.tpu_address != tpu_address { - continue; + continue 'tx_channel_loop; } if !sharder.matching(packet.shard_hash) { - continue; + continue 'tx_channel_loop; } let mut transactions_batch: Vec> = packet.transactions.clone(); - while let Ok(more) = per_connection_receiver.try_recv() { + 'more: while let Ok(more) = per_connection_receiver.try_recv() { if more.tpu_address != tpu_address { - continue; + continue 'more; } if !sharder.matching(more.shard_hash) { - continue; + continue 'more; } transactions_batch.extend(more.transactions.clone()); } @@ -170,21 +172,21 @@ pub async fn tx_forwarder( tpu_address, agent_exit_signal, created_at: now, - last_used_ms: AtomicU64::new(0), + age_ms: AtomicU64::new(0), } }); // -- new agent let agent = agents.get(&tpu_address).unwrap(); agent.touch(); - // TODO only call from time to time - cleanup_agents(&mut agents, &tpu_address); + if agent_shutdown_debouncer.can_fire() { + cleanup_agents(&mut agents, &tpu_address); + } if broadcast_in.len() > 5 { debug!("tx-forward queue len: {}", broadcast_in.len()) } - // TODO use agent_exit signal to clean them up broadcast_in .send(forward_packet) .expect("send must succeed"); @@ -193,9 +195,6 @@ pub async fn tx_forwarder( // not reachable } -// ms -const AGENT_SHUTDOWN_IDLE: u64 = 5_000; - fn cleanup_agents(agents: &mut HashMap, current_tpu_address: &SocketAddr) { let mut to_shutdown = Vec::new(); for (tpu_address, handle) in &*agents { @@ -203,7 +202,7 @@ fn cleanup_agents(agents: &mut HashMap, current_tpu_add continue; } - let last_used_ms = handle.last_used_ms.load(Ordering::Relaxed); + let last_used_ms = handle.age_ms.load(Ordering::Relaxed); if last_used_ms > AGENT_SHUTDOWN_IDLE { to_shutdown.push(tpu_address.to_owned()) @@ -211,15 +210,20 @@ fn cleanup_agents(agents: &mut HashMap, current_tpu_add } for tpu_address in to_shutdown.iter() { - if let Some(removed_agent) = agents.remove(&tpu_address) { - if let Ok(_) = removed_agent.agent_exit_signal.compare_exchange( - false, true, Ordering::Relaxed, Ordering::Relaxed) { - let last_used_ms = removed_agent.last_used_ms.load(Ordering::Relaxed); - debug!("Agent for tpu node {} was IDLE for {}ms - sending exit signal", removed_agent.tpu_address, last_used_ms); + if let Some(removed_agent) = agents.remove(tpu_address) { + let was_signaled = removed_agent + .agent_exit_signal + .compare_exchange(false, true, Ordering::SeqCst, Ordering::Relaxed) + .is_ok(); + if was_signaled { + let last_used_ms = removed_agent.age_ms.load(Ordering::Relaxed); + debug!( + "Idle Agent for tpu node {} idle for {}ms - sending exit signal", + removed_agent.tpu_address, last_used_ms + ); } } } - } /// takes a validator identity and creates a new QUIC client; appears as staked peer to TPU diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index d2f6b391..98f717e9 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -1,11 +1,12 @@ -use crate::util::timeout_fallback; use anyhow::{bail, Context}; use log::{info, warn}; use quinn::{Connection, ConnectionError, Endpoint}; use std::fmt; use std::net::SocketAddr; use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::Duration; use tokio::sync::RwLock; +use tokio::time::timeout; use tracing::debug; /// connection manager with automatic reconnect; designated for connection to Solana TPU nodes @@ -17,6 +18,8 @@ use tracing::debug; /// * TPU address might be wrong which then is a permanent problem /// * the ActiveConnection instance gets renewed on leader schedule change +const SEND_TIMEOUT: Duration = Duration::from_secs(5); + enum ConnectionState { NotConnected, Connection(Connection), @@ -42,7 +45,7 @@ impl AutoReconnect { } pub async fn send_uni(&self, payload: &Vec) -> anyhow::Result<()> { - let mut send_stream = timeout_fallback(self.refresh_and_get().await?.open_uni()) + let mut send_stream = timeout(SEND_TIMEOUT, self.refresh_and_get().await?.open_uni()) .await .context("open uni stream for sending")??; send_stream.write_all(payload.as_slice()).await?; @@ -67,7 +70,11 @@ impl AutoReconnect { let lock = self.current.read().await; if let ConnectionState::Connection(conn) = &*lock { if conn.close_reason().is_none() { - debug!("Reuse connection {} to {}", conn.stable_id(), self.target_address); + debug!( + "Reuse connection {} to {}", + conn.stable_id(), + self.target_address + ); return; } } @@ -92,16 +99,17 @@ impl AutoReconnect { if reconnect_count < 10 { info!( - "Replace closed connection {} with {} (retry {})", + "Replace closed connection {} with {} to target {} (retry {})", old_stable_id, new_connection.stable_id(), + self.target_address, reconnect_count ); } else { *lock = ConnectionState::PermanentError; warn!( - "Too many reconnect attempts {} with {} (retry {})", - old_stable_id, + "Too many reconnect attempts to {}, last one with {} (retry {})", + self.target_address, new_connection.stable_id(), reconnect_count ); @@ -116,7 +124,11 @@ impl AutoReconnect { } }; } else { - debug!("Reuse connection {} to {} with write-lock", current.stable_id(), self.target_address); + debug!( + "Reuse connection {} to {} with write-lock", + current.stable_id(), + self.target_address + ); } } ConnectionState::NotConnected => { @@ -142,7 +154,10 @@ impl AutoReconnect { } ConnectionState::PermanentError => { // no nothing - debug!("Not using connection to {} with permanent error", self.target_address); + debug!( + "Not using connection to {} with permanent error", + self.target_address + ); } } } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 89cd3054..0bf0d5e1 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -230,7 +230,7 @@ impl QuicProxyConnectionManager { let proxy_request_raw = bincode::serialize(&forwarding_request).expect("Expect to serialize transactions"); - let send_result = auto_connection.send_uni(proxy_request_raw).await; + let send_result = auto_connection.send_uni(&proxy_request_raw).await; match send_result { Ok(()) => { diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index e8dd13b0..77c226c4 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -1,6 +1,6 @@ -use anyhow::Context; -use log::warn; -use quinn::{Connection, Endpoint}; +use anyhow::{bail, Context}; +use log::{info, warn}; +use quinn::{Connection, ConnectionError, Endpoint}; use std::fmt; use std::net::SocketAddr; use std::sync::atomic::{AtomicU32, Ordering}; @@ -9,11 +9,22 @@ use tokio::sync::RwLock; use tokio::time::timeout; use tracing::debug; + /// copy of quic-proxy AutoReconnect - used that for reference + + +const SEND_TIMEOUT: Duration = Duration::from_secs(5); + +enum ConnectionState { + NotConnected, + Connection(Connection), + PermanentError, +} + pub struct AutoReconnect { // endoint should be configures with keep-alive and idle timeout endpoint: Endpoint, - current: RwLock>, + current: RwLock, pub target_address: SocketAddr, reconnect_count: AtomicU32, } @@ -22,15 +33,14 @@ impl AutoReconnect { pub fn new(endpoint: Endpoint, target_address: SocketAddr) -> Self { Self { endpoint, - current: RwLock::new(None), + current: RwLock::new(ConnectionState::NotConnected), target_address, reconnect_count: AtomicU32::new(0), } } - pub async fn send_uni(&self, payload: Vec) -> anyhow::Result<()> { - // TOOD do smart error handling + reconnect - let mut send_stream = timeout(Duration::from_secs(4), self.refresh().await.open_uni()) + pub async fn send_uni(&self, payload: &Vec) -> anyhow::Result<()> { + let mut send_stream = timeout(SEND_TIMEOUT, self.refresh_and_get().await?.open_uni()) .await .context("open uni stream for sending")??; send_stream.write_all(payload.as_slice()).await?; @@ -38,69 +48,142 @@ impl AutoReconnect { Ok(()) } - pub async fn refresh(&self) -> Connection { + pub async fn refresh_and_get(&self) -> anyhow::Result { + self.refresh().await; + + let lock = self.current.read().await; + match &*lock { + ConnectionState::NotConnected => bail!("not connected"), + ConnectionState::Connection(conn) => Ok(conn.clone()), + ConnectionState::PermanentError => bail!("permanent error"), + } + } + + pub async fn refresh(&self) { { + // first check for existing connection using a cheap read-lock let lock = self.current.read().await; - let maybe_conn = lock.as_ref(); - if maybe_conn - .filter(|conn| conn.close_reason().is_none()) - .is_some() - { - let reuse = maybe_conn.unwrap(); - debug!("Reuse connection {}", reuse.stable_id()); - return reuse.clone(); + if let ConnectionState::Connection(conn) = &*lock { + if conn.close_reason().is_none() { + debug!("Reuse connection {} to {}", conn.stable_id(), self.target_address); + return; + } } } let mut lock = self.current.write().await; - let maybe_conn = lock.as_ref(); - match maybe_conn { - Some(current) => { + match &*lock { + ConnectionState::Connection(current) => { if current.close_reason().is_some() { let old_stable_id = current.stable_id(); warn!( - "Connection {} is closed for reason: {:?}", + "Connection {} to {} is closed for reason: {:?}", old_stable_id, + self.target_address, current.close_reason() ); - let new_connection = self.create_connection().await; - *lock = Some(new_connection.clone()); - // let old_conn = lock.replace(new_connection.clone()); - self.reconnect_count.fetch_add(1, Ordering::SeqCst); + match self.create_connection().await { + Some(new_connection) => { + *lock = ConnectionState::Connection(new_connection.clone()); + let reconnect_count = + self.reconnect_count.fetch_add(1, Ordering::SeqCst); - debug!( - "Replace closed connection {} with {} (retry {})", - old_stable_id, - new_connection.stable_id(), - self.reconnect_count.load(Ordering::SeqCst) - ); - - new_connection + if reconnect_count < 10 { + info!( + "Replace closed connection {} with {} to target {} (retry {})", + old_stable_id, + new_connection.stable_id(), + self.target_address, + reconnect_count + ); + } else { + *lock = ConnectionState::PermanentError; + warn!( + "Too many reconnect attempts to {}, last one with {} (retry {})", + self.target_address, + new_connection.stable_id(), + reconnect_count + ); + } + } + None => { + warn!( + "Reconnect to {} failed for connection {}", + self.target_address, old_stable_id + ); + *lock = ConnectionState::PermanentError; + } + }; } else { - debug!("Reuse connection {} with write-lock", current.stable_id()); - current.clone() + debug!("Reuse connection {} to {} with write-lock", current.stable_id(), self.target_address); } } - None => { - let new_connection = self.create_connection().await; + ConnectionState::NotConnected => { + match self.create_connection().await { + Some(new_connection) => { + *lock = ConnectionState::Connection(new_connection.clone()); + self.reconnect_count.fetch_add(1, Ordering::SeqCst); - assert!(lock.is_none(), "old connection must be None"); - *lock = Some(new_connection.clone()); - // let old_conn = foo.replace(Some(new_connection.clone())); - debug!("Create initial connection {}", new_connection.stable_id()); - - new_connection + info!( + "Create initial connection {} to {}", + new_connection.stable_id(), + self.target_address + ); + } + None => { + warn!( + "Initial connection to {} failed permanently", + self.target_address + ); + *lock = ConnectionState::PermanentError; + } + }; + } + ConnectionState::PermanentError => { + // no nothing + debug!("Not using connection to {} with permanent error", self.target_address); } } } - async fn create_connection(&self) -> Connection { + async fn create_connection(&self) -> Option { let connection = self .endpoint .connect(self.target_address, "localhost") .expect("handshake"); - connection.await.expect("connection") + match connection.await { + Ok(conn) => Some(conn), + Err(ConnectionError::TimedOut) => None, + // maybe we should also treat TransportError explicitly + Err(unexpected_error) => { + panic!( + "Connection to {} failed with unexpected error: {}", + self.target_address, unexpected_error + ); + } + } + } + + // stable_id 140266619216912, rtt=2.156683ms, + // stats FrameStats { ACK: 3, CONNECTION_CLOSE: 0, CRYPTO: 3, + // DATA_BLOCKED: 0, DATAGRAM: 0, HANDSHAKE_DONE: 1, MAX_DATA: 0, + // MAX_STREAM_DATA: 1, MAX_STREAMS_BIDI: 0, MAX_STREAMS_UNI: 0, NEW_CONNECTION_ID: 4, + // NEW_TOKEN: 0, PATH_CHALLENGE: 0, PATH_RESPONSE: 0, PING: 0, RESET_STREAM: 0, + // RETIRE_CONNECTION_ID: 1, STREAM_DATA_BLOCKED: 0, STREAMS_BLOCKED_BIDI: 0, + // STREAMS_BLOCKED_UNI: 0, STOP_SENDING: 0, STREAM: 0 } + pub async fn connection_stats(&self) -> String { + let lock = self.current.read().await; + match &*lock { + ConnectionState::Connection(conn) => format!( + "stable_id {} stats {:?}, rtt={:?}", + conn.stable_id(), + conn.stats().frame_rx, + conn.stats().path.rtt + ), + ConnectionState::NotConnected => "n/c".to_string(), + ConnectionState::PermanentError => "n/a (permanent)".to_string(), + } } } From 5f0d6ae15c8d68f632d7d023da0253752904398b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 13:37:49 +0200 Subject: [PATCH 108/128] fmt --- .../src/tpu_utils/quinn_auto_reconnect.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 77c226c4..fb6d8ed8 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -9,10 +9,8 @@ use tokio::sync::RwLock; use tokio::time::timeout; use tracing::debug; - /// copy of quic-proxy AutoReconnect - used that for reference - const SEND_TIMEOUT: Duration = Duration::from_secs(5); enum ConnectionState { @@ -65,7 +63,11 @@ impl AutoReconnect { let lock = self.current.read().await; if let ConnectionState::Connection(conn) = &*lock { if conn.close_reason().is_none() { - debug!("Reuse connection {} to {}", conn.stable_id(), self.target_address); + debug!( + "Reuse connection {} to {}", + conn.stable_id(), + self.target_address + ); return; } } @@ -115,7 +117,11 @@ impl AutoReconnect { } }; } else { - debug!("Reuse connection {} to {} with write-lock", current.stable_id(), self.target_address); + debug!( + "Reuse connection {} to {} with write-lock", + current.stable_id(), + self.target_address + ); } } ConnectionState::NotConnected => { @@ -141,7 +147,10 @@ impl AutoReconnect { } ConnectionState::PermanentError => { // no nothing - debug!("Not using connection to {} with permanent error", self.target_address); + debug!( + "Not using connection to {} with permanent error", + self.target_address + ); } } } From 9a6542c7b75fd6686d59faf7bb4dbecdd53bb526 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 16:23:14 +0200 Subject: [PATCH 109/128] tpu auto connection with infinite retries --- .../src/quinn_auto_reconnect.rs | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 98f717e9..05c484bd 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -24,6 +24,7 @@ enum ConnectionState { NotConnected, Connection(Connection), PermanentError, + FailedAttempt(u32), } pub struct AutoReconnect { @@ -31,19 +32,23 @@ pub struct AutoReconnect { endpoint: Endpoint, current: RwLock, pub target_address: SocketAddr, - reconnect_count: AtomicU32, } + impl AutoReconnect { pub fn new(endpoint: Endpoint, target_address: SocketAddr) -> Self { Self { endpoint, current: RwLock::new(ConnectionState::NotConnected), target_address, - reconnect_count: AtomicU32::new(0), } } + pub async fn is_permanent_dead(&self) -> bool { + let lock = self.current.read().await; + matches!(&*lock, ConnectionState::PermanentError) + } + pub async fn send_uni(&self, payload: &Vec) -> anyhow::Result<()> { let mut send_stream = timeout(SEND_TIMEOUT, self.refresh_and_get().await?.open_uni()) .await @@ -61,6 +66,7 @@ impl AutoReconnect { ConnectionState::NotConnected => bail!("not connected"), ConnectionState::Connection(conn) => Ok(conn.clone()), ConnectionState::PermanentError => bail!("permanent error"), + ConnectionState::FailedAttempt(_) => bail!("failed connection attempt"), } } @@ -85,7 +91,7 @@ impl AutoReconnect { if current.close_reason().is_some() { let old_stable_id = current.stable_id(); warn!( - "Connection {} to {} is closed for reason: {:?}", + "Connection {} to {} is closed for reason: {:?} - reconnecting", old_stable_id, self.target_address, current.close_reason() @@ -94,33 +100,19 @@ impl AutoReconnect { match self.create_connection().await { Some(new_connection) => { *lock = ConnectionState::Connection(new_connection.clone()); - let reconnect_count = - self.reconnect_count.fetch_add(1, Ordering::SeqCst); - - if reconnect_count < 10 { - info!( - "Replace closed connection {} with {} to target {} (retry {})", + info!( + "Restored closed connection {} with {} to target {}", old_stable_id, new_connection.stable_id(), self.target_address, - reconnect_count ); - } else { - *lock = ConnectionState::PermanentError; - warn!( - "Too many reconnect attempts to {}, last one with {} (retry {})", - self.target_address, - new_connection.stable_id(), - reconnect_count - ); - } } None => { warn!( "Reconnect to {} failed for connection {}", self.target_address, old_stable_id ); - *lock = ConnectionState::PermanentError; + *lock = ConnectionState::FailedAttempt(1); } }; } else { @@ -135,7 +127,6 @@ impl AutoReconnect { match self.create_connection().await { Some(new_connection) => { *lock = ConnectionState::Connection(new_connection.clone()); - self.reconnect_count.fetch_add(1, Ordering::SeqCst); info!( "Create initial connection {} to {}", @@ -144,11 +135,9 @@ impl AutoReconnect { ); } None => { - warn!( - "Initial connection to {} failed permanently", - self.target_address - ); - *lock = ConnectionState::PermanentError; + warn!("Failed connect initially to target {}", + self.target_address); + *lock = ConnectionState::FailedAttempt(1); } }; } @@ -159,6 +148,18 @@ impl AutoReconnect { self.target_address ); } + ConnectionState::FailedAttempt(attempts) => { + match self.create_connection().await { + Some(new_connection) => { + *lock = ConnectionState::Connection(new_connection.clone()); + } + None => { + warn!("Reconnect to {} failed", + self.target_address); + *lock = ConnectionState::FailedAttempt(attempts + 1); + } + }; + } } } @@ -199,6 +200,7 @@ impl AutoReconnect { ), ConnectionState::NotConnected => "n/c".to_string(), ConnectionState::PermanentError => "n/a (permanent)".to_string(), + ConnectionState::FailedAttempt(_) => "fail".to_string(), } } } From 677801f5be0525f1520c6e5114a3b3f86c558a44 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 17:00:17 +0200 Subject: [PATCH 110/128] pack list of target tpu nodes in the wire format --- core/src/proxy_request_format.rs | 57 +++++++--------- .../src/inbound/proxy_listener.rs | 30 +++++---- quic-forward-proxy/src/outbound/tx_forward.rs | 16 +++-- quic-forward-proxy/src/proxy.rs | 2 +- .../src/proxy_request_format.rs | 47 ++++++------- .../src/quinn_auto_reconnect.rs | 34 ++++++---- .../tests/proxy_request_format.rs | 9 ++- .../quic_proxy_connection_manager.rs | 24 +++---- .../src/tpu_utils/quinn_auto_reconnect.rs | 66 +++++++++++-------- 9 files changed, 161 insertions(+), 124 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index ecf00398..0ccdb9ea 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -12,16 +12,21 @@ use std::net::SocketAddr; /// lite-rpc to proxy wire format /// compat info: non-public format ATM /// initial version -const FORMAT_VERSION1: u16 = 2400; +const FORMAT_VERSION1: u16 = 2500; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TxData(Signature, Vec); +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TpuNode { + pub tpu_socket_addr: SocketAddr, + pub identity_tpunode: Pubkey, // note: this is only used for debugging +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { format_version: u16, - tpu_socket_addr: SocketAddr, - identity_tpunode: Pubkey, // note: this is only used for debugging + tpu_nodes: Vec, transactions: Vec, } @@ -29,50 +34,38 @@ impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "TpuForwardingRequest for tpu target {} with identity {}", - &self.get_tpu_socket_addr(), - &self.get_identity_tpunode(), + "TpuForwardingRequest t9 {} tpu nodes", + &self.tpu_nodes.len(), ) } } impl TpuForwardingRequest { pub fn new( - tpu_socket_addr: SocketAddr, - identity_tpunode: Pubkey, + tpu_fanout_nodes: &Vec<(SocketAddr, Pubkey)>, transactions: Vec, ) -> Self { TpuForwardingRequest { format_version: FORMAT_VERSION1, - tpu_socket_addr, - identity_tpunode, - transactions: transactions.iter().map(Self::serialize).collect_vec(), + tpu_nodes: tpu_fanout_nodes.iter().map(|(tpu_addr, identity)| TpuNode { + tpu_socket_addr: *tpu_addr, + identity_tpunode: *identity, + }).collect_vec(), + transactions: transactions.iter() + .map(|tx| TxData(tx.signatures[0], + bincode::serialize(tx).unwrap())) + .collect_vec(), } } - fn serialize(tx: &VersionedTransaction) -> TxData { - TxData(tx.signatures[0], bincode::serialize(&tx).unwrap()) - } - - pub fn serialize_wire_format(&self) -> Vec { - bincode::serialize(&self).expect("Expect to serialize transactions") + pub fn try_serialize_wire_format(&self) -> anyhow::Result> { + bincode::serialize(&self) + .context("serialize proxy request") + .map_err(anyhow::Error::from) } - pub fn deserialize_from_raw_request(raw_proxy_request: &[u8]) -> TpuForwardingRequest { - let request = bincode::deserialize::(raw_proxy_request) - .context("deserialize proxy request") - .unwrap(); - - assert_eq!(request.format_version, 2301); - - request + pub fn get_tpu_nodes(&self) -> &Vec { + &self.tpu_nodes } - pub fn get_tpu_socket_addr(&self) -> SocketAddr { - self.tpu_socket_addr - } - - pub fn get_identity_tpunode(&self) -> Pubkey { - self.identity_tpunode - } } diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 948b436d..7b19226e 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -137,14 +137,12 @@ impl ProxyListener { .unwrap(); trace!("proxy request details: {}", proxy_request); - let _tpu_identity = proxy_request.get_identity_tpunode(); - let tpu_address = proxy_request.get_tpu_socket_addr(); let txs = proxy_request.get_transaction_bytes(); debug!( - "enqueue transaction batch of size {} to address {}", + "enqueue transaction batch of size {} to {} tpu nodes", txs.len(), - tpu_address + proxy_request.get_tpu_nodes().len(), ); if forwarder_channel_copy.capacity() < forwarder_channel_copy.max_capacity() { @@ -154,14 +152,22 @@ impl ProxyListener { forwarder_channel_copy.max_capacity() ); } - forwarder_channel_copy - .send_timeout( - ForwardPacket::new(txs, tpu_address, proxy_request.get_hash()), - FALLBACK_TIMEOUT, - ) - .await - .context("sending internal packet from proxy to forwarder") - .unwrap(); + + for tpu_node in proxy_request.get_tpu_nodes() { + let tpu_address = tpu_node.tpu_socket_addr; + forwarder_channel_copy + .send_timeout( + ForwardPacket::new( + txs.clone(), + tpu_address, + proxy_request.get_hash(), + ), + FALLBACK_TIMEOUT, + ) + .await + .context("sending internal packet from proxy to forwarder") + .unwrap(); + } }); debug!( diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 60e7d7dd..8980df6b 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -90,6 +90,9 @@ pub async fn tx_forwarder( let _exit_signal_copy = global_exit_signal.clone(); 'tx_channel_loop: loop { let timeout_result = timeout_fallback(per_connection_receiver.recv()).await; + if let Err(_elapsed) = timeout_result { + continue 'tx_channel_loop; + } if global_exit_signal.load(Ordering::Relaxed) { warn!("Caught global exit signal - stopping agent thread"); @@ -100,9 +103,6 @@ pub async fn tx_forwarder( return; } - if let Err(_elapsed) = timeout_result { - continue 'tx_channel_loop; - } let maybe_packet = timeout_result.unwrap(); if let Err(_recv_error) = maybe_packet { @@ -118,6 +118,14 @@ pub async fn tx_forwarder( continue 'tx_channel_loop; } + if auto_connection.is_permanent_dead().await { + warn!("Connection is considered permanently dead"); + // while let Ok(more) = per_connection_receiver.try_recv() { + // // drain + // } + // continue 'tx_channel_loop; + } + let mut transactions_batch: Vec> = packet.transactions.clone(); 'more: while let Ok(more) = per_connection_receiver.try_recv() { @@ -276,7 +284,7 @@ fn create_tpu_client_endpoint( transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); let timeout = IdleTimeout::try_from(Duration::from_millis(QUIC_MAX_TIMEOUT_MS as u64)).unwrap(); transport_config.max_idle_timeout(Some(timeout)); - transport_config.keep_alive_interval(None); + transport_config.keep_alive_interval(Some(Duration::from_millis(500))); config.transport_config(Arc::new(transport_config)); diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index ea790ffc..298958c3 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -36,7 +36,7 @@ impl QuicForwardProxy { pub async fn start_services(self) -> anyhow::Result<()> { let exit_signal = Arc::new(AtomicBool::new(false)); - let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(100_000); + let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); let proxy_listener = proxy_listener::ProxyListener::new(self.proxy_listener_addr, self.tls_config); diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 48807a6d..2e11c89d 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -14,16 +14,21 @@ use std::net::SocketAddr; /// lite-rpc to proxy wire format /// compat info: non-public format ATM /// initial version -pub const FORMAT_VERSION1: u16 = 2400; +pub const FORMAT_VERSION1: u16 = 2500; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TxData(Signature, Vec); +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TpuNode { + pub tpu_socket_addr: SocketAddr, + pub identity_tpunode: Pubkey, // note: this is only used for debugging +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { format_version: u16, - tpu_socket_addr: SocketAddr, - identity_tpunode: Pubkey, // note: this is only used for debugging + tpu_nodes: Vec, transactions: Vec, } @@ -31,32 +36,34 @@ impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "TpuForwardingRequest for tpu target {} with identity {}: payload {} tx", - &self.get_tpu_socket_addr(), - &self.get_identity_tpunode(), - &self.get_transaction_bytes().len() + "TpuForwardingRequest t9 {} tpu nodes", + &self.tpu_nodes.len(), ) } } impl TpuForwardingRequest { pub fn new( - tpu_socket_addr: SocketAddr, - identity_tpunode: Pubkey, + tpu_fanout_nodes: Vec<(SocketAddr, Pubkey)>, transactions: Vec, ) -> Self { TpuForwardingRequest { format_version: FORMAT_VERSION1, - tpu_socket_addr, - identity_tpunode, - transactions: transactions.into_iter().map(Self::serialize).collect_vec(), + tpu_nodes: tpu_fanout_nodes + .iter() + .map(|(tpu_addr, identity)| TpuNode { + tpu_socket_addr: *tpu_addr, + identity_tpunode: *identity, + }) + .collect_vec(), + transactions: transactions + .iter() + .map(|tx| TxData(tx.signatures[0], bincode::serialize(tx).unwrap())) + .collect_vec(), } } - fn serialize(tx: VersionedTransaction) -> TxData { - TxData(tx.signatures[0], bincode::serialize(&tx).unwrap()) - } - + // test only pub fn try_serialize_wire_format(&self) -> anyhow::Result> { bincode::serialize(&self) .context("serialize proxy request") @@ -77,12 +84,8 @@ impl TpuForwardingRequest { .map_err(anyhow::Error::from) } - pub fn get_tpu_socket_addr(&self) -> SocketAddr { - self.tpu_socket_addr - } - - pub fn get_identity_tpunode(&self) -> Pubkey { - self.identity_tpunode + pub fn get_tpu_nodes(&self) -> &Vec { + &self.tpu_nodes } pub fn get_transaction_bytes(&self) -> Vec> { diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 05c484bd..9bd3f11a 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -3,7 +3,6 @@ use log::{info, warn}; use quinn::{Connection, ConnectionError, Endpoint}; use std::fmt; use std::net::SocketAddr; -use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; use tokio::sync::RwLock; use tokio::time::timeout; @@ -19,6 +18,7 @@ use tracing::debug; /// * the ActiveConnection instance gets renewed on leader schedule change const SEND_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_RETRY_ATTEMPTS: u32 = 10; enum ConnectionState { NotConnected, @@ -34,7 +34,6 @@ pub struct AutoReconnect { pub target_address: SocketAddr, } - impl AutoReconnect { pub fn new(endpoint: Endpoint, target_address: SocketAddr) -> Self { Self { @@ -101,11 +100,11 @@ impl AutoReconnect { Some(new_connection) => { *lock = ConnectionState::Connection(new_connection.clone()); info!( - "Restored closed connection {} with {} to target {}", - old_stable_id, - new_connection.stable_id(), - self.target_address, - ); + "Restored closed connection {} with {} to target {}", + old_stable_id, + new_connection.stable_id(), + self.target_address, + ); } None => { warn!( @@ -135,8 +134,7 @@ impl AutoReconnect { ); } None => { - warn!("Failed connect initially to target {}", - self.target_address); + warn!("Failed connect initially to target {}", self.target_address); *lock = ConnectionState::FailedAttempt(1); } }; @@ -151,12 +149,22 @@ impl AutoReconnect { ConnectionState::FailedAttempt(attempts) => { match self.create_connection().await { Some(new_connection) => { - *lock = ConnectionState::Connection(new_connection.clone()); + *lock = ConnectionState::Connection(new_connection); } None => { - warn!("Reconnect to {} failed", - self.target_address); - *lock = ConnectionState::FailedAttempt(attempts + 1); + if *attempts < MAX_RETRY_ATTEMPTS { + warn!( + "Reconnect to {} failed (attempt {})", + self.target_address, attempts + ); + *lock = ConnectionState::FailedAttempt(attempts + 1); + } else { + warn!( + "Reconnect to {} failed permanently (attempt {})", + self.target_address, attempts + ); + *lock = ConnectionState::PermanentError; + } } }; } diff --git a/quic-forward-proxy/tests/proxy_request_format.rs b/quic-forward-proxy/tests/proxy_request_format.rs index 768c4e8d..6bc4628d 100644 --- a/quic-forward-proxy/tests/proxy_request_format.rs +++ b/quic-forward-proxy/tests/proxy_request_format.rs @@ -16,8 +16,10 @@ fn roundtrip() { let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); let wire_data = TpuForwardingRequest::new( - "127.0.0.1:5454".parse().unwrap(), - Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), + vec![( + "127.0.0.1:5454".parse().unwrap(), + Pubkey::from_str("Bm8rtweCQ19ksNebrLY92H7x4bCaeDJSSmEeWqkdCeop").unwrap(), + )], vec![tx.into()], ) .try_serialize_wire_format() @@ -27,7 +29,8 @@ fn roundtrip() { let request = TpuForwardingRequest::try_deserialize_from_wire_format(&wire_data).unwrap(); - assert!(request.get_tpu_socket_addr().is_ipv4()); + assert_eq!(request.get_tpu_nodes().len(), 1); + assert!(request.get_tpu_nodes()[0].tpu_socket_addr.is_ipv4()); assert_eq!(request.get_transaction_bytes().len(), 1); } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 0bf0d5e1..42258d46 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -8,7 +8,7 @@ use anyhow::bail; use std::time::Duration; use itertools::Itertools; -use log::{debug, error, info, trace}; +use log::{debug, error, info, trace, warn}; use quinn::{ ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt, }; @@ -132,7 +132,6 @@ impl QuicProxyConnectionManager { // note: this config must be aligned with quic-proxy's server config let mut transport_config = TransportConfig::default(); - let _timeout = IdleTimeout::try_from(Duration::from_secs(1)).unwrap(); // no remotely-initiated streams required transport_config.max_concurrent_uni_streams(VarInt::from_u32(0)); transport_config.max_concurrent_bidi_streams(VarInt::from_u32(0)); @@ -163,7 +162,6 @@ impl QuicProxyConnectionManager { } tokio::select! { - // TODO add timeout tx = transaction_receiver.recv() => { let first_tx: Vec = match tx { @@ -189,26 +187,26 @@ impl QuicProxyConnectionManager { trace!("Sending copy of transaction batch of {} txs to {} tpu nodes via quic proxy", txs.len(), tpu_fanout_nodes.len()); - for target_tpu_node in tpu_fanout_nodes { + let send_result = Self::send_copy_of_txs_to_quicproxy( &txs, &auto_connection, proxy_addr, - target_tpu_node.tpu_address, - target_tpu_node.tpu_identity) - .await.unwrap(); + tpu_fanout_nodes) + .await; + if let Err(err) = send_result { + warn!("Failed to send copy of txs to quic proxy - skip (error {})", err); } }, }; - } + } // -- loop } async fn send_copy_of_txs_to_quicproxy( raw_tx_batch: &[Vec], auto_connection: &AutoReconnect, _proxy_address: SocketAddr, - tpu_target_address: SocketAddr, - target_tpu_identity: Pubkey, + tpu_fanout_nodes: Vec, ) -> anyhow::Result<()> { let mut txs = vec![]; @@ -222,9 +220,13 @@ impl QuicProxyConnectionManager { txs.push(tx); } + let tpu_data = tpu_fanout_nodes.iter() + .map(|tpu| (tpu.tpu_address, tpu.tpu_identity)) + .collect_vec(); + for chunk in txs.chunks(CHUNK_SIZE_PER_STREAM) { let forwarding_request = - TpuForwardingRequest::new(tpu_target_address, target_tpu_identity, chunk.into()); + TpuForwardingRequest::new(&tpu_data, chunk.into()); debug!("forwarding_request: {}", forwarding_request); let proxy_request_raw = diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index fb6d8ed8..5d68592e 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -11,12 +11,15 @@ use tracing::debug; /// copy of quic-proxy AutoReconnect - used that for reference + const SEND_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_RETRY_ATTEMPTS: u32 = 10; enum ConnectionState { NotConnected, Connection(Connection), PermanentError, + FailedAttempt(u32), } pub struct AutoReconnect { @@ -24,19 +27,23 @@ pub struct AutoReconnect { endpoint: Endpoint, current: RwLock, pub target_address: SocketAddr, - reconnect_count: AtomicU32, } + impl AutoReconnect { pub fn new(endpoint: Endpoint, target_address: SocketAddr) -> Self { Self { endpoint, current: RwLock::new(ConnectionState::NotConnected), target_address, - reconnect_count: AtomicU32::new(0), } } + pub async fn is_permanent_dead(&self) -> bool { + let lock = self.current.read().await; + matches!(&*lock, ConnectionState::PermanentError) + } + pub async fn send_uni(&self, payload: &Vec) -> anyhow::Result<()> { let mut send_stream = timeout(SEND_TIMEOUT, self.refresh_and_get().await?.open_uni()) .await @@ -54,6 +61,7 @@ impl AutoReconnect { ConnectionState::NotConnected => bail!("not connected"), ConnectionState::Connection(conn) => Ok(conn.clone()), ConnectionState::PermanentError => bail!("permanent error"), + ConnectionState::FailedAttempt(_) => bail!("failed connection attempt"), } } @@ -78,7 +86,7 @@ impl AutoReconnect { if current.close_reason().is_some() { let old_stable_id = current.stable_id(); warn!( - "Connection {} to {} is closed for reason: {:?}", + "Connection {} to {} is closed for reason: {:?} - reconnecting", old_stable_id, self.target_address, current.close_reason() @@ -87,33 +95,19 @@ impl AutoReconnect { match self.create_connection().await { Some(new_connection) => { *lock = ConnectionState::Connection(new_connection.clone()); - let reconnect_count = - self.reconnect_count.fetch_add(1, Ordering::SeqCst); - - if reconnect_count < 10 { - info!( - "Replace closed connection {} with {} to target {} (retry {})", + info!( + "Restored closed connection {} with {} to target {}", old_stable_id, new_connection.stable_id(), self.target_address, - reconnect_count ); - } else { - *lock = ConnectionState::PermanentError; - warn!( - "Too many reconnect attempts to {}, last one with {} (retry {})", - self.target_address, - new_connection.stable_id(), - reconnect_count - ); - } } None => { warn!( "Reconnect to {} failed for connection {}", self.target_address, old_stable_id ); - *lock = ConnectionState::PermanentError; + *lock = ConnectionState::FailedAttempt(1); } }; } else { @@ -128,7 +122,6 @@ impl AutoReconnect { match self.create_connection().await { Some(new_connection) => { *lock = ConnectionState::Connection(new_connection.clone()); - self.reconnect_count.fetch_add(1, Ordering::SeqCst); info!( "Create initial connection {} to {}", @@ -137,11 +130,9 @@ impl AutoReconnect { ); } None => { - warn!( - "Initial connection to {} failed permanently", - self.target_address - ); - *lock = ConnectionState::PermanentError; + warn!("Failed connect initially to target {}", + self.target_address); + *lock = ConnectionState::FailedAttempt(1); } }; } @@ -152,6 +143,28 @@ impl AutoReconnect { self.target_address ); } + ConnectionState::FailedAttempt(attempts) => { + match self.create_connection().await { + Some(new_connection) => { + *lock = ConnectionState::Connection(new_connection); + } + None => { + if *attempts < MAX_RETRY_ATTEMPTS { + warn!( + "Reconnect to {} failed (attempt {})", + self.target_address, attempts + ); + *lock = ConnectionState::FailedAttempt(attempts + 1); + } else { + warn!( + "Reconnect to {} failed permanently (attempt {})", + self.target_address, attempts + ); + *lock = ConnectionState::PermanentError; + } + } + }; + } } } @@ -192,6 +205,7 @@ impl AutoReconnect { ), ConnectionState::NotConnected => "n/c".to_string(), ConnectionState::PermanentError => "n/a (permanent)".to_string(), + ConnectionState::FailedAttempt(_) => "fail".to_string(), } } } From 1e055fec2c559768b728267da521651c3d17083d Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 17:10:08 +0200 Subject: [PATCH 111/128] clippy+fmt --- core/src/proxy_request_format.rs | 20 ++++++++++--------- .../quic_proxy_connection_manager.rs | 10 ++++------ .../src/tpu_utils/quinn_auto_reconnect.rs | 16 ++++++--------- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index 0ccdb9ea..592b8b1c 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -42,18 +42,21 @@ impl Display for TpuForwardingRequest { impl TpuForwardingRequest { pub fn new( - tpu_fanout_nodes: &Vec<(SocketAddr, Pubkey)>, + tpu_fanout_nodes: &[(SocketAddr, Pubkey)], transactions: Vec, ) -> Self { TpuForwardingRequest { format_version: FORMAT_VERSION1, - tpu_nodes: tpu_fanout_nodes.iter().map(|(tpu_addr, identity)| TpuNode { - tpu_socket_addr: *tpu_addr, - identity_tpunode: *identity, - }).collect_vec(), - transactions: transactions.iter() - .map(|tx| TxData(tx.signatures[0], - bincode::serialize(tx).unwrap())) + tpu_nodes: tpu_fanout_nodes + .iter() + .map(|(tpu_addr, identity)| TpuNode { + tpu_socket_addr: *tpu_addr, + identity_tpunode: *identity, + }) + .collect_vec(), + transactions: transactions + .iter() + .map(|tx| TxData(tx.signatures[0], bincode::serialize(tx).unwrap())) .collect_vec(), } } @@ -67,5 +70,4 @@ impl TpuForwardingRequest { pub fn get_tpu_nodes(&self) -> &Vec { &self.tpu_nodes } - } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 42258d46..391fd457 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -9,9 +9,7 @@ use std::time::Duration; use itertools::Itertools; use log::{debug, error, info, trace, warn}; -use quinn::{ - ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt, -}; +use quinn::{ClientConfig, Endpoint, EndpointConfig, TokioRuntime, TransportConfig, VarInt}; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::VersionedTransaction; @@ -220,13 +218,13 @@ impl QuicProxyConnectionManager { txs.push(tx); } - let tpu_data = tpu_fanout_nodes.iter() + let tpu_data = tpu_fanout_nodes + .iter() .map(|tpu| (tpu.tpu_address, tpu.tpu_identity)) .collect_vec(); for chunk in txs.chunks(CHUNK_SIZE_PER_STREAM) { - let forwarding_request = - TpuForwardingRequest::new(&tpu_data, chunk.into()); + let forwarding_request = TpuForwardingRequest::new(&tpu_data, chunk.into()); debug!("forwarding_request: {}", forwarding_request); let proxy_request_raw = diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 5d68592e..32f05367 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -3,7 +3,6 @@ use log::{info, warn}; use quinn::{Connection, ConnectionError, Endpoint}; use std::fmt; use std::net::SocketAddr; -use std::sync::atomic::{AtomicU32, Ordering}; use std::time::Duration; use tokio::sync::RwLock; use tokio::time::timeout; @@ -11,7 +10,6 @@ use tracing::debug; /// copy of quic-proxy AutoReconnect - used that for reference - const SEND_TIMEOUT: Duration = Duration::from_secs(5); const MAX_RETRY_ATTEMPTS: u32 = 10; @@ -29,7 +27,6 @@ pub struct AutoReconnect { pub target_address: SocketAddr, } - impl AutoReconnect { pub fn new(endpoint: Endpoint, target_address: SocketAddr) -> Self { Self { @@ -96,11 +93,11 @@ impl AutoReconnect { Some(new_connection) => { *lock = ConnectionState::Connection(new_connection.clone()); info!( - "Restored closed connection {} with {} to target {}", - old_stable_id, - new_connection.stable_id(), - self.target_address, - ); + "Restored closed connection {} with {} to target {}", + old_stable_id, + new_connection.stable_id(), + self.target_address, + ); } None => { warn!( @@ -130,8 +127,7 @@ impl AutoReconnect { ); } None => { - warn!("Failed connect initially to target {}", - self.target_address); + warn!("Failed connect initially to target {}", self.target_address); *lock = ConnectionState::FailedAttempt(1); } }; From a27cc06711353bcdedc29df592d14bf5d934c3e2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 17:15:46 +0200 Subject: [PATCH 112/128] fix broken log output --- quic-forward-proxy/src/outbound/tx_forward.rs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 8980df6b..c58761b6 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -154,17 +154,17 @@ pub async fn tx_forwarder( auto_connection.target_address )); - if result.is_err() { - warn!( - "got send_txs_to_tpu_static error {:?} - loop over errors", - result - ); - } else { - debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); - debug!( - "Outbound connection stats: {}", - &auto_connection.connection_stats().await - ); + match result { + Ok(()) => { + debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); + debug!( + "Outbound connection stats: {}", + &auto_connection.connection_stats().await + ); + } + Err(err) => { + warn!("got send_txs_to_tpu_static error {} - loop over errors", err); + } } } // -- while all packtes from channel From 414216e5852edb2e2a4f1815351160a7994b6ea2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 17:29:42 +0200 Subject: [PATCH 113/128] handle 0 tpu case --- quic-forward-proxy/src/inbound/proxy_listener.rs | 5 +++++ services/src/tpu_utils/quic_proxy_connection_manager.rs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 7b19226e..a9a01c26 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -136,6 +136,11 @@ impl ProxyListener { TpuForwardingRequest::try_deserialize_from_wire_format(&raw_request) .unwrap(); + if proxy_request.get_tpu_nodes().is_empty() { + warn!("no tpu nodes in request - skip"); + return; + } + trace!("proxy request details: {}", proxy_request); let txs = proxy_request.get_transaction_bytes(); diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 391fd457..bf610215 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -182,6 +182,11 @@ impl QuicProxyConnectionManager { let tpu_fanout_nodes = current_tpu_nodes.read().await.clone(); + if tpu_fanout_nodes.is_empty() { + warn!("No tpu nodes to send transactions to - skip"); + continue; + } + trace!("Sending copy of transaction batch of {} txs to {} tpu nodes via quic proxy", txs.len(), tpu_fanout_nodes.len()); From c7a79ab6c01f9a2dcbd2c65696543bdbc06f914d Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 17:30:12 +0200 Subject: [PATCH 114/128] fmt --- quic-forward-proxy/src/inbound/proxy_listener.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index a9a01c26..b034c5f3 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -5,7 +5,7 @@ use crate::tls_config_provider_server::ProxyTlsConfigProvider; use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use crate::util::FALLBACK_TIMEOUT; use anyhow::{anyhow, bail, Context}; -use log::{debug, error, info, trace}; +use log::{debug, error, info, trace, warn}; use quinn::{Connection, Endpoint, ServerConfig, VarInt}; use solana_sdk::packet::PACKET_DATA_SIZE; use std::net::SocketAddr; From f4deb51c497420e297c6a9d0d040e84a0241d5ed Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 17:39:55 +0200 Subject: [PATCH 115/128] adjust broadcast channel capacity to pow2 --- quic-forward-proxy/src/outbound/tx_forward.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index c58761b6..22d5fc44 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -50,7 +50,7 @@ pub async fn tx_forwarder( let endpoint = new_endpoint_with_validator_identity(validator_identity).await; - let (broadcast_in, _) = tokio::sync::broadcast::channel::>(1000); + let (broadcast_in, _) = tokio::sync::broadcast::channel::>(1024); let mut agents: HashMap = HashMap::new(); let agent_shutdown_debouncer = Debouncer::new(Duration::from_millis(200)); From b080c23ffefa3cbeb18fb877a43ea9f124bfcf75 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Wed, 9 Aug 2023 17:43:49 +0200 Subject: [PATCH 116/128] inrease test timeouts --- .../tests/quic_proxy_tpu_integrationtest.rs | 4 ++-- services/tests/literpc_tpu_quic_server_integrationtest.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index d1d0c853..f7e9d873 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -47,7 +47,7 @@ struct TestCaseParams { proxy_mode: bool, } -const MAXIMUM_TRANSACTIONS_IN_QUEUE: usize = 200_000; +const MAXIMUM_TRANSACTIONS_IN_QUEUE: usize = 16_384; const MAX_QUIC_CONNECTIONS_PER_PEER: usize = 8; // like solana repo const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameters { @@ -258,7 +258,7 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { let warmup_tx_count: u32 = test_case_params.sample_tx_count / 2; while (count_map.len() as u32) < test_case_params.sample_tx_count { - if latest_tx.elapsed() > Duration::from_secs(25) { + if latest_tx.elapsed() > Duration::from_secs(50) { warn!("abort after timeout waiting for packet from quic streamer"); break; } diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index b133dec2..e0d4a54a 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -164,7 +164,7 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { CountMap::with_capacity(test_case_params.sample_tx_count as usize); let warmup_tx_count: u32 = test_case_params.sample_tx_count / 2; while (count_map.len() as u32) < test_case_params.sample_tx_count { - if latest_tx.elapsed() > Duration::from_secs(5) { + if latest_tx.elapsed() > Duration::from_secs(50) { warn!("abort after timeout waiting for packet from quic streamer"); break; } From e0f3538923ad79fad20b226fb817b65ec6af1c88 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 10 Aug 2023 10:31:31 +0200 Subject: [PATCH 117/128] move broadcast receiver --- .../tests/quic_proxy_tpu_integrationtest.rs | 14 +++++---- .../src/inbound/proxy_listener.rs | 2 +- .../src/proxy_request_format.rs | 1 + .../quic_proxy_connection_manager.rs | 30 ++++++++++++------- .../src/tpu_utils/tpu_connection_manager.rs | 6 ++-- services/src/tpu_utils/tpu_service.rs | 3 +- 6 files changed, 34 insertions(+), 22 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index f7e9d873..f3d91f42 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -503,6 +503,8 @@ async fn start_literpc_client_direct_mode( } sleep(Duration::from_secs(30)).await; + assert!(broadcast_sender.is_empty(), "broadcast channel must be empty"); + Ok(()) } @@ -580,17 +582,15 @@ async fn start_literpc_client_proxy_mode( // ) // .await; + let transaction_receiver = broadcast_sender.subscribe(); quic_proxy_connection_manager .update_connection( - broadcast_sender.clone(), + transaction_receiver, connections_to_keep, QUIC_CONNECTION_PARAMS, ) .await; - // TODO this is a race - sleep(Duration::from_millis(1500)).await; - for i in 0..test_case_params.sample_tx_count { let raw_sample_tx = build_raw_sample_tx(i); trace!( @@ -607,6 +607,8 @@ async fn start_literpc_client_proxy_mode( } sleep(Duration::from_secs(30)).await; + // TODO copy to direct_mode method + assert!(broadcast_sender.is_empty(), "broadcast channel must be empty"); Ok(()) } @@ -625,8 +627,8 @@ async fn start_quic_proxy(proxy_listen_addr: SocketAddr) -> anyhow::Result<()> { .start_services(); tokio::select! { - _ = proxy_service => { - panic!("Proxy service stopped unexpectedly") + res = proxy_service => { + panic!("Proxy service stopped unexpectedly: {:?}", res) }, } } diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index b034c5f3..670369cb 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -65,7 +65,7 @@ impl ProxyListener { } Err(err) => { error!( - "failed to accect connection from client: {reason} - skip", + "failed to accept connection from client: {reason} - skip", reason = err ); } diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 2e11c89d..15b7af3d 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -28,6 +28,7 @@ pub struct TpuNode { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuForwardingRequest { format_version: u16, + // note: this data gets stale tpu_nodes: Vec, transactions: Vec, } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index bf610215..d4f5f3a9 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -14,6 +14,7 @@ use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction::VersionedTransaction; use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; +use tokio::sync::broadcast::error::TryRecvError; use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; use solana_lite_rpc_core::quic_connection_utils::{ @@ -56,7 +57,7 @@ impl QuicProxyConnectionManager { pub async fn update_connection( &self, - transaction_sender: Arc)>>, + broadcast_receiver: Receiver<(String, Vec)>, // for duration of this slot these tpu nodes will receive the transactions connections_to_keep: HashMap, connection_parameters: QuicConnectionParameters, @@ -87,12 +88,10 @@ impl QuicProxyConnectionManager { info!("Starting very simple proxy thread"); - let transaction_receiver = transaction_sender.subscribe(); - let exit_signal = Arc::new(AtomicBool::new(false)); tokio::spawn(Self::read_transactions_and_broadcast( - transaction_receiver, + broadcast_receiver, self.current_tpu_nodes.clone(), self.proxy_addr, self.endpoint.clone(), @@ -167,17 +166,26 @@ impl QuicProxyConnectionManager { tx }, Err(e) => { - error!( - "Broadcast channel error (close) on recv: {} - aborting", e); + warn!("Broadcast channel error (close) on recv: {} - aborting", e); return; } }; let mut txs = vec![first_tx]; for _ in 1..connection_parameters.number_of_transactions_per_unistream { - if let Ok((_signature, tx)) = transaction_receiver.try_recv() { - txs.push(tx); - } + match transaction_receiver.try_recv() { + Ok((_sig, tx)) => { + txs.push(tx); + }, + Err(TryRecvError::Empty) => { + break; + } + Err(e) => { + warn!( + "Broadcast channel error (close) on more recv: {} - aborting", e); + return; + } + }; } let tpu_fanout_nodes = current_tpu_nodes.read().await.clone(); @@ -196,8 +204,8 @@ impl QuicProxyConnectionManager { proxy_addr, tpu_fanout_nodes) .await; - if let Err(err) = send_result { - warn!("Failed to send copy of txs to quic proxy - skip (error {})", err); + if let Err(e) = send_result { + warn!("Failed to send copy of txs to quic proxy - skip (error {})", e); } }, diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 2182895a..0a5990de 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -224,7 +224,7 @@ impl TpuConnectionManager { pub async fn update_connections( &self, - transaction_sender: Arc)>>, + broadcast_sender: Arc)>>, connections_to_keep: HashMap, identity_stakes: IdentityStakes, txs_sent_store: TxStore, @@ -244,8 +244,8 @@ impl TpuConnectionManager { // using mpsc as a oneshot channel/ because with one shot channel we cannot reuse the reciever let (sx, rx) = tokio::sync::mpsc::channel(1); - let transaction_reciever = transaction_sender.subscribe(); - active_connection.start_listening(transaction_reciever, rx, identity_stakes); + let broadcast_receiver = broadcast_sender.subscribe(); + active_connection.start_listening(broadcast_receiver, rx, identity_stakes); self.identity_to_active_connection.insert( *identity, Arc::new(ActiveConnectionWithExitChannel { diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 984c3101..59ea6dd0 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -202,9 +202,10 @@ impl TpuService { QuicProxy { quic_proxy_connection_manager, } => { + let transaction_receiver = self.broadcast_sender.subscribe(); quic_proxy_connection_manager .update_connection( - self.broadcast_sender.clone(), + transaction_receiver, connections_to_keep, self.config.quic_connection_params, ) From e08969357f1767fc972ebabb6b6159aea4110b98 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 10 Aug 2023 10:43:03 +0200 Subject: [PATCH 118/128] remove useless serialization to VersionedTransaction --- core/src/proxy_request_format.rs | 18 +++++----- .../tests/quic_proxy_tpu_integrationtest.rs | 11 ++++-- .../quic_proxy_connection_manager.rs | 35 ++++++------------- 3 files changed, 28 insertions(+), 36 deletions(-) diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index 592b8b1c..f1f32c4d 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -3,10 +3,10 @@ use itertools::Itertools; use serde::{Deserialize, Serialize}; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::Signature; -use solana_sdk::transaction::VersionedTransaction; use std::fmt; use std::fmt::Display; use std::net::SocketAddr; +use std::str::FromStr; /// /// lite-rpc to proxy wire format @@ -17,6 +17,12 @@ const FORMAT_VERSION1: u16 = 2500; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TxData(Signature, Vec); +impl TxData { + pub fn new(sig: String, tx_raw: Vec) -> Self { + TxData(Signature::from_str(sig.as_str()).unwrap(), tx_raw) + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TpuNode { pub tpu_socket_addr: SocketAddr, @@ -41,10 +47,7 @@ impl Display for TpuForwardingRequest { } impl TpuForwardingRequest { - pub fn new( - tpu_fanout_nodes: &[(SocketAddr, Pubkey)], - transactions: Vec, - ) -> Self { + pub fn new(tpu_fanout_nodes: &[(SocketAddr, Pubkey)], transactions: &[TxData]) -> Self { TpuForwardingRequest { format_version: FORMAT_VERSION1, tpu_nodes: tpu_fanout_nodes @@ -54,10 +57,7 @@ impl TpuForwardingRequest { identity_tpunode: *identity, }) .collect_vec(), - transactions: transactions - .iter() - .map(|tx| TxData(tx.signatures[0], bincode::serialize(tx).unwrap())) - .collect_vec(), + transactions: transactions.to_vec(), } } diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index f3d91f42..f3c9d182 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -503,8 +503,10 @@ async fn start_literpc_client_direct_mode( } sleep(Duration::from_secs(30)).await; - assert!(broadcast_sender.is_empty(), "broadcast channel must be empty"); - + assert!( + broadcast_sender.is_empty(), + "broadcast channel must be empty" + ); Ok(()) } @@ -608,7 +610,10 @@ async fn start_literpc_client_proxy_mode( sleep(Duration::from_secs(30)).await; // TODO copy to direct_mode method - assert!(broadcast_sender.is_empty(), "broadcast channel must be empty"); + assert!( + broadcast_sender.is_empty(), + "broadcast channel must be empty" + ); Ok(()) } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index d4f5f3a9..1f7f48d2 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -8,15 +8,14 @@ use anyhow::bail; use std::time::Duration; use itertools::Itertools; -use log::{debug, error, info, trace, warn}; +use log::{debug, info, trace, warn}; use quinn::{ClientConfig, Endpoint, EndpointConfig, TokioRuntime, TransportConfig, VarInt}; use solana_sdk::pubkey::Pubkey; -use solana_sdk::transaction::VersionedTransaction; -use tokio::sync::{broadcast::Receiver, broadcast::Sender, RwLock}; use tokio::sync::broadcast::error::TryRecvError; +use tokio::sync::{broadcast::Receiver, RwLock}; -use solana_lite_rpc_core::proxy_request_format::TpuForwardingRequest; +use solana_lite_rpc_core::proxy_request_format::{TpuForwardingRequest, TxData}; use solana_lite_rpc_core::quic_connection_utils::{ QuicConnectionParameters, SkipServerVerification, }; @@ -161,9 +160,9 @@ impl QuicProxyConnectionManager { tokio::select! { tx = transaction_receiver.recv() => { - let first_tx: Vec = match tx { - Ok((_sig, tx)) => { - tx + let first_tx: TxData = match tx { + Ok((sig, tx_raw)) => { + TxData::new(sig, tx_raw) }, Err(e) => { warn!("Broadcast channel error (close) on recv: {} - aborting", e); @@ -171,11 +170,11 @@ impl QuicProxyConnectionManager { } }; - let mut txs = vec![first_tx]; + let mut txs: Vec = vec![first_tx]; for _ in 1..connection_parameters.number_of_transactions_per_unistream { match transaction_receiver.try_recv() { - Ok((_sig, tx)) => { - txs.push(tx); + Ok((sig, tx_raw)) => { + txs.push(TxData::new(sig, tx_raw)); }, Err(TryRecvError::Empty) => { break; @@ -214,30 +213,18 @@ impl QuicProxyConnectionManager { } async fn send_copy_of_txs_to_quicproxy( - raw_tx_batch: &[Vec], + txs: &[TxData], auto_connection: &AutoReconnect, _proxy_address: SocketAddr, tpu_fanout_nodes: Vec, ) -> anyhow::Result<()> { - let mut txs = vec![]; - - for raw_tx in raw_tx_batch { - let tx = match bincode::deserialize::(raw_tx) { - Ok(tx) => tx, - Err(err) => { - bail!(err.to_string()); - } - }; - txs.push(tx); - } - let tpu_data = tpu_fanout_nodes .iter() .map(|tpu| (tpu.tpu_address, tpu.tpu_identity)) .collect_vec(); for chunk in txs.chunks(CHUNK_SIZE_PER_STREAM) { - let forwarding_request = TpuForwardingRequest::new(&tpu_data, chunk.into()); + let forwarding_request = TpuForwardingRequest::new(&tpu_data, chunk); debug!("forwarding_request: {}", forwarding_request); let proxy_request_raw = From 5a070370870111e7274801b22b3a43b9c2149c7b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 10 Aug 2023 10:44:24 +0200 Subject: [PATCH 119/128] cleanups --- .../tests/quic_proxy_tpu_integrationtest.rs | 2 +- services/src/tpu_utils/quic_proxy_connection_manager.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index f3c9d182..f45d4f39 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -183,7 +183,7 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { .expect("failed to build tokio runtime for lite-rpc-tpu-client"); // quic-forward-proxy - let runtime_quic_proxy = tokio::runtime::Builder::new_multi_thread() + let runtime_quic_proxy = Builder::new_multi_thread() .enable_all() .build() .expect("failed to build tokio runtime for quic-forward-proxy"); diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 1f7f48d2..19076c75 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -111,7 +111,7 @@ impl QuicProxyConnectionManager { .expect("create_endpoint bind_in_range") .1; let config = EndpointConfig::default(); - quinn::Endpoint::new(config, None, client_socket, TokioRuntime) + Endpoint::new(config, None, client_socket, TokioRuntime) .expect("create_endpoint quinn::Endpoint::new") }; @@ -153,7 +153,7 @@ impl QuicProxyConnectionManager { loop { // exit signal set - if exit_signal.load(Ordering::Relaxed) { + if exit_signal.load(Relaxed) { break; } @@ -208,7 +208,7 @@ impl QuicProxyConnectionManager { } }, - }; + } } // -- loop } From e246aa9d38fa7ab6fe612c4dcac50c3e712dfaa2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 10 Aug 2023 13:04:30 +0200 Subject: [PATCH 120/128] check for missing tx in test --- .../tests/quic_proxy_tpu_integrationtest.rs | 36 ++++++++++++++++++- .../quic_proxy_connection_manager.rs | 2 +- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index f45d4f39..3aff421d 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -20,9 +20,10 @@ use solana_streamer::packet::PacketBatch; use solana_streamer::quic::StreamStats; use solana_streamer::streamer::StakedNodes; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; +use itertools::Itertools; use std::str::FromStr; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; @@ -255,6 +256,8 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { let mut packet_count2 = 0; let mut count_map: CountMap = CountMap::with_capacity(test_case_params.sample_tx_count as usize); + let mut contents: HashSet = + HashSet::with_capacity(test_case_params.sample_tx_count as usize); let warmup_tx_count: u32 = test_case_params.sample_tx_count / 2; while (count_map.len() as u32) < test_case_params.sample_tx_count { @@ -296,6 +299,10 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { "read transaction from quic streaming server: {:?}", tx.get_signature() ); + // "hi 7292 " + let content = String::from_utf8(tx.message.instructions()[0].data.clone()).unwrap(); + contents.insert(content); + count_map.insert_or_increment(*tx.get_signature()); } @@ -320,6 +327,26 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { info!("got all expected packets - shutting down tokio runtime with lite-rpc client"); + // inspect content and find missing + let all_numbers: HashSet = contents + .iter() + .map(|content| parse_hi(content).unwrap()) + .sorted() + .collect(); + let max_number = *all_numbers.iter().max().unwrap(); + // OKEY: got 100000 distinct content strings with max 99999 + info!( + "got {} distinct content strings with max {}", + contents.len(), + max_number + ); + let missing = (0..=max_number) + .filter(|n| !all_numbers.contains(n)) + .collect_vec(); + for mi in missing { + info!("- missing {}", mi); + } + assert_eq!( count_map.len() as u32, test_case_params.sample_tx_count, @@ -701,6 +728,13 @@ pub fn build_raw_sample_tx(i: u32) -> (String, Vec) { (tx.get_signature().to_string(), raw_tx) } +// "hi 1234 " -> 1234 +fn parse_hi(input: &str) -> Option { + let input = input.trim(); + let input = input.replace("hi ", ""); + input.parse::().ok() +} + fn build_sample_tx(payer_keypair: &Keypair, i: u32) -> VersionedTransaction { let blockhash = Hash::default(); create_memo_tx(format!("hi {}", i).as_bytes(), payer_keypair, blockhash).into() diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 19076c75..f5e2523f 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::Relaxed; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use anyhow::bail; From 7df7f271a5ca66cbd6d5f5e07ee31fccc04033b7 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 10 Aug 2023 13:50:47 +0200 Subject: [PATCH 121/128] properly drain channel on agent shutdown --- .../tests/quic_proxy_tpu_integrationtest.rs | 9 +++++++-- quic-forward-proxy/src/outbound/tx_forward.rs | 18 ++++++++++++++---- .../tpu_utils/quic_proxy_connection_manager.rs | 11 +++++++++-- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 3aff421d..e97f8b82 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -635,13 +635,18 @@ async fn start_literpc_client_proxy_mode( } } - sleep(Duration::from_secs(30)).await; - // TODO copy to direct_mode method + while !broadcast_sender.is_empty() { + sleep(Duration::from_millis(1000)).await; + warn!("broadcast channel is not empty - wait before shutdown test client thread"); + } + assert!( broadcast_sender.is_empty(), "broadcast channel must be empty" ); + quic_proxy_connection_manager.signal_shutdown(); + Ok(()) } diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 22d5fc44..680f3c0d 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -95,12 +95,22 @@ pub async fn tx_forwarder( } if global_exit_signal.load(Ordering::Relaxed) { - warn!("Caught global exit signal - stopping agent thread"); - return; + warn!("Caught global exit signal, {} remaining - stopping agent thread", + per_connection_receiver.len()); + break 'tx_channel_loop; } + // gracefully drain unit queue is empty - then shut down if agent_exit_signal_copy.load(Ordering::Relaxed) { - warn!("Caught exit signal for this agent ({} #{}) - stopping agent thread", tpu_address, connection_idx); - return; + if per_connection_receiver.is_empty() { + warn!("Caught exit signal for this agent ({} #{}) - stopping agent thread", + tpu_address, connection_idx); + break 'tx_channel_loop; + } else { + warn!("Caught exit signal for this agent ({} #{}), {} remaining - continue", + tpu_address, connection_idx, + per_connection_receiver.len()); + } + } let maybe_packet = timeout_result.unwrap(); diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index f5e2523f..e2923097 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -33,6 +33,7 @@ pub struct QuicProxyConnectionManager { simple_thread_started: AtomicBool, proxy_addr: SocketAddr, current_tpu_nodes: Arc>>, + exit_signal: Arc, } const CHUNK_SIZE_PER_STREAM: usize = 20; @@ -51,9 +52,14 @@ impl QuicProxyConnectionManager { simple_thread_started: AtomicBool::from(false), proxy_addr, current_tpu_nodes: Arc::new(RwLock::new(vec![])), + exit_signal: Arc::new(AtomicBool::from(false)), } } + pub fn signal_shutdown(&self) { + self.exit_signal.store(true, Relaxed); + } + pub async fn update_connection( &self, broadcast_receiver: Receiver<(String, Vec)>, @@ -87,8 +93,7 @@ impl QuicProxyConnectionManager { info!("Starting very simple proxy thread"); - let exit_signal = Arc::new(AtomicBool::new(false)); - + let exit_signal = self.exit_signal.clone(); tokio::spawn(Self::read_transactions_and_broadcast( broadcast_receiver, self.current_tpu_nodes.clone(), @@ -141,6 +146,7 @@ impl QuicProxyConnectionManager { endpoint } + // send transactions to quic proxy async fn read_transactions_and_broadcast( mut transaction_receiver: Receiver<(String, Vec)>, current_tpu_nodes: Arc>>, @@ -154,6 +160,7 @@ impl QuicProxyConnectionManager { loop { // exit signal set if exit_signal.load(Relaxed) { + warn!("Caught exit signal - stopping sending transactions to quic proxy"); break; } From bd107c24c7ec58f8ff40173e6515ae9e1ee31f34 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 10 Aug 2023 14:01:20 +0200 Subject: [PATCH 122/128] add smarter agent shutdown --- quic-forward-proxy/src/outbound/tx_forward.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 680f3c0d..20d08c19 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -99,16 +99,20 @@ pub async fn tx_forwarder( per_connection_receiver.len()); break 'tx_channel_loop; } - // gracefully drain unit queue is empty - then shut down if agent_exit_signal_copy.load(Ordering::Relaxed) { if per_connection_receiver.is_empty() { warn!("Caught exit signal for this agent ({} #{}) - stopping agent thread", tpu_address, connection_idx); break 'tx_channel_loop; + } else if auto_connection.is_permanent_dead().await { + warn!("Caught exit signal for this agent ({} #{}), {} remaining but connection is dead - stopping", + tpu_address, connection_idx, + per_connection_receiver.len()); + break 'tx_channel_loop; } else { warn!("Caught exit signal for this agent ({} #{}), {} remaining - continue", - tpu_address, connection_idx, - per_connection_receiver.len()); + tpu_address, connection_idx, + per_connection_receiver.len()); } } From f2aa70652fa1d0f8a2f7e4bc020a1a9f6aae04ed Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 10 Aug 2023 14:16:35 +0200 Subject: [PATCH 123/128] refactor timeout --- quic-forward-proxy/src/outbound/tx_forward.rs | 19 +++++++++---------- .../src/quinn_auto_reconnect.rs | 4 ++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 20d08c19..d0d04fcc 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -90,9 +90,11 @@ pub async fn tx_forwarder( let _exit_signal_copy = global_exit_signal.clone(); 'tx_channel_loop: loop { let timeout_result = timeout_fallback(per_connection_receiver.recv()).await; - if let Err(_elapsed) = timeout_result { - continue 'tx_channel_loop; - } + + let maybe_packet = match timeout_result { + Ok(recv) => recv, + Err(_elapsed) => continue 'tx_channel_loop, + }; if global_exit_signal.load(Ordering::Relaxed) { warn!("Caught global exit signal, {} remaining - stopping agent thread", @@ -117,8 +119,6 @@ pub async fn tx_forwarder( } - let maybe_packet = timeout_result.unwrap(); - if let Err(_recv_error) = maybe_packet { break 'tx_channel_loop; } @@ -133,11 +133,10 @@ pub async fn tx_forwarder( } if auto_connection.is_permanent_dead().await { - warn!("Connection is considered permanently dead"); - // while let Ok(more) = per_connection_receiver.try_recv() { - // // drain - // } - // continue 'tx_channel_loop; + warn!("Agent ({} #{}) connection permanently dead, {} remaining - stopping", + tpu_address, connection_idx, + per_connection_receiver.len()); + break 'tx_channel_loop; } let mut transactions_batch: Vec> = packet.transactions.clone(); diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 9bd3f11a..24ed35af 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -149,6 +149,10 @@ impl AutoReconnect { ConnectionState::FailedAttempt(attempts) => { match self.create_connection().await { Some(new_connection) => { + debug!( + "Reconnected to {} succeeded after {} attempts", + self.target_address, attempts + ); *lock = ConnectionState::Connection(new_connection); } None => { From 10e4ea925aab3b73e5c222f5593849a45fd23739 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 10 Aug 2023 14:38:15 +0200 Subject: [PATCH 124/128] clarify test naming --- .../tests/quic_proxy_tpu_integrationtest.rs | 10 +++++----- services/src/tpu_utils/tpu_connection_manager.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index e97f8b82..fbe3ae83 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -62,7 +62,7 @@ const QUIC_CONNECTION_PARAMS: QuicConnectionParameters = QuicConnectionParameter }; #[test] -pub fn small_tx_batch_staked() { +pub fn small_tx_batch_staked_direct() { configure_logging(true); wireup_and_send_txs_via_channel(TestCaseParams { @@ -84,7 +84,7 @@ pub fn small_tx_batch_staked_proxy() { } #[test] -pub fn small_tx_batch_unstaked() { +pub fn small_tx_batch_unstake_direct() { configure_logging(true); wireup_and_send_txs_via_channel(TestCaseParams { @@ -95,7 +95,7 @@ pub fn small_tx_batch_unstaked() { } #[test] -pub fn with_100_transactions() { +pub fn with_100_transactions_direct() { configure_logging(false); wireup_and_send_txs_via_channel(TestCaseParams { @@ -106,7 +106,7 @@ pub fn with_100_transactions() { } #[test] -pub fn with_1000_transactions() { +pub fn with_1000_transactions_direct() { configure_logging(false); wireup_and_send_txs_via_channel(TestCaseParams { @@ -130,7 +130,7 @@ pub fn bench_proxy() { } #[test] -pub fn with_10000_transactions() { +pub fn with_10000_transactions_direct() { configure_logging(false); wireup_and_send_txs_via_channel(TestCaseParams { diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 0a5990de..a31f378e 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -129,7 +129,7 @@ impl ActiveConnection { }, Err(e) => { error!( - "Broadcast channel error on recv for {} error {}", + "Broadcast channel error on recv for {} error {} - continue", identity, e ); continue; From 4982325e674e694b994f92ecb54c45e3bff3fffa Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 10 Aug 2023 14:50:42 +0200 Subject: [PATCH 125/128] direct test - wait in loop --- .../tests/quic_proxy_tpu_integrationtest.rs | 10 +++++++++- services/src/tpu_utils/tpu_connection_manager.rs | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index fbe3ae83..b474d6a9 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -529,12 +529,18 @@ async fn start_literpc_client_direct_mode( broadcast_sender.send(raw_sample_tx)?; } - sleep(Duration::from_secs(30)).await; + while !broadcast_sender.is_empty() { + sleep(Duration::from_millis(1000)).await; + warn!("broadcast channel is not empty - wait before shutdown test client thread"); + } + assert!( broadcast_sender.is_empty(), "broadcast channel must be empty" ); + sleep(Duration::from_secs(3)).await; + Ok(()) } @@ -647,6 +653,8 @@ async fn start_literpc_client_proxy_mode( quic_proxy_connection_manager.signal_shutdown(); + sleep(Duration::from_secs(3)).await; + Ok(()) } diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index a31f378e..29dbe550 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -138,8 +138,8 @@ impl ActiveConnection { let mut txs = vec![first_tx]; for _ in 1..number_of_transactions_per_unistream { - if let Ok((signature, tx)) = transaction_reciever.try_recv() { - if Self::check_for_confirmation(&txs_sent_store, signature) { + if let Ok((sig, tx)) = transaction_reciever.try_recv() { + if Self::check_for_confirmation(&txs_sent_store, sig) { continue; } txs.push(tx); From 0aebd7b362b1db5e7a18462d9d21ebd1bb32bf37 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 11 Aug 2023 17:45:36 +0200 Subject: [PATCH 126/128] MAN-59 disable flaky tests --- .../tests/quic_proxy_tpu_integrationtest.rs | 7 +++++-- services/tests/literpc_tpu_quic_server_integrationtest.rs | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index b474d6a9..e52e513b 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -116,6 +116,8 @@ pub fn with_1000_transactions_direct() { }); } +// note: this tests are flakes on CI ond also local (see https://mangolana.atlassian.net/browse/MAN-59) +#[ignore] #[test] pub fn bench_proxy() { configure_logging(true); @@ -129,6 +131,8 @@ pub fn bench_proxy() { }); } +// note: this tests are flakes on CI ond also local (see https://mangolana.atlassian.net/browse/MAN-59) +#[ignore] #[test] pub fn with_10000_transactions_direct() { configure_logging(false); @@ -151,9 +155,8 @@ pub fn with_10000_transactions_proxy() { }); } -#[ignore] #[test] -pub fn too_many_transactions() { +pub fn many_transactions_proxy() { configure_logging(false); wireup_and_send_txs_via_channel(TestCaseParams { diff --git a/services/tests/literpc_tpu_quic_server_integrationtest.rs b/services/tests/literpc_tpu_quic_server_integrationtest.rs index e0d4a54a..e1a92bac 100644 --- a/services/tests/literpc_tpu_quic_server_integrationtest.rs +++ b/services/tests/literpc_tpu_quic_server_integrationtest.rs @@ -73,6 +73,8 @@ pub fn small_tx_batch_unstaked() { }); } +// note: this tests are flakes on CI ond also local (see https://mangolana.atlassian.net/browse/MAN-59) +#[ignore] #[test] pub fn many_transactions() { configure_logging(false); @@ -83,6 +85,7 @@ pub fn many_transactions() { }); } +// note: this tests are flakes on CI ond also local (see https://mangolana.atlassian.net/browse/MAN-59) #[ignore] #[test] pub fn too_many_transactions() { From 75a17779571ec85543e689234d395c13173c0896 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 11 Aug 2023 17:55:38 +0200 Subject: [PATCH 127/128] tweaked log levels on agent shutdown --- quic-forward-proxy/src/outbound/tx_forward.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index d0d04fcc..a821c3e1 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -103,16 +103,16 @@ pub async fn tx_forwarder( } if agent_exit_signal_copy.load(Ordering::Relaxed) { if per_connection_receiver.is_empty() { - warn!("Caught exit signal for this agent ({} #{}) - stopping agent thread", + debug!("Caught exit signal for this agent ({} #{}) - stopping agent thread", tpu_address, connection_idx); break 'tx_channel_loop; } else if auto_connection.is_permanent_dead().await { - warn!("Caught exit signal for this agent ({} #{}), {} remaining but connection is dead - stopping", + info!("Caught exit signal for this agent ({} #{}), {} remaining but connection is dead - stopping", tpu_address, connection_idx, per_connection_receiver.len()); break 'tx_channel_loop; } else { - warn!("Caught exit signal for this agent ({} #{}), {} remaining - continue", + trace!("Caught exit signal for this agent ({} #{}), {} remaining - continue", tpu_address, connection_idx, per_connection_receiver.len()); } From 3e4b7a0687741e7c0249a98f7680b25010e4658b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 11 Aug 2023 17:57:23 +0200 Subject: [PATCH 128/128] fix compile --- quic-forward-proxy/src/outbound/tx_forward.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index a821c3e1..d7224953 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -7,7 +7,7 @@ use crate::util::timeout_fallback; use crate::validator_identity::ValidatorIdentity; use anyhow::{bail, Context}; use futures::future::join_all; -use log::{debug, info, warn}; +use log::{debug, info, trace, warn}; use quinn::{ ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt, };