From c406814886084d41af4d5e0cab411e54b0b54ddb Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 6 Nov 2024 16:58:23 +0100 Subject: [PATCH 001/102] tcp conn tracker --- sdk/rust/nym-sdk/src/tcp_proxy.rs | 2 + .../src/tcp_proxy/connection_tracker.rs | 169 ++++++++++++++++++ .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 21 ++- 3 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs diff --git a/sdk/rust/nym-sdk/src/tcp_proxy.rs b/sdk/rust/nym-sdk/src/tcp_proxy.rs index 86aafcd8462..5e276ca4c94 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy.rs @@ -201,8 +201,10 @@ //! } //! ``` +mod connection_tracker; mod tcp_proxy_client; mod tcp_proxy_server; +pub use connection_tracker::TcpConnectionTracker; pub use tcp_proxy_client::NymProxyClient; pub use tcp_proxy_server::NymProxyServer; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs b/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs new file mode 100644 index 00000000000..7bd565222e2 --- /dev/null +++ b/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs @@ -0,0 +1,169 @@ +use anyhow::Error; +use anyhow::{bail, Result}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::thread; + +#[derive(Debug)] +pub struct TcpConnectionTracker { + count: Arc, +} + +impl TcpConnectionTracker { + pub fn new() -> Self { + Self { + count: Arc::new(AtomicUsize::new(0)), + } + } + + pub fn increment(&self) { + self.count.fetch_add(1, Ordering::SeqCst); + } + + pub fn decrement(&self) -> Result<()> { + if self.get_count() == 0 { + bail!("count already 0"); + } + self.count.fetch_sub(1, Ordering::SeqCst); + Ok(()) + } + + pub fn get_count(&self) -> usize { + self.count.load(Ordering::SeqCst) + } +} + +impl Clone for TcpConnectionTracker { + fn clone(&self) -> Self { + Self { + count: Arc::clone(&self.count), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::Result; + use std::sync::Arc; + use std::thread; + + #[test] + fn test_increment_decrement() -> Result<()> { + let tracker = TcpConnectionTracker::new(); + tracker.increment(); + tracker.increment(); + assert_eq!(tracker.get_count(), 2, "should be 2 after single increment"); + tracker.decrement()?; + assert_eq!( + tracker.get_count(), + 1, + "should be 1 after two increments and one decrement" + ); + Ok(()) + } + + #[test] + fn test_clone() { + let tracker = TcpConnectionTracker::new(); + let tracker_clone = tracker.clone(); + + tracker.increment(); + assert_eq!( + tracker_clone.get_count(), + 1, + "tracker clones should share the same count" + ); + } + + #[test] + fn test_multiple_threads() { + let tracker = TcpConnectionTracker::new(); + let mut handles = vec![]; + + for _ in 0..10 { + let thread_tracker = tracker.clone(); + let handle = thread::spawn(move || { + thread_tracker.increment(); + thread::sleep(std::time::Duration::from_millis(10)); + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!( + tracker.get_count(), + 10, + "should be 10 after 10 thread increments" + ); + } + + #[test] + fn test_concurrent_increment_decrement() -> Result<()> { + let tracker = TcpConnectionTracker::new(); + let mut handles = vec![]; + + for i in 0..10 { + let thread_tracker = tracker.clone(); + let handle = thread::spawn(move || { + if i < 5 { + thread_tracker.increment(); + } else { + thread_tracker.decrement().unwrap(); + } + thread::sleep(std::time::Duration::from_millis(10)); + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!( + tracker.get_count(), + 0, + "should be 0 after equal increments and decrements" + ); + Ok(()) + } + + #[test] + #[should_panic] + fn test_zero_floor() { + let tracker = TcpConnectionTracker::new(); + tracker.decrement().unwrap(); // should panic + } + + #[test] + fn test_stress() { + let tracker = TcpConnectionTracker::new(); + let mut handles = vec![]; + let num_threads = 100; + + for _ in 0..num_threads { + let thread_tracker = tracker.clone(); + let handle = thread::spawn(move || { + for _ in 0..100 { + thread_tracker.increment(); + thread::sleep(std::time::Duration::from_micros(1)); + thread_tracker.decrement().unwrap(); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!( + tracker.get_count(), + 0, + "should return to 0 after all increments and decrements" + ); + } +} diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index a0200d4c217..a072735e9b4 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -1,5 +1,8 @@ use crate::mixnet::{IncludedSurbs, MixnetClientBuilder, MixnetMessageSender, NymNetworkDetails}; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; +#[path = "connection_tracker.rs"] +mod connection_tracker; +use connection_tracker::TcpConnectionTracker; #[path = "utils.rs"] mod utils; use anyhow::Result; @@ -24,6 +27,7 @@ pub struct NymProxyClient { listen_address: String, listen_port: String, close_timeout: u64, + conn_tracker: TcpConnectionTracker, } impl NymProxyClient { @@ -41,6 +45,7 @@ impl NymProxyClient { listen_address: listen_address.to_string(), listen_port: listen_port.to_string(), close_timeout, + conn_tracker: TcpConnectionTracker::new(), }) } @@ -53,6 +58,7 @@ impl NymProxyClient { listen_address: DEFAULT_LISTEN_HOST.to_string(), listen_port: DEFAULT_LISTEN_PORT.to_string(), close_timeout: DEFAULT_CLOSE_TIMEOUT, + conn_tracker: TcpConnectionTracker::new(), }) } @@ -68,6 +74,7 @@ impl NymProxyClient { stream, self.server_address, self.close_timeout, + self.conn_tracker.clone(), )); } } @@ -89,7 +96,10 @@ impl NymProxyClient { stream: TcpStream, server_address: Recipient, close_timeout: u64, + conn_pool: TcpConnectionTracker, ) -> Result<()> { + conn_pool.increment(); + // ID for creation of session abstraction; new session ID per new connection accepted by our tcp listener above. let session_id = uuid::Uuid::new_v4(); @@ -137,6 +147,14 @@ impl NymProxyClient { // Wrap in an Arc for memsafe concurrent access let sent_messages_account = Arc::clone(&messages_account); + let overall_counter = conn_pool.clone(); + tokio::spawn(async move { + loop { + info!("active tcp connections: {}", overall_counter.get_count()); + tokio::time::sleep(Duration::from_secs(5)).await; + } + }); + // 'Outgoing' thread tokio::spawn(async move { let mut message_id = 0; @@ -217,6 +235,7 @@ impl NymProxyClient { info!(":: Closing write end of session: {}", session_id); info!(":: Triggering client shutdown"); client.disconnect().await; + conn_pool.clone().decrement()?; return Ok::<(), anyhow::Error>(()) } } From f53d2aa4cd6fde73aa427869811c39424b615f8a Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 6 Nov 2024 17:54:14 +0100 Subject: [PATCH 002/102] small tweak to tcp tracker --- sdk/rust/nym-sdk/src/tcp_proxy.rs | 6 ++-- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 30 +++++++++---------- .../{connection_tracker.rs => tcp_tracker.rs} | 3 -- 3 files changed, 18 insertions(+), 21 deletions(-) rename sdk/rust/nym-sdk/src/tcp_proxy/{connection_tracker.rs => tcp_tracker.rs} (98%) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy.rs b/sdk/rust/nym-sdk/src/tcp_proxy.rs index 5e276ca4c94..fcbaf4ad11f 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy.rs @@ -201,10 +201,12 @@ //! } //! ``` -mod connection_tracker; +// TODO UPDATE EXAMPLE ONCE STABLE + mod tcp_proxy_client; mod tcp_proxy_server; +mod tcp_tracker; -pub use connection_tracker::TcpConnectionTracker; pub use tcp_proxy_client::NymProxyClient; pub use tcp_proxy_server::NymProxyServer; +pub use tcp_tracker::TcpConnectionTracker; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index a072735e9b4..98c61406d1f 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -1,8 +1,8 @@ use crate::mixnet::{IncludedSurbs, MixnetClientBuilder, MixnetMessageSender, NymNetworkDetails}; use std::{sync::Arc, time::Duration}; -#[path = "connection_tracker.rs"] -mod connection_tracker; -use connection_tracker::TcpConnectionTracker; +#[path = "tcp_tracker.rs"] +mod tcp_tracker; +use tcp_tracker::TcpConnectionTracker; #[path = "utils.rs"] mod utils; use anyhow::Result; @@ -68,6 +68,14 @@ impl NymProxyClient { let listener = TcpListener::bind(format!("{}:{}", self.listen_address, self.listen_port)).await?; + let overall_counter = self.conn_tracker.clone(); + tokio::spawn(async move { + loop { + info!("active tcp connections: {}", overall_counter.get_count()); + tokio::time::sleep(Duration::from_secs(10)).await; + } + }); + loop { let (stream, _) = listener.accept().await?; tokio::spawn(NymProxyClient::handle_incoming( @@ -96,9 +104,9 @@ impl NymProxyClient { stream: TcpStream, server_address: Recipient, close_timeout: u64, - conn_pool: TcpConnectionTracker, + conn_tracker: TcpConnectionTracker, ) -> Result<()> { - conn_pool.increment(); + conn_tracker.increment(); // ID for creation of session abstraction; new session ID per new connection accepted by our tcp listener above. let session_id = uuid::Uuid::new_v4(); @@ -112,8 +120,6 @@ impl NymProxyClient { info!(":: creating client..."); let mut client = loop { let net = NymNetworkDetails::new_from_env(); - // TODO change to builder but ephemeral - // match MixnetClient::connect_new().await { match MixnetClientBuilder::new_ephemeral() .network_details(net) .build()? @@ -147,14 +153,6 @@ impl NymProxyClient { // Wrap in an Arc for memsafe concurrent access let sent_messages_account = Arc::clone(&messages_account); - let overall_counter = conn_pool.clone(); - tokio::spawn(async move { - loop { - info!("active tcp connections: {}", overall_counter.get_count()); - tokio::time::sleep(Duration::from_secs(5)).await; - } - }); - // 'Outgoing' thread tokio::spawn(async move { let mut message_id = 0; @@ -235,7 +233,7 @@ impl NymProxyClient { info!(":: Closing write end of session: {}", session_id); info!(":: Triggering client shutdown"); client.disconnect().await; - conn_pool.clone().decrement()?; + conn_tracker.clone().decrement()?; return Ok::<(), anyhow::Error>(()) } } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_tracker.rs similarity index 98% rename from sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs rename to sdk/rust/nym-sdk/src/tcp_proxy/tcp_tracker.rs index 7bd565222e2..1b6e434ce46 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_tracker.rs @@ -1,8 +1,6 @@ -use anyhow::Error; use anyhow::{bail, Result}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::thread; #[derive(Debug)] pub struct TcpConnectionTracker { @@ -45,7 +43,6 @@ impl Clone for TcpConnectionTracker { mod tests { use super::*; use anyhow::Result; - use std::sync::Arc; use std::thread; #[test] From 1130d0fe3ef6b8ba80ecac476d289dfbd5992be8 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 6 Nov 2024 18:11:52 +0100 Subject: [PATCH 003/102] name change + specific inc and dec logging --- sdk/rust/nym-sdk/src/tcp_proxy.rs | 4 ++-- .../{tcp_tracker.rs => connection_tracker.rs} | 18 +++++++-------- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 22 +++++++++++++------ 3 files changed, 26 insertions(+), 18 deletions(-) rename sdk/rust/nym-sdk/src/tcp_proxy/{tcp_tracker.rs => connection_tracker.rs} (90%) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy.rs b/sdk/rust/nym-sdk/src/tcp_proxy.rs index fcbaf4ad11f..beadc514aee 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy.rs @@ -203,10 +203,10 @@ // TODO UPDATE EXAMPLE ONCE STABLE +mod connection_tracker; mod tcp_proxy_client; mod tcp_proxy_server; -mod tcp_tracker; +pub use connection_tracker::ConnectionTracker; pub use tcp_proxy_client::NymProxyClient; pub use tcp_proxy_server::NymProxyServer; -pub use tcp_tracker::TcpConnectionTracker; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_tracker.rs b/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs similarity index 90% rename from sdk/rust/nym-sdk/src/tcp_proxy/tcp_tracker.rs rename to sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs index 1b6e434ce46..df6f195d529 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_tracker.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs @@ -3,11 +3,11 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; #[derive(Debug)] -pub struct TcpConnectionTracker { +pub struct ConnectionTracker { count: Arc, } -impl TcpConnectionTracker { +impl ConnectionTracker { pub fn new() -> Self { Self { count: Arc::new(AtomicUsize::new(0)), @@ -31,7 +31,7 @@ impl TcpConnectionTracker { } } -impl Clone for TcpConnectionTracker { +impl Clone for ConnectionTracker { fn clone(&self) -> Self { Self { count: Arc::clone(&self.count), @@ -47,7 +47,7 @@ mod tests { #[test] fn test_increment_decrement() -> Result<()> { - let tracker = TcpConnectionTracker::new(); + let tracker = ConnectionTracker::new(); tracker.increment(); tracker.increment(); assert_eq!(tracker.get_count(), 2, "should be 2 after single increment"); @@ -62,7 +62,7 @@ mod tests { #[test] fn test_clone() { - let tracker = TcpConnectionTracker::new(); + let tracker = ConnectionTracker::new(); let tracker_clone = tracker.clone(); tracker.increment(); @@ -75,7 +75,7 @@ mod tests { #[test] fn test_multiple_threads() { - let tracker = TcpConnectionTracker::new(); + let tracker = ConnectionTracker::new(); let mut handles = vec![]; for _ in 0..10 { @@ -100,7 +100,7 @@ mod tests { #[test] fn test_concurrent_increment_decrement() -> Result<()> { - let tracker = TcpConnectionTracker::new(); + let tracker = ConnectionTracker::new(); let mut handles = vec![]; for i in 0..10 { @@ -131,13 +131,13 @@ mod tests { #[test] #[should_panic] fn test_zero_floor() { - let tracker = TcpConnectionTracker::new(); + let tracker = ConnectionTracker::new(); tracker.decrement().unwrap(); // should panic } #[test] fn test_stress() { - let tracker = TcpConnectionTracker::new(); + let tracker = ConnectionTracker::new(); let mut handles = vec![]; let num_threads = 100; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 98c61406d1f..913794dd4a7 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -1,8 +1,8 @@ use crate::mixnet::{IncludedSurbs, MixnetClientBuilder, MixnetMessageSender, NymNetworkDetails}; use std::{sync::Arc, time::Duration}; -#[path = "tcp_tracker.rs"] -mod tcp_tracker; -use tcp_tracker::TcpConnectionTracker; +#[path = "connection_tracker.rs"] +mod connection_tracker; +use connection_tracker::ConnectionTracker; #[path = "utils.rs"] mod utils; use anyhow::Result; @@ -27,7 +27,7 @@ pub struct NymProxyClient { listen_address: String, listen_port: String, close_timeout: u64, - conn_tracker: TcpConnectionTracker, + conn_tracker: ConnectionTracker, } impl NymProxyClient { @@ -45,7 +45,7 @@ impl NymProxyClient { listen_address: listen_address.to_string(), listen_port: listen_port.to_string(), close_timeout, - conn_tracker: TcpConnectionTracker::new(), + conn_tracker: ConnectionTracker::new(), }) } @@ -58,7 +58,7 @@ impl NymProxyClient { listen_address: DEFAULT_LISTEN_HOST.to_string(), listen_port: DEFAULT_LISTEN_PORT.to_string(), close_timeout: DEFAULT_CLOSE_TIMEOUT, - conn_tracker: TcpConnectionTracker::new(), + conn_tracker: ConnectionTracker::new(), }) } @@ -104,9 +104,13 @@ impl NymProxyClient { stream: TcpStream, server_address: Recipient, close_timeout: u64, - conn_tracker: TcpConnectionTracker, + conn_tracker: ConnectionTracker, ) -> Result<()> { conn_tracker.increment(); + info!( + "new connection - current active tcp connections: {}", + conn_tracker.get_count() + ); // ID for creation of session abstraction; new session ID per new connection accepted by our tcp listener above. let session_id = uuid::Uuid::new_v4(); @@ -234,6 +238,10 @@ impl NymProxyClient { info!(":: Triggering client shutdown"); client.disconnect().await; conn_tracker.clone().decrement()?; + info!( + "dropped connection - current active tcp connections: {}", + conn_tracker.get_count() + ); return Ok::<(), anyhow::Error>(()) } } From d423cde71cf6f7f454eec9b5b8e8d328ced178a3 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 6 Nov 2024 18:16:57 +0100 Subject: [PATCH 004/102] changed logging --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 913794dd4a7..baa1e801b5a 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -71,7 +71,7 @@ impl NymProxyClient { let overall_counter = self.conn_tracker.clone(); tokio::spawn(async move { loop { - info!("active tcp connections: {}", overall_counter.get_count()); + info!("active clients: {}", overall_counter.get_count()); tokio::time::sleep(Duration::from_secs(10)).await; } }); @@ -108,7 +108,7 @@ impl NymProxyClient { ) -> Result<()> { conn_tracker.increment(); info!( - "new connection - current active tcp connections: {}", + "new connection - current active clients: {}", conn_tracker.get_count() ); @@ -239,7 +239,7 @@ impl NymProxyClient { client.disconnect().await; conn_tracker.clone().decrement()?; info!( - "dropped connection - current active tcp connections: {}", + "dropped connection - current active clients: {}", conn_tracker.get_count() ); return Ok::<(), anyhow::Error>(()) From 0c36e64526a1d2320fa888cef10d359b4de942f9 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 6 Nov 2024 18:20:15 +0100 Subject: [PATCH 005/102] tweak --- .../nym-sdk/src/tcp_proxy/connection_tracker.rs | 1 + .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs b/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs index df6f195d529..a24b49be14c 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs @@ -2,6 +2,7 @@ use anyhow::{bail, Result}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; +// This is used to keep track of the number of active ephemeral clients that are being called by handle_connection() #[derive(Debug)] pub struct ConnectionTracker { count: Arc, diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index baa1e801b5a..69a0150b916 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -45,7 +45,7 @@ impl NymProxyClient { listen_address: listen_address.to_string(), listen_port: listen_port.to_string(), close_timeout, - conn_tracker: ConnectionTracker::new(), + conn_tracker: ConnectionTracker::new(), // This is used to keep track of the number of active ephemeral clients that are being called by handle_connection() }) } @@ -237,14 +237,19 @@ impl NymProxyClient { info!(":: Closing write end of session: {}", session_id); info!(":: Triggering client shutdown"); client.disconnect().await; - conn_tracker.clone().decrement()?; - info!( - "dropped connection - current active clients: {}", - conn_tracker.get_count() - ); + // conn_tracker.clone().decrement()?; + // info!( + // "dropped connection - current active clients: {}", + // conn_tracker.get_count() + // ); return Ok::<(), anyhow::Error>(()) } } + conn_tracker.clone().decrement()?; + info!( + "dropped connection - current active clients: {}", + conn_tracker.get_count() + ); } }); tokio::signal::ctrl_c().await?; From 109de83128f80a8ba4aecc663016df8ae9c46205 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 6 Nov 2024 18:27:34 +0100 Subject: [PATCH 006/102] tweak --- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 69a0150b916..84a1ab1323c 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -237,19 +237,14 @@ impl NymProxyClient { info!(":: Closing write end of session: {}", session_id); info!(":: Triggering client shutdown"); client.disconnect().await; - // conn_tracker.clone().decrement()?; - // info!( - // "dropped connection - current active clients: {}", - // conn_tracker.get_count() - // ); + conn_tracker.clone().decrement()?; + info!( + "dropped connection - current active clients: {}", + conn_tracker.get_count() + ); return Ok::<(), anyhow::Error>(()) } } - conn_tracker.clone().decrement()?; - info!( - "dropped connection - current active clients: {}", - conn_tracker.get_count() - ); } }); tokio::signal::ctrl_c().await?; From 2e8e9a4c0a7f97dd43c1061c53e3231f19e25c0c Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 21 Nov 2024 12:03:03 +0100 Subject: [PATCH 007/102] tweak existing conn tracker logic --- .../src/tcp_proxy/connection_tracker.rs | 5 +-- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 35 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs b/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs index a24b49be14c..414c13e3d1c 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs @@ -2,7 +2,8 @@ use anyhow::{bail, Result}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -// This is used to keep track of the number of active ephemeral clients that are being called by handle_connection() +// Used by the ProxyClient for tracking # of active clients created on new TCP connection +// TODO used by the connection pool for maintaining # of clients in pool +/- #[derive(Debug)] pub struct ConnectionTracker { count: Arc, @@ -133,7 +134,7 @@ mod tests { #[should_panic] fn test_zero_floor() { let tracker = ConnectionTracker::new(); - tracker.decrement().unwrap(); // should panic + tracker.decrement().unwrap(); } #[test] diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 84a1ab1323c..067dd14eeb2 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -39,13 +39,13 @@ impl NymProxyClient { env: Option, ) -> Result { debug!("loading env file: {:?}", env); - setup_env(env); + setup_env(env); // Defaults to mainnet if empty Ok(NymProxyClient { server_address, listen_address: listen_address.to_string(), listen_port: listen_port.to_string(), close_timeout, - conn_tracker: ConnectionTracker::new(), // This is used to keep track of the number of active ephemeral clients that are being called by handle_connection() + conn_tracker: ConnectionTracker::new(), // Keep track of the number of active ephemeral clients that are in use }) } @@ -72,7 +72,7 @@ impl NymProxyClient { tokio::spawn(async move { loop { info!("active clients: {}", overall_counter.get_count()); - tokio::time::sleep(Duration::from_secs(10)).await; + tokio::time::sleep(Duration::from_secs(5)).await; } }); @@ -106,12 +106,6 @@ impl NymProxyClient { close_timeout: u64, conn_tracker: ConnectionTracker, ) -> Result<()> { - conn_tracker.increment(); - info!( - "new connection - current active clients: {}", - conn_tracker.get_count() - ); - // ID for creation of session abstraction; new session ID per new connection accepted by our tcp listener above. let session_id = uuid::Uuid::new_v4(); @@ -120,8 +114,8 @@ impl NymProxyClient { // Client creation can fail for multiple reasons like bad network connection: this loop just allows us to // retry in a loop until we can successfully connect without having to restart the entire function - info!(":: Starting session: {}", session_id); - info!(":: creating client..."); + info!("Starting session: {}", session_id); + info!("Creating client..."); let mut client = loop { let net = NymNetworkDetails::new_from_env(); match MixnetClientBuilder::new_ephemeral() @@ -132,13 +126,18 @@ impl NymProxyClient { { Ok(client) => break client, Err(err) => { - warn!(":: Error creating client: {:?}, will retry in 100ms", err); + warn!("Error creating client: {:?}, will retry in 100ms", err); tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; } } }; let client_addr = &client.nym_address(); - info!(":: client created: {}", &client_addr); + info!("Client created: {}", &client_addr); + conn_tracker.increment(); + info!( + "New connection - current active clients: {}", + conn_tracker.get_count() + ); // Split our tcpstream into OwnedRead and OwnedWrite halves for concurrent read/writing let (read, mut write) = stream.into_split(); @@ -192,7 +191,7 @@ impl NymProxyClient { .send_message(server_addr, &coded_message, IncludedSurbs::Amount(100)) .await?; - info!(":: Closing read end of session: {}", session_id); + info!("Closing read end of session: {}", session_id); tx.send(true) .map_err(|_| anyhow::anyhow!("Could not send close signal"))?; Ok::<(), anyhow::Error>(()) @@ -210,7 +209,7 @@ impl NymProxyClient { loop { tokio::select! { _ = &mut rx => { - info!(":: Closing write end of session: {} in {} seconds", session_id, close_timeout); + info!(" Closing write end of session: {} in {} seconds", session_id, close_timeout); break } Some(message) = client.next() => { @@ -234,12 +233,12 @@ impl NymProxyClient { msg_buffer.tick(&mut write).await?; }, _ = tokio::time::sleep(tokio::time::Duration::from_secs(close_timeout)) => { - info!(":: Closing write end of session: {}", session_id); - info!(":: Triggering client shutdown"); + info!(" Closing write end of session: {}", session_id); + info!(" Triggering client shutdown"); client.disconnect().await; conn_tracker.clone().decrement()?; info!( - "dropped connection - current active clients: {}", + "Dropped connection - current active clients: {}", conn_tracker.get_count() ); return Ok::<(), anyhow::Error>(()) From 530d744d6d672a9a4ce26bdaf77d5a331f6f8e81 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 21 Nov 2024 14:09:50 +0100 Subject: [PATCH 008/102] make default decay const --- sdk/rust/nym-sdk/src/tcp_proxy/utils.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs index b4bdad247e5..cafccf4c4cd 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs @@ -6,6 +6,8 @@ use tokio::{io::AsyncWriteExt as _, net::tcp::OwnedWriteHalf}; use tracing::{debug, info}; use uuid::Uuid; +const DEFAULT_DECAY: u64 = 2; // decay time in seconds + // Keeps track of // - incoming and unsorted messages wrapped in DecayWrapper for keeping track of when they were received // - the expected next message ID (reset on .tick()) @@ -122,7 +124,7 @@ impl DecayWrapper { DecayWrapper { value, start: Instant::now(), - decay: 6, + decay: DEFAULT_DECAY, } } From 546482916d57cb56f22d35fce3906fadb27464d5 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 25 Nov 2024 23:26:48 +0100 Subject: [PATCH 009/102] first pass connpool --- sdk/rust/nym-sdk/src/tcp_proxy.rs | 4 +- sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs | 271 ++++++++++++++++++ .../src/tcp_proxy/connection_tracker.rs | 168 ----------- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 106 +++---- 4 files changed, 331 insertions(+), 218 deletions(-) create mode 100644 sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs delete mode 100644 sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs diff --git a/sdk/rust/nym-sdk/src/tcp_proxy.rs b/sdk/rust/nym-sdk/src/tcp_proxy.rs index beadc514aee..af71dc6f0fc 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy.rs @@ -203,10 +203,10 @@ // TODO UPDATE EXAMPLE ONCE STABLE -mod connection_tracker; +// mod connection_tracker; mod tcp_proxy_client; mod tcp_proxy_server; -pub use connection_tracker::ConnectionTracker; +// pub use connection_tracker::ConnectionTracker; pub use tcp_proxy_client::NymProxyClient; pub use tcp_proxy_server::NymProxyServer; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs new file mode 100644 index 00000000000..24f9ac42be8 --- /dev/null +++ b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs @@ -0,0 +1,271 @@ +use crate::mixnet::{MixnetClient, MixnetClientBuilder, NymNetworkDetails}; +use anyhow::{bail, Result}; +use std::collections::VecDeque; +use std::fmt; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use tokio::sync::RwLock; +use tracing::{debug, info, instrument, warn}; + +// Make a set # of clients (low default) +// Make sure that # of clients is always 1 above the # of incoming conn requests +// Once a client is used, kill the client & remove it from the pool +pub struct ClientPool { + clients: Arc>>>, + default_pool_size: usize, + conn_count: Arc, // the actual # of connections running, denoting an incoming tcp request that is matched with a nym client + ready: bool, // default false, trigger true when clients.len() is default_pool_size/2 (currently arbitrary decision for 'floor' of ready clients) + // + // TODO not sure we need this, can just rely on trying to get a client and if there is none available then make one + // incoming_conns: Arc, // if we have incoming tcp connections that aren't matched: this will be used in order to work out the tcp conn 'backpressure' and to work out if we need to add clients to the pool +} + +impl fmt::Debug for ClientPool { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let clients_debug = match self.clients.try_read() { + Ok(clients) => { + if f.alternate() { + // pretty + clients + .iter() + .enumerate() + .map(|(i, client)| { + format!("\n {}: {}", i, client.nym_address().to_string()) + }) + .collect::>() + .join(",") + } else { + // compact + clients + .iter() + .map(|client| client.nym_address().to_string()) + .collect::>() + .join(", ") + } + } + Err(_) => "".to_string(), + }; + + let mut debug_struct = f.debug_struct("Pool"); + debug_struct + .field("default_pool_size", &self.default_pool_size) + .field("connection count", &*self.conn_count) + .field("clients", &format_args!("[{}]", clients_debug)) + .field("ready", &self.ready); + + debug_struct.finish() + } +} + +impl ClientPool { + pub fn new(default_pool_size: usize) -> Self { + ClientPool { + clients: Arc::new(RwLock::new(Vec::new())), + default_pool_size, + conn_count: Arc::new(AtomicUsize::new(0)), + // incoming_conns: Arc::new(AtomicUsize::new(0)), // see comment above + ready: false, + } + } + + // if clients == default, sleep + // if incoming conns > default - conn_count (aka in use clients) then make more clients + pub async fn start(&self) -> Result<()> { + loop { + let spawned_clients = self.clients.read().await.len(); + info!("Currently spawned clients: {}", spawned_clients); + + if spawned_clients >= self.default_pool_size { + debug!("got enough clients already: sleeping"); + } else { + info!("Spawning new client"); + let client = loop { + let net = NymNetworkDetails::new_from_env(); + match MixnetClientBuilder::new_ephemeral() + .network_details(net) + .build()? + .connect_to_mixnet() + .await + { + Ok(client) => break client, + Err(err) => { + warn!("Error creating client: {:?}, will retry in 100ms", err); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + } + }; + self.clients.write().await.push(Arc::new(client)); + } + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } + } + + pub async fn get_mixnet_client(&self) -> Result { + todo!() + } + + // disconnect ephemeral + // remove from vec + pub async fn disconnect_and_remove_client(&self, client: MixnetClient) -> Result<()> { + todo!() + } + + pub async fn get_client_count(&self) -> usize { + self.clients.read().await.len() + } + + pub fn get_conn_count(&self) -> usize { + self.conn_count.load(Ordering::SeqCst) + } + + pub fn increment_conn_count(&self) { + self.conn_count.fetch_add(1, Ordering::SeqCst); + } + + pub fn decrement_conn_count(&self) -> Result<()> { + if self.get_conn_count() == 0 { + bail!("count already 0"); + } + self.conn_count.fetch_sub(1, Ordering::SeqCst); + Ok(()) + } + + pub fn clone(&self) -> Self { + Self { + clients: Arc::clone(&self.clients), + default_pool_size: *&self.default_pool_size, + conn_count: Arc::clone(&self.conn_count), + // incoming_conns: Arc::clone(&self.incoming_conns), + ready: self.ready.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::Result; + use std::thread; + + #[test] + fn test_conn_count_increment_decrement() -> Result<()> { + let tracker = ClientPool::new(0); + tracker.increment_conn_count(); + tracker.increment_conn_count(); + assert_eq!( + tracker.get_conn_count(), + 2, + "should be 2 after single increment" + ); + tracker.decrement_conn_count()?; + assert_eq!( + tracker.get_conn_count(), + 1, + "should be 1 after two increments and one decrement" + ); + Ok(()) + } + #[test] + fn test_clone() { + let tracker = ClientPool::new(1); + let tracker_clone = tracker.clone(); + + tracker.increment_conn_count(); + assert_eq!( + tracker_clone.get_conn_count(), + 1, + "tracker clones should share the same count" + ); + } + + #[test] + fn test_conn_count_multiple_threads() { + let tracker = ClientPool::new(0); + let mut handles = vec![]; + + for _ in 0..10 { + let thread_tracker = tracker.clone(); + let handle = thread::spawn(move || { + thread_tracker.increment_conn_count(); + thread::sleep(std::time::Duration::from_millis(10)); + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!( + tracker.get_conn_count(), + 10, + "should be 10 after 10 thread increments" + ); + } + + #[test] + fn test_concurrent_increment_decrement() -> Result<()> { + let tracker = ClientPool::new(0); + let mut handles = vec![]; + + for i in 0..10 { + let thread_tracker = tracker.clone(); + let handle = thread::spawn(move || { + if i < 5 { + thread_tracker.increment_conn_count(); + } else { + thread_tracker.decrement_conn_count().unwrap(); + } + thread::sleep(std::time::Duration::from_millis(10)); + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!( + tracker.get_conn_count(), + 0, + "should be 0 after equal increments and decrements" + ); + Ok(()) + } + + #[test] + #[should_panic] + fn test_zero_floor() { + let tracker = ClientPool::new(0); + tracker.decrement_conn_count().unwrap(); + } + + #[test] + fn test_stress() { + let tracker = ClientPool::new(0); + let mut handles = vec![]; + let num_threads = 100; + + for _ in 0..num_threads { + let thread_tracker = tracker.clone(); + let handle = thread::spawn(move || { + for _ in 0..100 { + thread_tracker.increment_conn_count(); + thread::sleep(std::time::Duration::from_micros(1)); + thread_tracker.decrement_conn_count().unwrap(); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!( + tracker.get_conn_count(), + 0, + "should return to 0 after all increments and decrements" + ); + } +} diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs b/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs deleted file mode 100644 index 414c13e3d1c..00000000000 --- a/sdk/rust/nym-sdk/src/tcp_proxy/connection_tracker.rs +++ /dev/null @@ -1,168 +0,0 @@ -use anyhow::{bail, Result}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; - -// Used by the ProxyClient for tracking # of active clients created on new TCP connection -// TODO used by the connection pool for maintaining # of clients in pool +/- -#[derive(Debug)] -pub struct ConnectionTracker { - count: Arc, -} - -impl ConnectionTracker { - pub fn new() -> Self { - Self { - count: Arc::new(AtomicUsize::new(0)), - } - } - - pub fn increment(&self) { - self.count.fetch_add(1, Ordering::SeqCst); - } - - pub fn decrement(&self) -> Result<()> { - if self.get_count() == 0 { - bail!("count already 0"); - } - self.count.fetch_sub(1, Ordering::SeqCst); - Ok(()) - } - - pub fn get_count(&self) -> usize { - self.count.load(Ordering::SeqCst) - } -} - -impl Clone for ConnectionTracker { - fn clone(&self) -> Self { - Self { - count: Arc::clone(&self.count), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use anyhow::Result; - use std::thread; - - #[test] - fn test_increment_decrement() -> Result<()> { - let tracker = ConnectionTracker::new(); - tracker.increment(); - tracker.increment(); - assert_eq!(tracker.get_count(), 2, "should be 2 after single increment"); - tracker.decrement()?; - assert_eq!( - tracker.get_count(), - 1, - "should be 1 after two increments and one decrement" - ); - Ok(()) - } - - #[test] - fn test_clone() { - let tracker = ConnectionTracker::new(); - let tracker_clone = tracker.clone(); - - tracker.increment(); - assert_eq!( - tracker_clone.get_count(), - 1, - "tracker clones should share the same count" - ); - } - - #[test] - fn test_multiple_threads() { - let tracker = ConnectionTracker::new(); - let mut handles = vec![]; - - for _ in 0..10 { - let thread_tracker = tracker.clone(); - let handle = thread::spawn(move || { - thread_tracker.increment(); - thread::sleep(std::time::Duration::from_millis(10)); - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - assert_eq!( - tracker.get_count(), - 10, - "should be 10 after 10 thread increments" - ); - } - - #[test] - fn test_concurrent_increment_decrement() -> Result<()> { - let tracker = ConnectionTracker::new(); - let mut handles = vec![]; - - for i in 0..10 { - let thread_tracker = tracker.clone(); - let handle = thread::spawn(move || { - if i < 5 { - thread_tracker.increment(); - } else { - thread_tracker.decrement().unwrap(); - } - thread::sleep(std::time::Duration::from_millis(10)); - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - assert_eq!( - tracker.get_count(), - 0, - "should be 0 after equal increments and decrements" - ); - Ok(()) - } - - #[test] - #[should_panic] - fn test_zero_floor() { - let tracker = ConnectionTracker::new(); - tracker.decrement().unwrap(); - } - - #[test] - fn test_stress() { - let tracker = ConnectionTracker::new(); - let mut handles = vec![]; - let num_threads = 100; - - for _ in 0..num_threads { - let thread_tracker = tracker.clone(); - let handle = thread::spawn(move || { - for _ in 0..100 { - thread_tracker.increment(); - thread::sleep(std::time::Duration::from_micros(1)); - thread_tracker.decrement().unwrap(); - } - }); - handles.push(handle); - } - - for handle in handles { - handle.join().unwrap(); - } - - assert_eq!( - tracker.get_count(), - 0, - "should return to 0 after all increments and decrements" - ); - } -} diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 067dd14eeb2..dae4656046e 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -1,8 +1,10 @@ -use crate::mixnet::{IncludedSurbs, MixnetClientBuilder, MixnetMessageSender, NymNetworkDetails}; +use crate::mixnet::{ + IncludedSurbs, MixnetClient, MixnetClientBuilder, MixnetMessageSender, NymNetworkDetails, +}; use std::{sync::Arc, time::Duration}; -#[path = "connection_tracker.rs"] -mod connection_tracker; -use connection_tracker::ConnectionTracker; +#[path = "client_pool.rs"] +mod client_pool; +use client_pool::ClientPool; #[path = "utils.rs"] mod utils; use anyhow::Result; @@ -21,13 +23,14 @@ use utils::{MessageBuffer, Payload, ProxiedMessage}; const DEFAULT_CLOSE_TIMEOUT: u64 = 60; const DEFAULT_LISTEN_HOST: &str = "127.0.0.1"; const DEFAULT_LISTEN_PORT: &str = "8080"; +const DEFAULT_CLIENT_POOL_SIZE: usize = 4; pub struct NymProxyClient { server_address: Recipient, listen_address: String, listen_port: String, close_timeout: u64, - conn_tracker: ConnectionTracker, + conn_pool: ClientPool, } impl NymProxyClient { @@ -37,6 +40,7 @@ impl NymProxyClient { listen_port: &str, close_timeout: u64, env: Option, + default_client_amount: usize, ) -> Result { debug!("loading env file: {:?}", env); setup_env(env); // Defaults to mainnet if empty @@ -45,21 +49,21 @@ impl NymProxyClient { listen_address: listen_address.to_string(), listen_port: listen_port.to_string(), close_timeout, - conn_tracker: ConnectionTracker::new(), // Keep track of the number of active ephemeral clients that are in use + conn_pool: ClientPool::new(default_client_amount), }) } // server_address is the Nym address of the NymProxyServer to communicate with. pub async fn new_with_defaults(server_address: Recipient, env: Option) -> Result { - debug!("loading env file: {:?}", env); - setup_env(env); // Defaults to mainnet if empty - Ok(NymProxyClient { + NymProxyClient::new( server_address, - listen_address: DEFAULT_LISTEN_HOST.to_string(), - listen_port: DEFAULT_LISTEN_PORT.to_string(), - close_timeout: DEFAULT_CLOSE_TIMEOUT, - conn_tracker: ConnectionTracker::new(), - }) + DEFAULT_LISTEN_HOST, + DEFAULT_LISTEN_PORT, + DEFAULT_CLOSE_TIMEOUT, + env, + DEFAULT_CLIENT_POOL_SIZE, + ) + .await } pub async fn run(&self) -> Result<()> { @@ -68,22 +72,27 @@ impl NymProxyClient { let listener = TcpListener::bind(format!("{}:{}", self.listen_address, self.listen_port)).await?; - let overall_counter = self.conn_tracker.clone(); + let client_maker = self.conn_pool.clone(); + tokio::spawn(async move { client_maker.start().await.unwrap() }); + + let overall_counter = self.conn_pool.clone(); tokio::spawn(async move { loop { - info!("active clients: {}", overall_counter.get_count()); + info!("active connections: {}", overall_counter.get_conn_count()); tokio::time::sleep(Duration::from_secs(5)).await; } }); loop { - let (stream, _) = listener.accept().await?; - tokio::spawn(NymProxyClient::handle_incoming( - stream, - self.server_address, - self.close_timeout, - self.conn_tracker.clone(), - )); + if self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 { + let (stream, _) = listener.accept().await?; + tokio::spawn(NymProxyClient::handle_incoming( + stream, + self.server_address, + self.close_timeout, + self.conn_pool.clone(), + )); + } } } @@ -104,7 +113,7 @@ impl NymProxyClient { stream: TcpStream, server_address: Recipient, close_timeout: u64, - conn_tracker: ConnectionTracker, + conn_pool: ClientPool, ) -> Result<()> { // ID for creation of session abstraction; new session ID per new connection accepted by our tcp listener above. let session_id = uuid::Uuid::new_v4(); @@ -112,31 +121,29 @@ impl NymProxyClient { // Used to communicate end of session between 'Outgoing' and 'Incoming' tasks let (tx, mut rx) = oneshot::channel(); - // Client creation can fail for multiple reasons like bad network connection: this loop just allows us to - // retry in a loop until we can successfully connect without having to restart the entire function info!("Starting session: {}", session_id); - info!("Creating client..."); - let mut client = loop { - let net = NymNetworkDetails::new_from_env(); - match MixnetClientBuilder::new_ephemeral() - .network_details(net) - .build()? - .connect_to_mixnet() - .await - { - Ok(client) => break client, - Err(err) => { - warn!("Error creating client: {:?}, will retry in 100ms", err); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } + + let mut client: MixnetClient = match conn_pool.get_conn_count() >= DEFAULT_CLIENT_POOL_SIZE + { + true => { + debug!("grabbing client from pool"); + conn_pool.get_mixnet_client().await? + } + false => { + debug!("not enough clients in pool, creating ephemeral client"); + let net = NymNetworkDetails::new_from_env(); + MixnetClientBuilder::new_ephemeral() + .network_details(net) + .build()? + .connect_to_mixnet() + .await? } }; - let client_addr = &client.nym_address(); - info!("Client created: {}", &client_addr); - conn_tracker.increment(); + + conn_pool.increment_conn_count(); info!( - "New connection - current active clients: {}", - conn_tracker.get_count() + "New connection - current active connections: {}", + conn_pool.get_conn_count() ); // Split our tcpstream into OwnedRead and OwnedWrite halves for concurrent read/writing @@ -235,11 +242,12 @@ impl NymProxyClient { _ = tokio::time::sleep(tokio::time::Duration::from_secs(close_timeout)) => { info!(" Closing write end of session: {}", session_id); info!(" Triggering client shutdown"); + // TODO change this to be a fn in the conn_pool, disconnect and remove from vec client.disconnect().await; - conn_tracker.clone().decrement()?; + conn_pool.clone().decrement_conn_count()?; info!( - "Dropped connection - current active clients: {}", - conn_tracker.get_count() + "Dropped connection - current active connections: {}", + conn_pool.get_conn_count() ); return Ok::<(), anyhow::Error>(()) } @@ -250,3 +258,5 @@ impl NymProxyClient { Ok(()) } } + +// TODO tests From 8e8eceb894e40578b96af6ff20f11fb013b36d7a Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 26 Nov 2024 00:11:06 +0100 Subject: [PATCH 010/102] fix flipped lowerthan waiting for pool --- sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs | 12 ++---------- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs index 24f9ac42be8..41412dfc29c 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs @@ -14,10 +14,6 @@ pub struct ClientPool { clients: Arc>>>, default_pool_size: usize, conn_count: Arc, // the actual # of connections running, denoting an incoming tcp request that is matched with a nym client - ready: bool, // default false, trigger true when clients.len() is default_pool_size/2 (currently arbitrary decision for 'floor' of ready clients) - // - // TODO not sure we need this, can just rely on trying to get a client and if there is none available then make one - // incoming_conns: Arc, // if we have incoming tcp connections that aren't matched: this will be used in order to work out the tcp conn 'backpressure' and to work out if we need to add clients to the pool } impl fmt::Debug for ClientPool { @@ -50,8 +46,7 @@ impl fmt::Debug for ClientPool { debug_struct .field("default_pool_size", &self.default_pool_size) .field("connection count", &*self.conn_count) - .field("clients", &format_args!("[{}]", clients_debug)) - .field("ready", &self.ready); + .field("clients", &format_args!("[{}]", clients_debug)); debug_struct.finish() } @@ -63,8 +58,6 @@ impl ClientPool { clients: Arc::new(RwLock::new(Vec::new())), default_pool_size, conn_count: Arc::new(AtomicUsize::new(0)), - // incoming_conns: Arc::new(AtomicUsize::new(0)), // see comment above - ready: false, } } @@ -135,12 +128,11 @@ impl ClientPool { clients: Arc::clone(&self.clients), default_pool_size: *&self.default_pool_size, conn_count: Arc::clone(&self.conn_count), - // incoming_conns: Arc::clone(&self.incoming_conns), - ready: self.ready.clone(), } } } +// TODO COVER ALL FNS #[cfg(test)] mod tests { use super::*; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index dae4656046e..f75c52cd5ec 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -123,7 +123,7 @@ impl NymProxyClient { info!("Starting session: {}", session_id); - let mut client: MixnetClient = match conn_pool.get_conn_count() >= DEFAULT_CLIENT_POOL_SIZE + let mut client: MixnetClient = match conn_pool.get_conn_count() <= DEFAULT_CLIENT_POOL_SIZE { true => { debug!("grabbing client from pool"); From 415c30fcc9383dcc006f90fd8f4a1e9848cdfaea Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 26 Nov 2024 12:13:13 +0100 Subject: [PATCH 011/102] first pass disconnect --- .../examples/tcp_proxy_single_connection.rs | 13 ++++++--- sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs | 28 ++++++++++++------- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 27 ++++++++++-------- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index 5fcea31267b..402523c5e21 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -31,7 +31,6 @@ struct ExampleMessage { #[tokio::main] async fn main() -> anyhow::Result<()> { // Keep track of sent/received messages - // let counter = Arc::new(Mutex::new(0)); let counter = AtomicU8::new(0); // Comment this out to just see println! statements from this example, as Nym client logging is very informative but quite verbose. @@ -60,9 +59,15 @@ async fn main() -> anyhow::Result<()> { // We'll run the instance with a long timeout since we're sending everything down the same Tcp connection, so should be using a single session. // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. - let proxy_client = - tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 60, Some(env)) - .await?; + let proxy_client = tcp_proxy::NymProxyClient::new( + *proxy_nym_addr, + "127.0.0.1", + &client_port, + 20, + Some(env), + 2, + ) + .await?; tokio::spawn(async move { proxy_server.run_with_shutdown().await?; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs index 41412dfc29c..d7d8c5f7bc2 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs @@ -5,13 +5,14 @@ use std::fmt; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tokio::sync::RwLock; +use tokio::sync::Semaphore; use tracing::{debug, info, instrument, warn}; // Make a set # of clients (low default) -// Make sure that # of clients is always 1 above the # of incoming conn requests // Once a client is used, kill the client & remove it from the pool pub struct ClientPool { clients: Arc>>>, + semaphore: Arc, default_pool_size: usize, conn_count: Arc, // the actual # of connections running, denoting an incoming tcp request that is matched with a nym client } @@ -56,20 +57,20 @@ impl ClientPool { pub fn new(default_pool_size: usize) -> Self { ClientPool { clients: Arc::new(RwLock::new(Vec::new())), + semaphore: Arc::new(Semaphore::new(default_pool_size)), default_pool_size, conn_count: Arc::new(AtomicUsize::new(0)), } } - // if clients == default, sleep - // if incoming conns > default - conn_count (aka in use clients) then make more clients pub async fn start(&self) -> Result<()> { loop { + // TODO double check this.. let spawned_clients = self.clients.read().await.len(); - info!("Currently spawned clients: {}", spawned_clients); + debug!("Currently spawned clients: {}", spawned_clients); if spawned_clients >= self.default_pool_size { - debug!("got enough clients already: sleeping"); + debug!("Got enough clients already: sleeping"); } else { info!("Spawning new client"); let client = loop { @@ -93,14 +94,20 @@ impl ClientPool { } } - pub async fn get_mixnet_client(&self) -> Result { - todo!() + pub async fn get_mixnet_client(&self) -> Option { + let _permit = self.semaphore.acquire().await.ok()?; + let mut clients = self.clients.write().await; + clients + .pop() + .and_then(|arc_client| Arc::try_unwrap(arc_client).ok()) } - // disconnect ephemeral - // remove from vec pub async fn disconnect_and_remove_client(&self, client: MixnetClient) -> Result<()> { - todo!() + let mut clients = self.clients.write().await; + clients.retain(|arc_client| arc_client.as_ref().nym_address() != client.nym_address()); + client.disconnect().await; + self.semaphore.add_permits(1); + Ok(()) } pub async fn get_client_count(&self) -> usize { @@ -126,6 +133,7 @@ impl ClientPool { pub fn clone(&self) -> Self { Self { clients: Arc::clone(&self.clients), + semaphore: Arc::clone(&self.semaphore), default_pool_size: *&self.default_pool_size, conn_count: Arc::clone(&self.conn_count), } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index f75c52cd5ec..e893f796150 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -7,7 +7,7 @@ mod client_pool; use client_pool::ClientPool; #[path = "utils.rs"] mod utils; -use anyhow::Result; +use anyhow::{bail, Result}; use dashmap::DashSet; use nym_network_defaults::setup_env; use nym_sphinx::addressing::Recipient; @@ -42,7 +42,7 @@ impl NymProxyClient { env: Option, default_client_amount: usize, ) -> Result { - debug!("loading env file: {:?}", env); + debug!("Loading env file: {:?}", env); setup_env(env); // Defaults to mainnet if empty Ok(NymProxyClient { server_address, @@ -78,13 +78,16 @@ impl NymProxyClient { let overall_counter = self.conn_pool.clone(); tokio::spawn(async move { loop { - info!("active connections: {}", overall_counter.get_conn_count()); + info!("Active connections: {}", overall_counter.get_conn_count()); tokio::time::sleep(Duration::from_secs(5)).await; } }); + // if self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 { loop { - if self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 { + if DEFAULT_CLIENT_POOL_SIZE == 1 && self.conn_pool.get_client_count().await == 1 + || self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 + { let (stream, _) = listener.accept().await?; tokio::spawn(NymProxyClient::handle_incoming( stream, @@ -123,14 +126,13 @@ impl NymProxyClient { info!("Starting session: {}", session_id); - let mut client: MixnetClient = match conn_pool.get_conn_count() <= DEFAULT_CLIENT_POOL_SIZE - { - true => { - debug!("grabbing client from pool"); - conn_pool.get_mixnet_client().await? + let mut client = match conn_pool.get_mixnet_client().await { + Some(client) => { + info!("Grabbing client {} from pool", client.nym_address()); + client } - false => { - debug!("not enough clients in pool, creating ephemeral client"); + None => { + info!("Not enough clients in pool, creating ephemeral client"); let net = NymNetworkDetails::new_from_env(); MixnetClientBuilder::new_ephemeral() .network_details(net) @@ -243,7 +245,8 @@ impl NymProxyClient { info!(" Closing write end of session: {}", session_id); info!(" Triggering client shutdown"); // TODO change this to be a fn in the conn_pool, disconnect and remove from vec - client.disconnect().await; + // client.disconnect().await; + conn_pool.disconnect_and_remove_client(client).await?; conn_pool.clone().decrement_conn_count()?; info!( "Dropped connection - current active connections: {}", From 6f4421dc4a815210f588ac76b3b57dacd28c90ff Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 26 Nov 2024 13:50:15 +0100 Subject: [PATCH 012/102] trying to get disconnect to remove from pool --- .../examples/tcp_proxy_single_connection.rs | 12 +++--------- sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs | 18 +++++++++++------- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 2 -- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index 402523c5e21..cfe95466afe 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -59,15 +59,9 @@ async fn main() -> anyhow::Result<()> { // We'll run the instance with a long timeout since we're sending everything down the same Tcp connection, so should be using a single session. // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. - let proxy_client = tcp_proxy::NymProxyClient::new( - *proxy_nym_addr, - "127.0.0.1", - &client_port, - 20, - Some(env), - 2, - ) - .await?; + let proxy_client = + tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 2) + .await?; tokio::spawn(async move { proxy_server.run_with_shutdown().await?; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs index d7d8c5f7bc2..2ffc2785e09 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs @@ -1,19 +1,18 @@ use crate::mixnet::{MixnetClient, MixnetClientBuilder, NymNetworkDetails}; use anyhow::{bail, Result}; -use std::collections::VecDeque; use std::fmt; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tokio::sync::RwLock; use tokio::sync::Semaphore; -use tracing::{debug, info, instrument, warn}; +use tracing::{debug, info, warn}; // Make a set # of clients (low default) // Once a client is used, kill the client & remove it from the pool pub struct ClientPool { clients: Arc>>>, semaphore: Arc, - default_pool_size: usize, + default_pool_size: usize, // default # of clients to have running simultaneously, otherwise make ephemeral conn_count: Arc, // the actual # of connections running, denoting an incoming tcp request that is matched with a nym client } @@ -65,11 +64,15 @@ impl ClientPool { pub async fn start(&self) -> Result<()> { loop { - // TODO double check this.. let spawned_clients = self.clients.read().await.len(); - debug!("Currently spawned clients: {}", spawned_clients); + let addresses = self; + info!( + "Currently spawned clients: {}: {:?} ", + spawned_clients, addresses + ); - if spawned_clients >= self.default_pool_size { + // TODO check this logic in situation with larger pool + if spawned_clients == self.semaphore.available_permits() { debug!("Got enough clients already: sleeping"); } else { info!("Spawning new client"); @@ -102,11 +105,12 @@ impl ClientPool { .and_then(|arc_client| Arc::try_unwrap(arc_client).ok()) } + // TODO why is it not being removed? just passed back into pool? pub async fn disconnect_and_remove_client(&self, client: MixnetClient) -> Result<()> { let mut clients = self.clients.write().await; + self.semaphore.add_permits(1); clients.retain(|arc_client| arc_client.as_ref().nym_address() != client.nym_address()); client.disconnect().await; - self.semaphore.add_permits(1); Ok(()) } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index e893f796150..0c1a3a03e92 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -244,8 +244,6 @@ impl NymProxyClient { _ = tokio::time::sleep(tokio::time::Duration::from_secs(close_timeout)) => { info!(" Closing write end of session: {}", session_id); info!(" Triggering client shutdown"); - // TODO change this to be a fn in the conn_pool, disconnect and remove from vec - // client.disconnect().await; conn_pool.disconnect_and_remove_client(client).await?; conn_pool.clone().decrement_conn_count()?; info!( From 5083b420ecdefaec67ada79c100e79384391563d Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 26 Nov 2024 15:30:48 +0100 Subject: [PATCH 013/102] remove double accounting for moment --- .../examples/tcp_proxy_single_connection.rs | 2 +- sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs | 268 +++++++++--------- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 35 ++- 3 files changed, 154 insertions(+), 151 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index cfe95466afe..5222d90f989 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -60,7 +60,7 @@ async fn main() -> anyhow::Result<()> { // We'll run the instance with a long timeout since we're sending everything down the same Tcp connection, so should be using a single session. // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. let proxy_client = - tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 2) + tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 1) .await?; tokio::spawn(async move { diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs index 2ffc2785e09..f6f35f26552 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs @@ -12,8 +12,7 @@ use tracing::{debug, info, warn}; pub struct ClientPool { clients: Arc>>>, semaphore: Arc, - default_pool_size: usize, // default # of clients to have running simultaneously, otherwise make ephemeral - conn_count: Arc, // the actual # of connections running, denoting an incoming tcp request that is matched with a nym client + default_pool_size: usize, // default # of clients to have available in pool. If incoming tcp requests > this, make ephemeral client elsewhere } impl fmt::Debug for ClientPool { @@ -45,7 +44,7 @@ impl fmt::Debug for ClientPool { let mut debug_struct = f.debug_struct("Pool"); debug_struct .field("default_pool_size", &self.default_pool_size) - .field("connection count", &*self.conn_count) + // .field("connection count", &*self.conn_count) .field("clients", &format_args!("[{}]", clients_debug)); debug_struct.finish() @@ -58,7 +57,7 @@ impl ClientPool { clients: Arc::new(RwLock::new(Vec::new())), semaphore: Arc::new(Semaphore::new(default_pool_size)), default_pool_size, - conn_count: Arc::new(AtomicUsize::new(0)), + // conn_count: Arc::new(AtomicUsize::new(0)), } } @@ -70,8 +69,12 @@ impl ClientPool { "Currently spawned clients: {}: {:?} ", spawned_clients, addresses ); + info!( + "current avail permits {}", + self.semaphore.available_permits() + ); - // TODO check this logic in situation with larger pool + // TODO FIX THIS if spawned_clients == self.semaphore.available_permits() { debug!("Got enough clients already: sleeping"); } else { @@ -118,158 +121,159 @@ impl ClientPool { self.clients.read().await.len() } - pub fn get_conn_count(&self) -> usize { - self.conn_count.load(Ordering::SeqCst) - } + // pub fn get_conn_count(&self) -> usize { + // // self.conn_count.load(Ordering::SeqCst) + // self.default_pool_size - self.semaphore.available_permits() + // } - pub fn increment_conn_count(&self) { - self.conn_count.fetch_add(1, Ordering::SeqCst); - } + // pub fn increment_conn_count(&self) { + // self.conn_count.fetch_add(1, Ordering::SeqCst); + // } - pub fn decrement_conn_count(&self) -> Result<()> { - if self.get_conn_count() == 0 { - bail!("count already 0"); - } - self.conn_count.fetch_sub(1, Ordering::SeqCst); - Ok(()) - } + // pub fn decrement_conn_count(&self) -> Result<()> { + // if self.get_conn_count() == 0 { + // bail!("count already 0"); + // } + // self.conn_count.fetch_sub(1, Ordering::SeqCst); + // Ok(()) + // } pub fn clone(&self) -> Self { Self { clients: Arc::clone(&self.clients), semaphore: Arc::clone(&self.semaphore), default_pool_size: *&self.default_pool_size, - conn_count: Arc::clone(&self.conn_count), + // conn_count: Arc::clone(&self.conn_count), } } } // TODO COVER ALL FNS -#[cfg(test)] -mod tests { - use super::*; - use anyhow::Result; - use std::thread; +// #[cfg(test)] +// mod tests { +// use super::*; +// use anyhow::Result; +// use std::thread; - #[test] - fn test_conn_count_increment_decrement() -> Result<()> { - let tracker = ClientPool::new(0); - tracker.increment_conn_count(); - tracker.increment_conn_count(); - assert_eq!( - tracker.get_conn_count(), - 2, - "should be 2 after single increment" - ); - tracker.decrement_conn_count()?; - assert_eq!( - tracker.get_conn_count(), - 1, - "should be 1 after two increments and one decrement" - ); - Ok(()) - } - #[test] - fn test_clone() { - let tracker = ClientPool::new(1); - let tracker_clone = tracker.clone(); +// #[test] +// fn test_conn_count_increment_decrement() -> Result<()> { +// let tracker = ClientPool::new(0); +// tracker.increment_conn_count(); +// tracker.increment_conn_count(); +// assert_eq!( +// tracker.get_conn_count(), +// 2, +// "should be 2 after single increment" +// ); +// tracker.decrement_conn_count()?; +// assert_eq!( +// tracker.get_conn_count(), +// 1, +// "should be 1 after two increments and one decrement" +// ); +// Ok(()) +// } +// #[test] +// fn test_clone() { +// let tracker = ClientPool::new(1); +// let tracker_clone = tracker.clone(); - tracker.increment_conn_count(); - assert_eq!( - tracker_clone.get_conn_count(), - 1, - "tracker clones should share the same count" - ); - } +// tracker.increment_conn_count(); +// assert_eq!( +// tracker_clone.get_conn_count(), +// 1, +// "tracker clones should share the same count" +// ); +// } - #[test] - fn test_conn_count_multiple_threads() { - let tracker = ClientPool::new(0); - let mut handles = vec![]; +// #[test] +// fn test_conn_count_multiple_threads() { +// let tracker = ClientPool::new(0); +// let mut handles = vec![]; - for _ in 0..10 { - let thread_tracker = tracker.clone(); - let handle = thread::spawn(move || { - thread_tracker.increment_conn_count(); - thread::sleep(std::time::Duration::from_millis(10)); - }); - handles.push(handle); - } +// for _ in 0..10 { +// let thread_tracker = tracker.clone(); +// let handle = thread::spawn(move || { +// thread_tracker.increment_conn_count(); +// thread::sleep(std::time::Duration::from_millis(10)); +// }); +// handles.push(handle); +// } - for handle in handles { - handle.join().unwrap(); - } +// for handle in handles { +// handle.join().unwrap(); +// } - assert_eq!( - tracker.get_conn_count(), - 10, - "should be 10 after 10 thread increments" - ); - } +// assert_eq!( +// tracker.get_conn_count(), +// 10, +// "should be 10 after 10 thread increments" +// ); +// } - #[test] - fn test_concurrent_increment_decrement() -> Result<()> { - let tracker = ClientPool::new(0); - let mut handles = vec![]; +// #[test] +// fn test_concurrent_increment_decrement() -> Result<()> { +// let tracker = ClientPool::new(0); +// let mut handles = vec![]; - for i in 0..10 { - let thread_tracker = tracker.clone(); - let handle = thread::spawn(move || { - if i < 5 { - thread_tracker.increment_conn_count(); - } else { - thread_tracker.decrement_conn_count().unwrap(); - } - thread::sleep(std::time::Duration::from_millis(10)); - }); - handles.push(handle); - } +// for i in 0..10 { +// let thread_tracker = tracker.clone(); +// let handle = thread::spawn(move || { +// if i < 5 { +// thread_tracker.increment_conn_count(); +// } else { +// thread_tracker.decrement_conn_count().unwrap(); +// } +// thread::sleep(std::time::Duration::from_millis(10)); +// }); +// handles.push(handle); +// } - for handle in handles { - handle.join().unwrap(); - } +// for handle in handles { +// handle.join().unwrap(); +// } - assert_eq!( - tracker.get_conn_count(), - 0, - "should be 0 after equal increments and decrements" - ); - Ok(()) - } +// assert_eq!( +// tracker.get_conn_count(), +// 0, +// "should be 0 after equal increments and decrements" +// ); +// Ok(()) +// } - #[test] - #[should_panic] - fn test_zero_floor() { - let tracker = ClientPool::new(0); - tracker.decrement_conn_count().unwrap(); - } +// #[test] +// #[should_panic] +// fn test_zero_floor() { +// let tracker = ClientPool::new(0); +// tracker.decrement_conn_count().unwrap(); +// } - #[test] - fn test_stress() { - let tracker = ClientPool::new(0); - let mut handles = vec![]; - let num_threads = 100; +// #[test] +// fn test_stress() { +// let tracker = ClientPool::new(0); +// let mut handles = vec![]; +// let num_threads = 100; - for _ in 0..num_threads { - let thread_tracker = tracker.clone(); - let handle = thread::spawn(move || { - for _ in 0..100 { - thread_tracker.increment_conn_count(); - thread::sleep(std::time::Duration::from_micros(1)); - thread_tracker.decrement_conn_count().unwrap(); - } - }); - handles.push(handle); - } +// for _ in 0..num_threads { +// let thread_tracker = tracker.clone(); +// let handle = thread::spawn(move || { +// for _ in 0..100 { +// thread_tracker.increment_conn_count(); +// thread::sleep(std::time::Duration::from_micros(1)); +// thread_tracker.decrement_conn_count().unwrap(); +// } +// }); +// handles.push(handle); +// } - for handle in handles { - handle.join().unwrap(); - } +// for handle in handles { +// handle.join().unwrap(); +// } - assert_eq!( - tracker.get_conn_count(), - 0, - "should return to 0 after all increments and decrements" - ); - } -} +// assert_eq!( +// tracker.get_conn_count(), +// 0, +// "should return to 0 after all increments and decrements" +// ); +// } +// } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 0c1a3a03e92..b521ec97a67 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -75,15 +75,14 @@ impl NymProxyClient { let client_maker = self.conn_pool.clone(); tokio::spawn(async move { client_maker.start().await.unwrap() }); - let overall_counter = self.conn_pool.clone(); - tokio::spawn(async move { - loop { - info!("Active connections: {}", overall_counter.get_conn_count()); - tokio::time::sleep(Duration::from_secs(5)).await; - } - }); + // let overall_counter = self.conn_pool.clone(); + // tokio::spawn(async move { + // loop { + // info!("Active connections: {}", overall_counter.get_conn_count()); + // tokio::time::sleep(Duration::from_secs(5)).await; + // } + // }); - // if self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 { loop { if DEFAULT_CLIENT_POOL_SIZE == 1 && self.conn_pool.get_client_count().await == 1 || self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 @@ -142,11 +141,11 @@ impl NymProxyClient { } }; - conn_pool.increment_conn_count(); - info!( - "New connection - current active connections: {}", - conn_pool.get_conn_count() - ); + // conn_pool.increment_conn_count(); + // info!( + // "New connection - current active connections: {}", + // conn_pool.get_conn_count() + // ); // Split our tcpstream into OwnedRead and OwnedWrite halves for concurrent read/writing let (read, mut write) = stream.into_split(); @@ -245,11 +244,11 @@ impl NymProxyClient { info!(" Closing write end of session: {}", session_id); info!(" Triggering client shutdown"); conn_pool.disconnect_and_remove_client(client).await?; - conn_pool.clone().decrement_conn_count()?; - info!( - "Dropped connection - current active connections: {}", - conn_pool.get_conn_count() - ); + // conn_pool.clone().decrement_conn_count()?; + // info!( + // "Dropped connection - current active connections: {}", + // conn_pool.get_conn_count() + // ); return Ok::<(), anyhow::Error>(()) } } From 4519bd657181715ddac25961f9f393061f333840 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 12:07:44 +0100 Subject: [PATCH 014/102] remove comments and reduce logging verbosity --- sdk/rust/nym-sdk/Cargo.toml | 2 +- .../nym-sdk/examples/tcp_proxy_multistream.rs | 13 +- .../examples/tcp_proxy_single_connection.rs | 11 +- sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs | 179 ++---------------- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 29 +-- .../nym-sdk/src/tcp_proxy/tcp_proxy_server.rs | 10 +- 6 files changed, 44 insertions(+), 200 deletions(-) diff --git a/sdk/rust/nym-sdk/Cargo.toml b/sdk/rust/nym-sdk/Cargo.toml index 6d8f360effa..0c79a0b7f27 100644 --- a/sdk/rust/nym-sdk/Cargo.toml +++ b/sdk/rust/nym-sdk/Cargo.toml @@ -57,7 +57,7 @@ uuid = { version = "1", features = ["v4", "serde"] } bincode = "1.0" serde = { version = "1", features = ["derive"] } tracing.workspace = true -tracing-subscriber = "0.3" +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } dirs.workspace = true [dev-dependencies] diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs index ef3a6654a93..3f0ddcd143a 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs @@ -10,6 +10,7 @@ use tokio::net::TcpStream; use tokio::signal; use tokio_stream::StreamExt; use tokio_util::codec; +use tracing_subscriber::{fmt, prelude::*, EnvFilter}; #[derive(Serialize, Deserialize, Debug)] struct ExampleMessage { @@ -34,8 +35,12 @@ async fn main() -> anyhow::Result<()> { // Nym client logging is very informative but quite verbose. // The Message Decay related logging gives you an ideas of the internals of the proxy message ordering: you need to switch // to DEBUG to see the contents of the msg buffer, sphinx packet chunking, etc. - tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) + // tracing_subscriber::fmt() + // .with_max_level(tracing::Level::INFO) + // .init(); + tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::new("nym_sdk::tcp_proxy=info")) .init(); let env_path = env::args().nth(2).expect("Env file not specified"); @@ -45,7 +50,7 @@ async fn main() -> anyhow::Result<()> { // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. let proxy_client = - tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 45, Some(env)).await?; + tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 45, Some(env), 5).await?; tokio::spawn(async move { proxy_client.run().await?; @@ -57,7 +62,7 @@ async fn main() -> anyhow::Result<()> { println!("done. sending bytes"); // In the info traces you will see the different session IDs being set up, one for each TcpStream. - for i in 0..4 { + for i in 0..10 { let conn_id = i; println!("Starting TCP connection {}", conn_id); let local_tcp_addr = format!("127.0.0.1:{}", listen_port.clone()); diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index 5222d90f989..965cd3e78d5 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -10,6 +10,7 @@ use tokio::net::{TcpListener, TcpStream}; use tokio::signal; use tokio_stream::StreamExt; use tokio_util::codec; +use tracing_subscriber::{fmt, prelude::*, EnvFilter}; #[derive(Serialize, Deserialize, Debug)] struct ExampleMessage { @@ -35,8 +36,12 @@ async fn main() -> anyhow::Result<()> { // Comment this out to just see println! statements from this example, as Nym client logging is very informative but quite verbose. // The Message Decay related logging gives you an ideas of the internals of the proxy message ordering. To see the contents of the msg buffer, sphinx packet chunking, etc change the tracing::Level to DEBUG. - tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) + // tracing_subscriber::fmt() + // .with_max_level(tracing::Level::INFO) + // .init(); + tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::new("nym_sdk::tcp_proxy=info")) .init(); let server_port = env::args() @@ -60,7 +65,7 @@ async fn main() -> anyhow::Result<()> { // We'll run the instance with a long timeout since we're sending everything down the same Tcp connection, so should be using a single session. // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. let proxy_client = - tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 1) + tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 3) .await?; tokio::spawn(async move { diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs index f6f35f26552..a2bd3217afe 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs @@ -1,7 +1,6 @@ use crate::mixnet::{MixnetClient, MixnetClientBuilder, NymNetworkDetails}; -use anyhow::{bail, Result}; +use anyhow::Result; use std::fmt; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tokio::sync::RwLock; use tokio::sync::Semaphore; @@ -44,9 +43,7 @@ impl fmt::Debug for ClientPool { let mut debug_struct = f.debug_struct("Pool"); debug_struct .field("default_pool_size", &self.default_pool_size) - // .field("connection count", &*self.conn_count) .field("clients", &format_args!("[{}]", clients_debug)); - debug_struct.finish() } } @@ -57,7 +54,6 @@ impl ClientPool { clients: Arc::new(RwLock::new(Vec::new())), semaphore: Arc::new(Semaphore::new(default_pool_size)), default_pool_size, - // conn_count: Arc::new(AtomicUsize::new(0)), } } @@ -69,12 +65,15 @@ impl ClientPool { "Currently spawned clients: {}: {:?} ", spawned_clients, addresses ); + // TODO PROBLEM IS HERE: not updating / tracking the in use permits when grab_mixnet_client is called info!( "current avail permits {}", self.semaphore.available_permits() ); - - // TODO FIX THIS + info!( + "current in use permits {}", + self.default_pool_size - self.semaphore.available_permits() + ); if spawned_clients == self.semaphore.available_permits() { debug!("Got enough clients already: sleeping"); } else { @@ -101,179 +100,31 @@ impl ClientPool { } pub async fn get_mixnet_client(&self) -> Option { - let _permit = self.semaphore.acquire().await.ok()?; + info!("Grabbing client from pool"); + let permit = self.semaphore.acquire().await; + info!("{permit:?}"); + info!("Available permits: {}", self.semaphore.available_permits()); let mut clients = self.clients.write().await; + // gain ownership of client, tracking with semaphore once its working to stop constantly renewing size of pool to default_pool_size and instead have pool be (default_pool_size - in use clients) to stop bloat clients .pop() .and_then(|arc_client| Arc::try_unwrap(arc_client).ok()) } - // TODO why is it not being removed? just passed back into pool? - pub async fn disconnect_and_remove_client(&self, client: MixnetClient) -> Result<()> { - let mut clients = self.clients.write().await; - self.semaphore.add_permits(1); - clients.retain(|arc_client| arc_client.as_ref().nym_address() != client.nym_address()); - client.disconnect().await; - Ok(()) - } + // pub async fn disconnect_and_remove_client(&self, client: MixnetClient) -> Result<()> { + // client.disconnect().await; + // Ok(()) + // } pub async fn get_client_count(&self) -> usize { self.clients.read().await.len() } - // pub fn get_conn_count(&self) -> usize { - // // self.conn_count.load(Ordering::SeqCst) - // self.default_pool_size - self.semaphore.available_permits() - // } - - // pub fn increment_conn_count(&self) { - // self.conn_count.fetch_add(1, Ordering::SeqCst); - // } - - // pub fn decrement_conn_count(&self) -> Result<()> { - // if self.get_conn_count() == 0 { - // bail!("count already 0"); - // } - // self.conn_count.fetch_sub(1, Ordering::SeqCst); - // Ok(()) - // } - pub fn clone(&self) -> Self { Self { clients: Arc::clone(&self.clients), semaphore: Arc::clone(&self.semaphore), default_pool_size: *&self.default_pool_size, - // conn_count: Arc::clone(&self.conn_count), } } } - -// TODO COVER ALL FNS -// #[cfg(test)] -// mod tests { -// use super::*; -// use anyhow::Result; -// use std::thread; - -// #[test] -// fn test_conn_count_increment_decrement() -> Result<()> { -// let tracker = ClientPool::new(0); -// tracker.increment_conn_count(); -// tracker.increment_conn_count(); -// assert_eq!( -// tracker.get_conn_count(), -// 2, -// "should be 2 after single increment" -// ); -// tracker.decrement_conn_count()?; -// assert_eq!( -// tracker.get_conn_count(), -// 1, -// "should be 1 after two increments and one decrement" -// ); -// Ok(()) -// } -// #[test] -// fn test_clone() { -// let tracker = ClientPool::new(1); -// let tracker_clone = tracker.clone(); - -// tracker.increment_conn_count(); -// assert_eq!( -// tracker_clone.get_conn_count(), -// 1, -// "tracker clones should share the same count" -// ); -// } - -// #[test] -// fn test_conn_count_multiple_threads() { -// let tracker = ClientPool::new(0); -// let mut handles = vec![]; - -// for _ in 0..10 { -// let thread_tracker = tracker.clone(); -// let handle = thread::spawn(move || { -// thread_tracker.increment_conn_count(); -// thread::sleep(std::time::Duration::from_millis(10)); -// }); -// handles.push(handle); -// } - -// for handle in handles { -// handle.join().unwrap(); -// } - -// assert_eq!( -// tracker.get_conn_count(), -// 10, -// "should be 10 after 10 thread increments" -// ); -// } - -// #[test] -// fn test_concurrent_increment_decrement() -> Result<()> { -// let tracker = ClientPool::new(0); -// let mut handles = vec![]; - -// for i in 0..10 { -// let thread_tracker = tracker.clone(); -// let handle = thread::spawn(move || { -// if i < 5 { -// thread_tracker.increment_conn_count(); -// } else { -// thread_tracker.decrement_conn_count().unwrap(); -// } -// thread::sleep(std::time::Duration::from_millis(10)); -// }); -// handles.push(handle); -// } - -// for handle in handles { -// handle.join().unwrap(); -// } - -// assert_eq!( -// tracker.get_conn_count(), -// 0, -// "should be 0 after equal increments and decrements" -// ); -// Ok(()) -// } - -// #[test] -// #[should_panic] -// fn test_zero_floor() { -// let tracker = ClientPool::new(0); -// tracker.decrement_conn_count().unwrap(); -// } - -// #[test] -// fn test_stress() { -// let tracker = ClientPool::new(0); -// let mut handles = vec![]; -// let num_threads = 100; - -// for _ in 0..num_threads { -// let thread_tracker = tracker.clone(); -// let handle = thread::spawn(move || { -// for _ in 0..100 { -// thread_tracker.increment_conn_count(); -// thread::sleep(std::time::Duration::from_micros(1)); -// thread_tracker.decrement_conn_count().unwrap(); -// } -// }); -// handles.push(handle); -// } - -// for handle in handles { -// handle.join().unwrap(); -// } - -// assert_eq!( -// tracker.get_conn_count(), -// 0, -// "should return to 0 after all increments and decrements" -// ); -// } -// } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index b521ec97a67..aba8865178b 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -75,15 +75,8 @@ impl NymProxyClient { let client_maker = self.conn_pool.clone(); tokio::spawn(async move { client_maker.start().await.unwrap() }); - // let overall_counter = self.conn_pool.clone(); - // tokio::spawn(async move { - // loop { - // info!("Active connections: {}", overall_counter.get_conn_count()); - // tokio::time::sleep(Duration::from_secs(5)).await; - // } - // }); - loop { + // TODO change this to a proper 'ready' state if DEFAULT_CLIENT_POOL_SIZE == 1 && self.conn_pool.get_client_count().await == 1 || self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 { @@ -110,7 +103,7 @@ impl NymProxyClient { // Then we spawn 2 tasks: // - 'Outgoing' thread => frames incoming bytes from OwnedReadHalf and pipe through the mixnet & trigger session close. // - 'Incoming' thread => orders incoming messages from the Mixnet via placing them in a MessageBuffer and using tick(), as well as manage session closing. - #[instrument] + #[instrument(skip(stream, server_address, close_timeout, conn_pool))] async fn handle_incoming( stream: TcpStream, server_address: Recipient, @@ -127,11 +120,11 @@ impl NymProxyClient { let mut client = match conn_pool.get_mixnet_client().await { Some(client) => { - info!("Grabbing client {} from pool", client.nym_address()); + info!("Grabbed client {} from pool", client.nym_address()); client } None => { - info!("Not enough clients in pool, creating ephemeral client"); + info!("IF YOU SEE THIS not enough clients in pool, creating ephemeral client"); let net = NymNetworkDetails::new_from_env(); MixnetClientBuilder::new_ephemeral() .network_details(net) @@ -141,12 +134,6 @@ impl NymProxyClient { } }; - // conn_pool.increment_conn_count(); - // info!( - // "New connection - current active connections: {}", - // conn_pool.get_conn_count() - // ); - // Split our tcpstream into OwnedRead and OwnedWrite halves for concurrent read/writing let (read, mut write) = stream.into_split(); // Since we're just trying to pipe whatever bytes our client/server are normally sending to each other, @@ -243,12 +230,8 @@ impl NymProxyClient { _ = tokio::time::sleep(tokio::time::Duration::from_secs(close_timeout)) => { info!(" Closing write end of session: {}", session_id); info!(" Triggering client shutdown"); - conn_pool.disconnect_and_remove_client(client).await?; - // conn_pool.clone().decrement_conn_count()?; - // info!( - // "Dropped connection - current active connections: {}", - // conn_pool.get_conn_count() - // ); + client.disconnect().await; + // conn_pool.disconnect_and_remove_client(client).await?; return Ok::<(), anyhow::Error>(()) } } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs index 3d5917efebe..bf1f47869ef 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs @@ -34,12 +34,12 @@ impl NymProxyServer { config_dir: &str, env: Option, ) -> Result { - info!(":: creating client..."); + info!("Creating client"); // We're wanting to build a client with a constant address, vs the ephemeral in-memory data storage of the NymProxyClient clients. // Following a builder pattern, having to manually connect to the mixnet below. let config_dir = PathBuf::from(config_dir); - debug!("loading env file: {:?}", env); + debug!("Loading env file: {:?}", env); setup_env(env); // Defaults to mainnet if empty let net = NymNetworkDetails::new_from_env(); let storage_paths = StoragePaths::new_from_dir(&config_dir)?; @@ -57,7 +57,7 @@ impl NymProxyServer { let (tx, rx) = tokio::sync::watch::channel::>(None); - info!(":: client created: {}", client.nym_address()); + info!("Client created: {}", client.nym_address()); Ok(NymProxyServer { upstream_address: upstream_address.to_string(), @@ -134,7 +134,7 @@ impl NymProxyServer { // - Send serialised reply => Mixnet via SURB. // - If tick() returns true, close session. while let Some(Ok(bytes)) = framed_read.next().await { - info!("server received {} bytes", bytes.len()); + info!("Server received {} bytes", bytes.len()); let reply = ProxiedMessage::new(Payload::Data(bytes.to_vec()), session_id, message_id); message_id += 1; @@ -176,7 +176,7 @@ impl NymProxyServer { let should_close = msg_buffer.tick(&mut write).await?; if should_close { - info!(":: Closing write end of session: {}", session_id); + info!("Closing write end of session: {}", session_id); break; } } From 699aa6116ad7d7c203159c1d537452dea974f40b Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 13:01:49 +0100 Subject: [PATCH 015/102] comments --- sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs | 1 + sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs | 1 + sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 2 -- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs index 3f0ddcd143a..3c37fd2e737 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs @@ -57,6 +57,7 @@ async fn main() -> anyhow::Result<()> { Ok::<(), anyhow::Error>(()) }); + // TODO change this to wait on the actual client to be ready (pool -> client ready state kickback via oneshot) println!("waiting for everything to be set up.."); tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; println!("done. sending bytes"); diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index 965cd3e78d5..66ee8848707 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -120,6 +120,7 @@ async fn main() -> anyhow::Result<()> { }); // Just wait for Nym clients to connect, TCP clients to bind, etc. + // TODO change this to wait on the actual client to be ready (pool -> client ready state kickback via oneshot) println!("waiting for everything to be set up.."); tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; println!("done. sending bytes"); diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index aba8865178b..c7237e4869b 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -241,5 +241,3 @@ impl NymProxyClient { Ok(()) } } - -// TODO tests From 42739597ed4c5560e55c1c4c0a4ec70a0e67ba46 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 13:39:12 +0100 Subject: [PATCH 016/102] removed a bunch of commenting from example --- .../nym-sdk/examples/tcp_proxy_multistream.rs | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs index 3c37fd2e737..697ac2071cb 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs @@ -35,9 +35,6 @@ async fn main() -> anyhow::Result<()> { // Nym client logging is very informative but quite verbose. // The Message Decay related logging gives you an ideas of the internals of the proxy message ordering: you need to switch // to DEBUG to see the contents of the msg buffer, sphinx packet chunking, etc. - // tracing_subscriber::fmt() - // .with_max_level(tracing::Level::INFO) - // .init(); tracing_subscriber::registry() .with(fmt::layer()) .with(EnvFilter::new("nym_sdk::tcp_proxy=info")) @@ -63,9 +60,8 @@ async fn main() -> anyhow::Result<()> { println!("done. sending bytes"); // In the info traces you will see the different session IDs being set up, one for each TcpStream. - for i in 0..10 { + for i in 0..5 { let conn_id = i; - println!("Starting TCP connection {}", conn_id); let local_tcp_addr = format!("127.0.0.1:{}", listen_port.clone()); tokio::spawn(async move { // Now the client and server proxies are running we can create and pipe traffic to/from @@ -96,12 +92,7 @@ async fn main() -> anyhow::Result<()> { .write_all(&serialised) .await .expect("couldn't write to stream"); - println!( - ">> client sent {}: {} bytes on conn {}", - &i, - msg.message_bytes.len(), - &conn_id - ); + println!(">> client sent msg {} on conn {}", &i, &conn_id); } Ok::<(), anyhow::Error>(()) }); @@ -113,15 +104,15 @@ async fn main() -> anyhow::Result<()> { while let Some(Ok(bytes)) = framed_read.next().await { match bincode::deserialize::(&bytes) { Ok(msg) => { - println!( - "<< client received {}: {} bytes on conn {}", - msg.message_id, - msg.message_bytes.len(), - msg.tcp_conn - ); + // println!( + // "<< client received {}: {} bytes on conn {}", + // msg.message_id, + // msg.message_bytes.len(), + // msg.tcp_conn + // ); reply_counter += 1; println!( - "tcp connection {} replies received {}/4", + "<< connection {} received reply {}/4", msg.tcp_conn, reply_counter ); } From 8fcea5f3501c363ac8e3972a1d03a8a0bc4e20de Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 13:39:25 +0100 Subject: [PATCH 017/102] removed a bunch of commenting from example --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index c7237e4869b..306bdb5ec93 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -76,7 +76,6 @@ impl NymProxyClient { tokio::spawn(async move { client_maker.start().await.unwrap() }); loop { - // TODO change this to a proper 'ready' state if DEFAULT_CLIENT_POOL_SIZE == 1 && self.conn_pool.get_client_count().await == 1 || self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 { From 4bb05f3698e8d700612b34cc4b7540d8db4e54c5 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 14:06:26 +0100 Subject: [PATCH 018/102] err handling conpool start --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 306bdb5ec93..2b2c0107b29 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -73,8 +73,12 @@ impl NymProxyClient { TcpListener::bind(format!("{}:{}", self.listen_address, self.listen_port)).await?; let client_maker = self.conn_pool.clone(); - tokio::spawn(async move { client_maker.start().await.unwrap() }); + tokio::spawn(async move { + client_maker.start().await?; + Ok::<(), anyhow::Error>(()) + }); + // TODO add 'ready' marker for consuming code loop { if DEFAULT_CLIENT_POOL_SIZE == 1 && self.conn_pool.get_client_count().await == 1 || self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 From 8c8e0857962b919b276f8f3698b20bd785fbdbe6 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 14:25:19 +0100 Subject: [PATCH 019/102] added notes for next features --- sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs | 1 + sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs index a2bd3217afe..d0f69ddfd00 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs @@ -111,6 +111,7 @@ impl ClientPool { .and_then(|arc_client| Arc::try_unwrap(arc_client).ok()) } + // This might still be needed if it needs to be called with a cancellation token in various threads. keeping for the moment // pub async fn disconnect_and_remove_client(&self, client: MixnetClient) -> Result<()> { // client.disconnect().await; // Ok(()) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 2b2c0107b29..027d039169d 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -78,6 +78,12 @@ impl NymProxyClient { Ok::<(), anyhow::Error>(()) }); + tokio::spawn(async move { + tokio::signal::ctrl_c().await.unwrap(); + info!("RECEIVED KILL SIGNAL"); + // TODO client pool kill function + }); + // TODO add 'ready' marker for consuming code loop { if DEFAULT_CLIENT_POOL_SIZE == 1 && self.conn_pool.get_client_count().await == 1 @@ -236,11 +242,10 @@ impl NymProxyClient { client.disconnect().await; // conn_pool.disconnect_and_remove_client(client).await?; return Ok::<(), anyhow::Error>(()) - } + }, } } }); - tokio::signal::ctrl_c().await?; Ok(()) } } From c49f7f91ed0084e0d86e46a6ebcee82c9bebd967 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 18:59:58 +0100 Subject: [PATCH 020/102] first version working --- .../nym-sdk/examples/tcp_proxy_multistream.rs | 23 +++------ sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs | 47 +++++++------------ .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 20 ++++---- 3 files changed, 36 insertions(+), 54 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs index 697ac2071cb..c1cc0123248 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs @@ -37,7 +37,7 @@ async fn main() -> anyhow::Result<()> { // to DEBUG to see the contents of the msg buffer, sphinx packet chunking, etc. tracing_subscriber::registry() .with(fmt::layer()) - .with(EnvFilter::new("nym_sdk::tcp_proxy=info")) + .with(EnvFilter::new("nym_sdk::tcp_proxy=warn")) .init(); let env_path = env::args().nth(2).expect("Env file not specified"); @@ -45,9 +45,9 @@ async fn main() -> anyhow::Result<()> { let listen_port = env::args().nth(3).expect("Port not specified"); - // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. + // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. The final argument is how many clients to keep in reserve in the client pool. let proxy_client = - tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 45, Some(env), 5).await?; + tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 45, Some(env), 2).await?; tokio::spawn(async move { proxy_client.run().await?; @@ -60,7 +60,7 @@ async fn main() -> anyhow::Result<()> { println!("done. sending bytes"); // In the info traces you will see the different session IDs being set up, one for each TcpStream. - for i in 0..5 { + for i in 0..8 { let conn_id = i; let local_tcp_addr = format!("127.0.0.1:{}", listen_port.clone()); tokio::spawn(async move { @@ -77,7 +77,7 @@ async fn main() -> anyhow::Result<()> { // Lets just send a bunch of messages to the server with variable delays between them, with a message and tcp connection ids to keep track of ordering on the server side (for illustrative purposes **only**; keeping track of anonymous replies is handled by the proxy under the hood with Single Use Reply Blocks (SURBs); for this illustration we want some kind of app-level message id, but irl most of the time you'll probably be parsing on e.g. the incoming response type instead) tokio::spawn(async move { - for i in 0..4 { + for i in 0..8 { let mut rng = SmallRng::from_entropy(); let delay: f64 = rng.gen_range(2.5..5.0); tokio::time::sleep(tokio::time::Duration::from_secs_f64(delay)).await; @@ -104,17 +104,8 @@ async fn main() -> anyhow::Result<()> { while let Some(Ok(bytes)) = framed_read.next().await { match bincode::deserialize::(&bytes) { Ok(msg) => { - // println!( - // "<< client received {}: {} bytes on conn {}", - // msg.message_id, - // msg.message_bytes.len(), - // msg.tcp_conn - // ); reply_counter += 1; - println!( - "<< connection {} received reply {}/4", - msg.tcp_conn, reply_counter - ); + println!("<< conn {} received {}/8", msg.tcp_conn, reply_counter); } Err(e) => { println!("<< client received something that wasn't an example message of {} bytes. error: {}", bytes.len(), e); @@ -137,7 +128,7 @@ async fn main() -> anyhow::Result<()> { // emulate a series of small messages followed by a closing larger one fn gen_bytes_fixed(i: usize) -> Vec { - let amounts = [10, 15, 50, 1000]; + let amounts = [10, 15, 50, 1000, 10, 15, 500, 2000]; let len = amounts[i]; let mut rng = rand::thread_rng(); (0..len).map(|_| rng.gen::()).collect() diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs index d0f69ddfd00..90667c1be07 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs @@ -3,15 +3,11 @@ use anyhow::Result; use std::fmt; use std::sync::Arc; use tokio::sync::RwLock; -use tokio::sync::Semaphore; use tracing::{debug, info, warn}; -// Make a set # of clients (low default) -// Once a client is used, kill the client & remove it from the pool pub struct ClientPool { - clients: Arc>>>, - semaphore: Arc, - default_pool_size: usize, // default # of clients to have available in pool. If incoming tcp requests > this, make ephemeral client elsewhere + clients: Arc>>>, // collection of clients waiting to be used which are popped off in get_mixnet_client() + client_pool_reserve_number: usize, // default # of clients to have available in pool in reserve waiting for incoming connections } impl fmt::Debug for ClientPool { @@ -42,18 +38,20 @@ impl fmt::Debug for ClientPool { let mut debug_struct = f.debug_struct("Pool"); debug_struct - .field("default_pool_size", &self.default_pool_size) + .field( + "client_pool_reserve_number", + &self.client_pool_reserve_number, + ) .field("clients", &format_args!("[{}]", clients_debug)); debug_struct.finish() } } impl ClientPool { - pub fn new(default_pool_size: usize) -> Self { + pub fn new(client_pool_reserve_number: usize) -> Self { ClientPool { clients: Arc::new(RwLock::new(Vec::new())), - semaphore: Arc::new(Semaphore::new(default_pool_size)), - default_pool_size, + client_pool_reserve_number, } } @@ -61,23 +59,17 @@ impl ClientPool { loop { let spawned_clients = self.clients.read().await.len(); let addresses = self; - info!( + debug!( "Currently spawned clients: {}: {:?} ", spawned_clients, addresses ); - // TODO PROBLEM IS HERE: not updating / tracking the in use permits when grab_mixnet_client is called - info!( - "current avail permits {}", - self.semaphore.available_permits() - ); - info!( - "current in use permits {}", - self.default_pool_size - self.semaphore.available_permits() - ); - if spawned_clients == self.semaphore.available_permits() { + if spawned_clients >= self.client_pool_reserve_number { debug!("Got enough clients already: sleeping"); } else { - info!("Spawning new client"); + info!( + "Clients in reserve = {}, reserve amount = {}, spawning new client", + spawned_clients, self.client_pool_reserve_number + ); let client = loop { let net = NymNetworkDetails::new_from_env(); match MixnetClientBuilder::new_ephemeral() @@ -95,17 +87,13 @@ impl ClientPool { }; self.clients.write().await.push(Arc::new(client)); } - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; } } pub async fn get_mixnet_client(&self) -> Option { - info!("Grabbing client from pool"); - let permit = self.semaphore.acquire().await; - info!("{permit:?}"); - info!("Available permits: {}", self.semaphore.available_permits()); + debug!("Grabbing client from pool"); let mut clients = self.clients.write().await; - // gain ownership of client, tracking with semaphore once its working to stop constantly renewing size of pool to default_pool_size and instead have pool be (default_pool_size - in use clients) to stop bloat clients .pop() .and_then(|arc_client| Arc::try_unwrap(arc_client).ok()) @@ -124,8 +112,7 @@ impl ClientPool { pub fn clone(&self) -> Self { Self { clients: Arc::clone(&self.clients), - semaphore: Arc::clone(&self.semaphore), - default_pool_size: *&self.default_pool_size, + client_pool_reserve_number: *&self.client_pool_reserve_number, } } } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 027d039169d..0ebc0fa3eea 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -1,13 +1,11 @@ -use crate::mixnet::{ - IncludedSurbs, MixnetClient, MixnetClientBuilder, MixnetMessageSender, NymNetworkDetails, -}; -use std::{sync::Arc, time::Duration}; +use crate::mixnet::{IncludedSurbs, MixnetClientBuilder, MixnetMessageSender, NymNetworkDetails}; +use std::sync::Arc; #[path = "client_pool.rs"] mod client_pool; use client_pool::ClientPool; #[path = "utils.rs"] mod utils; -use anyhow::{bail, Result}; +use anyhow::Result; use dashmap::DashSet; use nym_network_defaults::setup_env; use nym_sphinx::addressing::Recipient; @@ -85,6 +83,7 @@ impl NymProxyClient { }); // TODO add 'ready' marker for consuming code + loop { if DEFAULT_CLIENT_POOL_SIZE == 1 && self.conn_pool.get_client_count().await == 1 || self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 @@ -133,13 +132,18 @@ impl NymProxyClient { client } None => { - info!("IF YOU SEE THIS not enough clients in pool, creating ephemeral client"); + warn!("Not enough clients in pool, creating ephemeral client"); let net = NymNetworkDetails::new_from_env(); - MixnetClientBuilder::new_ephemeral() + let client = MixnetClientBuilder::new_ephemeral() .network_details(net) .build()? .connect_to_mixnet() - .await? + .await?; + warn!( + "Using {} for the moment, created outside of the connection pool", + client.nym_address() + ); + client } }; From f87f57bddd5fd7de891528b324085c8ded5c1bec Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 22:02:26 +0100 Subject: [PATCH 021/102] first pass spin out client_pool --- sdk/rust/nym-sdk/src/client_pool.rs | 8 + .../{tcp_proxy => client_pool}/client_pool.rs | 0 sdk/rust/nym-sdk/src/lib.rs | 2 + sdk/rust/nym-sdk/src/tcp_proxy.rs | 205 ------------------ .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 4 +- 5 files changed, 11 insertions(+), 208 deletions(-) create mode 100644 sdk/rust/nym-sdk/src/client_pool.rs rename sdk/rust/nym-sdk/src/{tcp_proxy => client_pool}/client_pool.rs (100%) diff --git a/sdk/rust/nym-sdk/src/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool.rs new file mode 100644 index 00000000000..ddac9b3ff79 --- /dev/null +++ b/sdk/rust/nym-sdk/src/client_pool.rs @@ -0,0 +1,8 @@ +// Copyright 2023 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +// TODO EXAMPLE ONCE FINISHED + +mod client_pool; + +pub use client_pool::ClientPool; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs similarity index 100% rename from sdk/rust/nym-sdk/src/tcp_proxy/client_pool.rs rename to sdk/rust/nym-sdk/src/client_pool/client_pool.rs diff --git a/sdk/rust/nym-sdk/src/lib.rs b/sdk/rust/nym-sdk/src/lib.rs index 22e954e926f..8615645cce4 100644 --- a/sdk/rust/nym-sdk/src/lib.rs +++ b/sdk/rust/nym-sdk/src/lib.rs @@ -2,10 +2,12 @@ //! //! The main component currently is [`mixnet`]. //! [`tcp_proxy`] is probably a good place to start for anyone wanting to integrate with existing app code and read/write from a socket. +//! [`client_pool`] is a configurable client pool. mod error; pub mod bandwidth; +pub mod client_pool; pub mod mixnet; pub mod tcp_proxy; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy.rs b/sdk/rust/nym-sdk/src/tcp_proxy.rs index af71dc6f0fc..823c98c8d67 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy.rs @@ -1,212 +1,7 @@ -//! Proxy abstractions for interacting with the mixnet like a tcp socket //! -//! -//! # Basic example -//! -//! ```no_run -//! use bincode; -//! use dirs; -//! use nym_sdk::tcp_proxy; -//! use rand::rngs::SmallRng; -//! use rand::{Rng, SeedableRng}; -//! use serde::{Deserialize, Serialize}; -//! use std::env; -//! use std::fs; -//! use std::sync::atomic::{AtomicU8, Ordering}; -//! use tokio::io::AsyncWriteExt; -//! use tokio::net::{TcpListener, TcpStream}; -//! use tokio::signal; -//! use tokio_stream::StreamExt; -//! use tokio_util::codec; -//! use tracing_subscriber; -//! -//! #[derive(Serialize, Deserialize, Debug)] -//! struct ExampleMessage { -//! message_id: i8, -//! message_bytes: Vec, -//! } -//! -//! // This is a basic example which opens a single TCP connection //! and writes a bunch of messages between a client and an echo -//! // server, so only uses a single session under the hood and //! doesn't really show off the message ordering capabilities; this is mainly -//! // just a quick introductory illustration on how: -//! // - the mixnet does message ordering -//! // - the NymProxyClient and NymProxyServer can be hooked into //! and used to communicate between two otherwise pretty vanilla TcpStreams -//! // -//! // For a more irl example checkout tcp_proxy_multistream.rs -//! // -//! // Run this with: -//! // `cargo run --example tcp_proxy_single_connection ` e.g. -//! // `cargo run --example tcp_proxy_single_connection 8081 ../../../envs/canary.env 8080 ` -//! #[tokio::main] -//! async fn main() -> anyhow::Result<()> { -//! // Keep track of sent/received messages -//! // let counter = Arc::new(Mutex::new(0)); -//! let counter = AtomicU8::new(0); -//! -//! // Comment this out to just see println! statements from this example, as Nym client logging is very informative but quite verbose. -//! // The Message Decay related logging gives you an ideas of the internals of the proxy message ordering. To see the contents of the msg buffer, sphinx packet chunking, etc change the tracing::Level to DEBUG. -//! tracing_subscriber::fmt() -//! .with_max_level(tracing::Level::INFO) -//! .init(); -//! -//! let server_port = env::args() -//! .nth(1) -//! .expect("Server listen port not specified"); -//! let upstream_tcp_addr = format!("127.0.0.1:{}", server_port); -//! -//! // This dir gets cleaned up at the end: NOTE if you switch env between tests without letting the file do the automatic cleanup, make sure to manually remove this directory up before running again, otherwise your client will attempt to use these keys for the new env -//! let home_dir = dirs::home_dir().expect("Unable to get home directory"); -//! let conf_path = format!("{}/tmp/nym-proxy-server-config", home_dir.display()); -//! -//! let env_path = env::args().nth(2).expect("Env file not specified"); -//! let env = env_path.to_string(); -//! let client_port = env::args().nth(3).expect("Port not specified"); -//! -//! let mut proxy_server = -//! tcp_proxy::NymProxyServer::new(&upstream_tcp_addr, &conf_path, Some(env_path.clone())) -//! .await?; -//! let proxy_nym_addr = proxy_server.nym_address(); -//! -//! // We'll run the instance with a long timeout since we're sending everything down the same Tcp connection, so should be using a single session. -//! // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. -//! let proxy_client = -//! tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 60, Some(env)) -//! .await?; -//! -//! tokio::spawn(async move { -//! let _ = proxy_server.run_with_shutdown().await?; -//! Ok::<(), anyhow::Error>(()) -//! }); -//! -//! tokio::spawn(async move { -//! let _ = proxy_client.run().await?; -//! Ok::<(), anyhow::Error>(()) -//! }); -//! -//! // 'Server side' thread: echo back incoming as response to the messages sent in the 'client side' thread below -//! tokio::spawn(async move { -//! let listener = TcpListener::bind(upstream_tcp_addr).await?; -//! loop { -//! let (socket, _) = listener.accept().await.unwrap(); -//! let (read, mut write) = socket.into_split(); -//! let codec = codec::BytesCodec::new(); -//! let mut framed_read = codec::FramedRead::new(read, codec); -//! while let Some(Ok(bytes)) = framed_read.next().await { -//! match bincode::deserialize:: (&bytes) { -//! Ok(msg) => { -//! println!( -//! "<< server received {}: {} bytes", -//! msg.message_id, -//! msg.message_bytes.len() -//! ); -//! let msg = ExampleMessage { -//! message_id: msg.message_id, -//! message_bytes: msg.message_bytes, -//! }; -//! let serialised = bincode::serialize(&msg)?; -//! write -//! .write_all(&serialised) -//! .await -//! .expect("couldnt send reply"); -//! println!( -//! ">> server sent {}: {} bytes", -//! msg.message_id, -//! msg.message_bytes.len() -//! ); -//! } -//! Err(e) => { -//! println!("<< server received something that wasn't an example message of {} bytes. error: {}", bytes.len(), e); -//! } -//! } -//! } -//! } -//! #[allow(unreachable_code)] -//! Ok::<(), anyhow::Error>(()) -//! }); -//! -//! // Just wait for Nym clients to connect, TCP clients to bind, etc. -//! println!("waiting for everything to be set up.."); -//! tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; -//! println!("done. sending bytes"); -//! -//! // Now the client and server proxies are running we can create and pipe traffic to/from -//! // a socket on the same port as our ProxyClient instance as if we were just communicating -//! // between a client and host via a normal TcpStream - albeit with a decent amount of additional latency. -//! // -//! // The assumption regarding integration is that you know what you're sending, and will do proper -//! // framing before and after, know what data types you're expecting, etc; the proxies are just piping bytes -//! // back and forth using tokio's `Bytecodec` under the hood. -//! let local_tcp_addr = format!("127.0.0.1:{}", client_port); -//! let stream = TcpStream::connect(local_tcp_addr).await?; -//! let (read, mut write) = stream.into_split(); -//! -//! // 'Client side' thread; lets just send a bunch of messages to the server with variable delays between them, with an id to keep track of ordering in the printlns; the mixnet only guarantees message delivery, not ordering. You might not be necessarily streaming traffic in this manner IRL, but this example is a good illustration of how messages travel through the mixnet. -//! // - On the level of individual messages broken into multiple packets, the Proxy abstraction deals with making sure that everything is sent between the sockets in the correct order. -//! // - On the level of different messages, this is not enforced: you might see in the logs that message 1 arrives at the server and is reconstructed after message 2. -//! tokio::spawn(async move { -//! let mut rng = SmallRng::from_entropy(); -//! for i in 0..10 { -//! let random_bytes = gen_bytes_fixed(i as usize); -//! let msg = ExampleMessage { -//! message_id: i, -//! message_bytes: random_bytes, -//! }; -//! let serialised = bincode::serialize(&msg)?; -//! write -//! .write_all(&serialised) -//! .await -//! .expect("couldn't write to stream"); -//! println!(">> client sent {}: {} bytes", &i, msg.message_bytes.len()); -//! let delay = rng.gen_range(3.0..7.0); -//! tokio::time::sleep(tokio::time::Duration::from_secs_f64(delay.clone())).await; -//! } -//! Ok::<(), anyhow::Error>(()) -//! }); -//! -//! let codec = codec::BytesCodec::new(); -//! let mut framed_read = codec::FramedRead::new(read, codec); -//! while let Some(Ok(bytes)) = framed_read.next().await { -//! match bincode::deserialize::(&bytes) { -//! Ok(msg) => { -//! println!( -//! "<< client received {}: {} bytes", -//! msg.message_id, -//! msg.message_bytes.len() -//! ); -//! counter.fetch_add(1, Ordering::SeqCst); -//! println!( -//! ":: messages received back: {:?}/10", -//! counter.load(Ordering::SeqCst) -//! ); -//! } -//! Err(e) => { -//! println!("<< client received something that wasn't an example message of {} bytes. error: {}", bytes.len(), e); -//! } -//! } -//! } -//! -//! // Once timeout is passed, you can either wait for graceful shutdown or just hard stop it. -//! signal::ctrl_c().await?; -//! println!(":: CTRL+C received, shutting down + cleanup up proxy server config files"); -//! fs::remove_dir_all(conf_path)?; -//! Ok(()) -//! } -//! -//! fn gen_bytes_fixed(i: usize) -> Vec { -//! // let amounts = vec![1, 10, 50, 100, 150, 200, 350, 500, 750, 1000]; -//! let amounts = vec![158, 1088, 505, 1001, 150, 200, 3500, 500, 750, 100]; -//! let len = amounts[i]; -//! let mut rng = rand::thread_rng(); -//! (0..len).map(|_| rng.gen::()).collect() -//! } -//! ``` - // TODO UPDATE EXAMPLE ONCE STABLE - -// mod connection_tracker; mod tcp_proxy_client; mod tcp_proxy_server; -// pub use connection_tracker::ConnectionTracker; pub use tcp_proxy_client::NymProxyClient; pub use tcp_proxy_server::NymProxyServer; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 0ebc0fa3eea..40087eeddac 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -1,8 +1,6 @@ +use crate::client_pool::ClientPool; use crate::mixnet::{IncludedSurbs, MixnetClientBuilder, MixnetMessageSender, NymNetworkDetails}; use std::sync::Arc; -#[path = "client_pool.rs"] -mod client_pool; -use client_pool::ClientPool; #[path = "utils.rs"] mod utils; use anyhow::Result; From 4f1bac204c18da7e0193aca1b756f9d7b880aad5 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 23:24:58 +0100 Subject: [PATCH 022/102] cancel token --- sdk/rust/nym-sdk/src/client_pool.rs | 1 + .../nym-sdk/src/client_pool/client_pool.rs | 6 -- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 59 +++++++++++++------ 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/sdk/rust/nym-sdk/src/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool.rs index ddac9b3ff79..8173a63073f 100644 --- a/sdk/rust/nym-sdk/src/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool.rs @@ -1,5 +1,6 @@ // Copyright 2023 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 +//! // TODO EXAMPLE ONCE FINISHED diff --git a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs index 90667c1be07..63f5ef11778 100644 --- a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs @@ -99,12 +99,6 @@ impl ClientPool { .and_then(|arc_client| Arc::try_unwrap(arc_client).ok()) } - // This might still be needed if it needs to be called with a cancellation token in various threads. keeping for the moment - // pub async fn disconnect_and_remove_client(&self, client: MixnetClient) -> Result<()> { - // client.disconnect().await; - // Ok(()) - // } - pub async fn get_client_count(&self) -> usize { self.clients.read().await.len() } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 40087eeddac..76e309e22ea 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -13,6 +13,7 @@ use tokio::{ }; use tokio_stream::StreamExt; use tokio_util::codec::{BytesCodec, FramedRead}; +use tokio_util::sync::CancellationToken; use tracing::{debug, info, instrument, warn}; use utils::{MessageBuffer, Payload, ProxiedMessage}; @@ -74,25 +75,34 @@ impl NymProxyClient { Ok::<(), anyhow::Error>(()) }); - tokio::spawn(async move { - tokio::signal::ctrl_c().await.unwrap(); - info!("RECEIVED KILL SIGNAL"); - // TODO client pool kill function + let cancel_token = CancellationToken::new(); + + tokio::spawn({ + let token = cancel_token.clone(); + async move { + tokio::signal::ctrl_c().await.unwrap(); + info!("Shutdown signal triggered"); + token.cancel(); + } }); - // TODO add 'ready' marker for consuming code + // TODO add 'ready' marker for upstream lib to know when to start sending loop { - if DEFAULT_CLIENT_POOL_SIZE == 1 && self.conn_pool.get_client_count().await == 1 - || self.conn_pool.get_client_count().await >= DEFAULT_CLIENT_POOL_SIZE / 2 - { - let (stream, _) = listener.accept().await?; - tokio::spawn(NymProxyClient::handle_incoming( - stream, - self.server_address, - self.close_timeout, - self.conn_pool.clone(), - )); + tokio::select! { + stream = listener.accept() => { + let (stream, _) = stream?; + tokio::spawn(NymProxyClient::handle_incoming( + stream, + self.server_address, + self.close_timeout, + self.conn_pool.clone(), + cancel_token.clone(), + )); + } + _ = cancel_token.cancelled() => { + break Ok(()); + } } } } @@ -109,12 +119,13 @@ impl NymProxyClient { // Then we spawn 2 tasks: // - 'Outgoing' thread => frames incoming bytes from OwnedReadHalf and pipe through the mixnet & trigger session close. // - 'Incoming' thread => orders incoming messages from the Mixnet via placing them in a MessageBuffer and using tick(), as well as manage session closing. - #[instrument(skip(stream, server_address, close_timeout, conn_pool))] + #[instrument(skip(stream, server_address, close_timeout, conn_pool, cancel_token))] async fn handle_incoming( stream: TcpStream, server_address: Recipient, close_timeout: u64, conn_pool: ClientPool, + cancel_token: CancellationToken, ) -> Result<()> { // ID for creation of session abstraction; new session ID per new connection accepted by our tcp listener above. let session_id = uuid::Uuid::new_v4(); @@ -211,11 +222,12 @@ impl NymProxyClient { // Select!-ing one of following options: // - rx is triggered by tx to log the session will end in ARGS.close_timeout time, break from this loop to pass to loop below // - Deserialise incoming mixnet message, push to msg buffer and tick() to order and write to OwnedWriteHalf. - // - call tick() once per 100ms if neither of the above have occurred. + // - If the cancel_token is in cancelled state, break and kick down to the loop below. + // - Call tick() once per 100ms if neither of the above have occurred. loop { tokio::select! { _ = &mut rx => { - info!(" Closing write end of session: {} in {} seconds", session_id, close_timeout); + info!("Closing write end of session: {} in {} seconds", session_id, close_timeout); break } Some(message) = client.next() => { @@ -223,6 +235,10 @@ impl NymProxyClient { msg_buffer.push(message); msg_buffer.tick(&mut write).await?; }, + _ = cancel_token.cancelled() => { + info!("CTRL_C triggered in thread, triggering loop shutdown"); + break + }, _ = tokio::time::sleep(tokio::time::Duration::from_millis(100)) => { msg_buffer.tick(&mut write).await?; } @@ -230,6 +246,7 @@ impl NymProxyClient { } // Select!-ing one of following options: // - Deserialise incoming mixnet message, push to msg buffer and tick() to order and write next messageID in line to OwnedWriteHalf. + // - If the cancel_token is in cancelled state, shutdown client for this thread. // - Sleep for session timeout and return, kills thread with Ok(()). loop { tokio::select! { @@ -238,11 +255,15 @@ impl NymProxyClient { msg_buffer.push(message); msg_buffer.tick(&mut write).await?; }, + _ = cancel_token.cancelled() => { + info!("CTRL_C triggered in thread, triggering client shutdown"); + client.disconnect().await; + return Ok::<(), anyhow::Error>(()) + }, _ = tokio::time::sleep(tokio::time::Duration::from_secs(close_timeout)) => { info!(" Closing write end of session: {}", session_id); info!(" Triggering client shutdown"); client.disconnect().await; - // conn_pool.disconnect_and_remove_client(client).await?; return Ok::<(), anyhow::Error>(()) }, } From 13a6f0b5efa44e2375abef13237db99c91212ea7 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 27 Nov 2024 23:25:08 +0100 Subject: [PATCH 023/102] logging change --- sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs | 6 +++++- sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs index c1cc0123248..4ab10abdff8 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs @@ -37,7 +37,11 @@ async fn main() -> anyhow::Result<()> { // to DEBUG to see the contents of the msg buffer, sphinx packet chunking, etc. tracing_subscriber::registry() .with(fmt::layer()) - .with(EnvFilter::new("nym_sdk::tcp_proxy=warn")) + .with( + EnvFilter::new("info") + .add_directive("nym_sdk::client_pool=info".parse().unwrap()) + .add_directive("nym_sdk::tcp_proxy_client=debug".parse().unwrap()), + ) .init(); let env_path = env::args().nth(2).expect("Env file not specified"); diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index 66ee8848707..2638ee4152a 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -120,7 +120,7 @@ async fn main() -> anyhow::Result<()> { }); // Just wait for Nym clients to connect, TCP clients to bind, etc. - // TODO change this to wait on the actual client to be ready (pool -> client ready state kickback via oneshot) + // TODO change this to wait on the actual client to be ready (pool -> client ready state kickback) println!("waiting for everything to be set up.."); tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; println!("done. sending bytes"); From 56becc3a34fe7dec8b44d195c06d50cf434a4c01 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 28 Nov 2024 00:00:57 +0100 Subject: [PATCH 024/102] minor tweaks --- sdk/rust/nym-sdk/src/client_pool.rs | 2 -- sdk/rust/nym-sdk/src/client_pool/client_pool.rs | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sdk/rust/nym-sdk/src/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool.rs index 8173a63073f..59d00093d2a 100644 --- a/sdk/rust/nym-sdk/src/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool.rs @@ -1,5 +1,3 @@ -// Copyright 2023 - Nym Technologies SA -// SPDX-License-Identifier: Apache-2.0 //! // TODO EXAMPLE ONCE FINISHED diff --git a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs index 63f5ef11778..4c9280d277a 100644 --- a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs @@ -60,7 +60,7 @@ impl ClientPool { let spawned_clients = self.clients.read().await.len(); let addresses = self; debug!( - "Currently spawned clients: {}: {:?} ", + "Currently spawned clients: {}: {:?}", spawned_clients, addresses ); if spawned_clients >= self.client_pool_reserve_number { @@ -103,6 +103,10 @@ impl ClientPool { self.clients.read().await.len() } + pub async fn get_pool_reserve(&self) -> usize { + self.client_pool_reserve_number + } + pub fn clone(&self) -> Self { Self { clients: Arc::clone(&self.clients), From 4eccc59f6f8b8579e1505e6a08517876f39d8928 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 2 Dec 2024 14:23:37 +0100 Subject: [PATCH 025/102] bump default decay time --- sdk/rust/nym-sdk/src/tcp_proxy/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs index cafccf4c4cd..3495aaf773c 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs @@ -6,7 +6,7 @@ use tokio::{io::AsyncWriteExt as _, net::tcp::OwnedWriteHalf}; use tracing::{debug, info}; use uuid::Uuid; -const DEFAULT_DECAY: u64 = 2; // decay time in seconds +const DEFAULT_DECAY: u64 = 8; // decay time in seconds // Keeps track of // - incoming and unsorted messages wrapped in DecayWrapper for keeping track of when they were received From c8ca174b8717641df10a3513a6555930a4164ca5 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 2 Dec 2024 14:58:28 +0100 Subject: [PATCH 026/102] updated code commnet --- sdk/rust/nym-sdk/src/tcp_proxy/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs index 3495aaf773c..1e874fe5654 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs @@ -62,7 +62,7 @@ impl MessageBuffer { debug!("{}", msg.inner()); } - // Iterate over self, filtering messages where msg.decayed() = true (aka message is older than 2 seconds), or where msg.message_id is less than next_msg_id. Then collect and order according to message_id. + // Iterate over self, filtering messages where msg.decayed() = true (aka message is older than DEFAULT_DECAY seconds), or where msg.message_id is less than next_msg_id. Then collect and order according to message_id. let mut send_buffer = self .iter() .filter(|msg| msg.decayed() || msg.message_id() <= self.next_msg_id) From 95274e38839975977c080ef68a6a5e2c73074ce6 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 2 Dec 2024 15:11:59 +0100 Subject: [PATCH 027/102] bump default decay time --- sdk/rust/nym-sdk/src/tcp_proxy/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs index 1e874fe5654..9c724106be8 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs @@ -6,7 +6,7 @@ use tokio::{io::AsyncWriteExt as _, net::tcp::OwnedWriteHalf}; use tracing::{debug, info}; use uuid::Uuid; -const DEFAULT_DECAY: u64 = 8; // decay time in seconds +const DEFAULT_DECAY: u64 = 10; // decay time in seconds // Keeps track of // - incoming and unsorted messages wrapped in DecayWrapper for keeping track of when they were received From 7563cccf4947bb88ee373cc72724a58ba7da1040 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 2 Dec 2024 15:23:14 +0100 Subject: [PATCH 028/102] bump default decay time (again) --- sdk/rust/nym-sdk/src/tcp_proxy/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs index 9c724106be8..1e874fe5654 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs @@ -6,7 +6,7 @@ use tokio::{io::AsyncWriteExt as _, net::tcp::OwnedWriteHalf}; use tracing::{debug, info}; use uuid::Uuid; -const DEFAULT_DECAY: u64 = 10; // decay time in seconds +const DEFAULT_DECAY: u64 = 8; // decay time in seconds // Keeps track of // - incoming and unsorted messages wrapped in DecayWrapper for keeping track of when they were received From a0e55f49d4343647bc64110023d5abb8eec4e7e1 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 3 Dec 2024 13:31:45 +0100 Subject: [PATCH 029/102] temp commit --- sdk/ffi/go/go-nym/bindings/bindings.go | 30 ++++++- sdk/ffi/go/go-nym/bindings/bindings.h | 1 + sdk/ffi/go/proxy_example.go | 2 +- sdk/ffi/go/src/bindings.udl | 2 +- sdk/ffi/go/src/lib.rs | 2 + sdk/ffi/shared/src/lib.rs | 86 +++++++++++-------- ...niffi-bindgen.rs => uniffi-bindgen.rs.bak} | 0 7 files changed, 84 insertions(+), 39 deletions(-) rename sdk/ffi/shared/{uniffi-bindgen.rs => uniffi-bindgen.rs.bak} (100%) diff --git a/sdk/ffi/go/go-nym/bindings/bindings.go b/sdk/ffi/go/go-nym/bindings/bindings.go index 100e4a5bf22..00ad0e3a826 100644 --- a/sdk/ffi/go/go-nym/bindings/bindings.go +++ b/sdk/ffi/go/go-nym/bindings/bindings.go @@ -383,7 +383,7 @@ func uniffiCheckChecksums() { checksum := rustCall(func(uniffiStatus *C.RustCallStatus) C.uint16_t { return C.uniffi_nym_go_ffi_checksum_func_new_proxy_client(uniffiStatus) }) - if checksum != 14386 { + if checksum != 38844 { // If this happens try cleaning and rebuilding your project panic("bindings: uniffi_nym_go_ffi_checksum_func_new_proxy_client: UniFFI API checksum mismatch") } @@ -453,6 +453,30 @@ func uniffiCheckChecksums() { } } +type FfiConverterUint8 struct{} + +var FfiConverterUint8INSTANCE = FfiConverterUint8{} + +func (FfiConverterUint8) Lower(value uint8) C.uint8_t { + return C.uint8_t(value) +} + +func (FfiConverterUint8) Write(writer io.Writer, value uint8) { + writeUint8(writer, value) +} + +func (FfiConverterUint8) Lift(value C.uint8_t) uint8 { + return uint8(value) +} + +func (FfiConverterUint8) Read(reader io.Reader) uint8 { + return readUint8(reader) +} + +type FfiDestroyerUint8 struct{} + +func (FfiDestroyerUint8) Destroy(_ uint8) {} + type FfiConverterUint64 struct{} var FfiConverterUint64INSTANCE = FfiConverterUint64{} @@ -963,9 +987,9 @@ func ListenForIncoming() (IncomingMessage, error) { } } -func NewProxyClient(serverAddress string, listenAddress string, listenPort string, closeTimeout uint64, env *string) error { +func NewProxyClient(serverAddress string, listenAddress string, listenPort string, closeTimeout uint64, env *string, poolSize uint8) error { _, _uniffiErr := rustCallWithError(FfiConverterTypeGoWrapError{}, func(_uniffiStatus *C.RustCallStatus) bool { - C.uniffi_nym_go_ffi_fn_func_new_proxy_client(FfiConverterStringINSTANCE.Lower(serverAddress), FfiConverterStringINSTANCE.Lower(listenAddress), FfiConverterStringINSTANCE.Lower(listenPort), FfiConverterUint64INSTANCE.Lower(closeTimeout), FfiConverterOptionalStringINSTANCE.Lower(env), _uniffiStatus) + C.uniffi_nym_go_ffi_fn_func_new_proxy_client(FfiConverterStringINSTANCE.Lower(serverAddress), FfiConverterStringINSTANCE.Lower(listenAddress), FfiConverterStringINSTANCE.Lower(listenPort), FfiConverterUint64INSTANCE.Lower(closeTimeout), FfiConverterOptionalStringINSTANCE.Lower(env), FfiConverterUint8INSTANCE.Lower(poolSize), _uniffiStatus) return false }) return _uniffiErr diff --git a/sdk/ffi/go/go-nym/bindings/bindings.h b/sdk/ffi/go/go-nym/bindings/bindings.h index 9e586da85e8..1a28b13dc47 100644 --- a/sdk/ffi/go/go-nym/bindings/bindings.h +++ b/sdk/ffi/go/go-nym/bindings/bindings.h @@ -90,6 +90,7 @@ void uniffi_nym_go_ffi_fn_func_new_proxy_client( RustBuffer listen_port, uint64_t close_timeout, RustBuffer env, + uint8_t pool_size, RustCallStatus* out_status ); diff --git a/sdk/ffi/go/proxy_example.go b/sdk/ffi/go/proxy_example.go index c9e0667562b..b979410f20c 100644 --- a/sdk/ffi/go/proxy_example.go +++ b/sdk/ffi/go/proxy_example.go @@ -118,7 +118,7 @@ func main() { go runProxyServer() // initialise a proxy client - build_err := bindings.NewProxyClient(proxyAddr, "127.0.0.1", clientPort, clientTimeout, &env_path) + build_err := bindings.NewProxyClient(proxyAddr, "127.0.0.1", clientPort, clientTimeout, &env_path, 2) if build_err != nil { fmt.Println(build_err) return diff --git a/sdk/ffi/go/src/bindings.udl b/sdk/ffi/go/src/bindings.udl index 030e75a9a68..ec429404c0f 100644 --- a/sdk/ffi/go/src/bindings.udl +++ b/sdk/ffi/go/src/bindings.udl @@ -30,7 +30,7 @@ namespace bindings { [Throws=GoWrapError] IncomingMessage listen_for_incoming(); [Throws=GoWrapError] - void new_proxy_client(string server_address, string listen_address, string listen_port, u64 close_timeout, string? env); + void new_proxy_client(string server_address, string listen_address, string listen_port, u64 close_timeout, string? env, u8 pool_size); [Throws=GoWrapError] void new_proxy_client_default(string server_address, string? env); [Throws=GoWrapError] diff --git a/sdk/ffi/go/src/lib.rs b/sdk/ffi/go/src/lib.rs index a0a690c6913..83656c1b76c 100644 --- a/sdk/ffi/go/src/lib.rs +++ b/sdk/ffi/go/src/lib.rs @@ -100,6 +100,7 @@ fn new_proxy_client( listen_port: String, close_timeout: u64, env: Option, + pool_size: u8, ) -> Result<(), GoWrapError> { let server_nym_addr = Recipient::try_from_base58_string(server_address).expect("couldn't create Recipient"); @@ -109,6 +110,7 @@ fn new_proxy_client( &listen_port, close_timeout, env, + pool_size as usize, ) { Ok(_) => Ok(()), Err(_) => Err(GoWrapError::ProxyInitError {}), diff --git a/sdk/ffi/shared/src/lib.rs b/sdk/ffi/shared/src/lib.rs index de25f90443a..4a09a63c8db 100644 --- a/sdk/ffi/shared/src/lib.rs +++ b/sdk/ffi/shared/src/lib.rs @@ -1,7 +1,7 @@ // Copyright 2023-2024 - Nym Technologies SA // SPDX-License-Identifier: Apache-2.0 -use anyhow::{anyhow, bail}; +use anyhow::{anyhow, bail, Error, Result}; use lazy_static::lazy_static; use nym_sdk::mixnet::{ MixnetClient, MixnetClientBuilder, MixnetMessageSender, Recipient, ReconstructedMessage, @@ -13,11 +13,8 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use tokio::runtime::Runtime; -// NYM_CLIENT/PROXIES: Static reference (only init-ed once) to: -// - Arc: share ownership -// - Mutex: thread-safe way to share data between threads -// - Option: init-ed or not -// RUNTIME: Tokio runtime: no need to pass back to C and deal with raw pointers as it was previously +// NYM_CLIENT/PROXIES: Static thread-safe reference (init once) to Options. +// RUNTIME: Tokio runtime: no need to pass across FFI boundary and deal with raw pointers. lazy_static! { static ref NYM_PROXY_CLIENT: Arc>> = Arc::new(Mutex::new(None)); static ref NYM_PROXY_SERVER: Arc>> = Arc::new(Mutex::new(None)); @@ -25,7 +22,16 @@ lazy_static! { static ref RUNTIME: Runtime = Runtime::new().unwrap(); } -pub fn init_ephemeral_internal() -> anyhow::Result<(), anyhow::Error> { +// TODO create get_client() to use in fns and remove code repetition +fn get_client_as_ref() -> bool { + NYM_CLIENT.lock().unwrap().as_ref().is_some() +} + +fn get_client_guard() -> bool { + todo!() +} + +pub fn init_ephemeral_internal() -> Result<(), Error> { if NYM_CLIENT.lock().unwrap().as_ref().is_some() { bail!("client already exists"); } else { @@ -37,13 +43,27 @@ pub fn init_ephemeral_internal() -> anyhow::Result<(), anyhow::Error> { } else { return Err(anyhow!("couldnt lock ephemeral NYM_CLIENT")); } - Ok::<(), anyhow::Error>(()) + Ok::<(), Error>(()) })?; } + // if get_client_as_ref() { + // RUNTIME.block_on(async move { + // let init_client = MixnetClient::connect_new().await?; + // let mut client = NYM_CLIENT.try_lock(); + // if let Ok(ref mut client) = client { + // **client = Some(init_client); + // } else { + // return Err(anyhow!("couldnt lock ephemeral NYM_CLIENT")); + // } + // Ok::<(), Error>(()) + // })?; + // } else { + // bail!("client already exists: no need to reinitialise"); + // } Ok(()) } -pub fn init_default_storage_internal(config_dir: PathBuf) -> anyhow::Result<(), anyhow::Error> { +pub fn init_default_storage_internal(config_dir: PathBuf) -> Result<(), Error> { if NYM_CLIENT.lock().unwrap().as_ref().is_some() { bail!("client already exists"); } else { @@ -60,13 +80,13 @@ pub fn init_default_storage_internal(config_dir: PathBuf) -> anyhow::Result<(), } else { return Err(anyhow!("couldnt lock NYM_CLIENT")); } - Ok::<(), anyhow::Error>(()) + Ok::<(), Error>(()) })?; } Ok(()) } -pub fn get_self_address_internal() -> anyhow::Result { +pub fn get_self_address_internal() -> Result { let client = NYM_CLIENT.lock().expect("could not lock NYM_CLIENT"); if client.is_none() { bail!("Client is not yet initialised"); @@ -83,7 +103,8 @@ pub fn get_self_address_internal() -> anyhow::Result { pub fn send_message_internal( recipient: Recipient, message: &str, -) -> anyhow::Result<(), anyhow::Error> { + // TODO add Option, if Some(surb_amount) call send_message() instead with specified #, else send_plain_message as this uses the default +) -> Result<(), Error> { let client = NYM_CLIENT.lock().expect("could not lock NYM_CLIENT"); if client.is_none() { bail!("Client is not yet initialised"); @@ -94,17 +115,14 @@ pub fn send_message_internal( RUNTIME.block_on(async move { nym_client.send_plain_message(recipient, message).await?; - Ok::<(), anyhow::Error>(()) + Ok::<(), Error>(()) })?; Ok(()) } // TODO send_raw_message_internal -pub fn reply_internal( - recipient: AnonymousSenderTag, - message: &str, -) -> anyhow::Result<(), anyhow::Error> { +pub fn reply_internal(recipient: AnonymousSenderTag, message: &str) -> Result<(), Error> { let client = NYM_CLIENT.lock().expect("could not lock NYM_CLIENT"); if client.is_none() { bail!("Client is not yet initialised"); @@ -115,12 +133,12 @@ pub fn reply_internal( RUNTIME.block_on(async move { nym_client.send_reply(recipient, message).await?; - Ok::<(), anyhow::Error>(()) + Ok::<(), Error>(()) })?; Ok(()) } -pub fn listen_for_incoming_internal() -> anyhow::Result { +pub fn listen_for_incoming_internal() -> Result { let mut binding = NYM_CLIENT.lock().expect("could not lock NYM_CLIENT"); if binding.is_none() { bail!("recipient is null"); @@ -131,7 +149,7 @@ pub fn listen_for_incoming_internal() -> anyhow::Result(ReconstructedMessage { + Ok::(ReconstructedMessage { message: received.message, sender_tag: received.sender_tag, }) @@ -140,9 +158,7 @@ pub fn listen_for_incoming_internal() -> anyhow::Result anyhow::Result { +pub async fn wait_for_non_empty_message(client: &mut MixnetClient) -> Result { while let Some(mut new_message) = client.wait_for_messages().await { if !new_message.is_empty() { return new_message @@ -159,7 +175,8 @@ pub fn proxy_client_new_internal( listen_port: &str, close_timeout: u64, env: Option, -) -> anyhow::Result<(), anyhow::Error> { + pool_size: usize, +) -> Result<(), Error> { if NYM_PROXY_CLIENT.lock().unwrap().as_ref().is_some() { bail!("proxy client already exists"); } else { @@ -170,6 +187,7 @@ pub fn proxy_client_new_internal( listen_port, close_timeout, env, + pool_size, ) .await?; let mut client = NYM_PROXY_CLIENT.try_lock(); @@ -178,7 +196,7 @@ pub fn proxy_client_new_internal( } else { return Err(anyhow!("couldnt lock NYM_PROXY_CLIENT")); } - Ok::<(), anyhow::Error>(()) + Ok::<(), Error>(()) })?; } Ok(()) @@ -187,7 +205,7 @@ pub fn proxy_client_new_internal( pub fn proxy_client_new_defaults_internal( server_address: Recipient, env: Option, -) -> anyhow::Result<(), anyhow::Error> { +) -> Result<(), Error> { if NYM_PROXY_CLIENT.lock().unwrap().as_ref().is_some() { bail!("proxy client already exists"); } else { @@ -199,13 +217,13 @@ pub fn proxy_client_new_defaults_internal( } else { return Err(anyhow!("couldn't lock PROXY_CLIENT")); } - Ok::<(), anyhow::Error>(()) + Ok::<(), Error>(()) })?; } Ok(()) } -pub fn proxy_client_run_internal() -> anyhow::Result<(), anyhow::Error> { +pub fn proxy_client_run_internal() -> Result<(), Error> { let proxy_client = NYM_PROXY_CLIENT .lock() .expect("could not lock NYM_PROXY_CLIENT"); @@ -217,7 +235,7 @@ pub fn proxy_client_run_internal() -> anyhow::Result<(), anyhow::Error> { .ok_or_else(|| anyhow!("could not get proxy_client as_ref()"))?; RUNTIME.block_on(async move { proxy.run().await?; - Ok::<(), anyhow::Error>(()) + Ok::<(), Error>(()) })?; Ok(()) } @@ -226,7 +244,7 @@ pub fn proxy_server_new_internal( upstream_address: &str, config_dir: &str, env: Option, -) -> anyhow::Result<(), anyhow::Error> { +) -> Result<(), Error> { if NYM_PROXY_SERVER.lock().unwrap().as_ref().is_some() { bail!("proxy client already exists"); } else { @@ -238,13 +256,13 @@ pub fn proxy_server_new_internal( } else { return Err(anyhow!("couldn't lock PROXY_SERVER")); } - Ok::<(), anyhow::Error>(()) + Ok::<(), Error>(()) })?; } Ok(()) } -pub fn proxy_server_run_internal() -> anyhow::Result<(), anyhow::Error> { +pub fn proxy_server_run_internal() -> Result<(), Error> { let mut proxy_server = NYM_PROXY_SERVER .lock() .expect("could not lock NYM_PROXY_CLIENT"); @@ -256,12 +274,12 @@ pub fn proxy_server_run_internal() -> anyhow::Result<(), anyhow::Error> { .ok_or_else(|| anyhow!("could not get proxy_client as_ref()"))?; RUNTIME.block_on(async move { proxy.run_with_shutdown().await?; - Ok::<(), anyhow::Error>(()) + Ok::<(), Error>(()) })?; Ok(()) } -pub fn proxy_server_address_internal() -> anyhow::Result { +pub fn proxy_server_address_internal() -> Result { let mut proxy_server = NYM_PROXY_SERVER .lock() .expect("could not lock NYM_PROXY_CLIENT"); diff --git a/sdk/ffi/shared/uniffi-bindgen.rs b/sdk/ffi/shared/uniffi-bindgen.rs.bak similarity index 100% rename from sdk/ffi/shared/uniffi-bindgen.rs rename to sdk/ffi/shared/uniffi-bindgen.rs.bak From 731f69acc3dc1778cb1c549babdb009778374862 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 15:40:51 +0100 Subject: [PATCH 030/102] add duplicate packets received to troubleshooting --- .../docs/pages/developers/rust/mixnet/troubleshooting.md | 3 +++ documentation/docs/pages/developers/rust/tcpproxy/_meta.json | 3 ++- .../docs/pages/developers/rust/tcpproxy/troubleshooting.md | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 documentation/docs/pages/developers/rust/tcpproxy/troubleshooting.md diff --git a/documentation/docs/pages/developers/rust/mixnet/troubleshooting.md b/documentation/docs/pages/developers/rust/mixnet/troubleshooting.md index 06bba32f6d7..ff7b10577aa 100644 --- a/documentation/docs/pages/developers/rust/mixnet/troubleshooting.md +++ b/documentation/docs/pages/developers/rust/mixnet/troubleshooting.md @@ -112,3 +112,6 @@ Whether the `data` of a SURB request being empty is a feature or a bug is to be You can find a few helper functions [here](./message-helpers.md) to help deal with this issue in the meantime. > If you can think of a more succinct or different way of handling this do reach out - we're happy to hear other opinions + +## Lots of `duplicate fragment received` messages +You might see a lot of `WARN` level logs about duplicate fragments in your logs, depending on the log level you're using. This occurs when a packet is retransmitted somewhere in the Mixnet, but then the original makes it to the destination client as well. This is not something to do with your client logic, but instead the state of the Mixnet. diff --git a/documentation/docs/pages/developers/rust/tcpproxy/_meta.json b/documentation/docs/pages/developers/rust/tcpproxy/_meta.json index 4ee4d522ebc..7b7e6bb8217 100644 --- a/documentation/docs/pages/developers/rust/tcpproxy/_meta.json +++ b/documentation/docs/pages/developers/rust/tcpproxy/_meta.json @@ -1,4 +1,5 @@ { "architecture": "Architecture", - "examples": "Examples" + "examples": "Examples", + "troubleshooting": "Troubleshooting" } diff --git a/documentation/docs/pages/developers/rust/tcpproxy/troubleshooting.md b/documentation/docs/pages/developers/rust/tcpproxy/troubleshooting.md new file mode 100644 index 00000000000..8a3d6d5c233 --- /dev/null +++ b/documentation/docs/pages/developers/rust/tcpproxy/troubleshooting.md @@ -0,0 +1,4 @@ +# Troubleshooting + +## Lots of `duplicate fragment received` messages +You might see a lot of `WARN` level logs about duplicate fragments in your logs, depending on the log level you're using. This occurs when a packet is retransmitted somewhere in the Mixnet, but then the original makes it to the destination client as well. This is not something to do with your client logic, but instead the state of the Mixnet. From e9ade22b103085b48eace5f46bdd6aa71b09729b Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 15:57:06 +0100 Subject: [PATCH 031/102] slightly dropped default decay time --- sdk/rust/nym-sdk/src/tcp_proxy/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs index 1e874fe5654..884a63f2abc 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs @@ -6,7 +6,7 @@ use tokio::{io::AsyncWriteExt as _, net::tcp::OwnedWriteHalf}; use tracing::{debug, info}; use uuid::Uuid; -const DEFAULT_DECAY: u64 = 8; // decay time in seconds +const DEFAULT_DECAY: u64 = 6; // decay time in seconds // Keeps track of // - incoming and unsorted messages wrapped in DecayWrapper for keeping track of when they were received From fa6efbaa2aff781b78f2d67ee30e82a2d2b0f598 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 15:57:15 +0100 Subject: [PATCH 032/102] comments to examples --- sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs | 3 +-- sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs index 4ab10abdff8..ff540302de6 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs @@ -49,7 +49,7 @@ async fn main() -> anyhow::Result<()> { let listen_port = env::args().nth(3).expect("Port not specified"); - // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. The final argument is how many clients to keep in reserve in the client pool. + // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. The final argument is how many clients to keep in reserve in the client pool when running the TcpProxy. let proxy_client = tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 45, Some(env), 2).await?; @@ -58,7 +58,6 @@ async fn main() -> anyhow::Result<()> { Ok::<(), anyhow::Error>(()) }); - // TODO change this to wait on the actual client to be ready (pool -> client ready state kickback via oneshot) println!("waiting for everything to be set up.."); tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; println!("done. sending bytes"); diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index 2638ee4152a..76805253215 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -36,9 +36,6 @@ async fn main() -> anyhow::Result<()> { // Comment this out to just see println! statements from this example, as Nym client logging is very informative but quite verbose. // The Message Decay related logging gives you an ideas of the internals of the proxy message ordering. To see the contents of the msg buffer, sphinx packet chunking, etc change the tracing::Level to DEBUG. - // tracing_subscriber::fmt() - // .with_max_level(tracing::Level::INFO) - // .init(); tracing_subscriber::registry() .with(fmt::layer()) .with(EnvFilter::new("nym_sdk::tcp_proxy=info")) @@ -64,8 +61,9 @@ async fn main() -> anyhow::Result<()> { // We'll run the instance with a long timeout since we're sending everything down the same Tcp connection, so should be using a single session. // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. + // The final argument is how many clients to keep in reserve in the client pool when running the TcpProxy: since we're only expecting 1 connection in total, we can start with no clients in the pool, and the TcpProxyClient will just spin up an ephemeral client outside of the pool to use. let proxy_client = - tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 3) + tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 0) .await?; tokio::spawn(async move { From 6fc231821bf0d5277e3dd1a7087f77b53c29fee3 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 16:07:13 +0100 Subject: [PATCH 033/102] some inline comments --- sdk/rust/nym-sdk/src/client_pool/client_pool.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs index 4c9280d277a..ef7e3a90969 100644 --- a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs @@ -6,10 +6,11 @@ use tokio::sync::RwLock; use tracing::{debug, info, warn}; pub struct ClientPool { - clients: Arc>>>, // collection of clients waiting to be used which are popped off in get_mixnet_client() - client_pool_reserve_number: usize, // default # of clients to have available in pool in reserve waiting for incoming connections + clients: Arc>>>, // Collection of clients waiting to be used which are popped off in get_mixnet_client() + client_pool_reserve_number: usize, // Default # of clients to have available in pool in reserve waiting for incoming connections } +// This is only necessary for when you're wanting to check the addresses of the clients that are currently in the pool. impl fmt::Debug for ClientPool { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let clients_debug = match self.clients.try_read() { @@ -55,6 +56,7 @@ impl ClientPool { } } + // The loop here is simple: if there aren't enough clients, create more. If you set clients to 0, repeatedly just sleep. pub async fn start(&self) -> Result<()> { loop { let spawned_clients = self.clients.read().await.len(); From c7b3bd1608d74c2d5fe52fcd5fb4365a332a4d78 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 16:07:38 +0100 Subject: [PATCH 034/102] minor comments --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 76e309e22ea..1d566ba763d 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -17,7 +17,7 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, info, instrument, warn}; use utils::{MessageBuffer, Payload, ProxiedMessage}; -const DEFAULT_CLOSE_TIMEOUT: u64 = 60; +const DEFAULT_CLOSE_TIMEOUT: u64 = 60; // seconds const DEFAULT_LISTEN_HOST: &str = "127.0.0.1"; const DEFAULT_LISTEN_PORT: &str = "8080"; const DEFAULT_CLIENT_POOL_SIZE: usize = 4; @@ -86,8 +86,6 @@ impl NymProxyClient { } }); - // TODO add 'ready' marker for upstream lib to know when to start sending - loop { tokio::select! { stream = listener.accept() => { From 747675f52fd650a26aaf8bbbbbcc79e1beb314d7 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 16:18:54 +0100 Subject: [PATCH 035/102] client_pool.rs mod --- sdk/rust/nym-sdk/src/client_pool.rs | 117 +++++++++++++++++++++++++++- 1 file changed, 116 insertions(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool.rs index 59d00093d2a..e2fea7fb78f 100644 --- a/sdk/rust/nym-sdk/src/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool.rs @@ -1,6 +1,121 @@ +//! use crate::mixnet::{MixnetClient, MixnetClientBuilder, NymNetworkDetails}; +//! use anyhow::Result; +//! use std::fmt; +//! use std::sync::Arc; +//! use tokio::sync::RwLock; +//! use tracing::{debug, info, warn}; + +//! pub struct ClientPool { +//! clients: Arc>>>, // Collection of clients waiting to be used which are popped off in get_mixnet_client() +//! client_pool_reserve_number: usize, // Default # of clients to have available in pool in reserve waiting for incoming connections +//! } + +//! // This is only necessary for when you're wanting to check the addresses of the clients that are currently in the pool. +//! impl fmt::Debug for ClientPool { +//! fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +//! let clients_debug = match self.clients.try_read() { +//! Ok(clients) => { +//! if f.alternate() { +//! // pretty +//! clients +//! .iter() +//! .enumerate() +//! .map(|(i, client)| { +//! format!("\n {}: {}", i, client.nym_address().to_string()) +//! }) +//! .collect::>() +//! .join(",") +//! } else { +//! // compact +//! clients +//! .iter() +//! .map(|client| client.nym_address().to_string()) +//! .collect::>() +//! .join(", ") +//! } +//! } +//! Err(_) => "".to_string(), +//! }; //! +//! let mut debug_struct = f.debug_struct("Pool"); +//! debug_struct +//! .field( +//! "client_pool_reserve_number", +//! &self.client_pool_reserve_number, +//! ) +//! .field("clients", &format_args!("[{}]", clients_debug)); +//! debug_struct.finish() +//! } +//! } -// TODO EXAMPLE ONCE FINISHED +//! impl ClientPool { +//! pub fn new(client_pool_reserve_number: usize) -> Self { +//! ClientPool { +//! clients: Arc::new(RwLock::new(Vec::new())), +//! client_pool_reserve_number, +//! } +//! } +//! +//! // The loop here is simple: if there aren't enough clients, create more. If you set clients to 0, repeatedly just sleep. +//! pub async fn start(&self) -> Result<()> { +//! loop { +//! let spawned_clients = self.clients.read().await.len(); +//! let addresses = self; +//! debug!( +//! "Currently spawned clients: {}: {:?}", +//! spawned_clients, addresses +//! ); +//! if spawned_clients >= self.client_pool_reserve_number { +//! debug!("Got enough clients already: sleeping"); +//! } else { +//! info!( +//! "Clients in reserve = {}, reserve amount = {}, spawning new client", +//! spawned_clients, self.client_pool_reserve_number +//! ); +//! let client = loop { +//! let net = NymNetworkDetails::new_from_env(); +//! match MixnetClientBuilder::new_ephemeral() +//! .network_details(net) +//! .build()? +//! .connect_to_mixnet() +//! .await +//! { +//! Ok(client) => break client, +//! Err(err) => { +//! warn!("Error creating client: {:?}, will retry in 100ms", err); +//! tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; +//! } +//! } +//! }; +//! self.clients.write().await.push(Arc::new(client)); +//! } +//! tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; +//! } +//! } +//! +//! pub async fn get_mixnet_client(&self) -> Option { +//! debug!("Grabbing client from pool"); +//! let mut clients = self.clients.write().await; +//! clients +//! .pop() +//! .and_then(|arc_client| Arc::try_unwrap(arc_client).ok()) +//! } +//! +//! pub async fn get_client_count(&self) -> usize { +//! self.clients.read().await.len() +//! } +//! +//! pub async fn get_pool_reserve(&self) -> usize { +//! self.client_pool_reserve_number +//! } +//! +//! pub fn clone(&self) -> Self { +//! Self { +//! clients: Arc::clone(&self.clients), +//! client_pool_reserve_number: *&self.client_pool_reserve_number, +//! } +//! } +//! } mod client_pool; From 7660f007c884120b2b36d9ad15e6c248ab58b846 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 17:10:31 +0100 Subject: [PATCH 036/102] client pool example --- sdk/rust/nym-sdk/examples/client_pool.rs | 86 ++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 sdk/rust/nym-sdk/examples/client_pool.rs diff --git a/sdk/rust/nym-sdk/examples/client_pool.rs b/sdk/rust/nym-sdk/examples/client_pool.rs new file mode 100644 index 00000000000..38558d651a7 --- /dev/null +++ b/sdk/rust/nym-sdk/examples/client_pool.rs @@ -0,0 +1,86 @@ +// Copyright 2023 - Nym Technologies SA +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; +use nym_network_defaults::setup_env; +use nym_sdk::client_pool::ClientPool; +use nym_sdk::mixnet::{MixnetClientBuilder, NymNetworkDetails}; + +// This client pool is used internally by the TcpProxyClient but can also be used by the Mixnet module, in case you're quickly swapping clients in and out but won't want to use the TcpProxy module for whatever reason. +// +// Run with: cargo run --example client_pool -- ../../../envs/.env +#[tokio::main] +async fn main() -> Result<()> { + nym_bin_common::logging::setup_logging(); + setup_env(std::env::args().nth(1)); + + let conn_pool = ClientPool::new(1); // Start the client pool with 1 client always being kept in reserve + let client_maker = conn_pool.clone(); + tokio::spawn(async move { + client_maker.start().await?; + Ok::<(), anyhow::Error>(()) + }); + + let pool_clone_one = conn_pool.clone(); + let pool_clone_two = conn_pool.clone(); + + tokio::spawn(async move { + let client_one = match pool_clone_one.get_mixnet_client().await { + Some(client) => { + println!("Grabbed client {} from pool", client.nym_address()); + client + } + None => { + println!("Not enough clients in pool, creating ephemeral client"); + let net = NymNetworkDetails::new_from_env(); + let client = MixnetClientBuilder::new_ephemeral() + .network_details(net) + .build()? + .connect_to_mixnet() + .await?; + println!( + "Using {} for the moment, created outside of the connection pool", + client.nym_address() + ); + client + } + }; + let our_address = client_one.nym_address(); + println!("Client 1: {our_address}"); + client_one.disconnect().await; + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something + return Ok::<(), anyhow::Error>(()); + }); + + tokio::spawn(async move { + let client_two = match pool_clone_two.get_mixnet_client().await { + Some(client) => { + println!("Grabbed client {} from pool", client.nym_address()); + client + } + None => { + println!("Not enough clients in pool, creating ephemeral client"); + let net = NymNetworkDetails::new_from_env(); + let client = MixnetClientBuilder::new_ephemeral() + .network_details(net) + .build()? + .connect_to_mixnet() + .await?; + println!( + "Using {} for the moment, created outside of the connection pool", + client.nym_address() + ); + client + } + }; + let our_address = *client_two.nym_address(); + println!("Client 2: {our_address}"); + client_two.disconnect().await; + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something + return Ok::<(), anyhow::Error>(()); + }); + + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + // TODO clientpool disconnect rest + Ok(()) +} From fedaf00375421dcebe551a4eea60144c9a2c01c9 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 17:30:32 +0100 Subject: [PATCH 037/102] clippy --- sdk/rust/nym-sdk/src/client_pool/client_pool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs index ef7e3a90969..968c8ca47c8 100644 --- a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs @@ -21,7 +21,7 @@ impl fmt::Debug for ClientPool { .iter() .enumerate() .map(|(i, client)| { - format!("\n {}: {}", i, client.nym_address().to_string()) + format!("\n {}: {}", i, client.nym_address()) }) .collect::>() .join(",") @@ -112,7 +112,7 @@ impl ClientPool { pub fn clone(&self) -> Self { Self { clients: Arc::clone(&self.clients), - client_pool_reserve_number: *&self.client_pool_reserve_number, + client_pool_reserve_number: self.client_pool_reserve_number, } } } From 68a1d43cbf4c3304d6d6c6cc0589ea95f4fa8e17 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 17:37:33 +0100 Subject: [PATCH 038/102] comments --- sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index 76805253215..df41d503e70 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -117,10 +117,9 @@ async fn main() -> anyhow::Result<()> { Ok::<(), anyhow::Error>(()) }); - // Just wait for Nym clients to connect, TCP clients to bind, etc. - // TODO change this to wait on the actual client to be ready (pool -> client ready state kickback) + // Just wait for Nym clients to connect, TCP clients to bind, etc. If there isn't a client in the pool (or you started it with 0) already then the TcpProxyClient just spins up an ephemeral client itself. println!("waiting for everything to be set up.."); - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; println!("done. sending bytes"); // Now the client and server proxies are running we can create and pipe traffic to/from From 8630cefddb7aa9ce624cad2f5e321daf89eff4c9 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 18:29:50 +0100 Subject: [PATCH 039/102] client pool example done --- sdk/rust/nym-sdk/examples/client_pool.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/client_pool.rs b/sdk/rust/nym-sdk/examples/client_pool.rs index 38558d651a7..2fffe43812e 100644 --- a/sdk/rust/nym-sdk/examples/client_pool.rs +++ b/sdk/rust/nym-sdk/examples/client_pool.rs @@ -5,6 +5,7 @@ use anyhow::Result; use nym_network_defaults::setup_env; use nym_sdk::client_pool::ClientPool; use nym_sdk::mixnet::{MixnetClientBuilder, NymNetworkDetails}; +use tokio::signal::ctrl_c; // This client pool is used internally by the TcpProxyClient but can also be used by the Mixnet module, in case you're quickly swapping clients in and out but won't want to use the TcpProxy module for whatever reason. // @@ -14,13 +15,16 @@ async fn main() -> Result<()> { nym_bin_common::logging::setup_logging(); setup_env(std::env::args().nth(1)); - let conn_pool = ClientPool::new(1); // Start the client pool with 1 client always being kept in reserve + let conn_pool = ClientPool::new(2); // Start the client pool with 1 client always being kept in reserve let client_maker = conn_pool.clone(); tokio::spawn(async move { client_maker.start().await?; Ok::<(), anyhow::Error>(()) }); + println!("\n\nWaiting a few seconds to fill pool\n\n"); + tokio::time::sleep(tokio::time::Duration::from_secs(15)).await; + let pool_clone_one = conn_pool.clone(); let pool_clone_two = conn_pool.clone(); @@ -46,7 +50,7 @@ async fn main() -> Result<()> { } }; let our_address = client_one.nym_address(); - println!("Client 1: {our_address}"); + println!("\n\nClient 1: {our_address}\n\n"); client_one.disconnect().await; tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something return Ok::<(), anyhow::Error>(()); @@ -74,13 +78,20 @@ async fn main() -> Result<()> { } }; let our_address = *client_two.nym_address(); - println!("Client 2: {our_address}"); + println!("\n\nClient 2: {our_address}\n\n"); client_two.disconnect().await; tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something return Ok::<(), anyhow::Error>(()); }); - tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; - // TODO clientpool disconnect rest + wait_for_ctrl_c(conn_pool).await?; + Ok(()) +} + +async fn wait_for_ctrl_c(pool: ClientPool) -> Result<()> { + println!("\n\nPress CTRL_C to disconnect pool\n\n"); + ctrl_c().await?; + println!("CTRL_C received. Killing client pool"); + pool.disconnect_pool().await; Ok(()) } From 5047ed2722f1eeb23e155a6d0076e36ef72bc18a Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 18:30:04 +0100 Subject: [PATCH 040/102] added disconnect to client pool --- .../nym-sdk/src/client_pool/client_pool.rs | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs index 968c8ca47c8..37d20858245 100644 --- a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs @@ -3,11 +3,13 @@ use anyhow::Result; use std::fmt; use std::sync::Arc; use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; pub struct ClientPool { clients: Arc>>>, // Collection of clients waiting to be used which are popped off in get_mixnet_client() client_pool_reserve_number: usize, // Default # of clients to have available in pool in reserve waiting for incoming connections + cancel_token: CancellationToken, } // This is only necessary for when you're wanting to check the addresses of the clients that are currently in the pool. @@ -20,9 +22,7 @@ impl fmt::Debug for ClientPool { clients .iter() .enumerate() - .map(|(i, client)| { - format!("\n {}: {}", i, client.nym_address()) - }) + .map(|(i, client)| format!("\n {}: {}", i, client.nym_address())) .collect::>() .join(",") } else { @@ -53,10 +53,12 @@ impl ClientPool { ClientPool { clients: Arc::new(RwLock::new(Vec::new())), client_pool_reserve_number, + cancel_token: CancellationToken::new(), } } // The loop here is simple: if there aren't enough clients, create more. If you set clients to 0, repeatedly just sleep. + // disconnect_pool() will kill this loop via the cancellation token. pub async fn start(&self) -> Result<()> { loop { let spawned_clients = self.clients.read().await.len(); @@ -65,6 +67,9 @@ impl ClientPool { "Currently spawned clients: {}: {:?}", spawned_clients, addresses ); + if self.cancel_token.is_cancelled() { + break Ok(()); + } if spawned_clients >= self.client_pool_reserve_number { debug!("Got enough clients already: sleeping"); } else { @@ -93,6 +98,22 @@ impl ClientPool { } } + pub async fn disconnect_pool(&self) { + info!("Triggering Client Pool disconnect"); + self.cancel_token.cancel(); + info!( + "Client pool cancellation token cancelled: {}", + self.cancel_token.is_cancelled() + ); + let mut clients = self.clients.write().await; + while let Some(arc_client) = clients.pop() { + if let Ok(client) = Arc::try_unwrap(arc_client) { + info!("Killing reserve client {}", client.nym_address()); + client.disconnect().await; + } + } + } + pub async fn get_mixnet_client(&self) -> Option { debug!("Grabbing client from pool"); let mut clients = self.clients.write().await; @@ -113,6 +134,7 @@ impl ClientPool { Self { clients: Arc::clone(&self.clients), client_pool_reserve_number: self.client_pool_reserve_number, + cancel_token: self.cancel_token.clone(), } } } From 70c1e4fd85f4b1c4dce8d6639816abe10008c7da Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 13 Dec 2024 18:43:32 +0100 Subject: [PATCH 041/102] update mod file --- sdk/rust/nym-sdk/src/client_pool.rs | 48 +++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/sdk/rust/nym-sdk/src/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool.rs index e2fea7fb78f..adbe4dffdb4 100644 --- a/sdk/rust/nym-sdk/src/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool.rs @@ -3,14 +3,16 @@ //! use std::fmt; //! use std::sync::Arc; //! use tokio::sync::RwLock; +//! use tokio_util::sync::CancellationToken; //! use tracing::{debug, info, warn}; //! pub struct ClientPool { -//! clients: Arc>>>, // Collection of clients waiting to be used which are popped off in get_mixnet_client() +//! clients: Arc>>>, // Collection of clients waiting to be used which are popped off in get_mixnet_client() //! client_pool_reserve_number: usize, // Default # of clients to have available in pool in reserve waiting for incoming connections +//! cancel_token: CancellationToken, //! } -//! // This is only necessary for when you're wanting to check the addresses of the clients that are currently in the pool. +//! // This is only necessary for when you're wanting to check the addresses of the clients that are currently in the //! pool. //! impl fmt::Debug for ClientPool { //! fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { //! let clients_debug = match self.clients.try_read() { @@ -20,9 +22,7 @@ //! clients //! .iter() //! .enumerate() -//! .map(|(i, client)| { -//! format!("\n {}: {}", i, client.nym_address().to_string()) -//! }) +//! .map(|(i, client)| format!("\n {}: {}", i, client.nym_address())) //! .collect::>() //! .join(",") //! } else { @@ -36,7 +36,7 @@ //! } //! Err(_) => "".to_string(), //! }; -//! + //! let mut debug_struct = f.debug_struct("Pool"); //! debug_struct //! .field( @@ -53,10 +53,12 @@ //! ClientPool { //! clients: Arc::new(RwLock::new(Vec::new())), //! client_pool_reserve_number, +//! cancel_token: CancellationToken::new(), //! } //! } -//! -//! // The loop here is simple: if there aren't enough clients, create more. If you set clients to 0, repeatedly just sleep. + +//! // The loop here is simple: if there aren't enough clients, create more. If you set clients to 0, repeatedly //! just sleep. +//! // disconnect_pool() will kill this loop via the cancellation token. //! pub async fn start(&self) -> Result<()> { //! loop { //! let spawned_clients = self.clients.read().await.len(); @@ -65,6 +67,9 @@ //! "Currently spawned clients: {}: {:?}", //! spawned_clients, addresses //! ); +//! if self.cancel_token.is_cancelled() { +//! break Ok(()); +//! } //! if spawned_clients >= self.client_pool_reserve_number { //! debug!("Got enough clients already: sleeping"); //! } else { @@ -92,7 +97,23 @@ //! tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; //! } //! } -//! + +//! pub async fn disconnect_pool(&self) { +//! info!("Triggering Client Pool disconnect"); +//! self.cancel_token.cancel(); +//! info!( +//! "Client pool cancellation token cancelled: {}", +//! self.cancel_token.is_cancelled() +//! ); +//! let mut clients = self.clients.write().await; +//! while let Some(arc_client) = clients.pop() { +//! if let Ok(client) = Arc::try_unwrap(arc_client) { +//! info!("Killing reserve client {}", client.nym_address()); +//! client.disconnect().await; +//! } +//! } +//! } + //! pub async fn get_mixnet_client(&self) -> Option { //! debug!("Grabbing client from pool"); //! let mut clients = self.clients.write().await; @@ -100,19 +121,20 @@ //! .pop() //! .and_then(|arc_client| Arc::try_unwrap(arc_client).ok()) //! } -//! + //! pub async fn get_client_count(&self) -> usize { //! self.clients.read().await.len() //! } -//! + //! pub async fn get_pool_reserve(&self) -> usize { //! self.client_pool_reserve_number //! } -//! + //! pub fn clone(&self) -> Self { //! Self { //! clients: Arc::clone(&self.clients), -//! client_pool_reserve_number: *&self.client_pool_reserve_number, +//! client_pool_reserve_number: self.client_pool_reserve_number, +//! cancel_token: self.cancel_token.clone(), //! } //! } //! } From 0492d84ea89ed080309dd9042d76abc18db145f2 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 13:32:49 +0100 Subject: [PATCH 042/102] add cancel token disconnect fn --- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 1d566ba763d..d1f36686335 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -14,7 +14,7 @@ use tokio::{ use tokio_stream::StreamExt; use tokio_util::codec::{BytesCodec, FramedRead}; use tokio_util::sync::CancellationToken; -use tracing::{debug, info, instrument, warn}; +use tracing::{debug, info, instrument}; use utils::{MessageBuffer, Payload, ProxiedMessage}; const DEFAULT_CLOSE_TIMEOUT: u64 = 60; // seconds @@ -28,6 +28,7 @@ pub struct NymProxyClient { listen_port: String, close_timeout: u64, conn_pool: ClientPool, + cancel_token: CancellationToken, } impl NymProxyClient { @@ -47,6 +48,7 @@ impl NymProxyClient { listen_port: listen_port.to_string(), close_timeout, conn_pool: ClientPool::new(default_client_amount), + cancel_token: CancellationToken::new(), }) } @@ -75,17 +77,6 @@ impl NymProxyClient { Ok::<(), anyhow::Error>(()) }); - let cancel_token = CancellationToken::new(); - - tokio::spawn({ - let token = cancel_token.clone(); - async move { - tokio::signal::ctrl_c().await.unwrap(); - info!("Shutdown signal triggered"); - token.cancel(); - } - }); - loop { tokio::select! { stream = listener.accept() => { @@ -95,16 +86,21 @@ impl NymProxyClient { self.server_address, self.close_timeout, self.conn_pool.clone(), - cancel_token.clone(), + self.cancel_token.clone(), )); } - _ = cancel_token.cancelled() => { + _ = self.cancel_token.cancelled() => { break Ok(()); } } } } + pub async fn disconnect(&self) { + self.cancel_token.cancel(); + self.conn_pool.disconnect_pool().await; + } + // The main body of our logic, triggered on each accepted incoming tcp connection. To deal with assumptions about // streaming we have to implement an abstract session for each set of outgoing messages atop each connection, with message // IDs to deal with the fact that the mixnet does not enforce message ordering. @@ -139,14 +135,14 @@ impl NymProxyClient { client } None => { - warn!("Not enough clients in pool, creating ephemeral client"); + info!("Not enough clients in pool, creating ephemeral client"); let net = NymNetworkDetails::new_from_env(); let client = MixnetClientBuilder::new_ephemeral() .network_details(net) .build()? .connect_to_mixnet() .await?; - warn!( + info!( "Using {} for the moment, created outside of the connection pool", client.nym_address() ); @@ -259,8 +255,8 @@ impl NymProxyClient { return Ok::<(), anyhow::Error>(()) }, _ = tokio::time::sleep(tokio::time::Duration::from_secs(close_timeout)) => { - info!(" Closing write end of session: {}", session_id); - info!(" Triggering client shutdown"); + info!("Closing write end of session: {}", session_id); + info!("Triggering client shutdown"); client.disconnect().await; return Ok::<(), anyhow::Error>(()) }, From 1015251bd9fd8250ddc2df7a1aed812e183241b4 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 13:33:04 +0100 Subject: [PATCH 043/102] comments --- sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs | 3 ++- sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs index ff540302de6..6d61ea2ca11 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs @@ -124,8 +124,9 @@ async fn main() -> anyhow::Result<()> { } // Once timeout is passed, you can either wait for graceful shutdown or just hard stop it. + // TODO CHANGE make this a task listening for ctrl_c, add cancel token to loops + call client.disconnect() on ctrl_c signal::ctrl_c().await?; - println!("CTRL+C received, shutting down"); + println!("CTRL_C received, shutting down"); Ok(()) } diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index df41d503e70..c8388da8bf8 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -61,9 +61,9 @@ async fn main() -> anyhow::Result<()> { // We'll run the instance with a long timeout since we're sending everything down the same Tcp connection, so should be using a single session. // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. - // The final argument is how many clients to keep in reserve in the client pool when running the TcpProxy: since we're only expecting 1 connection in total, we can start with no clients in the pool, and the TcpProxyClient will just spin up an ephemeral client outside of the pool to use. + // The final argument is how many clients to keep in reserve in the client pool when running the TcpProxy. let proxy_client = - tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 0) + tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 1) .await?; tokio::spawn(async move { @@ -179,8 +179,9 @@ async fn main() -> anyhow::Result<()> { } // Once timeout is passed, you can either wait for graceful shutdown or just hard stop it. + // TODO CHANGE make this a task listening for ctrl_c, add cancel token to loops + call client.disconnect() on ctrl_c signal::ctrl_c().await?; - println!(":: CTRL+C received, shutting down + cleanup up proxy server config files"); + println!(":: CTRL_C received, shutting down + cleanup up proxy server config files"); fs::remove_dir_all(conf_path)?; Ok(()) } From 3573d1e6a6965f0ca610591ff4730d7f49b06a91 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 13:33:20 +0100 Subject: [PATCH 044/102] comments --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs index bf1f47869ef..55e6273dbb3 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs @@ -98,9 +98,9 @@ impl NymProxyServer { // IDs to deal with the fact that the mixnet does not enforce message ordering. // // There is an initial thread which does a bunch of setup logic: - // - Create a TcpStream connecting to our upstream server process. - // - Split incoming TcpStream into OwnedReadHalf and OwnedWriteHalf for concurrent read/write. - // - Create an Arc to store our session SURB - used for anonymous replies. + // - Create a TcpStream connecting to our upstream server process. + // - Split incoming TcpStream into OwnedReadHalf and OwnedWriteHalf for concurrent read/write. + // - Create an Arc to store our session SURB - used for anonymous replies. // // Then we spawn 2 tasks: // - 'Incoming' thread => deals with parsing and storing the SURB (used in Mixnet replies), deserialising and passing the incoming data from the Mixnet to the upstream server. From 08f5749887cccda31858b1a64160ccfd5a439611 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 14:25:27 +0100 Subject: [PATCH 045/102] add clone --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index d1f36686335..eb176f39d59 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -32,6 +32,17 @@ pub struct NymProxyClient { } impl NymProxyClient { + pub fn clone(&self) -> Self { + Self { + server_address: self.server_address.clone(), + listen_address: self.listen_address.clone(), + listen_port: self.listen_port.clone(), + close_timeout: self.close_timeout, + conn_pool: self.conn_pool.clone(), + cancel_token: self.cancel_token.clone(), + } + } + pub async fn new( server_address: Recipient, listen_address: &str, From 7d2b1ca9a8e7fa7c321e6c516ad8477d762d4e61 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 14:25:39 +0100 Subject: [PATCH 046/102] added disconnect thread --- .../nym-sdk/examples/tcp_proxy_multistream.rs | 28 ++++++++++++++--- .../examples/tcp_proxy_single_connection.rs | 30 +++++++++++++++---- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs index 6d61ea2ca11..4134ab325df 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs @@ -10,6 +10,7 @@ use tokio::net::TcpStream; use tokio::signal; use tokio_stream::StreamExt; use tokio_util::codec; +use tokio_util::sync::CancellationToken; use tracing_subscriber::{fmt, prelude::*, EnvFilter}; #[derive(Serialize, Deserialize, Debug)] @@ -53,17 +54,37 @@ async fn main() -> anyhow::Result<()> { let proxy_client = tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 45, Some(env), 2).await?; + // For our disconnect() logic below + let proxy_clone = proxy_client.clone(); + tokio::spawn(async move { proxy_client.run().await?; Ok::<(), anyhow::Error>(()) }); + let example_cancel_token = CancellationToken::new(); + let client_cancel_token = example_cancel_token.clone(); + let watcher_cancel_token = example_cancel_token.clone(); + + // Cancel listener thread + tokio::spawn(async move { + signal::ctrl_c().await?; + println!(":: CTRL_C received, shutting down + cleanup up proxy server config files"); + watcher_cancel_token.cancel(); + proxy_clone.disconnect().await; + Ok::<(), anyhow::Error>(()) + }); + println!("waiting for everything to be set up.."); tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; println!("done. sending bytes"); // In the info traces you will see the different session IDs being set up, one for each TcpStream. for i in 0..8 { + let client_cancel_inner_token = client_cancel_token.clone(); + if client_cancel_token.is_cancelled() { + break; + } let conn_id = i; let local_tcp_addr = format!("127.0.0.1:{}", listen_port.clone()); tokio::spawn(async move { @@ -81,6 +102,9 @@ async fn main() -> anyhow::Result<()> { // Lets just send a bunch of messages to the server with variable delays between them, with a message and tcp connection ids to keep track of ordering on the server side (for illustrative purposes **only**; keeping track of anonymous replies is handled by the proxy under the hood with Single Use Reply Blocks (SURBs); for this illustration we want some kind of app-level message id, but irl most of the time you'll probably be parsing on e.g. the incoming response type instead) tokio::spawn(async move { for i in 0..8 { + if client_cancel_inner_token.is_cancelled() { + break; + } let mut rng = SmallRng::from_entropy(); let delay: f64 = rng.gen_range(2.5..5.0); tokio::time::sleep(tokio::time::Duration::from_secs_f64(delay)).await; @@ -123,10 +147,6 @@ async fn main() -> anyhow::Result<()> { tokio::time::sleep(tokio::time::Duration::from_secs_f64(delay)).await; } - // Once timeout is passed, you can either wait for graceful shutdown or just hard stop it. - // TODO CHANGE make this a task listening for ctrl_c, add cancel token to loops + call client.disconnect() on ctrl_c - signal::ctrl_c().await?; - println!("CTRL_C received, shutting down"); Ok(()) } diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index c8388da8bf8..cc7c934f685 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -10,6 +10,7 @@ use tokio::net::{TcpListener, TcpStream}; use tokio::signal; use tokio_stream::StreamExt; use tokio_util::codec; +use tokio_util::sync::CancellationToken; use tracing_subscriber::{fmt, prelude::*, EnvFilter}; #[derive(Serialize, Deserialize, Debug)] @@ -66,6 +67,9 @@ async fn main() -> anyhow::Result<()> { tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 1) .await?; + // For our disconnect() logic below + let proxy_clone = proxy_client.clone(); + tokio::spawn(async move { proxy_server.run_with_shutdown().await?; Ok::<(), anyhow::Error>(()) @@ -76,10 +80,28 @@ async fn main() -> anyhow::Result<()> { Ok::<(), anyhow::Error>(()) }); + let example_cancel_token = CancellationToken::new(); + let server_cancel_token = example_cancel_token.clone(); + let client_cancel_token = example_cancel_token.clone(); + let watcher_cancel_token = example_cancel_token.clone(); + + // Cancel listener thread + tokio::spawn(async move { + signal::ctrl_c().await?; + println!(":: CTRL_C received, shutting down + cleanup up proxy server config files"); + fs::remove_dir_all(conf_path)?; + watcher_cancel_token.cancel(); + proxy_clone.disconnect().await; + Ok::<(), anyhow::Error>(()) + }); + // 'Server side' thread: echo back incoming as response to the messages sent in the 'client side' thread below tokio::spawn(async move { let listener = TcpListener::bind(upstream_tcp_addr).await?; loop { + if server_cancel_token.is_cancelled() { + break; + } let (socket, _) = listener.accept().await.unwrap(); let (read, mut write) = socket.into_split(); let codec = codec::BytesCodec::new(); @@ -139,6 +161,9 @@ async fn main() -> anyhow::Result<()> { tokio::spawn(async move { let mut rng = SmallRng::from_entropy(); for i in 0..10 { + if client_cancel_token.is_cancelled() { + break; + } let random_bytes = gen_bytes_fixed(i as usize); let msg = ExampleMessage { message_id: i, @@ -178,11 +203,6 @@ async fn main() -> anyhow::Result<()> { } } - // Once timeout is passed, you can either wait for graceful shutdown or just hard stop it. - // TODO CHANGE make this a task listening for ctrl_c, add cancel token to loops + call client.disconnect() on ctrl_c - signal::ctrl_c().await?; - println!(":: CTRL_C received, shutting down + cleanup up proxy server config files"); - fs::remove_dir_all(conf_path)?; Ok(()) } From f819b336287d7f5410774b28229f3495420af46d Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 15:00:54 +0100 Subject: [PATCH 047/102] update example files tcpproxy --- .../rust/tcpproxy/examples/multiconn.mdx | 68 +++++++++++-------- .../rust/tcpproxy/examples/singleconn.mdx | 54 ++++++++++++--- 2 files changed, 84 insertions(+), 38 deletions(-) diff --git a/documentation/docs/pages/developers/rust/tcpproxy/examples/multiconn.mdx b/documentation/docs/pages/developers/rust/tcpproxy/examples/multiconn.mdx index 3a97858119d..a003b1bee8d 100644 --- a/documentation/docs/pages/developers/rust/tcpproxy/examples/multiconn.mdx +++ b/documentation/docs/pages/developers/rust/tcpproxy/examples/multiconn.mdx @@ -18,6 +18,8 @@ use tokio::net::TcpStream; use tokio::signal; use tokio_stream::StreamExt; use tokio_util::codec; +use tokio_util::sync::CancellationToken; +use tracing_subscriber::{fmt, prelude::*, EnvFilter}; #[derive(Serialize, Deserialize, Debug)] struct ExampleMessage { @@ -26,6 +28,8 @@ struct ExampleMessage { tcp_conn: i8, } +// This example just starts off a bunch of Tcp connections on a loop to a remote endpoint: in this case the TcpListener behind the NymProxyServer instance on the echo server found in `nym/tools/echo-server/`. It pipes a few messages to it, logs the replies, and keeps track of the number of replies received per connection. +// // To run: // - run the echo server with `cargo run` // - run this example with `cargo run --example tcp_proxy_multistream -- ` e.g. @@ -40,8 +44,13 @@ async fn main() -> anyhow::Result<()> { // Nym client logging is very informative but quite verbose. // The Message Decay related logging gives you an ideas of the internals of the proxy message ordering: you need to switch // to DEBUG to see the contents of the msg buffer, sphinx packet chunking, etc. - tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) + tracing_subscriber::registry() + .with(fmt::layer()) + .with( + EnvFilter::new("info") + .add_directive("nym_sdk::client_pool=info".parse().unwrap()) + .add_directive("nym_sdk::tcp_proxy_client=debug".parse().unwrap()), + ) .init(); let env_path = env::args().nth(2).expect("Env file not specified"); @@ -49,23 +58,42 @@ async fn main() -> anyhow::Result<()> { let listen_port = env::args().nth(3).expect("Port not specified"); - // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. + // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. The final argument is how many clients to keep in reserve in the client pool when running the TcpProxy. let proxy_client = - tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 45, Some(env)).await?; + tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 45, Some(env), 2).await?; + + // For our disconnect() logic below + let proxy_clone = proxy_client.clone(); tokio::spawn(async move { proxy_client.run().await?; Ok::<(), anyhow::Error>(()) }); + let example_cancel_token = CancellationToken::new(); + let client_cancel_token = example_cancel_token.clone(); + let watcher_cancel_token = example_cancel_token.clone(); + + // Cancel listener thread + tokio::spawn(async move { + signal::ctrl_c().await?; + println!(":: CTRL_C received, shutting down + cleanup up proxy server config files"); + watcher_cancel_token.cancel(); + proxy_clone.disconnect().await; + Ok::<(), anyhow::Error>(()) + }); + println!("waiting for everything to be set up.."); tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; println!("done. sending bytes"); // In the info traces you will see the different session IDs being set up, one for each TcpStream. - for i in 0..4 { + for i in 0..8 { + let client_cancel_inner_token = client_cancel_token.clone(); + if client_cancel_token.is_cancelled() { + break; + } let conn_id = i; - println!("Starting TCP connection {}", conn_id); let local_tcp_addr = format!("127.0.0.1:{}", listen_port.clone()); tokio::spawn(async move { // Now the client and server proxies are running we can create and pipe traffic to/from @@ -81,7 +109,10 @@ async fn main() -> anyhow::Result<()> { // Lets just send a bunch of messages to the server with variable delays between them, with a message and tcp connection ids to keep track of ordering on the server side (for illustrative purposes **only**; keeping track of anonymous replies is handled by the proxy under the hood with Single Use Reply Blocks (SURBs); for this illustration we want some kind of app-level message id, but irl most of the time you'll probably be parsing on e.g. the incoming response type instead) tokio::spawn(async move { - for i in 0..4 { + for i in 0..8 { + if client_cancel_inner_token.is_cancelled() { + break; + } let mut rng = SmallRng::from_entropy(); let delay: f64 = rng.gen_range(2.5..5.0); tokio::time::sleep(tokio::time::Duration::from_secs_f64(delay)).await; @@ -96,12 +127,7 @@ async fn main() -> anyhow::Result<()> { .write_all(&serialised) .await .expect("couldn't write to stream"); - println!( - ">> client sent {}: {} bytes on conn {}", - &i, - msg.message_bytes.len(), - &conn_id - ); + println!(">> client sent msg {} on conn {}", &i, &conn_id); } Ok::<(), anyhow::Error>(()) }); @@ -113,17 +139,8 @@ async fn main() -> anyhow::Result<()> { while let Some(Ok(bytes)) = framed_read.next().await { match bincode::deserialize::(&bytes) { Ok(msg) => { - println!( - "<< client received {}: {} bytes on conn {}", - msg.message_id, - msg.message_bytes.len(), - msg.tcp_conn - ); reply_counter += 1; - println!( - "tcp connection {} replies received {}/4", - msg.tcp_conn, reply_counter - ); + println!("<< conn {} received {}/8", msg.tcp_conn, reply_counter); } Err(e) => { println!("<< client received something that wasn't an example message of {} bytes. error: {}", bytes.len(), e); @@ -138,15 +155,12 @@ async fn main() -> anyhow::Result<()> { tokio::time::sleep(tokio::time::Duration::from_secs_f64(delay)).await; } - // Once timeout is passed, you can either wait for graceful shutdown or just hard stop it. - signal::ctrl_c().await?; - println!("CTRL+C received, shutting down"); Ok(()) } // emulate a series of small messages followed by a closing larger one fn gen_bytes_fixed(i: usize) -> Vec { - let amounts = [10, 15, 50, 1000]; + let amounts = [10, 15, 50, 1000, 10, 15, 500, 2000]; let len = amounts[i]; let mut rng = rand::thread_rng(); (0..len).map(|_| rng.gen::()).collect() diff --git a/documentation/docs/pages/developers/rust/tcpproxy/examples/singleconn.mdx b/documentation/docs/pages/developers/rust/tcpproxy/examples/singleconn.mdx index 1ee11968684..f5f1ce8c7ff 100644 --- a/documentation/docs/pages/developers/rust/tcpproxy/examples/singleconn.mdx +++ b/documentation/docs/pages/developers/rust/tcpproxy/examples/singleconn.mdx @@ -21,6 +21,8 @@ use tokio::net::{TcpListener, TcpStream}; use tokio::signal; use tokio_stream::StreamExt; use tokio_util::codec; +use tokio_util::sync::CancellationToken; +use tracing_subscriber::{fmt, prelude::*, EnvFilter}; #[derive(Serialize, Deserialize, Debug)] struct ExampleMessage { @@ -28,7 +30,14 @@ struct ExampleMessage { message_bytes: Vec, } - +// This is a basic example which opens a single TCP connection and writes a bunch of messages between a client and an echo +// server, so only uses a single session under the hood and doesn't really show off the message ordering capabilities; this is mainly +// just a quick introductory illustration on how: +// - the mixnet does message ordering +// - the NymProxyClient and NymProxyServer can be hooked into and used to communicate between two otherwise pretty vanilla TcpStreams +// +// For a more irl example checkout tcp_proxy_multistream.rs +// // Run this with: // `cargo run --example tcp_proxy_single_connection ` e.g. // `cargo run --example tcp_proxy_single_connection 8081 ../../../envs/canary.env 8080 ` @@ -39,8 +48,9 @@ async fn main() -> anyhow::Result<()> { // Comment this out to just see println! statements from this example, as Nym client logging is very informative but quite verbose. // The Message Decay related logging gives you an ideas of the internals of the proxy message ordering. To see the contents of the msg buffer, sphinx packet chunking, etc change the tracing::Level to DEBUG. - tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) + tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::new("nym_sdk::tcp_proxy=info")) .init(); let server_port = env::args() @@ -63,10 +73,14 @@ async fn main() -> anyhow::Result<()> { // We'll run the instance with a long timeout since we're sending everything down the same Tcp connection, so should be using a single session. // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. + // The final argument is how many clients to keep in reserve in the client pool when running the TcpProxy. let proxy_client = - tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 60, Some(env)) + tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 1) .await?; + // For our disconnect() logic below + let proxy_clone = proxy_client.clone(); + tokio::spawn(async move { proxy_server.run_with_shutdown().await?; Ok::<(), anyhow::Error>(()) @@ -77,10 +91,28 @@ async fn main() -> anyhow::Result<()> { Ok::<(), anyhow::Error>(()) }); + let example_cancel_token = CancellationToken::new(); + let server_cancel_token = example_cancel_token.clone(); + let client_cancel_token = example_cancel_token.clone(); + let watcher_cancel_token = example_cancel_token.clone(); + + // Cancel listener thread + tokio::spawn(async move { + signal::ctrl_c().await?; + println!(":: CTRL_C received, shutting down + cleanup up proxy server config files"); + fs::remove_dir_all(conf_path)?; + watcher_cancel_token.cancel(); + proxy_clone.disconnect().await; + Ok::<(), anyhow::Error>(()) + }); + // 'Server side' thread: echo back incoming as response to the messages sent in the 'client side' thread below tokio::spawn(async move { let listener = TcpListener::bind(upstream_tcp_addr).await?; loop { + if server_cancel_token.is_cancelled() { + break; + } let (socket, _) = listener.accept().await.unwrap(); let (read, mut write) = socket.into_split(); let codec = codec::BytesCodec::new(); @@ -118,9 +150,9 @@ async fn main() -> anyhow::Result<()> { Ok::<(), anyhow::Error>(()) }); - // Just wait for Nym clients to connect, TCP clients to bind, etc. + // Just wait for Nym clients to connect, TCP clients to bind, etc. If there isn't a client in the pool (or you started it with 0) already then the TcpProxyClient just spins up an ephemeral client itself. println!("waiting for everything to be set up.."); - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; println!("done. sending bytes"); // Now the client and server proxies are running we can create and pipe traffic to/from @@ -140,6 +172,9 @@ async fn main() -> anyhow::Result<()> { tokio::spawn(async move { let mut rng = SmallRng::from_entropy(); for i in 0..10 { + if client_cancel_token.is_cancelled() { + break; + } let random_bytes = gen_bytes_fixed(i as usize); let msg = ExampleMessage { message_id: i, @@ -179,15 +214,12 @@ async fn main() -> anyhow::Result<()> { } } - // Once timeout is passed, you can either wait for graceful shutdown or just hard stop it. - signal::ctrl_c().await?; - println!(":: CTRL+C received, shutting down + cleanup up proxy server config files"); - fs::remove_dir_all(conf_path)?; Ok(()) } fn gen_bytes_fixed(i: usize) -> Vec { - let amounts = vec![1, 10, 50, 100, 150, 200, 350, 500, 750, 1000]; + // let amounts = vec![1, 10, 50, 100, 150, 200, 350, 500, 750, 1000]; + let amounts = [158, 1088, 505, 1001, 150, 200, 3500, 500, 750, 100]; let len = amounts[i]; let mut rng = rand::thread_rng(); (0..len).map(|_| rng.gen::()).collect() From d59f05f9d4afefb77cdc7c20c6583241841d43db Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 18:46:49 +0100 Subject: [PATCH 048/102] client pool docs --- .../docs/pages/developers/rust/_meta.json | 1 + .../pages/developers/rust/client-pool.mdx | 7 ++ .../developers/rust/client-pool/_meta.json | 4 + .../rust/client-pool/architecture.mdx | 19 ++++ .../developers/rust/client-pool/example.md | 100 ++++++++++++++++++ .../developers/rust/development-status.md | 3 + .../docs/pages/developers/rust/ffi.mdx | 9 +- .../developers/rust/tcpproxy/architecture.mdx | 2 +- 8 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 documentation/docs/pages/developers/rust/client-pool.mdx create mode 100644 documentation/docs/pages/developers/rust/client-pool/_meta.json create mode 100644 documentation/docs/pages/developers/rust/client-pool/architecture.mdx create mode 100644 documentation/docs/pages/developers/rust/client-pool/example.md diff --git a/documentation/docs/pages/developers/rust/_meta.json b/documentation/docs/pages/developers/rust/_meta.json index 3fca1da31fc..d193319720a 100644 --- a/documentation/docs/pages/developers/rust/_meta.json +++ b/documentation/docs/pages/developers/rust/_meta.json @@ -3,6 +3,7 @@ "development-status": "Development Status", "mixnet": "Mixnet Module", "tcpproxy": "TcpProxy Module", + "client-pool": "Client Pool", "ffi": "FFI", "tutorials": "Tutorials (Coming Soon)" } diff --git a/documentation/docs/pages/developers/rust/client-pool.mdx b/documentation/docs/pages/developers/rust/client-pool.mdx new file mode 100644 index 00000000000..584b642f719 --- /dev/null +++ b/documentation/docs/pages/developers/rust/client-pool.mdx @@ -0,0 +1,7 @@ +# Client Pool + +We have a configurable-size Client Pool for processes that require multiple clients in quick succession (this is used by default by the [`TcpProxyClient`](./tcpproxy) for instance) + +This will be useful for developers looking to build connection logic, or just are using raw SDK clients in a sitatuation where there are multiple connections with a lot of churn. + +> You can find this code [here](https://github.com/nymtech/nym/blob/develop/sdk/rust/nym-sdk/examples/client_pool.rs) diff --git a/documentation/docs/pages/developers/rust/client-pool/_meta.json b/documentation/docs/pages/developers/rust/client-pool/_meta.json new file mode 100644 index 00000000000..3d6dc8b87b3 --- /dev/null +++ b/documentation/docs/pages/developers/rust/client-pool/_meta.json @@ -0,0 +1,4 @@ +{ + "architecture": "Architecture", + "example": "Example" +} diff --git a/documentation/docs/pages/developers/rust/client-pool/architecture.mdx b/documentation/docs/pages/developers/rust/client-pool/architecture.mdx new file mode 100644 index 00000000000..a50b2126e65 --- /dev/null +++ b/documentation/docs/pages/developers/rust/client-pool/architecture.mdx @@ -0,0 +1,19 @@ +# Client Pool Architecture + +## Motivations +In situations where multiple connections are expected, and the number of connections can vary greatly, the Pool reduces time spent waiting for the creation of a Mixnet Client blocking your code sending traffic through the Mixnet. Instead, a configurable number of Clients can be generated and run in the background which can be very quickly grabbed, used, and disconnected. + +The Pool can be simply run as a background process for the runtime of your program. + +## Clients & Lifetimes +The pool currently operates on the principle of creating an **ephemeral Client** which is used once and then disconnected after use. Using the [`TcpProxy`](../tcpproxy) as an example, a client is used for the lifetime of a single incoming TCP connection; after the TCP connection is closed, the Mixnet client is disconnected. + +Clients are popped from the pool when in use, and another Client is created to take its place. If connections are coming in faster than Clients are replenished, logic can be easily created to instead generate an ephemeral Client or to wait; this is up to the developer to decide. You can see an example of this logic on the example on the next page. + +## Runtime Loop +Aside from a few helper / getter functions and a graceful `disconnect_pool()`, the Client Pool is mostly made up of a very simple loop around some conditional logic making up `start()`: +- if the number of Clients in the pool is `< client_pool_reserve_number` (set on `new()`) then create more, +- if the number of Clients in the pool `== client_pool_reserve_number` (set on `new()`) then `sleep`, +- if `client_pool_reserve_number == 0` just `sleep`. + +`disconnect_pool()` will cause this loop to `break` via cancellation token. diff --git a/documentation/docs/pages/developers/rust/client-pool/example.md b/documentation/docs/pages/developers/rust/client-pool/example.md new file mode 100644 index 00000000000..1b3c499f348 --- /dev/null +++ b/documentation/docs/pages/developers/rust/client-pool/example.md @@ -0,0 +1,100 @@ +# Client Pool Example + +> You can find this code [here](https://github.com/nymtech/nym/blob/develop/sdk/rust/nym-sdk/examples/client_pool.rs) + +```rust +use anyhow::Result; +use nym_network_defaults::setup_env; +use nym_sdk::client_pool::ClientPool; +use nym_sdk::mixnet::{MixnetClientBuilder, NymNetworkDetails}; +use tokio::signal::ctrl_c; + +// This client pool is used internally by the TcpProxyClient but can also be used by the Mixnet module, in case you're quickly swapping clients in and out but won't want to use the TcpProxy module. +// +// Run with: cargo run --example client_pool -- ../../../envs/.env +#[tokio::main] +async fn main() -> Result<()> { + nym_bin_common::logging::setup_logging(); + setup_env(std::env::args().nth(1)); + + let conn_pool = ClientPool::new(2); // Start the client pool with 1 client always being kept in reserve + let client_maker = conn_pool.clone(); + tokio::spawn(async move { + client_maker.start().await?; + Ok::<(), anyhow::Error>(()) + }); + + println!("\n\nWaiting a few seconds to fill pool\n\n"); + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + + let pool_clone_one = conn_pool.clone(); + let pool_clone_two = conn_pool.clone(); + + tokio::spawn(async move { + let client_one = match pool_clone_one.get_mixnet_client().await { + Some(client) => { + println!("Grabbed client {} from pool", client.nym_address()); + client + } + None => { + println!("Not enough clients in pool, creating ephemeral client"); + let net = NymNetworkDetails::new_from_env(); + let client = MixnetClientBuilder::new_ephemeral() + .network_details(net) + .build()? + .connect_to_mixnet() + .await?; + println!( + "Using {} for the moment, created outside of the connection pool", + client.nym_address() + ); + client + } + }; + let our_address = client_one.nym_address(); + println!("\n\nClient 1: {our_address}\n\n"); + client_one.disconnect().await; + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something + return Ok::<(), anyhow::Error>(()); + }); + + tokio::spawn(async move { + let client_two = match pool_clone_two.get_mixnet_client().await { + Some(client) => { + println!("Grabbed client {} from pool", client.nym_address()); + client + } + None => { + println!("Not enough clients in pool, creating ephemeral client"); + let net = NymNetworkDetails::new_from_env(); + let client = MixnetClientBuilder::new_ephemeral() + .network_details(net) + .build()? + .connect_to_mixnet() + .await?; + println!( + "Using {} for the moment, created outside of the connection pool", + client.nym_address() + ); + client + } + }; + let our_address = *client_two.nym_address(); + println!("\n\nClient 2: {our_address}\n\n"); + client_two.disconnect().await; + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something + return Ok::<(), anyhow::Error>(()); + }); + + wait_for_ctrl_c(conn_pool).await?; + Ok(()) +} + +async fn wait_for_ctrl_c(pool: ClientPool) -> Result<()> { + println!("\n\nPress CTRL_C to disconnect pool\n\n"); + ctrl_c().await?; + println!("CTRL_C received. Killing client pool"); + pool.disconnect_pool().await; + Ok(()) +} +``` diff --git a/documentation/docs/pages/developers/rust/development-status.md b/documentation/docs/pages/developers/rust/development-status.md index 71ec96057c1..cde8d04d720 100644 --- a/documentation/docs/pages/developers/rust/development-status.md +++ b/documentation/docs/pages/developers/rust/development-status.md @@ -7,9 +7,12 @@ In the future the SDK will be made up of several modules, each of which will all |-----------|---------------------------------------------------------------------------------------|----------| | Mixnet | Create / load clients & keypairs, subscribe to Mixnet events, send & receive messages | ✔️ | | TcpProxy | Utilise the TcpProxyClient and TcpProxyServer abstractions for streaming | ✔️ | +| ClientPool| Create a pool of quickly useable Mixnet clients | ✔️ | | Ecash | Create & verify Ecash credentials | ❌ | | Validator | Sign & broadcast Nyx blockchain transactions, query the blockchain | ❌ | The `Mixnet` module currently exposes the logic of two clients: the [websocket client](../clients/websocket), and the [socks client](../clients/socks5). The `TcpProxy` module exposes functionality to set up client/server instances that expose a localhost TcpSocket to read/write to. + +The `ClientPool` is a configurable pool of ephemeral clients which can be created as a background process and quickly grabbed. diff --git a/documentation/docs/pages/developers/rust/ffi.mdx b/documentation/docs/pages/developers/rust/ffi.mdx index 0f4aba8fa01..819b0307a1f 100644 --- a/documentation/docs/pages/developers/rust/ffi.mdx +++ b/documentation/docs/pages/developers/rust/ffi.mdx @@ -22,7 +22,7 @@ The main functionality of exposed functions will be imported from `sdk/ffi/share Furthermore, the `shared/` code makes sure that client access is thread-safe, and that client actions happen in blocking threads on the Rust side of the FFI boundary. -### Mixnet Module +## Mixnet Module This is the basic mixnet component of the SDK, exposing client functionality with which people can build custom interfaces with the Mixnet. These functions are exposed to both Go and C/C++ via the `sdk/ffi/shared/` crate. | `shared/lib.rs` function | Rust Function | @@ -36,13 +36,13 @@ This is the basic mixnet component of the SDK, exposing client functionality wit > We have also implemented `listen_for_incoming_internal()` which is a wrapper around the Mixnet client's `wait_for_messages()`. This is a helper method for listening out for and handling incoming messages. -#### Currently Unsupported Functionality +### Currently Unsupported Functionality At the time of writing the following functionality is not exposed to the shared FFI library: - `split_sender()`: the ability to [split a client into sender and receiver](./mixnet/examples/split-send) for concurrent send/receive. - The use of [custom network topologies](./mixnet/examples/custom-topology). - `Socks5::new()`: creation and use of the [socks5/4a/4 proxy client](./mixnet/examples/socks). -### TcpProxy Module +## TcpProxy Module A connection abstraction which exposes a local TCP socket which developers are able to interact with basically as expected, being able to read/write to/from a bytestream, without really having to take into account the workings of the Mixnet/Sphinx/the [message-based](../concepts/messages) format of the underlying client. @@ -58,3 +58,6 @@ A connection abstraction which exposes a local TCP socket which developers are a | `proxy_server_new_internal(upstream_address: &str, config_dir: &str, env: Option)` | `NymProxyServer::new(upstream_address, config_dir, env)` | | `proxy_server_run_internal()` | `NymProxyServer.run_with_shutdown()` | | `proxy_server_address_internal()` | `NymProxyServer.nym_address()` | + +## Client Pool +There are currently no FFI bindings for the Client Pool. This will be coming in the future. diff --git a/documentation/docs/pages/developers/rust/tcpproxy/architecture.mdx b/documentation/docs/pages/developers/rust/tcpproxy/architecture.mdx index 862d00c90fc..06880bdc2c7 100644 --- a/documentation/docs/pages/developers/rust/tcpproxy/architecture.mdx +++ b/documentation/docs/pages/developers/rust/tcpproxy/architecture.mdx @@ -13,7 +13,7 @@ The motivation behind the creation of the `TcpProxy` module is to allow develope ## Clients Each of the sub-modules exposed by the `TcpProxy` deal with Nym clients in a different way. -- the `NymProxyClient` creates an ephemeral client per new TCP connection, which is closed according to the configurable timeout: we map one ephemeral client per TCP connection. This is to deal with multiple simultaneous streams. In the future, this will be superceded by a connection pool in order to speed up new connections. +- the `NymProxyClient` relies on the [`Client Pool`](../client-pool) to create clients and keep a certain number of them in reserve. If the amount of incoming TCP connections rises quicker than the Client Pool can create clients, or you have the pool size set to `0`, the `TcpProxyClient` creates an ephemeral client per new TCP connection, which is closed according to the configurable timeout: we map one ephemeral client per TCP connection. This is to deal with multiple simultaneous streams. In the future, this will be superceded by a connection pool in order to speed up new connections. - the `NymProxyServer` has a single Nym client with a persistent identity. ## Framing From 7fa1cdba93c8e7692e8dc5b1c63bf73d35341a8a Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 18:51:05 +0100 Subject: [PATCH 049/102] remove comments for future ffi push + lower default pool size from 4 to 2 --- sdk/ffi/shared/src/lib.rs | 23 ------------------- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 2 +- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/sdk/ffi/shared/src/lib.rs b/sdk/ffi/shared/src/lib.rs index 4a09a63c8db..e7a770f2699 100644 --- a/sdk/ffi/shared/src/lib.rs +++ b/sdk/ffi/shared/src/lib.rs @@ -22,15 +22,6 @@ lazy_static! { static ref RUNTIME: Runtime = Runtime::new().unwrap(); } -// TODO create get_client() to use in fns and remove code repetition -fn get_client_as_ref() -> bool { - NYM_CLIENT.lock().unwrap().as_ref().is_some() -} - -fn get_client_guard() -> bool { - todo!() -} - pub fn init_ephemeral_internal() -> Result<(), Error> { if NYM_CLIENT.lock().unwrap().as_ref().is_some() { bail!("client already exists"); @@ -46,20 +37,6 @@ pub fn init_ephemeral_internal() -> Result<(), Error> { Ok::<(), Error>(()) })?; } - // if get_client_as_ref() { - // RUNTIME.block_on(async move { - // let init_client = MixnetClient::connect_new().await?; - // let mut client = NYM_CLIENT.try_lock(); - // if let Ok(ref mut client) = client { - // **client = Some(init_client); - // } else { - // return Err(anyhow!("couldnt lock ephemeral NYM_CLIENT")); - // } - // Ok::<(), Error>(()) - // })?; - // } else { - // bail!("client already exists: no need to reinitialise"); - // } Ok(()) } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index eb176f39d59..ae74464a699 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -20,7 +20,7 @@ use utils::{MessageBuffer, Payload, ProxiedMessage}; const DEFAULT_CLOSE_TIMEOUT: u64 = 60; // seconds const DEFAULT_LISTEN_HOST: &str = "127.0.0.1"; const DEFAULT_LISTEN_PORT: &str = "8080"; -const DEFAULT_CLIENT_POOL_SIZE: usize = 4; +const DEFAULT_CLIENT_POOL_SIZE: usize = 2; pub struct NymProxyClient { server_address: Recipient, From 7608ca768f751e262c329f9b3497555078b62e47 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 18:51:19 +0100 Subject: [PATCH 050/102] comment on ffi --- documentation/docs/pages/developers/rust/ffi.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/docs/pages/developers/rust/ffi.mdx b/documentation/docs/pages/developers/rust/ffi.mdx index 819b0307a1f..848ca1755e3 100644 --- a/documentation/docs/pages/developers/rust/ffi.mdx +++ b/documentation/docs/pages/developers/rust/ffi.mdx @@ -60,4 +60,4 @@ A connection abstraction which exposes a local TCP socket which developers are a | `proxy_server_address_internal()` | `NymProxyServer.nym_address()` | ## Client Pool -There are currently no FFI bindings for the Client Pool. This will be coming in the future. +There are currently no FFI bindings for the Client Pool. This will be coming in the future. The bindings for the TcpProxy have been updated to be able to use the Client Pool under the hood, but the standalone Pool is not yet exposed to FFI. From 8864921260f285e66edaf6aba1b5dd370a8f466c Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 16 Dec 2024 19:05:03 +0100 Subject: [PATCH 051/102] update command help --- .../docs/pages/developers/tools/standalone-tcpproxy.mdx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/documentation/docs/pages/developers/tools/standalone-tcpproxy.mdx b/documentation/docs/pages/developers/tools/standalone-tcpproxy.mdx index 177a080a5cb..659b1c70de5 100644 --- a/documentation/docs/pages/developers/tools/standalone-tcpproxy.mdx +++ b/documentation/docs/pages/developers/tools/standalone-tcpproxy.mdx @@ -39,6 +39,8 @@ Options: Listen port [default: 8080] -e, --env-path Optional env filepath - if none is supplied then the proxy defaults to using mainnet else just use a path to one of the supplied files in envs/ e.g. ./envs/sandbox.env + --client-pool-reserve + How many clients to have running in reserve for quick access by incoming connections [default: 2] -h, --help Print help ``` From 5cc54842979f0c85eb826a0d40fb5c6dcf31bc90 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 00:40:05 +0100 Subject: [PATCH 052/102] clone impl --- .../nym-sdk/src/client_pool/client_pool.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs index 37d20858245..49ec5c79df5 100644 --- a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool/client_pool.rs @@ -48,6 +48,16 @@ impl fmt::Debug for ClientPool { } } +impl std::clone::Clone for ClientPool { + fn clone(&self) -> Self { + Self { + clients: Arc::clone(&self.clients), + client_pool_reserve_number: self.client_pool_reserve_number, + cancel_token: self.cancel_token.clone(), + } + } +} + impl ClientPool { pub fn new(client_pool_reserve_number: usize) -> Self { ClientPool { @@ -129,12 +139,4 @@ impl ClientPool { pub async fn get_pool_reserve(&self) -> usize { self.client_pool_reserve_number } - - pub fn clone(&self) -> Self { - Self { - clients: Arc::clone(&self.clients), - client_pool_reserve_number: self.client_pool_reserve_number, - cancel_token: self.cancel_token.clone(), - } - } } From 349e95aa074defa37fecb9837b8c739ea40464e1 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 00:40:26 +0100 Subject: [PATCH 053/102] remove clone --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index ae74464a699..acc0a760707 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -34,7 +34,7 @@ pub struct NymProxyClient { impl NymProxyClient { pub fn clone(&self) -> Self { Self { - server_address: self.server_address.clone(), + server_address: self.server_address, listen_address: self.listen_address.clone(), listen_port: self.listen_port.clone(), close_timeout: self.close_timeout, From f78bb83eabb89e37122341ba9b053de01b6a3f22 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 00:59:47 +0100 Subject: [PATCH 054/102] fix clippy --- .../examples/tcp_proxy_single_connection.rs | 1 - sdk/rust/nym-sdk/src/client_pool.rs | 4 +- .../{client_pool.rs => mixnet_client_pool.rs} | 4 +- sdk/rust/nym-sdk/src/tcp_proxy.rs | 217 +++++++++++++++++- .../nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 12 +- 5 files changed, 220 insertions(+), 18 deletions(-) rename sdk/rust/nym-sdk/src/client_pool/{client_pool.rs => mixnet_client_pool.rs} (98%) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs index cc7c934f685..f2ca78c9311 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_single_connection.rs @@ -207,7 +207,6 @@ async fn main() -> anyhow::Result<()> { } fn gen_bytes_fixed(i: usize) -> Vec { - // let amounts = vec![1, 10, 50, 100, 150, 200, 350, 500, 750, 1000]; let amounts = [158, 1088, 505, 1001, 150, 200, 3500, 500, 750, 100]; let len = amounts[i]; let mut rng = rand::thread_rng(); diff --git a/sdk/rust/nym-sdk/src/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool.rs index adbe4dffdb4..0a2471275b3 100644 --- a/sdk/rust/nym-sdk/src/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool.rs @@ -139,6 +139,6 @@ //! } //! } -mod client_pool; +mod mixnet_client_pool; -pub use client_pool::ClientPool; +pub use mixnet_client_pool::ClientPool; diff --git a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs b/sdk/rust/nym-sdk/src/client_pool/mixnet_client_pool.rs similarity index 98% rename from sdk/rust/nym-sdk/src/client_pool/client_pool.rs rename to sdk/rust/nym-sdk/src/client_pool/mixnet_client_pool.rs index 49ec5c79df5..41656fe272d 100644 --- a/sdk/rust/nym-sdk/src/client_pool/client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool/mixnet_client_pool.rs @@ -12,7 +12,7 @@ pub struct ClientPool { cancel_token: CancellationToken, } -// This is only necessary for when you're wanting to check the addresses of the clients that are currently in the pool. +// This is only necessary for when you're wanting to check the addresses of the clients that are currently in the pool via logging. impl fmt::Debug for ClientPool { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let clients_debug = match self.clients.try_read() { @@ -48,7 +48,7 @@ impl fmt::Debug for ClientPool { } } -impl std::clone::Clone for ClientPool { +impl Clone for ClientPool { fn clone(&self) -> Self { Self { clients: Arc::clone(&self.clients), diff --git a/sdk/rust/nym-sdk/src/tcp_proxy.rs b/sdk/rust/nym-sdk/src/tcp_proxy.rs index 823c98c8d67..beb078ff4ab 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy.rs @@ -1,5 +1,218 @@ -//! -// TODO UPDATE EXAMPLE ONCE STABLE +//! use nym_sdk::tcp_proxy; +//! use rand::rngs::SmallRng; +//! use rand::{Rng, SeedableRng}; +//! use serde::{Deserialize, Serialize}; +//! use std::env; +//! use std::fs; +//! use std::sync::atomic::{AtomicU8, Ordering}; +//! use tokio::io::AsyncWriteExt; +//! use tokio::net::{TcpListener, TcpStream}; +//! use tokio::signal; +//! use tokio_stream::StreamExt; +//! use tokio_util::codec; +//! use tokio_util::sync::CancellationToken; +//! use tracing_subscriber::{fmt, prelude::*, EnvFilter}; + +//! #[derive(Serialize, Deserialize, Debug)] +//! struct ExampleMessage { +//! message_id: i8, +//! message_bytes: Vec, +//! } + +//! // This is a basic example which opens a single TCP connection and writes a bunch of messages between a client and an echo +//! // server, so only uses a single session under the hood and doesn't really show off the message ordering capabilities; this is mainly +//! // just a quick introductory illustration on how: +//! // - the mixnet does message ordering +//! // - the NymProxyClient and NymProxyServer can be hooked into and used to communicate between two otherwise pretty vanilla TcpStreams +//! // +//! // For a more irl example checkout tcp_proxy_multistream.rs +//! // +//! // Run this with: +//! // `cargo run --example tcp_proxy_single_connection ` e.g. +//! // `cargo run --example tcp_proxy_single_connection 8081 ../../../envs/canary.env 8080 ` +//! #[tokio::main] +//! async fn main() -> anyhow::Result<()> { +//! // Keep track of sent/received messages +//! let counter = AtomicU8::new(0); + +//! // Comment this out to just see println! statements from this example, as Nym client logging is very informative but quite verbose. +//! // The Message Decay related logging gives you an ideas of the internals of the proxy message ordering. To see the contents of the msg buffer, sphinx packet chunking, etc change the tracing::Level to DEBUG. +//! tracing_subscriber::registry() +//! .with(fmt::layer()) +//! .with(EnvFilter::new("nym_sdk::tcp_proxy=info")) +//! .init(); + +//! let server_port = env::args() +//! .nth(1) +//! .expect("Server listen port not specified"); +//! let upstream_tcp_addr = format!("127.0.0.1:{}", server_port); + +//! // This dir gets cleaned up at the end: NOTE if you switch env between tests without letting the file do the automatic cleanup, make sure to manually remove this directory up before running again, otherwise your client will attempt to use these keys for the new env +//! let home_dir = dirs::home_dir().expect("Unable to get home directory"); +//! let conf_path = format!("{}/tmp/nym-proxy-server-config", home_dir.display()); + +//! let env_path = env::args().nth(2).expect("Env file not specified"); +//! let env = env_path.to_string(); +//! let client_port = env::args().nth(3).expect("Port not specified"); + +//! let mut proxy_server = +//! tcp_proxy::NymProxyServer::new(&upstream_tcp_addr, &conf_path, Some(env_path.clone())) +//! .await?; +//! let proxy_nym_addr = proxy_server.nym_address(); + +//! // We'll run the instance with a long timeout since we're sending everything down the same Tcp connection, so should be using a single session. +//! // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. +//! // The final argument is how many clients to keep in reserve in the client pool when running the TcpProxy. +//! let proxy_client = +//! tcp_proxy::NymProxyClient::new(*proxy_nym_addr, "127.0.0.1", &client_port, 5, Some(env), 1) +//! .await?; + +//! // For our disconnect() logic below +//! let proxy_clone = proxy_client.clone(); + +//! tokio::spawn(async move { +//! proxy_server.run_with_shutdown().await?; +//! Ok::<(), anyhow::Error>(()) +//! }); + +//! tokio::spawn(async move { +//! proxy_client.run().await?; +//! Ok::<(), anyhow::Error>(()) +//! }); + +//! let example_cancel_token = CancellationToken::new(); +//! let server_cancel_token = example_cancel_token.clone(); +//! let client_cancel_token = example_cancel_token.clone(); +//! let watcher_cancel_token = example_cancel_token.clone(); + +//! // Cancel listener thread +//! tokio::spawn(async move { +//! signal::ctrl_c().await?; +//! println!(":: CTRL_C received, shutting down + cleanup up proxy server config files"); +//! fs::remove_dir_all(conf_path)?; +//! watcher_cancel_token.cancel(); +//! proxy_clone.disconnect().await; +//! Ok::<(), anyhow::Error>(()) +//! }); + +//! // 'Server side' thread: echo back incoming as response to the messages sent in the 'client side' thread below +//! tokio::spawn(async move { +//! let listener = TcpListener::bind(upstream_tcp_addr).await?; +//! loop { +//! if server_cancel_token.is_cancelled() { +//! break; +//! } +//! let (socket, _) = listener.accept().await.unwrap(); +//! let (read, mut write) = socket.into_split(); +//! let codec = codec::BytesCodec::new(); +//! let mut framed_read = codec::FramedRead::new(read, codec); +//! while let Some(Ok(bytes)) = framed_read.next().await { +//! match bincode::deserialize::(&bytes) { +//! Ok(msg) => { +//! println!( +//! "<< server received {}: {} bytes", +//! msg.message_id, +//! msg.message_bytes.len() +//! ); +//! let msg = ExampleMessage { +//! message_id: msg.message_id, +//! message_bytes: msg.message_bytes, +//! }; +//! let serialised = bincode::serialize(&msg)?; +//! write +//! .write_all(&serialised) +//! .await +//! .expect("couldnt send reply"); +//! println!( +//! ">> server sent {}: {} bytes", +//! msg.message_id, +//! msg.message_bytes.len() +//! ); +//! } +//! Err(e) => { +//! println!("<< server received something that wasn't an example message of {} bytes. error: {}", bytes.len(), e); +//! } +//! } +//! } +//! } +//! #[allow(unreachable_code)] +//! Ok::<(), anyhow::Error>(()) +//! }); + +//! // Just wait for Nym clients to connect, TCP clients to bind, etc. If there isn't a client in the pool (or you started it with 0) already then the TcpProxyClient just spins up an ephemeral client itself. +//! println!("waiting for everything to be set up.."); +//! tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; +//! println!("done. sending bytes"); + +//! // Now the client and server proxies are running we can create and pipe traffic to/from +//! // a socket on the same port as our ProxyClient instance as if we were just communicating +//! // between a client and host via a normal TcpStream - albeit with a decent amount of additional latency. +//! // +//! // The assumption regarding integration is that you know what you're sending, and will do proper +//! // framing before and after, know what data types you're expecting, etc; the proxies are just piping bytes +//! // back and forth using tokio's `Bytecodec` under the hood. +//! let local_tcp_addr = format!("127.0.0.1:{}", client_port); +//! let stream = TcpStream::connect(local_tcp_addr).await?; +//! let (read, mut write) = stream.into_split(); + +//! // 'Client side' thread; lets just send a bunch of messages to the server with variable delays between them, with an id to keep track of ordering in the printlns; the mixnet only guarantees message delivery, not ordering. You might not be necessarily streaming traffic in this manner IRL, but this example is a good illustration of how messages travel through the mixnet. +//! // - On the level of individual messages broken into multiple packets, the Proxy abstraction deals with making sure that everything is sent between the sockets in the //! corrent order. +//! // - On the level of different messages, this is not enforced: you might see in the logs that message 1 arrives at the server and is reconstructed after message 2. +//! tokio::spawn(async move { +//! let mut rng = SmallRng::from_entropy(); +//! for i in 0..10 { +//! if client_cancel_token.is_cancelled() { +//! break; +//! } +//! let random_bytes = gen_bytes_fixed(i as usize); +//! let msg = ExampleMessage { +//! message_id: i, +//! message_bytes: random_bytes, +//! }; +//! let serialised = bincode::serialize(&msg)?; +//! write +//! .write_all(&serialised) +//! .await +//! .expect("couldn't write to stream"); +//! println!(">> client sent {}: {} bytes", &i, msg.message_bytes.len()); +//! let delay = rng.gen_range(3.0..7.0); +//! tokio::time::sleep(tokio::time::Duration::from_secs_f64(delay)).await; +//! } +//! Ok::<(), anyhow::Error>(()) +//! }); + +//! let codec = codec::BytesCodec::new(); +//! let mut framed_read = codec::FramedRead::new(read, codec); +//! while let Some(Ok(bytes)) = framed_read.next().await { +//! match bincode::deserialize::(&bytes) { +//! Ok(msg) => { +//! println!( +//! "<< client received {}: {} bytes", +//! msg.message_id, +//! msg.message_bytes.len() +//! ); +//! counter.fetch_add(1, Ordering::SeqCst); +//! println!( +//! ":: messages received back: {:?}/10", +//! counter.load(Ordering::SeqCst) +//! ); +//! } +//! Err(e) => { +//! println!("<< client received something that wasn't an example message of {} bytes. error: {}", bytes.len(), e); +//! } +//! } +//! } + +//! Ok(()) +//! } + +//! fn gen_bytes_fixed(i: usize) -> Vec { +//! let amounts = [158, 1088, 505, 1001, 150, 200, 3500, 500, 750, 100]; +//! let len = amounts[i]; +//! let mut rng = rand::thread_rng(); +//! (0..len).map(|_| rng.gen::()).collect() +//! } + mod tcp_proxy_client; mod tcp_proxy_server; diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index acc0a760707..46ec3eceb56 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -22,6 +22,7 @@ const DEFAULT_LISTEN_HOST: &str = "127.0.0.1"; const DEFAULT_LISTEN_PORT: &str = "8080"; const DEFAULT_CLIENT_POOL_SIZE: usize = 2; +#[derive(Clone)] pub struct NymProxyClient { server_address: Recipient, listen_address: String, @@ -32,17 +33,6 @@ pub struct NymProxyClient { } impl NymProxyClient { - pub fn clone(&self) -> Self { - Self { - server_address: self.server_address, - listen_address: self.listen_address.clone(), - listen_port: self.listen_port.clone(), - close_timeout: self.close_timeout, - conn_pool: self.conn_pool.clone(), - cancel_token: self.cancel_token.clone(), - } - } - pub async fn new( server_address: Recipient, listen_address: &str, From 9400140f69e3d174715882aa6d0f1f291d08b24b Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 01:03:32 +0100 Subject: [PATCH 055/102] fix clippy again --- sdk/rust/nym-sdk/examples/client_pool.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/client_pool.rs b/sdk/rust/nym-sdk/examples/client_pool.rs index 2fffe43812e..1d082fcf649 100644 --- a/sdk/rust/nym-sdk/examples/client_pool.rs +++ b/sdk/rust/nym-sdk/examples/client_pool.rs @@ -53,7 +53,7 @@ async fn main() -> Result<()> { println!("\n\nClient 1: {our_address}\n\n"); client_one.disconnect().await; tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something - return Ok::<(), anyhow::Error>(()); + Ok::<(), anyhow::Error>(()); }); tokio::spawn(async move { @@ -81,7 +81,7 @@ async fn main() -> Result<()> { println!("\n\nClient 2: {our_address}\n\n"); client_two.disconnect().await; tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something - return Ok::<(), anyhow::Error>(()); + Ok::<(), anyhow::Error>(()); }); wait_for_ctrl_c(conn_pool).await?; From 99d2a807ddffdcb85d6acfafadf45f23253606a3 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 01:07:53 +0100 Subject: [PATCH 056/102] fix clippy again --- sdk/rust/nym-sdk/examples/client_pool.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/client_pool.rs b/sdk/rust/nym-sdk/examples/client_pool.rs index 1d082fcf649..4ac52999dad 100644 --- a/sdk/rust/nym-sdk/examples/client_pool.rs +++ b/sdk/rust/nym-sdk/examples/client_pool.rs @@ -39,9 +39,11 @@ async fn main() -> Result<()> { let net = NymNetworkDetails::new_from_env(); let client = MixnetClientBuilder::new_ephemeral() .network_details(net) - .build()? + .build() + .unwrap() .connect_to_mixnet() - .await?; + .await + .unwrap(); println!( "Using {} for the moment, created outside of the connection pool", client.nym_address() @@ -67,9 +69,11 @@ async fn main() -> Result<()> { let net = NymNetworkDetails::new_from_env(); let client = MixnetClientBuilder::new_ephemeral() .network_details(net) - .build()? + .build() + .unwrap() .connect_to_mixnet() - .await?; + .await + .unwrap(); println!( "Using {} for the moment, created outside of the connection pool", client.nym_address() From d99cb96994f842d44b955a2b3ab396ecd81c8422 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 01:11:50 +0100 Subject: [PATCH 057/102] version bump --- sdk/ffi/go/Cargo.toml | 6 +++--- sdk/ffi/shared/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/ffi/go/Cargo.toml b/sdk/ffi/go/Cargo.toml index e1aff9facac..c9346f247fa 100644 --- a/sdk/ffi/go/Cargo.toml +++ b/sdk/ffi/go/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "nym-go-ffi" #"goffitest" -version = "0.2.0" +name = "nym-go-ffi" +version = "0.3.0" edition = "2021" license.workspace = true [lib] crate-type = ["cdylib"] -name = "nym_go_ffi" #"go_ffi" +name = "nym_go_ffi" [dependencies] # Bindgen diff --git a/sdk/ffi/shared/Cargo.toml b/sdk/ffi/shared/Cargo.toml index 9f66e095eff..a4081f91fa6 100644 --- a/sdk/ffi/shared/Cargo.toml +++ b/sdk/ffi/shared/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nym-ffi-shared" -version = "0.2.0" +version = "0.3.0" edition = "2021" license.workspace = true From 72482180b9e3000a0fa13ab632931e724bc72ed1 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 01:12:18 +0100 Subject: [PATCH 058/102] version bump --- sdk/ffi/go/Cargo.toml | 2 +- sdk/ffi/shared/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/ffi/go/Cargo.toml b/sdk/ffi/go/Cargo.toml index c9346f247fa..3f20dec3cb4 100644 --- a/sdk/ffi/go/Cargo.toml +++ b/sdk/ffi/go/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nym-go-ffi" -version = "0.3.0" +version = "0.2.1" edition = "2021" license.workspace = true diff --git a/sdk/ffi/shared/Cargo.toml b/sdk/ffi/shared/Cargo.toml index a4081f91fa6..964be022174 100644 --- a/sdk/ffi/shared/Cargo.toml +++ b/sdk/ffi/shared/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nym-ffi-shared" -version = "0.3.0" +version = "0.2.1" edition = "2021" license.workspace = true From 9d8f14ddec02b3997c63b0ab6458337374a640f9 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 01:16:06 +0100 Subject: [PATCH 059/102] fix test --- Cargo.lock | 4 ++-- sdk/rust/nym-sdk/examples/client_pool.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 30a01aa784a..c169240c195 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5310,7 +5310,7 @@ dependencies = [ [[package]] name = "nym-ffi-shared" -version = "0.2.0" +version = "0.2.1" dependencies = [ "anyhow", "bs58", @@ -5471,7 +5471,7 @@ dependencies = [ [[package]] name = "nym-go-ffi" -version = "0.2.0" +version = "0.2.1" dependencies = [ "anyhow", "lazy_static", diff --git a/sdk/rust/nym-sdk/examples/client_pool.rs b/sdk/rust/nym-sdk/examples/client_pool.rs index 4ac52999dad..885323750d7 100644 --- a/sdk/rust/nym-sdk/examples/client_pool.rs +++ b/sdk/rust/nym-sdk/examples/client_pool.rs @@ -55,7 +55,7 @@ async fn main() -> Result<()> { println!("\n\nClient 1: {our_address}\n\n"); client_one.disconnect().await; tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something - Ok::<(), anyhow::Error>(()); + Ok::<(), anyhow::Error>(()) }); tokio::spawn(async move { @@ -85,7 +85,7 @@ async fn main() -> Result<()> { println!("\n\nClient 2: {our_address}\n\n"); client_two.disconnect().await; tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; // Emulate doing something - Ok::<(), anyhow::Error>(()); + Ok::<(), anyhow::Error>(()) }); wait_for_ctrl_c(conn_pool).await?; From 4f5d86a50c9c3d94444de39bad52e8e4f413df36 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 11:20:57 +0100 Subject: [PATCH 060/102] tweaked text grammar --- .../docs/pages/developers/rust/client-pool/architecture.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/docs/pages/developers/rust/client-pool/architecture.mdx b/documentation/docs/pages/developers/rust/client-pool/architecture.mdx index a50b2126e65..4159796c1e3 100644 --- a/documentation/docs/pages/developers/rust/client-pool/architecture.mdx +++ b/documentation/docs/pages/developers/rust/client-pool/architecture.mdx @@ -1,14 +1,14 @@ # Client Pool Architecture ## Motivations -In situations where multiple connections are expected, and the number of connections can vary greatly, the Pool reduces time spent waiting for the creation of a Mixnet Client blocking your code sending traffic through the Mixnet. Instead, a configurable number of Clients can be generated and run in the background which can be very quickly grabbed, used, and disconnected. +In situations where multiple connections are expected, and the number of connections can vary greatly, the Client Pool reduces time spent waiting for the creation of a Mixnet Client blocking your code sending traffic through the Mixnet. Instead, a configurable number of Clients can be generated and run in the background which can be very quickly grabbed, used, and disconnected. The Pool can be simply run as a background process for the runtime of your program. ## Clients & Lifetimes -The pool currently operates on the principle of creating an **ephemeral Client** which is used once and then disconnected after use. Using the [`TcpProxy`](../tcpproxy) as an example, a client is used for the lifetime of a single incoming TCP connection; after the TCP connection is closed, the Mixnet client is disconnected. +The Client Pool creates **ephemeral Mixnet Clients** which are used and then disconnected. Using the [`TcpProxy`](../tcpproxy) as an example, Clients are used for the lifetime of a single incoming TCP connection; after the TCP connection is closed, the Mixnet client is disconnected. -Clients are popped from the pool when in use, and another Client is created to take its place. If connections are coming in faster than Clients are replenished, logic can be easily created to instead generate an ephemeral Client or to wait; this is up to the developer to decide. You can see an example of this logic on the example on the next page. +Clients are popped from the pool when in use, and another Client is created to take its place. If connections are coming in faster than Clients are replenished, you can instead generate an ephemeral Client on the fly, or wait; this is up to the developer to decide. You can see an example of this logic in the example on the next page. ## Runtime Loop Aside from a few helper / getter functions and a graceful `disconnect_pool()`, the Client Pool is mostly made up of a very simple loop around some conditional logic making up `start()`: From 16c3e5ff9a5e2eb208a38025ca1bd4b83b96a6ca Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 11:28:07 +0100 Subject: [PATCH 061/102] updated comment in example --- documentation/docs/pages/developers/rust/client-pool/example.md | 2 +- sdk/rust/nym-sdk/examples/client_pool.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/docs/pages/developers/rust/client-pool/example.md b/documentation/docs/pages/developers/rust/client-pool/example.md index 1b3c499f348..11258658950 100644 --- a/documentation/docs/pages/developers/rust/client-pool/example.md +++ b/documentation/docs/pages/developers/rust/client-pool/example.md @@ -17,7 +17,7 @@ async fn main() -> Result<()> { nym_bin_common::logging::setup_logging(); setup_env(std::env::args().nth(1)); - let conn_pool = ClientPool::new(2); // Start the client pool with 1 client always being kept in reserve + let conn_pool = ClientPool::new(2); // Start the Client Pool with 2 Clients always being kept in reserve let client_maker = conn_pool.clone(); tokio::spawn(async move { client_maker.start().await?; diff --git a/sdk/rust/nym-sdk/examples/client_pool.rs b/sdk/rust/nym-sdk/examples/client_pool.rs index 885323750d7..27c452ec194 100644 --- a/sdk/rust/nym-sdk/examples/client_pool.rs +++ b/sdk/rust/nym-sdk/examples/client_pool.rs @@ -15,7 +15,7 @@ async fn main() -> Result<()> { nym_bin_common::logging::setup_logging(); setup_env(std::env::args().nth(1)); - let conn_pool = ClientPool::new(2); // Start the client pool with 1 client always being kept in reserve + let conn_pool = ClientPool::new(2); // Start the Client Pool with 2 Clients always being kept in reserve let client_maker = conn_pool.clone(); tokio::spawn(async move { client_maker.start().await?; From 3fdaa7bac3148368400fbeeb1fcb5ced3bf009c0 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 17 Dec 2024 11:29:14 +0100 Subject: [PATCH 062/102] future is now --- .../docs/pages/developers/rust/tcpproxy/architecture.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/docs/pages/developers/rust/tcpproxy/architecture.mdx b/documentation/docs/pages/developers/rust/tcpproxy/architecture.mdx index 06880bdc2c7..006cd1ce6ae 100644 --- a/documentation/docs/pages/developers/rust/tcpproxy/architecture.mdx +++ b/documentation/docs/pages/developers/rust/tcpproxy/architecture.mdx @@ -13,7 +13,7 @@ The motivation behind the creation of the `TcpProxy` module is to allow develope ## Clients Each of the sub-modules exposed by the `TcpProxy` deal with Nym clients in a different way. -- the `NymProxyClient` relies on the [`Client Pool`](../client-pool) to create clients and keep a certain number of them in reserve. If the amount of incoming TCP connections rises quicker than the Client Pool can create clients, or you have the pool size set to `0`, the `TcpProxyClient` creates an ephemeral client per new TCP connection, which is closed according to the configurable timeout: we map one ephemeral client per TCP connection. This is to deal with multiple simultaneous streams. In the future, this will be superceded by a connection pool in order to speed up new connections. +- the `NymProxyClient` relies on the [`Client Pool`](../client-pool) to create clients and keep a certain number of them in reserve. If the amount of incoming TCP connections rises quicker than the Client Pool can create clients, or you have the pool size set to `0`, the `TcpProxyClient` creates an ephemeral client per new TCP connection, which is closed according to the configurable timeout: we map one ephemeral client per TCP connection. This is to deal with multiple simultaneous streams. - the `NymProxyServer` has a single Nym client with a persistent identity. ## Framing From 0f35f5c90988515cbc4d20ce099949dbe8592577 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 18 Dec 2024 20:11:28 +0100 Subject: [PATCH 063/102] add configurable gw to server --- .../nym-sdk/src/tcp_proxy/tcp_proxy_server.rs | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs index 55e6273dbb3..c81b07431c2 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs @@ -4,6 +4,7 @@ use crate::mixnet::{ }; use anyhow::Result; use dashmap::DashSet; +use nym_crypto::asymmetric::ed25519; use nym_network_defaults::setup_env; use nym_sphinx::addressing::Recipient; use std::path::PathBuf; @@ -33,20 +34,31 @@ impl NymProxyServer { upstream_address: &str, config_dir: &str, env: Option, + gateway: Option, ) -> Result { info!("Creating client"); // We're wanting to build a client with a constant address, vs the ephemeral in-memory data storage of the NymProxyClient clients. // Following a builder pattern, having to manually connect to the mixnet below. + // Optional selectable Gateway to use. let config_dir = PathBuf::from(config_dir); debug!("Loading env file: {:?}", env); setup_env(env); // Defaults to mainnet if empty let net = NymNetworkDetails::new_from_env(); let storage_paths = StoragePaths::new_from_dir(&config_dir)?; - let client = MixnetClientBuilder::new_with_default_storage(storage_paths) - .await? - .network_details(net) - .build()?; + + let client = if let Some(gateway) = gateway { + MixnetClientBuilder::new_with_default_storage(storage_paths) + .await? + .network_details(net) + .request_gateway(gateway.to_string()) + .build()? + } else { + MixnetClientBuilder::new_with_default_storage(storage_paths) + .await? + .network_details(net) + .build()? + }; let client = client.connect_to_mixnet().await?; From 2fc359f58f6d3944c7fa8cb9a96214d3d9f5be7a Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 19 Dec 2024 12:55:57 +0100 Subject: [PATCH 064/102] first pass echo server lib --- Cargo.lock | 3 + Cargo.toml | 2 +- tools/echo-server/Cargo.toml | 11 ++ tools/echo-server/src/echo-server.rs | 17 +++ tools/echo-server/src/lib.rs | 185 +++++++++++++++++++++++++++ tools/echo-server/src/main.rs | 161 ----------------------- 6 files changed, 217 insertions(+), 162 deletions(-) create mode 100644 tools/echo-server/src/echo-server.rs create mode 100644 tools/echo-server/src/lib.rs delete mode 100644 tools/echo-server/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index c169240c195..58893ee0f5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2196,8 +2196,11 @@ dependencies = [ "bincode", "bytecodec", "bytes", + "clap 4.5.20", "dashmap", "dirs", + "nym-bin-common", + "nym-crypto", "nym-sdk", "serde", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 122784eb5a8..5d3dbdd8102 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,7 +128,7 @@ members = [ "nym-node-status-api/nym-node-status-client", "nym-outfox", "nym-validator-rewarder", - "tools/echo-server", + # "tools/echo-server", "tools/internal/ssl-inject", # "tools/internal/sdk-version-bump", "tools/internal/testnet-manager", diff --git a/tools/echo-server/Cargo.toml b/tools/echo-server/Cargo.toml index dbdf44c056e..c76a96e8450 100644 --- a/tools/echo-server/Cargo.toml +++ b/tools/echo-server/Cargo.toml @@ -9,6 +9,11 @@ edition.workspace = true license.workspace = true rust-version.workspace = true + +[[bin]] +name = "echo-server" +path = "src/echo-server.rs" + [dependencies] anyhow.workspace = true dashmap.workspace = true @@ -24,3 +29,9 @@ bytecodec = { workspace = true } nym-sdk = { path = "../../sdk/rust/nym-sdk/" } bytes.workspace = true dirs.workspace = true +clap.workspace = true +nym-bin-common = { path = "../../common/bin-common", features = [ + "basic_tracing", + "output_format", +] } +nym-crypto = { path = "../../common/crypto", features = ["asymmetric"] } diff --git a/tools/echo-server/src/echo-server.rs b/tools/echo-server/src/echo-server.rs new file mode 100644 index 00000000000..d6cfa13d59b --- /dev/null +++ b/tools/echo-server/src/echo-server.rs @@ -0,0 +1,17 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +// use clap::{Args, Parser, Subcommand}; +// use nym_bin_common::{bin_info, bin_info_owned}; +// use std::sync::OnceLock; + +// fn pretty_build_info_static() -> &'static str { +// static PRETTY_BUILD_INFORMATION: OnceLock = OnceLock::new(); +// PRETTY_BUILD_INFORMATION.get_or_init(|| bin_info!().pretty_print()) +// } + +// This will be a Clap CLI binary for standalone use use wrapping the lib + +fn main() { + todo!() +} diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs new file mode 100644 index 00000000000..4b035b46c57 --- /dev/null +++ b/tools/echo-server/src/lib.rs @@ -0,0 +1,185 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use anyhow::Result; +use bytes::Bytes; +use nym_crypto::asymmetric::ed25519; +use nym_sdk::mixnet::Recipient; +use nym_sdk::tcp_proxy; +use nym_sdk::tcp_proxy::NymProxyServer; +use std::env; +use std::fs; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use tokio::io::AsyncWriteExt; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::Mutex; +use tokio::task; +use tokio_stream::StreamExt; +use tokio_util::sync::CancellationToken; +use tracing::{error, info, warn}; + +pub struct Metrics { + total_conn: AtomicU64, + active_conn: AtomicU64, + bytes_recv: AtomicU64, + bytes_sent: AtomicU64, +} + +impl Metrics { + fn new() -> Self { + Self { + total_conn: AtomicU64::new(0), + active_conn: AtomicU64::new(0), + bytes_recv: AtomicU64::new(0), + bytes_sent: AtomicU64::new(0), + } + } +} + +pub struct NymEchoServer { + client: Arc>, + client_address: Recipient, + listen_addr: String, + metrics: Arc, + cancel_token: CancellationToken, +} + +impl NymEchoServer { + pub async fn new( + &mut self, + gateway: Option, + config_path: Option<&str>, + env: String, + listen_port: &str, + ) -> Result { + let home_dir = dirs::home_dir().expect("Unable to get home directory"); + let default_path = format!("{}/tmp/nym-proxy-server-config", home_dir.display()); + let config_path = config_path.unwrap_or(&default_path); + let listen_addr = format!("127.0.0.1:{}", listen_port); + + Ok(NymEchoServer { + client: Arc::new(Mutex::new( + tcp_proxy::NymProxyServer::new( + &listen_addr, + &config_path, + Some(env.clone()), + gateway, + ) + .await + .unwrap(), + )), + client_address: self.client.lock().await.nym_address().clone(), + listen_addr, + metrics: Arc::new(Metrics::new()), + cancel_token: CancellationToken::new(), + }) + } + + pub async fn run(&mut self) -> Result<()> { + let cancel_token = self.cancel_token.clone(); + + let client = Arc::clone(&self.client); + task::spawn(async move { + client.lock().await.run_with_shutdown().await?; + Ok::<(), anyhow::Error>(()) + }); + + let all_metrics = Arc::clone(&self.metrics); + + tokio::spawn(async move { + loop { + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + info!( + "Metrics: total_connections={}, active_connections={}, bytes_received={}, bytes_sent={}", + all_metrics.total_conn.load(Ordering::Relaxed), + all_metrics.active_conn.load(Ordering::Relaxed), + all_metrics.bytes_recv.load(Ordering::Relaxed), + all_metrics.bytes_sent.load(Ordering::Relaxed), + ); + } + }); + + let listener = TcpListener::bind(self.listen_addr.clone()).await?; + + loop { + tokio::select! { + stream = listener.accept() => { + let (stream, _) = stream?; + + let connection_metrics = Arc::clone(&self.metrics); + connection_metrics.total_conn.fetch_add(1, Ordering::Relaxed); + connection_metrics.active_conn.fetch_add(1, Ordering::Relaxed); + + tokio::spawn(NymEchoServer::handle_incoming( + stream, connection_metrics, cancel_token.clone() + )); + } + _ = self.cancel_token.cancelled() => { + break Ok(()); + } + } + } + } + + async fn handle_incoming( + socket: TcpStream, + metrics: Arc, + cancel_token: CancellationToken, + ) { + let (read, mut write) = socket.into_split(); + let codec = tokio_util::codec::BytesCodec::new(); + let mut framed_read = tokio_util::codec::FramedRead::new(read, codec); + + loop { + tokio::select! { + Some(result) = framed_read.next() => { + match result { + Ok(bytes) => { + let len = bytes.len(); + metrics.bytes_recv.fetch_add(len as u64, Ordering::Relaxed); + if let Err(e) = write.write_all(&bytes).await { + error!("Failed to write to stream with err: {}", e); + break; + } + metrics.bytes_sent.fetch_add(len as u64, Ordering::Relaxed); + } + Err(e) => { + error!("Failed to read from stream with err: {}", e); + break; + } + } + } + _ = cancel_token.cancelled() => { + warn!("Shutdown signal received, closing connection"); + break; + } + } + } + metrics + .active_conn + .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + info!("Connection closed"); + } + + // TODO + // - make disconnect for proxy_server + // - make disconnect for this + // pub async fn disconnect(&self) { + // self.cancel_token.cancel(); + // let client = Arc::clone(&self.client); + // client.lock().await.disconnect_pool().await; + // while metrics.active_conn.load(Ordering::Relaxed) > 0 { + // info!("Waiting on active connections to close: sleeping 100ms"); + // tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + // } + // } + + pub fn nym_address(&self) -> Recipient { + self.client_address + } + + pub fn listen_addr(&self) -> String { + self.listen_addr.clone() + } +} diff --git a/tools/echo-server/src/main.rs b/tools/echo-server/src/main.rs deleted file mode 100644 index 28b0a7b96d8..00000000000 --- a/tools/echo-server/src/main.rs +++ /dev/null @@ -1,161 +0,0 @@ -use anyhow::Result; -use bytes::Bytes; -use nym_sdk::tcp_proxy; -use std::env; -use std::fs; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use tokio::io::AsyncWriteExt; -use tokio::net::{TcpListener, TcpStream}; -use tokio::signal; -use tokio::sync::broadcast; -use tokio::task; -use tokio_stream::StreamExt; -use tracing::{error, info, warn}; - -struct Metrics { - total_conn: AtomicU64, - active_conn: AtomicU64, - bytes_recv: AtomicU64, - bytes_sent: AtomicU64, -} - -impl Metrics { - fn new() -> Self { - Self { - total_conn: AtomicU64::new(0), - active_conn: AtomicU64::new(0), - bytes_recv: AtomicU64::new(0), - bytes_sent: AtomicU64::new(0), - } - } -} - -#[tokio::main] -async fn main() -> Result<()> { - // if you run this with DEBUG you see the msg buffer on the ProxyServer, but its quite chatty - tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .init(); - - let server_port = env::args() - .nth(1) - .expect("Server listen port not specified"); - let tcp_addr = format!("127.0.0.1:{}", server_port); - - // This dir gets cleaned up at the end: NOTE if you switch env between tests without letting the file do the automatic cleanup, make sure to manually remove this directory up before running again, otherwise your client will attempt to use these keys for the new env - let home_dir = dirs::home_dir().expect("Unable to get home directory"); - let conf_path = format!("{}/tmp/nym-proxy-server-config", home_dir.display()); - - let env_path = env::args().nth(2).expect("Env file not specified"); - let env = env_path.to_string(); - - let mut proxy_server = tcp_proxy::NymProxyServer::new(&tcp_addr, &conf_path, Some(env.clone())) - .await - .unwrap(); - let proxy_nym_addr = *proxy_server.nym_address(); - info!("ProxyServer listening out on {}", proxy_nym_addr); - - task::spawn(async move { - proxy_server.run_with_shutdown().await?; - Ok::<(), anyhow::Error>(()) - }); - - let (shutdown_sender, _) = broadcast::channel(1); - let metrics = Arc::new(Metrics::new()); - let all_metrics = Arc::clone(&metrics); - - tokio::spawn(async move { - loop { - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - info!( - "Metrics: total_connections={}, active_connections={}, bytes_received={}, bytes_sent={}", - all_metrics.total_conn.load(Ordering::Relaxed), - all_metrics.active_conn.load(Ordering::Relaxed), - all_metrics.bytes_recv.load(Ordering::Relaxed), - all_metrics.bytes_sent.load(Ordering::Relaxed), - ); - } - }); - - let listener = TcpListener::bind(tcp_addr).await?; - - loop { - tokio::select! { - _ = signal::ctrl_c() => { - info!("Shutdown signal received, closing server..."); - let _ = shutdown_sender.send(()); - // TODO we need something like this for the ProxyServer client - break; - } - Ok((socket, _)) = listener.accept() => { - let connection_metrics = Arc::clone(&metrics); - let shutdown_rx = shutdown_sender.subscribe(); - connection_metrics.total_conn.fetch_add(1, Ordering::Relaxed); - connection_metrics.active_conn.fetch_add(1, Ordering::Relaxed); - tokio::spawn(async move { - handle_incoming(socket, connection_metrics, shutdown_rx).await; - }); - } - } - } - - signal::ctrl_c().await?; - info!("Received CTRL+C"); - fs::remove_dir_all(conf_path)?; - while metrics.active_conn.load(Ordering::Relaxed) > 0 { - info!("Waiting on active connections to close: sleeping 100ms"); - // TODO some kind of hard kill here for the ProxyServer - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } - Ok(()) -} - -async fn handle_incoming( - socket: TcpStream, - metrics: Arc, - mut shutdown_rx: broadcast::Receiver<()>, -) { - let (read, mut write) = socket.into_split(); - let codec = tokio_util::codec::BytesCodec::new(); - let mut framed_read = tokio_util::codec::FramedRead::new(read, codec); - - loop { - tokio::select! { - Some(result) = framed_read.next() => { - match result { - Ok(bytes) => { - let len = bytes.len(); - metrics.bytes_recv.fetch_add(len as u64, Ordering::Relaxed); - if let Err(e) = write.write_all(&bytes).await { - error!("Failed to write to stream with err: {}", e); - break; - } - metrics.bytes_sent.fetch_add(len as u64, Ordering::Relaxed); - } - Err(e) => { - error!("Failed to read from stream with err: {}", e); - break; - } - } - } - _ = shutdown_rx.recv() => { - warn!("Shutdown signal received, closing connection"); - break; - } - // TODO need to work out a way that if this timesout and breaks but you dont hang up the conn on the client end you can reconnect..maybe. If we just use this as a ping echo server I dont think this is a problem - // EDIT I'm not actually sure we want this functionality? Measuring active connections might be useful though - _ = tokio::time::sleep(tokio::time::Duration::from_secs(120)) => { - info!("Timeout reached, assuming we wont get more messages on this conn, closing"); - let close_message = "Closing conn, reconnect if you want to ping again"; - let bytes: Bytes = close_message.into(); - write.write_all(&bytes).await.expect("Couldn't write to socket"); - break; - } - } - } - metrics - .active_conn - .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); - info!("Connection closed"); -} From d247c88a74ce80029025f14e3c26c242a0824ed0 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 19 Dec 2024 12:57:27 +0100 Subject: [PATCH 065/102] cargo fix --- tools/echo-server/src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index 4b035b46c57..72d5dc74a9b 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -2,13 +2,10 @@ // SPDX-License-Identifier: GPL-3.0-only use anyhow::Result; -use bytes::Bytes; use nym_crypto::asymmetric::ed25519; use nym_sdk::mixnet::Recipient; use nym_sdk::tcp_proxy; use nym_sdk::tcp_proxy::NymProxyServer; -use std::env; -use std::fs; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use tokio::io::AsyncWriteExt; From e2114dd9e5c1e00775743ec24e7434bfd5766e7d Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 19 Dec 2024 13:40:30 +0100 Subject: [PATCH 066/102] add disconnect --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs index c81b07431c2..8c2a63aab17 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs @@ -13,6 +13,7 @@ use tokio::net::TcpStream; use tokio::sync::watch::Receiver; use tokio::sync::RwLock; use tokio_stream::StreamExt; +use tokio_util::sync::CancellationToken; use tracing::{debug, error, info}; #[allow(clippy::duplicate_mod)] #[path = "utils.rs"] @@ -27,6 +28,7 @@ pub struct NymProxyServer { mixnet_client_sender: Arc>, tx: tokio::sync::watch::Sender>, rx: tokio::sync::watch::Receiver>, + cancel_token: CancellationToken, } impl NymProxyServer { @@ -78,6 +80,7 @@ impl NymProxyServer { mixnet_client_sender: sender, tx, rx, + cancel_token: CancellationToken::new(), }) } @@ -122,6 +125,7 @@ impl NymProxyServer { session_id: Uuid, mut rx: Receiver>, sender: Arc>, + cancel_token: CancellationToken, ) -> Result<()> { let global_surb = Arc::new(RwLock::new(None)); let stream = TcpStream::connect(upstream_address).await?; @@ -193,6 +197,9 @@ impl NymProxyServer { } } } + _ = cancel_token.cancelled() => { + break; + } _ = tokio::time::sleep(tokio::time::Duration::from_millis(100)) => { msg_buffer.tick(&mut write).await?; } @@ -222,6 +229,7 @@ impl NymProxyServer { session_id, self.rx(), self.mixnet_client_sender(), + self.cancel_token.clone(), )); info!("Spawned a new session handler: {}", message.session_id()); } @@ -235,7 +243,10 @@ impl NymProxyServer { } } - tokio::signal::ctrl_c().await?; Ok(()) } + + pub async fn disconnect(&self) { + self.cancel_token.cancel(); + } } From d7d2aaff681e9ed36ea11b785040544fe75a9cfe Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 19 Dec 2024 13:40:52 +0100 Subject: [PATCH 067/102] comment --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 46ec3eceb56..87592400a54 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -231,7 +231,7 @@ impl NymProxyClient { msg_buffer.tick(&mut write).await?; }, _ = cancel_token.cancelled() => { - info!("CTRL_C triggered in thread, triggering loop shutdown"); + info!("Triggering loop shutdown"); break }, _ = tokio::time::sleep(tokio::time::Duration::from_millis(100)) => { @@ -251,7 +251,7 @@ impl NymProxyClient { msg_buffer.tick(&mut write).await?; }, _ = cancel_token.cancelled() => { - info!("CTRL_C triggered in thread, triggering client shutdown"); + info!("Triggering client shutdown"); client.disconnect().await; return Ok::<(), anyhow::Error>(()) }, From b6b8ec41ebf8048ab57e570cb02fd45186f11fe1 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 19 Dec 2024 13:41:00 +0100 Subject: [PATCH 068/102] first pass all fns --- tools/echo-server/src/lib.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index 72d5dc74a9b..a6d8ea6a1e2 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -159,18 +159,15 @@ impl NymEchoServer { info!("Connection closed"); } - // TODO - // - make disconnect for proxy_server - // - make disconnect for this - // pub async fn disconnect(&self) { - // self.cancel_token.cancel(); - // let client = Arc::clone(&self.client); - // client.lock().await.disconnect_pool().await; - // while metrics.active_conn.load(Ordering::Relaxed) > 0 { - // info!("Waiting on active connections to close: sleeping 100ms"); - // tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - // } - // } + pub async fn disconnect(&self) { + self.cancel_token.cancel(); + let client = Arc::clone(&self.client); + client.lock().await.disconnect().await; + while self.metrics.active_conn.load(Ordering::Relaxed) > 0 { + info!("Waiting on active connections to close: sleeping 100ms"); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + } pub fn nym_address(&self) -> Recipient { self.client_address From 9761644d0866acb2c36d798c66de20d883e58a49 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 20 Dec 2024 00:10:53 +0100 Subject: [PATCH 069/102] tweaks to lib --- tools/echo-server/src/lib.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index a6d8ea6a1e2..63a4d32085b 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -36,7 +36,6 @@ impl Metrics { pub struct NymEchoServer { client: Arc>, - client_address: Recipient, listen_addr: String, metrics: Arc, cancel_token: CancellationToken, @@ -44,7 +43,6 @@ pub struct NymEchoServer { impl NymEchoServer { pub async fn new( - &mut self, gateway: Option, config_path: Option<&str>, env: String, @@ -63,10 +61,8 @@ impl NymEchoServer { Some(env.clone()), gateway, ) - .await - .unwrap(), + .await?, )), - client_address: self.client.lock().await.nym_address().clone(), listen_addr, metrics: Arc::new(Metrics::new()), cancel_token: CancellationToken::new(), @@ -98,12 +94,13 @@ impl NymEchoServer { }); let listener = TcpListener::bind(self.listen_addr.clone()).await?; + info!("{listener:?}"); loop { tokio::select! { stream = listener.accept() => { let (stream, _) = stream?; - + info!("Handling new stream"); let connection_metrics = Arc::clone(&self.metrics); connection_metrics.total_conn.fetch_add(1, Ordering::Relaxed); connection_metrics.active_conn.fetch_add(1, Ordering::Relaxed); @@ -113,6 +110,7 @@ impl NymEchoServer { )); } _ = self.cancel_token.cancelled() => { + info!("Cancel token cancelled: {:?}", self.cancel_token.cancelled()); break Ok(()); } } @@ -164,13 +162,13 @@ impl NymEchoServer { let client = Arc::clone(&self.client); client.lock().await.disconnect().await; while self.metrics.active_conn.load(Ordering::Relaxed) > 0 { - info!("Waiting on active connections to close: sleeping 100ms"); + info!("Waiting on active connections to close: sleeping"); tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; } } - pub fn nym_address(&self) -> Recipient { - self.client_address + pub async fn nym_address(&self) -> Recipient { + self.client.lock().await.nym_address().clone() } pub fn listen_addr(&self) -> String { From a9ef20914eb3514009c9f7c74009540f45239aa1 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 20 Dec 2024 00:11:03 +0100 Subject: [PATCH 070/102] temp commit --- .../mixnet-check-all-gateways/Cargo.toml | 29 ++++ .../mixnet-check-all-gateways/src/main.rs | 153 ++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 tools/internal/mixnet-check-all-gateways/Cargo.toml create mode 100644 tools/internal/mixnet-check-all-gateways/src/main.rs diff --git a/tools/internal/mixnet-check-all-gateways/Cargo.toml b/tools/internal/mixnet-check-all-gateways/Cargo.toml new file mode 100644 index 00000000000..7e35bfc9141 --- /dev/null +++ b/tools/internal/mixnet-check-all-gateways/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "mixnet-check-all-gateways" +version = "0.1.0" +authors.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +readme.workspace = true + +[dependencies] +nym-bin-common = { path = "../../../common/bin-common", features = [ + "basic_tracing", + "output_format", +] } +nym-crypto = { path = "../../../common/crypto", features = ["asymmetric"] } +clap.workspace = true +anyhow.workspace = true +tracing.workspace = true +tracing-subscriber = "0.3" +nym-sdk = { path = "../../../sdk/rust/nym-sdk/" } +echo-server = { path = "../../echo-server" } +serde = { version = "1", features = ["derive"] } +tokio = { workspace = true, features = ["full"] } +reqwest = { workspace = true, features = ["json"] } +serde_json.workspace = true +futures = { workspace = true } diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs new file mode 100644 index 00000000000..49cda71615f --- /dev/null +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -0,0 +1,153 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + +use anyhow::Result; +use echo_server::NymEchoServer; +use futures::stream::StreamExt; +use nym_bin_common::logging::setup_logging; +use nym_crypto::asymmetric::ed25519; +use nym_sdk::mixnet; +use nym_sdk::mixnet::MixnetMessageSender; +use reqwest::{self, Response, Url}; +use serde_json::Value; +use std::time::Duration; +use tokio::time; +use tokio::time::timeout; + +#[tokio::main] +async fn main() -> Result<()> { + setup_logging(); + let entry_gw_keys = reqwest_and_parse( + Url::parse( + "https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/entry-gateways/all?no_legacy=true", // make const + + ) + .unwrap(), + "ed25519_identity_pubkey", + ) + .await?; + println!( + "got {} entry gws: \n{:?}", + entry_gw_keys.len(), + entry_gw_keys + ); + + let exit_gw_keys = reqwest_and_parse( + Url::parse( + "https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/exit-gateways/all?no_legacy=true", // make const + ) + .unwrap(), + "ed25519_identity_pubkey", + ) + .await?; + println!( + "got {} exit gws: \n{:?}\n", + exit_gw_keys.len(), + exit_gw_keys + ); + + for gw in exit_gw_keys { + println!("{}", gw); + time::sleep(Duration::from_secs(1)).await; + + let mut echo_server = NymEchoServer::new( + Some(ed25519::PublicKey::from_base58_string(gw)?), + None, + "../../../envs/mainnet.env".to_string(), // make const + "9000", // when you run concurrently you can probably iterate through ports here as well + ) + .await?; + + let echo_addr = echo_server.nym_address().await; + println!("echo addr: {echo_addr}"); + + tokio::task::spawn(async move { + echo_server.run().await?; + Ok::<(), anyhow::Error>(()) + }); + + for gw in entry_gw_keys.clone() { + let builder = mixnet::MixnetClientBuilder::new_ephemeral() + .request_gateway(gw) + .build()?; + + let mut client = match builder.connect_to_mixnet().await { + Ok(client) => { + println!("connected"); + client + } + Err(err) => { + println!("failed to connect: {err}"); + return Err(err.into()); + } + }; + let our_address = client.nym_address(); + println!("{our_address}"); + client.send_plain_message(echo_addr, "echo").await?; + + match timeout(Duration::from_secs(5), client.next()).await { + Err(_timeout) => { + println!("❌"); + println!("timed out while waiting for the response..."); + } + Ok(Some(received)) => match String::from_utf8(received.message) { + Ok(message) => { + println!("✅"); + println!("received '{message}' back!"); + } + Err(err) => { + println!("❌"); + println!("the received message got malformed on the way to us: {err}"); + } + }, + Ok(None) => { + println!("❌"); + println!("failed to receive any message back..."); + } + } + + println!("disconnecting the client before shutting down..."); + client.disconnect().await; + } + + time::sleep(Duration::from_secs(100)).await; + } + + Ok(()) +} + +async fn reqwest_and_parse(endpoint: Url, key: &str) -> Result> { + let response: Response = reqwest::get(endpoint).await?; + let json: Value = response.json().await?; + let parsed: Vec = json["nodes"]["data"] + .as_array() + .unwrap() + .iter() + .map(|node| node[key].as_str().unwrap().to_string()) + .collect(); + Ok(parsed) +} + +/* + +// let response = reqwest::get( +// "https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/entry-gateways/active", +// ) +// .await?; +// let json: Value = response.json().await?; + +// let exit_gw_keys: Vec = json["nodes"]["data"] +// .as_array() +// .unwrap() +// .iter() +// .map(|node| { +// node["ed25519_identity_pubkey"] +// .as_str() +// .unwrap() +// .to_string() +// }) +// .collect(); + +// println!("Got {} active exit gw keys", exit_gw_keys.len(),); + +*/ From 060f785ff0117fab8b8e099518e7490c35a0b8d9 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 20 Dec 2024 00:11:11 +0100 Subject: [PATCH 071/102] temp commit --- Cargo.lock | 19 +++++++++++++++++++ Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 58893ee0f5a..9d456c7028d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4064,6 +4064,25 @@ dependencies = [ "wasm-utils", ] +[[package]] +name = "mixnet-check-all-gateways" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.20", + "echo-server", + "futures", + "nym-bin-common", + "nym-crypto", + "nym-sdk", + "reqwest 0.12.4", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "mixnet-connectivity-check" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5d3dbdd8102..a0b0fe957d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,7 +149,7 @@ members = [ "tools/internal/testnet-manager", "tools/internal/testnet-manager/dkg-bypass-contract", "common/verloc", - "tools/internal/mixnet-connectivity-check", + "tools/internal/mixnet-connectivity-check", "tools/internal/mixnet-check-all-gateways", ] default-members = [ From 45bef0da687bddbb820226f26558f7c21f232e06 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 20 Dec 2024 13:13:45 +0100 Subject: [PATCH 072/102] temp commit --- .../mixnet-check-all-gateways/src/main.rs | 126 ++++++++---------- 1 file changed, 59 insertions(+), 67 deletions(-) diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index 49cda71615f..9801abf367a 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -8,7 +8,7 @@ use nym_bin_common::logging::setup_logging; use nym_crypto::asymmetric::ed25519; use nym_sdk::mixnet; use nym_sdk::mixnet::MixnetMessageSender; -use reqwest::{self, Response, Url}; +use reqwest::{self, Url}; use serde_json::Value; use std::time::Duration; use tokio::time; @@ -18,12 +18,8 @@ use tokio::time::timeout; async fn main() -> Result<()> { setup_logging(); let entry_gw_keys = reqwest_and_parse( - Url::parse( - "https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/entry-gateways/all?no_legacy=true", // make const - - ) - .unwrap(), - "ed25519_identity_pubkey", + Url::parse("https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/entry-gateways/all?no_legacy=true").unwrap(), + "EntryGateway", ) .await?; println!( @@ -34,10 +30,10 @@ async fn main() -> Result<()> { let exit_gw_keys = reqwest_and_parse( Url::parse( - "https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/exit-gateways/all?no_legacy=true", // make const + "https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/exit-gateways/all?no_legacy=true", ) .unwrap(), - "ed25519_identity_pubkey", + "ExitGateway", ) .await?; println!( @@ -48,23 +44,27 @@ async fn main() -> Result<()> { for gw in exit_gw_keys { println!("{}", gw); - time::sleep(Duration::from_secs(1)).await; - let mut echo_server = NymEchoServer::new( - Some(ed25519::PublicKey::from_base58_string(gw)?), - None, - "../../../envs/mainnet.env".to_string(), // make const - "9000", // when you run concurrently you can probably iterate through ports here as well - ) - .await?; + // TODO set up a client manually with a reply fn to troubleshoot wtf is going on + + // time::sleep(Duration::from_secs(1)).await; + // let mut echo_server = NymEchoServer::new( + // Some(ed25519::PublicKey::from_base58_string( + // exit_gw_keys[0].clone(), + // )?), + // None, + // "../../../envs/mainnet.env".to_string(), // make const + // "9000", // when you run concurrently you can probably iterate through ports here as well + // ) + // .await?; - let echo_addr = echo_server.nym_address().await; - println!("echo addr: {echo_addr}"); + // let echo_addr = echo_server.nym_address().await; + // println!("echo addr: {echo_addr}"); - tokio::task::spawn(async move { - echo_server.run().await?; - Ok::<(), anyhow::Error>(()) - }); + // tokio::task::spawn(async move { + // echo_server.run().await?; + // Ok::<(), anyhow::Error>(()) + // }); for gw in entry_gw_keys.clone() { let builder = mixnet::MixnetClientBuilder::new_ephemeral() @@ -72,10 +72,7 @@ async fn main() -> Result<()> { .build()?; let mut client = match builder.connect_to_mixnet().await { - Ok(client) => { - println!("connected"); - client - } + Ok(client) => client, Err(err) => { println!("failed to connect: {err}"); return Err(err.into()); @@ -83,71 +80,66 @@ async fn main() -> Result<()> { }; let our_address = client.nym_address(); println!("{our_address}"); - client.send_plain_message(echo_addr, "echo").await?; + client.send_plain_message(*our_address, "echo").await?; match timeout(Duration::from_secs(5), client.next()).await { Err(_timeout) => { - println!("❌"); - println!("timed out while waiting for the response..."); + println!("timed out"); } Ok(Some(received)) => match String::from_utf8(received.message) { Ok(message) => { - println!("✅"); println!("received '{message}' back!"); } Err(err) => { - println!("❌"); println!("the received message got malformed on the way to us: {err}"); } }, Ok(None) => { - println!("❌"); println!("failed to receive any message back..."); } } - println!("disconnecting the client before shutting down..."); client.disconnect().await; } - - time::sleep(Duration::from_secs(100)).await; } Ok(()) } async fn reqwest_and_parse(endpoint: Url, key: &str) -> Result> { - let response: Response = reqwest::get(endpoint).await?; + let response = reqwest::get(endpoint).await?; let json: Value = response.json().await?; - let parsed: Vec = json["nodes"]["data"] - .as_array() - .unwrap() - .iter() - .map(|node| node[key].as_str().unwrap().to_string()) - .collect(); - Ok(parsed) + let filtered_keys = filter_gateway_keys(&json, key)?; + Ok(filtered_keys) } -/* - -// let response = reqwest::get( -// "https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/entry-gateways/active", -// ) -// .await?; -// let json: Value = response.json().await?; - -// let exit_gw_keys: Vec = json["nodes"]["data"] -// .as_array() -// .unwrap() -// .iter() -// .map(|node| { -// node["ed25519_identity_pubkey"] -// .as_str() -// .unwrap() -// .to_string() -// }) -// .collect(); - -// println!("Got {} active exit gw keys", exit_gw_keys.len(),); - -*/ +fn filter_gateway_keys(json: &Value, key: &str) -> Result> { + let mut filtered_keys = Vec::new(); + + if let Some(nodes) = json["nodes"]["data"].as_array() { + for node in nodes { + if let Some(performance) = node.get("performance").and_then(|v| v.as_str()) { + let performance_value: f64 = performance.parse().unwrap_or(0.0); + + let inactive = node.get("role").and_then(|v| v.as_str()) == Some("Inactive"); + + if let Some(role) = node.get("role").and_then(|v| v.as_str()) { + let is_correct_gateway = role == key; + // println!("Node: {:?}", node); + // println!("Performance: {}", performance_value); + // println!("Blacklisted: {}", inactive); + if performance_value > 0.0 && !inactive && is_correct_gateway { + if let Some(gateway_identity_key) = + node.get("ed25519_identity_pubkey").and_then(|v| v.as_str()) + { + filtered_keys.push(gateway_identity_key.to_string()); + } + } + } + } + } + } else { + println!("No nodes found "); + } + Ok(filtered_keys) +} From 814aeb5e7c4550b880f93bccf3549dc643c9d975 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 30 Dec 2024 17:43:41 +0100 Subject: [PATCH 073/102] clap standalone echo_server --- tools/echo-server/src/echo-server.rs | 49 ++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/tools/echo-server/src/echo-server.rs b/tools/echo-server/src/echo-server.rs index d6cfa13d59b..b151f6aaedb 100644 --- a/tools/echo-server/src/echo-server.rs +++ b/tools/echo-server/src/echo-server.rs @@ -1,17 +1,46 @@ // Copyright 2024 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only -// use clap::{Args, Parser, Subcommand}; -// use nym_bin_common::{bin_info, bin_info_owned}; -// use std::sync::OnceLock; +use anyhow::Result; +use clap::Parser; +use echo_server::NymEchoServer; +use nym_crypto::asymmetric::ed25519; -// fn pretty_build_info_static() -> &'static str { -// static PRETTY_BUILD_INFORMATION: OnceLock = OnceLock::new(); -// PRETTY_BUILD_INFORMATION.get_or_init(|| bin_info!().pretty_print()) -// } +#[derive(Parser, Debug)] +struct Args { + /// Optional gateway to use + #[clap(short, long)] + gateway: Option, -// This will be a Clap CLI binary for standalone use use wrapping the lib + /// Optional config path to specify + #[clap(short, long)] + config_path: Option, -fn main() { - todo!() + /// Env file + #[clap(short, long)] + env: String, + + /// Listen port + #[clap(long, default_value = "8080")] + listen_port: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + nym_bin_common::logging::setup_tracing_logger(); + let args = Args::parse(); + let mut echo_server = NymEchoServer::new( + args.gateway, + args.config_path.as_deref(), + args.env, + args.listen_port.as_str(), + ) + .await?; + + let echo_addr = echo_server.nym_address().await; + println!("{echo_addr}"); + + echo_server.run().await?; + + Ok(()) } From fb63555ab051d5ab84413300b144f2a8cba6ad5c Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 30 Dec 2024 17:45:14 +0100 Subject: [PATCH 074/102] add metric getter + debug print --- tools/echo-server/src/lib.rs | 64 +++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index 63a4d32085b..fa4e9c46559 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -3,7 +3,7 @@ use anyhow::Result; use nym_crypto::asymmetric::ed25519; -use nym_sdk::mixnet::Recipient; +use nym_sdk::mixnet::{MixnetClient, MixnetClientSender, Recipient}; use nym_sdk::tcp_proxy; use nym_sdk::tcp_proxy::NymProxyServer; use std::sync::atomic::{AtomicU64, Ordering}; @@ -16,6 +16,7 @@ use tokio_stream::StreamExt; use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; +#[derive(Debug)] pub struct Metrics { total_conn: AtomicU64, active_conn: AtomicU64, @@ -34,6 +35,7 @@ impl Metrics { } } +#[derive(Clone)] pub struct NymEchoServer { client: Arc>, listen_addr: String, @@ -174,4 +176,64 @@ impl NymEchoServer { pub fn listen_addr(&self) -> String { self.listen_addr.clone() } + + pub fn metrics(&self) -> Arc { + self.metrics.clone() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::StreamExt; + use nym_sdk::mixnet::{MixnetClient, MixnetMessageSender, Recipient}; + + #[tokio::test] + async fn echoes_bytes() { + let mut echo_server = + NymEchoServer::new(None, None, "../../envs/mainnet.env".to_string(), "9000") + .await + .unwrap(); + + let echo_addr = echo_server.nym_address().await; + println!("{echo_addr}"); + let incoming_metrics = echo_server.clone().metrics(); + println!("{incoming_metrics:#?}"); + + tokio::task::spawn(async move { + echo_server.run().await.unwrap(); + }); + + let message = "test"; + + let mut client = MixnetClient::connect_new().await.unwrap(); + let sender = client.split_sender(); + tokio::spawn(async move { + sender.send_plain_message(echo_addr, message).await.unwrap(); + }); + + tokio::spawn(async move { + if let Some(received) = client.next().await { + println!("Received: {}", String::from_utf8_lossy(&received.message)); + assert_eq!( + message.to_string(), + String::from_utf8_lossy(&received.message) + ); + } + client.disconnect().await; + }); + + // tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + + // assert_eq!( + // incoming_metrics.bytes_recv.load(Ordering::SeqCst) as usize, + // message_bytes.len() + // ); + } + + // #[test] + // fn creates_a_valid_nym_addr_with_given_gw() { + // // check valid + // // parse end + // } } From 7066c115e576748a0896d01ac93874140a2eebbc Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 30 Dec 2024 17:45:31 +0100 Subject: [PATCH 075/102] deps --- Cargo.lock | 1 + tools/echo-server/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 9d456c7028d..41d67af8600 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2199,6 +2199,7 @@ dependencies = [ "clap 4.5.20", "dashmap", "dirs", + "futures", "nym-bin-common", "nym-crypto", "nym-sdk", diff --git a/tools/echo-server/Cargo.toml b/tools/echo-server/Cargo.toml index c76a96e8450..78d20536167 100644 --- a/tools/echo-server/Cargo.toml +++ b/tools/echo-server/Cargo.toml @@ -35,3 +35,4 @@ nym-bin-common = { path = "../../common/bin-common", features = [ "output_format", ] } nym-crypto = { path = "../../common/crypto", features = ["asymmetric"] } +futures = { workspace = true } From 60e8f8254e5505983e1980fe9b0f93aed6625c05 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 30 Dec 2024 17:45:41 +0100 Subject: [PATCH 076/102] fix test --- .../nym-sdk/examples/tcp_proxy_multistream.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs index 4134ab325df..caf5d74020a 100644 --- a/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs +++ b/sdk/rust/nym-sdk/examples/tcp_proxy_multistream.rs @@ -52,7 +52,7 @@ async fn main() -> anyhow::Result<()> { // Within the TcpProxyClient, individual client shutdown is triggered by the timeout. The final argument is how many clients to keep in reserve in the client pool when running the TcpProxy. let proxy_client = - tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 45, Some(env), 2).await?; + tcp_proxy::NymProxyClient::new(server, "127.0.0.1", &listen_port, 80, Some(env), 3).await?; // For our disconnect() logic below let proxy_clone = proxy_client.clone(); @@ -80,7 +80,7 @@ async fn main() -> anyhow::Result<()> { println!("done. sending bytes"); // In the info traces you will see the different session IDs being set up, one for each TcpStream. - for i in 0..8 { + for i in 0..4 { let client_cancel_inner_token = client_cancel_token.clone(); if client_cancel_token.is_cancelled() { break; @@ -101,7 +101,7 @@ async fn main() -> anyhow::Result<()> { // Lets just send a bunch of messages to the server with variable delays between them, with a message and tcp connection ids to keep track of ordering on the server side (for illustrative purposes **only**; keeping track of anonymous replies is handled by the proxy under the hood with Single Use Reply Blocks (SURBs); for this illustration we want some kind of app-level message id, but irl most of the time you'll probably be parsing on e.g. the incoming response type instead) tokio::spawn(async move { - for i in 0..8 { + for i in 0..4 { if client_cancel_inner_token.is_cancelled() { break; } @@ -132,7 +132,7 @@ async fn main() -> anyhow::Result<()> { match bincode::deserialize::(&bytes) { Ok(msg) => { reply_counter += 1; - println!("<< conn {} received {}/8", msg.tcp_conn, reply_counter); + println!("<< conn {} received {}/4", msg.tcp_conn, reply_counter); } Err(e) => { println!("<< client received something that wasn't an example message of {} bytes. error: {}", bytes.len(), e); @@ -147,12 +147,18 @@ async fn main() -> anyhow::Result<()> { tokio::time::sleep(tokio::time::Duration::from_secs_f64(delay)).await; } + loop { + if example_cancel_token.is_cancelled() { + break; + } + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + } Ok(()) } // emulate a series of small messages followed by a closing larger one fn gen_bytes_fixed(i: usize) -> Vec { - let amounts = [10, 15, 50, 1000, 10, 15, 500, 2000]; + let amounts = [10, 15, 50, 1000, 2000]; let len = amounts[i]; let mut rng = rand::thread_rng(); (0..len).map(|_| rng.gen::()).collect() From 2d7957a83bcb47fcf6fbd17a72ccdca575912e65 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 30 Dec 2024 17:49:10 +0100 Subject: [PATCH 077/102] remove active conn --- tools/echo-server/src/lib.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index fa4e9c46559..8f3b32662b1 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -19,7 +19,6 @@ use tracing::{error, info, warn}; #[derive(Debug)] pub struct Metrics { total_conn: AtomicU64, - active_conn: AtomicU64, bytes_recv: AtomicU64, bytes_sent: AtomicU64, } @@ -28,7 +27,6 @@ impl Metrics { fn new() -> Self { Self { total_conn: AtomicU64::new(0), - active_conn: AtomicU64::new(0), bytes_recv: AtomicU64::new(0), bytes_sent: AtomicU64::new(0), } @@ -86,9 +84,8 @@ impl NymEchoServer { loop { tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; info!( - "Metrics: total_connections={}, active_connections={}, bytes_received={}, bytes_sent={}", + "Metrics: total_connections_since_start={}, bytes_received={}, bytes_sent={}", all_metrics.total_conn.load(Ordering::Relaxed), - all_metrics.active_conn.load(Ordering::Relaxed), all_metrics.bytes_recv.load(Ordering::Relaxed), all_metrics.bytes_sent.load(Ordering::Relaxed), ); @@ -105,7 +102,6 @@ impl NymEchoServer { info!("Handling new stream"); let connection_metrics = Arc::clone(&self.metrics); connection_metrics.total_conn.fetch_add(1, Ordering::Relaxed); - connection_metrics.active_conn.fetch_add(1, Ordering::Relaxed); tokio::spawn(NymEchoServer::handle_incoming( stream, connection_metrics, cancel_token.clone() @@ -153,9 +149,7 @@ impl NymEchoServer { } } } - metrics - .active_conn - .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + info!("Connection closed"); } @@ -163,10 +157,6 @@ impl NymEchoServer { self.cancel_token.cancel(); let client = Arc::clone(&self.client); client.lock().await.disconnect().await; - while self.metrics.active_conn.load(Ordering::Relaxed) > 0 { - info!("Waiting on active connections to close: sleeping"); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } } pub async fn nym_address(&self) -> Recipient { From eef3405777a993120114418c2292e0dcab9d1b55 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 30 Dec 2024 18:41:00 +0100 Subject: [PATCH 078/102] logging tweak --- tools/echo-server/src/echo-server.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/echo-server/src/echo-server.rs b/tools/echo-server/src/echo-server.rs index b151f6aaedb..c192860b3df 100644 --- a/tools/echo-server/src/echo-server.rs +++ b/tools/echo-server/src/echo-server.rs @@ -5,6 +5,7 @@ use anyhow::Result; use clap::Parser; use echo_server::NymEchoServer; use nym_crypto::asymmetric::ed25519; +use tracing::info; #[derive(Parser, Debug)] struct Args { @@ -21,7 +22,7 @@ struct Args { env: String, /// Listen port - #[clap(long, default_value = "8080")] + #[clap(short, long, default_value = "8080")] listen_port: String, } @@ -38,7 +39,7 @@ async fn main() -> Result<()> { .await?; let echo_addr = echo_server.nym_address().await; - println!("{echo_addr}"); + info!("listening on {echo_addr}"); echo_server.run().await?; From 6606fc8cc4fbe1b8d727f9587ab9234af64aff32 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 30 Dec 2024 18:41:19 +0100 Subject: [PATCH 079/102] work on test --- tools/echo-server/src/lib.rs | 94 ++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 35 deletions(-) diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index 8f3b32662b1..c0ae92d75fc 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -3,7 +3,7 @@ use anyhow::Result; use nym_crypto::asymmetric::ed25519; -use nym_sdk::mixnet::{MixnetClient, MixnetClientSender, Recipient}; +use nym_sdk::mixnet::Recipient; use nym_sdk::tcp_proxy; use nym_sdk::tcp_proxy::NymProxyServer; use std::sync::atomic::{AtomicU64, Ordering}; @@ -14,7 +14,7 @@ use tokio::sync::Mutex; use tokio::task; use tokio_stream::StreamExt; use tokio_util::sync::CancellationToken; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; #[derive(Debug)] pub struct Metrics { @@ -84,7 +84,7 @@ impl NymEchoServer { loop { tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; info!( - "Metrics: total_connections_since_start={}, bytes_received={}, bytes_sent={}", + "Metrics: total_connections_since_start={}, bytes_received={}, bytes_sent={}", all_metrics.total_conn.load(Ordering::Relaxed), all_metrics.bytes_recv.load(Ordering::Relaxed), all_metrics.bytes_sent.load(Ordering::Relaxed), @@ -93,7 +93,7 @@ impl NymEchoServer { }); let listener = TcpListener::bind(self.listen_addr.clone()).await?; - info!("{listener:?}"); + debug!("{listener:?}"); loop { tokio::select! { @@ -178,49 +178,73 @@ mod tests { use futures::StreamExt; use nym_sdk::mixnet::{MixnetClient, MixnetMessageSender, Recipient}; + // debug test with known addr running alongside #[tokio::test] - async fn echoes_bytes() { - let mut echo_server = - NymEchoServer::new(None, None, "../../envs/mainnet.env".to_string(), "9000") - .await - .unwrap(); - - let echo_addr = echo_server.nym_address().await; + async fn echoes_bytes_manual() { + let echo_addr = Recipient::try_from_base58_string("FAtdiqCBoTEy9m9Bq4AmrNt6YLuehCM7gpcPpBWdJDJz.DyAzs1rVNn5K9xm5UyvZUsZJVFGwNb1Ak3K3WkugcRwj@5E9DnixDjGH4jKtGPQkq5ba3tCR9Pkh8nSNV63456foJ").unwrap(); println!("{echo_addr}"); - let incoming_metrics = echo_server.clone().metrics(); - println!("{incoming_metrics:#?}"); - - tokio::task::spawn(async move { - echo_server.run().await.unwrap(); - }); let message = "test"; let mut client = MixnetClient::connect_new().await.unwrap(); let sender = client.split_sender(); - tokio::spawn(async move { + let sending_task_handle = tokio::spawn(async move { + println!("sending to {echo_addr}"); sender.send_plain_message(echo_addr, message).await.unwrap(); + println!("sent"); }); - tokio::spawn(async move { - if let Some(received) = client.next().await { - println!("Received: {}", String::from_utf8_lossy(&received.message)); - assert_eq!( - message.to_string(), - String::from_utf8_lossy(&received.message) - ); - } - client.disconnect().await; - }); - - // tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; - - // assert_eq!( - // incoming_metrics.bytes_recv.load(Ordering::SeqCst) as usize, - // message_bytes.len() - // ); + sending_task_handle.await.unwrap(); + tokio::time::sleep(tokio::time::Duration::from_secs(100)).await; } + // #[tokio::test] + // async fn echoes_bytes() { + // let mut echo_server = + // NymEchoServer::new(None, None, "../../envs/mainnet.env".to_string(), "9000") + // .await + // .unwrap(); + + // let echo_addr = echo_server.nym_address().await; + + // println!("{echo_addr}"); + // let incoming_metrics = echo_server.clone().metrics(); + // println!("{incoming_metrics:#?}"); + + // tokio::task::spawn(async move { + // echo_server.run().await.unwrap(); + // }); + + // let message = "test"; + + // let mut client = MixnetClient::connect_new().await.unwrap(); + // let sender = client.split_sender(); + // let sending_task_handle = tokio::spawn(async move { + // println!("sending to {echo_addr}"); + // sender.send_plain_message(echo_addr, message).await.unwrap(); + // println!("sent"); + // }); + + // let receiving_task_handle = tokio::spawn(async move { + // if let Some(received) = client.next().await { + // println!("Received: {}", String::from_utf8_lossy(&received.message)); + // assert_eq!( + // message.to_string(), + // String::from_utf8_lossy(&received.message) + // ); + // } + // client.disconnect().await; + // }); + + // sending_task_handle.await.unwrap(); + // receiving_task_handle.await.unwrap(); + + // assert_eq!( + // incoming_metrics.bytes_recv.load(Ordering::SeqCst) as usize, + // message_bytes.len() + // ); + // } + // #[test] // fn creates_a_valid_nym_addr_with_given_gw() { // // check valid From 1cd42bde0b562d8ee56623e990fc886023302c1f Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 30 Dec 2024 18:41:41 +0100 Subject: [PATCH 080/102] working on script fr testing all --- .../mixnet-check-all-gateways/src/main.rs | 70 ++++++++++--------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index 9801abf367a..feaa562304a 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -7,7 +7,7 @@ use futures::stream::StreamExt; use nym_bin_common::logging::setup_logging; use nym_crypto::asymmetric::ed25519; use nym_sdk::mixnet; -use nym_sdk::mixnet::MixnetMessageSender; +use nym_sdk::mixnet::{AnonymousSenderTag, MixnetMessageSender, ReconstructedMessage}; use reqwest::{self, Url}; use serde_json::Value; use std::time::Duration; @@ -22,11 +22,12 @@ async fn main() -> Result<()> { "EntryGateway", ) .await?; - println!( - "got {} entry gws: \n{:?}", - entry_gw_keys.len(), - entry_gw_keys - ); + // println!( + // "got {} entry gws: \n{:?}", + // entry_gw_keys.len(), + // entry_gw_keys + // ); + println!("got {} entry gws", entry_gw_keys.len(),); let exit_gw_keys = reqwest_and_parse( Url::parse( @@ -36,35 +37,36 @@ async fn main() -> Result<()> { "ExitGateway", ) .await?; - println!( - "got {} exit gws: \n{:?}\n", - exit_gw_keys.len(), - exit_gw_keys - ); - - for gw in exit_gw_keys { + // println!( + // "got {} exit gws: \n{:?}\n", + // exit_gw_keys.len(), + // exit_gw_keys + // ); + println!("got {} exit gws", exit_gw_keys.len(),); + + for gw in exit_gw_keys.clone() { println!("{}", gw); - // TODO set up a client manually with a reply fn to troubleshoot wtf is going on + let mut echo_server = NymEchoServer::new( + Some(ed25519::PublicKey::from_base58_string( + exit_gw_keys[0].clone(), + )?), + None, + "../../../envs/mainnet.env".to_string(), // make const + "9000", // when you run concurrently you can iterate through a port range here + ) + .await?; - // time::sleep(Duration::from_secs(1)).await; - // let mut echo_server = NymEchoServer::new( - // Some(ed25519::PublicKey::from_base58_string( - // exit_gw_keys[0].clone(), - // )?), - // None, - // "../../../envs/mainnet.env".to_string(), // make const - // "9000", // when you run concurrently you can probably iterate through ports here as well - // ) - // .await?; + let echo_addr = echo_server.nym_address().await; + println!("echo addr: {echo_addr}"); - // let echo_addr = echo_server.nym_address().await; - // println!("echo addr: {echo_addr}"); + tokio::task::spawn(async move { + echo_server.run().await?; + Ok::<(), anyhow::Error>(()) + }); - // tokio::task::spawn(async move { - // echo_server.run().await?; - // Ok::<(), anyhow::Error>(()) - // }); + // dumb sleep to let it startup + time::sleep(Duration::from_secs(5)).await; for gw in entry_gw_keys.clone() { let builder = mixnet::MixnetClientBuilder::new_ephemeral() @@ -80,7 +82,7 @@ async fn main() -> Result<()> { }; let our_address = client.nym_address(); println!("{our_address}"); - client.send_plain_message(*our_address, "echo").await?; + client.send_plain_message(echo_addr, "ping").await.unwrap(); match timeout(Duration::from_secs(5), client.next()).await { Err(_timeout) => { @@ -125,9 +127,9 @@ fn filter_gateway_keys(json: &Value, key: &str) -> Result> { if let Some(role) = node.get("role").and_then(|v| v.as_str()) { let is_correct_gateway = role == key; - // println!("Node: {:?}", node); - // println!("Performance: {}", performance_value); - // println!("Blacklisted: {}", inactive); + // println!("node addr: {:?}", node); + // println!("perf score: {}", performance_value); + // println!("status: {}", inactive); if performance_value > 0.0 && !inactive && is_correct_gateway { if let Some(gateway_identity_key) = node.get("ed25519_identity_pubkey").and_then(|v| v.as_str()) From ec77f1e7b4847efc182318caff916391e89eb358 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 31 Dec 2024 13:17:19 +0100 Subject: [PATCH 081/102] minor log tweak --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 87592400a54..977a2a61129 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -67,7 +67,10 @@ impl NymProxyClient { } pub async fn run(&self) -> Result<()> { - info!("Connecting to mixnet server at {}", self.server_address); + info!( + "Outgoing Mixnet traffic will be sent to {}", + self.server_address + ); let listener = TcpListener::bind(format!("{}:{}", self.listen_address, self.listen_port)).await?; From 0337c04844f55564bfa945e4bb87d13aae7a04c1 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 31 Dec 2024 13:17:41 +0100 Subject: [PATCH 082/102] fixed test: needed proxiedMessage format --- tools/echo-server/src/lib.rs | 96 ++++++++++++---------------- tools/echo-server/src/tests/utils.rs | 54 ++++++++++++++++ 2 files changed, 95 insertions(+), 55 deletions(-) create mode 100644 tools/echo-server/src/tests/utils.rs diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index c0ae92d75fc..1a706b17f4c 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -176,77 +176,63 @@ impl NymEchoServer { mod tests { use super::*; use futures::StreamExt; - use nym_sdk::mixnet::{MixnetClient, MixnetMessageSender, Recipient}; + use nym_sdk::mixnet::{IncludedSurbs, MixnetClient, MixnetMessageSender, Recipient}; + #[path = "utils.rs"] + mod utils; + use utils::{Payload, ProxiedMessage}; - // debug test with known addr running alongside #[tokio::test] - async fn echoes_bytes_manual() { - let echo_addr = Recipient::try_from_base58_string("FAtdiqCBoTEy9m9Bq4AmrNt6YLuehCM7gpcPpBWdJDJz.DyAzs1rVNn5K9xm5UyvZUsZJVFGwNb1Ak3K3WkugcRwj@5E9DnixDjGH4jKtGPQkq5ba3tCR9Pkh8nSNV63456foJ").unwrap(); + async fn echoes_bytes() { + let mut echo_server = + NymEchoServer::new(None, None, "../../envs/mainnet.env".to_string(), "9000") + .await + .unwrap(); + + let echo_addr = echo_server.nym_address().await; println!("{echo_addr}"); - let message = "test"; + tokio::task::spawn(async move { + echo_server.run().await.unwrap(); + }); + + let session_id = uuid::Uuid::new_v4(); + let message_id = 0; + let outgoing = ProxiedMessage::new( + Payload::Data("test".as_bytes().to_vec()), + session_id, + message_id, + ); + let coded_message = bincode::serialize(&outgoing).unwrap(); let mut client = MixnetClient::connect_new().await.unwrap(); let sender = client.split_sender(); let sending_task_handle = tokio::spawn(async move { - println!("sending to {echo_addr}"); - sender.send_plain_message(echo_addr, message).await.unwrap(); - println!("sent"); + sender + .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(10)) + .await + .unwrap(); + }); + + let receiving_task_handle = tokio::spawn(async move { + if let Some(received) = client.next().await { + let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); + assert_eq!(outgoing.message, incoming.message); + } + client.disconnect().await; }); sending_task_handle.await.unwrap(); - tokio::time::sleep(tokio::time::Duration::from_secs(100)).await; + receiving_task_handle.await.unwrap(); } // #[tokio::test] - // async fn echoes_bytes() { - // let mut echo_server = - // NymEchoServer::new(None, None, "../../envs/mainnet.env".to_string(), "9000") - // .await - // .unwrap(); - - // let echo_addr = echo_server.nym_address().await; - - // println!("{echo_addr}"); - // let incoming_metrics = echo_server.clone().metrics(); - // println!("{incoming_metrics:#?}"); - - // tokio::task::spawn(async move { - // echo_server.run().await.unwrap(); - // }); - - // let message = "test"; - - // let mut client = MixnetClient::connect_new().await.unwrap(); - // let sender = client.split_sender(); - // let sending_task_handle = tokio::spawn(async move { - // println!("sending to {echo_addr}"); - // sender.send_plain_message(echo_addr, message).await.unwrap(); - // println!("sent"); - // }); - - // let receiving_task_handle = tokio::spawn(async move { - // if let Some(received) = client.next().await { - // println!("Received: {}", String::from_utf8_lossy(&received.message)); - // assert_eq!( - // message.to_string(), - // String::from_utf8_lossy(&received.message) - // ); - // } - // client.disconnect().await; - // }); - - // sending_task_handle.await.unwrap(); - // receiving_task_handle.await.unwrap(); - - // assert_eq!( - // incoming_metrics.bytes_recv.load(Ordering::SeqCst) as usize, - // message_bytes.len() - // ); + // async fn incoming_and_sent_bytes_metrics_work() { + // todo!() // } - // #[test] - // fn creates_a_valid_nym_addr_with_given_gw() { + // #[tokio::test] + // async fn creates_a_valid_nym_addr_with_specified_gw() { + // todo!() // // check valid // // parse end // } diff --git a/tools/echo-server/src/tests/utils.rs b/tools/echo-server/src/tests/utils.rs new file mode 100644 index 00000000000..0735b009db6 --- /dev/null +++ b/tools/echo-server/src/tests/utils.rs @@ -0,0 +1,54 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; +use uuid::Uuid; + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ProxiedMessage { + pub message: Payload, + session_id: Uuid, + message_id: u16, +} + +impl ProxiedMessage { + pub fn new(message: Payload, session_id: Uuid, message_id: u16) -> Self { + ProxiedMessage { + message, + session_id, + message_id, + } + } + + pub fn message(&self) -> &Payload { + &self.message + } + + pub fn session_id(&self) -> Uuid { + self.session_id + } + + pub fn message_id(&self) -> u16 { + self.message_id + } +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub enum Payload { + Data(Vec), + Close, +} + +impl fmt::Display for ProxiedMessage { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let message = match self.message() { + Payload::Data(ref data) => format!("Data({})", data.len()), + Payload::Close => "Close".to_string(), + }; + write!( + f, + "ProxiedMessage {{ message: {}, session_id: {}, message_id: {} }}", + message, + self.session_id(), + self.message_id() + ) + } +} From ffcc4e8fa564608fd125373e6a8fbe650315620a Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 2 Jan 2025 19:13:23 +0100 Subject: [PATCH 083/102] first pass tester --- .../mixnet-check-all-gateways/Cargo.toml | 3 + .../mixnet-check-all-gateways/src/main.rs | 181 ++++++++++++++---- ...W3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json | 1 + ...KqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json | 1 + ...BpaTsy3Xd7BE2ZAEKiaQZFsXEZPf9ZKrQtxgS.json | 0 ...dQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json | 1 + ...QgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json | 1 + .../mixnet-check-all-gateways/src/scratch | 35 ++++ .../mixnet-check-all-gateways/src/utils.rs | 54 ++++++ 9 files changed, 244 insertions(+), 33 deletions(-) create mode 100644 tools/internal/mixnet-check-all-gateways/src/results/2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json create mode 100644 tools/internal/mixnet-check-all-gateways/src/results/4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json create mode 100644 tools/internal/mixnet-check-all-gateways/src/results/6vtDnxeBpaTsy3Xd7BE2ZAEKiaQZFsXEZPf9ZKrQtxgS.json create mode 100644 tools/internal/mixnet-check-all-gateways/src/results/98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json create mode 100644 tools/internal/mixnet-check-all-gateways/src/results/Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json create mode 100644 tools/internal/mixnet-check-all-gateways/src/scratch create mode 100644 tools/internal/mixnet-check-all-gateways/src/utils.rs diff --git a/tools/internal/mixnet-check-all-gateways/Cargo.toml b/tools/internal/mixnet-check-all-gateways/Cargo.toml index 7e35bfc9141..16b4540983d 100644 --- a/tools/internal/mixnet-check-all-gateways/Cargo.toml +++ b/tools/internal/mixnet-check-all-gateways/Cargo.toml @@ -27,3 +27,6 @@ tokio = { workspace = true, features = ["full"] } reqwest = { workspace = true, features = ["json"] } serde_json.workspace = true futures = { workspace = true } +uuid = { version = "1", features = ["v4", "serde"] } +bincode = "1.0" +dirs.workspace = true diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index feaa562304a..0ea4bea331d 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -1,22 +1,49 @@ // Copyright 2024 - Nym Technologies SA // SPDX-License-Identifier: GPL-3.0-only -use anyhow::Result; +use anyhow::{anyhow, Result}; use echo_server::NymEchoServer; use futures::stream::StreamExt; use nym_bin_common::logging::setup_logging; use nym_crypto::asymmetric::ed25519; use nym_sdk::mixnet; -use nym_sdk::mixnet::{AnonymousSenderTag, MixnetMessageSender, ReconstructedMessage}; +use nym_sdk::mixnet::{ + AnonymousSenderTag, IncludedSurbs, MixnetMessageSender, ReconstructedMessage, +}; use reqwest::{self, Url}; -use serde_json::Value; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::fs::OpenOptions; +use std::io::Write; use std::time::Duration; -use tokio::time; use tokio::time::timeout; +use tokio::time::{self, Timeout}; +#[path = "utils.rs"] +// TODO make these exportable from tcp_proxy module and then import from there, ditto with echo server lib +mod utils; +use utils::{Payload, ProxiedMessage}; + +const TIMEOUT: u64 = 10; // message ping timeout + +#[derive(Serialize, Deserialize, Debug)] +struct TestResult { + entry_gw: String, + exit_gw: String, + error: TestError, +} + +#[derive(Serialize, Deserialize, Debug)] +enum TestError { + Timeout, + NoMessage, + None, + CouldNotCreateEchoServer(String), + Other(String), +} #[tokio::main] async fn main() -> Result<()> { - setup_logging(); + // setup_logging(); // TODO think about parsing and noise here, could just parse on errors from all libs and then have info from here? let entry_gw_keys = reqwest_and_parse( Url::parse("https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/entry-gateways/all?no_legacy=true").unwrap(), "EntryGateway", @@ -44,18 +71,50 @@ async fn main() -> Result<()> { // ); println!("got {} exit gws", exit_gw_keys.len(),); - for gw in exit_gw_keys.clone() { - println!("{}", gw); + let mut port_range: u64 = 9000; // port that we start iterating upwards from, will go from port_range to (port_range + exit_gws.len()) by the end of the run + + for exit_gw in exit_gw_keys.clone() { + println!("creating echo server connecting to {}", exit_gw); - let mut echo_server = NymEchoServer::new( - Some(ed25519::PublicKey::from_base58_string( - exit_gw_keys[0].clone(), - )?), - None, + let filepath = format!("./src/results/{}.json", exit_gw.clone()); + let mut results = OpenOptions::new() + .read(true) + .write(true) // .append(true) + .create(true) + .open(filepath)?; + + let mut results_vec: Vec = Vec::new(); + let home_dir = dirs::home_dir().expect("Unable to get home directory"); + let mut echo_server = match NymEchoServer::new( + Some(ed25519::PublicKey::from_base58_string(&exit_gw)?), + Some( + format!( + "{}/tmp/nym-proxy-server-config-{}", + home_dir.display(), + &exit_gw + ) + .as_str(), + ), "../../../envs/mainnet.env".to_string(), // make const - "9000", // when you run concurrently you can iterate through a port range here + port_range.to_string().as_str(), ) - .await?; + .await + { + Ok(echo_server) => echo_server, + Err(err) => { + let res = TestResult { + entry_gw: "".to_string(), + exit_gw: exit_gw.clone(), + error: TestError::CouldNotCreateEchoServer(err.to_string()), + }; + results_vec.push(json!(res)); + let json_array = json!(results_vec); + println!("{json_array}"); + results.write_all(json_array.to_string().as_bytes())?; + continue; + } + }; + port_range += 1; let echo_addr = echo_server.nym_address().await; println!("echo addr: {echo_addr}"); @@ -68,43 +127,97 @@ async fn main() -> Result<()> { // dumb sleep to let it startup time::sleep(Duration::from_secs(5)).await; - for gw in entry_gw_keys.clone() { + for entry_gw in entry_gw_keys.clone() { let builder = mixnet::MixnetClientBuilder::new_ephemeral() - .request_gateway(gw) + .request_gateway(entry_gw.clone()) .build()?; let mut client = match builder.connect_to_mixnet().await { Ok(client) => client, Err(err) => { - println!("failed to connect: {err}"); - return Err(err.into()); + let res = TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::Other(err.to_string()), + }; + println!("{res:#?}"); + results_vec.push(json!(res)); + // println!("failed to connect: {err}"); + continue; + } + }; + + let test_address = client.nym_address(); + println!("currently testing entry gateway: {test_address}"); + + // Has to be ProxiedMessage for the moment which is slightly annoying until I + // modify the ProxyServer to just stupidly echo back whatever it gets in a + // ReconstructedMessage format if it can't deseralise it to a ProxiedMessage + let session_id = uuid::Uuid::new_v4(); + let message_id = 0; + let outgoing = ProxiedMessage::new( + Payload::Data("echo test".as_bytes().to_vec()), + session_id, + message_id, + ); + let coded_message = bincode::serialize(&outgoing).unwrap(); + + match client + .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(10)) + .await + { + Ok(_) => { + println!("Message sent"); + } + Err(err) => { + let res = TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::Other(err.to_string()), + }; + println!("{res:#?}"); + results_vec.push(json!(res)); + continue; } }; - let our_address = client.nym_address(); - println!("{our_address}"); - client.send_plain_message(echo_addr, "ping").await.unwrap(); - match timeout(Duration::from_secs(5), client.next()).await { + let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { Err(_timeout) => { println!("timed out"); - } - Ok(Some(received)) => match String::from_utf8(received.message) { - Ok(message) => { - println!("received '{message}' back!"); + TestResult { + entry_gw, + exit_gw: exit_gw.clone(), + error: TestError::Timeout, } - Err(err) => { - println!("the received message got malformed on the way to us: {err}"); + } + Ok(Some(received)) => { + let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); + println!("\ngot echo: {incoming}\n"); + // TODO check incoming is same as outgoing else make a MangledReply err type + TestResult { + entry_gw, + exit_gw: exit_gw.clone(), + error: TestError::None, } - }, + } Ok(None) => { println!("failed to receive any message back..."); + TestResult { + entry_gw, + exit_gw: exit_gw.clone(), + error: TestError::NoMessage, + } } - } + }; + println!("{res:#?}"); + results_vec.push(json!(res)); println!("disconnecting the client before shutting down..."); client.disconnect().await; } + let json_array = json!(results_vec); + println!("{json_array}"); + results.write_all(json_array.to_string().as_bytes())?; } - Ok(()) } @@ -121,7 +234,7 @@ fn filter_gateway_keys(json: &Value, key: &str) -> Result> { if let Some(nodes) = json["nodes"]["data"].as_array() { for node in nodes { if let Some(performance) = node.get("performance").and_then(|v| v.as_str()) { - let performance_value: f64 = performance.parse().unwrap_or(0.0); + let performance_value: f64 = performance.parse().unwrap_or(0.0); // TODO could make this a const @ top? let inactive = node.get("role").and_then(|v| v.as_str()) == Some("Inactive"); @@ -141,7 +254,9 @@ fn filter_gateway_keys(json: &Value, key: &str) -> Result> { } } } else { - println!("No nodes found "); + // TODO make error and return / break + // println!("No nodes found "); + return Err(anyhow!("Could not parse any gateways")); } Ok(filtered_keys) } diff --git a/tools/internal/mixnet-check-all-gateways/src/results/2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json b/tools/internal/mixnet-check-all-gateways/src/results/2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json new file mode 100644 index 00000000000..154bdcd3bb0 --- /dev/null +++ b/tools/internal/mixnet-check-all-gateways/src/results/2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json @@ -0,0 +1 @@ +[{"entry_gw":"63ctaex57EvjJZu92jT2ve2ULgmjVYAQph83qNjMpFDZ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"4S4FPLrQ3ZncmGtFXrpgWLtYRn4hcDmb7CTLwk6dkJ7A","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"C7J8SwZQqjWqhBryyjJxLt7FacVuPTwAmR2otGy53ayi","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"EsQEJhuxeNGJXzXA6R51wLCPFKWP322dHnBGXUgmvPJH","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"AkUa1476EZfpBRfWSEWvkXrUp6AGaFwV67ZpSPvDKTQ7","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"EDEtSQ6dVEE6JQqQLZFMdLETCx9NNgqoxkbvoyvfQR48","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"E3Zyogy3C7X1yNpQKwhsaveGbfgFwPpobHZ8DrPEUwyM","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"Ca4bbfxVGtuGuiYvH7V5jSnj9Jp4VJa4Sznu2doZD5np","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"3B7PsbXFuqq6rerYFLw5HPbQb4UmBqAhfWURRovMmWoj","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"H7rDKAXAjJjtkJAgz3paB95zUowH5pHVMBQBdxrx6Yzg","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"4WgKhJdmUffz4e1o1ftVAGS3HnG56LiNAxA9dmaekrVd","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DLxLKsd3LTnfudSSmHanPaZACsh1x4MGEzfJS4jQibir","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DfNMqQRy6pPkU8Z5rBsxRwzDUzAMXHPFwMhjF16ScZqn","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"2zHiExNRKiCXVKS35SNKtK4apGfZELMpA1jJ2gVevJoz","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"9PG6vqoVniK7bWD7esueje9pD3P3iU3Md1T8FAuNQipW","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DgD7K1RHn7kMPC3ibg6HkJFkDqaFstxpEFHgronRdCqj","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"6CTSR2KbwhwHJ2kA3xqA9KLBw1dXcfSyPRKaXBB7Sh1C","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"2hg8pU5fG6WxygEx1pqrwUDkJ8gu6GAe1LysNTfrF4jX","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7mperTohH5oNonkZH7EJkuZaa5WMowdgYY7ah6UFgPxJ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"38zcSsvjXsAX7C28ko2H3Lt55X4TYxfZYkPADxKXZHUj","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7fiZtNL1RACQTwGrKLBT9nbY77bfwZnX9rqcWqc53qgv","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"9xJM74FwwHhEKKJHihD21QSZnHM2QBRMoFx9Wst6qNBS","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"6ftUpAtiNRxPb793eg9vKhHhBJcgm8wRcToF3UbJJyWn","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DiciBkjHovXzTDE2EFJKPNj3TGw2oQjr7HPSas2YQPiQ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"ADjpymCgjFsE5m7YvnZFxLMscg85dCUUesR5g8yN3Mzz","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"J22ULbeW3rXvkHseBS5UuQnZ5c8uQyFQD1eLxm9ULz3H","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"BeT6Hyt1cSvtC5rYZeabZ5E3FWqRXHqUxb62DTLh564K","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"FmbUngD26tUGvJN8QqL78iK96bZYV2bWkjWZU7fDwBN1","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"Hs463Wh5LtWZUNyAmt4trcCbNVsuUhry1wpEXpVnAAfn","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7SnUJy4rWH9hXCitpgwx7XoK5PGRBNjaiz7BWeaqRfXx","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"8ShVhzs9fpKyZPgdtTTRT8yviU2tqk3i5Hnk7rMfLbML","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"HMbxs892i1thjXtPhtz9TGU41ghZCiyC8HhYWkQZs4Cn","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"8R9CjJsfWbAmisHcarJYLrvTdkW1D4CLyy3iAY89j5QF","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"A2dvg4E4SYFqaziJdGPv9hWF7Yh1Po11YSp9p7eEv1xm","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"Fottgyv51kxDKTcVmqP5VfkaXbuTZtCuPWikksTNBRVQ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"Hm7Jn79ztfvXcQjTpUdyx2jqcv4UgBAHcgChhH6b5P4Q","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7WWGub1zjX5KxYHzqiUHsCd12He5hcwfi97GWXLuwnrZ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"HNw9Z6HqwuhUXrqx6CDWMLK2bGGhCaPTSsRRevLWGPXw","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"FfdYC39U4qREoUZSrdNzGLQNug6Ac1PdxkZrUifR4jBR","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"CcYinhLeFU8n6xs78FG6Rz3wvosGTCU2hLB1CZyfkMVe","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"4cdgnckjRBAaaP4ds68YEJPRc78paC4YWMGBDUS8G57C","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"58ceEFaLJh6zAo3cirzT1BDQm7D3L5acnQrxGH1D6TAY","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"2DFyK19kUDGdZYkAUE5pTwot6Hvf7uRhnjs5rKP8PDSk","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"98uf1hyzmWTinkyc5PGyCxDo3E9QQK5XhWQ8B8z8aFoX","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7VJQ4FGRmLESkmjTeTpF6o3VFLVZ5xe9CoJSR38bZawQ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"AY4uHZFYVxwT6NiEXGLmdp9mxVZpW33ViUUqPgzWcF59","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DAGQxdxwAkwjaLjTw1B9vndia4YyFD15qRgcTQxrmkom","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"GwPzj3VXALYZJKjbEaveL8CD9wp4eDU4DXxUjMTRxFn6","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7ntzmDZRvG4a1pnDBU4Bg1RiAmLwmqXV5sZGNw68Ce14","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"ELjxhHLX1ZxfnJgQhtBTUiVUzLbfZs9Y6bSTpLLkAkR7","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"}] \ No newline at end of file diff --git a/tools/internal/mixnet-check-all-gateways/src/results/4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json b/tools/internal/mixnet-check-all-gateways/src/results/4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json new file mode 100644 index 00000000000..884d1948bc6 --- /dev/null +++ b/tools/internal/mixnet-check-all-gateways/src/results/4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json @@ -0,0 +1 @@ +[{"entry_gw":"63ctaex57EvjJZu92jT2ve2ULgmjVYAQph83qNjMpFDZ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"4S4FPLrQ3ZncmGtFXrpgWLtYRn4hcDmb7CTLwk6dkJ7A","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"C7J8SwZQqjWqhBryyjJxLt7FacVuPTwAmR2otGy53ayi","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"EsQEJhuxeNGJXzXA6R51wLCPFKWP322dHnBGXUgmvPJH","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"AkUa1476EZfpBRfWSEWvkXrUp6AGaFwV67ZpSPvDKTQ7","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"EDEtSQ6dVEE6JQqQLZFMdLETCx9NNgqoxkbvoyvfQR48","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"E3Zyogy3C7X1yNpQKwhsaveGbfgFwPpobHZ8DrPEUwyM","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"Ca4bbfxVGtuGuiYvH7V5jSnj9Jp4VJa4Sznu2doZD5np","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"3B7PsbXFuqq6rerYFLw5HPbQb4UmBqAhfWURRovMmWoj","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"H7rDKAXAjJjtkJAgz3paB95zUowH5pHVMBQBdxrx6Yzg","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"4WgKhJdmUffz4e1o1ftVAGS3HnG56LiNAxA9dmaekrVd","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DLxLKsd3LTnfudSSmHanPaZACsh1x4MGEzfJS4jQibir","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DfNMqQRy6pPkU8Z5rBsxRwzDUzAMXHPFwMhjF16ScZqn","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"2zHiExNRKiCXVKS35SNKtK4apGfZELMpA1jJ2gVevJoz","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"9PG6vqoVniK7bWD7esueje9pD3P3iU3Md1T8FAuNQipW","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DgD7K1RHn7kMPC3ibg6HkJFkDqaFstxpEFHgronRdCqj","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"6CTSR2KbwhwHJ2kA3xqA9KLBw1dXcfSyPRKaXBB7Sh1C","error":"Timeout","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"2hg8pU5fG6WxygEx1pqrwUDkJ8gu6GAe1LysNTfrF4jX","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7mperTohH5oNonkZH7EJkuZaa5WMowdgYY7ah6UFgPxJ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"38zcSsvjXsAX7C28ko2H3Lt55X4TYxfZYkPADxKXZHUj","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7fiZtNL1RACQTwGrKLBT9nbY77bfwZnX9rqcWqc53qgv","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"9xJM74FwwHhEKKJHihD21QSZnHM2QBRMoFx9Wst6qNBS","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"6ftUpAtiNRxPb793eg9vKhHhBJcgm8wRcToF3UbJJyWn","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DiciBkjHovXzTDE2EFJKPNj3TGw2oQjr7HPSas2YQPiQ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"ADjpymCgjFsE5m7YvnZFxLMscg85dCUUesR5g8yN3Mzz","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"J22ULbeW3rXvkHseBS5UuQnZ5c8uQyFQD1eLxm9ULz3H","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"BeT6Hyt1cSvtC5rYZeabZ5E3FWqRXHqUxb62DTLh564K","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"FmbUngD26tUGvJN8QqL78iK96bZYV2bWkjWZU7fDwBN1","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"Hs463Wh5LtWZUNyAmt4trcCbNVsuUhry1wpEXpVnAAfn","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7SnUJy4rWH9hXCitpgwx7XoK5PGRBNjaiz7BWeaqRfXx","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"8ShVhzs9fpKyZPgdtTTRT8yviU2tqk3i5Hnk7rMfLbML","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"HMbxs892i1thjXtPhtz9TGU41ghZCiyC8HhYWkQZs4Cn","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"8R9CjJsfWbAmisHcarJYLrvTdkW1D4CLyy3iAY89j5QF","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"A2dvg4E4SYFqaziJdGPv9hWF7Yh1Po11YSp9p7eEv1xm","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"Fottgyv51kxDKTcVmqP5VfkaXbuTZtCuPWikksTNBRVQ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"Hm7Jn79ztfvXcQjTpUdyx2jqcv4UgBAHcgChhH6b5P4Q","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7WWGub1zjX5KxYHzqiUHsCd12He5hcwfi97GWXLuwnrZ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"HNw9Z6HqwuhUXrqx6CDWMLK2bGGhCaPTSsRRevLWGPXw","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"FfdYC39U4qREoUZSrdNzGLQNug6Ac1PdxkZrUifR4jBR","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"CcYinhLeFU8n6xs78FG6Rz3wvosGTCU2hLB1CZyfkMVe","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"4cdgnckjRBAaaP4ds68YEJPRc78paC4YWMGBDUS8G57C","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"58ceEFaLJh6zAo3cirzT1BDQm7D3L5acnQrxGH1D6TAY","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"2DFyK19kUDGdZYkAUE5pTwot6Hvf7uRhnjs5rKP8PDSk","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"98uf1hyzmWTinkyc5PGyCxDo3E9QQK5XhWQ8B8z8aFoX","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7VJQ4FGRmLESkmjTeTpF6o3VFLVZ5xe9CoJSR38bZawQ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"AY4uHZFYVxwT6NiEXGLmdp9mxVZpW33ViUUqPgzWcF59","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DAGQxdxwAkwjaLjTw1B9vndia4YyFD15qRgcTQxrmkom","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"GwPzj3VXALYZJKjbEaveL8CD9wp4eDU4DXxUjMTRxFn6","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7ntzmDZRvG4a1pnDBU4Bg1RiAmLwmqXV5sZGNw68Ce14","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"ELjxhHLX1ZxfnJgQhtBTUiVUzLbfZs9Y6bSTpLLkAkR7","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"}] \ No newline at end of file diff --git a/tools/internal/mixnet-check-all-gateways/src/results/6vtDnxeBpaTsy3Xd7BE2ZAEKiaQZFsXEZPf9ZKrQtxgS.json b/tools/internal/mixnet-check-all-gateways/src/results/6vtDnxeBpaTsy3Xd7BE2ZAEKiaQZFsXEZPf9ZKrQtxgS.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/internal/mixnet-check-all-gateways/src/results/98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json b/tools/internal/mixnet-check-all-gateways/src/results/98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json new file mode 100644 index 00000000000..615079aa96f --- /dev/null +++ b/tools/internal/mixnet-check-all-gateways/src/results/98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json @@ -0,0 +1 @@ +[{"entry_gw":"63ctaex57EvjJZu92jT2ve2ULgmjVYAQph83qNjMpFDZ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"4S4FPLrQ3ZncmGtFXrpgWLtYRn4hcDmb7CTLwk6dkJ7A","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"C7J8SwZQqjWqhBryyjJxLt7FacVuPTwAmR2otGy53ayi","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"EsQEJhuxeNGJXzXA6R51wLCPFKWP322dHnBGXUgmvPJH","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"AkUa1476EZfpBRfWSEWvkXrUp6AGaFwV67ZpSPvDKTQ7","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"EDEtSQ6dVEE6JQqQLZFMdLETCx9NNgqoxkbvoyvfQR48","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"E3Zyogy3C7X1yNpQKwhsaveGbfgFwPpobHZ8DrPEUwyM","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"Ca4bbfxVGtuGuiYvH7V5jSnj9Jp4VJa4Sznu2doZD5np","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"3B7PsbXFuqq6rerYFLw5HPbQb4UmBqAhfWURRovMmWoj","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"H7rDKAXAjJjtkJAgz3paB95zUowH5pHVMBQBdxrx6Yzg","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"4WgKhJdmUffz4e1o1ftVAGS3HnG56LiNAxA9dmaekrVd","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DLxLKsd3LTnfudSSmHanPaZACsh1x4MGEzfJS4jQibir","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DfNMqQRy6pPkU8Z5rBsxRwzDUzAMXHPFwMhjF16ScZqn","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"2zHiExNRKiCXVKS35SNKtK4apGfZELMpA1jJ2gVevJoz","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"9PG6vqoVniK7bWD7esueje9pD3P3iU3Md1T8FAuNQipW","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DgD7K1RHn7kMPC3ibg6HkJFkDqaFstxpEFHgronRdCqj","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"6CTSR2KbwhwHJ2kA3xqA9KLBw1dXcfSyPRKaXBB7Sh1C","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"2hg8pU5fG6WxygEx1pqrwUDkJ8gu6GAe1LysNTfrF4jX","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7mperTohH5oNonkZH7EJkuZaa5WMowdgYY7ah6UFgPxJ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"38zcSsvjXsAX7C28ko2H3Lt55X4TYxfZYkPADxKXZHUj","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7fiZtNL1RACQTwGrKLBT9nbY77bfwZnX9rqcWqc53qgv","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"9xJM74FwwHhEKKJHihD21QSZnHM2QBRMoFx9Wst6qNBS","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"6ftUpAtiNRxPb793eg9vKhHhBJcgm8wRcToF3UbJJyWn","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DiciBkjHovXzTDE2EFJKPNj3TGw2oQjr7HPSas2YQPiQ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"ADjpymCgjFsE5m7YvnZFxLMscg85dCUUesR5g8yN3Mzz","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"J22ULbeW3rXvkHseBS5UuQnZ5c8uQyFQD1eLxm9ULz3H","error":"Timeout","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"BeT6Hyt1cSvtC5rYZeabZ5E3FWqRXHqUxb62DTLh564K","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"FmbUngD26tUGvJN8QqL78iK96bZYV2bWkjWZU7fDwBN1","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"Hs463Wh5LtWZUNyAmt4trcCbNVsuUhry1wpEXpVnAAfn","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7SnUJy4rWH9hXCitpgwx7XoK5PGRBNjaiz7BWeaqRfXx","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"8ShVhzs9fpKyZPgdtTTRT8yviU2tqk3i5Hnk7rMfLbML","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"HMbxs892i1thjXtPhtz9TGU41ghZCiyC8HhYWkQZs4Cn","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"8R9CjJsfWbAmisHcarJYLrvTdkW1D4CLyy3iAY89j5QF","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"A2dvg4E4SYFqaziJdGPv9hWF7Yh1Po11YSp9p7eEv1xm","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"Fottgyv51kxDKTcVmqP5VfkaXbuTZtCuPWikksTNBRVQ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"Hm7Jn79ztfvXcQjTpUdyx2jqcv4UgBAHcgChhH6b5P4Q","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7WWGub1zjX5KxYHzqiUHsCd12He5hcwfi97GWXLuwnrZ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"HNw9Z6HqwuhUXrqx6CDWMLK2bGGhCaPTSsRRevLWGPXw","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"FfdYC39U4qREoUZSrdNzGLQNug6Ac1PdxkZrUifR4jBR","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"CcYinhLeFU8n6xs78FG6Rz3wvosGTCU2hLB1CZyfkMVe","error":"Timeout","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"4cdgnckjRBAaaP4ds68YEJPRc78paC4YWMGBDUS8G57C","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"58ceEFaLJh6zAo3cirzT1BDQm7D3L5acnQrxGH1D6TAY","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"2DFyK19kUDGdZYkAUE5pTwot6Hvf7uRhnjs5rKP8PDSk","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"98uf1hyzmWTinkyc5PGyCxDo3E9QQK5XhWQ8B8z8aFoX","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7VJQ4FGRmLESkmjTeTpF6o3VFLVZ5xe9CoJSR38bZawQ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"AY4uHZFYVxwT6NiEXGLmdp9mxVZpW33ViUUqPgzWcF59","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DAGQxdxwAkwjaLjTw1B9vndia4YyFD15qRgcTQxrmkom","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"GwPzj3VXALYZJKjbEaveL8CD9wp4eDU4DXxUjMTRxFn6","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7ntzmDZRvG4a1pnDBU4Bg1RiAmLwmqXV5sZGNw68Ce14","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"ELjxhHLX1ZxfnJgQhtBTUiVUzLbfZs9Y6bSTpLLkAkR7","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"}] \ No newline at end of file diff --git a/tools/internal/mixnet-check-all-gateways/src/results/Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json b/tools/internal/mixnet-check-all-gateways/src/results/Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json new file mode 100644 index 00000000000..67eea3fdaa8 --- /dev/null +++ b/tools/internal/mixnet-check-all-gateways/src/results/Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json @@ -0,0 +1 @@ +[{"entry_gw":"63ctaex57EvjJZu92jT2ve2ULgmjVYAQph83qNjMpFDZ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"4S4FPLrQ3ZncmGtFXrpgWLtYRn4hcDmb7CTLwk6dkJ7A","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"C7J8SwZQqjWqhBryyjJxLt7FacVuPTwAmR2otGy53ayi","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"EsQEJhuxeNGJXzXA6R51wLCPFKWP322dHnBGXUgmvPJH","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"AkUa1476EZfpBRfWSEWvkXrUp6AGaFwV67ZpSPvDKTQ7","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"EDEtSQ6dVEE6JQqQLZFMdLETCx9NNgqoxkbvoyvfQR48","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"E3Zyogy3C7X1yNpQKwhsaveGbfgFwPpobHZ8DrPEUwyM","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"Ca4bbfxVGtuGuiYvH7V5jSnj9Jp4VJa4Sznu2doZD5np","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"3B7PsbXFuqq6rerYFLw5HPbQb4UmBqAhfWURRovMmWoj","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"H7rDKAXAjJjtkJAgz3paB95zUowH5pHVMBQBdxrx6Yzg","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"4WgKhJdmUffz4e1o1ftVAGS3HnG56LiNAxA9dmaekrVd","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DLxLKsd3LTnfudSSmHanPaZACsh1x4MGEzfJS4jQibir","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DfNMqQRy6pPkU8Z5rBsxRwzDUzAMXHPFwMhjF16ScZqn","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"2zHiExNRKiCXVKS35SNKtK4apGfZELMpA1jJ2gVevJoz","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"9PG6vqoVniK7bWD7esueje9pD3P3iU3Md1T8FAuNQipW","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DgD7K1RHn7kMPC3ibg6HkJFkDqaFstxpEFHgronRdCqj","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"6CTSR2KbwhwHJ2kA3xqA9KLBw1dXcfSyPRKaXBB7Sh1C","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"2hg8pU5fG6WxygEx1pqrwUDkJ8gu6GAe1LysNTfrF4jX","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7mperTohH5oNonkZH7EJkuZaa5WMowdgYY7ah6UFgPxJ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"38zcSsvjXsAX7C28ko2H3Lt55X4TYxfZYkPADxKXZHUj","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7fiZtNL1RACQTwGrKLBT9nbY77bfwZnX9rqcWqc53qgv","error":"Timeout","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"9xJM74FwwHhEKKJHihD21QSZnHM2QBRMoFx9Wst6qNBS","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"6ftUpAtiNRxPb793eg9vKhHhBJcgm8wRcToF3UbJJyWn","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DiciBkjHovXzTDE2EFJKPNj3TGw2oQjr7HPSas2YQPiQ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"ADjpymCgjFsE5m7YvnZFxLMscg85dCUUesR5g8yN3Mzz","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"J22ULbeW3rXvkHseBS5UuQnZ5c8uQyFQD1eLxm9ULz3H","error":"Timeout","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"BeT6Hyt1cSvtC5rYZeabZ5E3FWqRXHqUxb62DTLh564K","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"FmbUngD26tUGvJN8QqL78iK96bZYV2bWkjWZU7fDwBN1","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"Hs463Wh5LtWZUNyAmt4trcCbNVsuUhry1wpEXpVnAAfn","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7SnUJy4rWH9hXCitpgwx7XoK5PGRBNjaiz7BWeaqRfXx","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"8ShVhzs9fpKyZPgdtTTRT8yviU2tqk3i5Hnk7rMfLbML","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"HMbxs892i1thjXtPhtz9TGU41ghZCiyC8HhYWkQZs4Cn","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"8R9CjJsfWbAmisHcarJYLrvTdkW1D4CLyy3iAY89j5QF","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"A2dvg4E4SYFqaziJdGPv9hWF7Yh1Po11YSp9p7eEv1xm","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"Fottgyv51kxDKTcVmqP5VfkaXbuTZtCuPWikksTNBRVQ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"Hm7Jn79ztfvXcQjTpUdyx2jqcv4UgBAHcgChhH6b5P4Q","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7WWGub1zjX5KxYHzqiUHsCd12He5hcwfi97GWXLuwnrZ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"HNw9Z6HqwuhUXrqx6CDWMLK2bGGhCaPTSsRRevLWGPXw","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"FfdYC39U4qREoUZSrdNzGLQNug6Ac1PdxkZrUifR4jBR","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"CcYinhLeFU8n6xs78FG6Rz3wvosGTCU2hLB1CZyfkMVe","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"4cdgnckjRBAaaP4ds68YEJPRc78paC4YWMGBDUS8G57C","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"58ceEFaLJh6zAo3cirzT1BDQm7D3L5acnQrxGH1D6TAY","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"2DFyK19kUDGdZYkAUE5pTwot6Hvf7uRhnjs5rKP8PDSk","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"98uf1hyzmWTinkyc5PGyCxDo3E9QQK5XhWQ8B8z8aFoX","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7VJQ4FGRmLESkmjTeTpF6o3VFLVZ5xe9CoJSR38bZawQ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"AY4uHZFYVxwT6NiEXGLmdp9mxVZpW33ViUUqPgzWcF59","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DAGQxdxwAkwjaLjTw1B9vndia4YyFD15qRgcTQxrmkom","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"GwPzj3VXALYZJKjbEaveL8CD9wp4eDU4DXxUjMTRxFn6","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7ntzmDZRvG4a1pnDBU4Bg1RiAmLwmqXV5sZGNw68Ce14","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"ELjxhHLX1ZxfnJgQhtBTUiVUzLbfZs9Y6bSTpLLkAkR7","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"}] \ No newline at end of file diff --git a/tools/internal/mixnet-check-all-gateways/src/scratch b/tools/internal/mixnet-check-all-gateways/src/scratch new file mode 100644 index 00000000000..d3ac6b9d01b --- /dev/null +++ b/tools/internal/mixnet-check-all-gateways/src/scratch @@ -0,0 +1,35 @@ + + // let echo = mixnet::MixnetClientBuilder::new_ephemeral() + // .request_gateway(gw) + // .build()?; + // let mut client = echo.connect_to_mixnet().await?; + // let echo_addr = client.nym_address().clone(); + + // tokio::task::spawn(async move { + // loop { + // println!("Waiting for message\n"); + // let mut message: Vec = Vec::new(); + // while let Some(new_message) = client.wait_for_messages().await { + // if new_message.is_empty() { + // continue; + // } + // message = new_message; + // break; + // } + + // let mut parsed = String::new(); + // if let Some(r) = message.first() { + // parsed = String::from_utf8(r.message.clone()).unwrap(); + // } + // let return_recipient: AnonymousSenderTag = message[0].sender_tag.unwrap(); + // println!("\nReceived: {} from {}", parsed, return_recipient); + + // println!("Replying using SURBs"); + // client + // .send_reply(return_recipient, parsed.to_string()) + // .await + // .unwrap(); + + // tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + // } + // }); diff --git a/tools/internal/mixnet-check-all-gateways/src/utils.rs b/tools/internal/mixnet-check-all-gateways/src/utils.rs new file mode 100644 index 00000000000..0735b009db6 --- /dev/null +++ b/tools/internal/mixnet-check-all-gateways/src/utils.rs @@ -0,0 +1,54 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; +use uuid::Uuid; + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ProxiedMessage { + pub message: Payload, + session_id: Uuid, + message_id: u16, +} + +impl ProxiedMessage { + pub fn new(message: Payload, session_id: Uuid, message_id: u16) -> Self { + ProxiedMessage { + message, + session_id, + message_id, + } + } + + pub fn message(&self) -> &Payload { + &self.message + } + + pub fn session_id(&self) -> Uuid { + self.session_id + } + + pub fn message_id(&self) -> u16 { + self.message_id + } +} + +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] +pub enum Payload { + Data(Vec), + Close, +} + +impl fmt::Display for ProxiedMessage { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let message = match self.message() { + Payload::Data(ref data) => format!("Data({})", data.len()), + Payload::Close => "Close".to_string(), + }; + write!( + f, + "ProxiedMessage {{ message: {}, session_id: {}, message_id: {} }}", + message, + self.session_id(), + self.message_id() + ) + } +} From 49e113472296a51472524ba357537ce26c728f30 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 2 Jan 2025 19:13:51 +0100 Subject: [PATCH 084/102] cargo + todo list --- Cargo.lock | 3 +++ tools/echo-server/README.md | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 41d67af8600..af1b62c9c37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4070,7 +4070,9 @@ name = "mixnet-check-all-gateways" version = "0.1.0" dependencies = [ "anyhow", + "bincode", "clap 4.5.20", + "dirs", "echo-server", "futures", "nym-bin-common", @@ -4082,6 +4084,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "uuid", ] [[package]] diff --git a/tools/echo-server/README.md b/tools/echo-server/README.md index 71d9f263c92..ddca88e74a6 100644 --- a/tools/echo-server/README.md +++ b/tools/echo-server/README.md @@ -2,8 +2,8 @@ This is an initial minimal implementation of an echo server built using the `NymProxyServer` Rust SDK abstraction. -## Usage -``` -cargo build --release -../../target/release/echo-server e.g. ../../target/release/echo-server 9000 ../../envs/canary.env -``` +TODO +- [ ] proper readme +- [ ] workspace +- [ ] clippy +- [ ] make utils exportable? From 5e9d25364f8c8c2a09f285678e8cc5d52b31702e Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 2 Jan 2025 21:28:47 +0100 Subject: [PATCH 085/102] cancel token --- Cargo.lock | 1 + .../mixnet-check-all-gateways/.gitignore | 1 + .../mixnet-check-all-gateways/Cargo.toml | 1 + .../mixnet-check-all-gateways/src/main.rs | 117 ++++++++++++------ 4 files changed, 81 insertions(+), 39 deletions(-) create mode 100644 tools/internal/mixnet-check-all-gateways/.gitignore diff --git a/Cargo.lock b/Cargo.lock index af1b62c9c37..34efcf92833 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4082,6 +4082,7 @@ dependencies = [ "serde", "serde_json", "tokio", + "tokio-util", "tracing", "tracing-subscriber", "uuid", diff --git a/tools/internal/mixnet-check-all-gateways/.gitignore b/tools/internal/mixnet-check-all-gateways/.gitignore new file mode 100644 index 00000000000..052bea6f872 --- /dev/null +++ b/tools/internal/mixnet-check-all-gateways/.gitignore @@ -0,0 +1 @@ +src/results/* diff --git a/tools/internal/mixnet-check-all-gateways/Cargo.toml b/tools/internal/mixnet-check-all-gateways/Cargo.toml index 16b4540983d..18bab516f85 100644 --- a/tools/internal/mixnet-check-all-gateways/Cargo.toml +++ b/tools/internal/mixnet-check-all-gateways/Cargo.toml @@ -30,3 +30,4 @@ futures = { workspace = true } uuid = { version = "1", features = ["v4", "serde"] } bincode = "1.0" dirs.workspace = true +tokio-util = { workspace = true } diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index 0ea4bea331d..f963daee79c 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -7,23 +7,25 @@ use futures::stream::StreamExt; use nym_bin_common::logging::setup_logging; use nym_crypto::asymmetric::ed25519; use nym_sdk::mixnet; -use nym_sdk::mixnet::{ - AnonymousSenderTag, IncludedSurbs, MixnetMessageSender, ReconstructedMessage, -}; +use nym_sdk::mixnet::{IncludedSurbs, MixnetMessageSender}; use reqwest::{self, Url}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::fs::OpenOptions; use std::io::Write; use std::time::Duration; +use tokio::signal; +use tokio::time; use tokio::time::timeout; -use tokio::time::{self, Timeout}; #[path = "utils.rs"] // TODO make these exportable from tcp_proxy module and then import from there, ditto with echo server lib mod utils; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info, warn}; use utils::{Payload, ProxiedMessage}; const TIMEOUT: u64 = 10; // message ping timeout +const MESSAGE: &str = "echo test"; #[derive(Serialize, Deserialize, Debug)] struct TestResult { @@ -43,18 +45,18 @@ enum TestError { #[tokio::main] async fn main() -> Result<()> { - // setup_logging(); // TODO think about parsing and noise here, could just parse on errors from all libs and then have info from here? + setup_logging(); // TODO think about parsing and noise here, could just parse on errors from all libs and then have info from here? echo server metrics + info logging from this code + error logs from elsewhere should be ok let entry_gw_keys = reqwest_and_parse( Url::parse("https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/entry-gateways/all?no_legacy=true").unwrap(), "EntryGateway", ) .await?; - // println!( - // "got {} entry gws: \n{:?}", - // entry_gw_keys.len(), - // entry_gw_keys - // ); - println!("got {} entry gws", entry_gw_keys.len(),); + debug!( + "got {} entry gws: \n{:?}", + entry_gw_keys.len(), + entry_gw_keys + ); + info!("got {} entry gws", entry_gw_keys.len(),); let exit_gw_keys = reqwest_and_parse( Url::parse( @@ -64,17 +66,35 @@ async fn main() -> Result<()> { "ExitGateway", ) .await?; - // println!( - // "got {} exit gws: \n{:?}\n", - // exit_gw_keys.len(), - // exit_gw_keys - // ); - println!("got {} exit gws", exit_gw_keys.len(),); + debug!( + "got {} exit gws: \n{:?}\n", + exit_gw_keys.len(), + exit_gw_keys + ); + info!("got {} exit gws", exit_gw_keys.len(),); let mut port_range: u64 = 9000; // port that we start iterating upwards from, will go from port_range to (port_range + exit_gws.len()) by the end of the run + let cancel_token = CancellationToken::new(); + let watcher_token = cancel_token.clone(); + + // Cancel listener thread + tokio::spawn(async move { + signal::ctrl_c().await?; + println!("CTRL_C received"); + watcher_token.cancel(); + Ok::<(), anyhow::Error>(()) + }); + for exit_gw in exit_gw_keys.clone() { - println!("creating echo server connecting to {}", exit_gw); + let loop_token = cancel_token.clone(); + let cancel_loop_token = loop_token.clone(); + let cancel_loop_token_inner = cancel_loop_token.clone(); + if cancel_loop_token_inner.is_cancelled() { + break; + } + + info!("creating echo server connecting to {}", exit_gw); let filepath = format!("./src/results/{}.json", exit_gw.clone()); let mut results = OpenOptions::new() @@ -95,7 +115,7 @@ async fn main() -> Result<()> { ) .as_str(), ), - "../../../envs/mainnet.env".to_string(), // make const + "../../../envs/mainnet.env".to_string(), // TODO make configurable port_range.to_string().as_str(), ) .await @@ -109,7 +129,7 @@ async fn main() -> Result<()> { }; results_vec.push(json!(res)); let json_array = json!(results_vec); - println!("{json_array}"); + info!("{json_array}"); results.write_all(json_array.to_string().as_bytes())?; continue; } @@ -117,10 +137,20 @@ async fn main() -> Result<()> { port_range += 1; let echo_addr = echo_server.nym_address().await; - println!("echo addr: {echo_addr}"); + debug!("echo addr: {echo_addr}"); tokio::task::spawn(async move { - echo_server.run().await?; + // echo_server.run().await?; + loop { + tokio::select! { + _ = loop_token.cancelled() => { + info!("loop over; disconnecting echo server {}", echo_addr.clone()); + echo_server.disconnect().await; + break; + } + _ = echo_server.run() => {} + } + } Ok::<(), anyhow::Error>(()) }); @@ -128,6 +158,10 @@ async fn main() -> Result<()> { time::sleep(Duration::from_secs(5)).await; for entry_gw in entry_gw_keys.clone() { + let cancel_loop_token_inner = cancel_loop_token.clone(); + if cancel_loop_token_inner.is_cancelled() { + break; + } let builder = mixnet::MixnetClientBuilder::new_ephemeral() .request_gateway(entry_gw.clone()) .build()?; @@ -140,23 +174,22 @@ async fn main() -> Result<()> { exit_gw: exit_gw.clone(), error: TestError::Other(err.to_string()), }; - println!("{res:#?}"); + info!("{res:#?}"); results_vec.push(json!(res)); - // println!("failed to connect: {err}"); continue; } }; let test_address = client.nym_address(); - println!("currently testing entry gateway: {test_address}"); + info!("currently testing entry gateway: {test_address}"); // Has to be ProxiedMessage for the moment which is slightly annoying until I // modify the ProxyServer to just stupidly echo back whatever it gets in a - // ReconstructedMessage format if it can't deseralise it to a ProxiedMessage + // ReconstructedMessage format if it can't deseralise incoming traffic to a ProxiedMessage let session_id = uuid::Uuid::new_v4(); let message_id = 0; let outgoing = ProxiedMessage::new( - Payload::Data("echo test".as_bytes().to_vec()), + Payload::Data(MESSAGE.as_bytes().to_vec()), session_id, message_id, ); @@ -167,7 +200,7 @@ async fn main() -> Result<()> { .await { Ok(_) => { - println!("Message sent"); + debug!("Message sent"); } Err(err) => { let res = TestResult { @@ -175,7 +208,7 @@ async fn main() -> Result<()> { exit_gw: exit_gw.clone(), error: TestError::Other(err.to_string()), }; - println!("{res:#?}"); + info!("{res:#?}"); results_vec.push(json!(res)); continue; } @@ -183,7 +216,7 @@ async fn main() -> Result<()> { let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { Err(_timeout) => { - println!("timed out"); + warn!("timed out"); TestResult { entry_gw, exit_gw: exit_gw.clone(), @@ -192,8 +225,13 @@ async fn main() -> Result<()> { } Ok(Some(received)) => { let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); - println!("\ngot echo: {incoming}\n"); - // TODO check incoming is same as outgoing else make a MangledReply err type + debug!("got echo: {:?}", incoming); + info!( + "sent message as lazy ref until I properly sort the utils for comparison: {:?}", + MESSAGE.as_bytes() + ); + info!("incoming message: {:?}", incoming.message); + // TODO check incoming is same as outgoing else make a MangledReply err type + ERR log TestResult { entry_gw, exit_gw: exit_gw.clone(), @@ -201,7 +239,7 @@ async fn main() -> Result<()> { } } Ok(None) => { - println!("failed to receive any message back..."); + info!("failed to receive any message back..."); TestResult { entry_gw, exit_gw: exit_gw.clone(), @@ -209,14 +247,15 @@ async fn main() -> Result<()> { } } }; - println!("{res:#?}"); + info!("{res:#?}"); results_vec.push(json!(res)); - println!("disconnecting the client before shutting down..."); + debug!("disconnecting the client before shutting down..."); client.disconnect().await; } let json_array = json!(results_vec); - println!("{json_array}"); + debug!("{json_array}"); results.write_all(json_array.to_string().as_bytes())?; + cancel_loop_token.cancel(); } Ok(()) } @@ -240,9 +279,9 @@ fn filter_gateway_keys(json: &Value, key: &str) -> Result> { if let Some(role) = node.get("role").and_then(|v| v.as_str()) { let is_correct_gateway = role == key; - // println!("node addr: {:?}", node); - // println!("perf score: {}", performance_value); - // println!("status: {}", inactive); + debug!("node addr: {:?}", node); + debug!("perf score: {}", performance_value); + debug!("status: {}", inactive); if performance_value > 0.0 && !inactive && is_correct_gateway { if let Some(gateway_identity_key) = node.get("ed25519_identity_pubkey").and_then(|v| v.as_str()) From a87753f1eea1cd7f0715e255428b3c30f3a9c659 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 2 Jan 2025 22:00:45 +0100 Subject: [PATCH 086/102] remove scratch --- .../mixnet-check-all-gateways/.gitignore | 1 + .../mixnet-check-all-gateways/src/scratch | 35 ------------------- 2 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 tools/internal/mixnet-check-all-gateways/src/scratch diff --git a/tools/internal/mixnet-check-all-gateways/.gitignore b/tools/internal/mixnet-check-all-gateways/.gitignore index 052bea6f872..3a63d5fc45f 100644 --- a/tools/internal/mixnet-check-all-gateways/.gitignore +++ b/tools/internal/mixnet-check-all-gateways/.gitignore @@ -1 +1,2 @@ src/results/* +scratch diff --git a/tools/internal/mixnet-check-all-gateways/src/scratch b/tools/internal/mixnet-check-all-gateways/src/scratch deleted file mode 100644 index d3ac6b9d01b..00000000000 --- a/tools/internal/mixnet-check-all-gateways/src/scratch +++ /dev/null @@ -1,35 +0,0 @@ - - // let echo = mixnet::MixnetClientBuilder::new_ephemeral() - // .request_gateway(gw) - // .build()?; - // let mut client = echo.connect_to_mixnet().await?; - // let echo_addr = client.nym_address().clone(); - - // tokio::task::spawn(async move { - // loop { - // println!("Waiting for message\n"); - // let mut message: Vec = Vec::new(); - // while let Some(new_message) = client.wait_for_messages().await { - // if new_message.is_empty() { - // continue; - // } - // message = new_message; - // break; - // } - - // let mut parsed = String::new(); - // if let Some(r) = message.first() { - // parsed = String::from_utf8(r.message.clone()).unwrap(); - // } - // let return_recipient: AnonymousSenderTag = message[0].sender_tag.unwrap(); - // println!("\nReceived: {} from {}", parsed, return_recipient); - - // println!("Replying using SURBs"); - // client - // .send_reply(return_recipient, parsed.to_string()) - // .await - // .unwrap(); - - // tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - // } - // }); From 20697fd5fce947f705591d48544b29f70a08917f Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 2 Jan 2025 22:02:59 +0100 Subject: [PATCH 087/102] remove runs --- .../results/2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json | 1 - .../results/4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json | 1 - .../results/6vtDnxeBpaTsy3Xd7BE2ZAEKiaQZFsXEZPf9ZKrQtxgS.json | 0 .../results/98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json | 1 - .../results/Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json | 1 - 5 files changed, 4 deletions(-) delete mode 100644 tools/internal/mixnet-check-all-gateways/src/results/2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json delete mode 100644 tools/internal/mixnet-check-all-gateways/src/results/4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json delete mode 100644 tools/internal/mixnet-check-all-gateways/src/results/6vtDnxeBpaTsy3Xd7BE2ZAEKiaQZFsXEZPf9ZKrQtxgS.json delete mode 100644 tools/internal/mixnet-check-all-gateways/src/results/98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json delete mode 100644 tools/internal/mixnet-check-all-gateways/src/results/Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json diff --git a/tools/internal/mixnet-check-all-gateways/src/results/2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json b/tools/internal/mixnet-check-all-gateways/src/results/2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json deleted file mode 100644 index 154bdcd3bb0..00000000000 --- a/tools/internal/mixnet-check-all-gateways/src/results/2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh.json +++ /dev/null @@ -1 +0,0 @@ -[{"entry_gw":"63ctaex57EvjJZu92jT2ve2ULgmjVYAQph83qNjMpFDZ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"4S4FPLrQ3ZncmGtFXrpgWLtYRn4hcDmb7CTLwk6dkJ7A","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"C7J8SwZQqjWqhBryyjJxLt7FacVuPTwAmR2otGy53ayi","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"EsQEJhuxeNGJXzXA6R51wLCPFKWP322dHnBGXUgmvPJH","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"AkUa1476EZfpBRfWSEWvkXrUp6AGaFwV67ZpSPvDKTQ7","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"EDEtSQ6dVEE6JQqQLZFMdLETCx9NNgqoxkbvoyvfQR48","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"E3Zyogy3C7X1yNpQKwhsaveGbfgFwPpobHZ8DrPEUwyM","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"Ca4bbfxVGtuGuiYvH7V5jSnj9Jp4VJa4Sznu2doZD5np","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"3B7PsbXFuqq6rerYFLw5HPbQb4UmBqAhfWURRovMmWoj","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"H7rDKAXAjJjtkJAgz3paB95zUowH5pHVMBQBdxrx6Yzg","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"4WgKhJdmUffz4e1o1ftVAGS3HnG56LiNAxA9dmaekrVd","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DLxLKsd3LTnfudSSmHanPaZACsh1x4MGEzfJS4jQibir","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DfNMqQRy6pPkU8Z5rBsxRwzDUzAMXHPFwMhjF16ScZqn","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"2zHiExNRKiCXVKS35SNKtK4apGfZELMpA1jJ2gVevJoz","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"9PG6vqoVniK7bWD7esueje9pD3P3iU3Md1T8FAuNQipW","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DgD7K1RHn7kMPC3ibg6HkJFkDqaFstxpEFHgronRdCqj","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"6CTSR2KbwhwHJ2kA3xqA9KLBw1dXcfSyPRKaXBB7Sh1C","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"2hg8pU5fG6WxygEx1pqrwUDkJ8gu6GAe1LysNTfrF4jX","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7mperTohH5oNonkZH7EJkuZaa5WMowdgYY7ah6UFgPxJ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"38zcSsvjXsAX7C28ko2H3Lt55X4TYxfZYkPADxKXZHUj","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7fiZtNL1RACQTwGrKLBT9nbY77bfwZnX9rqcWqc53qgv","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"9xJM74FwwHhEKKJHihD21QSZnHM2QBRMoFx9Wst6qNBS","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"6ftUpAtiNRxPb793eg9vKhHhBJcgm8wRcToF3UbJJyWn","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DiciBkjHovXzTDE2EFJKPNj3TGw2oQjr7HPSas2YQPiQ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"ADjpymCgjFsE5m7YvnZFxLMscg85dCUUesR5g8yN3Mzz","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"J22ULbeW3rXvkHseBS5UuQnZ5c8uQyFQD1eLxm9ULz3H","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"BeT6Hyt1cSvtC5rYZeabZ5E3FWqRXHqUxb62DTLh564K","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"FmbUngD26tUGvJN8QqL78iK96bZYV2bWkjWZU7fDwBN1","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"Hs463Wh5LtWZUNyAmt4trcCbNVsuUhry1wpEXpVnAAfn","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7SnUJy4rWH9hXCitpgwx7XoK5PGRBNjaiz7BWeaqRfXx","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"8ShVhzs9fpKyZPgdtTTRT8yviU2tqk3i5Hnk7rMfLbML","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"HMbxs892i1thjXtPhtz9TGU41ghZCiyC8HhYWkQZs4Cn","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"8R9CjJsfWbAmisHcarJYLrvTdkW1D4CLyy3iAY89j5QF","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"A2dvg4E4SYFqaziJdGPv9hWF7Yh1Po11YSp9p7eEv1xm","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"Fottgyv51kxDKTcVmqP5VfkaXbuTZtCuPWikksTNBRVQ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"Hm7Jn79ztfvXcQjTpUdyx2jqcv4UgBAHcgChhH6b5P4Q","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7WWGub1zjX5KxYHzqiUHsCd12He5hcwfi97GWXLuwnrZ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"HNw9Z6HqwuhUXrqx6CDWMLK2bGGhCaPTSsRRevLWGPXw","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"FfdYC39U4qREoUZSrdNzGLQNug6Ac1PdxkZrUifR4jBR","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"CcYinhLeFU8n6xs78FG6Rz3wvosGTCU2hLB1CZyfkMVe","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"4cdgnckjRBAaaP4ds68YEJPRc78paC4YWMGBDUS8G57C","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"58ceEFaLJh6zAo3cirzT1BDQm7D3L5acnQrxGH1D6TAY","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"2DFyK19kUDGdZYkAUE5pTwot6Hvf7uRhnjs5rKP8PDSk","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"98uf1hyzmWTinkyc5PGyCxDo3E9QQK5XhWQ8B8z8aFoX","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7VJQ4FGRmLESkmjTeTpF6o3VFLVZ5xe9CoJSR38bZawQ","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"AY4uHZFYVxwT6NiEXGLmdp9mxVZpW33ViUUqPgzWcF59","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"DAGQxdxwAkwjaLjTw1B9vndia4YyFD15qRgcTQxrmkom","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"GwPzj3VXALYZJKjbEaveL8CD9wp4eDU4DXxUjMTRxFn6","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"7ntzmDZRvG4a1pnDBU4Bg1RiAmLwmqXV5sZGNw68Ce14","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"},{"entry_gw":"ELjxhHLX1ZxfnJgQhtBTUiVUzLbfZs9Y6bSTpLLkAkR7","error":"None","exit_gw":"2BuMSfMW3zpeAjKXyKLhmY4QW1DXurrtSPEJ6CjX3SEh"}] \ No newline at end of file diff --git a/tools/internal/mixnet-check-all-gateways/src/results/4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json b/tools/internal/mixnet-check-all-gateways/src/results/4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json deleted file mode 100644 index 884d1948bc6..00000000000 --- a/tools/internal/mixnet-check-all-gateways/src/results/4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2.json +++ /dev/null @@ -1 +0,0 @@ -[{"entry_gw":"63ctaex57EvjJZu92jT2ve2ULgmjVYAQph83qNjMpFDZ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"4S4FPLrQ3ZncmGtFXrpgWLtYRn4hcDmb7CTLwk6dkJ7A","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"C7J8SwZQqjWqhBryyjJxLt7FacVuPTwAmR2otGy53ayi","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"EsQEJhuxeNGJXzXA6R51wLCPFKWP322dHnBGXUgmvPJH","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"AkUa1476EZfpBRfWSEWvkXrUp6AGaFwV67ZpSPvDKTQ7","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"EDEtSQ6dVEE6JQqQLZFMdLETCx9NNgqoxkbvoyvfQR48","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"E3Zyogy3C7X1yNpQKwhsaveGbfgFwPpobHZ8DrPEUwyM","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"Ca4bbfxVGtuGuiYvH7V5jSnj9Jp4VJa4Sznu2doZD5np","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"3B7PsbXFuqq6rerYFLw5HPbQb4UmBqAhfWURRovMmWoj","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"H7rDKAXAjJjtkJAgz3paB95zUowH5pHVMBQBdxrx6Yzg","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"4WgKhJdmUffz4e1o1ftVAGS3HnG56LiNAxA9dmaekrVd","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DLxLKsd3LTnfudSSmHanPaZACsh1x4MGEzfJS4jQibir","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DfNMqQRy6pPkU8Z5rBsxRwzDUzAMXHPFwMhjF16ScZqn","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"2zHiExNRKiCXVKS35SNKtK4apGfZELMpA1jJ2gVevJoz","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"9PG6vqoVniK7bWD7esueje9pD3P3iU3Md1T8FAuNQipW","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DgD7K1RHn7kMPC3ibg6HkJFkDqaFstxpEFHgronRdCqj","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"6CTSR2KbwhwHJ2kA3xqA9KLBw1dXcfSyPRKaXBB7Sh1C","error":"Timeout","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"2hg8pU5fG6WxygEx1pqrwUDkJ8gu6GAe1LysNTfrF4jX","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7mperTohH5oNonkZH7EJkuZaa5WMowdgYY7ah6UFgPxJ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"38zcSsvjXsAX7C28ko2H3Lt55X4TYxfZYkPADxKXZHUj","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7fiZtNL1RACQTwGrKLBT9nbY77bfwZnX9rqcWqc53qgv","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"9xJM74FwwHhEKKJHihD21QSZnHM2QBRMoFx9Wst6qNBS","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"6ftUpAtiNRxPb793eg9vKhHhBJcgm8wRcToF3UbJJyWn","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DiciBkjHovXzTDE2EFJKPNj3TGw2oQjr7HPSas2YQPiQ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"ADjpymCgjFsE5m7YvnZFxLMscg85dCUUesR5g8yN3Mzz","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"J22ULbeW3rXvkHseBS5UuQnZ5c8uQyFQD1eLxm9ULz3H","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"BeT6Hyt1cSvtC5rYZeabZ5E3FWqRXHqUxb62DTLh564K","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"FmbUngD26tUGvJN8QqL78iK96bZYV2bWkjWZU7fDwBN1","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"Hs463Wh5LtWZUNyAmt4trcCbNVsuUhry1wpEXpVnAAfn","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7SnUJy4rWH9hXCitpgwx7XoK5PGRBNjaiz7BWeaqRfXx","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"8ShVhzs9fpKyZPgdtTTRT8yviU2tqk3i5Hnk7rMfLbML","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"HMbxs892i1thjXtPhtz9TGU41ghZCiyC8HhYWkQZs4Cn","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"8R9CjJsfWbAmisHcarJYLrvTdkW1D4CLyy3iAY89j5QF","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"A2dvg4E4SYFqaziJdGPv9hWF7Yh1Po11YSp9p7eEv1xm","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"Fottgyv51kxDKTcVmqP5VfkaXbuTZtCuPWikksTNBRVQ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"Hm7Jn79ztfvXcQjTpUdyx2jqcv4UgBAHcgChhH6b5P4Q","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7WWGub1zjX5KxYHzqiUHsCd12He5hcwfi97GWXLuwnrZ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"HNw9Z6HqwuhUXrqx6CDWMLK2bGGhCaPTSsRRevLWGPXw","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"FfdYC39U4qREoUZSrdNzGLQNug6Ac1PdxkZrUifR4jBR","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"CcYinhLeFU8n6xs78FG6Rz3wvosGTCU2hLB1CZyfkMVe","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"4cdgnckjRBAaaP4ds68YEJPRc78paC4YWMGBDUS8G57C","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"58ceEFaLJh6zAo3cirzT1BDQm7D3L5acnQrxGH1D6TAY","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"2DFyK19kUDGdZYkAUE5pTwot6Hvf7uRhnjs5rKP8PDSk","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"98uf1hyzmWTinkyc5PGyCxDo3E9QQK5XhWQ8B8z8aFoX","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7VJQ4FGRmLESkmjTeTpF6o3VFLVZ5xe9CoJSR38bZawQ","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"AY4uHZFYVxwT6NiEXGLmdp9mxVZpW33ViUUqPgzWcF59","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"DAGQxdxwAkwjaLjTw1B9vndia4YyFD15qRgcTQxrmkom","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"GwPzj3VXALYZJKjbEaveL8CD9wp4eDU4DXxUjMTRxFn6","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"7ntzmDZRvG4a1pnDBU4Bg1RiAmLwmqXV5sZGNw68Ce14","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"},{"entry_gw":"ELjxhHLX1ZxfnJgQhtBTUiVUzLbfZs9Y6bSTpLLkAkR7","error":"None","exit_gw":"4h283ohKqPRq7dakfXgVgWYidXoSatvy4fXXJdZpfhA2"}] \ No newline at end of file diff --git a/tools/internal/mixnet-check-all-gateways/src/results/6vtDnxeBpaTsy3Xd7BE2ZAEKiaQZFsXEZPf9ZKrQtxgS.json b/tools/internal/mixnet-check-all-gateways/src/results/6vtDnxeBpaTsy3Xd7BE2ZAEKiaQZFsXEZPf9ZKrQtxgS.json deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/internal/mixnet-check-all-gateways/src/results/98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json b/tools/internal/mixnet-check-all-gateways/src/results/98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json deleted file mode 100644 index 615079aa96f..00000000000 --- a/tools/internal/mixnet-check-all-gateways/src/results/98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF.json +++ /dev/null @@ -1 +0,0 @@ -[{"entry_gw":"63ctaex57EvjJZu92jT2ve2ULgmjVYAQph83qNjMpFDZ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"4S4FPLrQ3ZncmGtFXrpgWLtYRn4hcDmb7CTLwk6dkJ7A","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"C7J8SwZQqjWqhBryyjJxLt7FacVuPTwAmR2otGy53ayi","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"EsQEJhuxeNGJXzXA6R51wLCPFKWP322dHnBGXUgmvPJH","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"AkUa1476EZfpBRfWSEWvkXrUp6AGaFwV67ZpSPvDKTQ7","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"EDEtSQ6dVEE6JQqQLZFMdLETCx9NNgqoxkbvoyvfQR48","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"E3Zyogy3C7X1yNpQKwhsaveGbfgFwPpobHZ8DrPEUwyM","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"Ca4bbfxVGtuGuiYvH7V5jSnj9Jp4VJa4Sznu2doZD5np","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"3B7PsbXFuqq6rerYFLw5HPbQb4UmBqAhfWURRovMmWoj","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"H7rDKAXAjJjtkJAgz3paB95zUowH5pHVMBQBdxrx6Yzg","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"4WgKhJdmUffz4e1o1ftVAGS3HnG56LiNAxA9dmaekrVd","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DLxLKsd3LTnfudSSmHanPaZACsh1x4MGEzfJS4jQibir","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DfNMqQRy6pPkU8Z5rBsxRwzDUzAMXHPFwMhjF16ScZqn","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"2zHiExNRKiCXVKS35SNKtK4apGfZELMpA1jJ2gVevJoz","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"9PG6vqoVniK7bWD7esueje9pD3P3iU3Md1T8FAuNQipW","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DgD7K1RHn7kMPC3ibg6HkJFkDqaFstxpEFHgronRdCqj","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"6CTSR2KbwhwHJ2kA3xqA9KLBw1dXcfSyPRKaXBB7Sh1C","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"2hg8pU5fG6WxygEx1pqrwUDkJ8gu6GAe1LysNTfrF4jX","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7mperTohH5oNonkZH7EJkuZaa5WMowdgYY7ah6UFgPxJ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"38zcSsvjXsAX7C28ko2H3Lt55X4TYxfZYkPADxKXZHUj","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7fiZtNL1RACQTwGrKLBT9nbY77bfwZnX9rqcWqc53qgv","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"9xJM74FwwHhEKKJHihD21QSZnHM2QBRMoFx9Wst6qNBS","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"6ftUpAtiNRxPb793eg9vKhHhBJcgm8wRcToF3UbJJyWn","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DiciBkjHovXzTDE2EFJKPNj3TGw2oQjr7HPSas2YQPiQ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"ADjpymCgjFsE5m7YvnZFxLMscg85dCUUesR5g8yN3Mzz","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"J22ULbeW3rXvkHseBS5UuQnZ5c8uQyFQD1eLxm9ULz3H","error":"Timeout","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"BeT6Hyt1cSvtC5rYZeabZ5E3FWqRXHqUxb62DTLh564K","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"FmbUngD26tUGvJN8QqL78iK96bZYV2bWkjWZU7fDwBN1","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"Hs463Wh5LtWZUNyAmt4trcCbNVsuUhry1wpEXpVnAAfn","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7SnUJy4rWH9hXCitpgwx7XoK5PGRBNjaiz7BWeaqRfXx","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"8ShVhzs9fpKyZPgdtTTRT8yviU2tqk3i5Hnk7rMfLbML","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"HMbxs892i1thjXtPhtz9TGU41ghZCiyC8HhYWkQZs4Cn","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"8R9CjJsfWbAmisHcarJYLrvTdkW1D4CLyy3iAY89j5QF","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"A2dvg4E4SYFqaziJdGPv9hWF7Yh1Po11YSp9p7eEv1xm","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"Fottgyv51kxDKTcVmqP5VfkaXbuTZtCuPWikksTNBRVQ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"Hm7Jn79ztfvXcQjTpUdyx2jqcv4UgBAHcgChhH6b5P4Q","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7WWGub1zjX5KxYHzqiUHsCd12He5hcwfi97GWXLuwnrZ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"HNw9Z6HqwuhUXrqx6CDWMLK2bGGhCaPTSsRRevLWGPXw","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"FfdYC39U4qREoUZSrdNzGLQNug6Ac1PdxkZrUifR4jBR","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"CcYinhLeFU8n6xs78FG6Rz3wvosGTCU2hLB1CZyfkMVe","error":"Timeout","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"4cdgnckjRBAaaP4ds68YEJPRc78paC4YWMGBDUS8G57C","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"58ceEFaLJh6zAo3cirzT1BDQm7D3L5acnQrxGH1D6TAY","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"2DFyK19kUDGdZYkAUE5pTwot6Hvf7uRhnjs5rKP8PDSk","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"98uf1hyzmWTinkyc5PGyCxDo3E9QQK5XhWQ8B8z8aFoX","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7VJQ4FGRmLESkmjTeTpF6o3VFLVZ5xe9CoJSR38bZawQ","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"AY4uHZFYVxwT6NiEXGLmdp9mxVZpW33ViUUqPgzWcF59","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"DAGQxdxwAkwjaLjTw1B9vndia4YyFD15qRgcTQxrmkom","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"GwPzj3VXALYZJKjbEaveL8CD9wp4eDU4DXxUjMTRxFn6","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"7ntzmDZRvG4a1pnDBU4Bg1RiAmLwmqXV5sZGNw68Ce14","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"},{"entry_gw":"ELjxhHLX1ZxfnJgQhtBTUiVUzLbfZs9Y6bSTpLLkAkR7","error":"None","exit_gw":"98FmUvDdQYEeV1ioi5NpFK7DoeHphVECndaG7fkRUsaF"}] \ No newline at end of file diff --git a/tools/internal/mixnet-check-all-gateways/src/results/Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json b/tools/internal/mixnet-check-all-gateways/src/results/Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json deleted file mode 100644 index 67eea3fdaa8..00000000000 --- a/tools/internal/mixnet-check-all-gateways/src/results/Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW.json +++ /dev/null @@ -1 +0,0 @@ -[{"entry_gw":"63ctaex57EvjJZu92jT2ve2ULgmjVYAQph83qNjMpFDZ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"4S4FPLrQ3ZncmGtFXrpgWLtYRn4hcDmb7CTLwk6dkJ7A","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"C7J8SwZQqjWqhBryyjJxLt7FacVuPTwAmR2otGy53ayi","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"EsQEJhuxeNGJXzXA6R51wLCPFKWP322dHnBGXUgmvPJH","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"AkUa1476EZfpBRfWSEWvkXrUp6AGaFwV67ZpSPvDKTQ7","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"EDEtSQ6dVEE6JQqQLZFMdLETCx9NNgqoxkbvoyvfQR48","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"E3Zyogy3C7X1yNpQKwhsaveGbfgFwPpobHZ8DrPEUwyM","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"Ca4bbfxVGtuGuiYvH7V5jSnj9Jp4VJa4Sznu2doZD5np","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"3B7PsbXFuqq6rerYFLw5HPbQb4UmBqAhfWURRovMmWoj","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"H7rDKAXAjJjtkJAgz3paB95zUowH5pHVMBQBdxrx6Yzg","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"4WgKhJdmUffz4e1o1ftVAGS3HnG56LiNAxA9dmaekrVd","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DLxLKsd3LTnfudSSmHanPaZACsh1x4MGEzfJS4jQibir","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DfNMqQRy6pPkU8Z5rBsxRwzDUzAMXHPFwMhjF16ScZqn","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"2zHiExNRKiCXVKS35SNKtK4apGfZELMpA1jJ2gVevJoz","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"9PG6vqoVniK7bWD7esueje9pD3P3iU3Md1T8FAuNQipW","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DgD7K1RHn7kMPC3ibg6HkJFkDqaFstxpEFHgronRdCqj","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"6CTSR2KbwhwHJ2kA3xqA9KLBw1dXcfSyPRKaXBB7Sh1C","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"2hg8pU5fG6WxygEx1pqrwUDkJ8gu6GAe1LysNTfrF4jX","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7mperTohH5oNonkZH7EJkuZaa5WMowdgYY7ah6UFgPxJ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"38zcSsvjXsAX7C28ko2H3Lt55X4TYxfZYkPADxKXZHUj","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7fiZtNL1RACQTwGrKLBT9nbY77bfwZnX9rqcWqc53qgv","error":"Timeout","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"9xJM74FwwHhEKKJHihD21QSZnHM2QBRMoFx9Wst6qNBS","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"6ftUpAtiNRxPb793eg9vKhHhBJcgm8wRcToF3UbJJyWn","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DiciBkjHovXzTDE2EFJKPNj3TGw2oQjr7HPSas2YQPiQ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"ADjpymCgjFsE5m7YvnZFxLMscg85dCUUesR5g8yN3Mzz","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"J22ULbeW3rXvkHseBS5UuQnZ5c8uQyFQD1eLxm9ULz3H","error":"Timeout","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"BeT6Hyt1cSvtC5rYZeabZ5E3FWqRXHqUxb62DTLh564K","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"FmbUngD26tUGvJN8QqL78iK96bZYV2bWkjWZU7fDwBN1","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"Hs463Wh5LtWZUNyAmt4trcCbNVsuUhry1wpEXpVnAAfn","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7SnUJy4rWH9hXCitpgwx7XoK5PGRBNjaiz7BWeaqRfXx","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"8ShVhzs9fpKyZPgdtTTRT8yviU2tqk3i5Hnk7rMfLbML","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"HMbxs892i1thjXtPhtz9TGU41ghZCiyC8HhYWkQZs4Cn","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"8R9CjJsfWbAmisHcarJYLrvTdkW1D4CLyy3iAY89j5QF","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"A2dvg4E4SYFqaziJdGPv9hWF7Yh1Po11YSp9p7eEv1xm","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"Fottgyv51kxDKTcVmqP5VfkaXbuTZtCuPWikksTNBRVQ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"Hm7Jn79ztfvXcQjTpUdyx2jqcv4UgBAHcgChhH6b5P4Q","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7WWGub1zjX5KxYHzqiUHsCd12He5hcwfi97GWXLuwnrZ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"HNw9Z6HqwuhUXrqx6CDWMLK2bGGhCaPTSsRRevLWGPXw","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"FfdYC39U4qREoUZSrdNzGLQNug6Ac1PdxkZrUifR4jBR","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"CcYinhLeFU8n6xs78FG6Rz3wvosGTCU2hLB1CZyfkMVe","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"4cdgnckjRBAaaP4ds68YEJPRc78paC4YWMGBDUS8G57C","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"58ceEFaLJh6zAo3cirzT1BDQm7D3L5acnQrxGH1D6TAY","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"2DFyK19kUDGdZYkAUE5pTwot6Hvf7uRhnjs5rKP8PDSk","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"98uf1hyzmWTinkyc5PGyCxDo3E9QQK5XhWQ8B8z8aFoX","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7VJQ4FGRmLESkmjTeTpF6o3VFLVZ5xe9CoJSR38bZawQ","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"AY4uHZFYVxwT6NiEXGLmdp9mxVZpW33ViUUqPgzWcF59","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"DAGQxdxwAkwjaLjTw1B9vndia4YyFD15qRgcTQxrmkom","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"GwPzj3VXALYZJKjbEaveL8CD9wp4eDU4DXxUjMTRxFn6","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"7ntzmDZRvG4a1pnDBU4Bg1RiAmLwmqXV5sZGNw68Ce14","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"},{"entry_gw":"ELjxhHLX1ZxfnJgQhtBTUiVUzLbfZs9Y6bSTpLLkAkR7","error":"None","exit_gw":"Eb15FTXQgnenwLmqdfCQNj6PmKjMszrmHhtXqKKRafMW"}] \ No newline at end of file From 54cec265e6bf1748a4d29b030b8d0505600e014c Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Thu, 2 Jan 2025 22:21:41 +0100 Subject: [PATCH 088/102] add dir creation + time dir to results --- .../internal/mixnet-check-all-gateways/src/main.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index f963daee79c..750d94bb505 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -11,6 +11,7 @@ use nym_sdk::mixnet::{IncludedSurbs, MixnetMessageSender}; use reqwest::{self, Url}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; +use std::fs; use std::fs::OpenOptions; use std::io::Write; use std::time::Duration; @@ -20,6 +21,7 @@ use tokio::time::timeout; #[path = "utils.rs"] // TODO make these exportable from tcp_proxy module and then import from there, ditto with echo server lib mod utils; +use std::time::{SystemTime, UNIX_EPOCH}; use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; use utils::{Payload, ProxiedMessage}; @@ -86,6 +88,9 @@ async fn main() -> Result<()> { Ok::<(), anyhow::Error>(()) }); + let start = SystemTime::now(); + let time_now = start.duration_since(UNIX_EPOCH).unwrap().as_secs(); + for exit_gw in exit_gw_keys.clone() { let loop_token = cancel_token.clone(); let cancel_loop_token = loop_token.clone(); @@ -94,9 +99,16 @@ async fn main() -> Result<()> { break; } + if !fs::metadata(format!("./src/results/{}", time_now)) + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) + { + fs::create_dir_all(format!("./src/results/{}", time_now))?; + } + info!("creating echo server connecting to {}", exit_gw); - let filepath = format!("./src/results/{}.json", exit_gw.clone()); + let filepath = format!("./src/results/{}/{}.json", time_now, exit_gw.clone()); let mut results = OpenOptions::new() .read(true) .write(true) // .append(true) From 376eca688a32b02e1b1ebded8164c0e19e2456bf Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 3 Jan 2025 00:27:36 +0100 Subject: [PATCH 089/102] cancel token fixes ongoing --- .../mixnet-check-all-gateways/src/main.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index 750d94bb505..2e604fa62b7 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -92,10 +92,10 @@ async fn main() -> Result<()> { let time_now = start.duration_since(UNIX_EPOCH).unwrap().as_secs(); for exit_gw in exit_gw_keys.clone() { - let loop_token = cancel_token.clone(); - let cancel_loop_token = loop_token.clone(); - let cancel_loop_token_inner = cancel_loop_token.clone(); - if cancel_loop_token_inner.is_cancelled() { + let exit_gw_loop_token = cancel_token.clone(); + let echo_server_token = CancellationToken::new(); + let thread_echo_server_token = echo_server_token.clone(); + if exit_gw_loop_token.clone().is_cancelled() { break; } @@ -155,7 +155,7 @@ async fn main() -> Result<()> { // echo_server.run().await?; loop { tokio::select! { - _ = loop_token.cancelled() => { + _ = thread_echo_server_token.cancelled() => { info!("loop over; disconnecting echo server {}", echo_addr.clone()); echo_server.disconnect().await; break; @@ -170,10 +170,6 @@ async fn main() -> Result<()> { time::sleep(Duration::from_secs(5)).await; for entry_gw in entry_gw_keys.clone() { - let cancel_loop_token_inner = cancel_loop_token.clone(); - if cancel_loop_token_inner.is_cancelled() { - break; - } let builder = mixnet::MixnetClientBuilder::new_ephemeral() .request_gateway(entry_gw.clone()) .build()?; @@ -267,7 +263,7 @@ async fn main() -> Result<()> { let json_array = json!(results_vec); debug!("{json_array}"); results.write_all(json_array.to_string().as_bytes())?; - cancel_loop_token.cancel(); + echo_server_token.cancel(); } Ok(()) } From 93f063bb01c17e12f214c7cde069fa4e78ba6a78 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 3 Jan 2025 13:28:02 +0100 Subject: [PATCH 090/102] added test start fn to client_pool: uses specified gw for all clients --- .../src/client_pool/mixnet_client_pool.rs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/sdk/rust/nym-sdk/src/client_pool/mixnet_client_pool.rs b/sdk/rust/nym-sdk/src/client_pool/mixnet_client_pool.rs index 41656fe272d..ff7b8b81981 100644 --- a/sdk/rust/nym-sdk/src/client_pool/mixnet_client_pool.rs +++ b/sdk/rust/nym-sdk/src/client_pool/mixnet_client_pool.rs @@ -1,5 +1,9 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + use crate::mixnet::{MixnetClient, MixnetClientBuilder, NymNetworkDetails}; use anyhow::Result; +use nym_crypto::asymmetric::ed25519; use std::fmt; use std::sync::Arc; use tokio::sync::RwLock; @@ -108,6 +112,49 @@ impl ClientPool { } } + // Even though this is basically start() with an extra param since I think this + // will only be used for testing scenarios, and I didn't want to unnecessarily add + // another param to the function that will be used elsewhere, hence this is its own fn + pub async fn start_with_specified_gateway(&self, gateway: ed25519::PublicKey) -> Result<()> { + loop { + let spawned_clients = self.clients.read().await.len(); + let addresses = self; + debug!( + "Currently spawned clients: {}: {:?}", + spawned_clients, addresses + ); + if self.cancel_token.is_cancelled() { + break Ok(()); + } + if spawned_clients >= self.client_pool_reserve_number { + debug!("Got enough clients already: sleeping"); + } else { + info!( + "Clients in reserve = {}, reserve amount = {}, spawning new client", + spawned_clients, self.client_pool_reserve_number + ); + let client = loop { + let net = NymNetworkDetails::new_from_env(); + match MixnetClientBuilder::new_ephemeral() + .network_details(net) + .request_gateway(gateway.to_string()) + .build()? + .connect_to_mixnet() + .await + { + Ok(client) => break client, + Err(err) => { + warn!("Error creating client: {:?}, will retry in 100ms", err); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + } + }; + self.clients.write().await.push(Arc::new(client)); + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + } + pub async fn disconnect_pool(&self) { info!("Triggering Client Pool disconnect"); self.cancel_token.cancel(); From 8dda5fba5fa4660bf247ad2d0980028757cb4a93 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 3 Jan 2025 13:28:49 +0100 Subject: [PATCH 091/102] comment --- tools/internal/mixnet-check-all-gateways/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index 2e604fa62b7..a308b032ef7 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -151,8 +151,8 @@ async fn main() -> Result<()> { let echo_addr = echo_server.nym_address().await; debug!("echo addr: {echo_addr}"); + // TODO change this to mpsc to do proper cancellation tokio::task::spawn(async move { - // echo_server.run().await?; loop { tokio::select! { _ = thread_echo_server_token.cancelled() => { From 54d7b37f1024fba33bc448986fd90acb4ec7dffa Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Fri, 3 Jan 2025 18:28:28 +0100 Subject: [PATCH 092/102] debugging failing lock quiring --- tools/echo-server/src/lib.rs | 13 +- .../mixnet-check-all-gateways/src/main.rs | 182 +++++++++--------- 2 files changed, 103 insertions(+), 92 deletions(-) diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index 1a706b17f4c..c89131062ce 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-only use anyhow::Result; +use clap::builder::styling::Reset; use nym_crypto::asymmetric::ed25519; use nym_sdk::mixnet::Recipient; use nym_sdk::tcp_proxy; @@ -12,6 +13,7 @@ use tokio::io::AsyncWriteExt; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::Mutex; use tokio::task; +use tokio::time::{timeout, Duration}; use tokio_stream::StreamExt; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -108,7 +110,7 @@ impl NymEchoServer { )); } _ = self.cancel_token.cancelled() => { - info!("Cancel token cancelled: {:?}", self.cancel_token.cancelled()); + info!("token cancelled, stopping handling streams"); break Ok(()); } } @@ -155,8 +157,15 @@ impl NymEchoServer { pub async fn disconnect(&self) { self.cancel_token.cancel(); + info!("token cancelled"); let client = Arc::clone(&self.client); - client.lock().await.disconnect().await; + info!("acquiring lock"); + if let Ok(guard) = timeout(Duration::from_secs(5), client.lock()).await { + guard.disconnect().await; + } else { + error!("Failed to acquire lock to trigger shutdown"); + // TODO somehow force aquire lock and kill + }; } pub async fn nym_address(&self) -> Recipient { diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index a308b032ef7..77762eac893 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -21,6 +21,7 @@ use tokio::time::timeout; #[path = "utils.rs"] // TODO make these exportable from tcp_proxy module and then import from there, ditto with echo server lib mod utils; +use nym_sdk::client_pool::ClientPool; use std::time::{SystemTime, UNIX_EPOCH}; use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; @@ -79,8 +80,6 @@ async fn main() -> Result<()> { let cancel_token = CancellationToken::new(); let watcher_token = cancel_token.clone(); - - // Cancel listener thread tokio::spawn(async move { signal::ctrl_c().await?; println!("CTRL_C received"); @@ -151,13 +150,13 @@ async fn main() -> Result<()> { let echo_addr = echo_server.nym_address().await; debug!("echo addr: {echo_addr}"); - // TODO change this to mpsc to do proper cancellation tokio::task::spawn(async move { loop { tokio::select! { _ = thread_echo_server_token.cancelled() => { info!("loop over; disconnecting echo server {}", echo_addr.clone()); - echo_server.disconnect().await; + echo_server.disconnect().await; // TODO fix this failing lock: can't aquire lock on the ProxyServer to kill it + info!("disconnected {}", echo_addr.clone()); break; } _ = echo_server.run() => {} @@ -170,100 +169,103 @@ async fn main() -> Result<()> { time::sleep(Duration::from_secs(5)).await; for entry_gw in entry_gw_keys.clone() { - let builder = mixnet::MixnetClientBuilder::new_ephemeral() - .request_gateway(entry_gw.clone()) - .build()?; + // let builder = mixnet::MixnetClientBuilder::new_ephemeral() + // .request_gateway(entry_gw.clone()) + // .build()?; - let mut client = match builder.connect_to_mixnet().await { - Ok(client) => client, - Err(err) => { - let res = TestResult { - entry_gw: entry_gw.clone(), - exit_gw: exit_gw.clone(), - error: TestError::Other(err.to_string()), - }; - info!("{res:#?}"); - results_vec.push(json!(res)); - continue; - } - }; + // let mut client = match builder.connect_to_mixnet().await { + // Ok(client) => client, + // Err(err) => { + // let res = TestResult { + // entry_gw: entry_gw.clone(), + // exit_gw: exit_gw.clone(), + // error: TestError::Other(err.to_string()), + // }; + // info!("{res:#?}"); + // results_vec.push(json!(res)); + // continue; + // } + // }; - let test_address = client.nym_address(); - info!("currently testing entry gateway: {test_address}"); + // let test_address = client.nym_address(); + // info!("currently testing entry gateway: {test_address}"); - // Has to be ProxiedMessage for the moment which is slightly annoying until I - // modify the ProxyServer to just stupidly echo back whatever it gets in a - // ReconstructedMessage format if it can't deseralise incoming traffic to a ProxiedMessage - let session_id = uuid::Uuid::new_v4(); - let message_id = 0; - let outgoing = ProxiedMessage::new( - Payload::Data(MESSAGE.as_bytes().to_vec()), - session_id, - message_id, - ); - let coded_message = bincode::serialize(&outgoing).unwrap(); + // // Has to be ProxiedMessage for the moment which is slightly annoying until I + // // modify the ProxyServer to just stupidly echo back whatever it gets in a + // // ReconstructedMessage format if it can't deseralise incoming traffic to a ProxiedMessage + // let session_id = uuid::Uuid::new_v4(); + // let message_id = 0; + // let outgoing = ProxiedMessage::new( + // Payload::Data(MESSAGE.as_bytes().to_vec()), + // session_id, + // message_id, + // ); + // let coded_message = bincode::serialize(&outgoing).unwrap(); - match client - .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(10)) - .await - { - Ok(_) => { - debug!("Message sent"); - } - Err(err) => { - let res = TestResult { - entry_gw: entry_gw.clone(), - exit_gw: exit_gw.clone(), - error: TestError::Other(err.to_string()), - }; - info!("{res:#?}"); - results_vec.push(json!(res)); - continue; - } - }; + // match client + // .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(10)) + // .await + // { + // Ok(_) => { + // debug!("Message sent"); + // } + // Err(err) => { + // let res = TestResult { + // entry_gw: entry_gw.clone(), + // exit_gw: exit_gw.clone(), + // error: TestError::Other(err.to_string()), + // }; + // info!("{res:#?}"); + // results_vec.push(json!(res)); + // continue; + // } + // }; - let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { - Err(_timeout) => { - warn!("timed out"); - TestResult { - entry_gw, - exit_gw: exit_gw.clone(), - error: TestError::Timeout, - } - } - Ok(Some(received)) => { - let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); - debug!("got echo: {:?}", incoming); - info!( - "sent message as lazy ref until I properly sort the utils for comparison: {:?}", - MESSAGE.as_bytes() - ); - info!("incoming message: {:?}", incoming.message); - // TODO check incoming is same as outgoing else make a MangledReply err type + ERR log - TestResult { - entry_gw, - exit_gw: exit_gw.clone(), - error: TestError::None, - } - } - Ok(None) => { - info!("failed to receive any message back..."); - TestResult { - entry_gw, - exit_gw: exit_gw.clone(), - error: TestError::NoMessage, - } - } - }; - info!("{res:#?}"); - results_vec.push(json!(res)); - debug!("disconnecting the client before shutting down..."); - client.disconnect().await; + // let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { + // Err(_timeout) => { + // warn!("timed out"); + // TestResult { + // entry_gw, + // exit_gw: exit_gw.clone(), + // error: TestError::Timeout, + // } + // } + // Ok(Some(received)) => { + // let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); + // debug!("got echo: {:?}", incoming); + // info!( + // "sent message as lazy ref until I properly sort the utils for comparison: {:?}", + // MESSAGE.as_bytes() + // ); + // info!("incoming message: {:?}", incoming.message); + // // TODO check incoming is same as outgoing else make a MangledReply err type + ERR log + // TestResult { + // entry_gw, + // exit_gw: exit_gw.clone(), + // error: TestError::None, + // } + // } + // Ok(None) => { + // info!("failed to receive any message back..."); + // TestResult { + // entry_gw, + // exit_gw: exit_gw.clone(), + // error: TestError::NoMessage, + // } + // } + // }; + // info!("{res:#?}"); + // results_vec.push(json!(res)); + // debug!("disconnecting the client before shutting down..."); + // client.disconnect().await; + + if Some(&entry_gw) == entry_gw_keys.last() { + echo_server_token.cancel(); + } } let json_array = json!(results_vec); debug!("{json_array}"); results.write_all(json_array.to_string().as_bytes())?; - echo_server_token.cancel(); } Ok(()) } @@ -281,7 +283,7 @@ fn filter_gateway_keys(json: &Value, key: &str) -> Result> { if let Some(nodes) = json["nodes"]["data"].as_array() { for node in nodes { if let Some(performance) = node.get("performance").and_then(|v| v.as_str()) { - let performance_value: f64 = performance.parse().unwrap_or(0.0); // TODO could make this a const @ top? + let performance_value: f64 = performance.parse().unwrap_or(0.0); // TODO make this configurable let inactive = node.get("role").and_then(|v| v.as_str()) == Some("Inactive"); From 50191cf80bd0c3b9e56f1df37508516c9c57a4b3 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 6 Jan 2025 13:06:37 +0100 Subject: [PATCH 093/102] re-add commented out code fr sharing --- .../mixnet-check-all-gateways/src/main.rs | 168 +++++++++--------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index 77762eac893..2b2c8c1df96 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -169,95 +169,95 @@ async fn main() -> Result<()> { time::sleep(Duration::from_secs(5)).await; for entry_gw in entry_gw_keys.clone() { - // let builder = mixnet::MixnetClientBuilder::new_ephemeral() - // .request_gateway(entry_gw.clone()) - // .build()?; + let builder = mixnet::MixnetClientBuilder::new_ephemeral() + .request_gateway(entry_gw.clone()) + .build()?; - // let mut client = match builder.connect_to_mixnet().await { - // Ok(client) => client, - // Err(err) => { - // let res = TestResult { - // entry_gw: entry_gw.clone(), - // exit_gw: exit_gw.clone(), - // error: TestError::Other(err.to_string()), - // }; - // info!("{res:#?}"); - // results_vec.push(json!(res)); - // continue; - // } - // }; + let mut client = match builder.connect_to_mixnet().await { + Ok(client) => client, + Err(err) => { + let res = TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::Other(err.to_string()), + }; + info!("{res:#?}"); + results_vec.push(json!(res)); + continue; + } + }; - // let test_address = client.nym_address(); - // info!("currently testing entry gateway: {test_address}"); + let test_address = client.nym_address(); + info!("currently testing entry gateway: {test_address}"); - // // Has to be ProxiedMessage for the moment which is slightly annoying until I - // // modify the ProxyServer to just stupidly echo back whatever it gets in a - // // ReconstructedMessage format if it can't deseralise incoming traffic to a ProxiedMessage - // let session_id = uuid::Uuid::new_v4(); - // let message_id = 0; - // let outgoing = ProxiedMessage::new( - // Payload::Data(MESSAGE.as_bytes().to_vec()), - // session_id, - // message_id, - // ); - // let coded_message = bincode::serialize(&outgoing).unwrap(); + // Has to be ProxiedMessage for the moment which is slightly annoying until I + // modify the ProxyServer to just stupidly echo back whatever it gets in a + // ReconstructedMessage format if it can't deseralise incoming traffic to a ProxiedMessage + let session_id = uuid::Uuid::new_v4(); + let message_id = 0; + let outgoing = ProxiedMessage::new( + Payload::Data(MESSAGE.as_bytes().to_vec()), + session_id, + message_id, + ); + let coded_message = bincode::serialize(&outgoing).unwrap(); - // match client - // .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(10)) - // .await - // { - // Ok(_) => { - // debug!("Message sent"); - // } - // Err(err) => { - // let res = TestResult { - // entry_gw: entry_gw.clone(), - // exit_gw: exit_gw.clone(), - // error: TestError::Other(err.to_string()), - // }; - // info!("{res:#?}"); - // results_vec.push(json!(res)); - // continue; - // } - // }; + match client + .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(10)) + .await + { + Ok(_) => { + debug!("Message sent"); + } + Err(err) => { + let res = TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::Other(err.to_string()), + }; + info!("{res:#?}"); + results_vec.push(json!(res)); + continue; + } + }; - // let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { - // Err(_timeout) => { - // warn!("timed out"); - // TestResult { - // entry_gw, - // exit_gw: exit_gw.clone(), - // error: TestError::Timeout, - // } - // } - // Ok(Some(received)) => { - // let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); - // debug!("got echo: {:?}", incoming); - // info!( - // "sent message as lazy ref until I properly sort the utils for comparison: {:?}", - // MESSAGE.as_bytes() - // ); - // info!("incoming message: {:?}", incoming.message); - // // TODO check incoming is same as outgoing else make a MangledReply err type + ERR log - // TestResult { - // entry_gw, - // exit_gw: exit_gw.clone(), - // error: TestError::None, - // } - // } - // Ok(None) => { - // info!("failed to receive any message back..."); - // TestResult { - // entry_gw, - // exit_gw: exit_gw.clone(), - // error: TestError::NoMessage, - // } - // } - // }; - // info!("{res:#?}"); - // results_vec.push(json!(res)); - // debug!("disconnecting the client before shutting down..."); - // client.disconnect().await; + let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { + Err(_timeout) => { + warn!("timed out"); + TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::Timeout, + } + } + Ok(Some(received)) => { + let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); + debug!("got echo: {:?}", incoming); + info!( + "sent message as lazy ref until I properly sort the utils for comparison: {:?}", + MESSAGE.as_bytes() + ); + info!("incoming message: {:?}", incoming.message); + // TODO check incoming is same as outgoing else make a MangledReply err type + ERR log + TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::None, + } + } + Ok(None) => { + info!("failed to receive any message back..."); + TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::NoMessage, + } + } + }; + info!("{res:#?}"); + results_vec.push(json!(res)); + debug!("disconnecting the client before shutting down..."); + client.disconnect().await; if Some(&entry_gw) == entry_gw_keys.last() { echo_server_token.cancel(); From 3e1ea8e855065d75ccb801c3b6b88b05a5ae9ccf Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 6 Jan 2025 19:41:01 +0100 Subject: [PATCH 094/102] remove comments --- tools/echo-server/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index c89131062ce..42808a6e0ba 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: GPL-3.0-only use anyhow::Result; -use clap::builder::styling::Reset; use nym_crypto::asymmetric::ed25519; use nym_sdk::mixnet::Recipient; use nym_sdk::tcp_proxy; @@ -164,7 +163,6 @@ impl NymEchoServer { guard.disconnect().await; } else { error!("Failed to acquire lock to trigger shutdown"); - // TODO somehow force aquire lock and kill }; } From 378ed8871422dba868e4563225ec0f347e042f60 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 6 Jan 2025 19:41:41 +0100 Subject: [PATCH 095/102] comments --- .../mixnet-check-all-gateways/src/main.rs | 175 +++++++++--------- 1 file changed, 85 insertions(+), 90 deletions(-) diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index 2b2c8c1df96..ec57bfa199d 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -21,7 +21,6 @@ use tokio::time::timeout; #[path = "utils.rs"] // TODO make these exportable from tcp_proxy module and then import from there, ditto with echo server lib mod utils; -use nym_sdk::client_pool::ClientPool; use std::time::{SystemTime, UNIX_EPOCH}; use tokio_util::sync::CancellationToken; use tracing::{debug, info, warn}; @@ -76,13 +75,14 @@ async fn main() -> Result<()> { ); info!("got {} exit gws", exit_gw_keys.len(),); - let mut port_range: u64 = 9000; // port that we start iterating upwards from, will go from port_range to (port_range + exit_gws.len()) by the end of the run + let mut port_range: u64 = 9000; // port that we start iterating upwards from, will go from port_range to (port_range + exit_gws.len()) by the end of the run. This was made configurable presuming at some point we'd try make this run concurrently for speedup. let cancel_token = CancellationToken::new(); let watcher_token = cancel_token.clone(); tokio::spawn(async move { signal::ctrl_c().await?; println!("CTRL_C received"); + // TODO shutdown logic for loops watcher_token.cancel(); Ok::<(), anyhow::Error>(()) }); @@ -91,12 +91,8 @@ async fn main() -> Result<()> { let time_now = start.duration_since(UNIX_EPOCH).unwrap().as_secs(); for exit_gw in exit_gw_keys.clone() { - let exit_gw_loop_token = cancel_token.clone(); let echo_server_token = CancellationToken::new(); let thread_echo_server_token = echo_server_token.clone(); - if exit_gw_loop_token.clone().is_cancelled() { - break; - } if !fs::metadata(format!("./src/results/{}", time_now)) .map(|metadata| metadata.is_dir()) @@ -169,95 +165,94 @@ async fn main() -> Result<()> { time::sleep(Duration::from_secs(5)).await; for entry_gw in entry_gw_keys.clone() { - let builder = mixnet::MixnetClientBuilder::new_ephemeral() - .request_gateway(entry_gw.clone()) - .build()?; + // let builder = mixnet::MixnetClientBuilder::new_ephemeral() + // .request_gateway(entry_gw.clone()) + // .build()?; - let mut client = match builder.connect_to_mixnet().await { - Ok(client) => client, - Err(err) => { - let res = TestResult { - entry_gw: entry_gw.clone(), - exit_gw: exit_gw.clone(), - error: TestError::Other(err.to_string()), - }; - info!("{res:#?}"); - results_vec.push(json!(res)); - continue; - } - }; + // let mut client = match builder.connect_to_mixnet().await { + // Ok(client) => client, + // Err(err) => { + // let res = TestResult { + // entry_gw: entry_gw.clone(), + // exit_gw: exit_gw.clone(), + // error: TestError::Other(err.to_string()), + // }; + // info!("{res:#?}"); + // results_vec.push(json!(res)); + // continue; + // } + // }; - let test_address = client.nym_address(); - info!("currently testing entry gateway: {test_address}"); + // let test_address = client.nym_address(); + // info!("currently testing entry gateway: {test_address}"); - // Has to be ProxiedMessage for the moment which is slightly annoying until I - // modify the ProxyServer to just stupidly echo back whatever it gets in a - // ReconstructedMessage format if it can't deseralise incoming traffic to a ProxiedMessage - let session_id = uuid::Uuid::new_v4(); - let message_id = 0; - let outgoing = ProxiedMessage::new( - Payload::Data(MESSAGE.as_bytes().to_vec()), - session_id, - message_id, - ); - let coded_message = bincode::serialize(&outgoing).unwrap(); + // // Has to be ProxiedMessage for the moment which is slightly annoying until I + // // modify the ProxyServer to just stupidly echo back whatever it gets in a + // // ReconstructedMessage format if it can't deseralise incoming traffic to a ProxiedMessage + // let session_id = uuid::Uuid::new_v4(); + // let message_id = 0; + // let outgoing = ProxiedMessage::new( + // Payload::Data(MESSAGE.as_bytes().to_vec()), + // session_id, + // message_id, + // ); + // let coded_message = bincode::serialize(&outgoing).unwrap(); - match client - .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(10)) - .await - { - Ok(_) => { - debug!("Message sent"); - } - Err(err) => { - let res = TestResult { - entry_gw: entry_gw.clone(), - exit_gw: exit_gw.clone(), - error: TestError::Other(err.to_string()), - }; - info!("{res:#?}"); - results_vec.push(json!(res)); - continue; - } - }; + // match client + // .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(30)) + // .await + // { + // Ok(_) => { + // debug!("Message sent"); + // } + // Err(err) => { + // let res = TestResult { + // entry_gw: entry_gw.clone(), + // exit_gw: exit_gw.clone(), + // error: TestError::Other(err.to_string()), + // }; + // info!("{res:#?}"); + // results_vec.push(json!(res)); + // continue; + // } + // }; - let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { - Err(_timeout) => { - warn!("timed out"); - TestResult { - entry_gw: entry_gw.clone(), - exit_gw: exit_gw.clone(), - error: TestError::Timeout, - } - } - Ok(Some(received)) => { - let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); - debug!("got echo: {:?}", incoming); - info!( - "sent message as lazy ref until I properly sort the utils for comparison: {:?}", - MESSAGE.as_bytes() - ); - info!("incoming message: {:?}", incoming.message); - // TODO check incoming is same as outgoing else make a MangledReply err type + ERR log - TestResult { - entry_gw: entry_gw.clone(), - exit_gw: exit_gw.clone(), - error: TestError::None, - } - } - Ok(None) => { - info!("failed to receive any message back..."); - TestResult { - entry_gw: entry_gw.clone(), - exit_gw: exit_gw.clone(), - error: TestError::NoMessage, - } - } - }; - info!("{res:#?}"); - results_vec.push(json!(res)); - debug!("disconnecting the client before shutting down..."); - client.disconnect().await; + // let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { + // Err(_timeout) => { + // warn!("timed out"); + // TestResult { + // entry_gw: entry_gw.clone(), + // exit_gw: exit_gw.clone(), + // error: TestError::Timeout, + // } + // } + // Ok(Some(received)) => { + // let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); + // info!("got echo: {:?}", incoming); + // debug!( + // "sent message as lazy ref until I properly sort the utils for comparison: {:?}", + // MESSAGE.as_bytes() + // ); + // debug!("incoming message: {:?}", incoming.message); + // TestResult { + // entry_gw: entry_gw.clone(), + // exit_gw: exit_gw.clone(), + // error: TestError::None, + // } + // } + // Ok(None) => { + // info!("failed to receive any message back..."); + // TestResult { + // entry_gw: entry_gw.clone(), + // exit_gw: exit_gw.clone(), + // error: TestError::NoMessage, + // } + // } + // }; + // debug!("{res:#?}"); + // results_vec.push(json!(res)); + // debug!("disconnecting the client before shutting down..."); + // client.disconnect().await; if Some(&entry_gw) == entry_gw_keys.last() { echo_server_token.cancel(); From b73d66e073eaee4d10c8adb1450ea94430a8c45c Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Mon, 6 Jan 2025 19:42:25 +0100 Subject: [PATCH 096/102] rearrange code, comment for rewrite of run() --- .../nym-sdk/src/tcp_proxy/tcp_proxy_server.rs | 105 +++++++++--------- 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs index 8c2a63aab17..11078f0b09f 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs @@ -84,28 +84,43 @@ impl NymProxyServer { }) } - pub fn nym_address(&self) -> &Recipient { - self.mixnet_client.nym_address() - } - - pub fn mixnet_client_mut(&mut self) -> &mut MixnetClient { - &mut self.mixnet_client - } + pub async fn run_with_shutdown(&mut self) -> Result<()> { + // TODO add a select! loop for the cancellation token to stop the main process as well + let loop_cancel_token = self.cancel_token.clone(); - pub fn session_map(&self) -> &DashSet { - &self.session_map - } + // On our Mixnet client getting a new message: + // - Check if the attached sessionID exists. + // - If !sessionID, spawn a new session_handler() task. + // - Send the message down tx => rx in our handler. + while let Some(new_message) = &self.mixnet_client_mut().next().await { + let message: ProxiedMessage = bincode::deserialize(&new_message.message)?; + let session_id = message.session_id(); + // If we've already got message from an existing session, continue, else add it to the session mapping and spawn a new handler(). + if self.session_map().contains(&message.session_id()) { + debug!("Got message for an existing session"); + } else { + self.session_map().insert(message.session_id()); + debug!("Got message for a new session"); + tokio::spawn(Self::session_handler( + self.upstream_address.clone(), + session_id, + self.rx(), + self.mixnet_client_sender(), + self.cancel_token.clone(), + )); + info!("Spawned a new session handler: {}", message.session_id()); + } - pub fn mixnet_client_sender(&self) -> Arc> { - Arc::clone(&self.mixnet_client_sender) - } + debug!("Sending message for session {}", message.session_id()); - pub fn tx(&self) -> tokio::sync::watch::Sender> { - self.tx.clone() - } + if let Some(sender_tag) = new_message.sender_tag { + self.tx.send(Some((message, sender_tag)))? + } else { + error!("No sender tag found, we can't send a reply without it!") + } + } - pub fn rx(&self) -> tokio::sync::watch::Receiver> { - self.rx.clone() + Ok(()) } // The main body of our logic, triggered on each received new sessionID. To deal with assumptions about @@ -210,43 +225,31 @@ impl NymProxyServer { Ok(()) } - pub async fn run_with_shutdown(&mut self) -> Result<()> { - // On our Mixnet client getting a new message: - // - Check if the attached sessionID exists. - // - If !sessionID, spawn a new session_handler() task. - // - Send the message down tx => rx in our handler. - while let Some(new_message) = &self.mixnet_client_mut().next().await { - let message: ProxiedMessage = bincode::deserialize(&new_message.message)?; - let session_id = message.session_id(); - // If we've already got message from an existing session, continue, else add it to the session mapping and spawn a new handler(). - if self.session_map().contains(&message.session_id()) { - debug!("Got message for an existing session"); - } else { - self.session_map().insert(message.session_id()); - debug!("Got message for a new session"); - tokio::spawn(Self::session_handler( - self.upstream_address.clone(), - session_id, - self.rx(), - self.mixnet_client_sender(), - self.cancel_token.clone(), - )); - info!("Spawned a new session handler: {}", message.session_id()); - } + pub async fn disconnect(&self) { + self.cancel_token.cancel(); + } - debug!("Sending message for session {}", message.session_id()); + pub fn nym_address(&self) -> &Recipient { + self.mixnet_client.nym_address() + } - if let Some(sender_tag) = new_message.sender_tag { - self.tx.send(Some((message, sender_tag)))? - } else { - error!("No sender tag found, we can't send a reply without it!") - } - } + pub fn mixnet_client_mut(&mut self) -> &mut MixnetClient { + &mut self.mixnet_client + } - Ok(()) + pub fn session_map(&self) -> &DashSet { + &self.session_map } - pub async fn disconnect(&self) { - self.cancel_token.cancel(); + pub fn mixnet_client_sender(&self) -> Arc> { + Arc::clone(&self.mixnet_client_sender) + } + + pub fn tx(&self) -> tokio::sync::watch::Sender> { + self.tx.clone() + } + + pub fn rx(&self) -> tokio::sync::watch::Receiver> { + self.rx.clone() } } From d86286ef54dfe32e6d9c9b8c12d4cd58d338f451 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Tue, 7 Jan 2025 14:42:51 +0100 Subject: [PATCH 097/102] trying to make run_with_shutdown killable --- Cargo.lock | 2 + sdk/rust/nym-sdk/Cargo.toml | 1 + .../nym-sdk/src/tcp_proxy/tcp_proxy_server.rs | 191 +++++++++++++++--- sdk/rust/nym-sdk/src/tcp_proxy/utils.rs | 4 +- 4 files changed, 167 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 34efcf92833..142239d59ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4081,6 +4081,7 @@ dependencies = [ "reqwest 0.12.4", "serde", "serde_json", + "tempfile", "tokio", "tokio-util", "tracing", @@ -6176,6 +6177,7 @@ dependencies = [ "reqwest 0.12.4", "serde", "tap", + "tempfile", "thiserror", "tokio", "tokio-stream", diff --git a/sdk/rust/nym-sdk/Cargo.toml b/sdk/rust/nym-sdk/Cargo.toml index 0c79a0b7f27..4c089be972e 100644 --- a/sdk/rust/nym-sdk/Cargo.toml +++ b/sdk/rust/nym-sdk/Cargo.toml @@ -59,6 +59,7 @@ serde = { version = "1", features = ["derive"] } tracing.workspace = true tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } dirs.workspace = true +tempfile.workspace = true [dev-dependencies] anyhow = { workspace = true } diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs index 11078f0b09f..bd109e00c7e 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs @@ -1,6 +1,9 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + use crate::mixnet::{ AnonymousSenderTag, MixnetClient, MixnetClientBuilder, MixnetClientSender, MixnetMessageSender, - NymNetworkDetails, StoragePaths, + NymNetworkDetails, ReconstructedMessage, StoragePaths, }; use anyhow::Result; use dashmap::DashSet; @@ -85,38 +88,89 @@ impl NymProxyServer { } pub async fn run_with_shutdown(&mut self) -> Result<()> { - // TODO add a select! loop for the cancellation token to stop the main process as well - let loop_cancel_token = self.cancel_token.clone(); - - // On our Mixnet client getting a new message: - // - Check if the attached sessionID exists. - // - If !sessionID, spawn a new session_handler() task. - // - Send the message down tx => rx in our handler. - while let Some(new_message) = &self.mixnet_client_mut().next().await { - let message: ProxiedMessage = bincode::deserialize(&new_message.message)?; - let session_id = message.session_id(); - // If we've already got message from an existing session, continue, else add it to the session mapping and spawn a new handler(). - if self.session_map().contains(&message.session_id()) { - debug!("Got message for an existing session"); - } else { - self.session_map().insert(message.session_id()); - debug!("Got message for a new session"); - tokio::spawn(Self::session_handler( - self.upstream_address.clone(), - session_id, - self.rx(), - self.mixnet_client_sender(), - self.cancel_token.clone(), - )); - info!("Spawned a new session handler: {}", message.session_id()); + let loop_cancel_token = self.cancel_token.child_token(); + let upstream_address = self.upstream_address.clone(); + let rx = self.rx(); + let mixnet_sender = self.mixnet_client_sender(); + let cancel_token = self.cancel_token.clone(); + let tx = self.tx.clone(); + let session_map = self.session_map().clone(); + + let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(100)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + let mut pending_message: Option = None; // No messages at the beginning of run() obviously + let message_stream = self.mixnet_client_mut(); // Pollable message stream from client + + loop { + tokio::task::yield_now().await; + + if loop_cancel_token.is_cancelled() { + debug!("Received cancellation signal, breaking run() loop"); + break; } - debug!("Sending message for session {}", message.session_id()); + tokio::select! { + biased; // Process in order + + // Process any pending message, 'async poll' hack + () = async { + tokio::task::yield_now().await; + }, if pending_message.is_some() => { + if let Some(new_message) = pending_message.take() { + let message: ProxiedMessage = match bincode::deserialize(&new_message.message) { + Ok(msg) => msg, + Err(e) => { + error!("Failed to deserialize ProxiedMessage: {}", e); + continue; + } + }; + + let session_id = message.session_id(); + + if session_map.insert(session_id) { + debug!("Got message for a new session"); + + tokio::spawn(Self::session_handler( + upstream_address.clone(), + session_id, + rx.clone(), + mixnet_sender.clone(), + cancel_token.clone(), + )); + + info!("Spawned a new session handler: {}", session_id); + } else { + debug!("Got message for an existing session"); + } + + debug!("Sending message for session {}", session_id); + + if let Some(sender_tag) = new_message.sender_tag { + if let Err(e) = tx.send(Some((message, sender_tag))) { + error!("Failed to send ProxiedMessage: {}", e); + } + } else { + error!("No sender tag found, we can't send a reply without it!"); + } + + // Yield after processing each message to allow for lock() to be aquired by whatever process is running server instance + tokio::task::yield_now().await; + } + } + + // Try to get new messages + maybe_message = message_stream.next(), if pending_message.is_none() => { + if let Some(msg) = maybe_message { + pending_message = Some(msg); + } + tokio::task::yield_now().await; + } - if let Some(sender_tag) = new_message.sender_tag { - self.tx.send(Some((message, sender_tag)))? - } else { - error!("No sender tag found, we can't send a reply without it!") + // Also yield on reg tick for same reason + _ = interval.tick() => { + tokio::task::yield_now().await; + } } } @@ -253,3 +307,80 @@ impl NymProxyServer { self.rx.clone() } } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + use tempfile::TempDir; + use tokio::sync::Mutex; + use tokio::time::timeout; + + async fn try_acquire_lock(server: &Arc>) -> bool { + match timeout(Duration::from_millis(500), server.lock()).await { + Ok(guard) => { + debug!("Successfully acquired lock"); + drop(guard); + true + } + Err(_) => { + debug!("Failed to acquire lock"); + false + } + } + } + + #[tokio::test] + async fn test_server_lock_acquisition() -> Result<()> { + let config_dir = TempDir::new()?; + let server = NymProxyServer::new( + "127.0.0.1:8000", + config_dir.path().to_str().unwrap(), + None, + None, + ) + .await?; + + let server = Arc::new(Mutex::new(server)); + let server_for_task = Arc::clone(&server); + + let _handle = tokio::spawn(async move { + let mut server = server_for_task.lock().await; + let _ = server.run_with_shutdown().await; + }); + + // let the server start up properly + tokio::time::sleep(Duration::from_secs(10)).await; + + let max_attempts = 10; + let retry_delay = Duration::from_millis(400); + let mut successful_lock = false; + + for attempt in 1..=max_attempts { + debug!("Lock acquisition attempt {} of {}", attempt, max_attempts); + + if try_acquire_lock(&server).await { + successful_lock = true; + break; + } + + tokio::task::yield_now().await; + tokio::time::sleep(retry_delay).await; + } + + assert!( + successful_lock, + "Failed to acquire lock after {} attempts", + max_attempts + ); + + // Actually try and kill the thing + let server_guard = timeout(Duration::from_secs(5), server.lock()) + .await + .expect("Failed to acquire final lock"); + server_guard.disconnect().await; + drop(server_guard); + + Ok(()) + } +} diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs index 884a63f2abc..2753ef28cf2 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/utils.rs @@ -1,7 +1,9 @@ -use std::{collections::HashSet, fmt, ops::Deref, time::Instant}; +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only use anyhow::Result; use serde::{Deserialize, Serialize}; +use std::{collections::HashSet, fmt, ops::Deref, time::Instant}; use tokio::{io::AsyncWriteExt as _, net::tcp::OwnedWriteHalf}; use tracing::{debug, info}; use uuid::Uuid; From e638f87729fc3aea7cada8322a84ab86602f0e78 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 8 Jan 2025 16:20:03 +0100 Subject: [PATCH 098/102] first pass done --- .../mixnet-check-all-gateways/Cargo.toml | 1 + .../mixnet-check-all-gateways/src/main.rs | 200 +++++++++--------- 2 files changed, 106 insertions(+), 95 deletions(-) diff --git a/tools/internal/mixnet-check-all-gateways/Cargo.toml b/tools/internal/mixnet-check-all-gateways/Cargo.toml index 18bab516f85..7c6ffc5f546 100644 --- a/tools/internal/mixnet-check-all-gateways/Cargo.toml +++ b/tools/internal/mixnet-check-all-gateways/Cargo.toml @@ -31,3 +31,4 @@ uuid = { version = "1", features = ["v4", "serde"] } bincode = "1.0" dirs.workspace = true tokio-util = { workspace = true } +tempfile.workspace = true diff --git a/tools/internal/mixnet-check-all-gateways/src/main.rs b/tools/internal/mixnet-check-all-gateways/src/main.rs index ec57bfa199d..b87090bcbb9 100644 --- a/tools/internal/mixnet-check-all-gateways/src/main.rs +++ b/tools/internal/mixnet-check-all-gateways/src/main.rs @@ -47,7 +47,7 @@ enum TestError { #[tokio::main] async fn main() -> Result<()> { - setup_logging(); // TODO think about parsing and noise here, could just parse on errors from all libs and then have info from here? echo server metrics + info logging from this code + error logs from elsewhere should be ok + setup_logging(); // TODO think about parsing and noise here if make it concurrent. Could just parse on errors from all libs and then have info from here? echo server metrics + info logging from this code + error logs from elsewhere should be ok let entry_gw_keys = reqwest_and_parse( Url::parse("https://validator.nymtech.net/api/v1/unstable/nym-nodes/skimmed/entry-gateways/all?no_legacy=true").unwrap(), "EntryGateway", @@ -75,14 +75,13 @@ async fn main() -> Result<()> { ); info!("got {} exit gws", exit_gw_keys.len(),); - let mut port_range: u64 = 9000; // port that we start iterating upwards from, will go from port_range to (port_range + exit_gws.len()) by the end of the run. This was made configurable presuming at some point we'd try make this run concurrently for speedup. + let mut port_range: u64 = 9000; // Port that we start iterating upwards from, will go from port_range to (port_range + exit_gws.len()) by the end of the run. This was made configurable presuming at some point we'd try make this run concurrently for speedup. let cancel_token = CancellationToken::new(); let watcher_token = cancel_token.clone(); tokio::spawn(async move { signal::ctrl_c().await?; println!("CTRL_C received"); - // TODO shutdown logic for loops watcher_token.cancel(); Ok::<(), anyhow::Error>(()) }); @@ -91,8 +90,13 @@ async fn main() -> Result<()> { let time_now = start.duration_since(UNIX_EPOCH).unwrap().as_secs(); for exit_gw in exit_gw_keys.clone() { - let echo_server_token = CancellationToken::new(); - let thread_echo_server_token = echo_server_token.clone(); + let loop_token = cancel_token.child_token(); + let inner_loop_token = cancel_token.child_token(); + if loop_token.is_cancelled() { + break; + } + let thread_token = cancel_token.child_token(); + let last_check_token = thread_token.clone(); if !fs::metadata(format!("./src/results/{}", time_now)) .map(|metadata| metadata.is_dir()) @@ -122,7 +126,7 @@ async fn main() -> Result<()> { ) .as_str(), ), - "../../../envs/mainnet.env".to_string(), // TODO make configurable + "../../../envs/mainnet.env".to_string(), // TODO replace with None port_range.to_string().as_str(), ) .await @@ -143,16 +147,16 @@ async fn main() -> Result<()> { }; port_range += 1; + let echo_disconnect_signal = echo_server.disconnect_signal(); let echo_addr = echo_server.nym_address().await; debug!("echo addr: {echo_addr}"); tokio::task::spawn(async move { loop { tokio::select! { - _ = thread_echo_server_token.cancelled() => { + _ = thread_token.cancelled() => { info!("loop over; disconnecting echo server {}", echo_addr.clone()); - echo_server.disconnect().await; // TODO fix this failing lock: can't aquire lock on the ProxyServer to kill it - info!("disconnected {}", echo_addr.clone()); + echo_disconnect_signal.send(()).await?; break; } _ = echo_server.run() => {} @@ -165,97 +169,104 @@ async fn main() -> Result<()> { time::sleep(Duration::from_secs(5)).await; for entry_gw in entry_gw_keys.clone() { - // let builder = mixnet::MixnetClientBuilder::new_ephemeral() - // .request_gateway(entry_gw.clone()) - // .build()?; + if inner_loop_token.is_cancelled() { + info!("Inner loop cancelled"); + break; + } + let builder = mixnet::MixnetClientBuilder::new_ephemeral() + .request_gateway(entry_gw.clone()) + .build()?; + + let mut client = match builder.connect_to_mixnet().await { + Ok(client) => client, + Err(err) => { + let res = TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::Other(err.to_string()), + }; + info!("{res:#?}"); + results_vec.push(json!(res)); + continue; + } + }; - // let mut client = match builder.connect_to_mixnet().await { - // Ok(client) => client, - // Err(err) => { - // let res = TestResult { - // entry_gw: entry_gw.clone(), - // exit_gw: exit_gw.clone(), - // error: TestError::Other(err.to_string()), - // }; - // info!("{res:#?}"); - // results_vec.push(json!(res)); - // continue; - // } - // }; + let test_address = client.nym_address(); + info!("currently testing entry gateway: {test_address}"); - // let test_address = client.nym_address(); - // info!("currently testing entry gateway: {test_address}"); + // Has to be ProxiedMessage for the moment which is slightly annoying until I + // modify the ProxyServer to just stupidly echo back whatever it gets in a + // ReconstructedMessage format if it can't deseralise incoming traffic to a ProxiedMessage + let session_id = uuid::Uuid::new_v4(); + let message_id = 0; + let outgoing = ProxiedMessage::new( + Payload::Data(MESSAGE.as_bytes().to_vec()), + session_id, + message_id, + ); + let coded_message = bincode::serialize(&outgoing).unwrap(); - // // Has to be ProxiedMessage for the moment which is slightly annoying until I - // // modify the ProxyServer to just stupidly echo back whatever it gets in a - // // ReconstructedMessage format if it can't deseralise incoming traffic to a ProxiedMessage - // let session_id = uuid::Uuid::new_v4(); - // let message_id = 0; - // let outgoing = ProxiedMessage::new( - // Payload::Data(MESSAGE.as_bytes().to_vec()), - // session_id, - // message_id, - // ); - // let coded_message = bincode::serialize(&outgoing).unwrap(); + match client + .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(30)) + .await + { + Ok(_) => { + debug!("Message sent"); + } + Err(err) => { + let res = TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::Other(err.to_string()), + }; + info!("{res:#?}"); + results_vec.push(json!(res)); + continue; + } + }; - // match client - // .send_message(echo_addr, &coded_message, IncludedSurbs::Amount(30)) - // .await - // { - // Ok(_) => { - // debug!("Message sent"); - // } - // Err(err) => { - // let res = TestResult { - // entry_gw: entry_gw.clone(), - // exit_gw: exit_gw.clone(), - // error: TestError::Other(err.to_string()), - // }; - // info!("{res:#?}"); - // results_vec.push(json!(res)); - // continue; - // } - // }; + let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { + Err(_timeout) => { + warn!("timed out"); + TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::Timeout, + } + } + Ok(Some(received)) => { + let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); + info!("got echo: {:?}", incoming); + debug!( + "sent message as lazy ref until I properly sort the utils for comparison: {:?}", + MESSAGE.as_bytes() + ); + debug!("incoming message: {:?}", incoming.message); + TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::None, + } + } + Ok(None) => { + info!("failed to receive any message back..."); + TestResult { + entry_gw: entry_gw.clone(), + exit_gw: exit_gw.clone(), + error: TestError::NoMessage, + } + } + }; + debug!("{res:#?}"); + results_vec.push(json!(res)); + debug!("disconnecting the client before shutting down..."); + client.disconnect().await; - // let res = match timeout(Duration::from_secs(TIMEOUT), client.next()).await { - // Err(_timeout) => { - // warn!("timed out"); - // TestResult { - // entry_gw: entry_gw.clone(), - // exit_gw: exit_gw.clone(), - // error: TestError::Timeout, - // } - // } - // Ok(Some(received)) => { - // let incoming: ProxiedMessage = bincode::deserialize(&received.message).unwrap(); - // info!("got echo: {:?}", incoming); - // debug!( - // "sent message as lazy ref until I properly sort the utils for comparison: {:?}", - // MESSAGE.as_bytes() - // ); - // debug!("incoming message: {:?}", incoming.message); - // TestResult { - // entry_gw: entry_gw.clone(), - // exit_gw: exit_gw.clone(), - // error: TestError::None, - // } - // } - // Ok(None) => { - // info!("failed to receive any message back..."); - // TestResult { - // entry_gw: entry_gw.clone(), - // exit_gw: exit_gw.clone(), - // error: TestError::NoMessage, - // } - // } - // }; - // debug!("{res:#?}"); - // results_vec.push(json!(res)); - // debug!("disconnecting the client before shutting down..."); - // client.disconnect().await; + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + info!("{}", &entry_gw); if Some(&entry_gw) == entry_gw_keys.last() { - echo_server_token.cancel(); + last_check_token.cancel(); } } let json_array = json!(results_vec); @@ -278,7 +289,7 @@ fn filter_gateway_keys(json: &Value, key: &str) -> Result> { if let Some(nodes) = json["nodes"]["data"].as_array() { for node in nodes { if let Some(performance) = node.get("performance").and_then(|v| v.as_str()) { - let performance_value: f64 = performance.parse().unwrap_or(0.0); // TODO make this configurable + let performance_value: f64 = performance.parse().unwrap_or(0.0); // TODO make this configurable? let inactive = node.get("role").and_then(|v| v.as_str()) == Some("Inactive"); @@ -299,7 +310,6 @@ fn filter_gateway_keys(json: &Value, key: &str) -> Result> { } } else { // TODO make error and return / break - // println!("No nodes found "); return Err(anyhow!("Could not parse any gateways")); } Ok(filtered_keys) From 3f90de1790e6116c46ec80a6413531e1c711b77c Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 8 Jan 2025 16:20:14 +0100 Subject: [PATCH 099/102] update lock --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 142239d59ec..f273ee7fbd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2204,6 +2204,7 @@ dependencies = [ "nym-crypto", "nym-sdk", "serde", + "tempfile", "tokio", "tokio-stream", "tokio-util", From 45617cd218d497a119cfbc3dd92a1fb16542332e Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 8 Jan 2025 16:20:29 +0100 Subject: [PATCH 100/102] proper disconnect signals --- tools/echo-server/Cargo.toml | 1 + tools/echo-server/README.md | 3 +- tools/echo-server/src/lib.rs | 141 +++++++++++++++++---------- tools/echo-server/src/tests/utils.rs | 3 + 4 files changed, 96 insertions(+), 52 deletions(-) diff --git a/tools/echo-server/Cargo.toml b/tools/echo-server/Cargo.toml index 78d20536167..279f4c8fa7a 100644 --- a/tools/echo-server/Cargo.toml +++ b/tools/echo-server/Cargo.toml @@ -36,3 +36,4 @@ nym-bin-common = { path = "../../common/bin-common", features = [ ] } nym-crypto = { path = "../../common/crypto", features = ["asymmetric"] } futures = { workspace = true } +tempfile.workspace = true diff --git a/tools/echo-server/README.md b/tools/echo-server/README.md index ddca88e74a6..e1fb8841367 100644 --- a/tools/echo-server/README.md +++ b/tools/echo-server/README.md @@ -6,4 +6,5 @@ TODO - [ ] proper readme - [ ] workspace - [ ] clippy -- [ ] make utils exportable? +- [ ] make utils exportable + import properly +- [ ] child tokens diff --git a/tools/echo-server/src/lib.rs b/tools/echo-server/src/lib.rs index 42808a6e0ba..b5dbdef5de1 100644 --- a/tools/echo-server/src/lib.rs +++ b/tools/echo-server/src/lib.rs @@ -6,16 +6,18 @@ use nym_crypto::asymmetric::ed25519; use nym_sdk::mixnet::Recipient; use nym_sdk::tcp_proxy; use nym_sdk::tcp_proxy::NymProxyServer; +use std::fmt::Debug; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use tokio::io::AsyncWriteExt; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::Mutex; use tokio::task; -use tokio::time::{timeout, Duration}; use tokio_stream::StreamExt; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, info}; + +const METRICS_TICK: u8 = 6; // Tempo of metrics logging in seconds #[derive(Debug)] pub struct Metrics { @@ -34,19 +36,21 @@ impl Metrics { } } -#[derive(Clone)] pub struct NymEchoServer { client: Arc>, listen_addr: String, metrics: Arc, cancel_token: CancellationToken, + client_shutdown_tx: tokio::sync::mpsc::Sender<()>, // This is the shutdown signal for the TcpProxyServer instance + shutdown_tx: tokio::sync::mpsc::Sender<()>, // These are the shutdown signals for the EchoServer + shutdown_rx: tokio::sync::mpsc::Receiver<()>, } impl NymEchoServer { pub async fn new( gateway: Option, config_path: Option<&str>, - env: String, + env: String, // TODO make Option listen_port: &str, ) -> Result { let home_dir = dirs::home_dir().expect("Unable to get home directory"); @@ -54,25 +58,33 @@ impl NymEchoServer { let config_path = config_path.unwrap_or(&default_path); let listen_addr = format!("127.0.0.1:{}", listen_port); - Ok(NymEchoServer { - client: Arc::new(Mutex::new( - tcp_proxy::NymProxyServer::new( - &listen_addr, - &config_path, - Some(env.clone()), - gateway, - ) + let client = Arc::new(Mutex::new( + tcp_proxy::NymProxyServer::new(&listen_addr, &config_path, Some(env.clone()), gateway) .await?, - )), + )); + + let client_shutdown_tx = client.lock().await.disconnect_signal(); + + let (shutdown_tx, shutdown_rx) = tokio::sync::mpsc::channel(1); + + Ok(NymEchoServer { + client, listen_addr, metrics: Arc::new(Metrics::new()), cancel_token: CancellationToken::new(), + client_shutdown_tx, + shutdown_tx, + shutdown_rx, }) } pub async fn run(&mut self) -> Result<()> { let cancel_token = self.cancel_token.clone(); + let mut interval = + tokio::time::interval(tokio::time::Duration::from_secs(METRICS_TICK as u64)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let client = Arc::clone(&self.client); task::spawn(async move { client.lock().await.run_with_shutdown().await?; @@ -81,23 +93,22 @@ impl NymEchoServer { let all_metrics = Arc::clone(&self.metrics); - tokio::spawn(async move { - loop { - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - info!( - "Metrics: total_connections_since_start={}, bytes_received={}, bytes_sent={}", - all_metrics.total_conn.load(Ordering::Relaxed), - all_metrics.bytes_recv.load(Ordering::Relaxed), - all_metrics.bytes_sent.load(Ordering::Relaxed), - ); - } - }); - let listener = TcpListener::bind(self.listen_addr.clone()).await?; debug!("{listener:?}"); + let mut shutdown_rx = + std::mem::replace(&mut self.shutdown_rx, tokio::sync::mpsc::channel(1).1); + loop { tokio::select! { + Some(()) = shutdown_rx.recv() => { + info!("Disconnect signal received"); + self.cancel_token.cancel(); + info!("Cancel token cancelled: killing handle_incoming loops"); + self.client_shutdown_tx.send(()).await?; + info!("Sent shutdown signal to ProxyServer instance"); + break; + } stream = listener.accept() => { let (stream, _) = stream?; info!("Handling new stream"); @@ -108,12 +119,17 @@ impl NymEchoServer { stream, connection_metrics, cancel_token.clone() )); } - _ = self.cancel_token.cancelled() => { - info!("token cancelled, stopping handling streams"); - break Ok(()); + _ = interval.tick() => { + info!("Metrics: total_connections_since_start={}, bytes_received={}, bytes_sent={}", + all_metrics.total_conn.load(Ordering::Relaxed), + all_metrics.bytes_recv.load(Ordering::Relaxed), + all_metrics.bytes_sent.load(Ordering::Relaxed), + ); } } } + self.shutdown_rx = shutdown_rx; + Ok(()) } async fn handle_incoming( @@ -145,7 +161,7 @@ impl NymEchoServer { } } _ = cancel_token.cancelled() => { - warn!("Shutdown signal received, closing connection"); + info!("Shutdown signal received, closing connection"); break; } } @@ -154,16 +170,8 @@ impl NymEchoServer { info!("Connection closed"); } - pub async fn disconnect(&self) { - self.cancel_token.cancel(); - info!("token cancelled"); - let client = Arc::clone(&self.client); - info!("acquiring lock"); - if let Ok(guard) = timeout(Duration::from_secs(5), client.lock()).await { - guard.disconnect().await; - } else { - error!("Failed to acquire lock to trigger shutdown"); - }; + pub fn disconnect_signal(&self) -> tokio::sync::mpsc::Sender<()> { + self.shutdown_tx.clone() } pub async fn nym_address(&self) -> Recipient { @@ -183,20 +191,54 @@ impl NymEchoServer { mod tests { use super::*; use futures::StreamExt; - use nym_sdk::mixnet::{IncludedSurbs, MixnetClient, MixnetMessageSender, Recipient}; + use nym_sdk::mixnet::{IncludedSurbs, MixnetClient, MixnetMessageSender}; #[path = "utils.rs"] mod utils; + use tempfile::TempDir; use utils::{Payload, ProxiedMessage}; #[tokio::test] - async fn echoes_bytes() { - let mut echo_server = - NymEchoServer::new(None, None, "../../envs/mainnet.env".to_string(), "9000") - .await - .unwrap(); + async fn shutdown_works() -> Result<()> { + let config_dir = TempDir::new()?; + let mut echo_server = NymEchoServer::new( + None, + Some(config_dir.path().to_str().unwrap()), + "../../envs/mainnet.env".to_string(), + "9000", + ) + .await + .unwrap(); + + // Getter for shutdown signal + let shutdown_tx = echo_server.disconnect_signal(); + + let server_handle = tokio::spawn(async move { echo_server.run().await.unwrap() }); + + // Let it start up + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; + + // Kill server + shutdown_tx.send(()).await?; + + // Wait for shutdown in handle + server_handle.await?; + + Ok(()) + } + + #[tokio::test] + async fn echoes_bytes() -> Result<()> { + let config_dir = TempDir::new()?; + let mut echo_server = NymEchoServer::new( + None, + Some(config_dir.path().to_str().unwrap()), + "../../envs/mainnet.env".to_string(), + "9000", + ) + .await + .unwrap(); let echo_addr = echo_server.nym_address().await; - println!("{echo_addr}"); tokio::task::spawn(async move { echo_server.run().await.unwrap(); @@ -230,12 +272,9 @@ mod tests { sending_task_handle.await.unwrap(); receiving_task_handle.await.unwrap(); - } - // #[tokio::test] - // async fn incoming_and_sent_bytes_metrics_work() { - // todo!() - // } + Ok(()) + } // #[tokio::test] // async fn creates_a_valid_nym_addr_with_specified_gw() { diff --git a/tools/echo-server/src/tests/utils.rs b/tools/echo-server/src/tests/utils.rs index 0735b009db6..508a0462fd2 100644 --- a/tools/echo-server/src/tests/utils.rs +++ b/tools/echo-server/src/tests/utils.rs @@ -1,3 +1,6 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + use serde::{Deserialize, Serialize}; use std::fmt; use uuid::Uuid; From 5b536eeaf6184e302c81ea097c54afa98fab58bf Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 8 Jan 2025 16:20:46 +0100 Subject: [PATCH 101/102] proper disconnect signal management --- .../nym-sdk/src/tcp_proxy/tcp_proxy_server.rs | 140 ++++++------------ 1 file changed, 42 insertions(+), 98 deletions(-) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs index bd109e00c7e..06411cc61a0 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_server.rs @@ -3,7 +3,7 @@ use crate::mixnet::{ AnonymousSenderTag, MixnetClient, MixnetClientBuilder, MixnetClientSender, MixnetMessageSender, - NymNetworkDetails, ReconstructedMessage, StoragePaths, + NymNetworkDetails, StoragePaths, }; use anyhow::Result; use dashmap::DashSet; @@ -32,6 +32,8 @@ pub struct NymProxyServer { tx: tokio::sync::watch::Sender>, rx: tokio::sync::watch::Receiver>, cancel_token: CancellationToken, + shutdown_tx: tokio::sync::mpsc::Sender<()>, + shutdown_rx: tokio::sync::mpsc::Receiver<()>, } impl NymProxyServer { @@ -74,6 +76,9 @@ impl NymProxyServer { let (tx, rx) = tokio::sync::watch::channel::>(None); + // Our shutdown signal channel + let (shutdown_tx, shutdown_rx) = tokio::sync::mpsc::channel(1); + info!("Client created: {}", client.nym_address()); Ok(NymProxyServer { @@ -84,40 +89,39 @@ impl NymProxyServer { tx, rx, cancel_token: CancellationToken::new(), + shutdown_tx, + shutdown_rx, }) } pub async fn run_with_shutdown(&mut self) -> Result<()> { - let loop_cancel_token = self.cancel_token.child_token(); + let handle_token = self.cancel_token.child_token(); let upstream_address = self.upstream_address.clone(); let rx = self.rx(); let mixnet_sender = self.mixnet_client_sender(); - let cancel_token = self.cancel_token.clone(); let tx = self.tx.clone(); let session_map = self.session_map().clone(); - let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(100)); - interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut shutdown_rx = + std::mem::replace(&mut self.shutdown_rx, tokio::sync::mpsc::channel(1).1); - let mut pending_message: Option = None; // No messages at the beginning of run() obviously - let message_stream = self.mixnet_client_mut(); // Pollable message stream from client + // Then get the message stream: poll this for incoming messages + let message_stream = self.mixnet_client_mut(); loop { - tokio::task::yield_now().await; - - if loop_cancel_token.is_cancelled() { - debug!("Received cancellation signal, breaking run() loop"); - break; - } - tokio::select! { - biased; // Process in order - - // Process any pending message, 'async poll' hack - () = async { - tokio::task::yield_now().await; - }, if pending_message.is_some() => { - if let Some(new_message) = pending_message.take() { + Some(()) = shutdown_rx.recv() => { + debug!("Received shutdown signal, stopping TcpProxyServer"); + handle_token.cancel(); + break; + } + // On our Mixnet client getting a new message: + // - Try deserialise into ProxiedMessage TODO also add check for ReconstructedMessage message innards if != ProxiedMessage + // - Check if the attached sessionID exists. + // - If !sessionID, spawn a new session_handler() task. + // - Send the message down tx => rx in our handler. + message = message_stream.next() => { + if let Some(new_message) = message { let message: ProxiedMessage = match bincode::deserialize(&new_message.message) { Ok(msg) => msg, Err(e) => { @@ -136,12 +140,10 @@ impl NymProxyServer { session_id, rx.clone(), mixnet_sender.clone(), - cancel_token.clone(), + handle_token.clone() )); info!("Spawned a new session handler: {}", session_id); - } else { - debug!("Got message for an existing session"); } debug!("Sending message for session {}", session_id); @@ -153,27 +155,12 @@ impl NymProxyServer { } else { error!("No sender tag found, we can't send a reply without it!"); } - - // Yield after processing each message to allow for lock() to be aquired by whatever process is running server instance - tokio::task::yield_now().await; - } - } - - // Try to get new messages - maybe_message = message_stream.next(), if pending_message.is_none() => { - if let Some(msg) = maybe_message { - pending_message = Some(msg); } - tokio::task::yield_now().await; - } - - // Also yield on reg tick for same reason - _ = interval.tick() => { - tokio::task::yield_now().await; } } } + self.shutdown_rx = shutdown_rx; Ok(()) } @@ -279,8 +266,8 @@ impl NymProxyServer { Ok(()) } - pub async fn disconnect(&self) { - self.cancel_token.cancel(); + pub fn disconnect_signal(&self) -> tokio::sync::mpsc::Sender<()> { + self.shutdown_tx.clone() } pub fn nym_address(&self) -> &Recipient { @@ -311,75 +298,32 @@ impl NymProxyServer { #[cfg(test)] mod tests { use super::*; - use std::time::Duration; use tempfile::TempDir; - use tokio::sync::Mutex; - use tokio::time::timeout; - - async fn try_acquire_lock(server: &Arc>) -> bool { - match timeout(Duration::from_millis(500), server.lock()).await { - Ok(guard) => { - debug!("Successfully acquired lock"); - drop(guard); - true - } - Err(_) => { - debug!("Failed to acquire lock"); - false - } - } - } #[tokio::test] - async fn test_server_lock_acquisition() -> Result<()> { + async fn shutdown_works() -> Result<()> { let config_dir = TempDir::new()?; - let server = NymProxyServer::new( + let mut server = NymProxyServer::new( "127.0.0.1:8000", config_dir.path().to_str().unwrap(), - None, - None, + None, // Mainnet + None, // Random gateway ) .await?; - let server = Arc::new(Mutex::new(server)); - let server_for_task = Arc::clone(&server); + // Getter for shutdown signal tx + let shutdown_tx = server.disconnect_signal(); - let _handle = tokio::spawn(async move { - let mut server = server_for_task.lock().await; - let _ = server.run_with_shutdown().await; - }); + let server_handle = tokio::spawn(async move { server.run_with_shutdown().await }); - // let the server start up properly - tokio::time::sleep(Duration::from_secs(10)).await; + // Let it start up + tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; - let max_attempts = 10; - let retry_delay = Duration::from_millis(400); - let mut successful_lock = false; - - for attempt in 1..=max_attempts { - debug!("Lock acquisition attempt {} of {}", attempt, max_attempts); - - if try_acquire_lock(&server).await { - successful_lock = true; - break; - } - - tokio::task::yield_now().await; - tokio::time::sleep(retry_delay).await; - } + // Kill server + shutdown_tx.send(()).await?; - assert!( - successful_lock, - "Failed to acquire lock after {} attempts", - max_attempts - ); - - // Actually try and kill the thing - let server_guard = timeout(Duration::from_secs(5), server.lock()) - .await - .expect("Failed to acquire final lock"); - server_guard.disconnect().await; - drop(server_guard); + // Wait for shutdown in handle + check Result != err + server_handle.await??; Ok(()) } From 499b1c7400d3a4a982977ab233bea62d813d2361 Mon Sep 17 00:00:00 2001 From: mfahampshire Date: Wed, 8 Jan 2025 16:20:54 +0100 Subject: [PATCH 102/102] license --- sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs index 977a2a61129..74f1c822667 100644 --- a/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs +++ b/sdk/rust/nym-sdk/src/tcp_proxy/tcp_proxy_client.rs @@ -1,3 +1,6 @@ +// Copyright 2024 - Nym Technologies SA +// SPDX-License-Identifier: GPL-3.0-only + use crate::client_pool::ClientPool; use crate::mixnet::{IncludedSurbs, MixnetClientBuilder, MixnetMessageSender, NymNetworkDetails}; use std::sync::Arc;