From 8c32228ef73d278f6b738cd1ec77b626a1a50cd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Sun, 24 Mar 2024 13:48:42 +0100 Subject: [PATCH 01/21] Make target interval more explicit --- nano/lib/interval.hpp | 8 +------- nano/lib/uniquer.hpp | 4 ++-- nano/node/vote_cache.cpp | 5 ++--- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/nano/lib/interval.hpp b/nano/lib/interval.hpp index 263f15ce70..a4b500fa64 100644 --- a/nano/lib/interval.hpp +++ b/nano/lib/interval.hpp @@ -7,12 +7,7 @@ namespace nano class interval { public: - explicit interval (std::chrono::milliseconds target) : - target{ target } - { - } - - bool elapsed () + bool elapsed (auto target) { auto const now = std::chrono::steady_clock::now (); if (now - last >= target) @@ -24,7 +19,6 @@ class interval } private: - std::chrono::milliseconds const target; std::chrono::steady_clock::time_point last{ std::chrono::steady_clock::now () }; }; } \ No newline at end of file diff --git a/nano/lib/uniquer.hpp b/nano/lib/uniquer.hpp index 351df334dc..022616fa3b 100644 --- a/nano/lib/uniquer.hpp +++ b/nano/lib/uniquer.hpp @@ -27,7 +27,7 @@ class uniquer final nano::lock_guard guard{ mutex }; - if (cleanup_interval.elapsed ()) + if (cleanup_interval.elapsed (cleanup_cutoff)) { cleanup (); } @@ -75,6 +75,6 @@ class uniquer final private: mutable nano::mutex mutex; std::unordered_map> values; - nano::interval cleanup_interval{ cleanup_cutoff }; + nano::interval cleanup_interval; }; } \ No newline at end of file diff --git a/nano/node/vote_cache.cpp b/nano/node/vote_cache.cpp index f3eaec23c0..92a78ff037 100644 --- a/nano/node/vote_cache.cpp +++ b/nano/node/vote_cache.cpp @@ -123,8 +123,7 @@ std::chrono::steady_clock::time_point nano::vote_cache_entry::last_vote () const nano::vote_cache::vote_cache (vote_cache_config const & config_a, nano::stats & stats_a) : config{ config_a }, - stats{ stats_a }, - cleanup_interval{ config_a.age_cutoff / 2 } + stats{ stats_a } { } @@ -239,7 +238,7 @@ std::vector nano::vote_cache::top (const nano::uint { nano::lock_guard lock{ mutex }; - if (cleanup_interval.elapsed ()) + if (cleanup_interval.elapsed (config.age_cutoff / 2)) { cleanup (); } From ba115cb5726c856967731acc7b3d9e58156d3f7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Thu, 21 Mar 2024 22:36:39 +0100 Subject: [PATCH 02/21] Refactor `socket_type` and `socket_endpoint` enums --- nano/core_test/bootstrap.cpp | 54 +++++++++---------- nano/core_test/socket.cpp | 4 +- nano/node/transport/socket.cpp | 19 +++++-- nano/node/transport/socket.hpp | 80 ++++++++++++++++------------ nano/node/transport/tcp.cpp | 6 +-- nano/node/transport/tcp_listener.cpp | 2 +- nano/node/transport/tcp_server.cpp | 14 ++--- 7 files changed, 99 insertions(+), 80 deletions(-) diff --git a/nano/core_test/bootstrap.cpp b/nano/core_test/bootstrap.cpp index 4442b7ede9..22c48327ad 100644 --- a/nano/core_test/bootstrap.cpp +++ b/nano/core_test/bootstrap.cpp @@ -16,7 +16,7 @@ using namespace std::chrono_literals; TEST (bulk_pull, no_address) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start = 1; req->end = 2; @@ -28,7 +28,7 @@ TEST (bulk_pull, no_address) TEST (bulk_pull, genesis_to_end) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis_key.pub; req->end.clear (); @@ -41,7 +41,7 @@ TEST (bulk_pull, genesis_to_end) TEST (bulk_pull, no_end) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis_key.pub; req->end = 1; @@ -73,7 +73,7 @@ TEST (bulk_pull, end_not_owned) open->signature = nano::sign_message (key2.prv, key2.pub, open->hash ()); system.nodes[0]->work_generate_blocking (*open); ASSERT_EQ (nano::block_status::progress, system.nodes[0]->process (open)); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start = key2.pub; req->end = nano::dev::genesis->hash (); @@ -84,7 +84,7 @@ TEST (bulk_pull, end_not_owned) TEST (bulk_pull, none) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis_key.pub; req->end = nano::dev::genesis->hash (); @@ -96,7 +96,7 @@ TEST (bulk_pull, none) TEST (bulk_pull, get_next_on_open) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis_key.pub; req->end.clear (); @@ -126,7 +126,7 @@ TEST (bulk_pull, ascending_one_hash) .build (); node.work_generate_blocking (*block1); ASSERT_EQ (nano::block_status::progress, node.process (block1)); - auto socket = std::make_shared (node, nano::transport::socket::endpoint_type_t::server); + auto socket = std::make_shared (node, nano::transport::socket_endpoint::server); auto connection = std::make_shared (socket, system.nodes[0]); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis->hash (); @@ -158,7 +158,7 @@ TEST (bulk_pull, ascending_two_account) .build (); node.work_generate_blocking (*block1); ASSERT_EQ (nano::block_status::progress, node.process (block1)); - auto socket = std::make_shared (node, nano::transport::socket::endpoint_type_t::server); + auto socket = std::make_shared (node, nano::transport::socket_endpoint::server); auto connection = std::make_shared (socket, system.nodes[0]); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis_key.pub; @@ -193,7 +193,7 @@ TEST (bulk_pull, ascending_end) .build (); node.work_generate_blocking (*block1); ASSERT_EQ (nano::block_status::progress, node.process (block1)); - auto socket = std::make_shared (node, nano::transport::socket::endpoint_type_t::server); + auto socket = std::make_shared (node, nano::transport::socket_endpoint::server); auto connection = std::make_shared (socket, system.nodes[0]); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis_key.pub; @@ -209,7 +209,7 @@ TEST (bulk_pull, ascending_end) TEST (bulk_pull, by_block) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis->hash (); req->end.clear (); @@ -225,7 +225,7 @@ TEST (bulk_pull, by_block) TEST (bulk_pull, by_block_single) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis->hash (); req->end = nano::dev::genesis->hash (); @@ -262,7 +262,7 @@ TEST (bulk_pull, count_limit) .build (); ASSERT_EQ (nano::block_status::progress, node0->process (receive1)); - auto connection (std::make_shared (std::make_shared (*node0, nano::transport::socket::endpoint_type_t::server), node0)); + auto connection (std::make_shared (std::make_shared (*node0, nano::transport::socket_endpoint::server), node0)); auto req = std::make_unique (nano::dev::network_params.network); req->start = receive1->hash (); req->set_count_present (true); @@ -1686,7 +1686,7 @@ TEST (frontier_req_response, DISABLED_destruction) TEST (frontier_req, begin) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start.clear (); req->age = std::numeric_limitsage)>::max (); @@ -1699,7 +1699,7 @@ TEST (frontier_req, begin) TEST (frontier_req, end) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start = nano::dev::genesis_key.pub.number () + 1; req->age = std::numeric_limitsage)>::max (); @@ -1740,7 +1740,7 @@ TEST (frontier_req, count) node1->work_generate_blocking (*receive1); ASSERT_EQ (nano::block_status::progress, node1->process (receive1)); - auto connection (std::make_shared (std::make_shared (*node1, nano::transport::socket::endpoint_type_t::server), node1)); + auto connection (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); auto req = std::make_unique (nano::dev::network_params.network); req->start.clear (); req->age = std::numeric_limitsage)>::max (); @@ -1753,7 +1753,7 @@ TEST (frontier_req, count) TEST (frontier_req, time_bound) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start.clear (); req->age = 1; @@ -1766,7 +1766,7 @@ TEST (frontier_req, time_bound) req2->start.clear (); req2->age = 1; req2->count = std::numeric_limitscount)>::max (); - auto connection2 (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection2 (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto request2 (std::make_shared (connection, std::move (req2))); ASSERT_TRUE (request2->current.is_zero ()); } @@ -1774,7 +1774,7 @@ TEST (frontier_req, time_bound) TEST (frontier_req, time_cutoff) { nano::test::system system (1); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto req = std::make_unique (nano::dev::network_params.network); req->start.clear (); req->age = 3; @@ -1788,7 +1788,7 @@ TEST (frontier_req, time_cutoff) req2->start.clear (); req2->age = 3; req2->count = std::numeric_limitscount)>::max (); - auto connection2 (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection2 (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); auto request2 (std::make_shared (connection, std::move (req2))); ASSERT_TRUE (request2->frontier.is_zero ()); } @@ -1860,7 +1860,7 @@ TEST (frontier_req, confirmed_frontier) ASSERT_EQ (nano::block_status::progress, node1->process (receive2)); // Request for all accounts (confirmed only) - auto connection (std::make_shared (std::make_shared (*node1, nano::transport::socket::endpoint_type_t::server), node1)); + auto connection (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); auto req = std::make_unique (nano::dev::network_params.network); req->start.clear (); req->age = std::numeric_limitsage)>::max (); @@ -1873,7 +1873,7 @@ TEST (frontier_req, confirmed_frontier) ASSERT_EQ (nano::dev::genesis->hash (), request->frontier); // Request starting with account before genesis (confirmed only) - auto connection2 (std::make_shared (std::make_shared (*node1, nano::transport::socket::endpoint_type_t::server), node1)); + auto connection2 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); auto req2 = std::make_unique (nano::dev::network_params.network); req2->start = key_before_genesis.pub; req2->age = std::numeric_limitsage)>::max (); @@ -1886,7 +1886,7 @@ TEST (frontier_req, confirmed_frontier) ASSERT_EQ (nano::dev::genesis->hash (), request2->frontier); // Request starting with account after genesis (confirmed only) - auto connection3 (std::make_shared (std::make_shared (*node1, nano::transport::socket::endpoint_type_t::server), node1)); + auto connection3 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); auto req3 = std::make_unique (nano::dev::network_params.network); req3->start = key_after_genesis.pub; req3->age = std::numeric_limitsage)>::max (); @@ -1899,7 +1899,7 @@ TEST (frontier_req, confirmed_frontier) ASSERT_TRUE (request3->frontier.is_zero ()); // Request for all accounts (unconfirmed blocks) - auto connection4 (std::make_shared (std::make_shared (*node1, nano::transport::socket::endpoint_type_t::server), node1)); + auto connection4 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); auto req4 = std::make_unique (nano::dev::network_params.network); req4->start.clear (); req4->age = std::numeric_limitsage)>::max (); @@ -1910,7 +1910,7 @@ TEST (frontier_req, confirmed_frontier) ASSERT_EQ (receive1->hash (), request4->frontier); // Request starting with account after genesis (unconfirmed blocks) - auto connection5 (std::make_shared (std::make_shared (*node1, nano::transport::socket::endpoint_type_t::server), node1)); + auto connection5 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); auto req5 = std::make_unique (nano::dev::network_params.network); req5->start = key_after_genesis.pub; req5->age = std::numeric_limitsage)>::max (); @@ -1923,7 +1923,7 @@ TEST (frontier_req, confirmed_frontier) // Confirm account before genesis (confirmed only) ASSERT_TRUE (nano::test::start_elections (system, *node1, { send1, receive1 }, true)); ASSERT_TIMELY (5s, node1->block_confirmed (send1->hash ()) && node1->block_confirmed (receive1->hash ())); - auto connection6 (std::make_shared (std::make_shared (*node1, nano::transport::socket::endpoint_type_t::server), node1)); + auto connection6 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); auto req6 = std::make_unique (nano::dev::network_params.network); req6->start = key_before_genesis.pub; req6->age = std::numeric_limitsage)>::max (); @@ -1938,7 +1938,7 @@ TEST (frontier_req, confirmed_frontier) // Confirm account after genesis (confirmed only) ASSERT_TRUE (nano::test::start_elections (system, *node1, { send2, receive2 }, true)); ASSERT_TIMELY (5s, node1->block_confirmed (send2->hash ()) && node1->block_confirmed (receive2->hash ())); - auto connection7 (std::make_shared (std::make_shared (*node1, nano::transport::socket::endpoint_type_t::server), node1)); + auto connection7 (std::make_shared (std::make_shared (*node1, nano::transport::socket_endpoint::server), node1)); auto req7 = std::make_unique (nano::dev::network_params.network); req7->start = key_after_genesis.pub; req7->age = std::numeric_limitsage)>::max (); @@ -2105,7 +2105,7 @@ TEST (bulk_pull_account, basics) auto send2 (system.wallet (0)->send_action (nano::dev::genesis_key.pub, key1.pub, 10)); auto send3 (system.wallet (0)->send_action (nano::dev::genesis_key.pub, key1.pub, 2)); ASSERT_TIMELY_EQ (5s, system.nodes[0]->balance (key1.pub), 25); - auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket::endpoint_type_t::server), system.nodes[0])); + auto connection (std::make_shared (std::make_shared (*system.nodes[0], nano::transport::socket_endpoint::server), system.nodes[0])); { auto req = std::make_unique (nano::dev::network_params.network); diff --git a/nano/core_test/socket.cpp b/nano/core_test/socket.cpp index 85d8211791..f8d58992e5 100644 --- a/nano/core_test/socket.cpp +++ b/nano/core_test/socket.cpp @@ -698,7 +698,7 @@ TEST (socket_timeout, write) // create a client socket and send lots of data to fill the socket queue on the local and remote side // eventually, the all tcp queues should fill up and async_write will not be able to progress // and the timeout should kick in and close the socket, which will cause the async_write to return an error - auto socket = std::make_shared (*node, nano::transport::socket::endpoint_type_t::client, 1024 * 64); // socket with a max queue size much larger than OS buffers + auto socket = std::make_shared (*node, nano::transport::socket_endpoint::client, 1024 * 64); // socket with a max queue size much larger than OS buffers std::atomic done = false; boost::system::error_code ec; socket->async_connect (acceptor.local_endpoint (), [&socket, &ec, &done] (boost::system::error_code const & ec_a) { @@ -812,7 +812,7 @@ TEST (socket_timeout, write_overlapped) // create a client socket and send lots of data to fill the socket queue on the local and remote side // eventually, the all tcp queues should fill up and async_write will not be able to progress // and the timeout should kick in and close the socket, which will cause the async_write to return an error - auto socket = std::make_shared (*node, nano::transport::socket::endpoint_type_t::client, 1024 * 64); // socket with a max queue size much larger than OS buffers + auto socket = std::make_shared (*node, nano::transport::socket_endpoint::client, 1024 * 64); // socket with a max queue size much larger than OS buffers std::atomic done = false; boost::system::error_code ec; socket->async_connect (acceptor.local_endpoint (), [&socket, &ec, &done] (boost::system::error_code const & ec_a) { diff --git a/nano/node/transport/socket.cpp b/nano/node/transport/socket.cpp index abf3bc0454..11f5405acf 100644 --- a/nano/node/transport/socket.cpp +++ b/nano/node/transport/socket.cpp @@ -19,7 +19,7 @@ * socket */ -nano::transport::socket::socket (nano::node & node_a, endpoint_type_t endpoint_type_a, std::size_t max_queue_size_a) : +nano::transport::socket::socket (nano::node & node_a, nano::transport::socket_endpoint endpoint_type_a, std::size_t max_queue_size_a) : send_queue{ max_queue_size_a }, strand{ node_a.io_ctx.get_executor () }, tcp_socket{ node_a.io_ctx }, @@ -47,7 +47,7 @@ void nano::transport::socket::start () void nano::transport::socket::async_connect (nano::tcp_endpoint const & endpoint_a, std::function callback_a) { debug_assert (callback_a); - debug_assert (endpoint_type () == endpoint_type_t::client); + debug_assert (endpoint_type () == socket_endpoint::client); start (); set_default_timeout (); @@ -248,7 +248,7 @@ void nano::transport::socket::ongoing_checkup () auto condition_to_disconnect{ false }; // if this is a server socket, and no data is received for silent_connection_tolerance_time seconds then disconnect - if (this_l->endpoint_type () == endpoint_type_t::server && (now - this_l->last_receive_time_or_init) > static_cast (this_l->silent_connection_tolerance_time.count ())) + if (this_l->endpoint_type () == socket_endpoint::server && (now - this_l->last_receive_time_or_init) > static_cast (this_l->silent_connection_tolerance_time.count ())) { this_l->node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_silent_connection_drop, nano::stat::dir::in); @@ -258,7 +258,7 @@ void nano::transport::socket::ongoing_checkup () // if there is no activity for timeout seconds then disconnect if ((now - this_l->last_completion_time_or_init) > this_l->timeout) { - this_l->node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_io_timeout_drop, this_l->endpoint_type () == endpoint_type_t::server ? nano::stat::dir::in : nano::stat::dir::out); + this_l->node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_io_timeout_drop, this_l->endpoint_type () == socket_endpoint::server ? nano::stat::dir::in : nano::stat::dir::out); condition_to_disconnect = true; } @@ -462,7 +462,16 @@ std::size_t network_prefix) return counted_connections; } -std::string_view nano::transport::to_string (nano::transport::socket::type_t type) +/* + * + */ + +std::string_view nano::transport::to_string (socket_type type) +{ + return magic_enum::enum_name (type); +} + +std::string_view nano::transport::to_string (socket_endpoint type) { return magic_enum::enum_name (type); } diff --git a/nano/node/transport/socket.hpp b/nano/node/transport/socket.hpp index b0b1096072..73c3d04460 100644 --- a/nano/node/transport/socket.hpp +++ b/nano/node/transport/socket.hpp @@ -39,8 +39,26 @@ enum class buffer_drop_policy no_socket_drop }; +enum class socket_type +{ + undefined, + bootstrap, + realtime, + realtime_response_server // special type for tcp channel response server +}; + +std::string_view to_string (socket_type); + +enum class socket_endpoint +{ + server, // Socket was created by accepting an incoming connection + client, // Socket was created by initiating an outgoing connection +}; + +std::string_view to_string (socket_endpoint); + /** Socket class for tcp clients and newly accepted connections */ -class socket final : public std::enable_shared_from_this +class socket final : public std::enable_shared_from_this { friend class tcp_server; friend class tcp_channels; @@ -49,37 +67,31 @@ class socket final : public std::enable_shared_from_this); - void async_read (std::shared_ptr> const &, std::size_t, std::function); - void async_write (nano::shared_const_buffer const &, std::function callback = {}, nano::transport::traffic_type = nano::transport::traffic_type::generic); + void async_connect ( + boost::asio::ip::tcp::endpoint const & endpoint, + std::function callback); + + void async_read ( + std::shared_ptr> const & buffer, + std::size_t size, + std::function callback); + + void async_write ( + nano::shared_const_buffer const &, + std::function callback = {}, + traffic_type = traffic_type::generic); void close (); + boost::asio::ip::tcp::endpoint remote_endpoint () const; boost::asio::ip::tcp::endpoint local_endpoint () const; + /** Returns true if the socket has timed out */ bool has_timed_out () const; /** This can be called to change the maximum idle time, e.g. based on the type of traffic detected. */ @@ -87,28 +99,28 @@ class socket final : public std::enable_shared_from_this> const & data_a, std::size_t size_a, std::function callback_a); private: - type_t type_m{ type_t::undefined }; - endpoint_type_t endpoint_type_m; + nano::transport::socket_type type_m{ socket_type::undefined }; + nano::transport::socket_endpoint endpoint_type_m; public: std::size_t const max_queue_size; @@ -211,8 +223,6 @@ class socket final : public std::enable_shared_from_this>; namespace socket_functions diff --git a/nano/node/transport/tcp.cpp b/nano/node/transport/tcp.cpp index 1313b77d8b..05ac10d0f1 100644 --- a/nano/node/transport/tcp.cpp +++ b/nano/node/transport/tcp.cpp @@ -344,9 +344,9 @@ void nano::transport::tcp_channels::process_message (nano::message const & messa temporary_channel->set_node_id (node_id_a); temporary_channel->set_network_version (message_a.header.version_using); temporary_channel->temporary = true; - debug_assert (type_a == nano::transport::socket::type_t::realtime || type_a == nano::transport::socket::type_t::realtime_response_server); + debug_assert (type_a == nano::transport::socket_type::realtime || type_a == nano::transport::socket_type::realtime_response_server); // Don't insert temporary channels for response_server - if (type_a == nano::transport::socket::type_t::realtime) + if (type_a == nano::transport::socket_type::realtime) { insert (temporary_channel, socket_a, nullptr); } @@ -767,7 +767,7 @@ void nano::transport::tcp_channels::start_tcp_receive_node_id (std::shared_ptr (socket_l, node_l); node_l->network.tcp_channels.insert (channel_a, socket_l, response_server); // Listen for possible responses - response_server->socket->type_set (nano::transport::socket::type_t::realtime_response_server); + response_server->socket->type_set (nano::transport::socket_type::realtime_response_server); response_server->remote_node_id = channel_a->get_node_id (); response_server->start (); }); diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index 07f3fc8d6e..47dac191a5 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -143,7 +143,7 @@ void nano::transport::tcp_listener::on_connection (std::function (this_l->node, socket::endpoint_type_t::server); + auto new_connection = std::make_shared (this_l->node, socket_endpoint::server); this_l->acceptor.async_accept (new_connection->tcp_socket, new_connection->remote, boost::asio::bind_executor (this_l->strand, [this_l, new_connection, cbk = std::move (callback)] (boost::system::error_code const & ec_a) mutable { diff --git a/nano/node/transport/tcp_server.cpp b/nano/node/transport/tcp_server.cpp index e4dc93a4d1..070a2882fb 100644 --- a/nano/node/transport/tcp_server.cpp +++ b/nano/node/transport/tcp_server.cpp @@ -38,11 +38,11 @@ nano::transport::tcp_server::~tcp_server () node->logger.debug (nano::log::type::tcp_server, "Exiting TCP server ({})", fmt::streamed (remote_endpoint)); - if (socket->type () == nano::transport::socket::type_t::bootstrap) + if (socket->type () == nano::transport::socket_type::bootstrap) { --node->tcp_listener->bootstrap_count; } - else if (socket->type () == nano::transport::socket::type_t::realtime) + else if (socket->type () == nano::transport::socket_type::realtime) { --node->tcp_listener->realtime_count; @@ -612,13 +612,13 @@ bool nano::transport::tcp_server::to_bootstrap_connection () { return false; } - if (socket->type () != nano::transport::socket::type_t::undefined) + if (socket->type () != nano::transport::socket_type::undefined) { return false; } ++node->tcp_listener->bootstrap_count; - socket->type_set (nano::transport::socket::type_t::bootstrap); + socket->type_set (nano::transport::socket_type::bootstrap); node->logger.debug (nano::log::type::tcp_server, "Switched to bootstrap mode ({})", fmt::streamed (remote_endpoint)); @@ -636,14 +636,14 @@ bool nano::transport::tcp_server::to_realtime_connection (nano::account const & { return false; } - if (socket->type () != nano::transport::socket::type_t::undefined) + if (socket->type () != nano::transport::socket_type::undefined) { return false; } remote_node_id = node_id; ++node->tcp_listener->realtime_count; - socket->type_set (nano::transport::socket::type_t::realtime); + socket->type_set (nano::transport::socket_type::realtime); node->logger.debug (nano::log::type::tcp_server, "Switched to realtime mode ({})", fmt::streamed (remote_endpoint)); @@ -652,7 +652,7 @@ bool nano::transport::tcp_server::to_realtime_connection (nano::account const & bool nano::transport::tcp_server::is_undefined_connection () const { - return socket->type () == nano::transport::socket::type_t::undefined; + return socket->type () == nano::transport::socket_type::undefined; } bool nano::transport::tcp_server::is_bootstrap_connection () const From a4a96ed8960da720526641403e5da5ae65cad813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 22 Mar 2024 11:29:20 +0100 Subject: [PATCH 03/21] Add `peer_exclusion::check` helper overload --- nano/node/peer_exclusion.cpp | 7 ++++++- nano/node/peer_exclusion.hpp | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/nano/node/peer_exclusion.cpp b/nano/node/peer_exclusion.cpp index 547ff65e4c..f890124fd3 100644 --- a/nano/node/peer_exclusion.cpp +++ b/nano/node/peer_exclusion.cpp @@ -67,10 +67,15 @@ std::chrono::steady_clock::time_point nano::peer_exclusion::until (const nano::t } bool nano::peer_exclusion::check (nano::tcp_endpoint const & endpoint) const +{ + return check (endpoint.address ()); +} + +bool nano::peer_exclusion::check (boost::asio::ip::address const & address) const { nano::lock_guard guard{ mutex }; - if (auto existing = peers.get ().find (endpoint.address ()); existing != peers.get ().end ()) + if (auto existing = peers.get ().find (address); existing != peers.get ().end ()) { if (existing->score >= score_limit && existing->exclude_until > std::chrono::steady_clock::now ()) { diff --git a/nano/node/peer_exclusion.hpp b/nano/node/peer_exclusion.hpp index d317cb6939..4ca738d9b9 100644 --- a/nano/node/peer_exclusion.hpp +++ b/nano/node/peer_exclusion.hpp @@ -50,6 +50,7 @@ class peer_exclusion final uint64_t score (nano::tcp_endpoint const &) const; std::chrono::steady_clock::time_point until (nano::tcp_endpoint const &) const; bool check (nano::tcp_endpoint const &) const; + bool check (boost::asio::ip::address const &) const; void remove (nano::tcp_endpoint const &); std::size_t size () const; From 2dcf07bfdc919314364e83772e70dc25d4e24715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:38:08 +0100 Subject: [PATCH 04/21] Add `is_same_ip` & `is_same_subnetwork` functions --- nano/node/transport/transport.cpp | 14 ++++++++++++-- nano/node/transport/transport.hpp | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/nano/node/transport/transport.cpp b/nano/node/transport/transport.cpp index 741b5d69f4..4b216b7187 100644 --- a/nano/node/transport/transport.cpp +++ b/nano/node/transport/transport.cpp @@ -18,12 +18,12 @@ nano::endpoint nano::transport::map_endpoint_to_v6 (nano::endpoint const & endpo nano::endpoint nano::transport::map_tcp_to_endpoint (nano::tcp_endpoint const & endpoint_a) { - return nano::endpoint (endpoint_a.address (), endpoint_a.port ()); + return { endpoint_a.address (), endpoint_a.port () }; } nano::tcp_endpoint nano::transport::map_endpoint_to_tcp (nano::endpoint const & endpoint_a) { - return nano::tcp_endpoint (endpoint_a.address (), endpoint_a.port ()); + return { endpoint_a.address (), endpoint_a.port () }; } boost::asio::ip::address nano::transport::map_address_to_subnetwork (boost::asio::ip::address const & address_a) @@ -41,6 +41,16 @@ boost::asio::ip::address nano::transport::ipv4_address_or_ipv6_subnet (boost::as return address_a.to_v6 ().is_v4_mapped () ? address_a : boost::asio::ip::make_network_v6 (address_a.to_v6 (), ipv6_address_prefix_length).network (); } +bool nano::transport::is_same_ip (boost::asio::ip::address const & address_a, boost::asio::ip::address const & address_b) +{ + return ipv4_address_or_ipv6_subnet (address_a) == ipv4_address_or_ipv6_subnet (address_b); +} + +bool nano::transport::is_same_subnetwork (boost::asio::ip::address const & address_a, boost::asio::ip::address const & address_b) +{ + return map_address_to_subnetwork (address_a) == map_address_to_subnetwork (address_b); +} + boost::asio::ip::address_v6 nano::transport::mapped_from_v4_bytes (unsigned long address_a) { return boost::asio::ip::address_v6::v4_mapped (boost::asio::ip::address_v4 (address_a)); diff --git a/nano/node/transport/transport.hpp b/nano/node/transport/transport.hpp index 6e177a3f87..48a3cc48fb 100644 --- a/nano/node/transport/transport.hpp +++ b/nano/node/transport/transport.hpp @@ -19,6 +19,8 @@ boost::asio::ip::address ipv4_address_or_ipv6_subnet (boost::asio::ip::address c boost::asio::ip::address_v6 mapped_from_v4_bytes (unsigned long); boost::asio::ip::address_v6 mapped_from_v4_or_v6 (boost::asio::ip::address const &); bool is_ipv4_or_v4_mapped_address (boost::asio::ip::address const &); +bool is_same_ip (boost::asio::ip::address const &, boost::asio::ip::address const &); +bool is_same_subnetwork (boost::asio::ip::address const &, boost::asio::ip::address const &); // Unassigned, reserved, self bool reserved_address (nano::endpoint const &, bool = false); From 32a654841f8de133e4d777db8ab4675aa88916b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 22 Mar 2024 18:34:58 +0100 Subject: [PATCH 05/21] Additional test logging --- nano/node/node.cpp | 4 +++- nano/test_common/testutil.cpp | 11 +++++++++++ nano/test_common/testutil.hpp | 5 +++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/nano/node/node.cpp b/nano/node/node.cpp index c21f9818dc..6ae1ee45dc 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -591,7 +591,9 @@ void nano::node::process_active (std::shared_ptr const & incoming) [[nodiscard]] nano::block_status nano::node::process (store::write_transaction const & transaction, std::shared_ptr block) { - return ledger.process (transaction, block); + auto status = ledger.process (transaction, block); + logger.debug (nano::log::type::node, "Directly processed block: {} (status: {})", block->hash ().to_string (), to_string (status)); + return status; } nano::block_status nano::node::process (std::shared_ptr block) diff --git a/nano/test_common/testutil.cpp b/nano/test_common/testutil.cpp index 75f26546f4..e3a7e24cc0 100644 --- a/nano/test_common/testutil.cpp +++ b/nano/test_common/testutil.cpp @@ -339,3 +339,14 @@ void nano::test::print_all_blocks (nano::node & node) std::cout << b->to_json (); } } + +std::vector> nano::test::all_blocks (nano::node & node) +{ + auto transaction = node.store.tx_begin_read (); + std::vector> result; + for (auto it = node.store.block.begin (transaction), end = node.store.block.end (); it != end; ++it) + { + result.push_back (it->second.block); + } + return result; +} \ No newline at end of file diff --git a/nano/test_common/testutil.hpp b/nano/test_common/testutil.hpp index a2706dc3f8..36ec9a3143 100644 --- a/nano/test_common/testutil.hpp +++ b/nano/test_common/testutil.hpp @@ -425,5 +425,10 @@ namespace test * \brief Debugging function to print all blocks in a node. Intented to be used to debug unit tests. */ void print_all_blocks (nano::node & node); + + /** + * Returns all blocks in the ledger + */ + std::vector> all_blocks (nano::node &); } } \ No newline at end of file From 9b4e16e8176cafb81521256103826bb06e9046f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:38:32 +0100 Subject: [PATCH 06/21] Convert tcp listener to use a dedicated thread --- nano/core_test/network.cpp | 10 +- nano/core_test/socket.cpp | 20 +- nano/lib/stats_enums.hpp | 16 +- nano/lib/thread_roles.cpp | 3 + nano/lib/thread_roles.hpp | 1 + nano/node/node.cpp | 12 +- nano/node/transport/socket.cpp | 9 +- nano/node/transport/socket.hpp | 13 +- nano/node/transport/tcp.cpp | 4 +- nano/node/transport/tcp_listener.cpp | 400 ++++++++++++++++----------- nano/node/transport/tcp_listener.hpp | 88 ++++-- nano/node/transport/tcp_server.cpp | 6 +- 12 files changed, 368 insertions(+), 214 deletions(-) diff --git a/nano/core_test/network.cpp b/nano/core_test/network.cpp index cafb622826..89711fbf27 100644 --- a/nano/core_test/network.cpp +++ b/nano/core_test/network.cpp @@ -715,7 +715,7 @@ TEST (network, peer_max_tcp_attempts) } ASSERT_EQ (0, node->network.size ()); ASSERT_FALSE (node->network.tcp_channels.track_reachout (nano::endpoint (node->network.endpoint ().address (), system.get_available_port ()))); - ASSERT_EQ (1, node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_ip, nano::stat::dir::out)); + ASSERT_EQ (1, node->stats.count (nano::stat::type::tcp, nano::stat::detail::max_per_ip, nano::stat::dir::out)); } #endif @@ -737,9 +737,9 @@ namespace transport ASSERT_TRUE (node->network.tcp_channels.track_reachout (endpoint)); } ASSERT_EQ (0, node->network.size ()); - ASSERT_EQ (0, node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_subnetwork, nano::stat::dir::out)); + ASSERT_EQ (0, node->stats.count (nano::stat::type::tcp, nano::stat::detail::max_per_subnetwork, nano::stat::dir::out)); ASSERT_FALSE (node->network.tcp_channels.track_reachout (nano::endpoint (boost::asio::ip::make_address_v6 ("::ffff:127.0.0.1"), system.get_available_port ()))); - ASSERT_EQ (1, node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_subnetwork, nano::stat::dir::out)); + ASSERT_EQ (1, node->stats.count (nano::stat::type::tcp, nano::stat::detail::max_per_subnetwork, nano::stat::dir::out)); } } } @@ -923,9 +923,9 @@ TEST (network, tcp_no_connect_excluded_peers) { node0->network.excluded_peers.add (endpoint1_tcp); } - ASSERT_EQ (0, node0->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_excluded)); + ASSERT_EQ (0, node0->stats.count (nano::stat::type::tcp, nano::stat::detail::excluded)); node1->network.merge_peer (node0->network.endpoint ()); - ASSERT_TIMELY (5s, node0->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_excluded) >= 1); + ASSERT_TIMELY (5s, node0->stats.count (nano::stat::type::tcp, nano::stat::detail::excluded) >= 1); ASSERT_EQ (nullptr, node0->network.find_node_id (node1->get_node_id ())); // Should not actively reachout to excluded peers diff --git a/nano/core_test/socket.cpp b/nano/core_test/socket.cpp index f8d58992e5..bfe0aec3dd 100644 --- a/nano/core_test/socket.cpp +++ b/nano/core_test/socket.cpp @@ -60,11 +60,11 @@ TEST (socket, max_connections) client3->async_connect (dst_endpoint, connect_handler); auto get_tcp_accept_failures = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_accept_failure, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_failure, nano::stat::dir::in); }; auto get_tcp_accept_successes = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_accept_success, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in); }; ASSERT_TIMELY_EQ (5s, get_tcp_accept_failures (), 1); @@ -156,11 +156,11 @@ TEST (socket, max_connections_per_ip) } auto get_tcp_max_per_ip = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_ip, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp, nano::stat::detail::max_per_ip, nano::stat::dir::in); }; auto get_tcp_accept_successes = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_accept_success, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in); }; ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), max_ip_connections); @@ -278,11 +278,11 @@ TEST (socket, max_connections_per_subnetwork) } auto get_tcp_max_per_subnetwork = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_subnetwork, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp, nano::stat::detail::max_per_subnetwork, nano::stat::dir::in); }; auto get_tcp_accept_successes = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_accept_success, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in); }; ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), max_subnetwork_connections); @@ -340,11 +340,11 @@ TEST (socket, disabled_max_peers_per_ip) } auto get_tcp_max_per_ip = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_ip, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp, nano::stat::detail::max_per_ip, nano::stat::dir::in); }; auto get_tcp_accept_successes = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_accept_success, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in); }; ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), max_ip_connections + 1); @@ -572,9 +572,9 @@ TEST (socket, concurrent_writes) runner.stop_event_processing (); runner.join (); - ASSERT_EQ (node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_accept_success, nano::stat::dir::in), client_count); + ASSERT_EQ (node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in), client_count); // We may exhaust max connections and have some tcp accept failures, but no more than the client count - ASSERT_LT (node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_accept_failure, nano::stat::dir::in), client_count); + ASSERT_LT (node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_failure, nano::stat::dir::in), client_count); for (auto & t : client_threads) { diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 01bf4b62d1..2106f69cb2 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -25,6 +25,7 @@ enum class type : uint8_t ipc, tcp, tcp_channels, + tcp_listener, channel, socket, confirmation_height, @@ -226,19 +227,24 @@ enum class detail : uint8_t merge_peer, // tcp - tcp_accept_success, - tcp_accept_failure, tcp_write_drop, tcp_write_no_socket_drop, - tcp_excluded, - tcp_max_per_ip, - tcp_max_per_subnetwork, tcp_silent_connection_drop, tcp_io_timeout_drop, tcp_connect_error, tcp_read_error, tcp_write_error, + // tcp_listener + accept_success, + accept_error, + accept_failure, + accept_limits_exceeded, + close_error, + max_per_ip, + max_per_subnetwork, + excluded, + // tcp_server handshake, handshake_abort, diff --git a/nano/lib/thread_roles.cpp b/nano/lib/thread_roles.cpp index 52eb3bba1e..9053897088 100644 --- a/nano/lib/thread_roles.cpp +++ b/nano/lib/thread_roles.cpp @@ -125,6 +125,9 @@ std::string nano::thread_role::get_string (nano::thread_role::name role) case nano::thread_role::name::network_reachout: thread_role_name_string = "Net reachout"; break; + case nano::thread_role::name::tcp_listener: + thread_role_name_string = "TCP listener"; + break; default: debug_assert (false && "nano::thread_role::get_string unhandled thread role"); } diff --git a/nano/lib/thread_roles.hpp b/nano/lib/thread_roles.hpp index c1dfce2d3e..b54815cf4d 100644 --- a/nano/lib/thread_roles.hpp +++ b/nano/lib/thread_roles.hpp @@ -48,6 +48,7 @@ enum class name network_cleanup, network_keepalive, network_reachout, + tcp_listener, }; std::string_view to_string (name); diff --git a/nano/node/node.cpp b/nano/node/node.cpp index 6ae1ee45dc..fba8014c0a 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -638,13 +638,7 @@ void nano::node::start () bool tcp_enabled = false; if (config.tcp_incoming_connections_max > 0 && !(flags.disable_bootstrap_listener && flags.disable_tcp_realtime)) { - tcp_listener->start ([this] (std::shared_ptr const & new_connection, boost::system::error_code const & ec_a) { - if (!ec_a) - { - tcp_listener->accept_action (ec_a, new_connection); - } - return true; - }); + tcp_listener->start (); tcp_enabled = true; if (network.port != tcp_listener->endpoint ().port ()) @@ -654,6 +648,10 @@ void nano::node::start () logger.info (nano::log::type::node, "Node peering port: {}", network.port.load ()); } + else + { + logger.warn (nano::log::type::node, "Node peering is disabled"); + } if (!flags.disable_backup) { diff --git a/nano/node/transport/socket.cpp b/nano/node/transport/socket.cpp index 11f5405acf..0de3f1bcaa 100644 --- a/nano/node/transport/socket.cpp +++ b/nano/node/transport/socket.cpp @@ -20,10 +20,17 @@ */ nano::transport::socket::socket (nano::node & node_a, nano::transport::socket_endpoint endpoint_type_a, std::size_t max_queue_size_a) : + socket{ boost::asio::ip::tcp::socket{ node_a.io_ctx }, {}, {}, node_a, endpoint_type_a, max_queue_size_a } +{ +} + +nano::transport::socket::socket (boost::asio::ip::tcp::socket boost_socket_a, boost::asio::ip::tcp::endpoint remote_endpoint_a, boost::asio::ip::tcp::endpoint local_endpoint_a, nano::node & node_a, nano::transport::socket_endpoint endpoint_type_a, std::size_t max_queue_size_a) : send_queue{ max_queue_size_a }, strand{ node_a.io_ctx.get_executor () }, - tcp_socket{ node_a.io_ctx }, + tcp_socket{ std::move (boost_socket_a) }, node{ node_a }, + remote{ remote_endpoint_a }, + local{ local_endpoint_a }, endpoint_type_m{ endpoint_type_a }, timeout{ std::numeric_limits::max () }, last_completion_time_or_init{ nano::seconds_since_epoch () }, diff --git a/nano/node/transport/socket.hpp b/nano/node/transport/socket.hpp index 73c3d04460..706f93dd93 100644 --- a/nano/node/transport/socket.hpp +++ b/nano/node/transport/socket.hpp @@ -69,9 +69,20 @@ class socket final : public std::enable_shared_from_this public: explicit socket (nano::node &, nano::transport::socket_endpoint = socket_endpoint::client, std::size_t max_queue_size = default_max_queue_size); + + // TODO: Accepting remote/local endpoints as a parameter is unnecessary, but is needed for now to keep compatibility with the legacy code + explicit socket ( + boost::asio::ip::tcp::socket, + boost::asio::ip::tcp::endpoint remote_endpoint, + boost::asio::ip::tcp::endpoint local_endpoint, + nano::node &, + nano::transport::socket_endpoint = socket_endpoint::server, + std::size_t max_queue_size = default_max_queue_size); + ~socket (); void start (); + void close (); void async_connect ( boost::asio::ip::tcp::endpoint const & endpoint, @@ -87,8 +98,6 @@ class socket final : public std::enable_shared_from_this std::function callback = {}, traffic_type = traffic_type::generic); - void close (); - boost::asio::ip::tcp::endpoint remote_endpoint () const; boost::asio::ip::tcp::endpoint local_endpoint () const; diff --git a/nano/node/transport/tcp.cpp b/nano/node/transport/tcp.cpp index 05ac10d0f1..eac61e5794 100644 --- a/nano/node/transport/tcp.cpp +++ b/nano/node/transport/tcp.cpp @@ -383,7 +383,7 @@ bool nano::transport::tcp_channels::max_ip_connections (nano::tcp_endpoint const } if (result) { - node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_ip, nano::stat::dir::out); + node.stats.inc (nano::stat::type::tcp, nano::stat::detail::max_per_ip, nano::stat::dir::out); } return result; } @@ -404,7 +404,7 @@ bool nano::transport::tcp_channels::max_subnetwork_connections (nano::tcp_endpoi } if (result) { - node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_subnetwork, nano::stat::dir::out); + node.stats.inc (nano::stat::type::tcp, nano::stat::detail::max_per_subnetwork, nano::stat::dir::out); } return result; } diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index 47dac191a5..3f0c4c9af1 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -6,6 +7,8 @@ #include +using namespace std::chrono_literals; + namespace { bool is_temporary_error (boost::system::error_code const & ec_a) @@ -30,252 +33,325 @@ bool is_temporary_error (boost::system::error_code const & ec_a) */ nano::transport::tcp_listener::tcp_listener (uint16_t port_a, nano::node & node_a, std::size_t max_inbound_connections) : - node (node_a), - strand{ node_a.io_ctx.get_executor () }, + node{ node_a }, + stats{ node_a.stats }, + logger{ node_a.logger }, + port{ port_a }, + max_inbound_connections{ max_inbound_connections }, acceptor{ node_a.io_ctx }, - local{ boost::asio::ip::tcp::endpoint{ boost::asio::ip::address_v6::any (), port_a } }, - max_inbound_connections{ max_inbound_connections } + local{ boost::asio::ip::tcp::endpoint{ boost::asio::ip::address_v6::any (), port_a } } { + connection_accepted.add ([this] (auto const & socket, auto const & server) { + node.observers.socket_accepted.notify (*socket); + }); } nano::transport::tcp_listener::~tcp_listener () { - debug_assert (stopped); + // Thread should be stopped before destruction + debug_assert (!thread.joinable ()); } void nano::transport::tcp_listener::start (std::function const &, boost::system::error_code const &)> callback_a) { - nano::lock_guard lock{ mutex }; + debug_assert (!thread.joinable ()); + debug_assert (!cleanup_thread.joinable ()); - acceptor.open (local.protocol ()); - acceptor.set_option (boost::asio::ip::tcp::acceptor::reuse_address (true)); - boost::system::error_code ec; - acceptor.bind (local, ec); - if (!ec) + try { - acceptor.listen (boost::asio::socket_base::max_listen_connections, ec); + acceptor.open (local.protocol ()); + acceptor.set_option (boost::asio::ip::tcp::acceptor::reuse_address (true)); + acceptor.bind (local); + acceptor.listen (boost::asio::socket_base::max_listen_connections); + + logger.info (nano::log::type::tcp_listener, "Listening for incoming connections on: {}", fmt::streamed (acceptor.local_endpoint ())); } - if (ec) + catch (boost::system::system_error const & ex) { - node.logger.critical (nano::log::type::tcp_listener, "Error while binding for incoming TCP: {} (port: {})", ec.message (), acceptor.local_endpoint ().port ()); - throw std::runtime_error (ec.message ()); + logger.critical (nano::log::type::tcp_listener, "Error while binding for incoming TCP: {} (port: {})", ex.what (), port); + + throw std::runtime_error (ex.code ().message ()); } - on_connection (callback_a); + thread = std::thread ([this, callback_a] { + nano::thread_role::set (nano::thread_role::name::tcp_listener); + try + { + logger.debug (nano::log::type::tcp_listener, "Starting acceptor thread"); + run (); + logger.debug (nano::log::type::tcp_listener, "Stopped acceptor thread"); + } + catch (std::exception const & ex) + { + logger.critical (nano::log::type::tcp_listener, "Error: {}", ex.what ()); + release_assert (false); // Should be handled earlier + } + catch (...) + { + logger.critical (nano::log::type::tcp_listener, "Unknown error"); + release_assert (false); // Should be handled earlier + } + }); + + cleanup_thread = std::thread ([this] { + nano::thread_role::set (nano::thread_role::name::tcp_listener); + run_cleanup (); + }); } void nano::transport::tcp_listener::stop () { - decltype (connections) connections_l; + logger.info (nano::log::type::tcp_listener, "Stopping listeninig for incoming connections and closing all sockets..."); { nano::lock_guard lock{ mutex }; stopped = true; - connections_l.swap (connections); } + condition.notify_all (); - nano::lock_guard lock{ mutex }; - boost::asio::dispatch (strand, [this_l = shared_from_this ()] () { - this_l->acceptor.close (); + acceptor.close (); + + if (thread.joinable ()) + { + thread.join (); + } + if (cleanup_thread.joinable ()) + { + cleanup_thread.join (); + } + + decltype (connections) connections_l; + { + nano::lock_guard lock{ mutex }; + connections_l.swap (connections); + } - for (auto & address_connection_pair : this_l->connections_per_address) + for (auto & connection : connections_l) + { + if (auto socket = connection.socket.lock ()) { - if (auto connection_l = address_connection_pair.second.lock ()) - { - connection_l->close (); - } + socket->close (); } - this_l->connections_per_address.clear (); - }); + if (auto server = connection.server.lock ()) + { + server->stop (); + } + } } -std::size_t nano::transport::tcp_listener::connection_count () +void nano::transport::tcp_listener::run_cleanup () { - nano::lock_guard lock{ mutex }; - cleanup (); - return connections.size (); + nano::unique_lock lock{ mutex }; + while (!stopped) + { + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::cleanup); + cleanup (); + condition.wait_for (lock, 1s, [this] () { return stopped.load (); }); + } } void nano::transport::tcp_listener::cleanup () { debug_assert (!mutex.try_lock ()); - erase_if (connections, [] (auto const & connection) { - return connection.second.expired (); + erase_if (connections, [this] (auto const & connection) { + if (connection.socket.expired () && connection.server.expired ()) + { + logger.debug (nano::log::type::tcp_listener, "Evicting dead connection: {}", fmt::streamed (connection.endpoint)); + return true; + } + else + { + return false; + } }); } -bool nano::transport::tcp_listener::limit_reached_for_incoming_subnetwork_connections (std::shared_ptr const & new_connection) +void nano::transport::tcp_listener::run () { - debug_assert (strand.running_in_this_thread ()); - if (node.flags.disable_max_peers_per_subnetwork || nano::transport::is_ipv4_or_v4_mapped_address (new_connection->remote.address ())) + nano::unique_lock lock{ mutex }; + while (!stopped && acceptor.is_open ()) + { + lock.unlock (); + + wait_available_slots (); + + if (stopped) + { + return; + } + + bool cooldown = false; + try + { + auto result = accept_one (); + if (result != accept_result::accepted) + { + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::accept_failure, nano::stat::dir::in); + // Refusal reason should be logged earlier + } + } + catch (boost::system::system_error const & ex) + { + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::accept_error, nano::stat::dir::in); + logger.log (stopped ? nano::log::level::debug : nano::log::level::error, // Avoid logging expected errors when stopping + nano::log::type::tcp_listener, "Error accepting incoming connection: {}", ex.what ()); + + cooldown = true; + } + + lock.lock (); + + // Sleep for a while to prevent busy loop with additional cooldown if an error occurred + condition.wait_for (lock, cooldown ? 100ms : 10ms, [this] () { return stopped.load (); }); + } + if (!stopped) { - // If the limit is disabled, then it is unreachable. - // If the address is IPv4 we don't check for a network limit, since its address space isn't big as IPv6 /64. - return false; + debug_assert (false, "acceptor stopped unexpectedly"); + logger.error (nano::log::type::tcp_listener, "Acceptor stopped unexpectedly"); } - auto const counted_connections = socket_functions::count_subnetwork_connections ( - connections_per_address, - new_connection->remote.address ().to_v6 (), - node.network_params.network.ipv6_subnetwork_prefix_for_limiting); - return counted_connections >= node.network_params.network.max_peers_per_subnetwork; } -bool nano::transport::tcp_listener::limit_reached_for_incoming_ip_connections (std::shared_ptr const & new_connection) +auto nano::transport::tcp_listener::accept_one () -> accept_result { - debug_assert (strand.running_in_this_thread ()); - if (node.flags.disable_max_peers_per_ip) + auto raw_socket = acceptor.accept (); + auto const remote_endpoint = raw_socket.remote_endpoint (); + auto const local_endpoint = raw_socket.local_endpoint (); + + if (auto result = check_limits (remote_endpoint.address ()); result != accept_result::accepted) + { + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::accept_limits_exceeded, nano::stat::dir::in); + // Refusal reason should be logged earlier + + try + { + // Best effor attempt to gracefully close the socket, shutdown before closing to avoid zombie sockets + raw_socket.shutdown (boost::asio::ip::tcp::socket::shutdown_both); + raw_socket.close (); + } + catch (boost::system::system_error const & ex) + { + logger.debug (nano::log::type::tcp_listener, "Error while closing socket after refusing connection: {}", ex.what ()); + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::close_error, nano::stat::dir::in); + } + + return result; + } + + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::accept_success, nano::stat::dir::in); + logger.debug (nano::log::type::tcp_listener, "Accepted incoming connection from: {}", fmt::streamed (remote_endpoint)); + + auto socket = std::make_shared (std::move (raw_socket), remote_endpoint, local_endpoint, node, socket_endpoint::server); + auto server = std::make_shared (socket, node.shared (), true); + { - // If the limit is disabled, then it is unreachable. - return false; + nano::lock_guard lock{ mutex }; + connections.emplace (entry{ remote_endpoint, socket, server }); } - auto const address_connections_range = connections_per_address.equal_range (new_connection->remote.address ()); - auto const counted_connections = static_cast (std::abs (std::distance (address_connections_range.first, address_connections_range.second))); - return counted_connections >= node.network_params.network.max_peers_per_ip; + + socket->set_timeout (node.network_params.network.idle_timeout); + socket->start (); + server->start (); + + connection_accepted.notify (socket, server); + + return accept_result::accepted; } -void nano::transport::tcp_listener::on_connection (std::function const &, boost::system::error_code const &)> callback_a) +void nano::transport::tcp_listener::wait_available_slots () { - boost::asio::post (strand, boost::asio::bind_executor (strand, [this_l = shared_from_this (), callback = std::move (callback_a)] () mutable { - if (!this_l->acceptor.is_open ()) + auto should_wait = [this] { + nano::lock_guard lock{ mutex }; + return connections.size () >= max_inbound_connections; + }; + + nano::interval log_interval; + while (!stopped && should_wait ()) + { + if (log_interval.elapsed (node.network_params.network.is_dev_network () ? 1s : 15s)) { - this_l->node.logger.error (nano::log::type::tcp_listener, "Acceptor is not open"); - return; + logger.warn (nano::log::type::tcp_listener, "Waiting for available slots to accept new connections (current: {} / max: {})", + connection_count (), max_inbound_connections); } - // Prepare new connection - auto new_connection = std::make_shared (this_l->node, socket_endpoint::server); - this_l->acceptor.async_accept (new_connection->tcp_socket, new_connection->remote, - boost::asio::bind_executor (this_l->strand, - [this_l, new_connection, cbk = std::move (callback)] (boost::system::error_code const & ec_a) mutable { - this_l->evict_dead_connections (); - - if (this_l->connections_per_address.size () >= this_l->max_inbound_connections) - { - this_l->node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_accept_failure, nano::stat::dir::in); - this_l->node.logger.debug (nano::log::type::tcp_listener, "Max connections reached ({}), unable to open new connection", this_l->connections_per_address.size ()); - - this_l->on_connection_requeue_delayed (std::move (cbk)); - return; - } + std::this_thread::sleep_for (100ms); + } +} - if (this_l->limit_reached_for_incoming_ip_connections (new_connection)) - { - this_l->node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_ip, nano::stat::dir::in); - this_l->node.logger.debug (nano::log::type::tcp_listener, "Max connections per IP reached ({}), unable to open new connection", new_connection->remote_endpoint ().address ().to_string ()); +auto nano::transport::tcp_listener::check_limits (boost::asio::ip::address const & ip) -> accept_result +{ + nano::lock_guard lock{ mutex }; - this_l->on_connection_requeue_delayed (std::move (cbk)); - return; - } + cleanup (); - if (this_l->limit_reached_for_incoming_subnetwork_connections (new_connection)) - { - auto const remote_ip_address = new_connection->remote_endpoint ().address (); - debug_assert (remote_ip_address.is_v6 ()); - auto const remote_subnet = socket_functions::get_ipv6_subnet_address (remote_ip_address.to_v6 (), this_l->node.network_params.network.max_peers_per_subnetwork); + debug_assert (connections.size () <= max_inbound_connections); // Should be checked earlier (wait_available_slots) - this_l->node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_max_per_subnetwork, nano::stat::dir::in); - this_l->node.logger.debug (nano::log::type::tcp_listener, "Max connections per subnetwork reached (subnetwork: {}, ip: {}), unable to open new connection", - remote_subnet.canonical ().to_string (), - remote_ip_address.to_string ()); + if (!node.flags.disable_max_peers_per_ip) + { + if (count_per_ip (ip) >= node.network_params.network.max_peers_per_ip) + { + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::max_per_ip, nano::stat::dir::in); + logger.debug (nano::log::type::tcp_listener, "Max connections per IP reached ({}), unable to open new connection", ip.to_string ()); - this_l->on_connection_requeue_delayed (std::move (cbk)); - return; - } + return accept_result::too_many_per_ip; + } + } - if (!ec_a) - { - { - // Best effort attempt to get endpoint addresses - boost::system::error_code ec; - new_connection->local = new_connection->tcp_socket.local_endpoint (ec); - } - - // Make sure the new connection doesn't idle. Note that in most cases, the callback is going to start - // an IO operation immediately, which will start a timer. - new_connection->start (); - new_connection->set_timeout (this_l->node.network_params.network.idle_timeout); - this_l->node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_accept_success, nano::stat::dir::in); - this_l->connections_per_address.emplace (new_connection->remote.address (), new_connection); - this_l->node.observers.socket_accepted.notify (*new_connection); - if (cbk (new_connection, ec_a)) - { - this_l->on_connection (std::move (cbk)); - return; - } - this_l->node.logger.warn (nano::log::type::tcp_listener, "Stopping to accept new connections"); - return; - } + // If the address is IPv4 we don't check for a network limit, since its address space isn't big as IPv6/64. + if (!node.flags.disable_max_peers_per_subnetwork && !nano::transport::is_ipv4_or_v4_mapped_address (ip)) + { + if (count_per_subnetwork (ip) >= node.network_params.network.max_peers_per_subnetwork) + { + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::max_per_subnetwork, nano::stat::dir::in); + logger.debug (nano::log::type::tcp_listener, "Max connections per subnetwork reached ({}), unable to open new connection", ip.to_string ()); - // accept error - this_l->node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_accept_failure, nano::stat::dir::in); - this_l->node.logger.error (nano::log::type::tcp_listener, "Unable to accept connection: {} ({})", ec_a.message (), new_connection->remote_endpoint ().address ().to_string ()); + return accept_result::too_many_per_subnetwork; + } + } - if (is_temporary_error (ec_a)) - { - // if it is a temporary error, just retry it - this_l->on_connection_requeue_delayed (std::move (cbk)); - return; - } + if (node.network.excluded_peers.check (ip)) // true => error + { + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::excluded, nano::stat::dir::in); + logger.debug (nano::log::type::tcp_listener, "Rejected connection from excluded peer: {}", ip.to_string ()); - // if it is not a temporary error, check how the listener wants to handle this error - if (cbk (new_connection, ec_a)) - { - this_l->on_connection_requeue_delayed (std::move (cbk)); - return; - } + return accept_result::excluded; + } - // No requeue if we reach here, no incoming socket connections will be handled - this_l->node.logger.warn (nano::log::type::tcp_listener, "Stopping to accept new connections"); - })); - })); + return accept_result::accepted; } -// If we are unable to accept a socket, for any reason, we wait just a little (1ms) before rescheduling the next connection accept. -// The intention is to throttle back the connection requests and break up any busy loops that could possibly form and -// give the rest of the system a chance to recover. -void nano::transport::tcp_listener::on_connection_requeue_delayed (std::function const &, boost::system::error_code const &)> callback_a) +std::size_t nano::transport::tcp_listener::connection_count () const { - node.workers.add_timed_task (std::chrono::steady_clock::now () + std::chrono::milliseconds (1), [this_l = shared_from_this (), callback = std::move (callback_a)] () mutable { - this_l->on_connection (std::move (callback)); - }); + nano::lock_guard lock{ mutex }; + return connections.size (); } -// This must be called from a strand -void nano::transport::tcp_listener::evict_dead_connections () +size_t nano::transport::tcp_listener::count_per_ip (boost::asio::ip::address const & ip) const { - debug_assert (strand.running_in_this_thread ()); + debug_assert (!mutex.try_lock ()); - erase_if (connections_per_address, [] (auto const & entry) { - return entry.second.expired (); + return std::count_if (connections.begin (), connections.end (), [&ip] (auto const & connection) { + return nano::transport::is_same_ip (connection.address (), ip); }); } -void nano::transport::tcp_listener::accept_action (boost::system::error_code const & ec, std::shared_ptr const & socket_a) +size_t nano::transport::tcp_listener::count_per_subnetwork (boost::asio::ip::address const & ip) const { - if (!node.network.excluded_peers.check (socket_a->remote_endpoint ())) - { - auto server = std::make_shared (socket_a, node.shared (), true); - nano::lock_guard lock{ mutex }; - connections[server.get ()] = server; - server->start (); - } - else - { - node.stats.inc (nano::stat::type::tcp, nano::stat::detail::tcp_excluded); - node.logger.debug (nano::log::type::tcp_server, "Rejected connection from excluded peer: {}", nano::util::to_str (socket_a->remote_endpoint ())); - } + debug_assert (!mutex.try_lock ()); + + return std::count_if (connections.begin (), connections.end (), [this, &ip] (auto const & connection) { + return nano::transport::is_same_subnetwork (connection.address (), ip); + }); } boost::asio::ip::tcp::endpoint nano::transport::tcp_listener::endpoint () const { - nano::lock_guard lock{ mutex }; if (!stopped) { - return boost::asio::ip::tcp::endpoint (boost::asio::ip::address_v6::loopback (), acceptor.local_endpoint ().port ()); + return { boost::asio::ip::address_v6::loopback (), acceptor.local_endpoint ().port () }; } else { - return boost::asio::ip::tcp::endpoint (boost::asio::ip::address_v6::loopback (), 0); + return { boost::asio::ip::address_v6::loopback (), 0 }; } } diff --git a/nano/node/transport/tcp_listener.hpp b/nano/node/transport/tcp_listener.hpp index ae049d97ee..7defb4a34f 100644 --- a/nano/node/transport/tcp_listener.hpp +++ b/nano/node/transport/tcp_listener.hpp @@ -1,9 +1,23 @@ #pragma once -#include #include +#include +#include +#include + #include +#include +#include + +namespace mi = boost::multi_index; + +namespace nano +{ +class node; +class stats; +class logger; +} namespace nano::transport { @@ -11,50 +25,90 @@ class socket; class tcp_server; /** - * Server side portion of bootstrap sessions. Listens for new socket connections and spawns tcp_server objects when connected. + * Server side portion of tcp sessions. Listens for new socket connections and spawns tcp_server objects when connected. */ -class tcp_listener final : public std::enable_shared_from_this +class tcp_listener final { public: tcp_listener (uint16_t port, nano::node &, std::size_t max_inbound_connections); ~tcp_listener (); - void start (std::function const &, boost::system::error_code const &)> callback); + void start (std::function const &, boost::system::error_code const &)> callback = {}); void stop (); - void accept_action (boost::system::error_code const &, std::shared_ptr const &); - - std::size_t connection_count (); + std::size_t connection_count () const; nano::tcp_endpoint endpoint () const; std::unique_ptr collect_container_info (std::string const & name); +public: // Events + using connection_accepted_event_t = nano::observer_set const &, std::shared_ptr>; + connection_accepted_event_t connection_accepted; + private: // Dependencies nano::node & node; + nano::stats & stats; + nano::logger & logger; private: - void on_connection (std::function const &, boost::system::error_code const &)> callback_a); - void evict_dead_connections (); - void on_connection_requeue_delayed (std::function const & new_connection, boost::system::error_code const &)>); - /** Checks whether the maximum number of connections per IP was reached. If so, it returns true. */ - bool limit_reached_for_incoming_ip_connections (std::shared_ptr const & new_connection); - bool limit_reached_for_incoming_subnetwork_connections (std::shared_ptr const & new_connection); + void run (); + void run_cleanup (); void cleanup (); + void wait_available_slots (); + + enum class accept_result + { + invalid, + accepted, + too_many_per_ip, + too_many_per_subnetwork, + excluded, + }; + + accept_result accept_one (); + accept_result check_limits (boost::asio::ip::address const & ip); + size_t count_per_ip (boost::asio::ip::address const & ip) const; + size_t count_per_subnetwork (boost::asio::ip::address const & ip) const; public: std::atomic bootstrap_count{ 0 }; std::atomic realtime_count{ 0 }; private: - std::unordered_map> connections; - std::multimap> connections_per_address; + struct entry + { + boost::asio::ip::tcp::endpoint endpoint; + std::weak_ptr socket; + std::weak_ptr server; + + boost::asio::ip::address address () const + { + return endpoint.address (); + } + }; + +private: + uint16_t const port; + std::size_t const max_inbound_connections; + + // clang-format off + class tag_address {}; + + using ordered_connections = boost::multi_index_container, + mi::const_mem_fun> + >>; + // clang-format on + ordered_connections connections; - boost::asio::strand strand; boost::asio::ip::tcp::acceptor acceptor; boost::asio::ip::tcp::endpoint local; - std::size_t const max_inbound_connections; std::atomic stopped; + nano::condition_variable condition; mutable nano::mutex mutex; + std::thread thread; + std::thread cleanup_thread; }; } \ No newline at end of file diff --git a/nano/node/transport/tcp_server.cpp b/nano/node/transport/tcp_server.cpp index 070a2882fb..85c8f01b89 100644 --- a/nano/node/transport/tcp_server.cpp +++ b/nano/node/transport/tcp_server.cpp @@ -36,7 +36,7 @@ nano::transport::tcp_server::~tcp_server () return; } - node->logger.debug (nano::log::type::tcp_server, "Exiting TCP server ({})", fmt::streamed (remote_endpoint)); + node->logger.debug (nano::log::type::tcp_server, "Exiting server: {}", fmt::streamed (remote_endpoint)); if (socket->type () == nano::transport::socket_type::bootstrap) { @@ -73,7 +73,7 @@ void nano::transport::tcp_server::start () return; } - node->logger.debug (nano::log::type::tcp_server, "Starting TCP server ({})", fmt::streamed (remote_endpoint)); + node->logger.debug (nano::log::type::tcp_server, "Starting server: {}", fmt::streamed (remote_endpoint)); receive_message (); } @@ -280,7 +280,7 @@ auto nano::transport::tcp_server::process_handshake (nano::node_id_handshake con if (node->flags.disable_tcp_realtime) { node->stats.inc (nano::stat::type::tcp_server, nano::stat::detail::handshake_error); - node->logger.debug (nano::log::type::tcp_server, "Handshake attempted with disabled realtime TCP ({})", fmt::streamed (remote_endpoint)); + node->logger.debug (nano::log::type::tcp_server, "Handshake attempted with disabled realtime mode ({})", fmt::streamed (remote_endpoint)); return handshake_status::abort; } From 01ac3d55a75672cc2a3d1c9ee626acf62c76cf0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 22 Mar 2024 22:09:02 +0100 Subject: [PATCH 07/21] Fix tests --- nano/core_test/network.cpp | 6 +- nano/core_test/node.cpp | 1 + nano/core_test/socket.cpp | 178 +++++++++++++++++-------------------- 3 files changed, 88 insertions(+), 97 deletions(-) diff --git a/nano/core_test/network.cpp b/nano/core_test/network.cpp index 89711fbf27..8fd4de95d6 100644 --- a/nano/core_test/network.cpp +++ b/nano/core_test/network.cpp @@ -910,7 +910,7 @@ TEST (peer_exclusion, validate) } } -TEST (network, tcp_no_connect_excluded_peers) +TEST (network, tcp_no_accept_excluded_peers) { nano::test::system system (1); auto node0 (system.nodes[0]); @@ -923,9 +923,9 @@ TEST (network, tcp_no_connect_excluded_peers) { node0->network.excluded_peers.add (endpoint1_tcp); } - ASSERT_EQ (0, node0->stats.count (nano::stat::type::tcp, nano::stat::detail::excluded)); + ASSERT_EQ (0, node0->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::excluded)); node1->network.merge_peer (node0->network.endpoint ()); - ASSERT_TIMELY (5s, node0->stats.count (nano::stat::type::tcp, nano::stat::detail::excluded) >= 1); + ASSERT_TIMELY (5s, node0->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::excluded) >= 1); ASSERT_EQ (nullptr, node0->network.find_node_id (node1->get_node_id ())); // Should not actively reachout to excluded peers diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index d396a2cba5..5435a66a11 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -39,6 +39,7 @@ TEST (node, null_account) TEST (node, stop) { nano::test::system system (1); + system.io_guard.reset (); ASSERT_NE (system.nodes[0]->wallets.items.end (), system.nodes[0]->wallets.items.begin ()); ASSERT_TRUE (true); } diff --git a/nano/core_test/socket.cpp b/nano/core_test/socket.cpp index bfe0aec3dd..3e651e9bb4 100644 --- a/nano/core_test/socket.cpp +++ b/nano/core_test/socket.cpp @@ -20,9 +20,7 @@ using namespace std::chrono_literals; TEST (socket, max_connections) { nano::test::system system; - auto node = system.add_node (); - auto server_port = system.get_available_port (); // successful incoming connections are stored in server_sockets to keep them alive (server side) @@ -30,14 +28,11 @@ TEST (socket, max_connections) // start a server socket that allows max 2 live connections auto listener = std::make_shared (server_port, *node, 2); - nano::test::stop_guard stop_guard{ *listener }; - listener->start ([&server_sockets] (std::shared_ptr const & new_connection, boost::system::error_code const & ec) { - if (!ec) - { - server_sockets.push_back (new_connection); - } - return true; + listener->connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { + server_sockets.push_back (socket); }); + nano::test::stop_guard stop_guard{ *listener }; + listener->start (); boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; @@ -60,15 +55,15 @@ TEST (socket, max_connections) client3->async_connect (dst_endpoint, connect_handler); auto get_tcp_accept_failures = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_failure, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::accept_failure, nano::stat::dir::in); }; auto get_tcp_accept_successes = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::accept_success, nano::stat::dir::in); }; - ASSERT_TIMELY_EQ (5s, get_tcp_accept_failures (), 1); - ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), 2); + ASSERT_TIMELY_EQ (10s, get_tcp_accept_successes (), 2); + ASSERT_ALWAYS_EQ (1s, get_tcp_accept_successes (), 2); ASSERT_TIMELY_EQ (5s, connection_attempts, 3); ASSERT_TIMELY_EQ (5s, server_sockets.size (), 2); @@ -82,8 +77,8 @@ TEST (socket, max_connections) auto client5 = std::make_shared (*node); client5->async_connect (dst_endpoint, connect_handler); - ASSERT_TIMELY_EQ (5s, get_tcp_accept_failures (), 2); - ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), 3); + ASSERT_TIMELY_EQ (10s, get_tcp_accept_successes (), 3); + ASSERT_ALWAYS_EQ (1s, get_tcp_accept_successes (), 3); ASSERT_TIMELY_EQ (5s, connection_attempts, 5); ASSERT_TIMELY_EQ (5s, server_sockets.size (), 3); @@ -102,8 +97,8 @@ TEST (socket, max_connections) auto client8 = std::make_shared (*node); client8->async_connect (dst_endpoint, connect_handler); - ASSERT_TIMELY_EQ (5s, get_tcp_accept_failures (), 3); ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), 5); + ASSERT_ALWAYS_EQ (1s, get_tcp_accept_successes (), 5); ASSERT_TIMELY_EQ (5s, connection_attempts, 8); // connections initiated by the client ASSERT_TIMELY_EQ (5s, server_sockets.size (), 5); // connections accepted by the server } @@ -126,14 +121,11 @@ TEST (socket, max_connections_per_ip) std::vector> server_sockets; auto listener = std::make_shared (server_port, *node, max_global_connections); - nano::test::stop_guard stop_guard{ *listener }; - listener->start ([&server_sockets] (std::shared_ptr const & new_connection, boost::system::error_code const & ec) { - if (!ec) - { - server_sockets.push_back (new_connection); - } - return true; + listener->connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { + server_sockets.push_back (socket); }); + nano::test::stop_guard stop_guard{ *listener }; + listener->start (); boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; @@ -156,11 +148,11 @@ TEST (socket, max_connections_per_ip) } auto get_tcp_max_per_ip = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::max_per_ip, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::max_per_ip, nano::stat::dir::in); }; auto get_tcp_accept_successes = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::accept_success, nano::stat::dir::in); }; ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), max_ip_connections); @@ -248,14 +240,11 @@ TEST (socket, max_connections_per_subnetwork) std::vector> server_sockets; auto listener = std::make_shared (server_port, *node, max_global_connections); - nano::test::stop_guard stop_guard{ *listener }; - listener->start ([&server_sockets] (std::shared_ptr const & new_connection, boost::system::error_code const & ec) { - if (!ec) - { - server_sockets.push_back (new_connection); - } - return true; + listener->connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { + server_sockets.push_back (socket); }); + nano::test::stop_guard stop_guard{ *listener }; + listener->start (); boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; @@ -278,11 +267,11 @@ TEST (socket, max_connections_per_subnetwork) } auto get_tcp_max_per_subnetwork = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::max_per_subnetwork, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::max_per_subnetwork, nano::stat::dir::in); }; auto get_tcp_accept_successes = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::accept_success, nano::stat::dir::in); }; ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), max_subnetwork_connections); @@ -309,17 +298,14 @@ TEST (socket, disabled_max_peers_per_ip) // successful incoming connections are stored in server_sockets to keep them alive (server side) std::vector> server_sockets; - auto server_socket = std::make_shared (server_port, *node, max_global_connections); - nano::test::stop_guard stop_guard{ *server_socket }; - server_socket->start ([&server_sockets] (std::shared_ptr const & new_connection, boost::system::error_code const & ec) { - if (!ec) - { - server_sockets.push_back (new_connection); - } - return true; + auto listener = std::make_shared (server_port, *node, max_global_connections); + listener->connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { + server_sockets.push_back (socket); }); + nano::test::stop_guard stop_guard{ *listener }; + listener->start (); - boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), server_socket->endpoint ().port () }; + boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; // client side connection tracking std::atomic connection_attempts = 0; @@ -340,11 +326,11 @@ TEST (socket, disabled_max_peers_per_ip) } auto get_tcp_max_per_ip = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::max_per_ip, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::max_per_ip, nano::stat::dir::in); }; auto get_tcp_accept_successes = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in); + return node->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::accept_success, nano::stat::dir::in); }; ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), max_ip_connections + 1); @@ -372,14 +358,11 @@ TEST (socket, disconnection_of_silent_connections) // start a server listening socket auto listener = std::make_shared (server_port, *node, 1); - nano::test::stop_guard stop_guard{ *listener }; - listener->start ([&server_data_socket] (std::shared_ptr const & new_connection, boost::system::error_code const & ec) { - if (!ec) - { - server_data_socket = new_connection; - } - return true; + listener->connection_accepted.add ([&server_data_socket] (auto const & socket, auto const & server) { + server_data_socket = socket; }); + nano::test::stop_guard stop_guard{ *listener }; + listener->start (); boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; @@ -422,36 +405,37 @@ TEST (socket, drop_policy) std::vector> connections; auto func = [&] (size_t total_message_count, nano::transport::buffer_drop_policy drop_policy) { - auto server_port (system.get_available_port ()); - - auto listener = std::make_shared (server_port, *node, 1); - nano::test::stop_guard stop_guard{ *listener }; - listener->start ([&connections] (std::shared_ptr const & new_connection, boost::system::error_code const & ec) { - if (!ec) - { - connections.push_back (new_connection); - } - return true; + boost::asio::ip::tcp::endpoint endpoint (boost::asio::ip::address_v6::loopback (), system.get_available_port ()); + boost::asio::ip::tcp::acceptor acceptor (node->io_ctx); + acceptor.open (endpoint.protocol ()); + acceptor.bind (endpoint); + acceptor.listen (boost::asio::socket_base::max_listen_connections); + + boost::asio::ip::tcp::socket newsock (*system.io_ctx); + acceptor.async_accept (newsock, [] (boost::system::error_code const & ec) { + EXPECT_FALSE (ec); }); auto client = std::make_shared (*node); auto channel = std::make_shared (*node, client); - nano::test::counted_completion write_completion (static_cast (total_message_count)); - client->async_connect (boost::asio::ip::tcp::endpoint (boost::asio::ip::address_v6::loopback (), listener->endpoint ().port ()), - [&channel, total_message_count, node, &write_completion, &drop_policy, client] (boost::system::error_code const & ec_a) mutable { + std::atomic completed_writes{ 0 }; + + client->async_connect (boost::asio::ip::tcp::endpoint (boost::asio::ip::address_v6::loopback (), acceptor.local_endpoint ().port ()), + [&channel, total_message_count, node, &completed_writes, &drop_policy, client] (boost::system::error_code const & ec_a) mutable { for (int i = 0; i < total_message_count; i++) { std::vector buff (1); channel->send_buffer ( - nano::shared_const_buffer (std::move (buff)), [&write_completion, client] (boost::system::error_code const & ec, size_t size_a) mutable { + nano::shared_const_buffer (std::move (buff)), [&completed_writes, client] (boost::system::error_code const & ec, size_t size_a) mutable { client.reset (); - write_completion.increment (); + ++completed_writes; }, drop_policy); } }); - ASSERT_FALSE (write_completion.await_count_for (std::chrono::seconds (5))); + + ASSERT_TIMELY_EQ (5s, completed_writes, total_message_count); ASSERT_EQ (1, client.use_count ()); }; @@ -492,14 +476,16 @@ TEST (socket, concurrent_writes) constexpr size_t total_message_count = client_count * message_count; // We're expecting client_count*4 messages - nano::test::counted_completion read_count_completion (total_message_count); - std::function const &)> reader = [&read_count_completion, &total_message_count, &reader] (std::shared_ptr const & socket_a) { + std::atomic completed_reads{ 0 }; + + using reader_callback_t = std::function const &)>; + reader_callback_t reader = [&completed_reads, &total_message_count, &reader] (std::shared_ptr const & socket_a) { auto buff (std::make_shared> ()); buff->resize (1); - socket_a->async_read (buff, 1, [&read_count_completion, &reader, &total_message_count, socket_a, buff] (boost::system::error_code const & ec, size_t size_a) { + socket_a->async_read (buff, 1, [&completed_reads, &reader, &total_message_count, socket_a, buff] (boost::system::error_code const & ec, size_t size_a) { if (!ec) { - if (read_count_completion.increment () < total_message_count) + if (completed_reads++ < total_message_count) { reader (socket_a); } @@ -511,46 +497,53 @@ TEST (socket, concurrent_writes) }); }; + std::vector> connections; + auto server_port (system.get_available_port ()); boost::asio::ip::tcp::endpoint endpoint (boost::asio::ip::address_v6::any (), server_port); + boost::asio::ip::tcp::acceptor acceptor (node->io_ctx); + acceptor.open (endpoint.protocol ()); + acceptor.bind (endpoint); + acceptor.listen (boost::asio::socket_base::max_listen_connections); - std::vector> connections; - - auto listener = std::make_shared (server_port, *node, max_connections); - nano::test::stop_guard stop_guard{ *listener }; - listener->start ([&connections, &reader] (std::shared_ptr const & new_connection, boost::system::error_code const & ec_a) { - if (ec_a) + using accept_callback_t = std::function; + accept_callback_t accept_callback = [&] (boost::system::error_code const & ec, boost::asio::ip::tcp::socket socket) { + if (!ec) { - std::cerr << "on_connection: " << ec_a.message () << std::endl; + auto new_connection = std::make_shared (std::move (socket), socket.remote_endpoint (), socket.local_endpoint (), *node); + connections.push_back (new_connection); + reader (new_connection); + + acceptor.async_accept (accept_callback); } else { - connections.push_back (new_connection); - reader (new_connection); + std::cerr << "async_accept: " << ec.message () << std::endl; } - // Keep accepting connections - return true; - }); + }; + acceptor.async_accept (accept_callback); + + std::atomic completed_connections{ 0 }; - nano::test::counted_completion connection_count_completion (client_count); std::vector> clients; for (unsigned i = 0; i < client_count; i++) { auto client = std::make_shared (*node); clients.push_back (client); - client->async_connect (boost::asio::ip::tcp::endpoint (boost::asio::ip::address_v4::loopback (), listener->endpoint ().port ()), - [&connection_count_completion] (boost::system::error_code const & ec_a) { + client->async_connect (boost::asio::ip::tcp::endpoint (boost::asio::ip::address_v4::loopback (), acceptor.local_endpoint ().port ()), + [&completed_connections] (boost::system::error_code const & ec_a) { if (ec_a) { std::cerr << "async_connect: " << ec_a.message () << std::endl; } else { - connection_count_completion.increment (); + ++completed_connections; } }); } - ASSERT_FALSE (connection_count_completion.await_count_for (10s)); + + ASSERT_TIMELY_EQ (10s, completed_connections, client_count); // Execute overlapping writes from multiple threads auto client (clients[0]); @@ -567,15 +560,12 @@ TEST (socket, concurrent_writes) }); } - ASSERT_FALSE (read_count_completion.await_count_for (10s)); + ASSERT_TIMELY_EQ (10s, completed_reads, total_message_count); + node->stop (); runner.stop_event_processing (); runner.join (); - ASSERT_EQ (node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_success, nano::stat::dir::in), client_count); - // We may exhaust max connections and have some tcp accept failures, but no more than the client count - ASSERT_LT (node->stats.count (nano::stat::type::tcp, nano::stat::detail::accept_failure, nano::stat::dir::in), client_count); - for (auto & t : client_threads) { t.join (); From ef1af6b31c7d9678decdfedf568d24eab56ffbe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 22 Mar 2024 23:11:49 +0100 Subject: [PATCH 08/21] Move `is_temporary_error` function --- nano/node/transport/tcp_listener.cpp | 19 ------------------- nano/node/transport/transport.cpp | 20 ++++++++++++++++++++ nano/node/transport/transport.hpp | 2 ++ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index 3f0c4c9af1..e4fe4a3914 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -9,25 +9,6 @@ using namespace std::chrono_literals; -namespace -{ -bool is_temporary_error (boost::system::error_code const & ec_a) -{ - switch (ec_a.value ()) - { -#if EAGAIN != EWOULDBLOCK - case EAGAIN: -#endif - - case EWOULDBLOCK: - case EINTR: - return true; - default: - return false; - } -} -} - /* * tcp_listener */ diff --git a/nano/node/transport/transport.cpp b/nano/node/transport/transport.cpp index 4b216b7187..7522de853a 100644 --- a/nano/node/transport/transport.cpp +++ b/nano/node/transport/transport.cpp @@ -164,3 +164,23 @@ bool nano::transport::reserved_address (nano::endpoint const & endpoint_a, bool } return result; } + +bool nano::transport::is_temporary_error (boost::system::error_code const & ec) +{ + switch (ec.value ()) + { + case boost::asio::error::try_again: + case boost::asio::error::no_buffer_space: + case boost::asio::error::no_memory: + case boost::asio::error::connection_reset: + case boost::asio::error::connection_aborted: + case boost::asio::error::connection_refused: + case boost::asio::error::broken_pipe: + case boost::asio::error::operation_aborted: + case boost::asio::error::network_reset: + case boost::asio::error::interrupted: + return true; + default: + return false; + } +} \ No newline at end of file diff --git a/nano/node/transport/transport.hpp b/nano/node/transport/transport.hpp index 48a3cc48fb..523e9d5e37 100644 --- a/nano/node/transport/transport.hpp +++ b/nano/node/transport/transport.hpp @@ -24,4 +24,6 @@ bool is_same_subnetwork (boost::asio::ip::address const &, boost::asio::ip::addr // Unassigned, reserved, self bool reserved_address (nano::endpoint const &, bool = false); + +bool is_temporary_error (boost::system::error_code const &); } \ No newline at end of file From cdba6e3238875008804693961ba55c8f60fc33c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 22 Mar 2024 23:40:49 +0100 Subject: [PATCH 09/21] Remove public counters and replace with getters --- nano/core_test/node.cpp | 2 +- nano/node/transport/tcp_listener.cpp | 28 +++++++++++++++++++++++++++- nano/node/transport/tcp_listener.hpp | 8 +++----- nano/node/transport/tcp_server.cpp | 12 ++---------- nano/test_common/system.cpp | 8 ++++---- 5 files changed, 37 insertions(+), 21 deletions(-) diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 5435a66a11..8f64fbfe89 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -2820,7 +2820,7 @@ TEST (node, peers) node2->start (); ASSERT_TIMELY (10s, !node2->network.empty () && !node1->network.empty ()) // Wait to finish TCP node ID handshakes - ASSERT_TIMELY (10s, node1->tcp_listener->realtime_count != 0 && node2->tcp_listener->realtime_count != 0); + ASSERT_TIMELY (10s, node1->tcp_listener->realtime_count () != 0 && node2->tcp_listener->realtime_count () != 0); // Confirm that the peers match with the endpoints we are expecting ASSERT_EQ (1, node1->network.size ()); auto list1 (node1->network.list (2)); diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index e4fe4a3914..0e4c83aeda 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -300,12 +300,38 @@ auto nano::transport::tcp_listener::check_limits (boost::asio::ip::address const return accept_result::accepted; } -std::size_t nano::transport::tcp_listener::connection_count () const +size_t nano::transport::tcp_listener::connection_count () const { nano::lock_guard lock{ mutex }; return connections.size (); } +size_t nano::transport::tcp_listener::realtime_count () const +{ + nano::lock_guard lock{ mutex }; + + return std::count_if (connections.begin (), connections.end (), [] (auto const & connection) { + if (auto socket = connection.socket.lock ()) + { + return socket->is_realtime_connection (); + } + return false; + }); +} + +size_t nano::transport::tcp_listener::bootstrap_count () const +{ + nano::lock_guard lock{ mutex }; + + return std::count_if (connections.begin (), connections.end (), [] (auto const & connection) { + if (auto socket = connection.socket.lock ()) + { + return socket->is_bootstrap_connection (); + } + return false; + }); +} + size_t nano::transport::tcp_listener::count_per_ip (boost::asio::ip::address const & ip) const { debug_assert (!mutex.try_lock ()); diff --git a/nano/node/transport/tcp_listener.hpp b/nano/node/transport/tcp_listener.hpp index 7defb4a34f..33652b4b15 100644 --- a/nano/node/transport/tcp_listener.hpp +++ b/nano/node/transport/tcp_listener.hpp @@ -36,8 +36,10 @@ class tcp_listener final void start (std::function const &, boost::system::error_code const &)> callback = {}); void stop (); - std::size_t connection_count () const; nano::tcp_endpoint endpoint () const; + size_t connection_count () const; + size_t realtime_count () const; + size_t bootstrap_count () const; std::unique_ptr collect_container_info (std::string const & name); @@ -70,10 +72,6 @@ class tcp_listener final size_t count_per_ip (boost::asio::ip::address const & ip) const; size_t count_per_subnetwork (boost::asio::ip::address const & ip) const; -public: - std::atomic bootstrap_count{ 0 }; - std::atomic realtime_count{ 0 }; - private: struct entry { diff --git a/nano/node/transport/tcp_server.cpp b/nano/node/transport/tcp_server.cpp index 85c8f01b89..3cb650010b 100644 --- a/nano/node/transport/tcp_server.cpp +++ b/nano/node/transport/tcp_server.cpp @@ -38,14 +38,8 @@ nano::transport::tcp_server::~tcp_server () node->logger.debug (nano::log::type::tcp_server, "Exiting server: {}", fmt::streamed (remote_endpoint)); - if (socket->type () == nano::transport::socket_type::bootstrap) + if (socket->type () == nano::transport::socket_type::realtime) { - --node->tcp_listener->bootstrap_count; - } - else if (socket->type () == nano::transport::socket_type::realtime) - { - --node->tcp_listener->realtime_count; - // Clear temporary channel auto exisiting_response_channel (node->network.tcp_channels.find_channel (remote_endpoint)); if (exisiting_response_channel != nullptr) @@ -608,7 +602,7 @@ bool nano::transport::tcp_server::to_bootstrap_connection () { return false; } - if (node->tcp_listener->bootstrap_count >= node->config.bootstrap_connections_max) + if (node->tcp_listener->bootstrap_count () >= node->config.bootstrap_connections_max) { return false; } @@ -617,7 +611,6 @@ bool nano::transport::tcp_server::to_bootstrap_connection () return false; } - ++node->tcp_listener->bootstrap_count; socket->type_set (nano::transport::socket_type::bootstrap); node->logger.debug (nano::log::type::tcp_server, "Switched to bootstrap mode ({})", fmt::streamed (remote_endpoint)); @@ -642,7 +635,6 @@ bool nano::transport::tcp_server::to_realtime_connection (nano::account const & } remote_node_id = node_id; - ++node->tcp_listener->realtime_count; socket->type_set (nano::transport::socket_type::realtime); node->logger.debug (nano::log::type::tcp_server, "Switched to realtime mode ({})", fmt::streamed (remote_endpoint)); diff --git a/nano/test_common/system.cpp b/nano/test_common/system.cpp index df300b0ea9..2e8faff16f 100644 --- a/nano/test_common/system.cpp +++ b/nano/test_common/system.cpp @@ -131,8 +131,8 @@ std::shared_ptr nano::test::system::add_node (nano::node_config cons auto starting_size_1 = node1->network.size (); auto starting_size_2 = node2->network.size (); - auto starting_realtime_1 = node1->tcp_listener->realtime_count.load (); - auto starting_realtime_2 = node2->tcp_listener->realtime_count.load (); + auto starting_realtime_1 = node1->tcp_listener->realtime_count (); + auto starting_realtime_2 = node2->tcp_listener->realtime_count (); auto starting_keepalives_1 = node1->stats.count (stat::type::message, stat::detail::keepalive, stat::dir::in); auto starting_keepalives_2 = node2->stats.count (stat::type::message, stat::detail::keepalive, stat::dir::in); @@ -155,8 +155,8 @@ std::shared_ptr nano::test::system::add_node (nano::node_config cons { // Wait for initial connection finish auto ec = poll_until_true (5s, [&node1, &node2, starting_realtime_1, starting_realtime_2] () { - auto realtime_1 = node1->tcp_listener->realtime_count.load (); - auto realtime_2 = node2->tcp_listener->realtime_count.load (); + auto realtime_1 = node1->tcp_listener->realtime_count (); + auto realtime_2 = node2->tcp_listener->realtime_count (); return realtime_1 > starting_realtime_1 && realtime_2 > starting_realtime_2; }); debug_assert (!ec); From 7e02c0def8c77a62b726df5ea43a133a7da79b10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 22 Mar 2024 23:49:36 +0100 Subject: [PATCH 10/21] Store component as unique_ptr inside the node --- nano/core_test/network.cpp | 18 +++++++++--------- nano/core_test/node.cpp | 2 +- nano/node/node.cpp | 13 +++++++------ nano/node/node.hpp | 3 ++- nano/node/transport/tcp_server.cpp | 2 +- nano/test_common/system.cpp | 8 ++++---- 6 files changed, 24 insertions(+), 22 deletions(-) diff --git a/nano/core_test/network.cpp b/nano/core_test/network.cpp index 8fd4de95d6..55b5bbf3ff 100644 --- a/nano/core_test/network.cpp +++ b/nano/core_test/network.cpp @@ -65,7 +65,7 @@ TEST (network, construction_with_specified_port) auto const node = system.add_node (nano::node_config{ port }); EXPECT_EQ (port, node->network.port); EXPECT_EQ (port, node->network.endpoint ().port ()); - EXPECT_EQ (port, node->tcp_listener->endpoint ().port ()); + EXPECT_EQ (port, node->tcp_listener.endpoint ().port ()); } TEST (network, construction_without_specified_port) @@ -75,7 +75,7 @@ TEST (network, construction_without_specified_port) auto const port = node->network.port.load (); EXPECT_NE (0, port); EXPECT_EQ (port, node->network.endpoint ().port ()); - EXPECT_EQ (port, node->tcp_listener->endpoint ().port ()); + EXPECT_EQ (port, node->tcp_listener.endpoint ().port ()); } TEST (network, send_node_id_handshake_tcp) @@ -614,7 +614,7 @@ TEST (tcp_listener, tcp_node_id_handshake) { nano::test::system system (1); auto socket (std::make_shared (*system.nodes[0])); - auto bootstrap_endpoint (system.nodes[0]->tcp_listener->endpoint ()); + auto bootstrap_endpoint (system.nodes[0]->tcp_listener.endpoint ()); auto cookie (system.nodes[0]->network.syn_cookies.assign (nano::transport::map_tcp_to_endpoint (bootstrap_endpoint))); ASSERT_TRUE (cookie); nano::node_id_handshake::query_payload query{ *cookie }; @@ -653,7 +653,7 @@ TEST (tcp_listener, DISABLED_tcp_listener_timeout_empty) auto node0 (system.nodes[0]); auto socket (std::make_shared (*node0)); std::atomic connected (false); - socket->async_connect (node0->tcp_listener->endpoint (), [&connected] (boost::system::error_code const & ec) { + socket->async_connect (node0->tcp_listener.endpoint (), [&connected] (boost::system::error_code const & ec) { ASSERT_FALSE (ec); connected = true; }); @@ -662,7 +662,7 @@ TEST (tcp_listener, DISABLED_tcp_listener_timeout_empty) system.deadline_set (std::chrono::seconds (6)); while (!disconnected) { - disconnected = node0->tcp_listener->connection_count () == 0; + disconnected = node0->tcp_listener.connection_count () == 0; ASSERT_NO_ERROR (system.poll ()); } } @@ -672,24 +672,24 @@ TEST (tcp_listener, tcp_listener_timeout_node_id_handshake) nano::test::system system (1); auto node0 (system.nodes[0]); auto socket (std::make_shared (*node0)); - auto cookie (node0->network.syn_cookies.assign (nano::transport::map_tcp_to_endpoint (node0->tcp_listener->endpoint ()))); + auto cookie (node0->network.syn_cookies.assign (nano::transport::map_tcp_to_endpoint (node0->tcp_listener.endpoint ()))); ASSERT_TRUE (cookie); nano::node_id_handshake::query_payload query{ *cookie }; nano::node_id_handshake node_id_handshake{ nano::dev::network_params.network, query }; auto channel = std::make_shared (*node0, socket); - socket->async_connect (node0->tcp_listener->endpoint (), [&node_id_handshake, channel] (boost::system::error_code const & ec) { + socket->async_connect (node0->tcp_listener.endpoint (), [&node_id_handshake, channel] (boost::system::error_code const & ec) { ASSERT_FALSE (ec); channel->send (node_id_handshake, [] (boost::system::error_code const & ec, size_t size_a) { ASSERT_FALSE (ec); }); }); ASSERT_TIMELY (5s, node0->stats.count (nano::stat::type::tcp_server, nano::stat::detail::node_id_handshake) != 0); - ASSERT_EQ (node0->tcp_listener->connection_count (), 1); + ASSERT_EQ (node0->tcp_listener.connection_count (), 1); bool disconnected (false); system.deadline_set (std::chrono::seconds (20)); while (!disconnected) { - disconnected = node0->tcp_listener->connection_count () == 0; + disconnected = node0->tcp_listener.connection_count () == 0; ASSERT_NO_ERROR (system.poll ()); } } diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 8f64fbfe89..2e47186870 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -2820,7 +2820,7 @@ TEST (node, peers) node2->start (); ASSERT_TIMELY (10s, !node2->network.empty () && !node1->network.empty ()) // Wait to finish TCP node ID handshakes - ASSERT_TIMELY (10s, node1->tcp_listener->realtime_count () != 0 && node2->tcp_listener->realtime_count () != 0); + ASSERT_TIMELY (10s, node1->tcp_listener.realtime_count () != 0 && node2->tcp_listener.realtime_count () != 0); // Confirm that the peers match with the endpoints we are expecting ASSERT_EQ (1, node1->network.size ()); auto list1 (node1->network.list (2)); diff --git a/nano/node/node.cpp b/nano/node/node.cpp index fba8014c0a..11155e4984 100644 --- a/nano/node/node.cpp +++ b/nano/node/node.cpp @@ -168,7 +168,8 @@ nano::node::node (std::shared_ptr io_ctx_a, std::filesy // Thus, be very careful if you change the order: if `bootstrap` gets constructed before `network`, // the latter would inherit the port from the former (if TCP is active, otherwise `network` picks first) // - tcp_listener{ std::make_shared (network.port, *this, config.tcp_incoming_connections_max) }, + tcp_listener_impl{ std::make_unique (network.port, *this, config.tcp_incoming_connections_max) }, + tcp_listener{ *tcp_listener_impl }, application_path (application_path_a), port_mapping (*this), block_processor (*this, write_database_queue), @@ -557,7 +558,7 @@ std::unique_ptr nano::collect_container_info (no composite->add_component (collect_container_info (node.ledger, "ledger")); composite->add_component (collect_container_info (node.active, "active")); composite->add_component (collect_container_info (node.bootstrap_initiator, "bootstrap_initiator")); - composite->add_component (node.tcp_listener->collect_container_info ("tcp_listener")); + composite->add_component (node.tcp_listener.collect_container_info ("tcp_listener")); composite->add_component (collect_container_info (node.network, "network")); composite->add_component (node.telemetry.collect_container_info ("telemetry")); composite->add_component (collect_container_info (node.workers, "workers")); @@ -638,12 +639,12 @@ void nano::node::start () bool tcp_enabled = false; if (config.tcp_incoming_connections_max > 0 && !(flags.disable_bootstrap_listener && flags.disable_tcp_realtime)) { - tcp_listener->start (); + tcp_listener.start (); tcp_enabled = true; - if (network.port != tcp_listener->endpoint ().port ()) + if (network.port != tcp_listener.endpoint ().port ()) { - network.port = tcp_listener->endpoint ().port (); + network.port = tcp_listener.endpoint ().port (); } logger.info (nano::log::type::node, "Node peering port: {}", network.port.load ()); @@ -727,7 +728,7 @@ void nano::node::stop () websocket.stop (); bootstrap_server.stop (); bootstrap_initiator.stop (); - tcp_listener->stop (); + tcp_listener.stop (); port_mapping.stop (); wallets.stop (); stats.stop (); diff --git a/nano/node/node.hpp b/nano/node/node.hpp index 53ba1f69a9..01d385af2e 100644 --- a/nano/node/node.hpp +++ b/nano/node/node.hpp @@ -163,7 +163,8 @@ class node final : public std::enable_shared_from_this nano::telemetry telemetry; nano::bootstrap_initiator bootstrap_initiator; nano::bootstrap_server bootstrap_server; - std::shared_ptr tcp_listener; + std::unique_ptr tcp_listener_impl; + nano::transport::tcp_listener & tcp_listener; std::filesystem::path application_path; nano::node_observers observers; nano::port_mapping port_mapping; diff --git a/nano/node/transport/tcp_server.cpp b/nano/node/transport/tcp_server.cpp index 3cb650010b..a0dc394de4 100644 --- a/nano/node/transport/tcp_server.cpp +++ b/nano/node/transport/tcp_server.cpp @@ -602,7 +602,7 @@ bool nano::transport::tcp_server::to_bootstrap_connection () { return false; } - if (node->tcp_listener->bootstrap_count () >= node->config.bootstrap_connections_max) + if (node->tcp_listener.bootstrap_count () >= node->config.bootstrap_connections_max) { return false; } diff --git a/nano/test_common/system.cpp b/nano/test_common/system.cpp index 2e8faff16f..4af8f1b749 100644 --- a/nano/test_common/system.cpp +++ b/nano/test_common/system.cpp @@ -131,8 +131,8 @@ std::shared_ptr nano::test::system::add_node (nano::node_config cons auto starting_size_1 = node1->network.size (); auto starting_size_2 = node2->network.size (); - auto starting_realtime_1 = node1->tcp_listener->realtime_count (); - auto starting_realtime_2 = node2->tcp_listener->realtime_count (); + auto starting_realtime_1 = node1->tcp_listener.realtime_count (); + auto starting_realtime_2 = node2->tcp_listener.realtime_count (); auto starting_keepalives_1 = node1->stats.count (stat::type::message, stat::detail::keepalive, stat::dir::in); auto starting_keepalives_2 = node2->stats.count (stat::type::message, stat::detail::keepalive, stat::dir::in); @@ -155,8 +155,8 @@ std::shared_ptr nano::test::system::add_node (nano::node_config cons { // Wait for initial connection finish auto ec = poll_until_true (5s, [&node1, &node2, starting_realtime_1, starting_realtime_2] () { - auto realtime_1 = node1->tcp_listener->realtime_count (); - auto realtime_2 = node2->tcp_listener->realtime_count (); + auto realtime_1 = node1->tcp_listener.realtime_count (); + auto realtime_2 = node2->tcp_listener.realtime_count (); return realtime_1 > starting_realtime_1 && realtime_2 > starting_realtime_2; }); debug_assert (!ec); From 037c350a4a9627daa4cb92058e03c3442817d0dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:44:56 +0100 Subject: [PATCH 11/21] Move tests for network functions to a separate file --- nano/core_test/CMakeLists.txt | 1 + nano/core_test/network.cpp | 56 -------------------------- nano/core_test/network_functions.cpp | 59 ++++++++++++++++++++++++++++ nano/node/transport/transport.hpp | 2 +- 4 files changed, 61 insertions(+), 57 deletions(-) create mode 100644 nano/core_test/network_functions.cpp diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index b4111eead3..df6ef3ccd8 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -32,6 +32,7 @@ add_executable( memory_pool.cpp network.cpp network_filter.cpp + network_functions.cpp node.cpp object_stream.cpp optimistic_scheduler.cpp diff --git a/nano/core_test/network.cpp b/nano/core_test/network.cpp index 55b5bbf3ff..aaf475029d 100644 --- a/nano/core_test/network.cpp +++ b/nano/core_test/network.cpp @@ -540,62 +540,6 @@ TEST (network, endpoint_bad_fd) ASSERT_TIMELY_EQ (10s, system.nodes[0]->network.endpoint ().port (), 0); } -TEST (network, reserved_address) -{ - nano::test::system system (1); - // 0 port test - ASSERT_TRUE (nano::transport::reserved_address (nano::endpoint (boost::asio::ip::make_address_v6 ("2001::"), 0))); - // Valid address test - ASSERT_FALSE (nano::transport::reserved_address (nano::endpoint (boost::asio::ip::make_address_v6 ("2001::"), 1))); - nano::endpoint loopback (boost::asio::ip::make_address_v6 ("::1"), 1); - ASSERT_FALSE (nano::transport::reserved_address (loopback)); - nano::endpoint private_network_peer (boost::asio::ip::make_address_v6 ("::ffff:10.0.0.0"), 1); - ASSERT_TRUE (nano::transport::reserved_address (private_network_peer, false)); - ASSERT_FALSE (nano::transport::reserved_address (private_network_peer, true)); -} - -TEST (network, ipv6_bind_subnetwork) -{ - auto address1 (boost::asio::ip::make_address_v6 ("a41d:b7b2:8298:cf45:672e:bd1a:e7fb:f713")); - auto subnet1 (boost::asio::ip::make_network_v6 (address1, 48)); - ASSERT_EQ (boost::asio::ip::make_address_v6 ("a41d:b7b2:8298::"), subnet1.network ()); - auto address1_subnet (nano::transport::ipv4_address_or_ipv6_subnet (address1)); - ASSERT_EQ (subnet1.network (), address1_subnet); - // Ipv4 should return initial address - auto address2 (boost::asio::ip::make_address_v6 ("::ffff:192.168.1.1")); - auto address2_subnet (nano::transport::ipv4_address_or_ipv6_subnet (address2)); - ASSERT_EQ (address2, address2_subnet); -} - -TEST (network, network_range_ipv6) -{ - auto address1 (boost::asio::ip::make_address_v6 ("a41d:b7b2:8298:cf45:672e:bd1a:e7fb:f713")); - auto subnet1 (boost::asio::ip::make_network_v6 (address1, 58)); - ASSERT_EQ (boost::asio::ip::make_address_v6 ("a41d:b7b2:8298:cf40::"), subnet1.network ()); - auto address2 (boost::asio::ip::make_address_v6 ("520d:2402:3d:5e65:11:f8:7c54:3f")); - auto subnet2 (boost::asio::ip::make_network_v6 (address2, 33)); - ASSERT_EQ (boost::asio::ip::make_address_v6 ("520d:2402:0::"), subnet2.network ()); - // Default settings test - auto address3 (boost::asio::ip::make_address_v6 ("a719:0f12:536e:d88a:1331:ba53:4598:04e5")); - auto subnet3 (boost::asio::ip::make_network_v6 (address3, 32)); - ASSERT_EQ (boost::asio::ip::make_address_v6 ("a719:0f12::"), subnet3.network ()); - auto address3_subnet (nano::transport::map_address_to_subnetwork (address3)); - ASSERT_EQ (subnet3.network (), address3_subnet); -} - -TEST (network, network_range_ipv4) -{ - auto address1 (boost::asio::ip::make_address_v6 ("::ffff:192.168.1.1")); - auto subnet1 (boost::asio::ip::make_network_v6 (address1, 96 + 16)); - ASSERT_EQ (boost::asio::ip::make_address_v6 ("::ffff:192.168.0.0"), subnet1.network ()); - // Default settings test - auto address2 (boost::asio::ip::make_address_v6 ("::ffff:80.67.148.225")); - auto subnet2 (boost::asio::ip::make_network_v6 (address2, 96 + 24)); - ASSERT_EQ (boost::asio::ip::make_address_v6 ("::ffff:80.67.148.0"), subnet2.network ()); - auto address2_subnet (nano::transport::map_address_to_subnetwork (address2)); - ASSERT_EQ (subnet2.network (), address2_subnet); -} - TEST (node, port_mapping) { nano::test::system system (1); diff --git a/nano/core_test/network_functions.cpp b/nano/core_test/network_functions.cpp new file mode 100644 index 0000000000..d0a8ae5b22 --- /dev/null +++ b/nano/core_test/network_functions.cpp @@ -0,0 +1,59 @@ +#include +#include + +#include + +TEST (network_functions, reserved_address) +{ + // 0 port test + ASSERT_TRUE (nano::transport::reserved_address (nano::endpoint (boost::asio::ip::make_address_v6 ("2001::"), 0))); + // Valid address test + ASSERT_FALSE (nano::transport::reserved_address (nano::endpoint (boost::asio::ip::make_address_v6 ("2001::"), 1))); + nano::endpoint loopback (boost::asio::ip::make_address_v6 ("::1"), 1); + ASSERT_FALSE (nano::transport::reserved_address (loopback)); + nano::endpoint private_network_peer (boost::asio::ip::make_address_v6 ("::ffff:10.0.0.0"), 1); + ASSERT_TRUE (nano::transport::reserved_address (private_network_peer, false)); + ASSERT_FALSE (nano::transport::reserved_address (private_network_peer, true)); +} + +TEST (network_functions, ipv6_bind_subnetwork) +{ + auto address1 (boost::asio::ip::make_address_v6 ("a41d:b7b2:8298:cf45:672e:bd1a:e7fb:f713")); + auto subnet1 (boost::asio::ip::make_network_v6 (address1, 48)); + ASSERT_EQ (boost::asio::ip::make_address_v6 ("a41d:b7b2:8298::"), subnet1.network ()); + auto address1_subnet (nano::transport::ipv4_address_or_ipv6_subnet (address1)); + ASSERT_EQ (subnet1.network (), address1_subnet); + // Ipv4 should return initial address + auto address2 (boost::asio::ip::make_address_v6 ("::ffff:192.168.1.1")); + auto address2_subnet (nano::transport::ipv4_address_or_ipv6_subnet (address2)); + ASSERT_EQ (address2, address2_subnet); +} + +TEST (network_functions, network_range_ipv6) +{ + auto address1 (boost::asio::ip::make_address_v6 ("a41d:b7b2:8298:cf45:672e:bd1a:e7fb:f713")); + auto subnet1 (boost::asio::ip::make_network_v6 (address1, 58)); + ASSERT_EQ (boost::asio::ip::make_address_v6 ("a41d:b7b2:8298:cf40::"), subnet1.network ()); + auto address2 (boost::asio::ip::make_address_v6 ("520d:2402:3d:5e65:11:f8:7c54:3f")); + auto subnet2 (boost::asio::ip::make_network_v6 (address2, 33)); + ASSERT_EQ (boost::asio::ip::make_address_v6 ("520d:2402:0::"), subnet2.network ()); + // Default settings test + auto address3 (boost::asio::ip::make_address_v6 ("a719:0f12:536e:d88a:1331:ba53:4598:04e5")); + auto subnet3 (boost::asio::ip::make_network_v6 (address3, 32)); + ASSERT_EQ (boost::asio::ip::make_address_v6 ("a719:0f12::"), subnet3.network ()); + auto address3_subnet (nano::transport::map_address_to_subnetwork (address3)); + ASSERT_EQ (subnet3.network (), address3_subnet); +} + +TEST (network_functions, network_range_ipv4) +{ + auto address1 (boost::asio::ip::make_address_v6 ("::ffff:192.168.1.1")); + auto subnet1 (boost::asio::ip::make_network_v6 (address1, 96 + 16)); + ASSERT_EQ (boost::asio::ip::make_address_v6 ("::ffff:192.168.0.0"), subnet1.network ()); + // Default settings test + auto address2 (boost::asio::ip::make_address_v6 ("::ffff:80.67.148.225")); + auto subnet2 (boost::asio::ip::make_network_v6 (address2, 96 + 24)); + ASSERT_EQ (boost::asio::ip::make_address_v6 ("::ffff:80.67.148.0"), subnet2.network ()); + auto address2_subnet (nano::transport::map_address_to_subnetwork (address2)); + ASSERT_EQ (subnet2.network (), address2_subnet); +} diff --git a/nano/node/transport/transport.hpp b/nano/node/transport/transport.hpp index 523e9d5e37..88a225b64e 100644 --- a/nano/node/transport/transport.hpp +++ b/nano/node/transport/transport.hpp @@ -23,7 +23,7 @@ bool is_same_ip (boost::asio::ip::address const &, boost::asio::ip::address cons bool is_same_subnetwork (boost::asio::ip::address const &, boost::asio::ip::address const &); // Unassigned, reserved, self -bool reserved_address (nano::endpoint const &, bool = false); +bool reserved_address (nano::endpoint const &, bool allow_local_peers = false); bool is_temporary_error (boost::system::error_code const &); } \ No newline at end of file From b652bf016f3b545e11f7ffa5a2eb42073b5080ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:21:48 +0100 Subject: [PATCH 12/21] Add tests for ip address manipulation functions --- nano/core_test/network_functions.cpp | 85 ++++++++++++++++++++++++++++ nano/node/transport/transport.cpp | 11 ++-- nano/node/transport/transport.hpp | 4 +- 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/nano/core_test/network_functions.cpp b/nano/core_test/network_functions.cpp index d0a8ae5b22..ed45a6f06f 100644 --- a/nano/core_test/network_functions.cpp +++ b/nano/core_test/network_functions.cpp @@ -57,3 +57,88 @@ TEST (network_functions, network_range_ipv4) auto address2_subnet (nano::transport::map_address_to_subnetwork (address2)); ASSERT_EQ (subnet2.network (), address2_subnet); } + +TEST (network_functions, ipv4_address_or_ipv6_subnet) +{ + // IPv4 mapped as IPv6 address should return the original IPv4 address + boost::asio::ip::address addr1 = boost::asio::ip::address::from_string ("192.168.1.1"); + boost::asio::ip::address addr2 = boost::asio::ip::address::from_string ("::ffff:192.168.1.1"); + ASSERT_EQ (nano::transport::ipv4_address_or_ipv6_subnet (addr1), addr2); + + // IPv6 address within the same /48 subnet should return the network prefix + boost::asio::ip::address addr3 = boost::asio::ip::address::from_string ("2001:0db8:85a3:0000:0000:8a2e:0370:7334"); + boost::asio::ip::address addr4 = boost::asio::ip::address::from_string ("2001:0db8:85a3::"); + ASSERT_EQ (nano::transport::ipv4_address_or_ipv6_subnet (addr3), addr4); + + // Different IPv6 address outside the /48 subnet should not match + boost::asio::ip::address addr5 = boost::asio::ip::address::from_string ("2001:0db8:85a4:0001:0000:8a2e:0370:7334"); + ASSERT_NE (nano::transport::ipv4_address_or_ipv6_subnet (addr3), nano::transport::ipv4_address_or_ipv6_subnet (addr5)); +} + +TEST (network_functions, is_same_ip) +{ + // Same IPv4 addresses + boost::asio::ip::address ipv4_addr1 = boost::asio::ip::address::from_string ("192.168.1.1"); + ASSERT_TRUE (nano::transport::is_same_ip (ipv4_addr1, ipv4_addr1)); + + // IPv4 and its IPv6 mapped form + boost::asio::ip::address ipv6_mapped_ipv4 = boost::asio::ip::address::from_string ("::ffff:192.168.1.1"); + ASSERT_TRUE (nano::transport::is_same_ip (ipv4_addr1, ipv6_mapped_ipv4)); +} + +TEST (network_functions, is_same_ipv4) +{ + // Same IPv4 addresses + boost::asio::ip::address ipv4_addr1 = boost::asio::ip::address::from_string ("192.168.1.1"); + ASSERT_TRUE (nano::transport::is_same_ip (ipv4_addr1, ipv4_addr1)); + + // IPv4 and its IPv6 mapped form + boost::asio::ip::address ipv6_mapped_ipv4 = boost::asio::ip::address::from_string ("::ffff:192.168.1.1"); + ASSERT_TRUE (nano::transport::is_same_ip (ipv4_addr1, ipv6_mapped_ipv4)); +} + +TEST (network_functions, is_same_ipv6) +{ + // Two different IPv6 addresses within the same /48 subnet + boost::asio::ip::address ipv6_addr1 = boost::asio::ip::address::from_string ("2001:db8::1"); + boost::asio::ip::address ipv6_addr2 = boost::asio::ip::address::from_string ("2001:db8::2"); + ASSERT_TRUE (nano::transport::is_same_ip (ipv6_addr1, ipv6_addr2)); + + // Two different IPv6 addresses in different /48 subnets + boost::asio::ip::address ipv6_addr3 = boost::asio::ip::address::from_string ("2001:db8:1::1"); + ASSERT_FALSE (nano::transport::is_same_ip (ipv6_addr1, ipv6_addr3)); +} + +TEST (network_functions, is_different_ip_family) +{ + boost::asio::ip::address addr1 = boost::asio::ip::address::from_string ("192.168.1.1"); + boost::asio::ip::address addr2 = boost::asio::ip::address::from_string ("::1"); + ASSERT_FALSE (nano::transport::is_same_ip (addr1, addr2)); +} + +TEST (network_functions, is_same_ip_v4_mapped) +{ + boost::asio::ip::address addr1 = boost::asio::ip::address::from_string ("::ffff:192.168.1.1"); + boost::asio::ip::address addr2 = boost::asio::ip::address::from_string ("192.168.1.1"); + ASSERT_TRUE (nano::transport::is_same_ip (addr1, addr2)); + + boost::asio::ip::address addr3 = boost::asio::ip::address::from_string ("10.0.0.1"); + ASSERT_FALSE (nano::transport::is_same_ip (addr1, addr3)); +} + +TEST (network_functions, map_ipv4_address_to_subnetwork) +{ + boost::asio::ip::address addr = boost::asio::ip::address::from_string ("192.168.1.100"); + auto subnetwork = nano::transport::map_address_to_subnetwork (addr); + // Assuming a /24 subnet mask for IPv4, all addresses in 192.168.1.x should map to the same network + // Automatically maps to IPv6 + ASSERT_EQ (subnetwork.to_string (), "::ffff:192.168.1.0"); +} + +TEST (network_functions, map_ipv6_address_to_subnetwork) +{ + boost::asio::ip::address addr = boost::asio::ip::address::from_string ("2001:db8:abcd:0012::0"); + auto subnetwork = nano::transport::map_address_to_subnetwork (addr); + // Assuming a /32 subnet mask for IPv6 + ASSERT_EQ (subnetwork.to_string (), "2001:db8::"); +} \ No newline at end of file diff --git a/nano/node/transport/transport.cpp b/nano/node/transport/transport.cpp index 7522de853a..ba42166e1f 100644 --- a/nano/node/transport/transport.cpp +++ b/nano/node/transport/transport.cpp @@ -26,17 +26,18 @@ nano::tcp_endpoint nano::transport::map_endpoint_to_tcp (nano::endpoint const & return { endpoint_a.address (), endpoint_a.port () }; } -boost::asio::ip::address nano::transport::map_address_to_subnetwork (boost::asio::ip::address const & address_a) +boost::asio::ip::address nano::transport::map_address_to_subnetwork (boost::asio::ip::address address_a) { - debug_assert (address_a.is_v6 ()); + address_a = mapped_from_v4_or_v6 (address_a); static short const ipv6_subnet_prefix_length = 32; // Equivalent to network prefix /32. - static short const ipv4_subnet_prefix_length = (128 - 32) + 24; // Limits for /24 IPv4 subnetwork + static short const ipv4_subnet_prefix_length = (128 - 32) + 24; // Limits for /24 IPv4 subnetwork (we're using mapped IPv4 to IPv6 addresses, hence (128 - 32)) return address_a.to_v6 ().is_v4_mapped () ? boost::asio::ip::make_network_v6 (address_a.to_v6 (), ipv4_subnet_prefix_length).network () : boost::asio::ip::make_network_v6 (address_a.to_v6 (), ipv6_subnet_prefix_length).network (); } -boost::asio::ip::address nano::transport::ipv4_address_or_ipv6_subnet (boost::asio::ip::address const & address_a) +boost::asio::ip::address nano::transport::ipv4_address_or_ipv6_subnet (boost::asio::ip::address address_a) { - debug_assert (address_a.is_v6 ()); + address_a = mapped_from_v4_or_v6 (address_a); + // Assuming /48 subnet prefix for IPv6 as it's relatively easy to acquire such a /48 address range static short const ipv6_address_prefix_length = 48; // /48 IPv6 subnetwork return address_a.to_v6 ().is_v4_mapped () ? address_a : boost::asio::ip::make_network_v6 (address_a.to_v6 (), ipv6_address_prefix_length).network (); } diff --git a/nano/node/transport/transport.hpp b/nano/node/transport/transport.hpp index 88a225b64e..434f362c94 100644 --- a/nano/node/transport/transport.hpp +++ b/nano/node/transport/transport.hpp @@ -14,8 +14,8 @@ namespace nano::transport nano::endpoint map_endpoint_to_v6 (nano::endpoint const &); nano::endpoint map_tcp_to_endpoint (nano::tcp_endpoint const &); nano::tcp_endpoint map_endpoint_to_tcp (nano::endpoint const &); -boost::asio::ip::address map_address_to_subnetwork (boost::asio::ip::address const &); -boost::asio::ip::address ipv4_address_or_ipv6_subnet (boost::asio::ip::address const &); +boost::asio::ip::address map_address_to_subnetwork (boost::asio::ip::address); +boost::asio::ip::address ipv4_address_or_ipv6_subnet (boost::asio::ip::address); boost::asio::ip::address_v6 mapped_from_v4_bytes (unsigned long); boost::asio::ip::address_v6 mapped_from_v4_or_v6 (boost::asio::ip::address const &); bool is_ipv4_or_v4_mapped_address (boost::asio::ip::address const &); From 3f0ba6cc7ddb55888ce5cc256693caae766b26ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Tue, 26 Mar 2024 10:48:02 +0100 Subject: [PATCH 13/21] Improvements --- nano/core_test/socket.cpp | 2 +- nano/lib/stats_enums.hpp | 1 + nano/node/transport/socket.cpp | 6 ++--- nano/node/transport/socket.hpp | 5 ++-- nano/node/transport/tcp_listener.cpp | 40 +++++++++++++--------------- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/nano/core_test/socket.cpp b/nano/core_test/socket.cpp index 3e651e9bb4..9b65e25b48 100644 --- a/nano/core_test/socket.cpp +++ b/nano/core_test/socket.cpp @@ -510,7 +510,7 @@ TEST (socket, concurrent_writes) accept_callback_t accept_callback = [&] (boost::system::error_code const & ec, boost::asio::ip::tcp::socket socket) { if (!ec) { - auto new_connection = std::make_shared (std::move (socket), socket.remote_endpoint (), socket.local_endpoint (), *node); + auto new_connection = std::make_shared (*node, std::move (socket), socket.remote_endpoint (), socket.local_endpoint ()); connections.push_back (new_connection); reader (new_connection); diff --git a/nano/lib/stats_enums.hpp b/nano/lib/stats_enums.hpp index 2106f69cb2..9d1b6920ba 100644 --- a/nano/lib/stats_enums.hpp +++ b/nano/lib/stats_enums.hpp @@ -244,6 +244,7 @@ enum class detail : uint8_t max_per_ip, max_per_subnetwork, excluded, + erase_dead, // tcp_server handshake, diff --git a/nano/node/transport/socket.cpp b/nano/node/transport/socket.cpp index 0de3f1bcaa..0fae87d4fe 100644 --- a/nano/node/transport/socket.cpp +++ b/nano/node/transport/socket.cpp @@ -20,15 +20,15 @@ */ nano::transport::socket::socket (nano::node & node_a, nano::transport::socket_endpoint endpoint_type_a, std::size_t max_queue_size_a) : - socket{ boost::asio::ip::tcp::socket{ node_a.io_ctx }, {}, {}, node_a, endpoint_type_a, max_queue_size_a } + socket{ node_a, boost::asio::ip::tcp::socket{ node_a.io_ctx }, {}, {}, endpoint_type_a, max_queue_size_a } { } -nano::transport::socket::socket (boost::asio::ip::tcp::socket boost_socket_a, boost::asio::ip::tcp::endpoint remote_endpoint_a, boost::asio::ip::tcp::endpoint local_endpoint_a, nano::node & node_a, nano::transport::socket_endpoint endpoint_type_a, std::size_t max_queue_size_a) : +nano::transport::socket::socket (nano::node & node_a, boost::asio::ip::tcp::socket boost_socket_a, boost::asio::ip::tcp::endpoint remote_endpoint_a, boost::asio::ip::tcp::endpoint local_endpoint_a, nano::transport::socket_endpoint endpoint_type_a, std::size_t max_queue_size_a) : send_queue{ max_queue_size_a }, + node{ node_a }, strand{ node_a.io_ctx.get_executor () }, tcp_socket{ std::move (boost_socket_a) }, - node{ node_a }, remote{ remote_endpoint_a }, local{ local_endpoint_a }, endpoint_type_m{ endpoint_type_a }, diff --git a/nano/node/transport/socket.hpp b/nano/node/transport/socket.hpp index 706f93dd93..5c558dea83 100644 --- a/nano/node/transport/socket.hpp +++ b/nano/node/transport/socket.hpp @@ -72,10 +72,10 @@ class socket final : public std::enable_shared_from_this // TODO: Accepting remote/local endpoints as a parameter is unnecessary, but is needed for now to keep compatibility with the legacy code explicit socket ( + nano::node &, boost::asio::ip::tcp::socket, boost::asio::ip::tcp::endpoint remote_endpoint, boost::asio::ip::tcp::endpoint local_endpoint, - nano::node &, nano::transport::socket_endpoint = socket_endpoint::server, std::size_t max_queue_size = default_max_queue_size); @@ -172,9 +172,10 @@ class socket final : public std::enable_shared_from_this write_queue send_queue; protected: + nano::node & node; + boost::asio::strand strand; boost::asio::ip::tcp::socket tcp_socket; - nano::node & node; /** The other end of the connection */ boost::asio::ip::tcp::endpoint remote; diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index 0e4c83aeda..04420a7f78 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -137,6 +137,7 @@ void nano::transport::tcp_listener::cleanup () erase_if (connections, [this] (auto const & connection) { if (connection.socket.expired () && connection.server.expired ()) { + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::erase_dead); logger.debug (nano::log::type::tcp_listener, "Evicting dead connection: {}", fmt::streamed (connection.endpoint)); return true; } @@ -187,8 +188,8 @@ void nano::transport::tcp_listener::run () } if (!stopped) { - debug_assert (false, "acceptor stopped unexpectedly"); logger.error (nano::log::type::tcp_listener, "Acceptor stopped unexpectedly"); + debug_assert (false, "acceptor stopped unexpectedly"); } } @@ -211,8 +212,8 @@ auto nano::transport::tcp_listener::accept_one () -> accept_result } catch (boost::system::system_error const & ex) { - logger.debug (nano::log::type::tcp_listener, "Error while closing socket after refusing connection: {}", ex.what ()); stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::close_error, nano::stat::dir::in); + logger.debug (nano::log::type::tcp_listener, "Error while closing socket after refusing connection: {}", ex.what ()); } return result; @@ -221,7 +222,7 @@ auto nano::transport::tcp_listener::accept_one () -> accept_result stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::accept_success, nano::stat::dir::in); logger.debug (nano::log::type::tcp_listener, "Accepted incoming connection from: {}", fmt::streamed (remote_endpoint)); - auto socket = std::make_shared (std::move (raw_socket), remote_endpoint, local_endpoint, node, socket_endpoint::server); + auto socket = std::make_shared (node, std::move (raw_socket), remote_endpoint, local_endpoint, socket_endpoint::server); auto server = std::make_shared (socket, node.shared (), true); { @@ -240,13 +241,8 @@ auto nano::transport::tcp_listener::accept_one () -> accept_result void nano::transport::tcp_listener::wait_available_slots () { - auto should_wait = [this] { - nano::lock_guard lock{ mutex }; - return connections.size () >= max_inbound_connections; - }; - nano::interval log_interval; - while (!stopped && should_wait ()) + while (connection_count () >= max_inbound_connections && !stopped) { if (log_interval.elapsed (node.network_params.network.is_dev_network () ? 1s : 15s)) { @@ -266,12 +262,21 @@ auto nano::transport::tcp_listener::check_limits (boost::asio::ip::address const debug_assert (connections.size () <= max_inbound_connections); // Should be checked earlier (wait_available_slots) + if (node.network.excluded_peers.check (ip)) // true => error + { + stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::excluded, nano::stat::dir::in); + logger.debug (nano::log::type::tcp_listener, "Rejected connection from excluded peer: {}", ip.to_string ()); + + return accept_result::excluded; + } + if (!node.flags.disable_max_peers_per_ip) { - if (count_per_ip (ip) >= node.network_params.network.max_peers_per_ip) + if (auto count = count_per_ip (ip); count >= node.network_params.network.max_peers_per_ip) { stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::max_per_ip, nano::stat::dir::in); - logger.debug (nano::log::type::tcp_listener, "Max connections per IP reached ({}), unable to open new connection", ip.to_string ()); + logger.debug (nano::log::type::tcp_listener, "Max connections per IP reached (ip: {}, count: {}), unable to open new connection", + ip.to_string (), count); return accept_result::too_many_per_ip; } @@ -280,23 +285,16 @@ auto nano::transport::tcp_listener::check_limits (boost::asio::ip::address const // If the address is IPv4 we don't check for a network limit, since its address space isn't big as IPv6/64. if (!node.flags.disable_max_peers_per_subnetwork && !nano::transport::is_ipv4_or_v4_mapped_address (ip)) { - if (count_per_subnetwork (ip) >= node.network_params.network.max_peers_per_subnetwork) + if (auto count = count_per_subnetwork (ip); count >= node.network_params.network.max_peers_per_subnetwork) { stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::max_per_subnetwork, nano::stat::dir::in); - logger.debug (nano::log::type::tcp_listener, "Max connections per subnetwork reached ({}), unable to open new connection", ip.to_string ()); + logger.debug (nano::log::type::tcp_listener, "Max connections per subnetwork reached (ip: {}, count: {}), unable to open new connection", + ip.to_string (), count); return accept_result::too_many_per_subnetwork; } } - if (node.network.excluded_peers.check (ip)) // true => error - { - stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::excluded, nano::stat::dir::in); - logger.debug (nano::log::type::tcp_listener, "Rejected connection from excluded peer: {}", ip.to_string ()); - - return accept_result::excluded; - } - return accept_result::accepted; } From 4bc45458bae94b8f60d73613becfd5f2a7e05671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Wed, 27 Mar 2024 10:37:53 +0100 Subject: [PATCH 14/21] Fix socket tests --- nano/core_test/socket.cpp | 98 ++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/nano/core_test/socket.cpp b/nano/core_test/socket.cpp index 9b65e25b48..51501e3cf1 100644 --- a/nano/core_test/socket.cpp +++ b/nano/core_test/socket.cpp @@ -20,21 +20,27 @@ using namespace std::chrono_literals; TEST (socket, max_connections) { nano::test::system system; - auto node = system.add_node (); + + auto node_flags = nano::inactive_node_flag_defaults (); + node_flags.read_only = false; + nano::inactive_node inactivenode (nano::unique_path (), node_flags); + auto node = inactivenode.node; + + nano::thread_runner runner{ node->io_ctx_shared }; + auto server_port = system.get_available_port (); // successful incoming connections are stored in server_sockets to keep them alive (server side) std::vector> server_sockets; // start a server socket that allows max 2 live connections - auto listener = std::make_shared (server_port, *node, 2); - listener->connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { + nano::transport::tcp_listener listener{ server_port, *node, 2 }; + listener.connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { server_sockets.push_back (socket); }); - nano::test::stop_guard stop_guard{ *listener }; - listener->start (); + nano::test::start_stop_guard stop_guard{ listener }; - boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; + boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener.endpoint ().port () }; // client side connection tracking std::atomic connection_attempts = 0; @@ -107,9 +113,14 @@ TEST (socket, max_connections_per_ip) { nano::test::system system; - auto node = system.add_node (); + auto node_flags = nano::inactive_node_flag_defaults (); + node_flags.read_only = false; + nano::inactive_node inactivenode (nano::unique_path (), node_flags); + auto node = inactivenode.node; ASSERT_FALSE (node->flags.disable_max_peers_per_ip); + nano::thread_runner runner{ node->io_ctx_shared }; + auto server_port = system.get_available_port (); const auto max_ip_connections = node->network_params.network.max_peers_per_ip; @@ -120,14 +131,13 @@ TEST (socket, max_connections_per_ip) // successful incoming connections are stored in server_sockets to keep them alive (server side) std::vector> server_sockets; - auto listener = std::make_shared (server_port, *node, max_global_connections); - listener->connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { + nano::transport::tcp_listener listener{ server_port, *node, max_global_connections }; + listener.connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { server_sockets.push_back (socket); }); - nano::test::stop_guard stop_guard{ *listener }; - listener->start (); + nano::test::start_stop_guard stop_guard{ listener }; - boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; + boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener.endpoint ().port () }; // client side connection tracking std::atomic connection_attempts = 0; @@ -220,14 +230,19 @@ TEST (socket, max_connections_per_subnetwork) { nano::test::system system; - nano::node_flags node_flags; + auto node_flags = nano::inactive_node_flag_defaults (); + node_flags.read_only = false; // disabling IP limit because it will be used the same IP address to check they come from the same subnetwork. node_flags.disable_max_peers_per_ip = true; node_flags.disable_max_peers_per_subnetwork = false; - auto node = system.add_node (node_flags); + nano::inactive_node inactivenode (nano::unique_path (), node_flags); + auto node = inactivenode.node; + ASSERT_TRUE (node->flags.disable_max_peers_per_ip); ASSERT_FALSE (node->flags.disable_max_peers_per_subnetwork); + nano::thread_runner runner{ node->io_ctx_shared }; + auto server_port = system.get_available_port (); boost::asio::ip::tcp::endpoint listen_endpoint{ boost::asio::ip::address_v6::any (), server_port }; @@ -239,14 +254,13 @@ TEST (socket, max_connections_per_subnetwork) // successful incoming connections are stored in server_sockets to keep them alive (server side) std::vector> server_sockets; - auto listener = std::make_shared (server_port, *node, max_global_connections); - listener->connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { + nano::transport::tcp_listener listener{ server_port, *node, max_global_connections }; + listener.connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { server_sockets.push_back (socket); }); - nano::test::stop_guard stop_guard{ *listener }; - listener->start (); + nano::test::start_stop_guard stop_guard{ listener }; - boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; + boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener.endpoint ().port () }; // client side connection tracking std::atomic connection_attempts = 0; @@ -283,11 +297,16 @@ TEST (socket, disabled_max_peers_per_ip) { nano::test::system system; - nano::node_flags node_flags; + auto node_flags = nano::inactive_node_flag_defaults (); + node_flags.read_only = false; node_flags.disable_max_peers_per_ip = true; - auto node = system.add_node (node_flags); + nano::inactive_node inactivenode (nano::unique_path (), node_flags); + auto node = inactivenode.node; + ASSERT_TRUE (node->flags.disable_max_peers_per_ip); + nano::thread_runner runner{ node->io_ctx_shared }; + auto server_port = system.get_available_port (); const auto max_ip_connections = node->network_params.network.max_peers_per_ip; @@ -298,14 +317,13 @@ TEST (socket, disabled_max_peers_per_ip) // successful incoming connections are stored in server_sockets to keep them alive (server side) std::vector> server_sockets; - auto listener = std::make_shared (server_port, *node, max_global_connections); - listener->connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { + nano::transport::tcp_listener listener = { server_port, *node, max_global_connections }; + listener.connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { server_sockets.push_back (socket); }); - nano::test::stop_guard stop_guard{ *listener }; - listener->start (); + nano::test::start_stop_guard stop_guard{ listener }; - boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; + boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener.endpoint ().port () }; // client side connection tracking std::atomic connection_attempts = 0; @@ -348,23 +366,15 @@ TEST (socket, disconnection_of_silent_connections) config.network_params.network.idle_timeout = std::chrono::seconds::max (); // Silent connections are connections open by external peers that don't contribute with any data. config.network_params.network.silent_connection_tolerance_time = std::chrono::seconds{ 5 }; - auto node = system.add_node (config); - auto server_port = system.get_available_port (); - - // on a connection, a server data socket is created. The shared pointer guarantees the object's lifecycle until the end of this test. + // On a connection, a server data socket is created. The shared pointer guarantees the object's lifecycle until the end of this test. std::shared_ptr server_data_socket; - - // start a server listening socket - auto listener = std::make_shared (server_port, *node, 1); - listener->connection_accepted.add ([&server_data_socket] (auto const & socket, auto const & server) { + node->tcp_listener.connection_accepted.add ([&server_data_socket] (auto const & socket, auto const & server) { server_data_socket = socket; }); - nano::test::stop_guard stop_guard{ *listener }; - listener->start (); - boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), listener->endpoint ().port () }; + boost::asio::ip::tcp::endpoint dst_endpoint{ boost::asio::ip::address_v6::loopback (), node->tcp_listener.endpoint ().port () }; // Instantiates a client to simulate an incoming connection. auto client_socket = std::make_shared (*node); @@ -379,16 +389,10 @@ TEST (socket, disconnection_of_silent_connections) ASSERT_TIMELY (10s, server_data_socket != nullptr); ASSERT_TIMELY (10s, server_data_socket->is_closed ()); - auto get_tcp_io_timeout_drops = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_io_timeout_drop, nano::stat::dir::in); - }; - auto get_tcp_silent_connection_drops = [&node] () { - return node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_silent_connection_drop, nano::stat::dir::in); - }; // Just to ensure the disconnection wasn't due to the timer timeout. - ASSERT_EQ (0, get_tcp_io_timeout_drops ()); + ASSERT_EQ (0, node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_io_timeout_drop, nano::stat::dir::in)); // Asserts the silent checker worked. - ASSERT_EQ (1, get_tcp_silent_connection_drops ()); + ASSERT_EQ (1, node->stats.count (nano::stat::type::tcp, nano::stat::detail::tcp_silent_connection_drop, nano::stat::dir::in)); } TEST (socket, drop_policy) @@ -400,7 +404,7 @@ TEST (socket, drop_policy) nano::inactive_node inactivenode (nano::unique_path (), node_flags); auto node = inactivenode.node; - nano::thread_runner runner (node->io_ctx_shared, 1); + nano::thread_runner runner{ node->io_ctx_shared }; std::vector> connections; @@ -468,7 +472,7 @@ TEST (socket, concurrent_writes) // This gives more realistic execution than using system#poll, allowing writes to // queue up and drain concurrently. - nano::thread_runner runner (node->io_ctx_shared, 1); + nano::thread_runner runner{ node->io_ctx_shared }; constexpr size_t max_connections = 4; constexpr size_t client_count = max_connections; From 3f9a2c5d335eab8443ecbe1e21ca47b4fabdd82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:13:39 +0100 Subject: [PATCH 15/21] Use async accept --- nano/node/transport/tcp_listener.cpp | 24 +++++++++++++++++++++--- nano/node/transport/tcp_listener.hpp | 2 ++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index 04420a7f78..00136bcb63 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include using namespace std::chrono_literals; @@ -86,11 +88,16 @@ void nano::transport::tcp_listener::stop () { nano::lock_guard lock{ mutex }; stopped = true; + + boost::system::error_code ec; + acceptor.close (ec); // Best effort to close the acceptor, ignore errors + if (ec) + { + logger.error (nano::log::type::tcp_listener, "Error while closing acceptor: {}", ec.message ()); + } } condition.notify_all (); - acceptor.close (); - if (thread.joinable ()) { thread.join (); @@ -193,9 +200,20 @@ void nano::transport::tcp_listener::run () } } +boost::asio::ip::tcp::socket nano::transport::tcp_listener::accept_socket () +{ + std::future future; + { + nano::unique_lock lock{ mutex }; + future = acceptor.async_accept (boost::asio::use_future); + } + future.wait (); + return future.get (); +} + auto nano::transport::tcp_listener::accept_one () -> accept_result { - auto raw_socket = acceptor.accept (); + auto raw_socket = accept_socket (); auto const remote_endpoint = raw_socket.remote_endpoint (); auto const local_endpoint = raw_socket.local_endpoint (); diff --git a/nano/node/transport/tcp_listener.hpp b/nano/node/transport/tcp_listener.hpp index 33652b4b15..4111992842 100644 --- a/nano/node/transport/tcp_listener.hpp +++ b/nano/node/transport/tcp_listener.hpp @@ -69,6 +69,8 @@ class tcp_listener final accept_result accept_one (); accept_result check_limits (boost::asio::ip::address const & ip); + boost::asio::ip::tcp::socket accept_socket (); + size_t count_per_ip (boost::asio::ip::address const & ip) const; size_t count_per_subnetwork (boost::asio::ip::address const & ip) const; From 7b65d9f75ce9390fe91a26ec7bb52ef41b4ea977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 1 Apr 2024 10:04:40 +0200 Subject: [PATCH 16/21] Use `asio` namespace alias --- nano/node/transport/tcp_listener.cpp | 26 +++++++++++++------------- nano/node/transport/tcp_listener.hpp | 19 ++++++++++--------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index 00136bcb63..9ac30e1f3a 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -22,7 +22,7 @@ nano::transport::tcp_listener::tcp_listener (uint16_t port_a, nano::node & node_ port{ port_a }, max_inbound_connections{ max_inbound_connections }, acceptor{ node_a.io_ctx }, - local{ boost::asio::ip::tcp::endpoint{ boost::asio::ip::address_v6::any (), port_a } } + local{ asio::ip::tcp::endpoint{ asio::ip::address_v6::any (), port_a } } { connection_accepted.add ([this] (auto const & socket, auto const & server) { node.observers.socket_accepted.notify (*socket); @@ -43,9 +43,9 @@ void nano::transport::tcp_listener::start (std::function future; + std::future future; { nano::unique_lock lock{ mutex }; - future = acceptor.async_accept (boost::asio::use_future); + future = acceptor.async_accept (asio::use_future); } future.wait (); return future.get (); @@ -225,7 +225,7 @@ auto nano::transport::tcp_listener::accept_one () -> accept_result try { // Best effor attempt to gracefully close the socket, shutdown before closing to avoid zombie sockets - raw_socket.shutdown (boost::asio::ip::tcp::socket::shutdown_both); + raw_socket.shutdown (asio::ip::tcp::socket::shutdown_both); raw_socket.close (); } catch (boost::system::system_error const & ex) @@ -272,7 +272,7 @@ void nano::transport::tcp_listener::wait_available_slots () } } -auto nano::transport::tcp_listener::check_limits (boost::asio::ip::address const & ip) -> accept_result +auto nano::transport::tcp_listener::check_limits (asio::ip::address const & ip) -> accept_result { nano::lock_guard lock{ mutex }; @@ -348,7 +348,7 @@ size_t nano::transport::tcp_listener::bootstrap_count () const }); } -size_t nano::transport::tcp_listener::count_per_ip (boost::asio::ip::address const & ip) const +size_t nano::transport::tcp_listener::count_per_ip (asio::ip::address const & ip) const { debug_assert (!mutex.try_lock ()); @@ -357,7 +357,7 @@ size_t nano::transport::tcp_listener::count_per_ip (boost::asio::ip::address con }); } -size_t nano::transport::tcp_listener::count_per_subnetwork (boost::asio::ip::address const & ip) const +size_t nano::transport::tcp_listener::count_per_subnetwork (asio::ip::address const & ip) const { debug_assert (!mutex.try_lock ()); @@ -366,15 +366,15 @@ size_t nano::transport::tcp_listener::count_per_subnetwork (boost::asio::ip::add }); } -boost::asio::ip::tcp::endpoint nano::transport::tcp_listener::endpoint () const +asio::ip::tcp::endpoint nano::transport::tcp_listener::endpoint () const { if (!stopped) { - return { boost::asio::ip::address_v6::loopback (), acceptor.local_endpoint ().port () }; + return { asio::ip::address_v6::loopback (), acceptor.local_endpoint ().port () }; } else { - return { boost::asio::ip::address_v6::loopback (), 0 }; + return { asio::ip::address_v6::loopback (), 0 }; } } diff --git a/nano/node/transport/tcp_listener.hpp b/nano/node/transport/tcp_listener.hpp index 4111992842..abcf2bb207 100644 --- a/nano/node/transport/tcp_listener.hpp +++ b/nano/node/transport/tcp_listener.hpp @@ -11,6 +11,7 @@ #include namespace mi = boost::multi_index; +namespace asio = boost::asio; namespace nano { @@ -68,20 +69,20 @@ class tcp_listener final }; accept_result accept_one (); - accept_result check_limits (boost::asio::ip::address const & ip); - boost::asio::ip::tcp::socket accept_socket (); + accept_result check_limits (asio::ip::address const & ip); + asio::ip::tcp::socket accept_socket (); - size_t count_per_ip (boost::asio::ip::address const & ip) const; - size_t count_per_subnetwork (boost::asio::ip::address const & ip) const; + size_t count_per_ip (asio::ip::address const & ip) const; + size_t count_per_subnetwork (asio::ip::address const & ip) const; private: struct entry { - boost::asio::ip::tcp::endpoint endpoint; + asio::ip::tcp::endpoint endpoint; std::weak_ptr socket; std::weak_ptr server; - boost::asio::ip::address address () const + asio::ip::address address () const { return endpoint.address (); } @@ -97,13 +98,13 @@ class tcp_listener final using ordered_connections = boost::multi_index_container, - mi::const_mem_fun> + mi::const_mem_fun> >>; // clang-format on ordered_connections connections; - boost::asio::ip::tcp::acceptor acceptor; - boost::asio::ip::tcp::endpoint local; + asio::ip::tcp::acceptor acceptor; + asio::ip::tcp::endpoint local; std::atomic stopped; nano::condition_variable condition; From 89196fbd00371e45bd97de78b5ff60f73d4ad756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 1 Apr 2024 10:24:20 +0200 Subject: [PATCH 17/21] Locking --- nano/node/transport/tcp_listener.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index 9ac30e1f3a..a0144069c5 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -370,6 +370,7 @@ asio::ip::tcp::endpoint nano::transport::tcp_listener::endpoint () const { if (!stopped) { + nano::lock_guard lock{ mutex }; return { asio::ip::address_v6::loopback (), acceptor.local_endpoint ().port () }; } else From 75fce7e39947f6a5a4fce505fa1c6c1d2e7797d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Mon, 1 Apr 2024 10:49:19 +0200 Subject: [PATCH 18/21] Unused callback --- nano/node/transport/tcp_listener.cpp | 4 ++-- nano/node/transport/tcp_listener.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index a0144069c5..4d419dcb0f 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -35,7 +35,7 @@ nano::transport::tcp_listener::~tcp_listener () debug_assert (!thread.joinable ()); } -void nano::transport::tcp_listener::start (std::function const &, boost::system::error_code const &)> callback_a) +void nano::transport::tcp_listener::start () { debug_assert (!thread.joinable ()); debug_assert (!cleanup_thread.joinable ()); @@ -56,7 +56,7 @@ void nano::transport::tcp_listener::start (std::function const &, boost::system::error_code const &)> callback = {}); + void start (); void stop (); nano::tcp_endpoint endpoint () const; From 406422345d227f6e18adf7f2760dea4d4c35e679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Tue, 9 Apr 2024 10:17:25 +0200 Subject: [PATCH 19/21] Fixes --- nano/core_test/node.cpp | 2 +- nano/node/transport/tcp_listener.cpp | 16 +++++++--------- nano/node/transport/transport.cpp | 20 -------------------- nano/node/transport/transport.hpp | 2 -- 4 files changed, 8 insertions(+), 32 deletions(-) diff --git a/nano/core_test/node.cpp b/nano/core_test/node.cpp index 2e47186870..388508983b 100644 --- a/nano/core_test/node.cpp +++ b/nano/core_test/node.cpp @@ -39,8 +39,8 @@ TEST (node, null_account) TEST (node, stop) { nano::test::system system (1); - system.io_guard.reset (); ASSERT_NE (system.nodes[0]->wallets.items.end (), system.nodes[0]->wallets.items.begin ()); + system.stop_node (*system.nodes[0]); ASSERT_TRUE (true); } diff --git a/nano/node/transport/tcp_listener.cpp b/nano/node/transport/tcp_listener.cpp index 4d419dcb0f..7cb676fb73 100644 --- a/nano/node/transport/tcp_listener.cpp +++ b/nano/node/transport/tcp_listener.cpp @@ -84,7 +84,9 @@ void nano::transport::tcp_listener::start () void nano::transport::tcp_listener::stop () { - logger.info (nano::log::type::tcp_listener, "Stopping listeninig for incoming connections and closing all sockets..."); + debug_assert (!stopped); + + logger.info (nano::log::type::tcp_listener, "Stopping listening for incoming connections and closing all sockets..."); { nano::lock_guard lock{ mutex }; stopped = true; @@ -169,7 +171,6 @@ void nano::transport::tcp_listener::run () return; } - bool cooldown = false; try { auto result = accept_one (); @@ -182,16 +183,13 @@ void nano::transport::tcp_listener::run () catch (boost::system::system_error const & ex) { stats.inc (nano::stat::type::tcp_listener, nano::stat::detail::accept_error, nano::stat::dir::in); - logger.log (stopped ? nano::log::level::debug : nano::log::level::error, // Avoid logging expected errors when stopping - nano::log::type::tcp_listener, "Error accepting incoming connection: {}", ex.what ()); - - cooldown = true; + logger.log (nano::log::level::debug, nano::log::type::tcp_listener, "Error accepting incoming connection: {}", ex.what ()); } lock.lock (); - // Sleep for a while to prevent busy loop with additional cooldown if an error occurred - condition.wait_for (lock, cooldown ? 100ms : 10ms, [this] () { return stopped.load (); }); + // Sleep for a while to prevent busy loop + condition.wait_for (lock, 10ms, [this] () { return stopped.load (); }); } if (!stopped) { @@ -368,9 +366,9 @@ size_t nano::transport::tcp_listener::count_per_subnetwork (asio::ip::address co asio::ip::tcp::endpoint nano::transport::tcp_listener::endpoint () const { + nano::lock_guard lock{ mutex }; if (!stopped) { - nano::lock_guard lock{ mutex }; return { asio::ip::address_v6::loopback (), acceptor.local_endpoint ().port () }; } else diff --git a/nano/node/transport/transport.cpp b/nano/node/transport/transport.cpp index ba42166e1f..4a90626f9e 100644 --- a/nano/node/transport/transport.cpp +++ b/nano/node/transport/transport.cpp @@ -164,24 +164,4 @@ bool nano::transport::reserved_address (nano::endpoint const & endpoint_a, bool } } return result; -} - -bool nano::transport::is_temporary_error (boost::system::error_code const & ec) -{ - switch (ec.value ()) - { - case boost::asio::error::try_again: - case boost::asio::error::no_buffer_space: - case boost::asio::error::no_memory: - case boost::asio::error::connection_reset: - case boost::asio::error::connection_aborted: - case boost::asio::error::connection_refused: - case boost::asio::error::broken_pipe: - case boost::asio::error::operation_aborted: - case boost::asio::error::network_reset: - case boost::asio::error::interrupted: - return true; - default: - return false; - } } \ No newline at end of file diff --git a/nano/node/transport/transport.hpp b/nano/node/transport/transport.hpp index 434f362c94..efeb6260eb 100644 --- a/nano/node/transport/transport.hpp +++ b/nano/node/transport/transport.hpp @@ -24,6 +24,4 @@ bool is_same_subnetwork (boost::asio::ip::address const &, boost::asio::ip::addr // Unassigned, reserved, self bool reserved_address (nano::endpoint const &, bool allow_local_peers = false); - -bool is_temporary_error (boost::system::error_code const &); } \ No newline at end of file From e1c9c978b8423bb5d51b128f3f5a041a347e3b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Fri, 19 Apr 2024 17:54:56 +0200 Subject: [PATCH 20/21] Use single threaded io context in tests --- nano/core_test/socket.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nano/core_test/socket.cpp b/nano/core_test/socket.cpp index 857242f3b1..ee59b6dc9b 100644 --- a/nano/core_test/socket.cpp +++ b/nano/core_test/socket.cpp @@ -27,7 +27,7 @@ TEST (socket, max_connections) nano::inactive_node inactivenode (nano::unique_path (), node_flags); auto node = inactivenode.node; - nano::thread_runner runner{ node->io_ctx_shared }; + nano::thread_runner runner{ node->io_ctx_shared, 1 }; auto server_port = system.get_available_port (); @@ -120,7 +120,7 @@ TEST (socket, max_connections_per_ip) auto node = inactivenode.node; ASSERT_FALSE (node->flags.disable_max_peers_per_ip); - nano::thread_runner runner{ node->io_ctx_shared }; + nano::thread_runner runner{ node->io_ctx_shared, 1 }; auto server_port = system.get_available_port (); @@ -242,7 +242,7 @@ TEST (socket, max_connections_per_subnetwork) ASSERT_TRUE (node->flags.disable_max_peers_per_ip); ASSERT_FALSE (node->flags.disable_max_peers_per_subnetwork); - nano::thread_runner runner{ node->io_ctx_shared }; + nano::thread_runner runner{ node->io_ctx_shared, 1 }; auto server_port = system.get_available_port (); boost::asio::ip::tcp::endpoint listen_endpoint{ boost::asio::ip::address_v6::any (), server_port }; @@ -306,7 +306,7 @@ TEST (socket, disabled_max_peers_per_ip) ASSERT_TRUE (node->flags.disable_max_peers_per_ip); - nano::thread_runner runner{ node->io_ctx_shared }; + nano::thread_runner runner{ node->io_ctx_shared, 1 }; auto server_port = system.get_available_port (); @@ -405,7 +405,7 @@ TEST (socket, drop_policy) nano::inactive_node inactivenode (nano::unique_path (), node_flags); auto node = inactivenode.node; - nano::thread_runner runner{ node->io_ctx_shared }; + nano::thread_runner runner{ node->io_ctx_shared, 1 }; std::vector> connections; @@ -473,7 +473,7 @@ TEST (socket, concurrent_writes) // This gives more realistic execution than using system#poll, allowing writes to // queue up and drain concurrently. - nano::thread_runner runner{ node->io_ctx_shared }; + nano::thread_runner runner{ node->io_ctx_shared, 1 }; constexpr size_t max_connections = 4; constexpr size_t client_count = max_connections; From e93f2437c767d9d26cb77d54506cf8449cd5b6f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Wo=CC=81jcik?= <3044353+pwojcikdev@users.noreply.github.com> Date: Sat, 20 Apr 2024 19:26:59 +0200 Subject: [PATCH 21/21] Fix `socket.max_connections` --- nano/core_test/socket.cpp | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/nano/core_test/socket.cpp b/nano/core_test/socket.cpp index ee59b6dc9b..5a7a627735 100644 --- a/nano/core_test/socket.cpp +++ b/nano/core_test/socket.cpp @@ -33,10 +33,12 @@ TEST (socket, max_connections) // successful incoming connections are stored in server_sockets to keep them alive (server side) std::vector> server_sockets; + std::mutex server_sockets_mutex; // start a server socket that allows max 2 live connections nano::transport::tcp_listener listener{ server_port, *node, 2 }; - listener.connection_accepted.add ([&server_sockets] (auto const & socket, auto const & server) { + listener.connection_accepted.add ([&] (auto const & socket, auto const & server) { + std::lock_guard guard{ server_sockets_mutex }; server_sockets.push_back (socket); }); nano::test::start_stop_guard stop_guard{ listener }; @@ -69,14 +71,22 @@ TEST (socket, max_connections) return node->stats.count (nano::stat::type::tcp_listener, nano::stat::detail::accept_success, nano::stat::dir::in); }; + auto server_sockets_size = [&] () { + std::lock_guard guard{ server_sockets_mutex }; + return server_sockets.size (); + }; + ASSERT_TIMELY_EQ (10s, get_tcp_accept_successes (), 2); ASSERT_ALWAYS_EQ (1s, get_tcp_accept_successes (), 2); ASSERT_TIMELY_EQ (5s, connection_attempts, 3); - ASSERT_TIMELY_EQ (5s, server_sockets.size (), 2); + ASSERT_TIMELY_EQ (5s, server_sockets_size (), 2); // create space for one socket and fill the connections table again - server_sockets[0].reset (); + { + std::lock_guard guard{ server_sockets_mutex }; + server_sockets[0].reset (); + } auto client4 = std::make_shared (*node); client4->async_connect (dst_endpoint, connect_handler); @@ -91,9 +101,11 @@ TEST (socket, max_connections) // close all existing sockets and fill the connections table again // start counting form 1 because 0 is the already closed socket - - server_sockets[1].reset (); - server_sockets[2].reset (); + { + std::lock_guard guard{ server_sockets_mutex }; + server_sockets[1].reset (); + server_sockets[2].reset (); + } auto client6 = std::make_shared (*node); client6->async_connect (dst_endpoint, connect_handler); @@ -107,7 +119,7 @@ TEST (socket, max_connections) ASSERT_TIMELY_EQ (5s, get_tcp_accept_successes (), 5); ASSERT_ALWAYS_EQ (1s, get_tcp_accept_successes (), 5); ASSERT_TIMELY_EQ (5s, connection_attempts, 8); // connections initiated by the client - ASSERT_TIMELY_EQ (5s, server_sockets.size (), 5); // connections accepted by the server + ASSERT_TIMELY_EQ (5s, server_sockets_size (), 5); // connections accepted by the server } TEST (socket, max_connections_per_ip)