diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gossip/SyncGossip.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gossip/SyncGossip.java index 9b89506541c8..1c4797f996d0 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gossip/SyncGossip.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gossip/SyncGossip.java @@ -388,7 +388,8 @@ private void buildSyncProtocolThreads( new NegotiationProtocols(List.of( heartbeatProtocolFactory.build(otherId), reconnectProtocolFactory.build(otherId), - syncProtocolFactory.build(otherId))))) + syncProtocolFactory.build(otherId))), + platformContext.getTime())) .build()); } } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/NetworkUtils.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/NetworkUtils.java index 80192a8f6bcf..97758cbe6cf4 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/NetworkUtils.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/NetworkUtils.java @@ -21,6 +21,7 @@ import com.swirlds.common.crypto.config.CryptoConfig; import com.swirlds.common.platform.NodeId; +import com.swirlds.common.utility.throttle.RateLimiter; import com.swirlds.config.api.Configuration; import com.swirlds.platform.Utilities; import com.swirlds.platform.crypto.KeysAndCerts; @@ -70,10 +71,12 @@ public static void close(final Closeable... closeables) { * * @param e the exception that was thrown * @param connection the connection used when the exception was thrown + * @param socketExceptionRateLimiter a rate limiter for reporting full stack traces for socket exceptions * @throws InterruptedException if the provided exception is an {@link InterruptedException}, it will be rethrown * once the connection is closed */ - public static void handleNetworkException(final Exception e, final Connection connection) + public static void handleNetworkException( + final Exception e, final Connection connection, final RateLimiter socketExceptionRateLimiter) throws InterruptedException { final String description; // always disconnect when an exception gets thrown @@ -90,8 +93,12 @@ public static void handleNetworkException(final Exception e, final Connection co // we use a different marker depending on what the root cause is final Marker marker = NetworkUtils.determineExceptionMarker(e); if (SOCKET_EXCEPTIONS.getMarker().equals(marker)) { - final String formattedException = NetworkUtils.formatException(e); - logger.warn(marker, "Connection broken: {} {}", description, formattedException); + if (socketExceptionRateLimiter.requestAndTrigger()) { + logger.warn(marker, "Connection broken: {}", description, e); + } else { + final String formattedException = NetworkUtils.formatException(e); + logger.warn(marker, "Connection broken: {} {}", description, formattedException); + } } else { logger.error(EXCEPTION.getMarker(), "Connection broken: {}", description, e); } diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/communication/ProtocolNegotiatorThread.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/communication/ProtocolNegotiatorThread.java index 4f9f1de6b1be..20389a2ca8a5 100644 --- a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/communication/ProtocolNegotiatorThread.java +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/communication/ProtocolNegotiatorThread.java @@ -16,19 +16,25 @@ package com.swirlds.platform.network.communication; +import com.swirlds.base.time.Time; import com.swirlds.common.threading.interrupt.InterruptableRunnable; +import com.swirlds.common.utility.throttle.RateLimiter; import com.swirlds.platform.network.Connection; import com.swirlds.platform.network.ConnectionManager; import com.swirlds.platform.network.NetworkProtocolException; import com.swirlds.platform.network.NetworkUtils; import com.swirlds.platform.network.protocol.ProtocolRunnable; import java.io.IOException; +import java.time.Duration; import java.util.List; /** * Continuously runs protocol negotiation and protocols over connections supplied by the connection manager */ public class ProtocolNegotiatorThread implements InterruptableRunnable { + /** A duration between reporting full stack traces for socket exceptions. */ + private static final Duration SOCKET_EXCEPTION_DURATION = Duration.ofMinutes(1); + /** * The number of milliseconds to sleep if a negotiation fails */ @@ -37,6 +43,7 @@ public class ProtocolNegotiatorThread implements InterruptableRunnable { private final ConnectionManager connectionManager; private final List handshakeProtocols; private final NegotiationProtocols protocols; + private final RateLimiter socketExceptionRateLimiter; /** * @param connectionManager @@ -47,17 +54,21 @@ public class ProtocolNegotiatorThread implements InterruptableRunnable { * the list of protocols to execute when a new connection is established * @param protocols * the protocols to negotiate and run + * @param time + * the Time object */ public ProtocolNegotiatorThread( final ConnectionManager connectionManager, final int sleepMillis, final List handshakeProtocols, - final NegotiationProtocols protocols) { + final NegotiationProtocols protocols, + final Time time) { this.connectionManager = connectionManager; this.sleepMillis = sleepMillis; this.handshakeProtocols = handshakeProtocols; this.protocols = protocols; + this.socketExceptionRateLimiter = new RateLimiter(time, SOCKET_EXCEPTION_DURATION); } @Override @@ -74,7 +85,7 @@ public void run() throws InterruptedException { negotiator.execute(); } } catch (final RuntimeException | IOException | NetworkProtocolException | NegotiationException e) { - NetworkUtils.handleNetworkException(e, currentConn); + NetworkUtils.handleNetworkException(e, currentConn, socketExceptionRateLimiter); } } } diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/network/NetworkUtilsTest.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/network/NetworkUtilsTest.java index 5b50a02b9614..c8fb639a1dff 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/network/NetworkUtilsTest.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/network/NetworkUtilsTest.java @@ -16,28 +16,41 @@ package com.swirlds.platform.test.network; +import com.swirlds.common.context.PlatformContext; +import com.swirlds.common.test.fixtures.platform.TestPlatformContextBuilder; +import com.swirlds.common.utility.throttle.RateLimiter; +import com.swirlds.config.api.Configuration; +import com.swirlds.config.extensions.test.fixtures.TestConfigBuilder; import com.swirlds.platform.network.Connection; import com.swirlds.platform.network.NetworkUtils; +import java.time.Duration; import javax.net.ssl.SSLException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; class NetworkUtilsTest { + private final Configuration configuration = new TestConfigBuilder().getOrCreateConfig(); + private final PlatformContext platformContext = + TestPlatformContextBuilder.create().withConfiguration(configuration).build(); + private final RateLimiter socketExceptionRateLimiter = + new RateLimiter(platformContext.getTime(), Duration.ofMinutes(1)); + @Test void handleNetworkExceptionTest() { final Connection c = new FakeConnection(); Assertions.assertDoesNotThrow( - () -> NetworkUtils.handleNetworkException(new Exception(), c), + () -> NetworkUtils.handleNetworkException(new Exception(), c, socketExceptionRateLimiter), "handling should not throw an exception"); Assertions.assertFalse(c.connected(), "method should have disconnected the connection"); Assertions.assertDoesNotThrow( - () -> NetworkUtils.handleNetworkException(new SSLException("test", new NullPointerException()), null), + () -> NetworkUtils.handleNetworkException( + new SSLException("test", new NullPointerException()), null, socketExceptionRateLimiter), "handling should not throw an exception"); Assertions.assertThrows( InterruptedException.class, - () -> NetworkUtils.handleNetworkException(new InterruptedException(), null), + () -> NetworkUtils.handleNetworkException(new InterruptedException(), null, socketExceptionRateLimiter), "an interrupted exception should be rethrown"); } } diff --git a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/network/communication/multithreaded/TestNegotiator.java b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/network/communication/multithreaded/TestNegotiator.java index 9509af3d1879..644691e0af09 100644 --- a/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/network/communication/multithreaded/TestNegotiator.java +++ b/platform-sdk/swirlds-unit-tests/core/swirlds-platform-test/src/test/java/com/swirlds/platform/test/network/communication/multithreaded/TestNegotiator.java @@ -16,6 +16,10 @@ package com.swirlds.platform.test.network.communication.multithreaded; +import com.swirlds.common.context.PlatformContext; +import com.swirlds.common.test.fixtures.platform.TestPlatformContextBuilder; +import com.swirlds.config.api.Configuration; +import com.swirlds.config.extensions.test.fixtures.TestConfigBuilder; import com.swirlds.platform.network.Connection; import com.swirlds.platform.network.ConnectionManager; import com.swirlds.platform.network.communication.NegotiationProtocols; @@ -28,6 +32,10 @@ * Used to run a negotiator in a separate thread and capture any exceptions it might throw */ class TestNegotiator { + private final Configuration configuration = new TestConfigBuilder().getOrCreateConfig(); + private final PlatformContext platformContext = + TestPlatformContextBuilder.create().withConfiguration(configuration).build(); + private final TestProtocol protocol; private final ProtocolNegotiatorThread negotiator; private final Thread thread; @@ -42,7 +50,8 @@ public TestNegotiator(final Connection connection, final TestProtocol protocol) connectionManager, 100, List.of(c -> handshakeRan.incrementAndGet()), - new NegotiationProtocols(List.of(protocol))); + new NegotiationProtocols(List.of(protocol)), + platformContext.getTime()); thread = new Thread(this::run); }