feat: log socket exception stack trace periodically (#14318)

Signed-off-by: Anthony Petrov <[email protected]>
hashgraph · Jul 19, 2024 · 962b4ba · 962b4ba
1 parent 5fc4b24
commit 962b4ba
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 10 deletions.
diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gossip/SyncGossip.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/gossip/SyncGossip.java
@@ -388,7 +388,8 @@ private void buildSyncProtocolThreads(
                             new NegotiationProtocols(List.of(
                                     heartbeatProtocolFactory.build(otherId),
                                     reconnectProtocolFactory.build(otherId),
-                                    syncProtocolFactory.build(otherId)))))
+                                    syncProtocolFactory.build(otherId))),
+                            platformContext.getTime()))
                     .build());
         }
     }

diff --git a/...rm-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/NetworkUtils.java b/...rm-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/network/NetworkUtils.java
@@ -21,6 +21,7 @@
 
 import com.swirlds.common.crypto.config.CryptoConfig;
 import com.swirlds.common.platform.NodeId;
+import com.swirlds.common.utility.throttle.RateLimiter;
 import com.swirlds.config.api.Configuration;
 import com.swirlds.platform.Utilities;
 import com.swirlds.platform.crypto.KeysAndCerts;
@@ -70,10 +71,12 @@ public static void close(final Closeable... closeables) {
      *
      * @param e          the exception that was thrown
      * @param connection the connection used when the exception was thrown
+     * @param socketExceptionRateLimiter a rate limiter for reporting full stack traces for socket exceptions
      * @throws InterruptedException if the provided exception is an {@link InterruptedException}, it will be rethrown
      *                              once the connection is closed
      */
-    public static void handleNetworkException(final Exception e, final Connection connection)
+    public static void handleNetworkException(
+            final Exception e, final Connection connection, final RateLimiter socketExceptionRateLimiter)
             throws InterruptedException {
         final String description;
         // always disconnect when an exception gets thrown
@@ -90,8 +93,12 @@ public static void handleNetworkException(final Exception e, final Connection co
         // we use a different marker depending on what the root cause is
         final Marker marker = NetworkUtils.determineExceptionMarker(e);
         if (SOCKET_EXCEPTIONS.getMarker().equals(marker)) {
-            final String formattedException = NetworkUtils.formatException(e);
-            logger.warn(marker, "Connection broken: {} {}", description, formattedException);
+            if (socketExceptionRateLimiter.requestAndTrigger()) {
+                logger.warn(marker, "Connection broken: {}", description, e);
+            } else {
+                final String formattedException = NetworkUtils.formatException(e);
+                logger.warn(marker, "Connection broken: {} {}", description, formattedException);
+            }
         } else {
             logger.error(EXCEPTION.getMarker(), "Connection broken: {}", description, e);
         }

diff --git a/...re/src/main/java/com/swirlds/platform/network/communication/ProtocolNegotiatorThread.java b/...re/src/main/java/com/swirlds/platform/network/communication/ProtocolNegotiatorThread.java
@@ -16,19 +16,25 @@
 
 package com.swirlds.platform.network.communication;
 
+import com.swirlds.base.time.Time;
 import com.swirlds.common.threading.interrupt.InterruptableRunnable;
+import com.swirlds.common.utility.throttle.RateLimiter;
 import com.swirlds.platform.network.Connection;
 import com.swirlds.platform.network.ConnectionManager;
 import com.swirlds.platform.network.NetworkProtocolException;
 import com.swirlds.platform.network.NetworkUtils;
 import com.swirlds.platform.network.protocol.ProtocolRunnable;
 import java.io.IOException;
+import java.time.Duration;
 import java.util.List;
 
 /**
  * Continuously runs protocol negotiation and protocols over connections supplied by the connection manager
  */
 public class ProtocolNegotiatorThread implements InterruptableRunnable {
+    /** A duration between reporting full stack traces for socket exceptions. */
+    private static final Duration SOCKET_EXCEPTION_DURATION = Duration.ofMinutes(1);
+
     /**
      * The number of milliseconds to sleep if a negotiation fails
      */
@@ -37,6 +43,7 @@ public class ProtocolNegotiatorThread implements InterruptableRunnable {
     private final ConnectionManager connectionManager;
     private final List<ProtocolRunnable> handshakeProtocols;
     private final NegotiationProtocols protocols;
+    private final RateLimiter socketExceptionRateLimiter;
 
     /**
      * @param connectionManager
@@ -47,17 +54,21 @@ public class ProtocolNegotiatorThread implements InterruptableRunnable {
      * 		the list of protocols to execute when a new connection is established
      * @param protocols
      * 		the protocols to negotiate and run
+     * @param time
+     *      the Time object
      */
     public ProtocolNegotiatorThread(
             final ConnectionManager connectionManager,
             final int sleepMillis,
             final List<ProtocolRunnable> handshakeProtocols,
-            final NegotiationProtocols protocols) {
+            final NegotiationProtocols protocols,
+            final Time time) {
 
         this.connectionManager = connectionManager;
         this.sleepMillis = sleepMillis;
         this.handshakeProtocols = handshakeProtocols;
         this.protocols = protocols;
+        this.socketExceptionRateLimiter = new RateLimiter(time, SOCKET_EXCEPTION_DURATION);
     }
 
     @Override
@@ -74,7 +85,7 @@ public void run() throws InterruptedException {
                 negotiator.execute();
             }
         } catch (final RuntimeException | IOException | NetworkProtocolException | NegotiationException e) {
-            NetworkUtils.handleNetworkException(e, currentConn);
+            NetworkUtils.handleNetworkException(e, currentConn, socketExceptionRateLimiter);
         }
     }
 }
diff --git a/...irlds-platform-test/src/test/java/com/swirlds/platform/test/network/NetworkUtilsTest.java b/...irlds-platform-test/src/test/java/com/swirlds/platform/test/network/NetworkUtilsTest.java
@@ -16,28 +16,41 @@
 
 package com.swirlds.platform.test.network;
 
+import com.swirlds.common.context.PlatformContext;
+import com.swirlds.common.test.fixtures.platform.TestPlatformContextBuilder;
+import com.swirlds.common.utility.throttle.RateLimiter;
+import com.swirlds.config.api.Configuration;
+import com.swirlds.config.extensions.test.fixtures.TestConfigBuilder;
 import com.swirlds.platform.network.Connection;
 import com.swirlds.platform.network.NetworkUtils;
+import java.time.Duration;
 import javax.net.ssl.SSLException;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 class NetworkUtilsTest {
+    private final Configuration configuration = new TestConfigBuilder().getOrCreateConfig();
+    private final PlatformContext platformContext =
+            TestPlatformContextBuilder.create().withConfiguration(configuration).build();
+    private final RateLimiter socketExceptionRateLimiter =
+            new RateLimiter(platformContext.getTime(), Duration.ofMinutes(1));
+
     @Test
     void handleNetworkExceptionTest() {
         final Connection c = new FakeConnection();
         Assertions.assertDoesNotThrow(
-                () -> NetworkUtils.handleNetworkException(new Exception(), c),
+                () -> NetworkUtils.handleNetworkException(new Exception(), c, socketExceptionRateLimiter),
                 "handling should not throw an exception");
         Assertions.assertFalse(c.connected(), "method should have disconnected the connection");
 
         Assertions.assertDoesNotThrow(
-                () -> NetworkUtils.handleNetworkException(new SSLException("test", new NullPointerException()), null),
+                () -> NetworkUtils.handleNetworkException(
+                        new SSLException("test", new NullPointerException()), null, socketExceptionRateLimiter),
                 "handling should not throw an exception");
 
         Assertions.assertThrows(
                 InterruptedException.class,
-                () -> NetworkUtils.handleNetworkException(new InterruptedException(), null),
+                () -> NetworkUtils.handleNetworkException(new InterruptedException(), null, socketExceptionRateLimiter),
                 "an interrupted exception should be rethrown");
     }
 }
diff --git a/...st/java/com/swirlds/platform/test/network/communication/multithreaded/TestNegotiator.java b/...st/java/com/swirlds/platform/test/network/communication/multithreaded/TestNegotiator.java
@@ -16,6 +16,10 @@
 
 package com.swirlds.platform.test.network.communication.multithreaded;
 
+import com.swirlds.common.context.PlatformContext;
+import com.swirlds.common.test.fixtures.platform.TestPlatformContextBuilder;
+import com.swirlds.config.api.Configuration;
+import com.swirlds.config.extensions.test.fixtures.TestConfigBuilder;
 import com.swirlds.platform.network.Connection;
 import com.swirlds.platform.network.ConnectionManager;
 import com.swirlds.platform.network.communication.NegotiationProtocols;
@@ -28,6 +32,10 @@
  * Used to run a negotiator in a separate thread and capture any exceptions it might throw
  */
 class TestNegotiator {
+    private final Configuration configuration = new TestConfigBuilder().getOrCreateConfig();
+    private final PlatformContext platformContext =
+            TestPlatformContextBuilder.create().withConfiguration(configuration).build();
+
     private final TestProtocol protocol;
     private final ProtocolNegotiatorThread negotiator;
     private final Thread thread;
@@ -42,7 +50,8 @@ public TestNegotiator(final Connection connection, final TestProtocol protocol)
                 connectionManager,
                 100,
                 List.of(c -> handshakeRan.incrementAndGet()),
-                new NegotiationProtocols(List.of(protocol)));
+                new NegotiationProtocols(List.of(protocol)),
+                platformContext.getTime());
         thread = new Thread(this::run);
     }