rpc accept loop: added backoff on logging

nomad/rpc.go

@@ -49,8 +49,9 @@ const (
 
 type rpcHandler struct {
 	*Server
-	logger   log.Logger
-	gologger *golog.Logger
+	logger          log.Logger
+	gologger        *golog.Logger
+	acceptLoopDelay time.Duration
 }
 
 func newRpcHandler(s *Server) *rpcHandler {
@@ -98,22 +99,44 @@ func (r *rpcHandler) listen(ctx context.Context) {
 			if r.shutdown {
 				return
 			}
-
-			select {
-			case <-ctx.Done():
-				return
-			default:
-			}
-
-			r.logger.Error("failed to accept RPC conn", "error", err)
+			r.handleAcceptErr(err, ctx)
 			continue
 		}
+		// No error, reset loop delay
+		r.acceptLoopDelay = 0
 
 		go r.handleConn(ctx, conn, &RPCContext{Conn: conn})
 		metrics.IncrCounter([]string{"nomad", "rpc", "accept_conn"}, 1)
 	}
 }
 
+// Sleep to avoid spamming the log, with a maximum delay according to whether or not the error is temporary
+func (r *rpcHandler) handleAcceptErr(err error, ctx context.Context) {
+	const baseAcceptLoopDelay = 5 * time.Millisecond
+	const maxAcceptLoopDelay = 5 * time.Second
+	const maxAcceptLoopDelayTemporaryError = 1 * time.Second
+
+	if r.acceptLoopDelay == 0 {
+		r.acceptLoopDelay = baseAcceptLoopDelay
+	} else {
+		r.acceptLoopDelay *= 2
+	}
+	temporaryError := false
+	if ne, ok := err.(net.Error); ok && ne.Temporary() {
+		temporaryError = true
+	}
+	if temporaryError && r.acceptLoopDelay > maxAcceptLoopDelayTemporaryError {
+		r.acceptLoopDelay = maxAcceptLoopDelayTemporaryError
+	} else if r.acceptLoopDelay > maxAcceptLoopDelay {
+		r.acceptLoopDelay = maxAcceptLoopDelay
+	}
+	r.logger.Error("failed to accept RPC conn", "error", err, "delay", r.acceptLoopDelay)
+	select {
+	case <-ctx.Done():
+	case <-time.After(r.acceptLoopDelay):
+	}
+}
+
 // handleConn is used to determine if this is a Raft or
 // Nomad type RPC connection and invoke the correct handler
 func (r *rpcHandler) handleConn(ctx context.Context, conn net.Conn, rpcCtx *RPCContext) {