Merge pull request #88245 from tbg/backport21.2-86645

cockroachdb · Sep 20, 2022 · 12b7c9d · 12b7c9d
2 parents 8612c8a + b1b17b2
commit 12b7c9d
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/pkg/kv/kvserver/raft_transport.go b/pkg/kv/kvserver/raft_transport.go
@@ -592,6 +592,9 @@ func (t *RaftTransport) SendAsync(req *RaftMessageRequest, class rpc.ConnectionC
 		}
 		return true
 	default:
+		if logRaftSendQueueFullEvery.ShouldLog() {
+			log.Warningf(t.AnnotateCtx(context.Background()), "raft send queue to n%d is full", toNodeID)
+		}
 		req.release()
 		return false
 	}

diff --git a/pkg/kv/kvserver/store_raft.go b/pkg/kv/kvserver/store_raft.go
@@ -27,6 +27,11 @@ import (
 	"go.etcd.io/etcd/raft/v3/raftpb"
 )
 
+var (
+	logRaftRecvQueueFullEvery = log.Every(1 * time.Second)
+	logRaftSendQueueFullEvery = log.Every(1 * time.Second)
+)
+
 type raftRequestInfo struct {
 	req        *RaftMessageRequest
 	respStream RaftMessageResponseStream
@@ -177,6 +182,9 @@ func (s *Store) HandleRaftUncoalescedRequest(
 		// TODO(peter): Return an error indicating the request was dropped. Note
 		// that dropping the request is safe. Raft will retry.
 		s.metrics.RaftRcvdMsgDropped.Inc(1)
+		if logRaftRecvQueueFullEvery.ShouldLog() {
+			log.Warningf(ctx, "raft receive queue for r%d is full", req.RangeID)
+		}
 		return false
 	}
 	q.infos = append(q.infos, raftRequestInfo{