kvserver: log when raft send/recv queue fills up

Inspired by cockroachlabs/support#1770.

If either the raft send or receive queue fills up, wide-spread outages
can occur as replication progress stalls. We have metrics that can
indicate this, but straightforward logging is also appropriate to direct
attention to the fact, which this commit achieves.

Touches #79755

Release justification: important logging improvement
Release note: None

pkg/kv/kvserver/raft_transport.go

@@ -592,6 +592,9 @@ func (t *RaftTransport) SendAsync(req *RaftMessageRequest, class rpc.ConnectionC
 		}
 		return true
 	default:
+		if logRaftSendQueueFullEvery.ShouldLog() {
+			log.Warningf(t.AnnotateCtx(context.Background()), "raft send queue to n%d is full", toNodeID)
+		}
 		req.release()
 		return false
 	}

pkg/kv/kvserver/store_raft.go

@@ -27,6 +27,11 @@ import (
 	"go.etcd.io/etcd/raft/v3/raftpb"
 )
 
+var (
+	logRaftRecvQueueFullEvery = log.Every(1 * time.Second)
+	logRaftSendQueueFullEvery = log.Every(1 * time.Second)
+)
+
 type raftRequestInfo struct {
 	req        *RaftMessageRequest
 	respStream RaftMessageResponseStream
@@ -177,6 +182,9 @@ func (s *Store) HandleRaftUncoalescedRequest(
 		// TODO(peter): Return an error indicating the request was dropped. Note
 		// that dropping the request is safe. Raft will retry.
 		s.metrics.RaftRcvdMsgDropped.Inc(1)
+		if logRaftRecvQueueFullEvery.ShouldLog() {
+			log.Warningf(ctx, "raft receive queue for r%d is full", req.RangeID)
+		}
 		return false
 	}
 	q.infos = append(q.infos, raftRequestInfo{