Merge #26908

26908: storage: harden artifical quiesce heartbeat r=spencerkimball a=spencerkimball

We previously had the assumption when sending quiesce messages
that the Commit field could always be set to the Raft group's
`status.Commit`. With upcoming changes to quiesce ranges even
with replicas that are behind but non-live, this value could be
set incorrectly and still received by a supposedly dead replica.

This change mirrors the logic in the raft implementation for
setting the `raftpb.Message.Commit` field.

Release note: None

Co-authored-by: Spencer Kimball <[email protected]>

pkg/storage/replica.go

@@ -4327,10 +4327,22 @@ func (r *Replica) quiesceAndNotifyLocked(ctx context.Context, status *raft.Statu
 	if !r.quiesceLocked() {
 		return false
 	}
-	for id := range status.Progress {
+
+	for id, prog := range status.Progress {
 		if roachpb.ReplicaID(id) == r.mu.replicaID {
 			continue
 		}
+		// In common operation, we only quiesce when all followers are
+		// up-to-date. However, when we quiesce in the presence of dead
+		// nodes, a follower which is behind but considered dead may not
+		// have the log entry referenced by status.Commit and would
+		// explode if it were told to commit up to that point. So if
+		// prog.Match for a replica is not up to date with status.Commit,
+		// assume the replica is considered dead and skip the quiesce
+		// heartbeat.
+		if prog.Match < status.Commit {
+			continue
+		}
 		toReplica, toErr := r.getReplicaDescriptorByIDRLocked(
 			roachpb.ReplicaID(id), r.mu.lastFromReplica)
 		if toErr != nil {