kvcoord: remove GCed DistSender circuit breakers from tripped metrics

Otherwise, these will register as tripped forever.

Epic: none
Release note: None

pkg/kv/kvclient/kvcoord/dist_sender_circuit_breaker.go

@@ -267,11 +267,6 @@ func (d *DistSenderCircuitBreakers) gcLoop(ctx context.Context) {
 				// replace. We don't expect this, since only this loop removes them.
 				if v, ok := d.replicas.LoadAndDelete(key); ok {
 					cb = v.(*ReplicaCircuitBreaker)
-					// TODO(erikgrinaker): this needs to remove tripped circuit breakers
-					// from the metrics, otherwise they'll appear as tripped forever.
-					// However, there are race conditions with concurrent probes that can
-					// lead to metrics gauge leaks (both positive and negative), so we'll
-					// have to make sure we reap the probes here first.
 					d.metrics.CircuitBreaker.Replicas.Dec(1)
 					gced++
 
@@ -281,6 +276,11 @@ func (d *DistSenderCircuitBreakers) gcLoop(ctx context.Context) {
 					// probes run longer than the GC threshold. The map removal above will
 					// prevent new probes being launched (checked by launchProbe).
 					cb.reapProbe()
+
+					// Remove the GCed breaker from tripped metrics.
+					if cb.isTripped() {
+						d.metrics.CircuitBreaker.ReplicasTripped.Dec(1)
+					}
 				}
 			}
 			return true