kvserver: metamorphically enable kv.raft.leader_fortification.fractio…

…n_enabled

Part of cockroachdb#123847.

This commit metamorphically enables the `kv.raft.leader_fortification.fraction_enabled`
to exercise raft fortification and leader leases.

The commit also includes a few other WIP changes to try to stabilize this. It
won't be fully stable until defortification is implemented.

Release note: None

pkg/kv/kvclient/kvcoord/dist_sender_ambiguous_test.go

@@ -341,6 +341,7 @@ func TestTransactionUnexpectedlyCommitted(t *testing.T) {
 
 	ctx := context.Background()
 	st := cluster.MakeTestingClusterSettings()
+	kvserver.OverrideLeaderLeaseMetamorphism(ctx, &st.SV)
 
 	// Disable closed timestamps for control over when transaction gets bumped.
 	closedts.TargetDuration.Override(ctx, &st.SV, 1*time.Hour)

pkg/kv/kvserver/client_merge_test.go

@@ -2991,13 +2991,17 @@ func TestStoreRangeMergeAbandonedFollowers(t *testing.T) {
 // RHS, as it interpreted destroyReasonMergePending to mean that the RHS replica
 // had already been garbage collected.
 func TestStoreRangeMergeAbandonedFollowersAutomaticallyGarbageCollected(t *testing.T) {
+
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
 	ctx := context.Background()
+	st := cluster.MakeTestingClusterSettings()
+	kvserver.OverrideLeaderLeaseMetamorphism(ctx, &st.SV)
 	tc := testcluster.StartTestCluster(t, 3,
 		base.TestClusterArgs{
 			ReplicationMode: base.ReplicationManual,
+			ServerArgs:      base.TestServerArgs{},
 		})
 	defer tc.Stopper().Stop(ctx)
 	scratch := tc.ScratchRange(t)

pkg/kv/kvserver/replica_store_liveness.go

@@ -16,9 +16,9 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/raft/raftstoreliveness"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
 	"github.com/cockroachdb/cockroach/pkg/settings"
-	"github.com/cockroachdb/cockroach/pkg/util/envutil"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/cockroach/pkg/util/metamorphic"
 )
 
 // RaftLeaderFortificationFractionEnabled controls the fraction of ranges for
@@ -33,9 +33,9 @@ var RaftLeaderFortificationFractionEnabled = settings.RegisterFloatSetting(
 		"by extension, use Leader leases for all ranges which do not require "+
 		"expiration-based leases. Set to a value between 0.0 and 1.0 to gradually "+
 		"roll out Leader leases across the ranges in a cluster.",
-	// TODO(nvanbenschoten): make this a metamorphic constant once raft leader
-	// fortification and leader leases are sufficiently stable.
-	envutil.EnvOrDefaultFloat64("COCKROACH_LEADER_FORTIFICATION_FRACTION_ENABLED", 0.0),
+	metamorphic.ConstantWithTestChoice("kv.raft.leader_fortification.fraction_enabled",
+		1.0, /* defaultValue */
+		1.0 /* otherValues */),
 	settings.FloatInRange(0.0, 1.0),
 	settings.WithPublic,
 )

pkg/kv/kvserver/storeliveness/support_manager.go

@@ -243,7 +243,7 @@ func (sm *SupportManager) startLoop(ctx context.Context) {
 		select {
 		case <-sm.storesToAdd.sig:
 			sm.maybeAddStores(ctx)
-			sm.sendHeartbeats(ctx)
+			continue
 
 		case <-heartbeatTicker.C:
 			sm.sendHeartbeats(ctx)