storage: tweak Raft tunables

Increase Raft MaxInFlightMsgs default from 4 to 64 which allows deeper
pipeling of Raft messages on high latency links.

Increase Raft log max-size from 1MB to 4MB. The 1MB setting was overly
aggressive on high latency links and seems to contribute to increased
likelihood of Raft snapshots.

See cockroachdb#13687

pkg/storage/raft_log_queue.go

@@ -45,7 +45,7 @@ const (
 )
 
 // raftLogMaxSize limits the maximum size of the Raft log.
-var raftLogMaxSize = envutil.EnvOrDefaultInt64("COCKROACH_RAFT_LOG_MAX_SIZE", 1<<20 /* 1 MB */)
+var raftLogMaxSize = envutil.EnvOrDefaultInt64("COCKROACH_RAFT_LOG_MAX_SIZE", 4<<20 /* 4 MB */)
 
 // raftLogQueue manages a queue of replicas slated to have their raft logs
 // truncated by removing unneeded entries.

pkg/storage/store.go

@@ -181,7 +181,7 @@ func TestStoreConfig(clock *hlc.Clock) StoreConfig {
 
 var (
 	raftMaxSizePerMsg   = envutil.EnvOrDefaultInt("COCKROACH_RAFT_MAX_SIZE_PER_MSG", 16*1024)
-	raftMaxInflightMsgs = envutil.EnvOrDefaultInt("COCKROACH_RAFT_MAX_INFLIGHT_MSGS", 4)
+	raftMaxInflightMsgs = envutil.EnvOrDefaultInt("COCKROACH_RAFT_MAX_INFLIGHT_MSGS", 64)
 )
 
 func newRaftConfig(
@@ -208,9 +208,9 @@ func newRaftConfig(
 		// MaxInflightMsgs controls how many "inflight" messages Raft will send to
 		// a follower without hearing a response. The total number of Raft log
 		// entries is a combination of this setting and MaxSizePerMsg. The current
-		// settings provide for up to 64 KB of raft log to be sent without
+		// settings provide for up to 1 MB of raft log to be sent without
 		// acknowledgement. With an average entry size of 1 KB that translates to
-		// ~64 commands that might be executed in the handling of a single
+		// ~1024 commands that might be executed in the handling of a single
 		// raft.Ready operation.
 		MaxInflightMsgs: raftMaxInflightMsgs,
 	}