Merge #66859

66859: kvcoord: increase max_intent_bytes default to 4MB r=andreimatei,erikgrinaker a=aliher1911

Previously limit was set conservatively to 256KB. Having a
conservative limit can cause intent cleanup batches to condense
aggressively resorting to range cleanups which are more expensive.
Increasing defaults will reduce contention during cleanup at the
expense of increased memory for large transactions on
tx_coord_sender.

Release note (performance improvement): Increase default value for
kv.transaction.max_intents_bytes cluster setting from 256KB to 4MB
to improve transaction performance for large transactions at the
expense of increased memory usage. Transactions above this size
limit use slower cleanup mode for commits and aborts.

#54029

Co-authored-by: Oleg Afanasyev <[email protected]>

docs/generated/settings/settings-for-tenants.txt

@@ -32,7 +32,7 @@ kv.range_split.by_load_enabled	boolean	true	allow automatic splits of ranges bas
 kv.range_split.load_qps_threshold	integer	2500	the QPS over which, the range becomes a candidate for load based splitting
 kv.rangefeed.enabled	boolean	false	if set, rangefeed registration is enabled
 kv.replication_reports.interval	duration	1m0s	the frequency for generating the replication_constraint_stats, replication_stats_report and replication_critical_localities reports (set to 0 to disable)
-kv.transaction.max_intents_bytes	integer	262144	maximum number of bytes used to track locks in transactions
+kv.transaction.max_intents_bytes	integer	4194304	maximum number of bytes used to track locks in transactions
 kv.transaction.max_refresh_spans_bytes	integer	256000	maximum number of bytes used to track refresh spans in serializable transactions
 security.ocsp.mode	enumeration	off	"use OCSP to check whether TLS certificates are revoked. If the OCSP
 server is unreachable, in strict mode all certificates will be rejected

docs/generated/settings/settings.html

@@ -36,7 +36,7 @@
 <tr><td><code>kv.replication_reports.interval</code></td><td>duration</td><td><code>1m0s</code></td><td>the frequency for generating the replication_constraint_stats, replication_stats_report and replication_critical_localities reports (set to 0 to disable)</td></tr>
 <tr><td><code>kv.snapshot_rebalance.max_rate</code></td><td>byte size</td><td><code>8.0 MiB</code></td><td>the rate limit (bytes/sec) to use for rebalance and upreplication snapshots</td></tr>
 <tr><td><code>kv.snapshot_recovery.max_rate</code></td><td>byte size</td><td><code>8.0 MiB</code></td><td>the rate limit (bytes/sec) to use for recovery snapshots</td></tr>
-<tr><td><code>kv.transaction.max_intents_bytes</code></td><td>integer</td><td><code>262144</code></td><td>maximum number of bytes used to track locks in transactions</td></tr>
+<tr><td><code>kv.transaction.max_intents_bytes</code></td><td>integer</td><td><code>4194304</code></td><td>maximum number of bytes used to track locks in transactions</td></tr>
 <tr><td><code>kv.transaction.max_refresh_spans_bytes</code></td><td>integer</td><td><code>256000</code></td><td>maximum number of bytes used to track refresh spans in serializable transactions</td></tr>
 <tr><td><code>security.ocsp.mode</code></td><td>enumeration</td><td><code>off</code></td><td>use OCSP to check whether TLS certificates are revoked. If the OCSP<br/>server is unreachable, in strict mode all certificates will be rejected<br/>and in lax mode all certificates will be accepted. [off = 0, lax = 1, strict = 2]</td></tr>
 <tr><td><code>security.ocsp.timeout</code></td><td>duration</td><td><code>3s</code></td><td>timeout before considering the OCSP server unreachable</td></tr>

pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go

@@ -67,10 +67,17 @@ var pipelinedWritesMaxBatchSize = settings.RegisterIntSetting(
 // NB: this is called "max_intents_bytes" instead of "max_lock_bytes" because
 // it was created before the concept of intents were generalized to locks.
 // Switching it would require a migration which doesn't seem worth it.
+//
+// Note: Default value was arbitrarily set to 256KB but practice showed that
+// it could be raised higher. When transaction reaches this limit, intent
+// cleanup switches to range instead of point cleanup which is expensive
+// because it needs to hold broad latches and iterate through the range to
+// find matching intents.
+// See #54029 for more details.
 var trackedWritesMaxSize = settings.RegisterIntSetting(
 	"kv.transaction.max_intents_bytes",
 	"maximum number of bytes used to track locks in transactions",
-	1<<18, /* 256 KB */
+	1<<22, /* 4 MB */
 ).WithPublic()
 
 // txnPipeliner is a txnInterceptor that pipelines transactional writes by using