Merge pull request grafana#306 from xperimental/fix-replay-ceiling-5.8

LOG-5415: Use a minimum value for replay memory ceiling
periklis · Jun 3, 2024 · 56c163c · 56c163c
2 parents 7d0e41f + 6214012
commit 56c163c
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 29 deletions.
diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md
@@ -1,5 +1,9 @@
 ## Main
 
+## Release 5.8.8
+
+- [13066](https://github.com/grafana/loki/pull/13066) **xperimental**: Use a minimum value for replay memory ceiling
+
 ## Release 5.8.7
 
 - [12874](https://github.com/grafana/loki/pull/12874) **periklis**: chore(operator): Update Loki operand to v2.9.8

diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go
@@ -68,7 +68,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 536870912
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -233,7 +233,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 0,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -319,7 +319,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -515,7 +515,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -598,7 +598,7 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) {
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -678,7 +678,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -944,7 +944,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -1030,7 +1030,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -1297,7 +1297,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -1383,7 +1383,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -1680,7 +1680,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -1769,7 +1769,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -2005,7 +2005,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -2094,7 +2094,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -2421,7 +2421,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -2521,7 +2521,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -2761,7 +2761,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -2847,7 +2847,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -3251,7 +3251,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -3338,7 +3338,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -3505,7 +3505,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -3593,7 +3593,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -3761,7 +3761,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -3849,7 +3849,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -4015,7 +4015,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -4106,7 +4106,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -4302,7 +4302,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,
@@ -4395,7 +4395,7 @@ ingester:
   wal:
     enabled: true
     dir: /tmp/wal
-    replay_memory_ceiling: 2500
+    replay_memory_ceiling: 2147483648
 ingester_client:
   grpc_client_config:
     max_recv_msg_size: 67108864
@@ -4591,7 +4591,7 @@ overrides:
 		},
 		WriteAheadLog: WriteAheadLog{
 			Directory:             "/tmp/wal",
-			IngesterMemoryRequest: 5000,
+			IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
 		},
 		ObjectStorage: storage.Options{
 			SharedStore: lokiv1.ObjectStorageSecretS3,

diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go
@@ -222,10 +222,20 @@ type WriteAheadLog struct {
 	IngesterMemoryRequest int64
 }
 
+const (
+	// minimumReplayCeiling contains the minimum value that will be used for the replay_memory_ceiling.
+	// It is set, so that even when the ingester has a low memory request, the replay will not flush each block
+	// on its own.
+	minimumReplayCeiling = 512 * 1024 * 1024
+)
+
 // ReplayMemoryCeiling calculates 50% of the ingester memory
 // for the ingester to use for the write-ahead-log capbability.
 func (w WriteAheadLog) ReplayMemoryCeiling() string {
 	value := int64(math.Ceil(float64(w.IngesterMemoryRequest) * float64(0.5)))
+	if value < minimumReplayCeiling {
+		value = minimumReplayCeiling
+	}
 	return fmt.Sprintf("%d", value)
 }