increase etcdHightCommitDurations alert window

scality · Jun 19, 2024 · 9faefc2 · 9faefc2
1 parent f770016
commit 9faefc2
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 ## Release 127.0.3 (in development)
 
+### Enhancements
+
+- The `etcdHightCommitDurations` window was increased to 30 minutes
+  in order to avoid false positives
+  (PR[#4341](https://github.com/scality/metalk8s/pull/4341))
+
 ## Release 127.0.2
 
 ### Enhancements

diff --git a/buildchain/buildchain/codegen.py b/buildchain/buildchain/codegen.py
@@ -197,6 +197,9 @@ def codegen_chart_kube_prometheus_stack() -> types.TaskDict:
         "--patch 'PrometheusRule,metalk8s-monitoring,"
         "prometheus-operator-kubernetes-system-kubelet,"
         'spec:groups:0:rules:1:for,"5m"\' '
+        "--patch 'PrometheusRule,metalk8s-monitoring,"
+        "prometheus-operator-etcd,"
+        'spec:groups:0:rules:11:for,"30m"\' '
         f"--drop-prometheus-rules {drop_rule_file} "
         f"--remove-manifest ConfigMap prometheus-operator-nodes-darwin "
         f"--output {target_sls}"

diff --git a/salt/metalk8s/addons/prometheus-operator/deployed/chart.sls b/salt/metalk8s/addons/prometheus-operator/deployed/chart.sls
@@ -52181,7 +52181,7 @@ spec:
       expr: |-
         histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
         > 0.25
-      for: 10m
+      for: 30m
       labels:
         severity: warning
     - alert: etcdDatabaseQuotaLowSpace