refactor: unique alert names (opendatahub-io#1397)

zdtsw-forking · Nov 28, 2024 · 76cae1b · 76cae1b
1 parent 55fd878
commit 76cae1b
Show file tree

Hide file tree

Showing 9 changed files with 248 additions and 177 deletions.
diff --git a/config/monitoring/prometheus/apps/prometheus-configs.yaml b/config/monitoring/prometheus/apps/prometheus-configs.yaml
diff --git a/tests/prometheus_unit_tests/codeflare-alerting.unit-tests.yaml b/tests/prometheus_unit_tests/codeflare-alerting.unit-tests.yaml
@@ -21,7 +21,13 @@ tests:
         values: "0x60"
     alert_rule_test:
       - eval_time: 1h
-        alertname: CodeFlare Operator Probe Success Burn Rate
+        alertname: CodeFlare Operator Probe Success 5m and 1h Burn Rate high
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: CodeFlare Operator Probe Success 30m and 6h Burn Rate high
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: CodeFlare Operator Probe Success 2h and 1d Burn Rate high
         exp_alerts: []
 
   - interval: 1m
@@ -32,16 +38,16 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 2m
-        alertname: CodeFlare Operator Probe Success Burn Rate
+        alertname: CodeFlare Operator Probe Success 5m and 1h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: CodeFlare Operator Probe Success Burn Rate
+              alertname: CodeFlare Operator Probe Success 5m and 1h Burn Rate high
               instance: "codeflare-operator"
               namespace: "redhat-ods-applications"
               severity: info
             exp_annotations:
               message: "High error budget burn for codeflare-operator (current value: 3)."
-              summary: CodeFlare Operator Probe Success Burn Rate
+              summary: CodeFlare Operator Probe Success 5m and 1h Burn Rate high
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Distributed-Workloads/codeflare-operator-availability.md'
 
   - interval: 1m
@@ -52,16 +58,16 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 15m
-        alertname: CodeFlare Operator Probe Success Burn Rate
+        alertname: CodeFlare Operator Probe Success 30m and 6h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: CodeFlare Operator Probe Success Burn Rate
+              alertname: CodeFlare Operator Probe Success 30m and 6h Burn Rate high
               instance: "codeflare-operator"
               namespace: "redhat-ods-applications"
               severity: info
             exp_annotations:
               message: "High error budget burn for codeflare-operator (current value: 16)."
-              summary: CodeFlare Operator Probe Success Burn Rate
+              summary: CodeFlare Operator Probe Success 30m and 6h Burn Rate high
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Distributed-Workloads/codeflare-operator-probe-success-burn-rate.md'
 
   - interval: 1m
@@ -72,16 +78,16 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 1h
-        alertname: CodeFlare Operator Probe Success Burn Rate
+        alertname: CodeFlare Operator Probe Success 2h and 1d Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: CodeFlare Operator Probe Success Burn Rate
+              alertname: CodeFlare Operator Probe Success 2h and 1d Burn Rate high
               instance: "codeflare-operator"
               namespace: "redhat-ods-applications"
               severity: info
             exp_annotations:
               message: "High error budget burn for codeflare-operator (current value: 61)."
-              summary: CodeFlare Operator Probe Success Burn Rate
+              summary: CodeFlare Operator Probe Success 2h and 1d Burn Rate high
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Distributed-Workloads/codeflare-operator-probe-success-burn-rate.md'
 
   # operator running

diff --git a/tests/prometheus_unit_tests/data-science-pipelines-operator-alerting.unit-tests.yaml b/tests/prometheus_unit_tests/data-science-pipelines-operator-alerting.unit-tests.yaml
@@ -18,9 +18,20 @@ tests:
         values: "0x60"
       - series: haproxy_backend_http_responses_total:burnrate1d{component="dsp"}
         values: "0x60"
+      - series: haproxy_backend_http_responses_total:burnrate3d{component="dsp"}
+        values: "0x60"
     alert_rule_test:
       - eval_time: 1h
-        alertname: Data Science Pipelines Application Route Error Burn Rate
+        alertname: Data Science Pipelines Application Route Error 5m and 1h Burn Rate high
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: Data Science Pipelines Application Route Error 30m and 6h Burn Rate high
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: Data Science Pipelines Application Route Error 2h and 1d Burn Rate high
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: Data Science Pipelines Application Route Error 6h and 3d Burn Rate high
         exp_alerts: []
 
   - interval: 1m
@@ -31,14 +42,14 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 2m
-        alertname: Data Science Pipelines Application Route Error Burn Rate
+        alertname: Data Science Pipelines Application Route Error 5m and 1h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Data Science Pipelines Application Route Error Burn Rate
+              alertname: Data Science Pipelines Application Route Error 5m and 1h Burn Rate high
               namespace: "redhat-ods-applications"
               severity: info
             exp_annotations:
-              summary: "Data Science Pipelines Application Route Error Burn Rate"
+              summary: "Data Science Pipelines Application Route Error 5m and 1h Burn Rate high"
               message: "High error budget burn for  (current value: 3)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Data-Science-Pipelines/data-science-pipelines-application-error-burn-rate.md'
 
@@ -50,14 +61,14 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 15m
-        alertname: Data Science Pipelines Application Route Error Burn Rate
+        alertname: Data Science Pipelines Application Route Error 30m and 6h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Data Science Pipelines Application Route Error Burn Rate
+              alertname: Data Science Pipelines Application Route Error 30m and 6h Burn Rate high
               namespace: "redhat-ods-applications"
               severity: info
             exp_annotations:
-              summary: "Data Science Pipelines Application Route Error Burn Rate"
+              summary: "Data Science Pipelines Application Route Error 30m and 6h Burn Rate high"
               message: "High error budget burn for  (current value: 16)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Data-Science-Pipelines/data-science-pipelines-application-error-burn-rate.md'
 
@@ -69,14 +80,14 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 1h
-        alertname: Data Science Pipelines Application Route Error Burn Rate
+        alertname: Data Science Pipelines Application Route Error 2h and 1d Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Data Science Pipelines Application Route Error Burn Rate
+              alertname: Data Science Pipelines Application Route Error 2h and 1d Burn Rate high
               namespace: "redhat-ods-applications"
               severity: info
             exp_annotations:
-              summary: "Data Science Pipelines Application Route Error Burn Rate"
+              summary: "Data Science Pipelines Application Route Error 2h and 1d Burn Rate high"
               message: "High error budget burn for  (current value: 61)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Data-Science-Pipelines/data-science-pipelines-application-error-burn-rate.md'
 
@@ -88,14 +99,14 @@ tests:
         values: "1+1x200"
     alert_rule_test:
       - eval_time: 3h
-        alertname: Data Science Pipelines Application Route Error Burn Rate
+        alertname: Data Science Pipelines Application Route Error 6h and 3d Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Data Science Pipelines Application Route Error Burn Rate
+              alertname: Data Science Pipelines Application Route Error 6h and 3d Burn Rate high
               namespace: "redhat-ods-applications"
               severity: info
             exp_annotations:
-              summary: "Data Science Pipelines Application Route Error Burn Rate"
+              summary: "Data Science Pipelines Application Route Error 6h and 3d Burn Rate high"
               message: "High error budget burn for  (current value: 181)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Data-Science-Pipelines/data-science-pipelines-application-error-burn-rate.md'
 
@@ -116,7 +127,13 @@ tests:
         values: "0x60"
     alert_rule_test:
       - eval_time: 3h
-        alertname: Data Science Pipelines Operator Probe Success Burn Rate
+        alertname: Data Science Pipelines Operator Probe Success 5m and 1h Burn Rate high
+        exp_alerts: []
+      - eval_time: 3h
+        alertname: Data Science Pipelines Operator Probe Success 30m and 6h Burn Rate high
+        exp_alerts: []
+      - eval_time: 3h
+        alertname: Data Science Pipelines Operator Probe Success 2h and 1d Burn Rate high
         exp_alerts: []
 
 
@@ -128,15 +145,15 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 2m
-        alertname: Data Science Pipelines Operator Probe Success Burn Rate
+        alertname: Data Science Pipelines Operator Probe Success 5m and 1h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Data Science Pipelines Operator Probe Success Burn Rate
+              alertname: Data Science Pipelines Operator Probe Success 5m and 1h Burn Rate high
               instance: "data-science-pipelines-operator"
               namespace: "redhat-ods-applications"
               severity: critical
             exp_annotations:
-              summary: "Data Science Pipelines Operator Probe Success Burn Rate"
+              summary: "Data Science Pipelines Operator Probe Success 5m and 1h Burn Rate high"
               message: "High error budget burn for data-science-pipelines-operator (current value: 3)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Data-Science-Pipelines/data-science-pipelines-operator-probe-success-burn-rate.md'
 
@@ -148,15 +165,15 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 15m
-        alertname: Data Science Pipelines Operator Probe Success Burn Rate
+        alertname: Data Science Pipelines Operator Probe Success 30m and 6h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Data Science Pipelines Operator Probe Success Burn Rate
+              alertname: Data Science Pipelines Operator Probe Success 30m and 6h Burn Rate high
               instance: "data-science-pipelines-operator"
               namespace: "redhat-ods-applications"
               severity: critical
             exp_annotations:
-              summary: "Data Science Pipelines Operator Probe Success Burn Rate"
+              summary: "Data Science Pipelines Operator Probe Success 30m and 6h Burn Rate high"
               message: "High error budget burn for data-science-pipelines-operator (current value: 16)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Data-Science-Pipelines/data-science-pipelines-operator-probe-success-burn-rate.md'
 
@@ -168,15 +185,15 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 1h
-        alertname: Data Science Pipelines Operator Probe Success Burn Rate
+        alertname: Data Science Pipelines Operator Probe Success 2h and 1d Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Data Science Pipelines Operator Probe Success Burn Rate
+              alertname: Data Science Pipelines Operator Probe Success 2h and 1d Burn Rate high
               instance: "data-science-pipelines-operator"
               namespace: "redhat-ods-applications"
               severity: warning
             exp_annotations:
-              summary: "Data Science Pipelines Operator Probe Success Burn Rate"
+              summary: "Data Science Pipelines Operator Probe Success 2h and 1d Burn Rate high"
               message: "High error budget burn for data-science-pipelines-operator (current value: 61)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Data-Science-Pipelines/data-science-pipelines-operator-probe-success-burn-rate.md'
 

diff --git a/tests/prometheus_unit_tests/kserve-alerting.unit-tests.yaml b/tests/prometheus_unit_tests/kserve-alerting.unit-tests.yaml
@@ -21,8 +21,14 @@ tests:
         values: "0x60"
     alert_rule_test:
       - eval_time: 1h
-        alertname: Kserve Controller Probe Success Burn Rate
+        alertname: Kserve Controller Probe Success 5m and 1h Burn Rate high
         exp_alerts: []
+      - eval_time: 1h
+        alertname: Kserve Controller Probe Success 30m and 6h Burn Rate high
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: Kserve Controller Probe Success 2h and 1d Burn Rate high
+        exp_alerts: [] 
 
   - interval: 1m
     input_series:
@@ -32,15 +38,15 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 2m
-        alertname: Kserve Controller Probe Success Burn Rate
+        alertname: Kserve Controller Probe Success 5m and 1h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Kserve Controller Probe Success Burn Rate
+              alertname: Kserve Controller Probe Success 5m and 1h Burn Rate high
               instance: "kserve-controller-manager"
               severity: critical
             exp_annotations:
               message: "High error budget burn for kserve-controller-manager (current value: 3)."
-              summary: Kserve Controller Probe Success Burn Rate
+              summary: Kserve Controller Probe Success 5m and 1h Burn Rate high
               triage: "https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Model-Serving/rhods-kserve-controller-probe-success-burn-rate.md"
 
   - interval: 1m
@@ -51,15 +57,15 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 15m
-        alertname: Kserve Controller Probe Success Burn Rate
+        alertname: Kserve Controller Probe Success 30m and 6h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Kserve Controller Probe Success Burn Rate
+              alertname: Kserve Controller Probe Success 30m and 6h Burn Rate high
               instance: "kserve-controller-manager"
               severity: critical
             exp_annotations:
               message: "High error budget burn for kserve-controller-manager (current value: 16)."
-              summary: Kserve Controller Probe Success Burn Rate
+              summary: Kserve Controller Probe Success 30m and 6h Burn Rate high
               triage: "https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Model-Serving/rhods-kserve-controller-probe-success-burn-rate.md"
 
   - interval: 1m
@@ -70,13 +76,13 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 1h
-        alertname: Kserve Controller Probe Success Burn Rate
+        alertname: Kserve Controller Probe Success 2h and 1d Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Kserve Controller Probe Success Burn Rate
+              alertname: Kserve Controller Probe Success 2h and 1d Burn Rate high
               instance: "kserve-controller-manager"
               severity: warning
             exp_annotations:
               message: "High error budget burn for kserve-controller-manager (current value: 61)."
-              summary: Kserve Controller Probe Success Burn Rate
+              summary: Kserve Controller Probe Success 2h and 1d Burn Rate high
               triage: "https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Model-Serving/rhods-kserve-controller-probe-success-burn-rate.md"
diff --git a/tests/prometheus_unit_tests/model-mesh-alerting.unit-tests.yaml b/tests/prometheus_unit_tests/model-mesh-alerting.unit-tests.yaml
@@ -20,7 +20,13 @@ tests:
         values: "0x60"
     alert_rule_test:
       - eval_time: 1h
-        alertname: Modelmesh Controller Probe Success Burn Rate
+        alertname: Modelmesh Controller Probe Success 5m and 1h Burn Rate high
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: Modelmesh Controller Probe Success 30m and 6h Burn Rate high
+        exp_alerts: []
+      - eval_time: 1h
+        alertname: Modelmesh Controller Probe Success 2h and 1d Burn Rate high
         exp_alerts: []
 
   - interval: 1m
@@ -31,15 +37,15 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 2m
-        alertname: Modelmesh Controller Probe Success Burn Rate
+        alertname: Modelmesh Controller Probe Success 5m and 1h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Modelmesh Controller Probe Success Burn Rate
+              alertname: Modelmesh Controller Probe Success 5m and 1h Burn Rate high
               namespace: "redhat-ods-applications"
               instance: "modelmesh-controller"
               severity: critical
             exp_annotations:
-              summary: "Modelmesh Controller Probe Success Burn Rate"
+              summary: "Modelmesh Controller Probe Success 5m and 1h Burn Rate high"
               message: "High error budget burn for modelmesh-controller (current value: 3)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Model-Serving/rhods-modelmesh-controller-probe-success-burn-rate.md'
 
@@ -51,15 +57,15 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 15m
-        alertname: Modelmesh Controller Probe Success Burn Rate
+        alertname: Modelmesh Controller Probe Success 30m and 6h Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Modelmesh Controller Probe Success Burn Rate
+              alertname: Modelmesh Controller Probe Success 30m and 6h Burn Rate high
               namespace: "redhat-ods-applications"
               instance: "modelmesh-controller"
               severity: critical
             exp_annotations:
-              summary: "Modelmesh Controller Probe Success Burn Rate"
+              summary: "Modelmesh Controller Probe Success 30m and 6h Burn Rate high"
               message: "High error budget burn for modelmesh-controller (current value: 16)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Model-Serving/rhods-modelmesh-controller-probe-success-burn-rate.md'
 
@@ -71,14 +77,14 @@ tests:
         values: "1+1x60"
     alert_rule_test:
       - eval_time: 1h
-        alertname: Modelmesh Controller Probe Success Burn Rate
+        alertname: Modelmesh Controller Probe Success 2h and 1d Burn Rate high
         exp_alerts:
           - exp_labels:
-              alertname: Modelmesh Controller Probe Success Burn Rate
+              alertname: Modelmesh Controller Probe Success 2h and 1d Burn Rate high
               namespace: "redhat-ods-applications"
               instance: "modelmesh-controller"
               severity: warning
             exp_annotations:
-              summary: "Modelmesh Controller Probe Success Burn Rate"
+              summary: "Modelmesh Controller Probe Success 2h and 1d Burn Rate high"
               message: "High error budget burn for modelmesh-controller (current value: 61)."
               triage: 'https://gitlab.cee.redhat.com/service/managed-tenants-sops/-/blob/main/RHODS/Model-Serving/rhods-modelmesh-controller-probe-success-burn-rate.md'