apps sc: Alert size over limit only on active indicies

elastisys · Jan 8, 2025 · c151444 · c151444
1 parent 4e52119
commit c151444
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/helmfile.d/charts/prometheus-alerts/templates/alerts/opensearch.yaml b/helmfile.d/charts/prometheus-alerts/templates/alerts/opensearch.yaml
@@ -67,13 +67,13 @@ spec:
         summary: Opensearch Cluster Red (instance {{`{{ $labels.instance }}`}})
         description: Opensearch Cluster is in a Red status VALUE = {{`{{ $value }}`}}  LABELS = {{`{{ $labels }}`}}
     {{- range $prefixes := .Values.osIndexAlerts }}
-    - alert: {{ $prefixes.prefix | title }}SizeIncreasedOverLimit
-      expr: elasticsearch_indices_store_size_bytes_primary{namespace="opensearch-system",index=~"{{ $prefixes.prefix }}.+"} / (1024^2) > {{ $prefixes.alertSizeMB }}
+    - alert: OpenSearch{{ $prefixes.prefix | trimSuffix "-" | title }}IndexSizeOverLimit
+      expr: (elasticsearch_indices_store_size_bytes_primary{namespace="opensearch-system", index=~"{{ $prefixes.prefix }}.+"} / (1024 ^ 2) > {{ $prefixes.alertSizeMB }}) * clamp(rate(elasticsearch_indices_docs_primary{namespace="opensearch-system", index=~"{{ $prefixes.prefix }}.+"}[4h]) > 0, 1, 1)
       for: 15m
       labels:
         severity: warning
       annotations:
-        message: Primary shard size for index {{`{{ $labels.index }}`}} has increased over the limit of {{ $prefixes.alertSizeMB }}MB current size is {{`{{ $value | printf "%.0f" }}`}}MB
-        description: If the size keeps increasing, that might indicate a problem with ISM
+        message: Active primary shard size for index {{`{{ $labels.index }}`}} has increased over the limit of {{ $prefixes.alertSizeMB }}MB, current size is {{`{{ $value | printf "%.0f" }}`}}MB
+        description: This indicates a problem with index state management preventing the alias to roll over to a new index.
     {{- end }}
 {{- end }}