Add prometheus-operator alerts support for manager (open-telemetry#266)

charts/opentelemetry-operator/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: opentelemetry-operator
-version: 0.9.0
+version: 0.9.1
 description: OpenTelemetry Operator Helm chart for Kubernetes
 type: application
 home: https://opentelemetry.io/

charts/opentelemetry-operator/templates/_helpers.tpl

@@ -5,6 +5,24 @@ Expand the name of the chart.
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "opentelemetry-operator.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
 {{/*
 Create chart name and version as used by the chart label.
 */}}

charts/opentelemetry-operator/templates/prometheusrule.yaml

@@ -0,0 +1,37 @@
+{{- if and .Values.manager.prometheusRule.enabled .Values.manager.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "opentelemetry-operator.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "opentelemetry-operator.labels" . | nindent 4 }}
+    {{- range $key, $value := .Values.manager.prometheusRule.extraLabels }}
+    {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }}
+    {{- end }}
+spec:
+  groups:
+{{- if .Values.manager.prometheusRule.groups }}
+    {{- toYaml .Values.manager.prometheusRule.groups | nindent 4 }}
+{{- end }}
+{{- if .Values.manager.prometheusRule.defaultRules.enabled }}
+  - name: managerRules
+    rules:
+    - alert: ReconcileErrors
+      expr: rate(controller_runtime_reconcile_total{result="error"}[5m]) > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        description: '{{`Reconciliation errors for {{ $labels.controller }} is increasing and has now reached {{ humanize $value }} `}}'
+        runbook_url: 'Check manager logs for reasons why this might happen'
+    - alert: WorkqueueDepth
+      expr: workqueue_depth > 0
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        description: '{{`Queue depth for {{ $labels.name }} has reached {{ $value }} `}}'
+        runbook_url: 'Check manager logs for reasons why this might happen'
+{{- end }}
+{{- end }}

charts/opentelemetry-operator/values.yaml

@@ -45,6 +45,17 @@ manager:
     metricsEndpoints:
     - port: metrics
 
+  prometheusRule:
+    enabled: false
+    groups: []
+    # Create default rules for monitoring the manager
+    defaultRules:
+      enabled: false
+
+    # additional labels for the PrometheusRule
+    extraLabels: {}
+
+
   ## List of additional cli arguments to configure the manager
   ## for example: --target-allocator-image, --labels, etc.
   extraArgs: []