Use metrics for readiness/liveness probes

Since the existing healthz/readyz don't do much, we leverage the existing metrics endpoint for the probes to avoid opening another port (8081) on the host. Signed-off-by: Ori Braunshtein <[email protected]>
metallb · Nov 11, 2024 · 5b13edb · 5b13edb
1 parent bfeacb8
commit 5b13edb
Show file tree

Hide file tree

Showing 9 changed files with 49 additions and 55 deletions.
diff --git a/charts/frr-k8s/README.md b/charts/frr-k8s/README.md
@@ -35,7 +35,6 @@ Kubernetes: `>= 1.19.0-0`
 | frrk8s.frr.resources | object | `{}` |  |
 | frrk8s.frr.secureMetricsPort | int | `9141` |  |
 | frrk8s.frrMetrics.resources | object | `{}` |  |
-| frrk8s.healthPort | int | `8081` |  |
 | frrk8s.image.pullPolicy | string | `nil` |  |
 | frrk8s.image.repository | string | `"quay.io/metallb/frr-k8s"` |  |
 | frrk8s.image.tag | string | `nil` |  |

diff --git a/charts/frr-k8s/templates/controller.yaml b/charts/frr-k8s/templates/controller.yaml
@@ -199,7 +199,6 @@ spec:
         {{- with .Values.frrk8s.logLevel }}
         - --log-level={{ . }}
         {{- end }}
-        - --health-probe-bind-address={{.Values.prometheus.metricsBindAddress}}:{{ .Values.frrk8s.healthPort }}
         {{- if .Values.frrk8s.alwaysBlock }}
         - --always-block={{ .Values.frrk8s.alwaysBlock }}
         {{- end }}
@@ -222,8 +221,8 @@ spec:
         {{- if .Values.frrk8s.livenessProbe.enabled }}
         livenessProbe:
           httpGet:
-            path: /healthz
-            port: {{ .Values.frrk8s.healthPort }}
+            path: /metrics
+            port: monitoring
             host: {{ .Values.prometheus.metricsBindAddress }}
           initialDelaySeconds: {{ .Values.frrk8s.livenessProbe.initialDelaySeconds }}
           periodSeconds: {{ .Values.frrk8s.livenessProbe.periodSeconds }}
@@ -234,8 +233,8 @@ spec:
         {{- if .Values.frrk8s.readinessProbe.enabled }}
         readinessProbe:
           httpGet:
-            path: /healthz
-            port: {{ .Values.frrk8s.healthPort }}
+            path: /metrics
+            port: monitoring
             host: {{ .Values.prometheus.metricsBindAddress }}
           initialDelaySeconds: {{ .Values.frrk8s.readinessProbe.initialDelaySeconds }}
           periodSeconds: {{ .Values.frrk8s.readinessProbe.periodSeconds }}

diff --git a/charts/frr-k8s/templates/webhooks.yaml b/charts/frr-k8s/templates/webhooks.yaml
@@ -42,7 +42,7 @@ spec:
         - "--restart-on-rotator-secret-refresh=true"
         {{- end }}
         - "--namespace=$(NAMESPACE)"
-        - --health-probe-bind-address=:8081
+        - "--metrics-bind-address=:{{ .Values.prometheus.metricsPort }}"
         env:
         - name: NAMESPACE
           valueFrom:
@@ -59,20 +59,23 @@ spec:
             drop:
               - ALL
           readOnlyRootFilesystem: true
+        ports:
+        - containerPort: {{ .Values.prometheus.metricsPort }}
+          name: monitoring
         {{- if .Values.frrk8s.livenessProbe.enabled }}
         livenessProbe:
           httpGet:
-            path: /healthz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: {{ .Values.frrk8s.livenessProbe.initialDelaySeconds }}
           periodSeconds: {{ .Values.frrk8s.livenessProbe.periodSeconds }}
           failureThreshold: {{ .Values.frrk8s.livenessProbe.failureThreshold }}
         {{- end }}
         {{- if .Values.frrk8s.readinessProbe.enabled }}
         readinessProbe:
           httpGet:
-            path: /readyz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: {{ .Values.frrk8s.readinessProbe.initialDelaySeconds }}
           periodSeconds: {{ .Values.frrk8s.readinessProbe.periodSeconds }}
           failureThreshold: {{ .Values.frrk8s.readinessProbe.failureThreshold }}

diff --git a/charts/frr-k8s/values.yaml b/charts/frr-k8s/values.yaml
@@ -128,7 +128,6 @@ frrk8s:
   podAnnotations: {}
   labels:
     app: frr-k8s
-  healthPort: 8081
   livenessProbe:
     enabled: true
     failureThreshold: 3

diff --git a/cmd/main.go b/cmd/main.go
@@ -37,7 +37,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/event"
-	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -70,7 +69,6 @@ func init() {
 func main() {
 	var (
 		metricsAddr                   string
-		probeAddr                     string
 		logLevel                      string
 		nodeName                      string
 		namespace                     string
@@ -84,7 +82,6 @@ func main() {
 	)
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "127.0.0.1:7572", "The address the metric endpoint binds to.")
-	flag.StringVar(&probeAddr, "health-probe-bind-address", "127.0.0.1:8081", "The address the probe endpoint binds to.")
 	flag.StringVar(&logLevel, "log-level", "info", fmt.Sprintf("log level. must be one of: [%s]", logging.Levels.String()))
 	flag.StringVar(&nodeName, "node-name", "", "The node this daemon is running on.")
 	flag.StringVar(&namespace, "namespace", "", "The namespace this daemon is deployed in")
@@ -114,7 +111,7 @@ func main() {
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                 scheme,
-		HealthProbeBindAddress: probeAddr,
+		HealthProbeBindAddress: "", // we use the metrics endpoint for healthchecks
 		Cache: cache.Options{
 			ByObject: map[client.Object]cache.ByObject{
 				&corev1.Secret{}: namespaceSelector,
@@ -139,15 +136,6 @@ func main() {
 
 	//+kubebuilder:scaffold:builder
 
-	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to set up health check")
-		os.Exit(1)
-	}
-	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
-		setupLog.Error(err, "unable to set up ready check")
-		os.Exit(1)
-	}
-
 	enableWebhook := webhookMode == "onlywebhook"
 	startListeners := make(chan struct{})
 	if enableWebhook && !disableCertRotation {

diff --git a/config/all-in-one/frr-k8s-prometheus.yaml b/config/all-in-one/frr-k8s-prometheus.yaml
@@ -963,7 +963,7 @@ spec:
         - --log-level=info
         - --webhook-mode=onlywebhook
         - --namespace=$(NAMESPACE)
-        - --health-probe-bind-address=:8081
+        - --metrics-bind-address=:7572
         command:
         - /frr-k8s
         env:
@@ -975,15 +975,18 @@ spec:
         imagePullPolicy: IfNotPresent
         livenessProbe:
           httpGet:
-            path: /healthz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 15
           periodSeconds: 20
         name: frr-k8s-webhook-server
+        ports:
+        - containerPort: 7572
+          name: monitoring
         readinessProbe:
           httpGet:
-            path: /readyz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:
@@ -1107,7 +1110,6 @@ spec:
             drop:
             - ALL
       - args:
-        - --health-probe-bind-address=127.0.0.1:8081
         - --metrics-bind-address=127.0.0.1:7572
         - --node-name=$(NODE_NAME)
         - --namespace=$(NAMESPACE)
@@ -1132,8 +1134,8 @@ spec:
         livenessProbe:
           httpGet:
             host: 127.0.0.1
-            path: /healthz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 15
           periodSeconds: 20
         name: frr-k8s
@@ -1143,8 +1145,8 @@ spec:
         readinessProbe:
           httpGet:
             host: 127.0.0.1
-            path: /readyz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:

diff --git a/config/all-in-one/frr-k8s.yaml b/config/all-in-one/frr-k8s.yaml
@@ -932,7 +932,7 @@ spec:
         - --log-level=info
         - --webhook-mode=onlywebhook
         - --namespace=$(NAMESPACE)
-        - --health-probe-bind-address=:8081
+        - --metrics-bind-address=:7572
         command:
         - /frr-k8s
         env:
@@ -944,15 +944,18 @@ spec:
         imagePullPolicy: IfNotPresent
         livenessProbe:
           httpGet:
-            path: /healthz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 15
           periodSeconds: 20
         name: frr-k8s-webhook-server
+        ports:
+        - containerPort: 7572
+          name: monitoring
         readinessProbe:
           httpGet:
-            path: /readyz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:
@@ -1076,7 +1079,6 @@ spec:
             drop:
             - ALL
       - args:
-        - --health-probe-bind-address=127.0.0.1:8081
         - --metrics-bind-address=127.0.0.1:7572
         - --node-name=$(NODE_NAME)
         - --namespace=$(NAMESPACE)
@@ -1101,8 +1103,8 @@ spec:
         livenessProbe:
           httpGet:
             host: 127.0.0.1
-            path: /healthz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 15
           periodSeconds: 20
         name: frr-k8s
@@ -1112,8 +1114,8 @@ spec:
         readinessProbe:
           httpGet:
             host: 127.0.0.1
-            path: /readyz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:

diff --git a/config/default/frr-k8s_auth_proxy_patch.yaml b/config/default/frr-k8s_auth_proxy_patch.yaml
@@ -73,7 +73,6 @@ spec:
             - ALL
       - name: frr-k8s
         args:
-        - "--health-probe-bind-address=127.0.0.1:8081"
         - "--metrics-bind-address=127.0.0.1:7572"
         - "--node-name=$(NODE_NAME)"
         - "--namespace=$(NAMESPACE)"

diff --git a/config/frr-k8s/frr-k8s.yaml b/config/frr-k8s/frr-k8s.yaml
@@ -57,15 +57,15 @@ spec:
           readOnlyRootFilesystem: true
         livenessProbe:
           httpGet:
-            path: /healthz
-            port: 8081
+            path: /metrics
+            port: monitoring
             host: 127.0.0.1
           initialDelaySeconds: 15
           periodSeconds: 20
         readinessProbe:
           httpGet:
-            path: /readyz
-            port: 8081
+            path: /metrics
+            port: monitoring
             host: 127.0.0.1
           initialDelaySeconds: 5
           periodSeconds: 10
@@ -243,7 +243,7 @@ spec:
         - "--log-level=info"
         - "--webhook-mode=onlywebhook"
         - "--namespace=$(NAMESPACE)"
-        - "--health-probe-bind-address=:8081"
+        - "--metrics-bind-address=:7572"
         env:
         - name: NAMESPACE
           valueFrom:
@@ -258,16 +258,19 @@ spec:
             drop:
               - ALL
           readOnlyRootFilesystem: true
+        ports:
+        - containerPort: 7572
+          name: monitoring
         livenessProbe:
           httpGet:
-            path: /healthz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 15
           periodSeconds: 20
         readinessProbe:
           httpGet:
-            path: /readyz
-            port: 8081
+            path: /metrics
+            port: monitoring
           initialDelaySeconds: 5
           periodSeconds: 10
         # TODO(user): Configure the resources accordingly based on the project requirements.