Add livenessProbe to target allocator deployment for operator (open-t…

…elemetry#2356)

* add liveness probe to ta manifest generation

* add livenessProbe to e2e-ta-features

* add chlog

* clean up oopsies

* add liveness e2e test to target allocator

* change healthz to livez

.chloggen/TA-liveness-probe.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. operator, target allocator, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: add liveness probe to target allocator deployment generation
+
+# One or more tracking issues related to the change
+issues: [2258]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

internal/manifests/targetallocator/container.go

@@ -18,6 +18,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/operator-framework/operator-lib/proxy"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 
 	"github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1"
 	"github.com/open-telemetry/opentelemetry-operator/internal/config"
@@ -69,14 +70,24 @@ func Container(cfg config.Config, logger logr.Logger, otelcol v1alpha1.OpenTelem
 	if otelcol.Spec.TargetAllocator.PrometheusCR.Enabled {
 		args = append(args, "--enable-prometheus-cr-watcher")
 	}
+	livenessProbe := &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path: "/livez",
+				Port: intstr.FromInt(8080),
+			},
+		},
+	}
+
 	envVars = append(envVars, proxy.ReadProxyVarsFromEnv()...)
 	return corev1.Container{
-		Name:         naming.TAContainer(),
-		Image:        image,
-		Ports:        ports,
-		Env:          envVars,
-		VolumeMounts: volumeMounts,
-		Resources:    otelcol.Spec.TargetAllocator.Resources,
-		Args:         args,
+		Name:          naming.TAContainer(),
+		Image:         image,
+		Ports:         ports,
+		Env:           envVars,
+		VolumeMounts:  volumeMounts,
+		Resources:     otelcol.Spec.TargetAllocator.Resources,
+		Args:          args,
+		LivenessProbe: livenessProbe,
 	}
 }

internal/manifests/targetallocator/container_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1"
@@ -196,6 +197,14 @@ func TestContainerHasEnvVars(t *testing.T) {
 				Protocol:      corev1.ProtocolTCP,
 			},
 		},
+		LivenessProbe: &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path: "/livez",
+					Port: intstr.FromInt(8080),
+				},
+			},
+		},
 	}
 
 	// test
@@ -278,6 +287,14 @@ func TestContainerDoesNotOverrideEnvVars(t *testing.T) {
 				Protocol:      corev1.ProtocolTCP,
 			},
 		},
+		LivenessProbe: &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				HTTPGet: &corev1.HTTPGetAction{
+					Path: "/livez",
+					Port: intstr.FromInt(8080),
+				},
+			},
+		},
 	}
 
 	// test
@@ -286,3 +303,28 @@ func TestContainerDoesNotOverrideEnvVars(t *testing.T) {
 	// verify
 	assert.Equal(t, expected, c)
 }
+func TestLivenessProbe(t *testing.T) {
+	// prepare
+	otelcol := v1alpha1.OpenTelemetryCollector{
+		Spec: v1alpha1.OpenTelemetryCollectorSpec{
+			TargetAllocator: v1alpha1.OpenTelemetryTargetAllocator{
+				Enabled: true,
+			},
+		},
+	}
+	cfg := config.New()
+	expected := &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path: "/livez",
+				Port: intstr.FromInt(8080),
+			},
+		},
+	}
+
+	// test
+	c := Container(cfg, logger, otelcol)
+
+	// verify
+	assert.Equal(t, expected, c.LivenessProbe)
+}

tests/e2e/targetallocator-features/00-assert.yaml

@@ -70,6 +70,13 @@ spec:
           volumeMounts:
           - mountPath: /conf
             name: ta-internal
+          livenessProbe:
+            httpGet:
+              path: /livez
+            successThreshold: 1
+            failureThreshold: 3
+            timeoutSeconds: 1
+            periodSeconds: 10
       volumes:
         - configMap:
             items:

tests/e2e/targetallocator-features/01-assert.yaml

@@ -0,0 +1,14 @@
+# waiting for liveness probe to work
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 5
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/component: opentelemetry-targetallocator
+status:
+  containerStatuses:
+    - name: ta-container
+      restartCount: 0

tests/e2e/targetallocator-features/01-liveness.yaml

@@ -0,0 +1,5 @@
+# waiting for liveness probe to work
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- command: sleep 35