admission controller: Set failure policy

pkg/clusteragent/admission/controllers/webhook/config.go

@@ -27,6 +27,7 @@ type Config struct {
 	svcName                  string
 	svcPort                  int32
 	timeout                  int32
+	failurePolicy            string
 }
 
 // NewConfig creates a webhook controller configuration
@@ -40,6 +41,7 @@ func NewConfig(admissionV1Enabled, namespaceSelectorEnabled bool) Config {
 		svcName:                  config.Datadog.GetString("admission_controller.service_name"),
 		svcPort:                  int32(443),
 		timeout:                  config.Datadog.GetInt32("admission_controller.timeout_seconds"),
+		failurePolicy:            config.Datadog.GetString("admission_controller.failure_policy"),
 	}
 }
 
@@ -52,6 +54,7 @@ func (w *Config) getServiceNs() string       { return w.namespace }
 func (w *Config) getServiceName() string     { return w.svcName }
 func (w *Config) getServicePort() int32      { return w.svcPort }
 func (w *Config) getTimeout() int32          { return w.timeout }
+func (w *Config) getFailurePolicy() string   { return w.failurePolicy }
 func (w *Config) configName(suffix string) string {
 	return strings.ReplaceAll(fmt.Sprintf("%s.%s", w.webhookName, suffix), "-", ".")
 }

pkg/clusteragent/admission/controllers/webhook/controller_v1.go

@@ -10,6 +10,7 @@ package webhook
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"github.com/DataDog/datadog-agent/pkg/config"
@@ -198,11 +199,11 @@ func (c *ControllerV1) generateTemplates() {
 }
 
 func (c *ControllerV1) getWebhookSkeleton(nameSuffix, path string) admiv1.MutatingWebhook {
-	failurePolicy := admiv1.Ignore
 	matchPolicy := admiv1.Exact
 	sideEffects := admiv1.SideEffectClassNone
 	port := c.config.getServicePort()
 	timeout := c.config.getTimeout()
+	failurePolicy := c.getAdmiV1FailurePolicy()
 	webhook := admiv1.MutatingWebhook{
 		Name: c.config.configName(nameSuffix),
 		ClientConfig: admiv1.WebhookClientConfig{
@@ -242,3 +243,16 @@ func (c *ControllerV1) getWebhookSkeleton(nameSuffix, path string) admiv1.Mutati
 
 	return webhook
 }
+
+func (c *ControllerV1) getAdmiV1FailurePolicy() admiv1.FailurePolicyType {
+	policy := strings.ToLower(c.config.getFailurePolicy())
+	switch policy {
+	case "ignore":
+		return admiv1.Ignore
+	case "fail":
+		return admiv1.Fail
+	default:
+		_ = log.Warnf("Unknown failure policy %s - defaulting to 'Ignore'", policy)
+		return admiv1.Ignore
+	}
+}

pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go

@@ -119,6 +119,59 @@ func TestUpdateOutdatedWebhookV1(t *testing.T) {
 	}
 }
 
+func TestAdmissionControllerFailureModeIgnore(t *testing.T) {
+	f := newFixtureV1(t)
+	c := f.run(t)
+	c.config = NewConfig(true, false)
+
+	holdValue := config.Datadog.Get("admission_controller.failure_policy")
+	defer config.Datadog.Set("admission_controller.failure_policy", holdValue)
+
+	config.Datadog.Set("admission_controller.failure_policy", "Ignore")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton := c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1.Ignore, *webhookSkeleton.FailurePolicy)
+
+	config.Datadog.Set("admission_controller.failure_policy", "ignore")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton = c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1.Ignore, *webhookSkeleton.FailurePolicy)
+
+	config.Datadog.Set("admission_controller.failure_policy", "BadVal")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton = c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1.Ignore, *webhookSkeleton.FailurePolicy)
+
+	config.Datadog.Set("admission_controller.failure_policy", "")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton = c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1.Ignore, *webhookSkeleton.FailurePolicy)
+}
+
+func TestAdmissionControllerFailureModeFail(t *testing.T) {
+	holdValue := config.Datadog.Get("admission_controller.failure_policy")
+	defer config.Datadog.Set("admission_controller.failure_policy", holdValue)
+
+	f := newFixtureV1(t)
+	c := f.run(t)
+
+	config.Datadog.Set("admission_controller.failure_policy", "Fail")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton := c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1.Fail, *webhookSkeleton.FailurePolicy)
+
+	config.Datadog.Set("admission_controller.failure_policy", "fail")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton = c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1.Fail, *webhookSkeleton.FailurePolicy)
+}
+
 func TestGenerateTemplatesV1(t *testing.T) {
 	mockConfig := config.Mock()
 	failurePolicy := admiv1.Ignore

pkg/clusteragent/admission/controllers/webhook/controller_v1beta1.go

@@ -10,6 +10,7 @@ package webhook
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"github.com/DataDog/datadog-agent/pkg/config"
@@ -198,11 +199,11 @@ func (c *ControllerV1beta1) generateTemplates() {
 }
 
 func (c *ControllerV1beta1) getWebhookSkeleton(nameSuffix, path string) admiv1beta1.MutatingWebhook {
-	failurePolicy := admiv1beta1.Ignore
 	matchPolicy := admiv1beta1.Exact
 	sideEffects := admiv1beta1.SideEffectClassNone
 	port := c.config.getServicePort()
 	timeout := c.config.getTimeout()
+	failurePolicy := c.getAdmiV1Beta1FailurePolicy()
 	webhook := admiv1beta1.MutatingWebhook{
 		Name: c.config.configName(nameSuffix),
 		ClientConfig: admiv1beta1.WebhookClientConfig{
@@ -242,3 +243,16 @@ func (c *ControllerV1beta1) getWebhookSkeleton(nameSuffix, path string) admiv1be
 
 	return webhook
 }
+
+func (c *ControllerV1beta1) getAdmiV1Beta1FailurePolicy() admiv1beta1.FailurePolicyType {
+	policy := strings.ToLower(c.config.getFailurePolicy())
+	switch policy {
+	case "ignore":
+		return admiv1beta1.Ignore
+	case "fail":
+		return admiv1beta1.Fail
+	default:
+		_ = log.Warnf("Unknown failure policy %s - defaulting to 'Ignore'", policy)
+		return admiv1beta1.Ignore
+	}
+}

pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go

@@ -119,6 +119,59 @@ func TestUpdateOutdatedWebhookV1beta1(t *testing.T) {
 	}
 }
 
+func TestAdmissionControllerFailureModeIgnoreV1beta1(t *testing.T) {
+	f := newFixtureV1beta1(t)
+	c := f.run(t)
+	c.config = NewConfig(true, false)
+
+	holdValue := config.Datadog.Get("admission_controller.failure_policy")
+	defer config.Datadog.Set("admission_controller.failure_policy", holdValue)
+
+	config.Datadog.Set("admission_controller.failure_policy", "Ignore")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton := c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1beta1.Ignore, *webhookSkeleton.FailurePolicy)
+
+	config.Datadog.Set("admission_controller.failure_policy", "ignore")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton = c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1beta1.Ignore, *webhookSkeleton.FailurePolicy)
+
+	config.Datadog.Set("admission_controller.failure_policy", "BadVal")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton = c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1beta1.Ignore, *webhookSkeleton.FailurePolicy)
+
+	config.Datadog.Set("admission_controller.failure_policy", "")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton = c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1beta1.Ignore, *webhookSkeleton.FailurePolicy)
+}
+
+func TestAdmissionControllerFailureModeFailV1beta1(t *testing.T) {
+	holdValue := config.Datadog.Get("admission_controller.failure_policy")
+	defer config.Datadog.Set("admission_controller.failure_policy", holdValue)
+
+	f := newFixtureV1beta1(t)
+	c := f.run(t)
+
+	config.Datadog.Set("admission_controller.failure_policy", "Fail")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton := c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1beta1.Fail, *webhookSkeleton.FailurePolicy)
+
+	config.Datadog.Set("admission_controller.failure_policy", "fail")
+	c.config = NewConfig(true, false)
+
+	webhookSkeleton = c.getWebhookSkeleton("foo", "/bar")
+	assert.Equal(t, admiv1beta1.Fail, *webhookSkeleton.FailurePolicy)
+}
+
 func TestGenerateTemplatesV1beta1(t *testing.T) {
 	mockConfig := config.Mock()
 	failurePolicy := admiv1beta1.Ignore

pkg/config/config.go

@@ -906,6 +906,7 @@ func InitConfig(config Config) {
 	config.BindEnvAndSetDefault("admission_controller.inject_tags.endpoint", "/injecttags")
 	config.BindEnvAndSetDefault("admission_controller.pod_owners_cache_validity", 10) // in minutes
 	config.BindEnvAndSetDefault("admission_controller.namespace_selector_fallback", false)
+	config.BindEnvAndSetDefault("admission_controller.failure_policy", "Ignore")
 
 	// Telemetry
 	// Enable telemetry metrics on the internals of the Agent.

pkg/config/config_template.yaml

@@ -2433,6 +2433,14 @@ api_key:
     ## Admission controller's endpoint responsible for handling tags injection requests.
     #
     # endpoint: /injecttags
+
+  ## @param failure_policy - string - optional - default: Ignore
+  ## @env DD_ADMISSION_CONTROLLER_FAILURE_POLICY - string - optional - default: Ignore
+  ## Set the failure policy for dynamic admission control.
+  ## The default of Ignore means that pods will still be admitted even if the webhook is unavailable to inject them.
+  ## Setting to Fail will require the admission controller to be present and pods to be injected before they are allowed to run.
+  #
+  # failure_policy: Ignore
 {{ end -}}
 {{- if .DockerTagging }}
 

releasenotes-dca/notes/add-admission-controller-failure-policy-b147223ae7cfbf88.yaml

@@ -0,0 +1,16 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+enhancements:
+  - |
+    Added a configuration option to admission controller to allow
+    configuration of the failure policy. Defaults to Ignore which
+    was the previous default. The default of Ignore means that pods
+    will still be admitted even if the webhook is unavailable to
+    inject them. Setting to Fail will require the admission controller
+    to be present and pods to be injected before they are allowed to run.