feat: improve e2e tests

Signed-off-by: Bence Csati <[email protected]>
bank-vaults · Jun 30, 2024 · 0985b07 · 0985b07
1 parent a846081
commit 0985b07
Show file tree

Hide file tree

Showing 6 changed files with 158 additions and 28 deletions.
diff --git a/e2e/main_test.go b/e2e/main_test.go
@@ -64,8 +64,7 @@ func TestMain(m *testing.M) {
 
 	// Set up cluster
 	if useRealCluster {
-		path := conf.ResolveKubeConfigFile()
-		cfg := envconf.NewWithKubeConfig(path)
+		cfg := envconf.NewWithKubeConfig(conf.ResolveKubeConfigFile())
 
 		if context := os.Getenv("USE_CONTEXT"); context != "" {
 			cfg.WithKubeContext(context)
@@ -233,8 +232,6 @@ func (e *reverseFinishEnvironment) Run(m *testing.M) int {
 	return e.Environment.Run(m)
 }
 
-// ======== VAULT ========
-
 func installVault(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
 	r, err := resources.New(cfg.Client().RESTConfig())
 	if err != nil {

diff --git a/e2e/test/deployment-crd-vault.yaml b/e2e/test/deployment-crd-vault.yaml
@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-deployment-crd-vault
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: test-deployment-crd-vault
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: test-deployment-crd-vault
+      annotations:
+        secrets-webhook.security.bank-vaults.io/provider: "vault"
+        secrets-webhook.security.bank-vaults.io/vault-addr: "https://vault.default.svc.cluster.local:8200"
+        secrets-webhook.security.bank-vaults.io/vault-path: "kubernetes"
+        secrets-webhook.security.bank-vaults.io/vault-role: "default"
+        secrets-webhook.security.bank-vaults.io/vault-skip-verify: "false"
+        secrets-webhook.security.bank-vaults.io/vault-tls-secret: vault-tls
+        secrets-webhook.security.bank-vaults.io/vault-serviceaccount: "sa"
+    spec:
+      containers:
+        - name: alpine
+          image: alpine
+          command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
+          env:
+            - name: AWS_SECRET_ACCESS_KEY
+              value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY
+          resources:
+            limits:
+              memory: "128Mi"
+              cpu: "100m"
diff --git a/e2e/test/deployment-init-seccontext-vault.yaml b/e2e/test/deployment-init-seccontext-vault.yaml
@@ -25,7 +25,7 @@ spec:
       containers:
         - name: alpine
           image: alpine
-          command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
+          command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
           env:
           - name: AWS_SECRET_ACCESS_KEY
             value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY

diff --git a/e2e/test/deployment-seccontext-vault.yaml b/e2e/test/deployment-seccontext-vault.yaml
@@ -25,7 +25,7 @@ spec:
       initContainers:
       - name: init-ubuntu
         image: ubuntu
-        command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
+        command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
         env:
         - name: AWS_SECRET_ACCESS_KEY
           value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY
@@ -36,7 +36,7 @@ spec:
       containers:
       - name: alpine
         image: alpine
-        command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
+        command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
         env:
         - name: AWS_SECRET_ACCESS_KEY
           value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY

diff --git a/e2e/test/deployment-vault.yaml b/e2e/test/deployment-vault.yaml
@@ -22,7 +22,7 @@ spec:
       initContainers:
       - name: init-ubuntu
         image: ubuntu
-        command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
+        command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
         env:
         - name: AWS_SECRET_ACCESS_KEY
           value: vault:secret/data/accounts/aws#${.AWS_SECRET_ACCESS_KEY} # Go templates are also supported with ${} delimiters
@@ -33,7 +33,7 @@ spec:
       containers:
       - name: alpine
         image: alpine
-        command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
+        command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
         env:
         - name: AWS_SECRET_ACCESS_KEY
           value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY

diff --git a/e2e/webhook_test.go b/e2e/webhook_test.go
@@ -30,6 +30,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/e2e-framework/klient/decoder"
 	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
 	"sigs.k8s.io/e2e-framework/klient/wait"
@@ -137,24 +138,41 @@ func TestPodMutation(t *testing.T) {
 		}).
 		Assess("security context defaults are correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			r := cfg.Client().Resources()
-
 			pods := &v1.PodList{}
 
 			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault"))
 			require.NoError(t, err)
 
-			if len(pods.Items) == 0 {
-				t.Fatal("no pods found")
-			}
+			assert.NotEmpty(t, pods.Items, "no pods found")
 
 			securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext
-
 			assert.Nil(t, securityContext.RunAsNonRoot)
 			assert.Nil(t, securityContext.RunAsUser)
 			assert.Nil(t, securityContext.RunAsGroup)
 
 			return ctx
 		}).
+		Assess("secret values are injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := cfg.Client().Resources()
+			pods := &v1.PodList{}
+
+			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault"))
+			require.NoError(t, err)
+
+			assert.NotEmpty(t, pods.Items, "no pods found")
+
+			// wait for the container to become available
+			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
+			require.NoError(t, err)
+
+			initContainerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.InitContainers[1].Name)
+			assert.Contains(t, initContainerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
+			assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			return ctx
+		}).
 		Feature()
 
 	deploymentSeccontextVault := applyResource(features.New("deployment-seccontext-vault"), "deployment-seccontext-vault.yaml").
@@ -169,6 +187,27 @@ func TestPodMutation(t *testing.T) {
 
 			return ctx
 		}).
+		Assess("secret values are injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := cfg.Client().Resources()
+			pods := &v1.PodList{}
+
+			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault"))
+			require.NoError(t, err)
+
+			assert.NotEmpty(t, pods.Items, "no pods found")
+
+			// wait for the container to become available
+			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
+			require.NoError(t, err)
+
+			initContainerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.InitContainers[1].Name)
+			assert.Contains(t, initContainerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
+			assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			return ctx
+		}).
 		Feature()
 
 	deploymentTemplatingVault := applyResource(features.New("deployment-template-vault"), "deployment-template-vault.yaml").
@@ -187,29 +226,23 @@ func TestPodMutation(t *testing.T) {
 		}).
 		Assess("config template", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			r := cfg.Client().Resources()
-
 			pods := &v1.PodList{}
 
 			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-template-vault"))
 			require.NoError(t, err)
 
-			if len(pods.Items) == 0 {
-				t.Fatal("no pods found")
-			}
+			assert.NotEmpty(t, pods.Items, "no pods found")
 
 			// wait for the container to become available
 			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
 			require.NoError(t, err)
 
 			var stdout, stderr bytes.Buffer
-			podName := pods.Items[0].Name
 			command := []string{"cat", "/vault/secrets/config.yaml"}
-
-			if err := r.ExecInPod(context.TODO(), cfg.Namespace(), podName, "alpine", command, &stdout, &stderr); err != nil {
+			if err := r.ExecInPod(ctx, cfg.Namespace(), pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name, command, &stdout, &stderr); err != nil {
 				t.Log(stderr.String())
 				t.Fatal(err)
 			}
-
 			assert.Equal(t, "\n    {\n      \"id\": \"secretId\",\n      \"key\": \"s3cr3t\"\n    }\n    \n  ", stdout.String())
 
 			return ctx
@@ -230,22 +263,18 @@ func TestPodMutation(t *testing.T) {
 		}).
 		Assess("security context is correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			r := cfg.Client().Resources()
-
 			pods := &v1.PodList{}
 
 			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext-vault"))
 			require.NoError(t, err)
 
-			if len(pods.Items) == 0 {
-				t.Fatal("no pods found")
-			}
+			assert.NotEmpty(t, pods.Items, "no pods found")
 
 			// wait for the container to become available
 			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
 			require.NoError(t, err)
 
 			securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext
-
 			require.NotNil(t, securityContext.RunAsNonRoot)
 			assert.Equal(t, true, *securityContext.RunAsNonRoot)
 			require.NotNil(t, securityContext.RunAsUser)
@@ -255,9 +284,59 @@ func TestPodMutation(t *testing.T) {
 
 			return ctx
 		}).
+		Assess("secret value is injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := cfg.Client().Resources()
+			pods := &v1.PodList{}
+
+			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext-vault"))
+			require.NoError(t, err)
+
+			assert.NotEmpty(t, pods.Items, "no pods found")
+
+			// wait for the container to become available
+			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
+			require.NoError(t, err)
+
+			containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
+			assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			return ctx
+		}).
+		Feature()
+
+	deploymentCRDVault := applyResource(features.New("deployment-crd-vault"), "deployment-crd-vault.yaml").
+		Assess("available", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			deployment := &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "test-deployment-crd-vault", Namespace: cfg.Namespace()},
+			}
+
+			// wait for the deployment to become available
+			err := wait.For(conditions.New(cfg.Client().Resources()).DeploymentConditionMatch(deployment, appsv1.DeploymentAvailable, v1.ConditionTrue), wait.WithTimeout(defaultTimeout))
+			require.NoError(t, err)
+
+			return ctx
+		}).
+		Assess("secret value is injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := cfg.Client().Resources()
+			pods := &v1.PodList{}
+
+			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-crd-vault"))
+			require.NoError(t, err)
+
+			assert.NotEmpty(t, pods.Items, "no pods found")
+
+			// wait for the container to become available
+			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
+			require.NoError(t, err)
+
+			logs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
+			assert.Contains(t, logs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			return ctx
+		}).
 		Feature()
 
-	testenv.Test(t, deploymentVault, deploymentSeccontextVault, deploymentTemplatingVault, deploymentInitSeccontextVault)
+	testenv.Test(t, deploymentVault, deploymentSeccontextVault, deploymentTemplatingVault, deploymentInitSeccontextVault, deploymentCRDVault)
 }
 
 func applyResource(builder *features.FeatureBuilder, file string) *features.FeatureBuilder {
@@ -283,3 +362,24 @@ func applyResource(builder *features.FeatureBuilder, file string) *features.Feat
 			return ctx
 		})
 }
+
+func getLogsFromContainer(t *testing.T, ctx context.Context, cfg *envconf.Config, podName string, containerName string) string {
+	clientset, err := kubernetes.NewForConfig(cfg.Client().RESTConfig())
+	require.NoError(t, err)
+
+	req := clientset.CoreV1().Pods(cfg.Namespace()).GetLogs(
+		podName,
+		&v1.PodLogOptions{
+			Container: containerName,
+		})
+
+	podLogs, err := req.Stream(ctx)
+	require.NoError(t, err)
+	defer podLogs.Close()
+
+	var buf bytes.Buffer
+	_, err = buf.ReadFrom(podLogs)
+	require.NoError(t, err)
+
+	return buf.String()
+}