From 751df82c4163d4a7bffad0df2922815f1a1a77a3 Mon Sep 17 00:00:00 2001 From: Bence Csati Date: Sun, 30 Jun 2024 13:26:05 +0200 Subject: [PATCH] feat: improve e2e tests Signed-off-by: Bence Csati feat: improve e2e tests Signed-off-by: Bence Csati --- e2e/main_test.go | 5 +- .../deployment-init-seccontext-vault.yaml | 2 +- e2e/test/deployment-seccontext-vault.yaml | 4 +- e2e/test/deployment-vault.yaml | 4 +- e2e/webhook_test.go | 104 +++++++++++++++--- 5 files changed, 92 insertions(+), 27 deletions(-) diff --git a/e2e/main_test.go b/e2e/main_test.go index 18c0fb5..3be96dd 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -64,8 +64,7 @@ func TestMain(m *testing.M) { // Set up cluster if useRealCluster { - path := conf.ResolveKubeConfigFile() - cfg := envconf.NewWithKubeConfig(path) + cfg := envconf.NewWithKubeConfig(conf.ResolveKubeConfigFile()) if context := os.Getenv("USE_CONTEXT"); context != "" { cfg.WithKubeContext(context) @@ -233,8 +232,6 @@ func (e *reverseFinishEnvironment) Run(m *testing.M) int { return e.Environment.Run(m) } -// ======== VAULT ======== - func installVault(ctx context.Context, cfg *envconf.Config) (context.Context, error) { r, err := resources.New(cfg.Client().RESTConfig()) if err != nil { diff --git a/e2e/test/deployment-init-seccontext-vault.yaml b/e2e/test/deployment-init-seccontext-vault.yaml index b7536ff..d2bfc17 100644 --- a/e2e/test/deployment-init-seccontext-vault.yaml +++ b/e2e/test/deployment-init-seccontext-vault.yaml @@ -25,7 +25,7 @@ spec: containers: - name: alpine image: alpine - command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"] + command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"] env: - name: AWS_SECRET_ACCESS_KEY value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY diff --git a/e2e/test/deployment-seccontext-vault.yaml b/e2e/test/deployment-seccontext-vault.yaml index 6ff8447..0d0eab5 100644 --- a/e2e/test/deployment-seccontext-vault.yaml +++ b/e2e/test/deployment-seccontext-vault.yaml @@ -25,7 +25,7 @@ spec: initContainers: - name: init-ubuntu image: ubuntu - command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo initContainers ready"] + command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo initContainers ready"] env: - name: AWS_SECRET_ACCESS_KEY value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY @@ -36,7 +36,7 @@ spec: containers: - name: alpine image: alpine - command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"] + command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"] env: - name: AWS_SECRET_ACCESS_KEY value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY diff --git a/e2e/test/deployment-vault.yaml b/e2e/test/deployment-vault.yaml index b190a3c..3a28f7a 100644 --- a/e2e/test/deployment-vault.yaml +++ b/e2e/test/deployment-vault.yaml @@ -22,7 +22,7 @@ spec: initContainers: - name: init-ubuntu image: ubuntu - command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo initContainers ready"] + command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo initContainers ready"] env: - name: AWS_SECRET_ACCESS_KEY value: vault:secret/data/accounts/aws#${.AWS_SECRET_ACCESS_KEY} # Go templates are also supported with ${} delimiters @@ -33,7 +33,7 @@ spec: containers: - name: alpine image: alpine - command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"] + command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"] env: - name: AWS_SECRET_ACCESS_KEY value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY diff --git a/e2e/webhook_test.go b/e2e/webhook_test.go index 4a0389f..5a62e8d 100644 --- a/e2e/webhook_test.go +++ b/e2e/webhook_test.go @@ -30,6 +30,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/e2e-framework/klient/decoder" "sigs.k8s.io/e2e-framework/klient/k8s/resources" "sigs.k8s.io/e2e-framework/klient/wait" @@ -137,24 +138,41 @@ func TestPodMutation(t *testing.T) { }). Assess("security context defaults are correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { r := cfg.Client().Resources() - pods := &v1.PodList{} err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault")) require.NoError(t, err) - if len(pods.Items) == 0 { - t.Fatal("no pods found") - } + assert.NotEmpty(t, pods.Items, "no pods found") securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext - assert.Nil(t, securityContext.RunAsNonRoot) assert.Nil(t, securityContext.RunAsUser) assert.Nil(t, securityContext.RunAsGroup) return ctx }). + Assess("secret values are injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + r := cfg.Client().Resources() + pods := &v1.PodList{} + + err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault")) + require.NoError(t, err) + + assert.NotEmpty(t, pods.Items, "no pods found") + + // wait for the container to become available + err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout)) + require.NoError(t, err) + + initContainerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.InitContainers[1].Name) + assert.Contains(t, initContainerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t") + + containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name) + assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t") + + return ctx + }). Feature() deploymentSeccontextVault := applyResource(features.New("deployment-seccontext-vault"), "deployment-seccontext-vault.yaml"). @@ -169,6 +187,27 @@ func TestPodMutation(t *testing.T) { return ctx }). + Assess("secret values are injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + r := cfg.Client().Resources() + pods := &v1.PodList{} + + err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault")) + require.NoError(t, err) + + assert.NotEmpty(t, pods.Items, "no pods found") + + // wait for the container to become available + err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout)) + require.NoError(t, err) + + initContainerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.InitContainers[1].Name) + assert.Contains(t, initContainerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t") + + containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name) + assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t") + + return ctx + }). Feature() deploymentTemplatingVault := applyResource(features.New("deployment-template-vault"), "deployment-template-vault.yaml"). @@ -187,29 +226,23 @@ func TestPodMutation(t *testing.T) { }). Assess("config template", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { r := cfg.Client().Resources() - pods := &v1.PodList{} err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-template-vault")) require.NoError(t, err) - if len(pods.Items) == 0 { - t.Fatal("no pods found") - } + assert.NotEmpty(t, pods.Items, "no pods found") // wait for the container to become available err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout)) require.NoError(t, err) var stdout, stderr bytes.Buffer - podName := pods.Items[0].Name command := []string{"cat", "/vault/secrets/config.yaml"} - - if err := r.ExecInPod(context.TODO(), cfg.Namespace(), podName, "alpine", command, &stdout, &stderr); err != nil { + if err := r.ExecInPod(ctx, cfg.Namespace(), pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name, command, &stdout, &stderr); err != nil { t.Log(stderr.String()) t.Fatal(err) } - assert.Equal(t, "\n {\n \"id\": \"secretId\",\n \"key\": \"s3cr3t\"\n }\n \n ", stdout.String()) return ctx @@ -230,22 +263,18 @@ func TestPodMutation(t *testing.T) { }). Assess("security context is correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { r := cfg.Client().Resources() - pods := &v1.PodList{} err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext-vault")) require.NoError(t, err) - if len(pods.Items) == 0 { - t.Fatal("no pods found") - } + assert.NotEmpty(t, pods.Items, "no pods found") // wait for the container to become available err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout)) require.NoError(t, err) securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext - require.NotNil(t, securityContext.RunAsNonRoot) assert.Equal(t, true, *securityContext.RunAsNonRoot) require.NotNil(t, securityContext.RunAsUser) @@ -255,6 +284,24 @@ func TestPodMutation(t *testing.T) { return ctx }). + Assess("secret value is injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + r := cfg.Client().Resources() + pods := &v1.PodList{} + + err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext-vault")) + require.NoError(t, err) + + assert.NotEmpty(t, pods.Items, "no pods found") + + // wait for the container to become available + err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout)) + require.NoError(t, err) + + containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name) + assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t") + + return ctx + }). Feature() testenv.Test(t, deploymentVault, deploymentSeccontextVault, deploymentTemplatingVault, deploymentInitSeccontextVault) @@ -283,3 +330,24 @@ func applyResource(builder *features.FeatureBuilder, file string) *features.Feat return ctx }) } + +func getLogsFromContainer(t *testing.T, ctx context.Context, cfg *envconf.Config, podName string, containerName string) string { + clientset, err := kubernetes.NewForConfig(cfg.Client().RESTConfig()) + require.NoError(t, err) + + req := clientset.CoreV1().Pods(cfg.Namespace()).GetLogs( + podName, + &v1.PodLogOptions{ + Container: containerName, + }) + + podLogs, err := req.Stream(ctx) + require.NoError(t, err) + defer podLogs.Close() + + var buf bytes.Buffer + _, err = buf.ReadFrom(podLogs) + require.NoError(t, err) + + return buf.String() +}