Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: improve e2e tests #105

Merged
merged 1 commit into from
Aug 8, 2024
Merged

feat: improve e2e tests #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions e2e/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,7 @@ func TestMain(m *testing.M) {

// Set up cluster
if useRealCluster {
path := conf.ResolveKubeConfigFile()
cfg := envconf.NewWithKubeConfig(path)
cfg := envconf.NewWithKubeConfig(conf.ResolveKubeConfigFile())

if context := os.Getenv("USE_CONTEXT"); context != "" {
cfg.WithKubeContext(context)
Expand Down Expand Up @@ -233,8 +232,6 @@ func (e *reverseFinishEnvironment) Run(m *testing.M) int {
return e.Environment.Run(m)
}

// ======== VAULT ========

func installVault(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
r, err := resources.New(cfg.Client().RESTConfig())
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion e2e/test/deployment-init-seccontext-vault.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ spec:
containers:
- name: alpine
image: alpine
command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
env:
- name: AWS_SECRET_ACCESS_KEY
value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY
Expand Down
4 changes: 2 additions & 2 deletions e2e/test/deployment-seccontext-vault.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ spec:
initContainers:
- name: init-ubuntu
image: ubuntu
command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
env:
- name: AWS_SECRET_ACCESS_KEY
value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY
Expand All @@ -36,7 +36,7 @@ spec:
containers:
- name: alpine
image: alpine
command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
env:
- name: AWS_SECRET_ACCESS_KEY
value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY
Expand Down
4 changes: 2 additions & 2 deletions e2e/test/deployment-vault.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ spec:
initContainers:
- name: init-ubuntu
image: ubuntu
command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
env:
- name: AWS_SECRET_ACCESS_KEY
value: vault:secret/data/accounts/aws#${.AWS_SECRET_ACCESS_KEY} # Go templates are also supported with ${} delimiters
Expand All @@ -33,7 +33,7 @@ spec:
containers:
- name: alpine
image: alpine
command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
env:
- name: AWS_SECRET_ACCESS_KEY
value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY
Expand Down
104 changes: 86 additions & 18 deletions e2e/webhook_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/e2e-framework/klient/decoder"
"sigs.k8s.io/e2e-framework/klient/k8s/resources"
"sigs.k8s.io/e2e-framework/klient/wait"
Expand Down Expand Up @@ -137,24 +138,41 @@ func TestPodMutation(t *testing.T) {
}).
Assess("security context defaults are correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
r := cfg.Client().Resources()

pods := &v1.PodList{}

err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault"))
require.NoError(t, err)

if len(pods.Items) == 0 {
t.Fatal("no pods found")
}
assert.NotEmpty(t, pods.Items, "no pods found")

securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext

assert.Nil(t, securityContext.RunAsNonRoot)
assert.Nil(t, securityContext.RunAsUser)
assert.Nil(t, securityContext.RunAsGroup)

return ctx
}).
Assess("secret values are injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
r := cfg.Client().Resources()
pods := &v1.PodList{}

err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault"))
require.NoError(t, err)

assert.NotEmpty(t, pods.Items, "no pods found")

// wait for the container to become available
err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
require.NoError(t, err)

initContainerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.InitContainers[1].Name)
assert.Contains(t, initContainerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")

containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")

return ctx
}).
Feature()

deploymentSeccontextVault := applyResource(features.New("deployment-seccontext-vault"), "deployment-seccontext-vault.yaml").
Expand All @@ -169,6 +187,27 @@ func TestPodMutation(t *testing.T) {

return ctx
}).
Assess("secret values are injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
r := cfg.Client().Resources()
pods := &v1.PodList{}

err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault"))
require.NoError(t, err)

assert.NotEmpty(t, pods.Items, "no pods found")

// wait for the container to become available
err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
require.NoError(t, err)

initContainerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.InitContainers[1].Name)
assert.Contains(t, initContainerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")

containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")

return ctx
}).
Feature()

deploymentTemplatingVault := applyResource(features.New("deployment-template-vault"), "deployment-template-vault.yaml").
Expand All @@ -187,29 +226,23 @@ func TestPodMutation(t *testing.T) {
}).
Assess("config template", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
r := cfg.Client().Resources()

pods := &v1.PodList{}

err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-template-vault"))
require.NoError(t, err)

if len(pods.Items) == 0 {
t.Fatal("no pods found")
}
assert.NotEmpty(t, pods.Items, "no pods found")

// wait for the container to become available
err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
require.NoError(t, err)

var stdout, stderr bytes.Buffer
podName := pods.Items[0].Name
command := []string{"cat", "/vault/secrets/config.yaml"}

if err := r.ExecInPod(context.TODO(), cfg.Namespace(), podName, "alpine", command, &stdout, &stderr); err != nil {
if err := r.ExecInPod(ctx, cfg.Namespace(), pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name, command, &stdout, &stderr); err != nil {
t.Log(stderr.String())
t.Fatal(err)
}

assert.Equal(t, "\n {\n \"id\": \"secretId\",\n \"key\": \"s3cr3t\"\n }\n \n ", stdout.String())

return ctx
Expand All @@ -230,22 +263,18 @@ func TestPodMutation(t *testing.T) {
}).
Assess("security context is correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
r := cfg.Client().Resources()

pods := &v1.PodList{}

err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext-vault"))
require.NoError(t, err)

if len(pods.Items) == 0 {
t.Fatal("no pods found")
}
assert.NotEmpty(t, pods.Items, "no pods found")

// wait for the container to become available
err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
require.NoError(t, err)

securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext

require.NotNil(t, securityContext.RunAsNonRoot)
assert.Equal(t, true, *securityContext.RunAsNonRoot)
require.NotNil(t, securityContext.RunAsUser)
Expand All @@ -255,6 +284,24 @@ func TestPodMutation(t *testing.T) {

return ctx
}).
Assess("secret value is injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
r := cfg.Client().Resources()
pods := &v1.PodList{}

err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext-vault"))
require.NoError(t, err)

assert.NotEmpty(t, pods.Items, "no pods found")

// wait for the container to become available
err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
require.NoError(t, err)

containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")

return ctx
}).
Feature()

testenv.Test(t, deploymentVault, deploymentSeccontextVault, deploymentTemplatingVault, deploymentInitSeccontextVault)
Expand Down Expand Up @@ -283,3 +330,24 @@ func applyResource(builder *features.FeatureBuilder, file string) *features.Feat
return ctx
})
}

func getLogsFromContainer(t *testing.T, ctx context.Context, cfg *envconf.Config, podName string, containerName string) string {
clientset, err := kubernetes.NewForConfig(cfg.Client().RESTConfig())
require.NoError(t, err)

req := clientset.CoreV1().Pods(cfg.Namespace()).GetLogs(
podName,
&v1.PodLogOptions{
Container: containerName,
})

podLogs, err := req.Stream(ctx)
require.NoError(t, err)
defer podLogs.Close()

var buf bytes.Buffer
_, err = buf.ReadFrom(podLogs)
require.NoError(t, err)

return buf.String()
}
Loading