From db593a00e8587a0960c4a945dc6ca6b84c94541d Mon Sep 17 00:00:00 2001 From: Ernest Wong Date: Tue, 18 May 2021 11:44:00 -0700 Subject: [PATCH] test: add an e2e job for arc scenario (#34) Signed-off-by: Ernest Wong --- .pipelines/pr.yaml | 3 ++ .pipelines/publish.yaml | 2 + Makefile | 14 +++---- config/manager/manager.yaml | 2 + scripts/ci-e2e.sh | 1 + test/e2e/e2e.go | 2 - test/e2e/e2e_test.go | 8 ++++ test/e2e/{webhook => }/webhook.go | 62 +++++++++++++++++++++++-------- 8 files changed, 70 insertions(+), 24 deletions(-) rename test/e2e/{webhook => }/webhook.go (70%) diff --git a/.pipelines/pr.yaml b/.pipelines/pr.yaml index c281d130c..47e438b75 100644 --- a/.pipelines/pr.yaml +++ b/.pipelines/pr.yaml @@ -53,6 +53,9 @@ jobs: matrix: aks_linux: REGISTRY: upstreamk8sci.azurecr.io/aad-pod-managed-identity + arc: + REGISTRY: upstreamk8sci.azurecr.io/aad-pod-managed-identity + ARC_CLUSTER: "true" kind_v1_19_7: KIND_NODE_VERSION: v1.19.7 LOCAL_ONLY: "true" diff --git a/.pipelines/publish.yaml b/.pipelines/publish.yaml index e5ffd1614..495c643c0 100644 --- a/.pipelines/publish.yaml +++ b/.pipelines/publish.yaml @@ -1,6 +1,8 @@ trigger: - main +pr: none + pool: Upstream Pool jobs: diff --git a/Makefile b/Makefile index 3b2409296..01647b091 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,10 @@ run: generate fmt vet manifests go run .cmd/webhook/main.go # Deploy controller in the configured Kubernetes cluster in ~/.kube/config +ARC_CLUSTER ?= false +AZURE_ENVIRONMENT ?= +AZURE_TENANT_ID ?= + .PHONY: deploy deploy: $(KUBECTL) $(KUSTOMIZE) $(ENVSUBST) $(MAKE) manifests install-cert-manager @@ -204,7 +208,7 @@ GINKGO_FOCUS ?= GINKGO_SKIP ?= GINKGO_NODES ?= 1 GINKGO_NO_COLOR ?= false -GINKGO_ARGS ?= +GINKGO_ARGS ?= -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" -nodes=$(GINKGO_NODES) -noColor=$(GINKGO_NO_COLOR) # E2E configurations E2E_ARGS ?= @@ -212,12 +216,8 @@ KUBECONFIG ?= $(HOME)/.kube/config .PHONY: test-e2e-run test-e2e-run: $(E2E_TEST) $(GINKGO) - $(GINKGO) -v -trace \ - -focus="$(GINKGO_FOCUS)" \ - -skip="$(GINKGO_SKIP)" \ - -nodes=$(GINKGO_NODES) \ - -noColor=$(GINKGO_NO_COLOR) \ - $(E2E_TEST) -- -kubeconfig=$(KUBECONFIG) $(E2E_ARGS) + $(GINKGO) -v -trace $(GINKGO_ARGS) \ + $(E2E_TEST) -- -kubeconfig=$(KUBECONFIG) -e2e.arc-cluster=$(ARC_CLUSTER) $(E2E_ARGS) .PHONY: test-e2e test-e2e: $(KUBECTL) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 2a0db2734..008396193 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -25,6 +25,8 @@ spec: containers: - command: - /manager + args: + - -arc-cluster=${ARC_CLUSTER:-false} image: manager:latest imagePullPolicy: IfNotPresent name: manager diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 2c85fb386..740bf8afc 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -40,6 +40,7 @@ create_cluster_and_deploy() { if [[ "$(should_create_aks_cluster)" == "true" ]]; then echo "Creating an AKS cluster '${CLUSTER_NAME}'" az group create --name "${CLUSTER_NAME}" --location "$(get_random_region)" > /dev/null + # TODO(chewong): ability to create an arc-enabled cluster az aks create \ --resource-group "${CLUSTER_NAME}" \ --name "${CLUSTER_NAME}" \ diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 8d9ea259b..c5ffa7358 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -17,8 +17,6 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" - - _ "github.com/Azure/aad-pod-managed-identity/test/e2e/webhook" ) var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d5dfe1c2c..a58d2ffe6 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -11,6 +11,14 @@ import ( "k8s.io/kubernetes/test/e2e/framework/config" ) +var ( + arcCluster bool +) + +func init() { + flag.BoolVar(&arcCluster, "e2e.arc-cluster", false, "Running on arc cluster") +} + // handleFlags sets up all flags and parses the command line. func handleFlags() { config.CopyFlags(config.Flags, flag.CommandLine) diff --git a/test/e2e/webhook/webhook.go b/test/e2e/webhook.go similarity index 70% rename from test/e2e/webhook/webhook.go rename to test/e2e/webhook.go index 79b93f1dd..8d5222933 100644 --- a/test/e2e/webhook/webhook.go +++ b/test/e2e/webhook.go @@ -1,6 +1,6 @@ // +build e2e -package webhook +package e2e import ( "context" @@ -23,7 +23,7 @@ var _ = ginkgo.Describe("Webhook", func() { ginkgo.It("should mutate a pod with a labeled service account", func() { serviceAccount := createServiceAccount(f, map[string]string{webhook.UsePodIdentityLabel: "true"}, nil) pod := createPodWithServiceAccount(f, serviceAccount) - validateMutatedPod(pod) + validateMutatedPod(f, pod) }) }) @@ -74,7 +74,7 @@ func createPodWithServiceAccount(f *framework.Framework, serviceAccount string) return createdPod } -func validateMutatedPod(pod *corev1.Pod) { +func validateMutatedPod(f *framework.Framework, pod *corev1.Pod) { for _, container := range pod.Spec.Containers { m := make(map[string]struct{}) for _, env := range container.Env { @@ -110,8 +110,7 @@ func validateMutatedPod(pod *corev1.Pod) { } } - framework.Logf("ensuring that the service account token volume is projected to %s as azure-identity-token", pod.Name) - expirationSeconds := webhook.DefaultServiceAccountTokenExpiration + framework.Logf("ensuring that the token volume is projected to %s as azure-identity-token", pod.Name) defaultMode := int32(420) found := false for _, volume := range pod.Spec.Volumes { @@ -121,15 +120,7 @@ func validateMutatedPod(pod *corev1.Pod) { Name: webhook.TokenFilePathName, VolumeSource: corev1.VolumeSource{ Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{ - { - ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Path: webhook.TokenFilePathName, - ExpirationSeconds: &expirationSeconds, - Audience: fmt.Sprintf("%s/federatedidentity", strings.TrimRight(azure.PublicCloud.ActiveDirectoryEndpoint, "/")), - }, - }, - }, + Sources: getVolumeProjectionSources(f, pod.Spec.ServiceAccountName), DefaultMode: &defaultMode, }, }, @@ -138,6 +129,47 @@ func validateMutatedPod(pod *corev1.Pod) { } } if !found { - framework.Failf("pod %s does not have azure-identity-token as a projected service account token volume") + framework.Failf("pod %s does not have azure-identity-token as a projected token volume", pod.Name) + } +} + +func getVolumeProjectionSources(f *framework.Framework, serviceAccountName string) []corev1.VolumeProjection { + expirationSeconds := webhook.DefaultServiceAccountTokenExpiration + if arcCluster { + // TODO(chewong): remove this secret creation process once we stopped using fake arc cluster + secretName := fmt.Sprintf("localtoken-%s", serviceAccountName) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: f.Namespace.Name, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + "token": []byte("fake token"), + }, + } + _, err := f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(context.TODO(), secret, metav1.CreateOptions{}) + framework.ExpectNoError(err, "failed to create secret %s", secretName) + + return []corev1.VolumeProjection{{ + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + Items: []corev1.KeyToPath{ + { + Key: "token", + Path: webhook.TokenFilePathName, + }, + }, + }, + }} + } + return []corev1.VolumeProjection{{ + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Path: webhook.TokenFilePathName, + ExpirationSeconds: &expirationSeconds, + Audience: fmt.Sprintf("%s/federatedidentity", strings.TrimRight(azure.PublicCloud.ActiveDirectoryEndpoint, "/")), + }}, } }