test: add an e2e job for arc scenario (#34)

Signed-off-by: Ernest Wong <[email protected]>

.pipelines/pr.yaml

@@ -53,6 +53,9 @@ jobs:
       matrix:
         aks_linux:
           REGISTRY: upstreamk8sci.azurecr.io/aad-pod-managed-identity
+        arc:
+          REGISTRY: upstreamk8sci.azurecr.io/aad-pod-managed-identity
+          ARC_CLUSTER: "true"
         kind_v1_19_7:
           KIND_NODE_VERSION: v1.19.7
           LOCAL_ONLY: "true"

.pipelines/publish.yaml

@@ -1,6 +1,8 @@
 trigger:
   - main
 
+pr: none
+
 pool: Upstream Pool
 
 jobs:

Makefile

@@ -106,6 +106,10 @@ run: generate fmt vet manifests
 	go run .cmd/webhook/main.go
 
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
+ARC_CLUSTER ?= false
+AZURE_ENVIRONMENT ?=
+AZURE_TENANT_ID ?=
+
 .PHONY: deploy
 deploy: $(KUBECTL) $(KUSTOMIZE) $(ENVSUBST)
 	$(MAKE) manifests install-cert-manager
@@ -204,20 +208,16 @@ GINKGO_FOCUS ?=
 GINKGO_SKIP ?=
 GINKGO_NODES ?= 1
 GINKGO_NO_COLOR ?= false
-GINKGO_ARGS ?=
+GINKGO_ARGS ?= -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" -nodes=$(GINKGO_NODES) -noColor=$(GINKGO_NO_COLOR)
 
 # E2E configurations
 E2E_ARGS ?=
 KUBECONFIG ?= $(HOME)/.kube/config
 
 .PHONY: test-e2e-run
 test-e2e-run: $(E2E_TEST) $(GINKGO)
-	$(GINKGO) -v -trace \
-		-focus="$(GINKGO_FOCUS)" \
-		-skip="$(GINKGO_SKIP)" \
-		-nodes=$(GINKGO_NODES) \
-		-noColor=$(GINKGO_NO_COLOR) \
-		$(E2E_TEST) -- -kubeconfig=$(KUBECONFIG) $(E2E_ARGS)
+	$(GINKGO) -v -trace $(GINKGO_ARGS) \
+		$(E2E_TEST) -- -kubeconfig=$(KUBECONFIG) -e2e.arc-cluster=$(ARC_CLUSTER) $(E2E_ARGS)
 
 .PHONY: test-e2e
 test-e2e: $(KUBECTL)

config/manager/manager.yaml

@@ -25,6 +25,8 @@ spec:
       containers:
       - command:
         - /manager
+        args:
+        - -arc-cluster=${ARC_CLUSTER:-false}
         image: manager:latest
         imagePullPolicy: IfNotPresent
         name: manager

scripts/ci-e2e.sh

@@ -40,6 +40,7 @@ create_cluster_and_deploy() {
     if [[ "$(should_create_aks_cluster)" == "true" ]]; then
       echo "Creating an AKS cluster '${CLUSTER_NAME}'"
       az group create --name "${CLUSTER_NAME}" --location "$(get_random_region)" > /dev/null
+      # TODO(chewong): ability to create an arc-enabled cluster
       az aks create \
         --resource-group "${CLUSTER_NAME}" \
         --name "${CLUSTER_NAME}" \

test/e2e/e2e.go

@@ -17,8 +17,6 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
-
-	_ "github.com/Azure/aad-pod-managed-identity/test/e2e/webhook"
 )
 
 var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {

test/e2e/e2e_test.go

@@ -11,6 +11,14 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework/config"
 )
 
+var (
+	arcCluster bool
+)
+
+func init() {
+	flag.BoolVar(&arcCluster, "e2e.arc-cluster", false, "Running on arc cluster")
+}
+
 // handleFlags sets up all flags and parses the command line.
 func handleFlags() {
 	config.CopyFlags(config.Flags, flag.CommandLine)

test/e2e/webhook/webhook.go → test/e2e/webhook.go

@@ -1,6 +1,6 @@
 // +build e2e
 
-package webhook
+package e2e
 
 import (
 	"context"
@@ -23,7 +23,7 @@ var _ = ginkgo.Describe("Webhook", func() {
 	ginkgo.It("should mutate a pod with a labeled service account", func() {
 		serviceAccount := createServiceAccount(f, map[string]string{webhook.UsePodIdentityLabel: "true"}, nil)
 		pod := createPodWithServiceAccount(f, serviceAccount)
-		validateMutatedPod(pod)
+		validateMutatedPod(f, pod)
 	})
 })
 
@@ -74,7 +74,7 @@ func createPodWithServiceAccount(f *framework.Framework, serviceAccount string)
 	return createdPod
 }
 
-func validateMutatedPod(pod *corev1.Pod) {
+func validateMutatedPod(f *framework.Framework, pod *corev1.Pod) {
 	for _, container := range pod.Spec.Containers {
 		m := make(map[string]struct{})
 		for _, env := range container.Env {
@@ -110,8 +110,7 @@ func validateMutatedPod(pod *corev1.Pod) {
 		}
 	}
 
-	framework.Logf("ensuring that the service account token volume is projected to %s as azure-identity-token", pod.Name)
-	expirationSeconds := webhook.DefaultServiceAccountTokenExpiration
+	framework.Logf("ensuring that the token volume is projected to %s as azure-identity-token", pod.Name)
 	defaultMode := int32(420)
 	found := false
 	for _, volume := range pod.Spec.Volumes {
@@ -121,15 +120,7 @@ func validateMutatedPod(pod *corev1.Pod) {
 				Name: webhook.TokenFilePathName,
 				VolumeSource: corev1.VolumeSource{
 					Projected: &corev1.ProjectedVolumeSource{
-						Sources: []corev1.VolumeProjection{
-							{
-								ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
-									Path:              webhook.TokenFilePathName,
-									ExpirationSeconds: &expirationSeconds,
-									Audience:          fmt.Sprintf("%s/federatedidentity", strings.TrimRight(azure.PublicCloud.ActiveDirectoryEndpoint, "/")),
-								},
-							},
-						},
+						Sources:     getVolumeProjectionSources(f, pod.Spec.ServiceAccountName),
 						DefaultMode: &defaultMode,
 					},
 				},
@@ -138,6 +129,47 @@ func validateMutatedPod(pod *corev1.Pod) {
 		}
 	}
 	if !found {
-		framework.Failf("pod %s does not have azure-identity-token as a projected service account token volume")
+		framework.Failf("pod %s does not have azure-identity-token as a projected token volume", pod.Name)
+	}
+}
+
+func getVolumeProjectionSources(f *framework.Framework, serviceAccountName string) []corev1.VolumeProjection {
+	expirationSeconds := webhook.DefaultServiceAccountTokenExpiration
+	if arcCluster {
+		// TODO(chewong): remove this secret creation process once we stopped using fake arc cluster
+		secretName := fmt.Sprintf("localtoken-%s", serviceAccountName)
+		secret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      secretName,
+				Namespace: f.Namespace.Name,
+			},
+			Type: corev1.SecretTypeOpaque,
+			Data: map[string][]byte{
+				"token": []byte("fake token"),
+			},
+		}
+		_, err := f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(context.TODO(), secret, metav1.CreateOptions{})
+		framework.ExpectNoError(err, "failed to create secret %s", secretName)
+
+		return []corev1.VolumeProjection{{
+			Secret: &corev1.SecretProjection{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: secretName,
+				},
+				Items: []corev1.KeyToPath{
+					{
+						Key:  "token",
+						Path: webhook.TokenFilePathName,
+					},
+				},
+			},
+		}}
+	}
+	return []corev1.VolumeProjection{{
+		ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
+			Path:              webhook.TokenFilePathName,
+			ExpirationSeconds: &expirationSeconds,
+			Audience:          fmt.Sprintf("%s/federatedidentity", strings.TrimRight(azure.PublicCloud.ActiveDirectoryEndpoint, "/")),
+		}},
 	}
 }