test: poll for service account creation and webhook readiness

.pipelines/nightly.yaml

@@ -31,7 +31,6 @@ jobs:
       # - SERVICE_ACCOUNT_ISSUER
       - group: e2e-environment-variables
     steps:
-      - template: templates/setup-go.yaml
       - script: |
           openssl genrsa -out sa.key 2048
           openssl rsa -in sa.key -pubout -out sa.pub
@@ -41,6 +40,7 @@ jobs:
           SKIP_PREFLIGHT: "true"
           SERVICE_ACCOUNT_ISSUER: $(SERVICE_ACCOUNT_ISSUER)
       - script: |
+          set -o errexit
           sed -i "s/AZURE_TENANT_ID: .*/AZURE_TENANT_ID: ${AZURE_TENANT_ID}/" manifest_staging/deploy/aad-pi-webhook.yaml
           sed -i "s/AZURE_ENVIRONMENT: .*/AZURE_ENVIRONMENT: AzurePublicCloud/" manifest_staging/deploy/aad-pi-webhook.yaml
           KUBECTL=$(pwd)/hack/tools/bin/kubectl
@@ -80,7 +80,6 @@ jobs:
           ARC_CLUSTER: "true"
           CLUSTER_NAME: "pmi-aks-arc"
     steps:
-      - template: templates/setup-go.yaml
       - script: make test-e2e
         displayName: Webhook E2E test suite
       - template: templates/publish-logs.yaml
@@ -108,7 +107,6 @@ jobs:
         upgrade_arc:
           ARC_CLUSTER: "true"
     steps:
-      - template: templates/setup-go.yaml
       - script: echo "##vso[task.setvariable variable=CLUSTER_NAME]pod-managed-identity-e2e-$(openssl rand -hex 2)"
         displayName: Set CLUSTER_NAME
       - script: make test-e2e
@@ -119,6 +117,8 @@ jobs:
         displayName: Upgrade cluster
       - script: make test-e2e
         displayName: Webhook E2E test suite
+        env:
+          SKIP_IMAGE_BUILD: "true"
       - script: az group delete --name "${CLUSTER_NAME}" --yes --no-wait || true
         displayName: Cleanup
         condition: always()
@@ -141,6 +141,8 @@ jobs:
       - group: e2e-environment-variables
       - name: LOCAL_ONLY
         value: "true"
+      - name: TEST_HELM_CHART
+        value: "true"
     strategy:
       matrix:
         kind_v1_18_19:
@@ -152,7 +154,6 @@ jobs:
         kind_v1_21_2:
           KIND_NODE_VERSION: v1.21.2
     steps:
-      - template: templates/setup-go.yaml
       - script: make test-e2e
         displayName: Webhook E2E test suite
         env:

.pipelines/pr.yaml

@@ -25,7 +25,6 @@ jobs:
     workspace:
       clean: all
     steps:
-      - template: templates/setup-go.yaml
       - script: make lint
         displayName: golangci-lint
       - script: make helm-lint
@@ -37,15 +36,13 @@ jobs:
     workspace:
       clean: all
     steps:
-      - template: templates/setup-go.yaml
       - script: make test
         displayName: Unit test
   - job: shellcheck
     timeoutInMinutes: 5
     workspace:
       clean: all
     steps:
-      - template: templates/setup-go.yaml
       - script: make shellcheck
         displayName: shellcheck
   - job:
@@ -101,7 +98,6 @@ jobs:
           LOCAL_ONLY: "true"
           TEST_HELM_CHART: "true"
     steps:
-      - template: templates/setup-go.yaml
       - script: echo "##vso[task.setvariable variable=CLUSTER_NAME]pod-managed-identity-e2e-$(openssl rand -hex 2)"
         displayName: Set CLUSTER_NAME
         condition: ne(variables.LOCAL_ONLY, 'true')

.pipelines/templates/publish-images.yaml

@@ -13,7 +13,6 @@ parameters:
     default: type=docker
 
 steps:
-  - template: setup-go.yaml
   - script: |
       echo "##vso[task.setvariable variable=REGISTRY]${{ parameters.registry }}"
       echo "##vso[task.setvariable variable=IMAGE_VERSION]${{ parameters.image_version }}"

scripts/ci-e2e.sh

@@ -12,15 +12,15 @@ cd "${REPO_ROOT}" || exit 1
 readonly CLUSTER_NAME="${CLUSTER_NAME:-pod-managed-identity-e2e-$(openssl rand -hex 2)}"
 readonly KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl"
 
-IMAGE_VERSION="$(git rev-parse --short HEAD)"
+IMAGE_VERSION="$(git describe --tags --always --dirty)"
 export IMAGE_VERSION
 
 create_cluster() {
   if [[ "${LOCAL_ONLY:-}" == "true" ]]; then
     download_service_account_keys
-    # create a kind cluster, then build and load the webhook manager image to the cluster
+    # create a kind cluster, then build and load the webhook manager image to the kind cluster
     make kind-create
-    OUTPUT_TYPE="type=docker" make docker-build-webhook
+    [[ "${SKIP_IMAGE_BUILD:-}" == "true" ]] || OUTPUT_TYPE="type=docker" make docker-build-webhook
     make kind-load-image
   else
     : "${REGISTRY:?Environment variable empty or not defined.}"
@@ -48,7 +48,7 @@ create_cluster() {
     fi
 
     echo "Building controller and deploying webhook to the cluster"
-    make docker-build-webhook
+    [[ "${SKIP_IMAGE_BUILD:-}" == "true" ]] || make docker-build-webhook
   fi
   ${KUBECTL} get nodes -owide
 }
@@ -81,8 +81,7 @@ main() {
 
   create_cluster
   make clean deploy
-  # adding a sleep here because the cert mount is not ready immediately after the deploy
-  sleep 120
+  poll_webhook_readiness
 
   if [[ -n "${WINDOWS_NODE_NAME:-}" ]]; then
     E2E_ARGS="--node-os-distro=windows ${E2E_ARGS:-}"
@@ -102,15 +101,12 @@ test_helm_chart() {
   ${KUBECTL} create namespace aad-pi-webhook-system
 
   # test helm upgrade from chart to manifest_staging/chart
-  if [[ -d "${REPO_ROOT}/charts/pod-identity-webhook" ]]; then
-    ${HELM} install pod-identity-webhook "${REPO_ROOT}/charts/pod-identity-webhook" \
-      --set azureTenantID="${AZURE_TENANT_ID}" \
-      --namespace aad-pi-webhook-system \
-      --wait
-    # adding a sleep here because the cert mount is not ready immediately after the deploy
-    sleep 120
-    make test-e2e-run
-  fi
+  ${HELM} install pod-identity-webhook "${REPO_ROOT}/charts/pod-identity-webhook" \
+    --set azureTenantID="${AZURE_TENANT_ID}" \
+    --namespace aad-pi-webhook-system \
+    --wait
+  poll_webhook_readiness
+  make test-e2e-run
 
   ${HELM} upgrade --install pod-identity-webhook "${REPO_ROOT}/manifest_staging/charts/pod-identity-webhook" \
     --set image.repository="${REGISTRY:-mcr.microsoft.com/oss/azure/aad-pod-managed-identity/webhook}" \
@@ -119,9 +115,50 @@ test_helm_chart() {
     --namespace aad-pi-webhook-system \
     --reuse-values \
     --wait
-  # adding a sleep here because the cert mount is not ready immediately after the deploy
-  sleep 120
+  poll_webhook_readiness
   make test-e2e-run
 }
 
+poll_webhook_readiness() {
+  TEST_RESOURCE=$(cat <<-EOF
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: aad-pi-webhook-system-test
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: test-service-account
+  namespace: aad-pi-webhook-system-test
+  labels:
+    azure.pod.identity/use: "true"
+  annotations:
+    azure.pod.identity/service-account-token-expiration: "100"
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-pod
+  namespace: aad-pi-webhook-system-test
+spec:
+  serviceAccountName: test-service-account
+  containers:
+  - name: nginx
+    image: nginx:1.15.8
+EOF
+)
+  for _ in {1..30}; do
+    # webhook is considered ready when it starts denying requests
+    # with invalid service account token expiration
+    if echo "${TEST_RESOURCE}" | ${KUBECTL} apply -f -; then
+      echo "${TEST_RESOURCE}" | ${KUBECTL} delete --grace-period=1 --ignore-not-found -f -
+      sleep 4
+    else
+      break
+    fi
+  done
+  echo "${TEST_RESOURCE}" | ${KUBECTL} delete --ignore-not-found -f -
+}
+
 main

test/e2e/helpers.go

@@ -12,6 +12,7 @@ import (
 	"github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -36,6 +37,17 @@ func createServiceAccount(c kubernetes.Interface, namespace, name string, labels
 	}
 	_, err := c.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), account, metav1.CreateOptions{})
 	framework.ExpectNoError(err, "failed to create service account %s", name)
+
+	// make sure the service account is created
+	// ref: https://github.com/Azure/aad-pod-managed-identity/issues/114
+	gomega.Eventually(func() bool {
+		_, err := c.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+		if apierrors.IsNotFound(err) {
+			framework.Logf("service account %s/%s is not found", namespace, name)
+		}
+		return err == nil
+	}, framework.PollShortTimeout, framework.Poll).Should(gomega.BeTrue())
+
 	framework.Logf("created service account %s", name)
 	return name
 }