Merge pull request #907 from wongma7/metadatatest

Add test-e2e-external-eks make rule that tests EKS with pod instance metadata disabled. Remove hostNetwork from DaemonSet

Makefile

@@ -108,6 +108,18 @@ test-e2e-external:
 	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
 	./hack/e2e/run.sh
 
+.PHONY: test-e2e-external-eks
+test-e2e-external-eks:
+	CLUSTER_TYPE=eksctl \
+	K8S_VERSION="1.20" \
+	HELM_VALUES_FILE="./hack/values_eksctl.yaml" \
+	AWS_REGION=us-west-2 \
+	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \
+	TEST_PATH=./tests/e2e-kubernetes/... \
+	GINKGO_FOCUS="External.Storage" \
+	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
+	./hack/e2e/run.sh
+
 .PHONY: image-release
 image-release:
 	docker build -t $(IMAGE):$(VERSION) . --target debian-base

charts/aws-ebs-csi-driver/templates/node.yaml

@@ -37,7 +37,6 @@ spec:
         {{- with .Values.node.nodeSelector }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-      hostNetwork: true
       serviceAccountName: {{ .Values.serviceAccount.node.name }}
       priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }}
       tolerations:

deploy/kubernetes/base/node.yaml

@@ -29,7 +29,6 @@ spec:
                 - fargate
       nodeSelector:
         kubernetes.io/os: linux
-      hostNetwork: true
       serviceAccountName: ebs-csi-node-sa
       priorityClassName: system-node-critical
       tolerations:

hack/e2e/eksctl.sh

@@ -20,6 +20,7 @@ function eksctl_create_cluster() {
   K8S_VERSION=${6}
   CLUSTER_FILE=${7}
   KUBECONFIG=${8}
+  EKSCTL_PATCH_FILE=${9}
 
   generate_ssh_key "${SSH_KEY_PATH}"
 
@@ -38,10 +39,13 @@ function eksctl_create_cluster() {
       --nodes=3 \
       --instance-types="${INSTANCE_TYPE}" \
       --version="${K8S_VERSION}" \
+      --disable-pod-imds \
       --dry-run \
       "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
 
-    # TODO implement patching
+    if test -f "$EKSCTL_PATCH_FILE"; then
+      eksctl_patch_cluster_file "$CLUSTER_FILE" "$EKSCTL_PATCH_FILE"
+    fi
 
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
     ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
@@ -73,3 +77,23 @@ function eksctl_delete_cluster() {
   loudecho "Deleting cluster ${CLUSTER_NAME}"
   ${BIN} delete cluster "${CLUSTER_NAME}"
 }
+
+function eksctl_patch_cluster_file() {
+  CLUSTER_FILE=${1}      # input must be yaml
+  EKSCTL_PATCH_FILE=${2} # input must be yaml
+
+  loudecho "Patching cluster $CLUSTER_NAME with $EKSCTL_PATCH_FILE"
+
+  # Temporary intermediate files for patching
+  CLUSTER_FILE_0=$CLUSTER_FILE.0
+  CLUSTER_FILE_1=$CLUSTER_FILE.1
+
+  cp "$CLUSTER_FILE" "$CLUSTER_FILE_0"
+
+  # Patch only the Cluster
+  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml > "$CLUSTER_FILE_1"
+  mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
+
+  # Done patching, overwrite original CLUSTER_FILE
+  mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml
+}

hack/e2e/kops.sh

@@ -46,7 +46,9 @@ function kops_create_cluster() {
       -o json \
       "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
 
-    kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE"
+    if test -f "$KOPS_PATCH_FILE"; then
+      kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE"
+    fi
 
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
     ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
@@ -86,10 +88,11 @@ function kops_delete_cluster() {
   ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
 }
 
-# TODO switch this to python, all this hacking with jq stinks!
+# TODO switch this to python or work exclusively with yaml, all this
+# hacking with jq stinks!
 function kops_patch_cluster_file() {
-  CLUSTER_FILE=${1}
-  KOPS_PATCH_FILE=${2}
+  CLUSTER_FILE=${1}    # input must be json
+  KOPS_PATCH_FILE=${2} # input must be yaml
 
   loudecho "Patching cluster $CLUSTER_NAME with $KOPS_PATCH_FILE"
 
@@ -116,5 +119,5 @@ function kops_patch_cluster_file() {
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
   # Done patching, overwrite original CLUSTER_FILE
-  mv "$CLUSTER_FILE_0" "$CLUSTER_FILE"
+  mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml
 }

hack/e2e/run.sh

@@ -36,7 +36,7 @@ TEST_DIR=${BASE_DIR}/csi-test-artifacts
 BIN_DIR=${TEST_DIR}/bin
 SSH_KEY_PATH=${TEST_DIR}/id_rsa
 CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.json
-KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.kubeconfig"}
+KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"}
 
 REGION=${AWS_REGION:-us-west-2}
 ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
@@ -55,6 +55,8 @@ KOPS_VERSION=${KOPS_VERSION:-1.20.0}
 KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e}
 KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml}
 
+EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml}
+
 HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml}
 
 TEST_PATH=${TEST_PATH:-"./tests/e2e/..."}
@@ -127,22 +129,29 @@ elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     "$INSTANCE_TYPE" \
     "$K8S_VERSION" \
     "$CLUSTER_FILE" \
-    "$KUBECONFIG"
+    "$KUBECONFIG" \
+    "$EKSCTL_PATCH_FILE"
   if [[ $? -ne 0 ]]; then
     exit 1
   fi
 fi
 
 loudecho "Deploying driver"
 startSec=$(date +'%s')
-"${HELM_BIN}" upgrade --install "${DRIVER_NAME}" \
-  --namespace kube-system \
-  --set image.repository="${IMAGE_NAME}" \
-  --set image.tag="${IMAGE_TAG}" \
-  -f "${HELM_VALUES_FILE}" \
-  --wait \
-  --kubeconfig "${KUBECONFIG}" \
-  ./charts/"${DRIVER_NAME}"
+
+HELM_ARGS=(upgrade --install "${DRIVER_NAME}"
+  --namespace kube-system
+  --set image.repository="${IMAGE_NAME}"
+  --set image.tag="${IMAGE_TAG}"
+  --wait
+  --kubeconfig "${KUBECONFIG}"
+  ./charts/"${DRIVER_NAME}")
+if test -f "$HELM_VALUES_FILE"; then
+  HELM_ARGS+=(-f "${HELM_VALUES_FILE}")
+fi
+set -x
+"${HELM_BIN}" "${HELM_ARGS[@]}"
+set +x
 
 if [[ -r "${EBS_SNAPSHOT_CRD}" ]]; then
   loudecho "Deploying snapshot CRD"

hack/eksctl-patch.yaml

@@ -0,0 +1,9 @@
+iam:
+  vpcResourceControllerPolicy: true
+  withOIDC: true
+  serviceAccounts:
+    - metadata:
+        name: ebs-csi-controller-sa
+        namespace: kube-system
+      wellKnownPolicies:
+        ebsCSIController: true

hack/values.yaml

@@ -3,4 +3,3 @@ controller:
   logLevel: 5
 node:
   logLevel: 5
-

hack/values_eksctl.yaml

@@ -0,0 +1,8 @@
+enableVolumeSnapshot: true
+controller:
+  logLevel: 5
+node:
+  logLevel: 5
+serviceAccount:
+  controller:
+    create: false # let eksctl create it