Refactor e2e testing scripts to be more reusable and use them instead…

Makefile

@@ -73,22 +73,31 @@ test-sanity:
 	#go test -v ./tests/sanity/...
 	echo "succeed"
 
-bin/k8s-e2e-tester: | bin
-	go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master
-
 .PHONY: test-e2e-single-az
-test-e2e-single-az: bin/k8s-e2e-tester
-	TESTCONFIG=./tester/single-az-config.yaml ${GOBIN}/k8s-e2e-tester
+test-e2e-single-az:
+	AWS_REGION=us-west-2 \
+	AWS_AVAILABILITY_ZONES=us-west-2a \
+	TEST_PATH=./tests/e2e/... \
+	GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \
+	GINKGO_SKIP="\"sc1\"|\"st1\"" \
+	./hack/e2e/run.sh
 
 .PHONY: test-e2e-multi-az
-test-e2e-multi-az: bin/k8s-e2e-tester
-	TESTCONFIG=./tester/multi-az-config.yaml ${GOBIN}/k8s-e2e-tester
+test-e2e-multi-az:
+	AWS_REGION=us-west-2 \
+	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
+	TEST_PATH=./tests/e2e/... \
+	GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" \
+	./hack/e2e/run.sh
 
 .PHONY: test-e2e-migration
 test-e2e-migration:
-	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-migration\]" ./hack/run-e2e-test
-	# TODO: enable migration test to use new framework
-	#TESTCONFIG=./tester/migration-test-config.yaml go run tester/cmd/main.go
+	AWS_REGION=us-west-2 \
+	AWS_AVAILABILITY_ZONES=us-west-2a \
+	TEST_PATH=./tests/e2e-migration/... \
+	GINKGO_FOCUS="\[ebs-csi-migration\]" \
+	EBS_CHECK_MIGRATION=true \
+	./hack/e2e/run.sh
 
 .PHONY: image-release
 image-release:

hack/e2e/.gitignore

@@ -0,0 +1 @@
+/csi-test-artifacts

hack/e2e/README.md

@@ -0,0 +1,26 @@
+# Usage
+
+run.sh will build and push a driver image, create a kops cluster, helm install the driver pointing to the built image, run ginkgo tests, then clean everything up.
+
+See below for an example.
+
+KOPS_STATE_FILE is an S3 bucket you have write access to.
+
+TEST_ID is a token used for idempotency.
+
+For more details, see the script itself.
+
+For more examples, see the top-level Makefile.
+
+```
+TEST_PATH=./tests/e2e-migration/... \
+EBS_CHECK_MIGRATION=true \
+TEST_ID=18512 \
+CLEAN=false \
+KOPS_STATE_FILE=s3://mattwon \
+AWS_REGION=us-west-2 \
+AWS_AVAILABILITY_ZONES=us-west-2a \
+GINKGO_FOCUS=Dynamic.\*xfs.\*should.store.data \
+GINKGO_NODES=1 \
+./hack/e2e/run.sh
+```

hack/e2e/ebs.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -uo pipefail
+
+BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+source "${BASE_DIR}"/util.sh
+
+function ebs_check_migration() {
+  loudecho "Checking migration"
+  # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
+  # Find the controller-manager log and read its metrics to verify
+  NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
+  kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system &
+
+  # Ensure port forwarding succeeded
+  n=0
+  until [ "$n" -ge 30 ]; do
+    set +e
+    HEALTHZ=$(curl -s 127.0.0.1:10252/healthz)
+    set -e
+    if [[ ${HEALTHZ} == "ok" ]]; then
+      loudecho "Port forwarding succeeded"
+      break
+    else
+      loudecho "Port forwarding is not yet ready"
+    fi
+    n=$((n + 1))
+    sleep 1
+  done
+  if [[ "$n" -eq 30 ]]; then
+    loudecho "Timed out waiting for port forward"
+    for PROC in $(jobs -p); do
+      kill "${PROC}"
+    done
+    return 1
+  fi
+
+  set +e
+  curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"'
+  CSI_CALLED=${PIPESTATUS[1]}
+  set -e
+
+  set +e
+  curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"'
+  INTREE_CALLED=${PIPESTATUS[1]}
+  set -e
+
+  for PROC in $(jobs -p); do
+    kill "${PROC}"
+  done
+
+  loudecho "CSI_CALLED: ${CSI_CALLED}"
+  loudecho "INTREE_CALLED: ${INTREE_CALLED}"
+
+  # if CSI was called and In-tree was not called, return 0/true/success
+  if [ "${CSI_CALLED}" == 0 ] && [ "${INTREE_CALLED}" == 1 ]; then
+    echo 0
+  else
+    echo 1
+  fi
+}

hack/e2e/ecr.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -uo pipefail
+
+BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+source "${BASE_DIR}"/util.sh
+
+function ecr_build_and_push() {
+  REGION=${1}
+  AWS_ACCOUNT_ID=${2}
+  IMAGE_NAME=${3}
+  IMAGE_TAG=${4}
+  set +e
+  if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then
+    set -e
+    loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
+  else
+    set -e
+    loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
+    aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com
+    docker build -t "${IMAGE_NAME}":"${IMAGE_TAG}" .
+    docker push "${IMAGE_NAME}":"${IMAGE_TAG}"
+  fi
+}

hack/utils/helm.sh → hack/e2e/helm.sh

@@ -2,7 +2,7 @@
 
 set -uo pipefail
 
-helm::install() {
+function helm_install() {
   INSTALL_PATH=${1}
   if [[ ! -e ${INSTALL_PATH}/helm ]]; then
     curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3

hack/e2e/kops.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+set -uo pipefail
+
+OS_ARCH=$(go env GOOS)-amd64
+
+BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
+source "${BASE_DIR}"/util.sh
+
+function kops_install() {
+  INSTALL_PATH=${1}
+  KOPS_VERSION=${2}
+  if [[ ! -e ${INSTALL_PATH}/kops ]]; then
+    KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH}
+    curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops
+    chmod +x "${INSTALL_PATH}"/kops
+  fi
+}
+
+function kops_create_cluster() {
+  SSH_KEY_PATH=${1}
+  KOPS_STATE_FILE=${2}
+  CLUSTER_NAME=${3}
+  KOPS_BIN=${4}
+  ZONES=${5}
+  INSTANCE_TYPE=${6}
+  K8S_VERSION=${7}
+  TEST_DIR=${8}
+  KOPS_FEATURE_GATES_FILE=${10}
+  KOPS_ADDITIONAL_POLICIES_FILE=${11}
+
+  loudecho "Generating SSH key $SSH_KEY_PATH"
+  if [[ ! -e ${SSH_KEY_PATH} ]]; then
+    ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
+  fi
+
+  set +e
+  if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
+    set -e
+    loudecho "Updating cluster $CLUSTER_NAME"
+  else
+    set -e
+    loudecho "Creating cluster $CLUSTER_NAME"
+    ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
+      --zones "${ZONES}" \
+      --node-count=3 \
+      --node-size="${INSTANCE_TYPE}" \
+      --kubernetes-version="${K8S_VERSION}" \
+      --ssh-public-key="${SSH_KEY_PATH}".pub \
+      "${CLUSTER_NAME}"
+  fi
+
+  CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml
+  ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o yaml > "${CLUSTER_YAML_PATH}"
+  cat "${KOPS_FEATURE_GATES_FILE}" >> "${CLUSTER_YAML_PATH}"
+  cat "${KOPS_ADDITIONAL_POLICIES_FILE}" >> "${CLUSTER_YAML_PATH}"
+  ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}"
+  ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
+
+  loudecho "Validating cluster $CLUSTER_NAME"
+  ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m
+  return $?
+}
+
+function kops_delete_cluster() {
+  KOPS_BIN=${1}
+  CLUSTER_NAME=${2}
+  KOPS_STATE_FILE=${3}
+  loudecho "Deleting cluster ${CLUSTER_NAME}"
+  ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
+}