diff --git a/hack/e2e/ebs.sh b/hack/e2e/ebs.sh index 0ca3646bc1..9a80611d20 100644 --- a/hack/e2e/ebs.sh +++ b/hack/e2e/ebs.sh @@ -6,11 +6,13 @@ BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") source "${BASE_DIR}"/util.sh function ebs_check_migration() { + KUBECONFIG=${1} + loudecho "Checking migration" # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com # Find the controller-manager log and read its metrics to verify - NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name") - kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system & + NODE=$(kubectl get node -l kubernetes.io/role=master -o json --kubeconfig "${KUBECONFIG}" | jq -r ".items[].metadata.name") + kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system --kubeconfig "${KUBECONFIG}" & # Ensure port forwarding succeeded n=0 diff --git a/hack/e2e/eksctl.sh b/hack/e2e/eksctl.sh new file mode 100644 index 0000000000..2d96860405 --- /dev/null +++ b/hack/e2e/eksctl.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +set -euo pipefail + +function eksctl_install() { + INSTALL_PATH=${1} + if [[ ! -e ${INSTALL_PATH}/eksctl ]]; then + EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" + curl --silent --location "${EKSCTL_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}" + chmod +x "${INSTALL_PATH}"/eksctl + fi +} + +function eksctl_create_cluster() { + SSH_KEY_PATH=${1} + CLUSTER_NAME=${2} + BIN=${3} + ZONES=${4} + INSTANCE_TYPE=${5} + K8S_VERSION=${6} + CLUSTER_FILE=${7} + KUBECONFIG=${8} + + generate_ssh_key "${SSH_KEY_PATH}" + + CLUSTER_NAME="${CLUSTER_NAME//./-}" + + if eksctl_cluster_exists "${CLUSTER_NAME}" "${BIN}"; then + loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE" + ${BIN} upgrade cluster -f "${CLUSTER_FILE}" + else + loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)" + ${BIN} create cluster \ + --managed \ + --ssh-access \ + --ssh-public-key "${SSH_KEY_PATH}".pub \ + --zones "${ZONES}" \ + --nodes=3 \ + --instance-types="${INSTANCE_TYPE}" \ + --version="${K8S_VERSION}" \ + --dry-run \ + "${CLUSTER_NAME}" > "${CLUSTER_FILE}" + + # TODO implement patching + + loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE" + ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}" + fi + + loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}" + + loudecho "Getting cluster ${CLUSTER_NAME}" + ${BIN} get cluster "${CLUSTER_NAME}" + return $? +} + +function eksctl_cluster_exists() { + CLUSTER_NAME=${1} + BIN=${2} + set +e + if ${BIN} get cluster "${CLUSTER_NAME}"; then + set -e + return 0 + else + set -e + return 1 + fi +} + +function eksctl_delete_cluster() { + BIN=${1} + CLUSTER_NAME=${2} + loudecho "Deleting cluster ${CLUSTER_NAME}" + ${BIN} delete cluster "${CLUSTER_NAME}" +} diff --git a/hack/e2e/kops.sh b/hack/e2e/kops.sh index c023366767..40bca20d4f 100644 --- a/hack/e2e/kops.sh +++ b/hack/e2e/kops.sh @@ -1,6 +1,6 @@ #!/bin/bash -set -uo pipefail +set -euo pipefail OS_ARCH=$(go env GOOS)-amd64 @@ -19,33 +19,25 @@ function kops_install() { function kops_create_cluster() { SSH_KEY_PATH=${1} - KOPS_STATE_FILE=${2} - CLUSTER_NAME=${3} - KOPS_BIN=${4} - ZONES=${5} - INSTANCE_TYPE=${6} - K8S_VERSION=${7} - TEST_DIR=${8} - KOPS_PATCH_FILE=${10} - - if [[ ! -e ${SSH_KEY_PATH} ]]; then - loudecho "Generating SSH key $SSH_KEY_PATH" - ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}" - else - loudecho "Reusing SSH key $SSH_KEY_PATH" - fi - - CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.json - - set +e - if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then - set -e + CLUSTER_NAME=${2} + BIN=${3} + ZONES=${4} + INSTANCE_TYPE=${5} + K8S_VERSION=${6} + CLUSTER_FILE=${7} + KUBECONFIG=${8} + KOPS_PATCH_FILE=${9} + KOPS_STATE_FILE=${10} + + generate_ssh_key "${SSH_KEY_PATH}" + + if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then loudecho "Replacing cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" + ${BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" else - set -e loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)" - ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \ + ${BIN} create cluster --state "${KOPS_STATE_FILE}" \ + --ssh-public-key="${SSH_KEY_PATH}".pub \ --zones "${ZONES}" \ --node-count=3 \ --node-size="${INSTANCE_TYPE}" \ @@ -57,26 +49,41 @@ function kops_create_cluster() { kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${KOPS_BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" + ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" + kops create secret --state "${KOPS_STATE_FILE}" --name "${CLUSTER_NAME}" sshpublickey admin -i "${SSH_KEY_PATH}".pub fi loudecho "Updating cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" \ - --ssh-public-key="${SSH_KEY_PATH}".pub --yes + ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes - ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin + loudecho "Exporting cluster ${CLUSTER_NAME} kubecfg to ${KUBECONFIG}" + ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" loudecho "Validating cluster ${CLUSTER_NAME}" - ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m + ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}" return $? } +function kops_cluster_exists() { + CLUSTER_NAME=${1} + BIN=${2} + KOPS_STATE_FILE=${3} + set +e + if ${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then + set -e + return 0 + else + set -e + return 1 + fi +} + function kops_delete_cluster() { - KOPS_BIN=${1} + BIN=${1} CLUSTER_NAME=${2} KOPS_STATE_FILE=${3} loudecho "Deleting cluster ${CLUSTER_NAME}" - ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes + ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes } # TODO switch this to python, all this hacking with jq stinks! diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index 8f883ad6cc..3eeac41142 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -19,6 +19,7 @@ set -euo pipefail BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") source "${BASE_DIR}"/ebs.sh source "${BASE_DIR}"/ecr.sh +source "${BASE_DIR}"/eksctl.sh source "${BASE_DIR}"/helm.sh source "${BASE_DIR}"/kops.sh source "${BASE_DIR}"/util.sh @@ -29,10 +30,13 @@ DRIVER_START_TIME_THRESHOLD_SECONDS=60 TEST_ID=${TEST_ID:-$RANDOM} CLUSTER_NAME=test-cluster-${TEST_ID}.k8s.local +CLUSTER_TYPE=${CLUSTER_TYPE:-kops} TEST_DIR=${BASE_DIR}/csi-test-artifacts BIN_DIR=${TEST_DIR}/bin SSH_KEY_PATH=${TEST_DIR}/id_rsa +CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.json +KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.kubeconfig"} REGION=${AWS_REGION:-us-west-2} ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c} @@ -43,15 +47,17 @@ AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}} IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}} -K8S_VERSION=${K8S_VERSION:-1.20.4} -KOPS_VERSION=${KOPS_VERSION:-1.20.0-beta.2} +# kops: must include patch version (e.g. 1.19.1) +# eksctl: mustn't include patch version (e.g. 1.19) +K8S_VERSION=${K8S_VERSION:-1.20.6} + +KOPS_VERSION=${KOPS_VERSION:-1.20.0} KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e} KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml} HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml} TEST_PATH=${TEST_PATH:-"./tests/e2e/..."} -KUBECONFIG=${KUBECONFIG:-"${HOME}/.kube/config"} ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"} GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-e2e\]"} GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"} @@ -67,9 +73,18 @@ loudecho "Testing in region ${REGION} and zones ${ZONES}" mkdir -p "${BIN_DIR}" export PATH=${PATH}:${BIN_DIR} -loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}" -kops_install "${BIN_DIR}" "${KOPS_VERSION}" -KOPS_BIN=${BIN_DIR}/kops +if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}" + kops_install "${BIN_DIR}" "${KOPS_VERSION}" + KOPS_BIN=${BIN_DIR}/kops +elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + loudecho "Installing eksctl latest to ${BIN_DIR}" + eksctl_install "${BIN_DIR}" + EKSCTL_BIN=${BIN_DIR}/eksctl +else + loudecho "${CLUSTER_TYPE} must be kops or eksctl!" + exit 1 +fi loudecho "Installing helm to ${BIN_DIR}" helm_install "${BIN_DIR}" @@ -88,19 +103,34 @@ ecr_build_and_push "${REGION}" \ "${IMAGE_NAME}" \ "${IMAGE_TAG}" -kops_create_cluster \ - "$SSH_KEY_PATH" \ - "$KOPS_STATE_FILE" \ - "$CLUSTER_NAME" \ - "$KOPS_BIN" \ - "$ZONES" \ - "$INSTANCE_TYPE" \ - "$K8S_VERSION" \ - "$TEST_DIR" \ - "$BASE_DIR" \ - "$KOPS_PATCH_FILE" -if [[ $? -ne 0 ]]; then - exit 1 +if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + kops_create_cluster \ + "$SSH_KEY_PATH" \ + "$CLUSTER_NAME" \ + "$KOPS_BIN" \ + "$ZONES" \ + "$INSTANCE_TYPE" \ + "$K8S_VERSION" \ + "$CLUSTER_FILE" \ + "$KUBECONFIG" \ + "$KOPS_PATCH_FILE" \ + "$KOPS_STATE_FILE" + if [[ $? -ne 0 ]]; then + exit 1 + fi +elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + eksctl_create_cluster \ + "$SSH_KEY_PATH" \ + "$CLUSTER_NAME" \ + "$EKSCTL_BIN" \ + "$ZONES" \ + "$INSTANCE_TYPE" \ + "$K8S_VERSION" \ + "$CLUSTER_FILE" \ + "$KUBECONFIG" + if [[ $? -ne 0 ]]; then + exit 1 + fi fi loudecho "Deploying driver" @@ -111,15 +141,17 @@ startSec=$(date +'%s') --set image.tag="${IMAGE_TAG}" \ -f "${HELM_VALUES_FILE}" \ --wait \ + --kubeconfig "${KUBECONFIG}" \ ./charts/"${DRIVER_NAME}" if [[ -r "${EBS_SNAPSHOT_CRD}" ]]; then loudecho "Deploying snapshot CRD" - kubectl apply -f "$EBS_SNAPSHOT_CRD" + kubectl apply -f "$EBS_SNAPSHOT_CRD" \ + --kubeconfig "${KUBECONFIG}" # TODO deploy snapshot controller too instead of including in helm chart fi endSec=$(date +'%s') -secondUsed=$(( (endSec-startSec)/1 )) +secondUsed=$(((endSec - startSec) / 1)) # Set timeout threshold as 20 seconds for now, usually it takes less than 10s to startup if [ $secondUsed -gt $DRIVER_START_TIME_THRESHOLD_SECONDS ]; then loudecho "Driver start timeout, took $secondUsed but the threshold is $DRIVER_START_TIME_THRESHOLD_SECONDS. Fail the test." @@ -127,7 +159,6 @@ if [ $secondUsed -gt $DRIVER_START_TIME_THRESHOLD_SECONDS ]; then fi loudecho "Driver deployment complete, time used: $secondUsed seconds" - loudecho "Testing focus ${GINKGO_FOCUS}" eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS" set -x @@ -141,7 +172,7 @@ loudecho "TEST_PASSED: ${TEST_PASSED}" OVERALL_TEST_PASSED="${TEST_PASSED}" if [[ "${EBS_CHECK_MIGRATION}" == true ]]; then exec 5>&1 - OUTPUT=$(ebs_check_migration | tee /dev/fd/5) + OUTPUT=$(ebs_check_migration "${KUBECONFIG}" | tee /dev/fd/5) MIGRATION_PASSED=$(echo "${OUTPUT}" | tail -1) loudecho "MIGRATION_PASSED: ${MIGRATION_PASSED}" if [ "${TEST_PASSED}" == 0 ] && [ "${MIGRATION_PASSED}" == 0 ]; then @@ -153,12 +184,13 @@ if [[ "${EBS_CHECK_MIGRATION}" == true ]]; then fi fi -PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json | jq -r .items[].metadata.name) +PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name) while IFS= read -r POD; do loudecho "Printing pod ${POD} ${CONTAINER_NAME} container logs" set +e - kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" + kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" \ + --kubeconfig "${KUBECONFIG}" set -e done <<< "${PODS}" @@ -167,12 +199,19 @@ if [[ "${CLEAN}" == true ]]; then loudecho "Removing driver" ${HELM_BIN} del "${DRIVER_NAME}" \ - --namespace kube-system - - kops_delete_cluster \ - "${KOPS_BIN}" \ - "${CLUSTER_NAME}" \ - "${KOPS_STATE_FILE}" + --namespace kube-system \ + --kubeconfig "${KUBECONFIG}" + + if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + kops_delete_cluster \ + "${KOPS_BIN}" \ + "${CLUSTER_NAME}" \ + "${KOPS_STATE_FILE}" + elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + eksctl_delete_cluster \ + "${EKSCTL_BIN}" \ + "${CLUSTER_NAME}" + fi else loudecho "Not cleaning" fi diff --git a/hack/e2e/util.sh b/hack/e2e/util.sh index 46bb5b16e0..56f23e3f86 100644 --- a/hack/e2e/util.sh +++ b/hack/e2e/util.sh @@ -7,3 +7,13 @@ function loudecho() { echo "## ${1}" echo "#" } + +function generate_ssh_key() { + SSH_KEY_PATH=${1} + if [[ ! -e ${SSH_KEY_PATH} ]]; then + loudecho "Generating SSH key $SSH_KEY_PATH" + ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}" + else + loudecho "Reusing SSH key $SSH_KEY_PATH" + fi +}