diff --git a/Makefile b/Makefile index 53fd4b14f9..ce57f72498 100644 --- a/Makefile +++ b/Makefile @@ -73,22 +73,31 @@ test-sanity: #go test -v ./tests/sanity/... echo "succeed" -bin/k8s-e2e-tester: | bin - go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master - .PHONY: test-e2e-single-az -test-e2e-single-az: bin/k8s-e2e-tester - TESTCONFIG=./tester/single-az-config.yaml ${GOBIN}/k8s-e2e-tester +test-e2e-single-az: + AWS_REGION=us-west-2 \ + AWS_AVAILABILITY_ZONES=us-west-2a \ + TEST_PATH=./tests/e2e/... \ + GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \ + GINKGO_SKIP="\"sc1\"|\"st1\"" \ + ./hack/e2e/run.sh .PHONY: test-e2e-multi-az -test-e2e-multi-az: bin/k8s-e2e-tester - TESTCONFIG=./tester/multi-az-config.yaml ${GOBIN}/k8s-e2e-tester +test-e2e-multi-az: + AWS_REGION=us-west-2 \ + AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ + TEST_PATH=./tests/e2e/... \ + GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" \ + ./hack/e2e/run.sh .PHONY: test-e2e-migration test-e2e-migration: - AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-migration\]" ./hack/run-e2e-test - # TODO: enable migration test to use new framework - #TESTCONFIG=./tester/migration-test-config.yaml go run tester/cmd/main.go + AWS_REGION=us-west-2 \ + AWS_AVAILABILITY_ZONES=us-west-2a \ + TEST_PATH=./tests/e2e-migration/... \ + GINKGO_FOCUS="\[ebs-csi-migration\]" \ + EBS_CHECK_MIGRATION=true \ + ./hack/e2e/run.sh .PHONY: image-release image-release: diff --git a/hack/e2e/.gitignore b/hack/e2e/.gitignore new file mode 100644 index 0000000000..7eb9af1faf --- /dev/null +++ b/hack/e2e/.gitignore @@ -0,0 +1 @@ +/csi-test-artifacts diff --git a/hack/e2e/README.md b/hack/e2e/README.md new file mode 100644 index 0000000000..108d66ac68 --- /dev/null +++ b/hack/e2e/README.md @@ -0,0 +1,26 @@ +# Usage + +run.sh will build and push a driver image, create a kops cluster, helm install the driver pointing to the built image, run ginkgo tests, then clean everything up. + +See below for an example. + +KOPS_STATE_FILE is an S3 bucket you have write access to. + +TEST_ID is a token used for idempotency. + +For more details, see the script itself. + +For more examples, see the top-level Makefile. + +``` +TEST_PATH=./tests/e2e-migration/... \ +EBS_CHECK_MIGRATION=true \ +TEST_ID=18512 \ +CLEAN=false \ +KOPS_STATE_FILE=s3://mattwon \ +AWS_REGION=us-west-2 \ +AWS_AVAILABILITY_ZONES=us-west-2a \ +GINKGO_FOCUS=Dynamic.\*xfs.\*should.store.data \ +GINKGO_NODES=1 \ +./hack/e2e/run.sh +``` diff --git a/hack/e2e/ebs.sh b/hack/e2e/ebs.sh new file mode 100644 index 0000000000..0ca3646bc1 --- /dev/null +++ b/hack/e2e/ebs.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -uo pipefail + +BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +source "${BASE_DIR}"/util.sh + +function ebs_check_migration() { + loudecho "Checking migration" + # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com + # Find the controller-manager log and read its metrics to verify + NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name") + kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system & + + # Ensure port forwarding succeeded + n=0 + until [ "$n" -ge 30 ]; do + set +e + HEALTHZ=$(curl -s 127.0.0.1:10252/healthz) + set -e + if [[ ${HEALTHZ} == "ok" ]]; then + loudecho "Port forwarding succeeded" + break + else + loudecho "Port forwarding is not yet ready" + fi + n=$((n + 1)) + sleep 1 + done + if [[ "$n" -eq 30 ]]; then + loudecho "Timed out waiting for port forward" + for PROC in $(jobs -p); do + kill "${PROC}" + done + return 1 + fi + + set +e + curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"' + CSI_CALLED=${PIPESTATUS[1]} + set -e + + set +e + curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"' + INTREE_CALLED=${PIPESTATUS[1]} + set -e + + for PROC in $(jobs -p); do + kill "${PROC}" + done + + loudecho "CSI_CALLED: ${CSI_CALLED}" + loudecho "INTREE_CALLED: ${INTREE_CALLED}" + + # if CSI was called and In-tree was not called, return 0/true/success + if [ "${CSI_CALLED}" == 0 ] && [ "${INTREE_CALLED}" == 1 ]; then + echo 0 + else + echo 1 + fi +} diff --git a/hack/e2e/ecr.sh b/hack/e2e/ecr.sh new file mode 100644 index 0000000000..8f710d95be --- /dev/null +++ b/hack/e2e/ecr.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -uo pipefail + +BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +source "${BASE_DIR}"/util.sh + +function ecr_build_and_push() { + REGION=${1} + AWS_ACCOUNT_ID=${2} + IMAGE_NAME=${3} + IMAGE_TAG=${4} + set +e + if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then + set -e + loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed" + else + set -e + loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}" + aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com + docker build -t "${IMAGE_NAME}":"${IMAGE_TAG}" . + docker push "${IMAGE_NAME}":"${IMAGE_TAG}" + fi +} diff --git a/hack/utils/helm.sh b/hack/e2e/helm.sh old mode 100755 new mode 100644 similarity index 92% rename from hack/utils/helm.sh rename to hack/e2e/helm.sh index 95ab5c388e..500441e1ae --- a/hack/utils/helm.sh +++ b/hack/e2e/helm.sh @@ -2,7 +2,7 @@ set -uo pipefail -helm::install() { +function helm_install() { INSTALL_PATH=${1} if [[ ! -e ${INSTALL_PATH}/helm ]]; then curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 diff --git a/hack/e2e/kops.sh b/hack/e2e/kops.sh new file mode 100644 index 0000000000..4f7cb989b6 --- /dev/null +++ b/hack/e2e/kops.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -uo pipefail + +OS_ARCH=$(go env GOOS)-amd64 + +BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +source "${BASE_DIR}"/util.sh + +function kops_install() { + INSTALL_PATH=${1} + KOPS_VERSION=${2} + if [[ ! -e ${INSTALL_PATH}/kops ]]; then + KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH} + curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops + chmod +x "${INSTALL_PATH}"/kops + fi +} + +function kops_create_cluster() { + SSH_KEY_PATH=${1} + KOPS_STATE_FILE=${2} + CLUSTER_NAME=${3} + KOPS_BIN=${4} + ZONES=${5} + INSTANCE_TYPE=${6} + K8S_VERSION=${7} + TEST_DIR=${8} + KOPS_FEATURE_GATES_FILE=${10} + KOPS_ADDITIONAL_POLICIES_FILE=${11} + + loudecho "Generating SSH key $SSH_KEY_PATH" + if [[ ! -e ${SSH_KEY_PATH} ]]; then + ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}" + fi + + set +e + if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then + set -e + loudecho "Updating cluster $CLUSTER_NAME" + else + set -e + loudecho "Creating cluster $CLUSTER_NAME" + ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \ + --zones "${ZONES}" \ + --node-count=3 \ + --node-size="${INSTANCE_TYPE}" \ + --kubernetes-version="${K8S_VERSION}" \ + --ssh-public-key="${SSH_KEY_PATH}".pub \ + "${CLUSTER_NAME}" + fi + + CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml + ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o yaml > "${CLUSTER_YAML_PATH}" + cat "${KOPS_FEATURE_GATES_FILE}" >> "${CLUSTER_YAML_PATH}" + cat "${KOPS_ADDITIONAL_POLICIES_FILE}" >> "${CLUSTER_YAML_PATH}" + ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}" + ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes + + loudecho "Validating cluster $CLUSTER_NAME" + ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m + return $? +} + +function kops_delete_cluster() { + KOPS_BIN=${1} + CLUSTER_NAME=${2} + KOPS_STATE_FILE=${3} + loudecho "Deleting cluster ${CLUSTER_NAME}" + ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes +} diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh new file mode 100755 index 0000000000..12db9c227a --- /dev/null +++ b/hack/e2e/run.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") +source "${BASE_DIR}"/ebs.sh +source "${BASE_DIR}"/ecr.sh +source "${BASE_DIR}"/helm.sh +source "${BASE_DIR}"/kops.sh +source "${BASE_DIR}"/util.sh + +DRIVER_NAME=${DRIVER_NAME:-aws-ebs-csi-driver} + +TEST_ID=${TEST_ID:-$RANDOM} +CLUSTER_NAME=test-cluster-${TEST_ID}.k8s.local + +TEST_DIR=${BASE_DIR}/csi-test-artifacts +BIN_DIR=${TEST_DIR}/bin +SSH_KEY_PATH=${TEST_DIR}/id_rsa + +REGION=${AWS_REGION:-us-west-2} +ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c} +INSTANCE_TYPE=${INSTANCE_TYPE:-c4.large} + +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}} +IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}} + +K8S_VERSION=${K8S_VERSION:-1.18.10} +KOPS_VERSION=${KOPS_VERSION:-1.18.2} +KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e} +KOPS_FEATURE_GATES_FILE=${KOPS_FEATURE_GATES_FILE:-./hack/feature-gates.yaml} +KOPS_ADDITIONAL_POLICIES_FILE=${KOPS_ADDITIONAL_POLICIES_FILE:-./hack/additional-policies.yaml} + +TEST_PATH=${TEST_PATH:-"./tests/e2e/..."} +KUBECONFIG=${KUBECONFIG:-"${HOME}/.kube/config"} +ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"} +GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-e2e\]"} +GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"} +GINKGO_NODES=${GINKGO_NODES:-4} +TEST_EXTRA_FLAGS=${TEST_EXTRA_FLAGS:-} + +EBS_SNAPSHOT_CRD=${EBS_SNAPSHOT_CRD:-"./deploy/kubernetes/cluster/crd_snapshotter.yaml"} +EBS_CHECK_MIGRATION=${EBS_CHECK_MIGRATION:-"false"} + +CLEAN=${CLEAN:-"true"} + +loudecho "Testing in region ${REGION} and zones ${ZONES}" +mkdir -p "${BIN_DIR}" +export PATH=${PATH}:${BIN_DIR} + +loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}" +kops_install "${BIN_DIR}" "${KOPS_VERSION}" +KOPS_BIN=${BIN_DIR}/kops + +loudecho "Installing helm to ${BIN_DIR}" +helm_install "${BIN_DIR}" +HELM_BIN=${BIN_DIR}/helm + +loudecho "Installing ginkgo to ${BIN_DIR}" +GINKGO_BIN=${BIN_DIR}/ginkgo +if [[ ! -e ${GINKGO_BIN} ]]; then + pushd /tmp + GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} GO111MODULE=on go get github.com/onsi/ginkgo/ginkgo@v1.12.0 + popd +fi + +ecr_build_and_push "${REGION}" \ + "${AWS_ACCOUNT_ID}" \ + "${IMAGE_NAME}" \ + "${IMAGE_TAG}" + +kops_create_cluster \ + "$SSH_KEY_PATH" \ + "$KOPS_STATE_FILE" \ + "$CLUSTER_NAME" \ + "$KOPS_BIN" \ + "$ZONES" \ + "$INSTANCE_TYPE" \ + "$K8S_VERSION" \ + "$TEST_DIR" \ + "$BASE_DIR" \ + "$KOPS_FEATURE_GATES_FILE" \ + "$KOPS_ADDITIONAL_POLICIES_FILE" +if [[ $? -ne 0 ]]; then + exit 1 +fi + +loudecho "Deploying driver" +"${HELM_BIN}" upgrade --install "${DRIVER_NAME}" \ + --namespace kube-system \ + --set enableVolumeScheduling=true \ + --set enableVolumeResizing=true \ + --set enableVolumeSnapshot=true \ + --set image.repository="${IMAGE_NAME}" \ + --set image.tag="${IMAGE_TAG}" \ + ./charts/"${DRIVER_NAME}" + +if [[ -n "${EBS_SNAPSHOT_CRD}" ]]; then + loudecho "Deploying snapshot CRD" + kubectl apply -f "$EBS_SNAPSHOT_CRD" + # TODO deploy snapshot controller too instead of including in helm chart +fi + +loudecho "Testing focus ${GINKGO_FOCUS}" +eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS" +set -x +${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "${TEST_PATH}" -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${ZONES%,*}" "${EXPANDED_TEST_EXTRA_FLAGS}" +TEST_PASSED=$? +set +x +loudecho "TEST_PASSED: ${TEST_PASSED}" + +OVERALL_TEST_PASSED="${TEST_PASSED}" +if [[ "${EBS_CHECK_MIGRATION}" == true ]]; then + exec 5>&1 + OUTPUT=$(ebs_check_migration | tee /dev/fd/5) + MIGRATION_PASSED=$(echo "${OUTPUT}" | tail -1) + loudecho "MIGRATION_PASSED: ${MIGRATION_PASSED}" + if [ "${TEST_PASSED}" == 0 ] && [ "${MIGRATION_PASSED}" == 0 ]; then + loudecho "Both test and migration passed" + OVERALL_TEST_PASSED=0 + else + loudecho "One of test or migration failed" + OVERALL_TEST_PASSED=1 + fi +fi + +if [[ "${CLEAN}" == true ]]; then + loudecho "Cleaning" + + loudecho "Removing driver" + ${HELM_BIN} del "${DRIVER_NAME}" \ + --namespace kube-system + + kops_delete_cluster \ + "${KOPS_BIN}" \ + "${CLUSTER_NAME}" \ + "${KOPS_STATE_FILE}" +else + loudecho "Not cleaning" +fi + +loudecho "OVERALL_TEST_PASSED: ${OVERALL_TEST_PASSED}" +if [[ $OVERALL_TEST_PASSED -ne 0 ]]; then + loudecho "FAIL!" + exit 1 +else + loudecho "SUCCESS!" +fi diff --git a/hack/e2e/util.sh b/hack/e2e/util.sh new file mode 100644 index 0000000000..46bb5b16e0 --- /dev/null +++ b/hack/e2e/util.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -uo pipefail + +function loudecho() { + echo "###" + echo "## ${1}" + echo "#" +} diff --git a/hack/run-e2e-test b/hack/run-e2e-test deleted file mode 100755 index 0245fb8101..0000000000 --- a/hack/run-e2e-test +++ /dev/null @@ -1,205 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -TEST_ID=${TEST_ID:-$RANDOM} -CLUSTER_NAME=test-cluster-${TEST_ID} - -BASE=$(realpath "${BASH_SOURCE[0]}") -BASE_DIR=$(dirname "${BASE}") -TEST_DIR=${BASE_DIR}/ebs-e2e-test -BIN_DIR=${TEST_DIR}/bin -SSH_KEY_PATH=${TEST_DIR}/id_rsa - -REGION=${AWS_REGION-us-west-2} -ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c} -INSTANCE_TYPE=${INSTANCE_TYPE:-c4.large} - -K8S_VERSION=${K8S_VERSION:-1.18.10} -KOPS_VERSION=${KOPS_VERSION:-1.18.2} -KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e} - -KUBECONFIG=${KUBECONFIG:-"${HOME}/.kube/config"} -ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"} -GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-migration\]"} -GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"} -GINKGO_NODES=${GINKGO_NODES:-4} -CHECK_MIGRATION=${CHECK_MIGRATION:-"true"} - -CLEAN=${CLEAN:-"true"} - -loudecho() { - echo "###" - echo "## ${1}" - echo "#" -} - -loudecho "Testing in region ${REGION} and zones ${ZONES}" -mkdir -p "${BIN_DIR}" -export PATH=${PATH}:${BIN_DIR} - -loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}" -source "${BASE_DIR}"/utils/kops.sh -kops::install "${BIN_DIR}" "${KOPS_VERSION}" -KOPS_BIN=${BIN_DIR}/kops - -loudecho "Installing helm to ${BIN_DIR}" -source "${BASE_DIR}"/utils/helm.sh -helm::install "${BIN_DIR}" -HELM_BIN=${BIN_DIR}/helm - -loudecho "Installing ginkgo to ${BIN_DIR}" -GINKGO_BIN=${BIN_DIR}/ginkgo -if [[ ! -e ${GINKGO_BIN} ]]; then - pushd /tmp - GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} GO111MODULE=on go get github.com/onsi/ginkgo/ginkgo@v1.12.0 - popd -fi - -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -IMAGE_NAME=${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/aws-ebs-csi-driver -IMAGE_TAG=${TEST_ID} -set +e -if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then - set -e - loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed" -else - set -e - loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}" - aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com - docker build -t "${IMAGE_NAME}":"${IMAGE_TAG}" . - docker push "${IMAGE_NAME}":"${IMAGE_TAG}" -fi - -loudecho "Generating SSH key $SSH_KEY_PATH" -if [[ ! -e ${SSH_KEY_PATH} ]]; then - ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}" -fi - -set +e -if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}".k8s.local; then - set -e - loudecho "Updating cluster $CLUSTER_NAME" -else - set -e - loudecho "Creating cluster $CLUSTER_NAME" - ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \ - --zones "${ZONES}" \ - --node-count=3 \ - --node-size="${INSTANCE_TYPE}" \ - --kubernetes-version="${K8S_VERSION}" \ - --ssh-public-key="${SSH_KEY_PATH}".pub \ - "${CLUSTER_NAME}".k8s.local -fi - -CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml -${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}".k8s.local -o yaml > "${CLUSTER_YAML_PATH}" -cat "${BASE_DIR}"/feature-gates.yaml >> "${CLUSTER_YAML_PATH}" -cat "${BASE_DIR}"/additional-policies.yaml >> "${CLUSTER_YAML_PATH}" -${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}" -${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}".k8s.local --yes - -loudecho "Validating cluster $CLUSTER_NAME" -${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m -VALID=$? -if [[ $VALID -ne 0 ]]; then - exit 1 -fi - -loudecho "Deploying driver" -${HELM_BIN} upgrade --install aws-ebs-csi-driver \ - --namespace kube-system \ - --set enableVolumeScheduling=true \ - --set enableVolumeResizing=true \ - --set enableVolumeSnapshot=true \ - --set image.repository="${IMAGE_NAME}" \ - --set image.tag="${IMAGE_TAG}" \ - ./charts/aws-ebs-csi-driver - -loudecho "Testing focus ${GINKGO_FOCUS}" -set -x -${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" ./tests/e2e-migration/... -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${ZONES%,*}" -TEST_PASSED=$? -set +x - -if [[ "${CHECK_MIGRATION}" == true ]]; then - loudecho "Checking migration" - # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com - # Find the controller-manager log and read its metrics to verify - NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name") - kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system & - - # Ensure port forwarding succeeded - while true; do - set +e - HEALTHZ=$(curl -s 127.0.0.1:10252/healthz) - set -e - if [[ ${HEALTHZ} == "ok" ]]; then - loudecho "Port forwarding succeeded" - break - else - loudecho "Port forwarding is not yet ready" - fi - sleep 1 - done - - set +e - curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"' - CSI_CALLED=${PIPESTATUS[1]} - set -e - - set +e - curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"' - INTREE_CALLED=${PIPESTATUS[1]} - set -e - - for PROC in $(jobs -p); do - kill "${PROC}" - done - - loudecho "CSI_CALLED: ${CSI_CALLED}" - loudecho "INTREE_CALLED: ${INTREE_CALLED}" - - # TEST_PASSED if tests passed, CSI was called, and In-tree was not called - if [ "${TEST_PASSED}" == 0 ] && [ "${CSI_CALLED}" == 0 ] && [ "${INTREE_CALLED}" == 1 ]; then - TEST_PASSED=0 - else - TEST_PASSED=1 - fi -fi - -if [[ "${CLEAN}" == true ]]; then - loudecho "Cleaning" - - loudecho "Removing driver" - ${HELM_BIN} del aws-ebs-csi-driver - - loudecho "Deleting cluster ${CLUSTER_NAME}" - ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}".k8s.local --state "${KOPS_STATE_FILE}" --yes - - rm -rf "${TEST_DIR}" -else - loudecho "Not cleaning" -fi - -loudecho "TEST_PASSED: ${TEST_PASSED}" -if [[ $TEST_PASSED -ne 0 ]]; then - loudecho "FAIL!" - exit 1 -else - loudecho "SUCCESS!" -fi diff --git a/hack/run-integration-test b/hack/run-integration-test deleted file mode 100755 index 81b154a199..0000000000 --- a/hack/run-integration-test +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Hard-coded version is needed in case GitHub API rate limit is exceeded. -# TODO: When aws-k8s-tester becomes a full release (https://developer.github.com/v3/repos/releases/#get-the-latest-release), use: -# $(shell curl -s --request GET --url https://api.github.com/repos/aws/aws-k8s-tester/releases/latest | jq -r '.tag_name? // ""') -AWS_K8S_TESTER_VERSION=$(curl -s --request GET --url https://api.github.com/repos/aws/aws-k8s-tester/tags | jq -r '.[0]?.name // "0.2.5"') -AWS_K8S_TESTER_OS_ARCH=$(go env GOOS)-amd64 -AWS_K8S_TESTER_DOWNLOAD_URL=https://github.com/aws/aws-k8s-tester/releases/download/${AWS_K8S_TESTER_VERSION}/aws-k8s-tester-${AWS_K8S_TESTER_VERSION}-${AWS_K8S_TESTER_OS_ARCH} -AWS_K8S_TESTER_PATH=/tmp/aws-k8s-tester - -VPC_ID_FLAG="" -if [ -z "$AWS_K8S_TESTER_VPC_ID" ]; then - VPC_ID_FLAG=--vpc-id=${AWS_K8S_TESTER_VPC_ID} -fi - -PR_NUM_FLAG="" -if [ -z "$PULL_NUMBER" ]; then - PR_NUM_FLAG=--pr-num=${PULL_NUMBER} -fi - -curl -L ${AWS_K8S_TESTER_DOWNLOAD_URL} -o ${AWS_K8S_TESTER_PATH} -chmod +x ${AWS_K8S_TESTER_PATH} -${AWS_K8S_TESTER_PATH} csi test integration --terminate-on-exit=true --timeout=20m ${PR_NUM_FLAG} ${VPC_ID_FLAG} diff --git a/hack/utils/kops.sh b/hack/utils/kops.sh deleted file mode 100644 index 0cddae6c4c..0000000000 --- a/hack/utils/kops.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -uo pipefail - -OS_ARCH=$(go env GOOS)-amd64 - -kops::install() { - INSTALL_PATH=${1} - KOPS_VERSION=${2} - if [[ ! -e ${INSTALL_PATH}/kops ]]; then - KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH} - curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops - chmod +x "${INSTALL_PATH}"/kops - fi -} diff --git a/hack/utils/tiller-rbac.yaml b/hack/utils/tiller-rbac.yaml deleted file mode 100644 index 1fcf47dca7..0000000000 --- a/hack/utils/tiller-rbac.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: tiller - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: tiller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: - - kind: ServiceAccount - name: tiller - namespace: kube-system diff --git a/tester/migration-test-config.yaml b/tester/migration-test-config.yaml deleted file mode 100644 index 77c7a253b0..0000000000 --- a/tester/migration-test-config.yaml +++ /dev/null @@ -1,139 +0,0 @@ -cluster: - kops: - stateFile: s3://k8s-kops-csi-e2e - zones: us-west-2a - nodeCount: 3 - nodeSize: c5.large - kubernetesVersion: 1.18.9 - featureGates: |2 - kubeAPIServer: - featureGates: - CSIDriverRegistry: "true" - CSINodeInfo: "true" - CSIBlockVolume: "true" - CSIMigration: "true" - CSIMigrationAWS: "true" - CSIMigrationAWSComplete: "true" - ExpandCSIVolumes: "true" - VolumeSnapshotDataSource: "true" - CSIInlineVolume: "true" - kubeControllerManager: - featureGates: - CSIDriverRegistry: "true" - CSINodeInfo: "true" - CSIBlockVolume: "true" - CSIMigration: "true" - CSIMigrationAWS: "true" - CSIMigrationAWSComplete: "true" - ExpandCSIVolumes: "true" - CSIInlineVolume: "true" - kubelet: - featureGates: - CSIDriverRegistry: "true" - CSINodeInfo: "true" - CSIBlockVolume: "true" - CSIMigration: "true" - CSIMigrationAWS: "true" - CSIMigrationAWSComplete: "true" - ExpandCSIVolumes: "true" - CSIInlineVolume: "true" - iamPolicies: |2 - additionalPolicies: - node: | - [ - { - "Effect": "Allow", - "Action": [ - "ec2:AttachVolume", - "ec2:CreateSnapshot", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:DeleteSnapshot", - "ec2:DeleteTags", - "ec2:DeleteVolume", - "ec2:DescribeInstances", - "ec2:DescribeSnapshots", - "ec2:DescribeTags", - "ec2:DescribeVolumes", - "ec2:DetachVolume", - "ec2:ModifyVolume", - "ec2:DescribeVolumesModifications" - ], - "Resource": "*" - } - ] - -build: | - eval $(aws ecr get-login --region us-west-2 --no-include-email) - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - IMAGE_TAG={{TEST_ID}} - IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver - docker build -t $IMAGE_NAME:$IMAGE_TAG . - docker push $IMAGE_NAME:$IMAGE_TAG - -install: | - echo "Deploying driver" - # install helm - OS_ARCH=$(go env GOOS)-amd64 - helm_name=helm-v2.14.1-$OS_ARCH.tar.gz - wget https://get.helm.sh/$helm_name - tar xvzf $helm_name - mv $OS_ARCH/helm /usr/local/bin/helm - - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - IMAGE_TAG={{TEST_ID}} - IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver - - # install tiller - kubectl apply -f ./hack/utils/tiller-rbac.yaml - helm init --service-account tiller --history-max 200 --wait - kubectl get po -n kube-system - - helm install --name aws-ebs-csi-driver \ - --set enableVolumeScheduling=true \ - --set enableVolumeResizing=true \ - --set enableVolumeSnapshot=true \ - --set image.repository=$IMAGE_NAME \ - --set image.tag=$IMAGE_TAG \ - ./charts/aws-ebs-csi-driver - -uninstall: | - echo "Removing driver" - helm del --purge aws-ebs-csi-driver - -test: | - # TODO known test failures to skip temporarily - # - should not allow expansion of pvcs without AllowVolumeExpansion property - # - Test passes but cleanup fails, need https://github.com/kubernetes/kubernetes/pull/81107 - # - (block volmode) Verify if offline PVC expansion works / should resize volume when PVC is edited while pod is using it - # - NodeExpand for BlockVolumes not well-defined, need more investigation and possibly https://github.com/container-storage-interface/spec/issues/380 - # - should provision storage with mount options - # - Known bug, need https://github.com/kubernetes/kubernetes/pull/80191 but not yet in a patch release - cd ./tests/e2e-migration - go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[ebs-csi-migration\]" -ginkgo.skip="\[Disruptive\]\ - |should.not.allow.expansion\ - |block.volmode.+volume-expand\ - |should.provision.storage.with.mount.options\ - |should.not.mount./.map.unused.volumes.in.a.pod" -gce-zone=us-west-2a - TEST_PASS=$? - cd ../.. - - # There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com - # Find the controller-manager log and read its metrics to verify - NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name") - kubectl port-forward kube-controller-manager-$NODE 10252:10252 -n kube-system& - curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"' - CSI_CALLED=$? - curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"' - INTREE_CALLED=$? - - # TEST_PASS if tests passed, CSI was called, and In-tree was not called - if [ "$TEST_PASS" == 0 ] && [ "$CSI_CALLED" == 0 ] && [ "$INTREE_CALLED" == 1 ]; then - TEST_PASS=0 - else - TEST_PASS=1 - fi - - if [[ $TEST_PASS -ne 0 ]]; then - exit 1 - fi diff --git a/tester/multi-az-config.yaml b/tester/multi-az-config.yaml deleted file mode 100644 index 55fa187db8..0000000000 --- a/tester/multi-az-config.yaml +++ /dev/null @@ -1,99 +0,0 @@ -cluster: - kops: - stateFile: s3://k8s-kops-csi-e2e - zones: us-west-2a,us-west-2b,us-west-2c - nodeCount: 3 - nodeSize: c5.large - kubernetesVersion: 1.18.9 - featureGates: |2 - kubeAPIServer: - featureGates: - CSIDriverRegistry: "true" - CSINodeInfo: "true" - CSIBlockVolume: "true" - CSIMigration: "true" - CSIMigrationAWS: "true" - ExpandCSIVolumes: "true" - VolumeSnapshotDataSource: "true" - CSIInlineVolume: "true" - kubeControllerManager: - featureGates: - CSIDriverRegistry: "true" - CSINodeInfo: "true" - CSIBlockVolume: "true" - CSIMigration: "true" - CSIMigrationAWS: "true" - ExpandCSIVolumes: "true" - CSIInlineVolume: "true" - kubelet: - featureGates: - CSIDriverRegistry: "true" - CSINodeInfo: "true" - CSIBlockVolume: "true" - CSIMigration: "true" - CSIMigrationAWS: "true" - ExpandCSIVolumes: "true" - CSIInlineVolume: "true" - iamPolicies: |2 - additionalPolicies: - node: | - [ - { - "Effect": "Allow", - "Action": [ - "ec2:AttachVolume", - "ec2:CreateSnapshot", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:DeleteSnapshot", - "ec2:DeleteTags", - "ec2:DeleteVolume", - "ec2:DescribeInstances", - "ec2:DescribeSnapshots", - "ec2:DescribeTags", - "ec2:DescribeVolumes", - "ec2:DetachVolume", - "ec2:ModifyVolume", - "ec2:DescribeVolumesModifications" - ], - "Resource": "*" - } - ] - -build: | - eval $(aws ecr get-login --region us-west-2 --no-include-email) - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - IMAGE_TAG={{TEST_ID}} - IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver - docker build -t $IMAGE_NAME:$IMAGE_TAG . - docker push $IMAGE_NAME:$IMAGE_TAG - -install: | - echo "Deploying driver" - # install helm - make bin/helm - - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - IMAGE_TAG={{TEST_ID}} - IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver - - bin/helm install aws-ebs-csi-driver \ - --set enableVolumeScheduling=true \ - --set enableVolumeResizing=true \ - --set enableVolumeSnapshot=true \ - --set image.repository=$IMAGE_NAME \ - --set image.tag=$IMAGE_TAG \ - ./charts/aws-ebs-csi-driver - - kubectl get po -n kube-system - -uninstall: | - echo "Removing driver" - bin/helm delete aws-ebs-csi-driver - -test: | - go get github.com/onsi/ginkgo/ginkgo - export KUBECONFIG=$HOME/.kube/config - export AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c - ARTIFACTS=${ARTIFACTS-/tmp/ebs-csi/artifacts} - $(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[multi-az\]" tests/e2e -- -report-dir=$ARTIFACTS diff --git a/tester/single-az-config.yaml b/tester/single-az-config.yaml deleted file mode 100644 index a45340b9f7..0000000000 --- a/tester/single-az-config.yaml +++ /dev/null @@ -1,102 +0,0 @@ -cluster: - kops: - stateFile: s3://k8s-kops-csi-e2e - zones: us-west-2a - nodeCount: 3 - nodeSize: c5.large - kubernetesVersion: 1.18.9 - featureGates: |2 - kubeAPIServer: - featureGates: - CSIDriverRegistry: "true" - CSINodeInfo: "true" - CSIBlockVolume: "true" - CSIMigration: "true" - CSIMigrationAWS: "true" - ExpandCSIVolumes: "true" - VolumeSnapshotDataSource: "true" - CSIInlineVolume: "true" - kubeControllerManager: - featureGates: - CSIDriverRegistry: "true" - CSINodeInfo: "true" - CSIBlockVolume: "true" - CSIMigration: "true" - CSIMigrationAWS: "true" - ExpandCSIVolumes: "true" - CSIInlineVolume: "true" - kubelet: - featureGates: - CSIDriverRegistry: "true" - CSINodeInfo: "true" - CSIBlockVolume: "true" - CSIMigration: "true" - CSIMigrationAWS: "true" - ExpandCSIVolumes: "true" - CSIInlineVolume: "true" - iamPolicies: |2 - additionalPolicies: - node: | - [ - { - "Effect": "Allow", - "Action": [ - "ec2:AttachVolume", - "ec2:CreateSnapshot", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:DeleteSnapshot", - "ec2:DeleteTags", - "ec2:DeleteVolume", - "ec2:DescribeInstances", - "ec2:DescribeSnapshots", - "ec2:DescribeTags", - "ec2:DescribeVolumes", - "ec2:DetachVolume", - "ec2:ModifyVolume", - "ec2:DescribeVolumesModifications" - ], - "Resource": "*" - } - ] - -build: | - eval $(aws ecr get-login --region us-west-2 --no-include-email) - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - IMAGE_TAG={{TEST_ID}} - IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver - docker build -t $IMAGE_NAME:$IMAGE_TAG . - docker push $IMAGE_NAME:$IMAGE_TAG - -install: | - echo "Deploying driver" - # install helm - make bin/helm - - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - IMAGE_TAG={{TEST_ID}} - IMAGE_NAME=$AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/aws-ebs-csi-driver - - # install snapshot CRD - kubectl apply -f deploy/kubernetes/cluster/ - - bin/helm install aws-ebs-csi-driver \ - --set enableVolumeScheduling=true \ - --set enableVolumeResizing=true \ - --set enableVolumeSnapshot=true \ - --set image.repository=$IMAGE_NAME \ - --set image.tag=$IMAGE_TAG \ - ./charts/aws-ebs-csi-driver - - kubectl get po -n kube-system - -uninstall: | - echo "Removing driver" - bin/helm delete aws-ebs-csi-driver - -test: | - go get github.com/onsi/ginkgo/ginkgo - export KUBECONFIG=$HOME/.kube/config - export AWS_AVAILABILITY_ZONES=us-west-2a - ARTIFACTS=${ARTIFACTS-/tmp/ebs-csi/artifacts} - $(go env GOBIN)/ginkgo -p -nodes=32 -v --focus="\[ebs-csi-e2e\] \[single-az\]" --skip="\"sc1\"|\"st1\"" tests/e2e -- -report-dir=$ARTIFACTS