Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor e2e testing scripts to be more reusable and use them instead… #694

Merged
merged 3 commits into from
Jan 13, 2021
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
@@ -73,22 +73,31 @@ test-sanity:
#go test -v ./tests/sanity/...
echo "succeed"

bin/k8s-e2e-tester: | bin
go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master

.PHONY: test-e2e-single-az
test-e2e-single-az: bin/k8s-e2e-tester
TESTCONFIG=./tester/single-az-config.yaml ${GOBIN}/k8s-e2e-tester
test-e2e-single-az:
AWS_REGION=us-west-2 \
AWS_AVAILABILITY_ZONES=us-west-2a \
TEST_PATH=./tests/e2e/... \
GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \
GINKGO_SKIP="\"sc1\"|\"st1\"" \
./hack/e2e/run.sh

.PHONY: test-e2e-multi-az
test-e2e-multi-az: bin/k8s-e2e-tester
TESTCONFIG=./tester/multi-az-config.yaml ${GOBIN}/k8s-e2e-tester
test-e2e-multi-az:
AWS_REGION=us-west-2 \
AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
TEST_PATH=./tests/e2e/... \
GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" \
./hack/e2e/run.sh

.PHONY: test-e2e-migration
test-e2e-migration:
AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-migration\]" ./hack/run-e2e-test
# TODO: enable migration test to use new framework
#TESTCONFIG=./tester/migration-test-config.yaml go run tester/cmd/main.go
AWS_REGION=us-west-2 \
AWS_AVAILABILITY_ZONES=us-west-2a \
TEST_PATH=./tests/e2e-migration/... \
GINKGO_FOCUS="\[ebs-csi-migration\]" \
EBS_CHECK_MIGRATION=true \
./hack/e2e/run.sh

.PHONY: image-release
image-release:
1 change: 1 addition & 0 deletions hack/e2e/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/csi-test-artifacts
26 changes: 26 additions & 0 deletions hack/e2e/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Usage

run.sh will build and push a driver image, create a kops cluster, helm install the driver pointing to the built image, run ginkgo tests, then clean everything up.

See below for an example.

KOPS_STATE_FILE is an S3 bucket you have write access to.

TEST_ID is a token used for idempotency.

For more details, see the script itself.

For more examples, see the top-level Makefile.

```
TEST_PATH=./tests/e2e-migration/... \
EBS_CHECK_MIGRATION=true \
TEST_ID=18512 \
CLEAN=false \
KOPS_STATE_FILE=s3://mattwon \
AWS_REGION=us-west-2 \
AWS_AVAILABILITY_ZONES=us-west-2a \
GINKGO_FOCUS=Dynamic.\*xfs.\*should.store.data \
GINKGO_NODES=1 \
./hack/e2e/run.sh
```
52 changes: 52 additions & 0 deletions hack/e2e/ebs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
#!/bin/bash

set -uo pipefail

BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
source "${BASE_DIR}"/util.sh

function ebs_check_migration() {
loudecho "Checking migration"
# There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
# Find the controller-manager log and read its metrics to verify
NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system &

# Ensure port forwarding succeeded
while true; do
wongma7 marked this conversation as resolved.
Show resolved Hide resolved
set +e
HEALTHZ=$(curl -s 127.0.0.1:10252/healthz)
set -e
if [[ ${HEALTHZ} == "ok" ]]; then
loudecho "Port forwarding succeeded"
break
else
loudecho "Port forwarding is not yet ready"
fi
sleep 1
done

set +e
curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"'
CSI_CALLED=${PIPESTATUS[1]}
set -e

set +e
curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"'
INTREE_CALLED=${PIPESTATUS[1]}
set -e

for PROC in $(jobs -p); do
kill "${PROC}"
done

loudecho "CSI_CALLED: ${CSI_CALLED}"
loudecho "INTREE_CALLED: ${INTREE_CALLED}"

# if CSI was called and In-tree was not called, return 0/true/success
if [ "${CSI_CALLED}" == 0 ] && [ "${INTREE_CALLED}" == 1 ]; then
echo 0
else
echo 1
fi
}
24 changes: 24 additions & 0 deletions hack/e2e/ecr.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/bin/bash

set -uo pipefail

BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
source "${BASE_DIR}"/util.sh

function ecr_build_and_push() {
REGION=${1}
AWS_ACCOUNT_ID=${2}
IMAGE_NAME=${3}
IMAGE_TAG=${4}
set +e
if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then
set -e
loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
else
set -e
loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com
docker build -t "${IMAGE_NAME}":"${IMAGE_TAG}" .
docker push "${IMAGE_NAME}":"${IMAGE_TAG}"
fi
}
2 changes: 1 addition & 1 deletion hack/utils/helm.sh → hack/e2e/helm.sh
100755 → 100644
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@

set -uo pipefail

helm::install() {
function helm_install() {
INSTALL_PATH=${1}
if [[ ! -e ${INSTALL_PATH}/helm ]]; then
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
71 changes: 71 additions & 0 deletions hack/e2e/kops.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
#!/bin/bash

set -uo pipefail

OS_ARCH=$(go env GOOS)-amd64

BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
source "${BASE_DIR}"/util.sh

function kops_install() {
INSTALL_PATH=${1}
KOPS_VERSION=${2}
if [[ ! -e ${INSTALL_PATH}/kops ]]; then
KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH}
curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops
chmod +x "${INSTALL_PATH}"/kops
fi
}

function kops_create_cluster() {
SSH_KEY_PATH=${1}
KOPS_STATE_FILE=${2}
CLUSTER_NAME=${3}
KOPS_BIN=${4}
ZONES=${5}
INSTANCE_TYPE=${6}
K8S_VERSION=${7}
TEST_DIR=${8}
KOPS_FEATURE_GATES_FILE=${10}
KOPS_ADDITIONAL_POLICIES_FILE=${11}

loudecho "Generating SSH key $SSH_KEY_PATH"
if [[ ! -e ${SSH_KEY_PATH} ]]; then
ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
fi

set +e
if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
set -e
loudecho "Updating cluster $CLUSTER_NAME"
else
set -e
loudecho "Creating cluster $CLUSTER_NAME"
${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
--zones "${ZONES}" \
--node-count=3 \
--node-size="${INSTANCE_TYPE}" \
--kubernetes-version="${K8S_VERSION}" \
--ssh-public-key="${SSH_KEY_PATH}".pub \
"${CLUSTER_NAME}"
fi

CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml
${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o yaml > "${CLUSTER_YAML_PATH}"
cat "${KOPS_FEATURE_GATES_FILE}" >> "${CLUSTER_YAML_PATH}"
cat "${KOPS_ADDITIONAL_POLICIES_FILE}" >> "${CLUSTER_YAML_PATH}"
${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}"
${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes

loudecho "Validating cluster $CLUSTER_NAME"
${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m
return $?
}

function kops_delete_cluster() {
KOPS_BIN=${1}
CLUSTER_NAME=${2}
KOPS_STATE_FILE=${3}
loudecho "Deleting cluster ${CLUSTER_NAME}"
${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
}
156 changes: 156 additions & 0 deletions hack/e2e/run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
#!/bin/bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -euo pipefail

BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
source "${BASE_DIR}"/ebs.sh
source "${BASE_DIR}"/ecr.sh
source "${BASE_DIR}"/helm.sh
source "${BASE_DIR}"/kops.sh
source "${BASE_DIR}"/util.sh

DRIVER_NAME=${DRIVER_NAME:-aws-ebs-csi-driver}

TEST_ID=${TEST_ID:-$RANDOM}
CLUSTER_NAME=test-cluster-${TEST_ID}.k8s.local

TEST_DIR=${BASE_DIR}/csi-test-artifacts
BIN_DIR=${TEST_DIR}/bin
SSH_KEY_PATH=${TEST_DIR}/id_rsa

REGION=${AWS_REGION:-us-west-2}
ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
INSTANCE_TYPE=${INSTANCE_TYPE:-c4.large}

AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}}
IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}}

K8S_VERSION=${K8S_VERSION:-1.18.10}
KOPS_VERSION=${KOPS_VERSION:-1.18.2}
KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e}
KOPS_FEATURE_GATES_FILE=${KOPS_FEATURE_GATES_FILE:-./hack/feature-gates.yaml}
KOPS_ADDITIONAL_POLICIES_FILE=${KOPS_ADDITIONAL_POLICIES_FILE:-./hack/additional-policies.yaml}

TEST_PATH=${TEST_PATH:-"./tests/e2e/..."}
KUBECONFIG=${KUBECONFIG:-"${HOME}/.kube/config"}
ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"}
GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-e2e\]"}
GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"}
GINKGO_NODES=${GINKGO_NODES:-4}
TEST_EXTRA_FLAGS=${TEST_EXTRA_FLAGS:-}

EBS_CHECK_MIGRATION=${EBS_CHECK_MIGRATION:-"false"}

CLEAN=${CLEAN:-"true"}

loudecho "Testing in region ${REGION} and zones ${ZONES}"
mkdir -p "${BIN_DIR}"
export PATH=${PATH}:${BIN_DIR}

loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}"
kops_install "${BIN_DIR}" "${KOPS_VERSION}"
KOPS_BIN=${BIN_DIR}/kops

loudecho "Installing helm to ${BIN_DIR}"
helm_install "${BIN_DIR}"
HELM_BIN=${BIN_DIR}/helm

loudecho "Installing ginkgo to ${BIN_DIR}"
GINKGO_BIN=${BIN_DIR}/ginkgo
if [[ ! -e ${GINKGO_BIN} ]]; then
pushd /tmp
GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} GO111MODULE=on go get github.com/onsi/ginkgo/[email protected]
popd
fi

ecr_build_and_push "${REGION}" \
"${AWS_ACCOUNT_ID}" \
"${IMAGE_NAME}" \
"${IMAGE_TAG}"

kops_create_cluster \
"$SSH_KEY_PATH" \
"$KOPS_STATE_FILE" \
"$CLUSTER_NAME" \
"$KOPS_BIN" \
"$ZONES" \
"$INSTANCE_TYPE" \
"$K8S_VERSION" \
"$TEST_DIR" \
"$BASE_DIR" \
"$KOPS_FEATURE_GATES_FILE" \
"$KOPS_ADDITIONAL_POLICIES_FILE"
if [[ $? -ne 0 ]]; then
exit 1
fi

loudecho "Deploying driver"
${HELM_BIN} upgrade --install ${DRIVER_NAME} \
--namespace kube-system \
--set enableVolumeScheduling=true \
--set enableVolumeResizing=true \
--set enableVolumeSnapshot=true \
--set image.repository="${IMAGE_NAME}" \
--set image.tag="${IMAGE_TAG}" \
./charts/${DRIVER_NAME}

loudecho "Testing focus ${GINKGO_FOCUS}"
eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS"
set -x
${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "${TEST_PATH}" -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${ZONES%,*}" "${EXPANDED_TEST_EXTRA_FLAGS}"
TEST_PASSED=$?
set +x
loudecho "TEST_PASSED: ${TEST_PASSED}"

OVERALL_TEST_PASSED="${TEST_PASSED}"
if [[ "${EBS_CHECK_MIGRATION}" == true ]]; then
OUTPUT=$(ebs_check_migration)
echo "${OUTPUT}"
MIGRATION_PASSED=$(echo "${OUTPUT}" | tail -1)
loudecho "MIGRATION_PASSED: ${MIGRATION_PASSED}"
if [ "${TEST_PASSED}" == 0 ] && [ "${MIGRATION_PASSED}" == 0 ]; then
loudecho "Both test and migration passed"
OVERALL_TEST_PASSED=0
else
loudecho "One of test or migration failed"
OVERALL_TEST_PASSED=1
fi
fi

if [[ "${CLEAN}" == true ]]; then
loudecho "Cleaning"

loudecho "Removing driver"
${HELM_BIN} del "${DRIVER_NAME}" \
--namespace kube-system

kops_delete_cluster \
"${KOPS_BIN}" \
"${CLUSTER_NAME}" \
"${KOPS_STATE_FILE}"
else
loudecho "Not cleaning"
fi

loudecho "OVERALL_TEST_PASSED: ${OVERALL_TEST_PASSED}"
if [[ $OVERALL_TEST_PASSED -ne 0 ]]; then
loudecho "FAIL!"
exit 1
else
loudecho "SUCCESS!"
fi
9 changes: 9 additions & 0 deletions hack/e2e/util.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/bin/bash

set -uo pipefail

function loudecho() {
echo "###"
echo "## ${1}"
echo "#"
}
Loading