diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh index d3d780a96d..cbc426d9bb 100644 --- a/scripts/lib/cluster.sh +++ b/scripts/lib/cluster.sh @@ -41,3 +41,43 @@ function up-test-cluster() { echo "ok." fi } + +function up-kops-cluster { + aws s3api create-bucket --bucket kops-cni-test-temp --region $AWS_DEFAULT_REGION --create-bucket-configuration LocationConstraint=$AWS_DEFAULT_REGION + curl -LO https://github.com/kubernetes/kops/releases/download/$(curl -s https://api.github.com/repos/kubernetes/kops/releases/latest | grep tag_name | cut -d '"' -f 4)/kops-linux-amd64 + chmod +x kops-linux-amd64 + sudo mv kops-linux-amd64 /usr/local/bin/kops + CLUSTER_NAME=kops-cni-test-cluster-${TEST_ID}.k8s.local + export KOPS_STATE_STORE=s3://kops-cni-test-temp + + SSH_KEYS=~/.ssh/devopsinuse + if [ ! -f "$SSH_KEYS" ] + then + echo -e "\nCreating SSH keys ..." + ssh-keygen -t rsa -N '' -f ~/.ssh/devopsinuse + else + echo -e "\nSSH keys are already in place!" + fi + + kops create cluster \ + --zones ${AWS_DEFAULT_REGION}a,${AWS_DEFAULT_REGION}b \ + --networking amazon-vpc-routed-eni \ + --node-count 2 \ + --ssh-public-key=~/.ssh/devopsinuse.pub \ + --kubernetes-version ${K8S_VERSION} \ + ${CLUSTER_NAME} + kops update cluster --name ${CLUSTER_NAME} --yes + sleep 100 + while [[ ! $(kops validate cluster | grep "is ready") ]] + do + sleep 5 + echo "Waiting for cluster validation" + done + kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/${MANIFEST_CNI_VERSION}/config/v1.6/cni-metrics-helper.yaml +} + +function down-kops-cluster { + kops delete cluster --name ${CLUSTER_NAME} --yes + aws s3 rm ${KOPS_STATE_STORE} --recursive + aws s3 rb ${KOPS_STATE_STORE} --region us-west-2 +} \ No newline at end of file diff --git a/scripts/lib/integration.sh b/scripts/lib/integration.sh new file mode 100644 index 0000000000..b9ffa1fe44 --- /dev/null +++ b/scripts/lib/integration.sh @@ -0,0 +1,32 @@ +function run_kops_conformance() { + START=$SECONDS + + export KUBECONFIG=~/.kube/config + kubectl apply -f "$TEST_CONFIG_PATH" + sleep 5 + while [[ $(kubectl describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]] + do + sleep 5 + echo "Waiting for daemonset update" + done + echo "Updated!" + + go install github.com/onsi/ginkgo/ginkgo + wget -qO- https://dl.k8s.io/v$K8S_VERSION/kubernetes-test.tar.gz | tar -zxvf - --strip-components=4 -C /tmp kubernetes/platforms/linux/amd64/e2e.test + + $GOPATH/bin/ginkgo -p --focus="Conformance" --failFast --flakeAttempts 2 \ + --skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|(Basic StatefulSet functionality [StatefulSetBasic])|\[Slow\]|\[Serial\]" /tmp/e2e.test -- --kubeconfig=$KUBECONFIG + + /tmp/e2e.test --ginkgo.focus="\[Serial\].*Conformance" --kubeconfig=$KUBECONFIG --ginkgo.failFast --ginkgo.flakeAttempts 2 \ + --ginkgo.skip="(should support remote command execution over websockets)|(should support retrieving logs from the container over websockets)|\[Slow\]" + echo "Kops conformance tests ran successfully!" + + KOPS_TEST_DURATION=$((SECONDS - START)) + echo "TIMELINE: KOPS tests took $KOPS_TEST_DURATION seconds." + + START=$SECONDS + down-kops-cluster + DOWN_KOPS_DURATION=$((SECONDS - START)) + echo "TIMELINE: Down KOPS cluster took $DOWN_KOPS_DURATION seconds." + exit 0 +} \ No newline at end of file diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh index 0abac20081..24254e2d6d 100755 --- a/scripts/run-integration-tests.sh +++ b/scripts/run-integration-tests.sh @@ -8,6 +8,7 @@ DIR=$(cd "$(dirname "$0")"; pwd) source "$DIR"/lib/common.sh source "$DIR"/lib/aws.sh source "$DIR"/lib/cluster.sh +source "$DIR"/lib/integration.sh # Variables used in /lib/aws.sh OS=$(go env GOOS) @@ -19,6 +20,7 @@ ARCH=$(go env GOARCH) : "${DEPROVISION:=true}" : "${BUILD:=true}" : "${RUN_CONFORMANCE:=false}" +: "${RUN_KOPS_TEST:=false}" __cluster_created=0 __cluster_deprovisioned=0 @@ -27,11 +29,18 @@ on_error() { # Make sure we destroy any cluster that was created if we hit run into an # error when attempting to run tests against the cluster if [[ $__cluster_created -eq 1 && $__cluster_deprovisioned -eq 0 && "$DEPROVISION" == true ]]; then - # prevent double-deprovisioning with ctrl-c during deprovisioning... - __cluster_deprovisioned=1 - echo "Cluster was provisioned already. Deprovisioning it..." - down-test-cluster + if [[ $RUN_KOPS_TEST == true ]]; then + __cluster_deprovisioned=1 + echo "Cluster was provisioned already. Deprovisioning it..." + down-kops-cluster + else + # prevent double-deprovisioning with ctrl-c during deprovisioning... + __cluster_deprovisioned=1 + echo "Cluster was provisioned already. Deprovisioning it..." + down-test-cluster + fi fi + exit 1 } @@ -49,6 +58,7 @@ TEST_CLUSTER_DIR=/tmp/cni-test/cluster-$CLUSTER_NAME CLUSTER_MANAGE_LOG_PATH=$TEST_CLUSTER_DIR/cluster-manage.log : "${CLUSTER_CONFIG:=${TEST_CLUSTER_DIR}/${CLUSTER_NAME}.yaml}" : "${KUBECONFIG_PATH:=${TEST_CLUSTER_DIR}/kubeconfig}" +: "${ADDONS_CNI_IMAGE:=""}" # shared binaries : "${TESTER_DIR:=/tmp/aws-k8s-tester}" @@ -140,13 +150,16 @@ mkdir -p "$REPORT_DIR" mkdir -p "$TEST_CLUSTER_DIR" mkdir -p "$TEST_CONFIG_DIR" -if [[ "$PROVISION" == true ]]; then - START=$SECONDS +START=$SECONDS +if [[ "$PROVISION" == true && "$RUN_KOPS_TEST" == true ]]; then + up-kops-cluster +elif [[ "$PROVISION" == true ]]; then up-test-cluster - UP_CLUSTER_DURATION=$((SECONDS - START)) - echo "TIMELINE: Upping test cluster took $UP_CLUSTER_DURATION seconds." - __cluster_created=1 fi +__cluster_created=1 + +UP_CLUSTER_DURATION=$((SECONDS - START)) +echo "TIMELINE: Upping test cluster took $UP_CLUSTER_DURATION seconds." echo "Using $BASE_CONFIG_PATH as a template" cp "$BASE_CONFIG_PATH" "$TEST_CONFIG_PATH" @@ -157,7 +170,12 @@ sed -i'.bak' "s,:$MANIFEST_IMAGE_VERSION,:$TEST_IMAGE_VERSION," "$TEST_CONFIG_PA sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init,$INIT_IMAGE_NAME," "$TEST_CONFIG_PATH" sed -i'.bak' "s,:$MANIFEST_IMAGE_VERSION,:$TEST_IMAGE_VERSION," "$TEST_CONFIG_PATH" -export KUBECONFIG=$KUBECONFIG_PATH +if [[ $RUN_KOPS_TEST != true ]]; then + export KUBECONFIG=$KUBECONFIG_PATH +else + run_kops_conformance + KUBECTL_PATH=kubectl +fi ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]') echo "*******************************************************************************" @@ -177,11 +195,14 @@ echo "Updating CNI to image $IMAGE_NAME:$TEST_IMAGE_VERSION" echo "Using init container $INIT_IMAGE_NAME:$TEST_IMAGE_VERSION" START=$SECONDS $KUBECTL_PATH apply -f "$TEST_CONFIG_PATH" +sleep 5 +while [[ $($KUBECTL_PATH describe ds aws-node -n=kube-system | grep "Available Pods: 0") ]] +do + sleep 5 + echo "Waiting for daemonset update" +done +echo "Updated!" -# Delay based on 3 nodes, 30s grace period per CNI pod -echo "TODO: Poll and wait for updates to complete instead!" -echo "Sleeping for 110s" -sleep 110 CNI_IMAGE_UPDATE_DURATION=$((SECONDS - START)) echo "TIMELINE: Updating CNI image took $CNI_IMAGE_UPDATE_DURATION seconds." @@ -216,7 +237,11 @@ fi if [[ "$DEPROVISION" == true ]]; then START=$SECONDS - down-test-cluster + if [[ "$RUN_KOPS_TEST" == true ]]; then + down-kops-cluster + else + down-test-cluster + fi DOWN_DURATION=$((SECONDS - START)) echo "TIMELINE: Down processes took $DOWN_DURATION seconds."