Run upstream e2e test suites with migration

Makefile

@@ -52,6 +52,10 @@ test-e2e-single-az:
 test-e2e-multi-az:
 	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c INSTANCE_TYPE=c5.large GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" ./hack/run-e2e-test
 
+.PHONY: test-e2e-migration
+test-e2e-migration:
+	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-migration\]" ./hack/run-e2e-test
+
 .PHONY: image-release
 image-release:
 	docker build -t $(IMAGE):$(VERSION) .

aws-ebs-csi-driver/Chart.yaml

@@ -3,7 +3,7 @@ appVersion: "0.3.0"
 name: aws-ebs-csi-driver
 description: A Helm chart for AWS EBS CSI Driver
 version: 0.1.0
-kubeVersion: ">=1.13.0"
+kubeVersion: ">=1.13.0-0"
 home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver
 sources:
   - https://github.com/kubernetes-sigs/aws-ebs-csi-driver

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/onsi/ginkgo v1.7.0
+	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.4.3
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect

go.sum

@@ -151,6 +151,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=

hack/additional-policies.yaml

@@ -15,7 +15,9 @@
             "ec2:DescribeSnapshots",
             "ec2:DescribeTags",
             "ec2:DescribeVolumes",
-            "ec2:DetachVolume"
+            "ec2:DetachVolume",
+            "ec2:ModifyVolume",
+            "ec2:DescribeVolumesModifications"
           ],
           "Resource": "*"
         }

hack/feature-gates.yaml

@@ -3,9 +3,23 @@
       CSIDriverRegistry: "true"
       CSINodeInfo: "true"
       CSIBlockVolume: "true"
+      CSIMigration: "true"
+      CSIMigrationAWS: "true"
+      ExpandCSIVolumes: "true"
       VolumeSnapshotDataSource: "true"
+  kubeControllerManager:
+    featureGates:
+      CSIDriverRegistry: "true"
+      CSINodeInfo: "true"
+      CSIBlockVolume: "true"
+      CSIMigration: "true"
+      CSIMigrationAWS: "true"
+      ExpandCSIVolumes: "true"
   kubelet:
     featureGates:
       CSIDriverRegistry: "true"
       CSINodeInfo: "true"
       CSIBlockVolume: "true"
+      CSIMigration: "true"
+      CSIMigrationAWS: "true"
+      ExpandCSIVolumes: "true"

hack/run-e2e-test

@@ -17,15 +17,15 @@
 set -uo pipefail
 
 OS_ARCH=$(go env GOOS)-amd64
-TEST_ID=$RANDOM
+TEST_ID=11027 # TODO $RANDOM
 CLUSTER_NAME=test-cluster-$TEST_ID
 TEST_DIR=/tmp/ebs-e2e-test
 BASE_DIR=$(dirname $0)
-REGION=${AWS_REGION-us-east-1}
-ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c}
+REGION=${AWS_REGION-us-west-2}
+ZONES=${AWS_AVAILABILITY_ZONES-us-west-2a,us-west-2b,us-west-2c}
 FOCUS=${GINKGO_FOCUS-"[ebs-csi-e2e]"}
 NODES=${GINKGO_NODES:-32}
-K8S_VERSION=${K8S_VERSION-1.14.0}
+K8S_VERSION=${K8S_VERSION-1.15.3-beta.0}
 INSTANCE_TYPE=${INSTANCE_TYPE-c4.large}
 
 source $(dirname "${BASH_SOURCE}")/utils/helm.sh
@@ -38,7 +38,7 @@ KOPS_STATE_FILE=s3://k8s-kops-csi-e2e
 
 # Download kops if not yet
 if [[ ! -e $KOPS_PATH ]]; then
-    mkdir -p $TEST_DIR 
+    mkdir -p $TEST_DIR
     echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH"
     curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH
 fi
@@ -87,7 +87,6 @@ docker push $IMAGE_NAME:$IMAGE_TAG
 
 echo "Deploying driver"
 helm::init
-helm::wait_tiller
 
 helm install --name aws-ebs-csi-driver \
     --set enableVolumeScheduling=true \
@@ -98,11 +97,43 @@ helm install --name aws-ebs-csi-driver \
     ./aws-ebs-csi-driver
 
 # Run the test
-go get -u github.com/onsi/ginkgo/ginkgo
-export KUBECONFIG=$HOME/.kube/config
-ginkgo -p -nodes=$NODES -v --focus="$FOCUS" tests/e2e -- -report-dir=$ARTIFACTS
+if [[ "$GINKGO_FOCUS" == "\[ebs-csi-migration\]" ]]; then
+    DEP_DOWNLOAD_URL=https://github.com/golang/dep/releases/download/v0.5.4/dep-$OS_ARCH
+    DEP_PATH=$TEST_DIR/dep
+
+    # Download dep if not yet
+    if [[ ! -e $DEP_PATH ]]; then
+      mkdir -p $TEST_DIR
+      echo "Downloading dep from $DEP_DOWNLOAD_URL to $DEP_PATH"
+      curl -L -X GET $DEP_DOWNLOAD_URL -o $DEP_PATH
+      chmod +x $DEP_PATH
+    fi
+
+    pushd ./tests/e2e-migration
+    $DEP_PATH ensure
+    popd
+    # TODO known test failures to skip temporarily
+    # - should not allow expansion of pvcs without AllowVolumeExpansion property
+    #   - Test passes but cleanup fails, need https://github.com/kubernetes/kubernetes/pull/81107
+    # - (block volmode) Verify if offline PVC expansion works / should resize volume when PVC is edited while pod is using it
+    #   - NodeExpand for BlockVolumes not well-defined, need more investigation and possibly https://github.com/container-storage-interface/spec/issues/380
+    # - should provision storage with mount options
+    #   - Known bug, need https://github.com/kubernetes/kubernetes/pull/80191 but not yet in a patch release
+    GO111MODULE=off go test -v -timeout 0 ./tests/e2e-migration/... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="$FOCUS" -ginkgo.skip="\[Disruptive\]\
+|should.not.allow.expansion\
+|block.volmode.+volume-expand\
+|should.provision.storage.with.mount.options" -gce-zone=$REGION
+else
+    go get -u github.com/onsi/ginkgo/ginkgo
+    export KUBECONFIG=$HOME/.kube/config
+    ginkgo -p -nodes=$NODES -v --focus="$FOCUS" tests/e2e -- -report-dir=$ARTIFACTS
+fi
 TEST_PASS=$?
 
+if [[ "$GINKGO_FOCUS" == "\[ebs-csi-migration\]" ]]; then
+    echo "TODO: gather controller-manager metrics to verify no in-tree calls were made."
+fi
+
 echo "Removing driver"
 helm del --purge aws-ebs-csi-driver
 

hack/utils/helm.sh

@@ -14,10 +14,6 @@ helm::install() {
 helm::init() {
     declare -r rbac_file_path=$(dirname "${BASH_SOURCE}")/tiller-rbac.yaml
     kubectl apply -f $rbac_file_path
-    helm init --service-account tiller --history-max 200
-}
-
-helm::wait_tiller() {
-    kubectl wait --for=condition=Ready --timeout=600s deployment/tiller-deploy -n kube-system
+    helm init --service-account tiller --history-max 200 --wait
     kubectl get po -n kube-system
-}
+}