WIP

Makefile

@@ -52,6 +52,10 @@ test-e2e-single-az:
 test-e2e-multi-az:
 	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c INSTANCE_TYPE=c5.large GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" ./hack/run-e2e-test
 
+.PHONY: test-e2e-migration
+test-e2e-migration:
+	AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-migration\]" ./hack/run-e2e-test
+
 .PHONY: image-release
 image-release:
 	docker build -t $(IMAGE):$(VERSION) .

aws-ebs-csi-driver/Chart.yaml

@@ -3,7 +3,7 @@ appVersion: "0.3.0"
 name: aws-ebs-csi-driver
 description: A Helm chart for AWS EBS CSI Driver
 version: 0.1.0
-kubeVersion: ">=1.13.0"
+kubeVersion: ">=1.13.0-0"
 home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver
 sources:
   - https://github.com/kubernetes-sigs/aws-ebs-csi-driver

go.mod

@@ -52,7 +52,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/onsi/ginkgo v1.7.0
+	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.4.3
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect

go.sum

@@ -151,6 +151,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=

hack/additional-policies.yaml

@@ -15,7 +15,9 @@
             "ec2:DescribeSnapshots",
             "ec2:DescribeTags",
             "ec2:DescribeVolumes",
-            "ec2:DetachVolume"
+            "ec2:DetachVolume",
+            "ec2:ModifyVolume",
+            "ec2:DescribeVolumesModifications"
           ],
           "Resource": "*"
         }

hack/feature-gates.yaml

@@ -5,6 +5,7 @@
       CSIBlockVolume: "true"
       CSIMigration: "true"
       CSIMigrationAWS: "true"
+      ExpandCSIVolumes: "true"
       VolumeSnapshotDataSource: "true"
   kubeControllerManager:
     featureGates:
@@ -13,10 +14,12 @@
       CSIBlockVolume: "true"
       CSIMigration: "true"
       CSIMigrationAWS: "true"
+      ExpandCSIVolumes: "true"
   kubelet:
     featureGates:
       CSIDriverRegistry: "true"
       CSINodeInfo: "true"
       CSIBlockVolume: "true"
       CSIMigration: "true"
       CSIMigrationAWS: "true"
+      ExpandCSIVolumes: "true"

hack/run-e2e-test

@@ -17,15 +17,15 @@
 set -uo pipefail
 
 OS_ARCH=$(go env GOOS)-amd64
-TEST_ID=$RANDOM
+TEST_ID=11027 # TODO $RANDOM
 CLUSTER_NAME=test-cluster-$TEST_ID
 TEST_DIR=/tmp/ebs-e2e-test
 BASE_DIR=$(dirname $0)
-REGION=${AWS_REGION-us-east-1}
-ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c}
+REGION=${AWS_REGION-us-west-2}
+ZONES=${AWS_AVAILABILITY_ZONES-us-west-2a,us-west-2b,us-west-2c}
 FOCUS=${GINKGO_FOCUS-"[ebs-csi-e2e]"}
 NODES=${GINKGO_NODES:-32}
-K8S_VERSION=${K8S_VERSION-1.14.0}
+K8S_VERSION=${K8S_VERSION-1.15.3-beta.0}
 INSTANCE_TYPE=${INSTANCE_TYPE-c4.large}
 
 source $(dirname "${BASH_SOURCE}")/utils/helm.sh
@@ -38,7 +38,7 @@ KOPS_STATE_FILE=s3://k8s-kops-csi-e2e
 
 # Download kops if not yet
 if [[ ! -e $KOPS_PATH ]]; then
-    mkdir -p $TEST_DIR 
+    mkdir -p $TEST_DIR
     echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH"
     curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH
 fi
@@ -87,7 +87,6 @@ docker push $IMAGE_NAME:$IMAGE_TAG
 
 echo "Deploying driver"
 helm::init
-helm::wait_tiller
 
 helm install --name aws-ebs-csi-driver \
     --set enableVolumeScheduling=true \
@@ -98,11 +97,33 @@ helm install --name aws-ebs-csi-driver \
     ./aws-ebs-csi-driver
 
 # Run the test
-go get -u github.com/onsi/ginkgo/ginkgo
-export KUBECONFIG=$HOME/.kube/config
-ginkgo -p -nodes=$NODES -v --focus="$FOCUS" tests/e2e -- -report-dir=$ARTIFACTS
+if [[ "$GINKGO_FOCUS" == "\[ebs-csi-migration\]" ]]; then
+    DEP_DOWNLOAD_URL=https://github.com/golang/dep/releases/download/v0.5.4/dep-$OS_ARCH
+    DEP_PATH=$TEST_DIR/dep
+
+    # Download dep if not yet
+    if [[ ! -e $DEP_PATH ]]; then
+      mkdir -p $TEST_DIR
+      echo "Downloading dep from $DEP_DOWNLOAD_URL to $DEP_PATH"
+      curl -L -X GET $DEP_DOWNLOAD_URL -o $DEP_PATH
+      chmod +x $DEP_PATH
+    fi
+
+    pushd ./tests/e2e-migration
+    $DEP_PATH ensure
+    popd
+    GO111MODULE=off go test -v -timeout 0 ./tests/e2e-migration/... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="$FOCUS" -ginkgo.skip="\[Disruptive\]" -gce-zone=$REGION
+else
+    go get -u github.com/onsi/ginkgo/ginkgo
+    export KUBECONFIG=$HOME/.kube/config
+    ginkgo -p -nodes=$NODES -v --focus="$FOCUS" tests/e2e -- -report-dir=$ARTIFACTS
+fi
 TEST_PASS=$?
 
+if [[ "$GINKGO_FOCUS" == "\[ebs-csi-migration\]" ]]; then
+    echo "TODO: gather controller-manager metrics to verify no in-tree calls were made."
+fi
+
 echo "Removing driver"
 helm del --purge aws-ebs-csi-driver
 

hack/utils/helm.sh

@@ -14,10 +14,6 @@ helm::install() {
 helm::init() {
     declare -r rbac_file_path=$(dirname "${BASH_SOURCE}")/tiller-rbac.yaml
     kubectl apply -f $rbac_file_path
-    helm init --service-account tiller --history-max 200
-}
-
-helm::wait_tiller() {
-    kubectl wait --for=condition=Ready --timeout=600s deployment/tiller-deploy -n kube-system
+    helm init --service-account tiller --history-max 200 --wait
     kubectl get po -n kube-system
-}
+}

tests/e2e-in-tree/README.md → tests/e2e-migration/README.md

@@ -4,8 +4,7 @@
 # Run
 ```sh
 dep ensure
-GO111MODULE=off go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -ginkgo.focus="\[ebs-csi-migration\]" -ginkgo.skip="\[Disruptive\]|should.access.volume.from.different.nodes|block.volmode|volume-expand" -gce-zone=us-west-2
-
+GO111MODULE=off go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[ebs-csi-migration\]" -ginkgo.skip="\[Disruptive\]" -gce-zone=us-west-2
 ```
 # FAQ
 - empty `go.mod` because: https://github.com/golang/go/wiki/Modules#can-an-additional-gomod-exclude-unnecessary-content-do-modules-have-the-equivalent-of-a-gitignore-file

tests/e2e-in-tree/e2e_test.go → tests/e2e-migration/e2e_test.go

@@ -49,7 +49,7 @@ func init() {
 
 func TestEBSCSI(t *testing.T) {
 	gomega.RegisterFailHandler(ginkgo.Fail)
-	ginkgo.RunSpecs(t, "EBS CSI Suite")
+	ginkgo.RunSpecs(t, "EBS CSI Migration Suite")
 }
 
 // List of testSuites to be executed in below loop
@@ -61,8 +61,7 @@ var csiTestSuites = []func() testsuites.TestSuite{
 	testsuites.InitProvisioningTestSuite,
 	testsuites.InitSnapshottableTestSuite,
 	testsuites.InitVolumeExpandTestSuite,
-	// TODO kops provisions 1 node per zone & test can't tolerate that situation.
-	//testsuites.InitMultiVolumeTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
 }
 
 var _ = ginkgo.Describe("[ebs-csi-migration] EBS CSI Migration", func() {
@@ -71,7 +70,7 @@ var _ = ginkgo.Describe("[ebs-csi-migration] EBS CSI Migration", func() {
 	// The storage.migratedPlugins flag must be set to "kubernetes.io/aws-ebs". Then the tests will
 	// validate that CSI, not in-tree, operations are happening.
 	driver := drivers.InitAwsDriver()
-	ginkgo.Context(testsuites.GetDriverNameWithFeatureTags( drivers.InitAwsDriver()), func() {
+	ginkgo.Context(testsuites.GetDriverNameWithFeatureTags(driver), func() {
 		testsuites.DefineTestSuite(driver, csiTestSuites)
 	})
 })