Skip to content

Commit

Permalink
Run upstream e2e test suites with migration
Browse files Browse the repository at this point in the history
  • Loading branch information
wongma7 committed Aug 24, 2019
1 parent debfdb4 commit ec56a59
Show file tree
Hide file tree
Showing 13 changed files with 880 additions and 19 deletions.
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ test-e2e-single-az:
test-e2e-multi-az:
AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c INSTANCE_TYPE=c5.large GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" ./hack/run-e2e-test

.PHONY: test-e2e-migration
test-e2e-migration:
AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-migration\]" ./hack/run-e2e-test

.PHONY: image-release
image-release:
docker build -t $(IMAGE):$(VERSION) .
Expand Down
2 changes: 1 addition & 1 deletion aws-ebs-csi-driver/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ appVersion: "0.3.0"
name: aws-ebs-csi-driver
description: A Helm chart for AWS EBS CSI Driver
version: 0.1.0
kubeVersion: ">=1.13.0"
kubeVersion: ">=1.13.0-0"
home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver
sources:
- https://github.com/kubernetes-sigs/aws-ebs-csi-driver
Expand Down
5 changes: 4 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@ require (
github.com/json-iterator/go v1.1.5 // indirect
github.com/kubernetes-csi/csi-test v2.0.0+incompatible
github.com/kubernetes-csi/external-snapshotter v1.0.1
github.com/onsi/ginkgo v1.7.0
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/onsi/ginkgo v1.8.0
github.com/onsi/gomega v1.4.3
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/soheilhy/cmux v0.1.4 // indirect
Expand Down
3 changes: 2 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,8 @@ github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6Yf
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
Expand Down
4 changes: 3 additions & 1 deletion hack/additional-policies.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
"ec2:DescribeSnapshots",
"ec2:DescribeTags",
"ec2:DescribeVolumes",
"ec2:DetachVolume"
"ec2:DetachVolume",
"ec2:ModifyVolume",
"ec2:DescribeVolumesModifications"
],
"Resource": "*"
}
Expand Down
17 changes: 17 additions & 0 deletions hack/feature-gates.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,26 @@
CSIDriverRegistry: "true"
CSINodeInfo: "true"
CSIBlockVolume: "true"
CSIMigration: "true"
CSIMigrationAWS: "true"
ExpandCSIVolumes: "true"
VolumeSnapshotDataSource: "true"
CSIInlineVolume: "true"
kubeControllerManager:
featureGates:
CSIDriverRegistry: "true"
CSINodeInfo: "true"
CSIBlockVolume: "true"
CSIMigration: "true"
CSIMigrationAWS: "true"
ExpandCSIVolumes: "true"
CSIInlineVolume: "true"
kubelet:
featureGates:
CSIDriverRegistry: "true"
CSINodeInfo: "true"
CSIBlockVolume: "true"
CSIMigration: "true"
CSIMigrationAWS: "true"
ExpandCSIVolumes: "true"
CSIInlineVolume: "true"
50 changes: 41 additions & 9 deletions hack/run-e2e-test
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ TEST_ID=$RANDOM
CLUSTER_NAME=test-cluster-$TEST_ID
TEST_DIR=/tmp/ebs-e2e-test
BASE_DIR=$(dirname $0)
REGION=${AWS_REGION-us-east-1}
ZONES=${AWS_AVAILABILITY_ZONES-us-east-1a,us-east-1b,us-east-1c}
REGION=${AWS_REGION-us-west-2}
ZONES=${AWS_AVAILABILITY_ZONES-us-west-2a,us-west-2b,us-west-2c}
FOCUS=${GINKGO_FOCUS-"[ebs-csi-e2e]"}
NODES=${GINKGO_NODES:-32}
K8S_VERSION=${K8S_VERSION-1.14.0}
K8S_VERSION=${K8S_VERSION-1.15.3}
INSTANCE_TYPE=${INSTANCE_TYPE-c4.large}

source $(dirname "${BASH_SOURCE}")/utils/helm.sh
Expand All @@ -38,7 +38,7 @@ KOPS_STATE_FILE=s3://k8s-kops-csi-e2e

# Download kops if not yet
if [[ ! -e $KOPS_PATH ]]; then
mkdir -p $TEST_DIR
mkdir -p $TEST_DIR
echo "Downloading KOPS from $KOPS_DOWNLOAD_URL to $KOPS_PATH"
curl -L -X GET $KOPS_DOWNLOAD_URL -o $KOPS_PATH
fi
Expand Down Expand Up @@ -88,7 +88,6 @@ done;

echo "Deploying driver"
helm::init
helm::wait_tiller

helm install --name aws-ebs-csi-driver \
--set enableVolumeScheduling=true \
Expand All @@ -99,10 +98,43 @@ helm install --name aws-ebs-csi-driver \
./aws-ebs-csi-driver

# Run the test
go get -u github.com/onsi/ginkgo/ginkgo
export KUBECONFIG=$HOME/.kube/config
ginkgo -p -nodes=$NODES -v --focus="$FOCUS" tests/e2e -- -report-dir=$ARTIFACTS
TEST_PASS=$?
if [[ "$GINKGO_FOCUS" == "\[ebs-csi-migration\]" ]]; then
# TODO known test failures to skip temporarily
# - should not allow expansion of pvcs without AllowVolumeExpansion property
# - Test passes but cleanup fails, need https://github.com/kubernetes/kubernetes/pull/81107
# - (block volmode) Verify if offline PVC expansion works / should resize volume when PVC is edited while pod is using it
# - NodeExpand for BlockVolumes not well-defined, need more investigation and possibly https://github.com/container-storage-interface/spec/issues/380
# - should provision storage with mount options
# - Known bug, need https://github.com/kubernetes/kubernetes/pull/80191 but not yet in a patch release
pushd ./tests/e2e-migration
go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="$FOCUS" -ginkgo.skip="\[Disruptive\]\
|should.not.allow.expansion\
|block.volmode.+volume-expand\
|should.provision.storage.with.mount.options" -gce-zone=${ZONES%,*}
TEST_PASS=$?
popd

# There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
# Find the controller-manager log and read its metrics to verify
NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
kubectl port-forward kube-controller-manager-$NODE 10252:10252 -n kube-system&
curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"'
CSI_CALLED=$?
curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"'
INTREE_CALLED=$?

# TEST_PASS if tests passed, CSI was called, and In-tree was not called
if [ "$TEST_PASS" == 0 ] && [ "$CSI_CALLED" == 0 ] && [ "$INTREE_CALLED" == 1 ]; then
TEST_PASS=0
else
TEST_PASS=1
fi
else
go get -u github.com/onsi/ginkgo/ginkgo
export KUBECONFIG=$HOME/.kube/config
ginkgo -p -nodes=$NODES -v --focus="$FOCUS" tests/e2e -- -report-dir=$ARTIFACTS
TEST_PASS=$?
fi

echo "Removing driver"
helm del --purge aws-ebs-csi-driver
Expand Down
8 changes: 2 additions & 6 deletions hack/utils/helm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,6 @@ helm::install() {
helm::init() {
declare -r rbac_file_path=$(dirname "${BASH_SOURCE}")/tiller-rbac.yaml
kubectl apply -f $rbac_file_path
helm init --service-account tiller --history-max 200
}

helm::wait_tiller() {
kubectl wait --for=condition=Ready --timeout=600s deployment/tiller-deploy -n kube-system
helm init --service-account tiller --history-max 200 --wait
kubectl get po -n kube-system
}
}
13 changes: 13 additions & 0 deletions tests/e2e-migration/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Prerequisites
- kubernetes 1.16+ AWS cluster

# Run
```sh
go test -v -timeout 0 ./... -kubeconfig=$HOME/.kube/config -report-dir=$ARTIFACTS -ginkgo.focus="\[ebs-csi-migration\]" -ginkgo.skip="\[Disruptive\]" -gce-zone=us-west-2a
```

# Update dependencies
```sh
go mod edit -require=k8s.io/[email protected]
./hack/update-gomod.sh v1.15.3
```
93 changes: 93 additions & 0 deletions tests/e2e-migration/e2e_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"flag"
"fmt"
"log"
"os"
"path"
"path/filepath"
"testing"

"github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/config"
"github.com/onsi/ginkgo/reporters"
"github.com/onsi/gomega"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/storage/drivers"
"k8s.io/kubernetes/test/e2e/storage/testsuites"

// ensure that cloud provider is loaded
_ "k8s.io/kubernetes/test/e2e/framework/providers/aws"
)

const kubeconfigEnvVar = "KUBECONFIG"

func init() {
// k8s.io/kubernetes/test/e2e/framework requires env KUBECONFIG to be set
// it does not fall back to defaults
if os.Getenv(kubeconfigEnvVar) == "" {
kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config")
os.Setenv(kubeconfigEnvVar, kubeconfig)
}
framework.RegisterCommonFlags(flag.CommandLine)
framework.RegisterClusterFlags(flag.CommandLine)
_ = flag.Set("storage.migratedPlugins", "kubernetes.io/aws-ebs")
_ = flag.Set("provider", "aws")
flag.Parse()
framework.AfterReadingAllFlags(&framework.TestContext)
}

func TestEBSCSI(t *testing.T) {
gomega.RegisterFailHandler(ginkgo.Fail)

// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
var r []ginkgo.Reporter
if framework.TestContext.ReportDir != "" {
if err := os.MkdirAll(framework.TestContext.ReportDir, 0755); err != nil {
log.Fatalf("Failed creating report directory: %v", err)
} else {
r = append(r, reporters.NewJUnitReporter(path.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%v%02d.xml", framework.TestContext.ReportPrefix, config.GinkgoConfig.ParallelNode))))
}
}
log.Printf("Starting e2e run %q on Ginkgo node %d", framework.RunID, config.GinkgoConfig.ParallelNode)

ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "EBS CSI Migration Suite", r)
}

// List of testSuites to be executed in below loop
var csiTestSuites = []func() testsuites.TestSuite{
testsuites.InitVolumesTestSuite,
testsuites.InitVolumeIOTestSuite,
testsuites.InitVolumeModeTestSuite,
testsuites.InitSubPathTestSuite,
testsuites.InitProvisioningTestSuite,
testsuites.InitSnapshottableTestSuite,
testsuites.InitVolumeExpandTestSuite,
testsuites.InitMultiVolumeTestSuite,
}

var _ = ginkgo.Describe("[ebs-csi-migration] EBS CSI Migration", func() {
// Init the *in-tree* driver.
// The CSIMigration & CSIMigrationAWS feature gates must be enabled on the cluster.
// The storage.migratedPlugins flag must be set to "kubernetes.io/aws-ebs". Then the tests will
// validate that CSI, not in-tree, operations are happening.
driver := drivers.InitAwsDriver()
ginkgo.Context(testsuites.GetDriverNameWithFeatureTags(driver), func() {
testsuites.DefineTestSuite(driver, csiTestSuites)
})
})
57 changes: 57 additions & 0 deletions tests/e2e-migration/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
module github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e-migration

go 1.12

replace k8s.io/api => k8s.io/api v0.0.0-20190822053644-6185379c914a

replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190822063004-0670dc4fec4e

replace k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190820074809-31b1e1ea64dc

replace k8s.io/apiserver => k8s.io/apiserver v0.0.0-20190822060508-785eacbd19ae

replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20190822063658-442a64f3fed7

replace k8s.io/client-go => k8s.io/client-go v0.0.0-20190822054823-0a74433fb222

replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20190822065847-2058b41dfbb6

replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.0.0-20190822065536-566e5fc137f7

replace k8s.io/code-generator => k8s.io/code-generator v0.0.0-20190820100630-060a3d12ce80

replace k8s.io/component-base => k8s.io/component-base v0.0.0-20190822055535-1f6a258f5d89

replace k8s.io/cri-api => k8s.io/cri-api v0.0.0-20190820110325-95eec93e2395

replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.0.0-20190822070154-f51cd605b3ee

replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.0.0-20190822061015-a4f93a8219ed

replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.0.0-20190822065235-826221481525

replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.0.0-20190822064323-7e0495d8a3ff

replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.0.0-20190822064931-4470440ed041

replace k8s.io/kubectl => k8s.io/kubectl v0.0.0-20190822071625-14af4a63a1e1

replace k8s.io/kubelet => k8s.io/kubelet v0.0.0-20190822064626-fa8f3d935631

replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.0.0-20190822070624-3a30a18bba71

replace k8s.io/metrics => k8s.io/metrics v0.0.0-20190822063337-6c03eb8600ee

replace k8s.io/node-api => k8s.io/node-api v0.0.0-20190822070940-24e163ffb9e7

replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.0.0-20190822061642-ab22eab63834

replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.0.0-20190822064016-bcca3cc588da

replace k8s.io/sample-controller => k8s.io/sample-controller v0.0.0-20190822062306-1b561d990eb5

require (
github.com/onsi/ginkgo v1.9.0
github.com/onsi/gomega v1.6.0
k8s.io/kubernetes v1.16.0-beta.1
)
Loading

0 comments on commit ec56a59

Please sign in to comment.