test: add version skew testing

This has two facets:
- switching the entire driver deployment from one version to another
  while there is persistent state like active volumes
- combining components from different releases in a deployment,
  which can happen during a rolling update

Upgrade and downgrade testing is done in two directions: from 0.6 to
master and from master to 0.6. In both cases and for all deployment
methods and most volume types, some sanity checks are run:

- an unused volume must be deletable
- an unused volume must be usable for a pod
- a volume used by a pod can be removed

Different volume types are covered via test patterns, i.e. each volume
type goes through driver downgrade/upgrade separately. Persistent
filesystem types are covered in some varieties, including cache
volumes. Block and CSI inline volumes only get tested once.

This is a compromise between keeping tests small and overall runtime,
because reinstalling the driver is slow. Therefore these tests also
don't run in our pre-submit testing.

Skew testing is done by switching to an old release and replacing the
controller image.

go.mod

@@ -65,10 +65,12 @@ replace (
 	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.19.0-rc.4
 	k8s.io/kubectl => k8s.io/kubectl v0.19.0-rc.4
 	k8s.io/kubelet => k8s.io/kubelet v0.19.0-rc.4
-	k8s.io/kubernetes => k8s.io/kubernetes v1.19.0-rc.4
 	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.19.0-rc.4
 	k8s.io/metrics => k8s.io/metrics v0.19.0-rc.4
 	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.19.0-rc.4
 	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.19.0-rc.4
 	k8s.io/sample-controller => k8s.io/sample-controller v0.19.0-rc.4
 )
+
+// We need the fix from https://github.com/kubernetes/kubernetes/pull/94283
+replace k8s.io/kubernetes => github.com/jingxu97/kubernetes v1.3.0-alpha.3.0.20200827160453-c99083e14e00

go.sum

@@ -54,6 +54,7 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM=
 github.com/aws/aws-sdk-go v1.6.10/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
+github.com/aws/aws-sdk-go v1.28.2 h1:j5IXG9CdyLfcVfICqo1PXVv+rua+QQHbkXuvuU/JF+8=
 github.com/aws/aws-sdk-go v1.28.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -162,6 +163,8 @@ github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdR
 github.com/evanphx/json-patch v0.0.0-20190815234213-e83c0a1c26c8/go.mod h1:pmLOTb3x90VhIKxsA9yeQG5yfOkkKnkk1h+Ql8NDYDw=
 github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -333,7 +336,10 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
+github.com/jingxu97/kubernetes v1.3.0-alpha.3.0.20200827160453-c99083e14e00 h1:C7PUZC8k1ow7E3Sd9SpKkb+pen7Bd+TahiBm7v5Ta1M=
+github.com/jingxu97/kubernetes v1.3.0-alpha.3.0.20200827160453-c99083e14e00/go.mod h1:yhT1/ltQajQsha3tnYc9QPFYSumGM45nlZdjf7WqE1A=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -565,6 +571,8 @@ go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345 h1:2gOG36vt1BhUqpzxwZLZJxUim2dHB05vw+RAn4Q6YOU=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200716221620-18dfb9cca345/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5 h1:Gqga3zA9tdAcfqobUGjSoCob5L3f8Dt5EuOp3ihNZko=
+go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8=
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
@@ -879,14 +887,14 @@ k8s.io/kube-aggregator v0.19.0-rc.4/go.mod h1:WOkoTISv7iVFk7fTTbQUoRpj+4dA/jZLmv
 k8s.io/kube-controller-manager v0.19.0-rc.4/go.mod h1:WDg85vnU/Kbfuo6wOTPFNb/R9Y25sMM2dh/iAD3UUtI=
 k8s.io/kube-openapi v0.0.0-20200427153329-656914f816f9 h1:5NC2ITmvg8RoxoH0wgmL4zn4VZqXGsKbxrikjaQx6s4=
 k8s.io/kube-openapi v0.0.0-20200427153329-656914f816f9/go.mod h1:bfCVj+qXcEaE5SCvzBaqpOySr6tuCcpPKqF6HD8nyCw=
+k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ=
+k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
 k8s.io/kube-proxy v0.19.0-rc.4/go.mod h1:2friA88LA6cS23RiBSUa1qMsGcN/IkigqETnIMfZ5YA=
 k8s.io/kube-scheduler v0.19.0-rc.4 h1:Iv/wtf8xjQl1cNVjPhmuCbPKF4Ei82sSPKYL6E7cGuU=
 k8s.io/kube-scheduler v0.19.0-rc.4/go.mod h1:v8ypc520PG0qK4DnzDTnHpFdCEY/WZxqd4CIyLI3m98=
 k8s.io/kubectl v0.19.0-rc.4 h1:x8d3ZWkubQZhM0WZjE8BBDBwWjbzmmznfa6a1NuFIfg=
 k8s.io/kubectl v0.19.0-rc.4/go.mod h1:Mcv3axnSyOwmupb6XZ06uAwaA+pRKJ525w3dHpH7xLk=
 k8s.io/kubelet v0.19.0-rc.4/go.mod h1:Shb5kTRpyOO5sY15p/8knPcVky1HHZGM/yPYckKgllI=
-k8s.io/kubernetes v1.19.0-rc.4 h1:E3pgtybJgSaKE9T1yurW3a6y/yHbh6C69Ro9GnDqtng=
-k8s.io/kubernetes v1.19.0-rc.4/go.mod h1:+wgXVVu9ZLggkhtx861Vq254Y/ZrjuCU9jMzVZ+e/kM=
 k8s.io/legacy-cloud-providers v0.19.0-rc.4/go.mod h1:KBs0kOkl0NDrAlCE8BIS23PoToVdz3Rd18yQlpc2GGI=
 k8s.io/metrics v0.19.0-rc.4/go.mod h1:nTrsL5F9u8RRz4Bf3AtfwNoPVahjNWD3Utxwp1oKwoY=
 k8s.io/sample-apiserver v0.19.0-rc.4/go.mod h1:x/uam2CKHv7dQuDjgoBvaoYxuUEkvFfsDwIFZ81Y6tc=
@@ -911,6 +919,7 @@ sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.1-0.20200706213357-43c19bbb7fba h1:AAbnc5KQuTWKuh2QSnyghKIOTFzB0Jayv7/OFDn3Cy4=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.1-0.20200706213357-43c19bbb7fba/go.mod h1:V06abazjHneE37ZdSY/UUwPVgcJMKI/jU5XGUjgIKoc=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=