diff --git a/deploy/kubernetes-1.17/README.md b/deploy/kubernetes-1.17/README.md new file mode 100644 index 000000000..d7178167d --- /dev/null +++ b/deploy/kubernetes-1.17/README.md @@ -0,0 +1,2 @@ +The deployment for Kubernetes 1.17 uses VolumeSnapshot Beta CRDs and thus is imcompatible +with Kubernetes < 1.17 when the VolumeSnapshot CRDs were Alpha. diff --git a/deploy/kubernetes-1.17/csi-hostpath-snapshotclass.yaml b/deploy/kubernetes-1.17/csi-hostpath-snapshotclass.yaml new file mode 100644 index 000000000..892dfd0c8 --- /dev/null +++ b/deploy/kubernetes-1.17/csi-hostpath-snapshotclass.yaml @@ -0,0 +1,6 @@ +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshotClass +metadata: + name: csi-hostpath-snapclass +driver: hostpath.csi.k8s.io #csi-hostpath +deletionPolicy: Delete diff --git a/deploy/kubernetes-1.17/deploy-hostpath.sh b/deploy/kubernetes-1.17/deploy-hostpath.sh new file mode 120000 index 000000000..589c43f62 --- /dev/null +++ b/deploy/kubernetes-1.17/deploy-hostpath.sh @@ -0,0 +1 @@ +../util/deploy-hostpath.sh \ No newline at end of file diff --git a/deploy/kubernetes-1.17/hostpath/csi-hostpath-attacher.yaml b/deploy/kubernetes-1.17/hostpath/csi-hostpath-attacher.yaml new file mode 100644 index 000000000..823763a91 --- /dev/null +++ b/deploy/kubernetes-1.17/hostpath/csi-hostpath-attacher.yaml @@ -0,0 +1,55 @@ +kind: Service +apiVersion: v1 +metadata: + name: csi-hostpath-attacher + labels: + app: csi-hostpath-attacher +spec: + selector: + app: csi-hostpath-attacher + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-attacher +spec: + serviceName: "csi-hostpath-attacher" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-attacher + template: + metadata: + labels: + app: csi-hostpath-attacher + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + serviceAccountName: csi-attacher + containers: + - name: csi-attacher + image: quay.io/k8scsi/csi-attacher:v2.0.0 + args: + - --v=5 + - --csi-address=/csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.17/hostpath/csi-hostpath-driverinfo.yaml b/deploy/kubernetes-1.17/hostpath/csi-hostpath-driverinfo.yaml new file mode 100644 index 000000000..47d6486be --- /dev/null +++ b/deploy/kubernetes-1.17/hostpath/csi-hostpath-driverinfo.yaml @@ -0,0 +1,12 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: hostpath.csi.k8s.io +spec: + # Supports persistent and ephemeral inline volumes. + volumeLifecycleModes: + - Persistent + - Ephemeral + # To determine at runtime which mode a volume uses, pod info and its + # "csi.storage.k8s.io/ephemeral" entry are needed. + podInfoOnMount: true diff --git a/deploy/kubernetes-1.17/hostpath/csi-hostpath-plugin.yaml b/deploy/kubernetes-1.17/hostpath/csi-hostpath-plugin.yaml new file mode 100644 index 000000000..2d54b74e8 --- /dev/null +++ b/deploy/kubernetes-1.17/hostpath/csi-hostpath-plugin.yaml @@ -0,0 +1,143 @@ +# Service defined here, plus serviceName below in StatefulSet, +# are needed only because of condition explained in +# https://github.com/kubernetes/kubernetes/issues/69608 + +kind: Service +apiVersion: v1 +metadata: + name: csi-hostpathplugin + labels: + app: csi-hostpathplugin +spec: + selector: + app: csi-hostpathplugin + ports: + - name: dummy + port: 12345 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpathplugin +spec: + serviceName: "csi-hostpathplugin" + # One replica only: + # Host path driver only works when everything runs + # on a single node. We achieve that by starting it once and then + # co-locate all other pods via inter-pod affinity + replicas: 1 + selector: + matchLabels: + app: csi-hostpathplugin + template: + metadata: + labels: + app: csi-hostpathplugin + spec: + hostNetwork: true + containers: + - name: node-driver-registrar + image: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "rm -rf /registration/csi-hostpath /registration/csi-hostpath-reg.sock"] + args: + - --v=5 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/csi-hostpath/csi.sock + securityContext: + privileged: true + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /registration + name: registration-dir + - mountPath: /csi-data-dir + name: csi-data-dir + + - name: hostpath + image: quay.io/k8scsi/hostpathplugin:v1.2.0 + args: + - "--drivername=hostpath.csi.k8s.io" + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + ports: + - containerPort: 9898 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 2 + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /var/lib/kubelet/plugins + mountPropagation: Bidirectional + name: plugins-dir + - mountPath: /csi-data-dir + name: csi-data-dir + - mountPath: /dev + name: dev-dir + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: quay.io/k8scsi/livenessprobe:v1.1.0 + args: + - --csi-address=/csi/csi.sock + - --connection-timeout=3s + - --health-port=9898 + + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/pods + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet/plugins + type: Directory + name: plugins-dir + - hostPath: + # 'path' is where PV data is persisted on host. + # using /tmp is also possible while the PVs will not available after plugin container recreation or host reboot + path: /var/lib/csi-hostpath-data/ + type: DirectoryOrCreate + name: csi-data-dir + - hostPath: + path: /dev + type: Directory + name: dev-dir diff --git a/deploy/kubernetes-1.17/hostpath/csi-hostpath-provisioner.yaml b/deploy/kubernetes-1.17/hostpath/csi-hostpath-provisioner.yaml new file mode 100644 index 000000000..85904dfb1 --- /dev/null +++ b/deploy/kubernetes-1.17/hostpath/csi-hostpath-provisioner.yaml @@ -0,0 +1,55 @@ +kind: Service +apiVersion: v1 +metadata: + name: csi-hostpath-provisioner + labels: + app: csi-hostpath-provisioner +spec: + selector: + app: csi-hostpath-provisioner + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-provisioner +spec: + serviceName: "csi-hostpath-provisioner" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-provisioner + template: + metadata: + labels: + app: csi-hostpath-provisioner + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + serviceAccountName: csi-provisioner + containers: + - name: csi-provisioner + image: quay.io/k8scsi/csi-provisioner:v1.5.0-rc1 + args: + - -v=5 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.17/hostpath/csi-hostpath-resizer.yaml b/deploy/kubernetes-1.17/hostpath/csi-hostpath-resizer.yaml new file mode 100644 index 000000000..77af2afb0 --- /dev/null +++ b/deploy/kubernetes-1.17/hostpath/csi-hostpath-resizer.yaml @@ -0,0 +1,54 @@ +kind: Service +apiVersion: v1 +metadata: + name: csi-hostpath-resizer + labels: + app: csi-hostpath-resizer +spec: + selector: + app: csi-hostpath-resizer + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-resizer +spec: + serviceName: "csi-hostpath-resizer" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-resizer + template: + metadata: + labels: + app: csi-hostpath-resizer + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + serviceAccountName: csi-resizer + containers: + - name: csi-resizer + image: quay.io/k8scsi/csi-resizer:v0.3.0 + args: + - -v=5 + - -csi-address=/csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.17/hostpath/csi-hostpath-snapshotter.yaml b/deploy/kubernetes-1.17/hostpath/csi-hostpath-snapshotter.yaml new file mode 100644 index 000000000..131dc9cb9 --- /dev/null +++ b/deploy/kubernetes-1.17/hostpath/csi-hostpath-snapshotter.yaml @@ -0,0 +1,56 @@ +kind: Service +apiVersion: v1 +metadata: + name: csi-hostpath-snapshotter + labels: + app: csi-hostpath-snapshotter +spec: + selector: + app: csi-hostpath-snapshotter + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-snapshotter +spec: + serviceName: "csi-hostpath-snapshotter" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-snapshotter + template: + metadata: + labels: + app: csi-hostpath-snapshotter + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + serviceAccount: csi-snapshotter + containers: + - name: csi-snapshotter + # TODO: change to official image when released + image: quay.io/k8scsi/csi-snapshotter:v2.0.0-rc2 + imagePullPolicy: IfNotPresent + args: + - -v=5 + - --csi-address=/csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.17/hostpath/csi-hostpath-testing.yaml b/deploy/kubernetes-1.17/hostpath/csi-hostpath-testing.yaml new file mode 100644 index 000000000..3e6b837d1 --- /dev/null +++ b/deploy/kubernetes-1.17/hostpath/csi-hostpath-testing.yaml @@ -0,0 +1,59 @@ +# WARNING: this is only for testing purposes. Do not install in a production +# cluster. +# +# This exposes the hostpath's Unix domain csi.sock as a TCP port to the +# outside world. The mapping from Unix domain socket to TCP is done +# by socat. +# +# This is useful for testing with csi-sanity or csc. + +apiVersion: v1 +kind: Service +metadata: + name: hostpath-service +spec: + type: NodePort + selector: + app: csi-hostpath-socat + ports: + - port: 10000 # fixed port inside the pod, dynamically allocated port outside +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-hostpath-socat +spec: + serviceName: "csi-hostpath-socat" + replicas: 1 + selector: + matchLabels: + app: csi-hostpath-socat + template: + metadata: + labels: + app: csi-hostpath-socat + spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - csi-hostpathplugin + topologyKey: kubernetes.io/hostname + containers: + - name: socat + image: alpine/socat:1.0.3 + args: + - tcp-listen:10000,fork,reuseaddr + - unix-connect:/csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/csi-hostpath + type: DirectoryOrCreate + name: socket-dir diff --git a/deploy/kubernetes-1.17/snapshotter/csi-hostpath-snapshotclass.yaml b/deploy/kubernetes-1.17/snapshotter/csi-hostpath-snapshotclass.yaml new file mode 100644 index 000000000..892dfd0c8 --- /dev/null +++ b/deploy/kubernetes-1.17/snapshotter/csi-hostpath-snapshotclass.yaml @@ -0,0 +1,6 @@ +apiVersion: snapshot.storage.k8s.io/v1beta1 +kind: VolumeSnapshotClass +metadata: + name: csi-hostpath-snapclass +driver: hostpath.csi.k8s.io #csi-hostpath +deletionPolicy: Delete diff --git a/deploy/kubernetes-latest b/deploy/kubernetes-latest index 5ca56f454..588027544 120000 --- a/deploy/kubernetes-latest +++ b/deploy/kubernetes-latest @@ -1 +1 @@ -kubernetes-1.16 \ No newline at end of file +kubernetes-1.17 \ No newline at end of file diff --git a/deploy/util/deploy-hostpath.sh b/deploy/util/deploy-hostpath.sh index 81e92bd9c..69d790de0 100755 --- a/deploy/util/deploy-hostpath.sh +++ b/deploy/util/deploy-hostpath.sh @@ -82,13 +82,46 @@ function rbac_version () { echo "$version" } +# version_gt returns true if arg1 is greater than arg2. +# +# This function expects versions to be one of the following formats: +# X.Y.Z, release-X.Y.Z, vX.Y.Z +# +# where X,Y, and Z are any number. +# +# Partial versions (1.2, release-1.2) work as well. +# The follow substrings are stripped before version comparison: +# - "v" +# - "release-" +# +# Usage: +# version_gt release-1.3 v1.2.0 (returns true) +# version_gt v1.1.1 v1.2.0 (returns false) +# version_gt 1.1.1 v1.2.0 (returns false) +# version_gt 1.3.1 v1.2.0 (returns true) +# version_gt 1.1.1 release-1.2.0 (returns false) +# version_gt 1.2.0 1.2.2 (returns false) +function version_gt() { + versions=$(for ver in "$@"; do ver=${ver#release-}; echo ${ver#v}; done) + greaterVersion=${1#"release-"}; + greaterVersion=${greaterVersion#"v"}; + test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion" +} + # In addition, the RBAC rules can be overridden separately. +# For snapshotter 2.0+, the directory has changed. +SNAPSHOTTER_RBAC_RELATIVE_PATH="rbac.yaml" +if version_gt $(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter "${UPDATE_RBAC_RULES}") "v1.255.255"; then + SNAPSHOTTER_RBAC_RELATIVE_PATH="csi-snapshotter/rbac-csi-snapshotter.yaml" +fi +echo "SNAPSHOTTER_RBAC_RELATIVE_PATH $SNAPSHOTTER_RBAC_RELATIVE_PATH" + CSI_PROVISIONER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-provisioner.yaml" csi-provisioner false)/deploy/kubernetes/rbac.yaml" : ${CSI_PROVISIONER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-provisioner/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-provisioner.yaml" csi-provisioner "${UPDATE_RBAC_RULES}")/deploy/kubernetes/rbac.yaml} CSI_ATTACHER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-attacher/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-attacher.yaml" csi-attacher false)/deploy/kubernetes/rbac.yaml" : ${CSI_ATTACHER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-attacher/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-attacher.yaml" csi-attacher "${UPDATE_RBAC_RULES}")/deploy/kubernetes/rbac.yaml} -CSI_SNAPSHOTTER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter false)/deploy/kubernetes/rbac.yaml" -: ${CSI_SNAPSHOTTER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter "${UPDATE_RBAC_RULES}")/deploy/kubernetes/rbac.yaml} +CSI_SNAPSHOTTER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter false)/deploy/kubernetes/${SNAPSHOTTER_RBAC_RELATIVE_PATH}" +: ${CSI_SNAPSHOTTER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-snapshotter.yaml" csi-snapshotter "${UPDATE_RBAC_RULES}")/deploy/kubernetes/${SNAPSHOTTER_RBAC_RELATIVE_PATH} CSI_RESIZER_RBAC_YAML="https://raw.githubusercontent.com/kubernetes-csi/external-resizer/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-resizer.yaml" csi-resizer false)/deploy/kubernetes/rbac.yaml" : ${CSI_RESIZER_RBAC:=https://raw.githubusercontent.com/kubernetes-csi/external-resizer/$(rbac_version "${BASE_DIR}/hostpath/csi-hostpath-resizer.yaml" csi-resizer "${UPDATE_RBAC_RULES}")/deploy/kubernetes/rbac.yaml} @@ -146,7 +179,7 @@ for i in $(ls ${BASE_DIR}/hostpath/*.yaml | sort); do [ -f ${BASE_DIR}/canary-blacklist.txt ] && grep -q "^$name\$" ${BASE_DIR}/canary-blacklist.txt; then # Ignore IMAGE_TAG=canary for this particular image because its - # canary image is blacklisted in the deployment's blacklist. + # canary image is blacklisted in the deployment blacklist. suffix=$(eval echo :\${${varname}_TAG:-${tag}}) else suffix=$(eval echo :\${${varname}_TAG:-${IMAGE_TAG:-${tag}}}) @@ -186,4 +219,5 @@ done # deploy snapshotclass echo "deploying snapshotclass" -kubectl apply -f ${BASE_DIR}/snapshotter/csi-hostpath-snapshotclass.yaml +SNAPSHOTCLASS_PATH="${BASE_DIR}/snapshotter/csi-hostpath-snapshotclass.yaml" +kubectl apply -f $SNAPSHOTCLASS_PATH diff --git a/release-tools/build.make b/release-tools/build.make index 1b6f35fe1..a9b9d25dc 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -70,6 +70,7 @@ build-%: check-go-version-go CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \ fi container-%: build-% diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 3b9621a21..9bfdc2e28 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -107,8 +107,7 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39 -configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind" +configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -322,13 +321,20 @@ configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha f configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" +# Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag" + # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply # doesn't work. Such tests should be written in a way that they verify # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" +# +# "different node" test skips can be removed once +# https://github.com/kubernetes/kubernetes/pull/82678 has been backported +# to all the K8s versions we test against +configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" # This is the directory for additional result files. Usually set by Prow, but # if not (for example, when invoking manually) it defaults to the work directory. @@ -524,6 +530,7 @@ apiVersion: kind.sigs.k8s.io/v1alpha3 nodes: - role: control-plane - role: worker +- role: worker EOF # kubeadm has API dependencies between apiVersion and Kubernetes version @@ -576,8 +583,20 @@ EOF die "Cluster creation failed again, giving up. See the 'kind-cluster' artifact directory for additional logs." fi fi - KUBECONFIG="$(kind get kubeconfig-path --name=csi-prow)" - export KUBECONFIG + export KUBECONFIG="${HOME}/.kube/config" +} + +# Deletes kind cluster inside a prow job +delete_cluster_inside_prow_job() { + # Inside a real Prow job it is better to clean up at runtime + # instead of leaving that to the Prow job cleanup code + # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872). + if [ "$JOB_NAME" ]; then + if kind get clusters | grep -q csi-prow; then + run kind delete cluster --name=csi-prow || die "kind delete failed" + fi + unset KUBECONFIG + fi } # Looks for the deployment as specified by CSI_PROW_DEPLOYMENT and CSI_PROW_KUBERNETES_VERSION @@ -657,6 +676,60 @@ install_hostpath () { fi } +# Installs all nessesary snapshotter CRDs +install_snapshot_crds() { + # Wait until volumesnapshot CRDs are in place. + CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd" + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false + cnt=0 + until kubectl get volumesnapshotclasses.snapshot.storage.k8s.io \ + && kubectl get volumesnapshots.snapshot.storage.k8s.io \ + && kubectl get volumesnapshotcontents.snapshot.storage.k8s.io; do + if [ $cnt -gt 30 ]; then + echo >&2 "ERROR: snapshot CRDs not ready after over 1 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot CRDs, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 2 + done +} + +# Install snapshot controller and associated RBAC, retrying until the pod is running. +install_snapshot_controller() { + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" + cnt=0 + until kubectl get clusterrolebinding snapshot-controller-role; do + if [ $cnt -gt 30 ]; then + echo "Cluster role bindings:" + kubectl describe clusterrolebinding + echo >&2 "ERROR: snapshot controller RBAC not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done + + + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + cnt=0 + expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-) + while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do + if [ $cnt -gt 30 ]; then + echo "snapshot-controller pod status:" + kubectl describe pods -l app=snapshot-controller + echo >&2 "ERROR: snapshot controller not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done +} + # collect logs and cluster status (like the version of all components, Kubernetes version, test version) collect_cluster_info () { cat <"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again @@ -927,6 +996,32 @@ make_test_to_junit () { fi } +# version_gt returns true if arg1 is greater than arg2. +# +# This function expects versions to be one of the following formats: +# X.Y.Z, release-X.Y.Z, vX.Y.Z +# +# where X,Y, and Z are any number. +# +# Partial versions (1.2, release-1.2) work as well. +# The follow substrings are stripped before version comparison: +# - "v" +# - "release-" +# +# Usage: +# version_gt release-1.3 v1.2.0 (returns true) +# version_gt v1.1.1 v1.2.0 (returns false) +# version_gt 1.1.1 v1.2.0 (returns false) +# version_gt 1.3.1 v1.2.0 (returns true) +# version_gt 1.1.1 release-1.2.0 (returns false) +# version_gt 1.2.0 1.2.2 (returns false) +function version_gt() { + versions=$(for ver in "$@"; do ver=${ver#release-}; echo "${ver#v}"; done) + greaterVersion=${1#"release-"}; + greaterVersion=${greaterVersion#"v"}; + test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion" +} + main () { local images ret ret=0 @@ -987,6 +1082,16 @@ main () { if tests_need_non_alpha_cluster; then start_cluster || die "starting the non-alpha cluster failed" + # Install necessary snapshot CRDs and snapshot controller + # For Kubernetes 1.17+, we will install the CRDs and snapshot controller. + if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then + info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller" + install_snapshot_crds + install_snapshot_controller + else + info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller" + fi + # Installing the driver might be disabled. if install_hostpath "$images"; then collect_cluster_info @@ -1017,12 +1122,23 @@ main () { fi fi fi + delete_cluster_inside_prow_job fi if tests_need_alpha_cluster && [ "${CSI_PROW_E2E_ALPHA_GATES}" ]; then # Need to (re)create the cluster. start_cluster "${CSI_PROW_E2E_ALPHA_GATES}" || die "starting alpha cluster failed" + # Install necessary snapshot CRDs and snapshot controller + # For Kubernetes 1.17+, we will install the CRDs and snapshot controller. + if version_gt "${CSI_PROW_KUBERNETES_VERSION}" "1.16.255" || "${CSI_PROW_KUBERNETES_VERSION}" == "latest"; then + info "Version ${CSI_PROW_KUBERNETES_VERSION}, installing CRDs and snapshot controller" + install_snapshot_crds + install_snapshot_controller + else + info "Version ${CSI_PROW_KUBERNETES_VERSION}, skipping CRDs and snapshot controller" + fi + # Installing the driver might be disabled. if install_hostpath "$images"; then collect_cluster_info @@ -1047,6 +1163,7 @@ main () { fi fi fi + delete_cluster_inside_prow_job fi fi