From 2a81c003c69c57a70652ecc44ab1b85af927170d Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 6 May 2019 08:55:34 +0200 Subject: [PATCH 1/3] kubernetes-csi: configure new base image in one place This image gets updated in the generated jobs via global search/replace by others without also updating the generator script. Storing the image name in one place simplifies the task of keeping up with those changes. --- config/jobs/kubernetes-csi/gen-jobs.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/config/jobs/kubernetes-csi/gen-jobs.sh b/config/jobs/kubernetes-csi/gen-jobs.sh index 2b1c539b9770..485ff1ecfaf5 100755 --- a/config/jobs/kubernetes-csi/gen-jobs.sh +++ b/config/jobs/kubernetes-csi/gen-jobs.sh @@ -20,6 +20,9 @@ base="$(dirname $0)" +# We need this image because it has Docker in Docker and go. +dind_image="gcr.io/k8s-testimages/kubekins-e2e:v20190420-93fab49-master" + # All kubernetes-csi repos which are part of the hostpath driver example. # For these repos we generate the full test matrix. hostpath_example_repos=" @@ -192,7 +195,7 @@ EOF spec: containers: # We need this image because it has Docker in Docker and go. - - image: gcr.io/k8s-testimages/kubekins-e2e:v20190329-811f7954b-master + - image: ${dind_image} command: - runner.sh args: @@ -238,7 +241,7 @@ EOF spec: containers: # We need this image because it has Docker in Docker and go. - - image: gcr.io/k8s-testimages/kubekins-e2e:v20190329-811f7954b-master + - image: ${dind_image} command: - runner.sh args: @@ -270,7 +273,7 @@ EOF spec: containers: # We need this image because it has Docker in Docker and go. - - image: gcr.io/k8s-testimages/kubekins-e2e:v20190329-811f7954b-master + - image: ${dind_image} command: - runner.sh args: @@ -308,7 +311,7 @@ EOF spec: containers: # We need this image because it has Docker in Docker and go. - - image: gcr.io/k8s-testimages/kubekins-e2e:v20190329-811f7954b-master + - image: ${dind_image} command: - runner.sh args: @@ -347,7 +350,7 @@ EOF spec: containers: # We need this image because it has Docker in Docker and go. - - image: gcr.io/k8s-testimages/kubekins-e2e:v20190329-811f7954b-master + - image: ${dind_image} command: - runner.sh args: @@ -402,7 +405,7 @@ for tests in non-alpha alpha; do spec: containers: # We need this image because it has Docker in Docker and go. - - image: gcr.io/k8s-testimages/kubekins-e2e:v20190329-811f7954b-master + - image: ${dind_image} command: - runner.sh args: @@ -451,7 +454,7 @@ for kubernetes in 1.13.3 1.14.0 master; do spec: containers: # We need this image because it has Docker in Docker and go. - - image: gcr.io/k8s-testimages/kubekins-e2e:v20190329-811f7954b-master + - image: ${dind_image} command: - runner.sh args: From c0e327735aebc5820a68b4805b54434cf5587fb3 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 6 May 2019 09:05:43 +0200 Subject: [PATCH 2/3] kubernetes-csi: test 1.14 deployment periodically With the kubernetes-1.14 deployment finalized in the csi-driver-host-path master branch (no more canary images!) we can start testing it periodically against all Kubernetes versions for which it works. As for kubernetes-1.13, alpha testing is limited to the matching Kubernetes release. --- .../csi-driver-host-path-config.yaml | 140 ++++++++++++++++++ config/jobs/kubernetes-csi/gen-jobs.sh | 3 +- testgrid/config.yaml | 20 +++ 3 files changed, 161 insertions(+), 2 deletions(-) diff --git a/config/jobs/kubernetes-csi/csi-driver-host-path/csi-driver-host-path-config.yaml b/config/jobs/kubernetes-csi/csi-driver-host-path/csi-driver-host-path-config.yaml index ea9d6354624b..65a586a5638d 100644 --- a/config/jobs/kubernetes-csi/csi-driver-host-path/csi-driver-host-path-config.yaml +++ b/config/jobs/kubernetes-csi/csi-driver-host-path/csi-driver-host-path-config.yaml @@ -355,6 +355,111 @@ periodics: resources: requests: cpu: 2000m +- interval: 6h + name: ci-kubernetes-csi-1-14-on-kubernetes-1-13 + decorate: true + extra_refs: + - org: kubernetes-csi + repo: csi-driver-host-path + base_ref: master + labels: + preset-service-account: "true" + preset-dind-enabled: "true" + preset-bazel-remote-cache-enabled: "false" + preset-kind-volume-mounts: "true" + spec: + containers: + # We need this image because it has Docker in Docker and go. + - image: gcr.io/k8s-testimages/kubekins-e2e:v20190420-93fab49-master + command: + - runner.sh + args: + - ./.prow.sh + env: + - name: CSI_PROW_KUBERNETES_VERSION + value: "release-1.13" + - name: CSI_PROW_BUILD_JOB + value: "false" + - name: CSI_PROW_DEPLOYMENT + value: "kubernetes-1.14" + - name: CSI_PROW_TESTS + value: "sanity serial parallel" + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + cpu: 2000m +- interval: 6h + name: ci-kubernetes-csi-1-14-on-kubernetes-1-14 + decorate: true + extra_refs: + - org: kubernetes-csi + repo: csi-driver-host-path + base_ref: master + labels: + preset-service-account: "true" + preset-dind-enabled: "true" + preset-bazel-remote-cache-enabled: "false" + preset-kind-volume-mounts: "true" + spec: + containers: + # We need this image because it has Docker in Docker and go. + - image: gcr.io/k8s-testimages/kubekins-e2e:v20190420-93fab49-master + command: + - runner.sh + args: + - ./.prow.sh + env: + - name: CSI_PROW_KUBERNETES_VERSION + value: "release-1.14" + - name: CSI_PROW_BUILD_JOB + value: "false" + - name: CSI_PROW_DEPLOYMENT + value: "kubernetes-1.14" + - name: CSI_PROW_TESTS + value: "sanity serial parallel" + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + cpu: 2000m +- interval: 6h + name: ci-kubernetes-csi-1-14-on-kubernetes-master + decorate: true + extra_refs: + - org: kubernetes-csi + repo: csi-driver-host-path + base_ref: master + labels: + preset-service-account: "true" + preset-dind-enabled: "true" + preset-bazel-remote-cache-enabled: "true" + preset-kind-volume-mounts: "true" + spec: + containers: + # We need this image because it has Docker in Docker and go. + - image: gcr.io/k8s-testimages/kubekins-e2e:v20190420-93fab49-master + command: + - runner.sh + args: + - ./.prow.sh + env: + - name: CSI_PROW_KUBERNETES_VERSION + value: "latest" + - name: CSI_PROW_BUILD_JOB + value: "false" + - name: CSI_PROW_DEPLOYMENT + value: "kubernetes-1.14" + - name: CSI_PROW_TESTS + value: "sanity serial parallel" + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + cpu: 2000m - interval: 6h name: ci-kubernetes-csi-alpha-1-13-on-kubernetes-1-13 decorate: true @@ -390,6 +495,41 @@ periodics: resources: requests: cpu: 2000m +- interval: 6h + name: ci-kubernetes-csi-alpha-1-14-on-kubernetes-1-14 + decorate: true + extra_refs: + - org: kubernetes-csi + repo: csi-driver-host-path + base_ref: master + labels: + preset-service-account: "true" + preset-dind-enabled: "true" + preset-bazel-remote-cache-enabled: "false" + preset-kind-volume-mounts: "true" + spec: + containers: + # We need this image because it has Docker in Docker and go. + - image: gcr.io/k8s-testimages/kubekins-e2e:v20190420-93fab49-master + command: + - runner.sh + args: + - ./.prow.sh + env: + - name: CSI_PROW_KUBERNETES_VERSION + value: "release-1.14" + - name: CSI_PROW_BUILD_JOB + value: "false" + - name: CSI_PROW_DEPLOYMENT + value: "kubernetes-1.14" + - name: CSI_PROW_TESTS + value: "serial-alpha parallel-alpha" + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + cpu: 2000m - interval: 6h name: ci-kubernetes-csi-canary-on-kubernetes-1-13 decorate: true diff --git a/config/jobs/kubernetes-csi/gen-jobs.sh b/config/jobs/kubernetes-csi/gen-jobs.sh index 485ff1ecfaf5..d283d75a02cc 100755 --- a/config/jobs/kubernetes-csi/gen-jobs.sh +++ b/config/jobs/kubernetes-csi/gen-jobs.sh @@ -380,9 +380,8 @@ cat >>"$base/csi-driver-host-path/csi-driver-host-path-config.yaml" < Date: Tue, 7 May 2019 09:37:55 +0200 Subject: [PATCH 3/3] kubernetes-csi: deploy latest RBAC rules for *-on-master jobs The latest external-snapshotter fails in the alpha-canary-on-master job because it needs modified RBAC rules. For on-master jobs it makes sense to use also the latest RBAC rules, while for other canary jobs we want to keep the RBAC rules unmodified to detect when an upcoming release breaks compatibility. --- .../csi-driver-host-path-config.yaml | 20 +++++++++++++------ config/jobs/kubernetes-csi/gen-jobs.sh | 6 ++++-- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/config/jobs/kubernetes-csi/csi-driver-host-path/csi-driver-host-path-config.yaml b/config/jobs/kubernetes-csi/csi-driver-host-path/csi-driver-host-path-config.yaml index 65a586a5638d..dc2b009c1269 100644 --- a/config/jobs/kubernetes-csi/csi-driver-host-path/csi-driver-host-path-config.yaml +++ b/config/jobs/kubernetes-csi/csi-driver-host-path/csi-driver-host-path-config.yaml @@ -558,7 +558,9 @@ periodics: # Replace images.... - name: CSI_PROW_HOSTPATH_CANARY value: "canary" - # ... but not the RBAC rules. + # ... but the RBAC rules only when testing on master. + # The other jobs test against the unmodified deployment for + # that Kubernetes version, i.e. with the original RBAC rules. - name: UPDATE_RBAC_RULES value: "false" - name: CSI_PROW_TESTS @@ -597,7 +599,9 @@ periodics: # Replace images.... - name: CSI_PROW_HOSTPATH_CANARY value: "canary" - # ... but not the RBAC rules. + # ... but the RBAC rules only when testing on master. + # The other jobs test against the unmodified deployment for + # that Kubernetes version, i.e. with the original RBAC rules. - name: UPDATE_RBAC_RULES value: "false" - name: CSI_PROW_TESTS @@ -636,9 +640,11 @@ periodics: # Replace images.... - name: CSI_PROW_HOSTPATH_CANARY value: "canary" - # ... but not the RBAC rules. + # ... but the RBAC rules only when testing on master. + # The other jobs test against the unmodified deployment for + # that Kubernetes version, i.e. with the original RBAC rules. - name: UPDATE_RBAC_RULES - value: "false" + value: "true" - name: CSI_PROW_TESTS value: "sanity serial parallel" # docker-in-docker needs privileged mode @@ -675,9 +681,11 @@ periodics: # Replace images.... - name: CSI_PROW_HOSTPATH_CANARY value: "canary" - # ... but not the RBAC rules. + # ... but the RBAC rules only when testing on master. + # The other jobs test against the unmodified deployment for + # that Kubernetes version, i.e. with the original RBAC rules. - name: UPDATE_RBAC_RULES - value: "false" + value: "true" - name: CSI_PROW_TESTS value: "serial-alpha parallel-alpha" # docker-in-docker needs privileged mode diff --git a/config/jobs/kubernetes-csi/gen-jobs.sh b/config/jobs/kubernetes-csi/gen-jobs.sh index d283d75a02cc..639045e3056f 100755 --- a/config/jobs/kubernetes-csi/gen-jobs.sh +++ b/config/jobs/kubernetes-csi/gen-jobs.sh @@ -466,9 +466,11 @@ for kubernetes in 1.13.3 1.14.0 master; do # Replace images.... - name: CSI_PROW_HOSTPATH_CANARY value: "canary" - # ... but not the RBAC rules. + # ... but the RBAC rules only when testing on master. + # The other jobs test against the unmodified deployment for + # that Kubernetes version, i.e. with the original RBAC rules. - name: UPDATE_RBAC_RULES - value: "false" + value: "$([ "$kubernetes" = "master" ] && echo "true" || echo "false")" - name: CSI_PROW_TESTS value: "$(expand_tests "$tests")" # docker-in-docker needs privileged mode