From 98e84df846b2fcb73b12442981d8a68f1205b1b4 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 21 May 2021 09:29:57 +0200 Subject: [PATCH] Squashed 'release-tools/' changes from 6616a6b..f325590 https://github.com/kubernetes-csi/csi-release-tools/commit/f325590 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/149 from pohly/cluster-logs https://github.com/kubernetes-csi/csi-release-tools/commit/4b03b30 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/155 from pohly/owners https://github.com/kubernetes-csi/csi-release-tools/commit/a6453c8 owners: introduce aliases https://github.com/kubernetes-csi/csi-release-tools/commit/ad83def Merge https://github.com/kubernetes-csi/csi-release-tools/pull/153 from pohly/fix-image-builds https://github.com/kubernetes-csi/csi-release-tools/commit/5561780 build.make: fix image publishng https://github.com/kubernetes-csi/csi-release-tools/commit/29bd39b Merge https://github.com/kubernetes-csi/csi-release-tools/pull/152 from pohly/bump-csi-test https://github.com/kubernetes-csi/csi-release-tools/commit/bc42793 prow.sh: use csi-test v4.2.0 https://github.com/kubernetes-csi/csi-release-tools/commit/b546baa Merge https://github.com/kubernetes-csi/csi-release-tools/pull/150 from mauriciopoppe/windows-multiarch-args https://github.com/kubernetes-csi/csi-release-tools/commit/bfbb6f3 add parameter base_image and addon_image to BUILD_PARAMETERS https://github.com/kubernetes-csi/csi-release-tools/commit/2d61d3b Merge https://github.com/kubernetes-csi/csi-release-tools/pull/151 from humblec/cm https://github.com/kubernetes-csi/csi-release-tools/commit/48e71f0 Replace `which` command ( non standard) with `command -v` builtin https://github.com/kubernetes-csi/csi-release-tools/commit/feb20e2 prow.sh: collect cluster logs https://github.com/kubernetes-csi/csi-release-tools/commit/7b96bea Merge https://github.com/kubernetes-csi/csi-release-tools/pull/148 from dobsonj/add-checkpathcmd-to-prow https://github.com/kubernetes-csi/csi-release-tools/commit/2d2e03b prow.sh: enable -csi.checkpathcmd option in csi-sanity https://github.com/kubernetes-csi/csi-release-tools/commit/09d4151 Merge https://github.com/kubernetes-csi/csi-release-tools/pull/147 from pohly/mock-testing https://github.com/kubernetes-csi/csi-release-tools/commit/74cfbc9 prow.sh: support mock tests https://github.com/kubernetes-csi/csi-release-tools/commit/4a3f110 prow.sh: remove obsolete test suppression git-subtree-dir: release-tools git-subtree-split: f3255906abf2e4134cb3db4ff79ddfb5ce1e91aa --- KUBERNETES_CSI_OWNERS_ALIASES | 44 ++++++++++++++ OWNERS | 7 +-- OWNERS_ALIASES | 1 + build.make | 49 ++++++++++++---- cloudbuild.yaml | 2 +- prow.sh | 107 +++++++++++++++++++++++++++------- verify-shellcheck.sh | 2 +- 7 files changed, 173 insertions(+), 39 deletions(-) create mode 100644 KUBERNETES_CSI_OWNERS_ALIASES create mode 120000 OWNERS_ALIASES diff --git a/KUBERNETES_CSI_OWNERS_ALIASES b/KUBERNETES_CSI_OWNERS_ALIASES new file mode 100644 index 000000000..6fb858aa3 --- /dev/null +++ b/KUBERNETES_CSI_OWNERS_ALIASES @@ -0,0 +1,44 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +aliases: + + # SIG-Storage chairs and leads should always have approval rights in all repos. + # Others may be added as needed here or in each repo. + kubernetes-csi-approvers: + - jsafrane + - msau42 + - saad-ali + - xing-yang + + # Reviewers are automatically assigned to new PRs. The following + # reviewers will be active in all repos. Other reviewers can be + # added in each repo. + # + # Reviewers are encouraged to set the "Busy" flag in their GitHub status + # when they are temporarily unable to review PRs. + kubernetes-csi-reviewers: + - andyzhangx + - chrishenzie + - ggriffiths + - gnufied + - j-griffith + - Jiawei0227 + - jingxu97 + - jsafrane + - pohly + - xing-yang + +# This documents who previously contributed to Kubernetes-CSI +# as approver. +emeritus_approver: +- lpabon +- sbezverk +- vladimirvivien + +# This documents who previously contributed to Kubernetes-CSI +# as reviewer. +emeritus_reviewer: +- lpabon +- saad-ali +- sbezverk +- vladimirvivien diff --git a/OWNERS b/OWNERS index 6d2f474e1..1fb74587b 100644 --- a/OWNERS +++ b/OWNERS @@ -1,11 +1,8 @@ # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md approvers: -- saad-ali -- msau42 +- kubernetes-csi-approvers - pohly reviewers: -- saad-ali -- msau42 -- pohly +- kubernetes-csi-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES new file mode 120000 index 000000000..7ec6c034d --- /dev/null +++ b/OWNERS_ALIASES @@ -0,0 +1 @@ +KUBERNETES_CSI_OWNERS_ALIASES \ No newline at end of file diff --git a/build.make b/build.make index 1faaf3b90..0ed51e80a 100644 --- a/build.make +++ b/build.make @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# force the usage of /bin/bash instead of /bin/sh +SHELL := /bin/bash + .PHONY: build-% build container-% container push-% push clean test # A space-separated list of all commands in the repository, must be @@ -63,26 +66,35 @@ endif # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. -# BUILD_PLATFORMS contains a set of triplets, +# BUILD_PLATFORMS contains a set of tuples [os arch suffix base_image addon_image] # separated by semicolon. An empty variable or empty entry (= just a # semicolon) builds for the default platform of the current Go # toolchain. BUILD_PLATFORMS = # Add go ldflags using LDFLAGS at the time of compilation. -IMPORTPATH_LDFLAGS = -X main.version=$(REV) +IMPORTPATH_LDFLAGS = -X main.version=$(REV) EXT_LDFLAGS = -extldflags "-static" -LDFLAGS = +LDFLAGS = FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) $(EXT_LDFLAGS) # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. $(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin - echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ + # os_arch_seen captures all of the $$os-$$arch seen for the current binary + # that we want to build, if we've seen an $$os-$$arch before it means that + # we don't need to build it again, this is done to avoid building + # the windows binary multiple times (see the default value of $$BUILD_PLATFORMS) + export os_arch_seen="" && echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + os_arch_seen_pre=$${os_arch_seen%%$$os-$$arch*}; \ + if ! [ $${#os_arch_seen_pre} = $${#os_arch_seen} ]; then \ + continue; \ + fi; \ if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \ echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ exit 1; \ fi; \ + os_arch_seen+=";$$os-$$arch"; \ done $(CMDS:%=container-%): container-%: build-% @@ -131,30 +143,46 @@ DOCKER_BUILDX_CREATE_ARGS ?= # the tag for the resulting multiarch image. $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ - DOCKER_CLI_EXPERIMENTAL=enabled; \ - export DOCKER_CLI_EXPERIMENTAL; \ + export DOCKER_CLI_EXPERIMENTAL=enabled; \ docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ trap "docker buildx rm multiarchimage-buildertest" EXIT; \ dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ if ! [ -f "$$dockerfile_windows" ]; then \ - build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe *[^ ]* *[^ ]*//g' -e 's/; *;/;/g' -e 's/;[ ]*$$//')"; \ fi; \ pushMultiArch () { \ tag=$$1; \ - echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ docker buildx build --push \ - --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --tag $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag \ --platform=$$os/$$arch \ --file $$(eval echo \$${dockerfile_$$os}) \ --build-arg binary=./bin/$*$$suffix \ --build-arg ARCH=$$arch \ + --build-arg BASE_IMAGE=$$base_image \ + --build-arg ADDON_IMAGE=$$addon_image \ --label revision=$(REV) \ .; \ done; \ - images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + echo $(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + done); \ docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix base_image addon_image; do \ + if [ $$os = "windows" ]; then \ + escaped_base_image=$${base_image/:/-}; \ + if ! [ -z $$escaped_base_image ]; then escaped_base_image+="-"; fi; \ + image=$(IMAGE_NAME):$$arch-$$os-$$escaped_base_image$$tag; \ + os_version=$$(docker manifest inspect mcr.microsoft.com/windows/$${base_image} | grep "os.version" | head -n 1 | awk '{print $$2}' | sed -e 's/"//g') || true; \ + docker manifest annotate --os-version $$os_version $(IMAGE_NAME):$$tag $$image; \ + fi; \ + done; \ docker manifest push -p $(IMAGE_NAME):$$tag; \ }; \ if [ $(PULL_BASE_REF) = "master" ]; then \ @@ -288,4 +316,3 @@ test-spelling: test-boilerplate: @ echo; echo "### $@:" @ ./release-tools/verify-boilerplate.sh "$(pwd)" - diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1e02ba6cb..b39a6db33 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -27,7 +27,7 @@ steps: # The image must contain bash and curl. Ideally it should also contain # the desired version of Go (currently defined in release-tools/prow.sh), # but that just speeds up the build and is not required. - - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20210331-c732583' entrypoint: ./.cloudbuild.sh env: - GIT_TAG=${_GIT_TAG} diff --git a/prow.sh b/prow.sh index fe23cf516..55f2a91cf 100755 --- a/prow.sh +++ b/prow.sh @@ -77,7 +77,10 @@ version_to_git () { esac } -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +# the list of windows versions was matched from: +# - https://hub.docker.com/_/microsoft-windows-nanoserver +# - https://hub.docker.com/_/microsoft-windows-servercore +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64; windows amd64 .exe nanoserver:1809 servercore:ltsc2019; windows amd64 .exe nanoserver:1909 servercore:1909; windows amd64 .exe nanoserver:2004 servercore:2004; windows amd64 .exe nanoserver:20H2 servercore:20H2" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because @@ -236,7 +239,7 @@ configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package" # of the cluster. The alternative would have been to (cross-)compile csi-sanity # and install it inside the cluster, which is not necessarily easier. configvar CSI_PROW_SANITY_REPO https://github.com/kubernetes-csi/csi-test "csi-test repo" -configvar CSI_PROW_SANITY_VERSION v4.0.2 "csi-test version" # v4.0.2 +configvar CSI_PROW_SANITY_VERSION v4.2.0 "csi-test version" configvar CSI_PROW_SANITY_PACKAGE_PATH github.com/kubernetes-csi/csi-test "csi-test package" configvar CSI_PROW_SANITY_SERVICE "hostpath-service" "Kubernetes TCP service name that exposes csi.sock" configvar CSI_PROW_SANITY_POD "csi-hostpathplugin-0" "Kubernetes pod with CSI driver" @@ -296,6 +299,17 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } +# Enabling mock tests adds the "CSI mock volume" tests from https://github.com/kubernetes/kubernetes/blob/master/test/e2e/storage/csi_mock_volume.go +# to the e2e.test invocations (serial, parallel, and the corresponding alpha variants). +# When testing canary images, those get used instead of the images specified +# in the e2e.test's normal YAML files. +# +# The default is to enable this for all jobs which use canary images +# because we want to know whether our release candidates will pass all +# existing tests (the storage testsuites and mock testing in +# Kubernetes). +configvar CSI_PROW_E2E_MOCK "$(if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then echo true; else echo false; fi)" "enable CSI mock volume tests" + # Regex for non-alpha, feature-tagged tests that should be run. # configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for latest Kubernetes version" @@ -354,11 +368,7 @@ configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -# -# "different node" test skips can be removed once -# https://github.com/kubernetes/kubernetes/pull/82678 has been backported -# to all the K8s versions we test against -configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" +configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" # This creates directories that are required for testing. ensure_paths () { @@ -625,11 +635,16 @@ EOF # Deletes kind cluster inside a prow job delete_cluster_inside_prow_job() { + local name="$1" + # Inside a real Prow job it is better to clean up at runtime # instead of leaving that to the Prow job cleanup code # because the later sometimes times out (https://github.com/kubernetes-csi/csi-release-tools/issues/24#issuecomment-554765872). + # + # This is also a good time to collect logs. if [ "$JOB_NAME" ]; then if kind get clusters | grep -q csi-prow; then + run kind export logs --name=csi-prow "${ARTIFACTS}/cluster-logs/$name" run kind delete cluster --name=csi-prow || die "kind delete failed" fi unset KUBECONFIG @@ -716,7 +731,7 @@ install_csi_driver () { fi } -# Installs all nessesary snapshotter CRDs +# Installs all nessesary snapshotter CRDs install_snapshot_crds() { # Wait until volumesnapshot CRDs are in place. CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/client/config/crd" @@ -763,7 +778,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" @@ -832,7 +847,7 @@ install_snapshot_controller() { fi echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" cnt=$((cnt + 1)) - sleep 10 + sleep 10 done } @@ -877,6 +892,29 @@ start_loggers () { done } +# Patches the image versions of test/e2e/testing-manifests/storage-csi/mock in the k/k +# source code, if needed. +patch_kubernetes () { + local source="$1" target="$2" + + if [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then + # We cannot replace k8s.gcr.io/sig-storage with gcr.io/k8s-staging-sig-storage because + # e2e.test does not support it (see test/utils/image/manifest.go). Instead we + # invoke the e2e.test binary with KUBE_TEST_REPO_LIST set to a file that + # overrides that registry. + find "$source/test/e2e/testing-manifests/storage-csi/mock" -name '*.yaml' -print0 | xargs -0 sed -i -e 's;k8s.gcr.io/sig-storage/\(.*\):v.*;k8s.gcr.io/sig-storage/\1:canary;' + cat >"$target/e2e-repo-list" <&2 <&1) + +EOF + fi +} + # Makes the E2E test suite binary available as "${CSI_PROW_WORK}/e2e.test". install_e2e () { if [ -e "${CSI_PROW_WORK}/e2e.test" ]; then @@ -885,6 +923,7 @@ install_e2e () { git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 && if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then + patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" && go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" && run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" @@ -936,7 +975,7 @@ run_e2e () ( trap move_junit EXIT cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" && - run_with_loggers ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" + run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml" ) # Run csi-sanity against installed CSI driver. @@ -948,7 +987,7 @@ run_sanity () ( kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- mkdir "\$@" && echo "\$@" EOF # Using "rm -rf" as fallback for "rmdir" is a workaround for: - # Node Service + # Node Service # should work # /nvme/gopath.tmp/src/github.com/kubernetes-csi/csi-test/pkg/sanity/node.go:624 # STEP: reusing connection to CSI driver at dns:///172.17.0.2:30896 @@ -975,6 +1014,24 @@ if ! kubectl exec "${CSI_PROW_SANITY_POD}" -c "${CSI_PROW_SANITY_CONTAINER}" -- exit 1 fi EOF + + cat >"${CSI_PROW_WORK}/checkdir_in_pod.sh" </dev/null; then +if command -v shellcheck &>/dev/null; then detected_version="$(shellcheck --version | grep 'version: .*')" if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then HAVE_SHELLCHECK=true