Skip to content

Commit

Permalink
Merge pull request #54 from pohly/prow-update-master
Browse files Browse the repository at this point in the history
master: update release-tools
  • Loading branch information
k8s-ci-robot authored Nov 5, 2019
2 parents 69b974f + 40bd270 commit ea69850
Show file tree
Hide file tree
Showing 26 changed files with 190 additions and 5,520 deletions.
61 changes: 18 additions & 43 deletions release-tools/build.make
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
# including build.make.
REGISTRY_NAME=quay.io/k8scsi

# Can be set to -mod=vendor to ensure that the "vendor" directory is used.
GOFLAGS_VENDOR=

# Revision that gets built into each binary via the main.version
# string. Uses the `git describe` output based on the most recent
# version tag with a short revision suffix or, if nothing has been
Expand Down Expand Up @@ -62,11 +65,11 @@ ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))
# Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
# to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.

build-%:
build-%: check-go-version-go
mkdir -p bin
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
if [ "$$ARCH" = "amd64" ]; then \
CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \
CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \
fi

container-%: build-%
Expand Down Expand Up @@ -97,19 +100,19 @@ push: $(CMDS:%=push-%)
clean:
-rm -rf bin

test:
test: check-go-version-go

.PHONY: test-go
test: test-go
test-go:
@ echo; echo "### $@:"
go test `go list ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)
go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)

.PHONY: test-vet
test: test-vet
test-vet:
@ echo; echo "### $@:"
go vet `go list ./... | grep -v vendor $(TEST_VET_FILTER_CMD)`
go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)`

.PHONY: test-fmt
test: test-fmt
Expand Down Expand Up @@ -154,43 +157,7 @@ test-fmt:
test: test-vendor
test-vendor:
@ echo; echo "### $@:"
@ if [ -f Gopkg.toml ]; then \
echo "Repo uses 'dep' for vendoring."; \
case "$$(dep version 2>/dev/null | grep 'version *:')" in \
*v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \
*) echo "skipping check, dep >= 0.5 required";; \
esac; \
elif [ -f go.mod ]; then \
echo "Repo uses 'go mod'."; \
if [ "$${JOB_NAME}" ] && \
( [ "$${JOB_TYPE}" != "presubmit" ] || \
[ $$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools; \
git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') | \
wc -l) -eq 0 ] ); then \
echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies."; \
elif ! GO111MODULE=on go mod tidy; then \
echo "ERROR: vendor check failed."; \
false; \
elif [ $$(git status --porcelain -- go.mod go.sum | wc -l) -gt 0 ]; then \
echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; \
git diff -- go.mod go.sum; \
false; \
elif [ -d vendor ]; then \
if ! GO111MODULE=on go mod vendor; then \
echo "ERROR: vendor check failed."; \
false; \
elif [ $$(git status --porcelain -- vendor | wc -l) -gt 0 ]; then \
echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"; \
git status -- vendor; \
git diff -- vendor; \
false; \
else \
echo "Go dependencies and vendor directory up-to-date."; \
fi; \
else \
echo "Go dependencies up-to-date."; \
fi; \
fi
@ ./release-tools/verify-vendor.sh

.PHONY: test-subtree
test: test-subtree
Expand All @@ -216,3 +183,11 @@ test-shellcheck:
./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \
done; \
exit $$ret

# Targets in the makefile can depend on check-go-version-<path to go binary>
# to trigger a warning if the x.y version of that binary does not match
# what the project uses. Make ensures that this is only checked once per
# invocation.
.PHONY: check-go-version-%
check-go-version-%:
./release-tools/verify-go-version.sh "$*"
50 changes: 19 additions & 31 deletions release-tools/prow.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,12 @@ get_versioned_variable () {
echo "$value"
}

# If we have a vendor directory, then use it. We must be careful to only
# use this for "make" invocations inside the project's repo itself because
# setting it globally can break other go usages (like "go get <some command>"
# which is disabled with GOFLAGS=-mod=vendor).
configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory"

# Go versions can be specified seperately for different tasks
# If the pre-installed Go is missing or a different
# version, the required version here will get installed
Expand All @@ -101,7 +107,8 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version
# kind version to use. If the pre-installed version is different,
# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/
# (if available), otherwise it is built from source.
configvar CSI_PROW_KIND_VERSION "v0.5.0" "kind"
# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39
configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind"

# ginkgo test runner version to use. If the pre-installed version is
# different, the desired version is built from source.
Expand Down Expand Up @@ -136,7 +143,6 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes"
#
# If the version is prefixed with "release-", then nothing
# is overridden.
override_k8s_version "1.13.10"
override_k8s_version "1.14.6"
override_k8s_version "1.15.3"

Expand Down Expand Up @@ -183,7 +189,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
#
# When no deploy script is found (nothing in `deploy` directory,
# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed.
configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0-rc8" "hostpath driver"
configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0" "hostpath driver"
configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo"
configvar CSI_PROW_DEPLOYMENT "" "deployment"
configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name"
Expand All @@ -200,9 +206,10 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image"
# all generated files are present.
#
# CSI_PROW_E2E_REPO=none disables E2E testing.
configvar CSI_PROW_E2E_VERSION_1_13 v1.14.0 "E2E version for Kubernetes 1.13.x" # we can't use the one from 1.13.x because it didn't have --storage.testdriver
# TOOO: remove versioned variables and make e2e version match k8s version
configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x"
configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x"
configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x"
# TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases
configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version
configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions
Expand Down Expand Up @@ -292,11 +299,6 @@ regex_join () {
# alpha in previous Kubernetes releases. This was considered too
# error prone. Therefore we use E2E tests that match the Kubernetes
# version that is getting tested.
#
# However, for 1.13.x testing we have to use the E2E tests from 1.14
# because 1.13 didn't have --storage.testdriver yet, so for that (and only
# that version) we have to define alpha tests differently.
configvar CSI_PROW_E2E_ALPHA_1_13 '\[Feature: \[Testpattern:.Dynamic.PV..block.volmode.\] should.create.and.delete.block.persistent.volumes' "alpha tests for Kubernetes 1.13" # Raw block was an alpha feature in 1.13.
configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough
configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests"

Expand All @@ -312,12 +314,12 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi
# kubernetes-csi components must be updated, either by disabling
# the failing test for "latest" or by updating the test and not running
# it anymore for older releases.
configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13"
configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14"
configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15"
configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16"
# TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and
# add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST.
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes"
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes"
configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates"

# Some tests are known to be unusable in a KinD cluster. For example,
Expand Down Expand Up @@ -723,22 +725,6 @@ install_sanity () (
run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed"
)

# Whether the hostpath driver supports raw block devices depends on which version
# we are testing. It would be much nicer if we could determine that by querying the
# installed driver's capabilities instead of having to do a version check.
hostpath_supports_block () {
local result
result="$(docker exec csi-prow-control-plane docker image ls --format='{{.Repository}} {{.Tag}} {{.ID}}' | grep hostpath | while read -r repo tag id; do
if [ "$tag" == "v1.0.1" ]; then
# Old version because the revision label is missing: didn't have support yet.
echo "false"
return
fi
done)"
# If not set, then it must be a newer driver with support.
echo "${result:-true}"
}
# The default implementation of this function generates a external
# driver test configuration for the hostpath driver.
#
Expand All @@ -755,12 +741,14 @@ SnapshotClass:
DriverInfo:
Name: ${CSI_PROW_HOSTPATH_DRIVER_NAME}
Capabilities:
block: $(hostpath_supports_block)
block: true
persistence: true
dataSource: true
multipods: true
nodeExpansion: true
controllerExpansion: true
snapshotDataSource: true
singleNodeVolume: true
EOF
}

Expand Down Expand Up @@ -946,7 +934,7 @@ main () {
images=
if ${CSI_PROW_BUILD_JOB}; then
# A successful build is required for testing.
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all || die "'make all' failed"
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make all' failed"
# We don't want test failures to prevent E2E testing below, because the failure
# might have been minor or unavoidable, for example when experimenting with
# changes in "release-tools" in a PR (that fails the "is release-tools unmodified"
Expand All @@ -956,13 +944,13 @@ main () {
warn "installing 'dep' failed, cannot test vendoring"
ret=1
fi
if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test 2>&1 | make_test_to_junit; then
if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" 2>&1 | make_test_to_junit; then
warn "'make test' failed, proceeding anyway"
ret=1
fi
fi
# Required for E2E testing.
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container || die "'make container' failed"
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make container' failed"
fi

if tests_need_kind; then
Expand Down
6 changes: 3 additions & 3 deletions release-tools/travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,16 @@ git:
depth: false
matrix:
include:
- go: 1.12.4
- go: 1.13.3
before_script:
- mkdir -p bin
- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep
- chmod u+x bin/dep
- export PATH=$PWD/bin:$PATH
script:
- make -k all test
- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' )
after_success:
- if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io;
make push;
make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' );
fi
23 changes: 23 additions & 0 deletions release-tools/update-vendor.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/usr/bin/env bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if [ -f Gopkg.toml ]; then
echo "Repo uses 'dep' for vendoring."
(set -x; dep ensure)
elif [ -f go.mod ]; then
release-tools/verify-go-version.sh "go"
(set -x; env GO111MODULE=on go mod tidy && env GO111MODULE=on go mod vendor)
fi
51 changes: 51 additions & 0 deletions release-tools/verify-go-version.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

GO="$1"

if [ ! "$GO" ]; then
echo >&2 "usage: $0 <path to go binary>"
exit 1
fi

die () {
echo "ERROR: $*"
exit 1
}

version=$("$GO" version) || die "determining version of $GO failed"
# shellcheck disable=SC2001
majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
# shellcheck disable=SC2001
expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/')

if [ "$majorminor" != "$expected" ]; then
cat >&2 <<EOF
======================================================
WARNING
This projects is tested with Go v$expected.
Your current Go version is v$majorminor.
This may or may not be close enough.
In particular test-gofmt and test-vendor
are known to be sensitive to the version of
Go.
======================================================
EOF
fi
Loading

0 comments on commit ea69850

Please sign in to comment.