Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

master: update release-tools #54

Merged
merged 9 commits into from
Nov 5, 2019
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 18 additions & 43 deletions release-tools/build.make
Original file line number Diff line number Diff line change
@@ -22,6 +22,9 @@
# including build.make.
REGISTRY_NAME=quay.io/k8scsi

# Can be set to -mod=vendor to ensure that the "vendor" directory is used.
GOFLAGS_VENDOR=

# Revision that gets built into each binary via the main.version
# string. Uses the `git describe` output based on the most recent
# version tag with a short revision suffix or, if nothing has been
@@ -62,11 +65,11 @@ ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))
# Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
# to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.

build-%:
build-%: check-go-version-go
mkdir -p bin
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
if [ "$$ARCH" = "amd64" ]; then \
CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \
CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \
fi

container-%: build-%
@@ -97,19 +100,19 @@ push: $(CMDS:%=push-%)
clean:
-rm -rf bin

test:
test: check-go-version-go

.PHONY: test-go
test: test-go
test-go:
@ echo; echo "### $@:"
go test `go list ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)
go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)

.PHONY: test-vet
test: test-vet
test-vet:
@ echo; echo "### $@:"
go vet `go list ./... | grep -v vendor $(TEST_VET_FILTER_CMD)`
go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)`

.PHONY: test-fmt
test: test-fmt
@@ -154,43 +157,7 @@ test-fmt:
test: test-vendor
test-vendor:
@ echo; echo "### $@:"
@ if [ -f Gopkg.toml ]; then \
echo "Repo uses 'dep' for vendoring."; \
case "$$(dep version 2>/dev/null | grep 'version *:')" in \
*v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \
*) echo "skipping check, dep >= 0.5 required";; \
esac; \
elif [ -f go.mod ]; then \
echo "Repo uses 'go mod'."; \
if [ "$${JOB_NAME}" ] && \
( [ "$${JOB_TYPE}" != "presubmit" ] || \
[ $$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools; \
git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') | \
wc -l) -eq 0 ] ); then \
echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies."; \
elif ! GO111MODULE=on go mod tidy; then \
echo "ERROR: vendor check failed."; \
false; \
elif [ $$(git status --porcelain -- go.mod go.sum | wc -l) -gt 0 ]; then \
echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; \
git diff -- go.mod go.sum; \
false; \
elif [ -d vendor ]; then \
if ! GO111MODULE=on go mod vendor; then \
echo "ERROR: vendor check failed."; \
false; \
elif [ $$(git status --porcelain -- vendor | wc -l) -gt 0 ]; then \
echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"; \
git status -- vendor; \
git diff -- vendor; \
false; \
else \
echo "Go dependencies and vendor directory up-to-date."; \
fi; \
else \
echo "Go dependencies up-to-date."; \
fi; \
fi
@ ./release-tools/verify-vendor.sh

.PHONY: test-subtree
test: test-subtree
@@ -216,3 +183,11 @@ test-shellcheck:
./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \
done; \
exit $$ret

# Targets in the makefile can depend on check-go-version-<path to go binary>
# to trigger a warning if the x.y version of that binary does not match
# what the project uses. Make ensures that this is only checked once per
# invocation.
.PHONY: check-go-version-%
check-go-version-%:
./release-tools/verify-go-version.sh "$*"
50 changes: 19 additions & 31 deletions release-tools/prow.sh
Original file line number Diff line number Diff line change
@@ -85,6 +85,12 @@ get_versioned_variable () {
echo "$value"
}

# If we have a vendor directory, then use it. We must be careful to only
# use this for "make" invocations inside the project's repo itself because
# setting it globally can break other go usages (like "go get <some command>"
# which is disabled with GOFLAGS=-mod=vendor).
configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory"

# Go versions can be specified seperately for different tasks
# If the pre-installed Go is missing or a different
# version, the required version here will get installed
@@ -101,7 +107,8 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version
# kind version to use. If the pre-installed version is different,
# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/
# (if available), otherwise it is built from source.
configvar CSI_PROW_KIND_VERSION "v0.5.0" "kind"
# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39
configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind"

# ginkgo test runner version to use. If the pre-installed version is
# different, the desired version is built from source.
@@ -136,7 +143,6 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes"
#
# If the version is prefixed with "release-", then nothing
# is overridden.
override_k8s_version "1.13.10"
override_k8s_version "1.14.6"
override_k8s_version "1.15.3"

@@ -183,7 +189,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
#
# When no deploy script is found (nothing in `deploy` directory,
# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed.
configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0-rc8" "hostpath driver"
configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0" "hostpath driver"
configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo"
configvar CSI_PROW_DEPLOYMENT "" "deployment"
configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name"
@@ -200,9 +206,10 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image"
# all generated files are present.
#
# CSI_PROW_E2E_REPO=none disables E2E testing.
configvar CSI_PROW_E2E_VERSION_1_13 v1.14.0 "E2E version for Kubernetes 1.13.x" # we can't use the one from 1.13.x because it didn't have --storage.testdriver
# TOOO: remove versioned variables and make e2e version match k8s version
configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x"
configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x"
configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x"
# TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases
configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version
configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions
@@ -292,11 +299,6 @@ regex_join () {
# alpha in previous Kubernetes releases. This was considered too
# error prone. Therefore we use E2E tests that match the Kubernetes
# version that is getting tested.
#
# However, for 1.13.x testing we have to use the E2E tests from 1.14
# because 1.13 didn't have --storage.testdriver yet, so for that (and only
# that version) we have to define alpha tests differently.
configvar CSI_PROW_E2E_ALPHA_1_13 '\[Feature: \[Testpattern:.Dynamic.PV..block.volmode.\] should.create.and.delete.block.persistent.volumes' "alpha tests for Kubernetes 1.13" # Raw block was an alpha feature in 1.13.
configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough
configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests"

@@ -312,12 +314,12 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi
# kubernetes-csi components must be updated, either by disabling
# the failing test for "latest" or by updating the test and not running
# it anymore for older releases.
configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13"
configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14"
configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15"
configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16"
# TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and
# add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST.
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes"
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes"
configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates"

# Some tests are known to be unusable in a KinD cluster. For example,
@@ -723,22 +725,6 @@ install_sanity () (
run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed"
)

# Whether the hostpath driver supports raw block devices depends on which version
# we are testing. It would be much nicer if we could determine that by querying the
# installed driver's capabilities instead of having to do a version check.
hostpath_supports_block () {
local result
result="$(docker exec csi-prow-control-plane docker image ls --format='{{.Repository}} {{.Tag}} {{.ID}}' | grep hostpath | while read -r repo tag id; do
if [ "$tag" == "v1.0.1" ]; then
# Old version because the revision label is missing: didn't have support yet.
echo "false"
return
fi
done)"
# If not set, then it must be a newer driver with support.
echo "${result:-true}"
}

# The default implementation of this function generates a external
# driver test configuration for the hostpath driver.
#
@@ -755,12 +741,14 @@ SnapshotClass:
DriverInfo:
Name: ${CSI_PROW_HOSTPATH_DRIVER_NAME}
Capabilities:
block: $(hostpath_supports_block)
block: true
persistence: true
dataSource: true
multipods: true
nodeExpansion: true
controllerExpansion: true
snapshotDataSource: true
singleNodeVolume: true
EOF
}

@@ -946,7 +934,7 @@ main () {
images=
if ${CSI_PROW_BUILD_JOB}; then
# A successful build is required for testing.
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all || die "'make all' failed"
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make all' failed"
# We don't want test failures to prevent E2E testing below, because the failure
# might have been minor or unavoidable, for example when experimenting with
# changes in "release-tools" in a PR (that fails the "is release-tools unmodified"
@@ -956,13 +944,13 @@ main () {
warn "installing 'dep' failed, cannot test vendoring"
ret=1
fi
if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test 2>&1 | make_test_to_junit; then
if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" 2>&1 | make_test_to_junit; then
warn "'make test' failed, proceeding anyway"
ret=1
fi
fi
# Required for E2E testing.
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container || die "'make container' failed"
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make container' failed"
fi

if tests_need_kind; then
6 changes: 3 additions & 3 deletions release-tools/travis.yml
Original file line number Diff line number Diff line change
@@ -6,16 +6,16 @@ git:
depth: false
matrix:
include:
- go: 1.12.4
- go: 1.13.3
before_script:
- mkdir -p bin
- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep
- chmod u+x bin/dep
- export PATH=$PWD/bin:$PATH
script:
- make -k all test
- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' )
after_success:
- if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io;
make push;
make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' );
fi
23 changes: 23 additions & 0 deletions release-tools/update-vendor.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/usr/bin/env bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if [ -f Gopkg.toml ]; then
echo "Repo uses 'dep' for vendoring."
(set -x; dep ensure)
elif [ -f go.mod ]; then
release-tools/verify-go-version.sh "go"
(set -x; env GO111MODULE=on go mod tidy && env GO111MODULE=on go mod vendor)
fi
51 changes: 51 additions & 0 deletions release-tools/verify-go-version.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

GO="$1"

if [ ! "$GO" ]; then
echo >&2 "usage: $0 <path to go binary>"
exit 1
fi

die () {
echo "ERROR: $*"
exit 1
}

version=$("$GO" version) || die "determining version of $GO failed"
# shellcheck disable=SC2001
majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
# shellcheck disable=SC2001
expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/')

if [ "$majorminor" != "$expected" ]; then
cat >&2 <<EOF

======================================================
WARNING

This projects is tested with Go v$expected.
Your current Go version is v$majorminor.
This may or may not be close enough.

In particular test-gofmt and test-vendor
are known to be sensitive to the version of
Go.
======================================================

EOF
fi
60 changes: 60 additions & 0 deletions release-tools/verify-vendor.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/usr/bin/env bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if [ -f Gopkg.toml ]; then
echo "Repo uses 'dep' for vendoring."
case "$(dep version 2>/dev/null | grep 'version *:')" in
*v0.[56789]*)
if dep check; then
echo "vendor up-to-date"
else
exit 1
fi
;;
*) echo "skipping check, dep >= 0.5 required";;
esac
elif [ -f go.mod ]; then
echo "Repo uses 'go mod'."
# shellcheck disable=SC2235
if [ "${JOB_NAME}" ] &&
( [ "${JOB_TYPE}" != "presubmit" ] ||
[ "$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools;
git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') |
wc -l)" -eq 0 ] ); then
echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies."
elif ! (set -x; env GO111MODULE=on go mod tidy); then
echo "ERROR: vendor check failed."
exit 1
elif [ "$(git status --porcelain -- go.mod go.sum | wc -l)" -gt 0 ]; then
echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':";
git diff -- go.mod go.sum
exit 1
elif [ -d vendor ]; then
if ! (set -x; env GO111MODULE=on go mod vendor); then
echo "ERROR: vendor check failed."
exit 1
elif [ "$(git status --porcelain -- vendor | wc -l)" -gt 0 ]; then
echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"
git status -- vendor
git diff -- vendor
exit 1
else
echo "Go dependencies and vendor directory up-to-date."
fi
else
echo "Go dependencies up-to-date."
fi
fi
61 changes: 0 additions & 61 deletions vendor/golang.org/x/sys/unix/mkasm_darwin.go

This file was deleted.

106 changes: 0 additions & 106 deletions vendor/golang.org/x/sys/unix/mkpost.go

This file was deleted.

407 changes: 0 additions & 407 deletions vendor/golang.org/x/sys/unix/mksyscall.go

This file was deleted.

404 changes: 0 additions & 404 deletions vendor/golang.org/x/sys/unix/mksyscall_aix_ppc.go

This file was deleted.

602 changes: 0 additions & 602 deletions vendor/golang.org/x/sys/unix/mksyscall_aix_ppc64.go

This file was deleted.

335 changes: 0 additions & 335 deletions vendor/golang.org/x/sys/unix/mksyscall_solaris.go

This file was deleted.

190 changes: 0 additions & 190 deletions vendor/golang.org/x/sys/unix/mksysnum.go

This file was deleted.

236 changes: 0 additions & 236 deletions vendor/golang.org/x/sys/unix/types_aix.go

This file was deleted.

283 changes: 0 additions & 283 deletions vendor/golang.org/x/sys/unix/types_darwin.go

This file was deleted.

263 changes: 0 additions & 263 deletions vendor/golang.org/x/sys/unix/types_dragonfly.go

This file was deleted.

356 changes: 0 additions & 356 deletions vendor/golang.org/x/sys/unix/types_freebsd.go

This file was deleted.

289 changes: 0 additions & 289 deletions vendor/golang.org/x/sys/unix/types_netbsd.go

This file was deleted.

282 changes: 0 additions & 282 deletions vendor/golang.org/x/sys/unix/types_openbsd.go

This file was deleted.

266 changes: 0 additions & 266 deletions vendor/golang.org/x/sys/unix/types_solaris.go

This file was deleted.

133 changes: 0 additions & 133 deletions vendor/golang.org/x/text/unicode/bidi/gen.go

This file was deleted.

57 changes: 0 additions & 57 deletions vendor/golang.org/x/text/unicode/bidi/gen_ranges.go

This file was deleted.

64 changes: 0 additions & 64 deletions vendor/golang.org/x/text/unicode/bidi/gen_trieval.go

This file was deleted.

Loading