From 6e0516f042d2db7a2fba1d075f633731e4d77a9b Mon Sep 17 00:00:00 2001 From: Connor Catlett <conncatl@amazon.com> Date: Thu, 7 Dec 2023 00:31:26 +0000 Subject: [PATCH] Refactor the Makefile and hack/ scripts See the description of PR 1856 for more information about this change Signed-off-by: Connor Catlett <conncatl@amazon.com> --- .dockerignore | 8 + .github/workflows/publish-ecr.yaml | 2 +- .gitignore | 4 + Dockerfile | 2 +- Makefile | 372 ++++++++---------- .../base/clusterrolebinding-attacher.yaml | 1 - .../base/clusterrolebinding-csi-node.yaml | 1 - .../base/clusterrolebinding-provisioner.yaml | 1 - .../base/clusterrolebinding-resizer.yaml | 1 - .../base/clusterrolebinding-snapshotter.yaml | 1 - docs/makefile.md | 202 ++++++++++ hack/.gitignore | 1 - hack/e2e/README.md | 57 --- hack/e2e/chart-testing.sh | 14 - hack/e2e/config.sh | 62 +++ hack/e2e/create-cluster.sh | 74 ++++ hack/e2e/delete-cluster.sh | 44 +++ hack/e2e/ecr.sh | 58 +-- hack/e2e/{ => eksctl}/eksctl.sh | 60 +-- .../eksctl/patch.yaml} | 0 .../eksctl/values.yaml} | 0 .../vpc-resource-controller-configmap.yaml | 0 hack/e2e/helm.sh | 15 - hack/e2e/{ => kops}/kops.sh | 75 ++-- .../kops/patch-cluster.yaml} | 0 .../kops/patch-node.yaml} | 0 hack/{ => e2e/kops}/values.yaml | 0 hack/e2e/metrics/metrics.sh | 14 +- hack/e2e/run.sh | 308 +++++---------- hack/{provenance => provenance.sh} | 2 +- hack/prow-e2e.sh | 74 ++++ hack/prow.sh | 4 +- hack/release | 134 ------- hack/release-scripts/generate-release-pr | 32 +- hack/release-scripts/generate-sidecar-tags | 21 +- .../release-scripts/get-latest-sidecar-images | 31 +- hack/test-integration.sh | 26 -- hack/tools/install.sh | 183 +++++++++ hack/{verify-all => tools/python-runner.sh} | 13 +- hack/update-gofmt | 19 - hack/update-gomock | 33 -- hack/update-gomod | 38 -- hack/update-kustomize.sh | 32 ++ hack/update-mockgen.sh | 36 ++ hack/verify-gofmt | 28 -- hack/verify-govet | 23 -- hack/{verify-kustomize => verify-update.sh} | 31 +- hack/verify-vendor.sh | 40 -- 48 files changed, 1171 insertions(+), 1006 deletions(-) create mode 100644 .dockerignore create mode 100644 docs/makefile.md delete mode 100644 hack/.gitignore delete mode 100644 hack/e2e/README.md delete mode 100644 hack/e2e/chart-testing.sh create mode 100644 hack/e2e/config.sh create mode 100755 hack/e2e/create-cluster.sh create mode 100755 hack/e2e/delete-cluster.sh rename hack/e2e/{ => eksctl}/eksctl.sh (59%) rename hack/{eksctl-patch.yaml => e2e/eksctl/patch.yaml} (100%) rename hack/{values_eksctl.yaml => e2e/eksctl/values.yaml} (100%) rename hack/{ => e2e/eksctl}/vpc-resource-controller-configmap.yaml (100%) delete mode 100644 hack/e2e/helm.sh rename hack/e2e/{ => kops}/kops.sh (64%) rename hack/{kops-patch.yaml => e2e/kops/patch-cluster.yaml} (100%) rename hack/{kops-patch-node.yaml => e2e/kops/patch-node.yaml} (100%) rename hack/{ => e2e/kops}/values.yaml (100%) rename hack/{provenance => provenance.sh} (96%) create mode 100755 hack/prow-e2e.sh delete mode 100755 hack/release delete mode 100755 hack/test-integration.sh create mode 100755 hack/tools/install.sh rename hack/{verify-all => tools/python-runner.sh} (69%) delete mode 100755 hack/update-gofmt delete mode 100755 hack/update-gomock delete mode 100755 hack/update-gomod create mode 100755 hack/update-kustomize.sh create mode 100755 hack/update-mockgen.sh delete mode 100755 hack/verify-gofmt delete mode 100755 hack/verify-govet rename hack/{verify-kustomize => verify-update.sh} (50%) delete mode 100755 hack/verify-vendor.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..2f526620ac --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.github/ +.idea/ +bin/ +hack/ +charts/ +deploy/ +docs/ +examples/ diff --git a/.github/workflows/publish-ecr.yaml b/.github/workflows/publish-ecr.yaml index 1c32b44ee4..e6a3267431 100644 --- a/.github/workflows/publish-ecr.yaml +++ b/.github/workflows/publish-ecr.yaml @@ -39,7 +39,7 @@ jobs: echo "VERSION=${GITHUB_REF_NAME}" >> $GITHUB_ENV - name: Build, tag, and push manifest to Amazon ECR - run: make -j `nproc` all-push + run: make -j `nproc` all-push-with-a1compat ecr-public: name: Push to ECR Public diff --git a/.gitignore b/.gitignore index 000ff70b0d..4be608e597 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ vendor/ # Files used by Makefile when upgrading sidecars hack/release-scripts/image-digests.yaml + +# E2E artifacts +_rundir/ +_artifacts/ diff --git a/Dockerfile b/Dockerfile index 5ea6b1b045..a60b4be999 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ COPY . . ARG TARGETOS ARG TARGETARCH ARG VERSION -RUN OS=$TARGETOS ARCH=$TARGETARCH make $TARGETOS/$TARGETARCH +RUN OS=$TARGETOS ARCH=$TARGETARCH make FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-csi-ebs:latest-al23 AS linux-al2023 COPY --from=builder /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver/bin/aws-ebs-csi-driver /bin/aws-ebs-csi-driver diff --git a/Makefile b/Makefile index 86b25a87f7..da02b4232d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright 2019 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,29 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +### +### This Makefile is documented in docs/makefile.md +### + +## Variables/Functions + VERSION?=v1.26.1 PKG=github.com/kubernetes-sigs/aws-ebs-csi-driver GIT_COMMIT?=$(shell git rev-parse HEAD) BUILD_DATE?=$(shell date -u -Iseconds) - LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/cloud.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE} -s -w" -GO111MODULE=on -GOPATH=$(shell go env GOPATH) -GOOS=$(shell go env GOOS) -GOBIN=$(shell pwd)/bin +OS?=$(shell go env GOHOSTOS) +ARCH?=$(shell go env GOHOSTARCH) +ifeq ($(OS),windows) + BINARY=aws-ebs-csi-driver.exe + OSVERSION?=ltsc2022 +else + BINARY=aws-ebs-csi-driver + OSVERSION?=al2023 +endif + +GO_SOURCES=go.mod go.sum $(shell find pkg cmd -type f -name "*.go") REGISTRY?=gcr.io/k8s-staging-provider-aws IMAGE?=$(REGISTRY)/aws-ebs-csi-driver TAG?=$(GIT_COMMIT) -OUTPUT_TYPE?=docker - -OS?=linux -ARCH?=amd64 -OSVERSION?=al2023 - ALL_OS?=linux windows ALL_ARCH_linux?=amd64 arm64 ALL_OSVERSION_linux?=al2023 @@ -43,244 +49,108 @@ ALL_OS_ARCH_OSVERSION_linux=$(foreach arch, $(ALL_ARCH_linux), $(foreach osversi ALL_ARCH_windows?=amd64 ALL_OSVERSION_windows?=ltsc2019 ltsc2022 ALL_OS_ARCH_OSVERSION_windows=$(foreach arch, $(ALL_ARCH_windows), $(foreach osversion, ${ALL_OSVERSION_windows}, windows-$(arch)-${osversion})) - ALL_OS_ARCH_OSVERSION=$(foreach os, $(ALL_OS), ${ALL_OS_ARCH_OSVERSION_${os}}) +CLUSTER_NAME?=ebs-csi-e2e.k8s.local +CLUSTER_TYPE?=kops +WINDOWS?=false + # split words on hyphen, access by 1-index word-hyphen = $(word $2,$(subst -, ,$1)) .EXPORT_ALL_VARIABLES: -.PHONY: linux/$(ARCH) bin/aws-ebs-csi-driver -linux/$(ARCH): bin/aws-ebs-csi-driver -bin/aws-ebs-csi-driver: | bin - CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -mod=mod -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver ./cmd/ - -.PHONY: windows/$(ARCH) bin/aws-ebs-csi-driver.exe -windows/$(ARCH): bin/aws-ebs-csi-driver.exe -bin/aws-ebs-csi-driver.exe: | bin - CGO_ENABLED=0 GOOS=windows GOARCH=$(ARCH) go build -mod=mod -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver.exe ./cmd/ - -# Builds all linux images (not windows because it can't be exported with OUTPUT_TYPE=docker) -.PHONY: all -all: all-image-docker - -# Builds all linux and windows images and pushes them -.PHONY: all-push -all-push: all-image-registry push-manifest - -.PHONY: push-manifest -push-manifest: create-manifest - docker manifest push --purge $(IMAGE):$(TAG) - -.PHONY: create-manifest -create-manifest: all-image-registry -# sed expression: -# LHS: match 0 or more not space characters -# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS - docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g") - -# Only linux for OUTPUT_TYPE=docker because windows image cannot be exported -# "Currently, multi-platform images cannot be exported with the docker export type. The most common usecase for multi-platform images is to directly push to a registry (see registry)." -# https://docs.docker.com/engine/reference/commandline/buildx_build/#output -.PHONY: all-image-docker -all-image-docker: $(addprefix sub-image-docker-,$(ALL_OS_ARCH_OSVERSION_linux)) -.PHONY: all-image-registry -all-image-registry: sub-image-registry-linux-arm64-al2 $(addprefix sub-image-registry-,$(ALL_OS_ARCH_OSVERSION)) - -sub-image-%: - $(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) image +## Default target +# When no target is supplied, make runs the first target that does not begin with a . +# Alias that to building the binary +.PHONY: default +default: bin/$(BINARY) -.PHONY: image -image: .image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) -.image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION): - docker buildx build \ - --platform=$(OS)/$(ARCH) \ - --progress=plain \ - --target=$(OS)-$(OSVERSION) \ - --output=type=$(OUTPUT_TYPE) \ - -t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \ - --build-arg=GOPROXY=$(GOPROXY) \ - --build-arg=VERSION=$(VERSION) \ - `./hack/provenance` \ - . - touch $@ +## Top level targets .PHONY: clean clean: - rm -rf .*image-* bin/ - -bin /tmp/helm /tmp/kubeval: - @mkdir -p $@ + rm -rf bin/ -bin/helm: | /tmp/helm bin - @curl -o /tmp/helm/helm.tar.gz -sSL https://get.helm.sh/helm-v3.11.2-${GOOS}-amd64.tar.gz - @tar -zxf /tmp/helm/helm.tar.gz -C bin --strip-components=1 - @rm -rf /tmp/helm/* +.PHONY: test +test: + go test -v -race ./cmd/... ./pkg/... -bin/kubeval: | /tmp/kubeval bin - @curl -o /tmp/kubeval/kubeval.tar.gz -sSL https://github.com/instrumenta/kubeval/releases/download/0.16.1/kubeval-linux-amd64.tar.gz - @tar -zxf /tmp/kubeval/kubeval.tar.gz -C bin kubeval - @rm -rf /tmp/kubeval/* +.PHONY: test-sanity +test-sanity: + go test -v -race ./tests/sanity/... -bin/mockgen: | bin - go install github.com/golang/mock/mockgen@v1.6.0 +.PHONY: tools +tools: bin/aws bin/ct bin/eksctl bin/ginkgo bin/golangci-lint bin/helm bin/kops bin/kubetest2 bin/mockgen bin/shfmt -bin/golangci-lint: | bin - echo "Installing golangci-lint..." - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.54.0 +.PHONY: update +update: update/gofmt update/kustomize update/mockgen update/gomod update/shfmt + @echo "All updates succeeded!" -.PHONY: kubeval -kubeval: bin/kubeval - bin/kubeval -d deploy/kubernetes/base,deploy/kubernetes/cluster,deploy/kubernetes/overlays -i kustomization.yaml,crd_.+\.yaml,controller_add +.PHONY: verify +verify: verify/govet verify/golangci-lint verify/update + @echo "All verifications passed!" -.PHONY: mockgen -mockgen: bin/mockgen - ./hack/update-gomock +.PHONY: all-push +all-push: all-image-registry push-manifest -.PHONY: verify -verify: bin/golangci-lint - echo "verifying and linting files ..." - ./hack/verify-all - echo "Congratulations! All Go source files have been linted." +.PHONY: cluster/create +cluster/create: bin/kops bin/eksctl + ./hack/e2e/create-cluster.sh -.PHONY: test -test: - go test -v -race ./cmd/... ./pkg/... +.PHONY: cluster/delete +cluster/delete: bin/kops bin/eksctl + ./hack/e2e/delete-cluster.sh -.PHONY: test-sanity -test-sanity: - go test -v -race ./tests/sanity/... +## E2E targets +# Targets to run e2e tests -.PHONY: test-e2e-single-az -test-e2e-single-az: - AWS_REGION=us-west-2 \ +.PHONY: e2e/single-az +e2e/single-az: bin/helm bin/ginkgo AWS_AVAILABILITY_ZONES=us-west-2a \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME,controller.volumeModificationFeature.enabled=true' \ - EBS_INSTALL_SNAPSHOT="true" \ TEST_PATH=./tests/e2e/... \ GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \ - GINKGO_SKIP="\"sc1\"|\"st1\"" \ + GINKGO_PARALLEL=5 \ + HELM_EXTRA_FLAGS="--set=controller.volumeModificationFeature.enabled=true" \ ./hack/e2e/run.sh -.PHONY: test-e2e-multi-az -test-e2e-multi-az: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EBS_INSTALL_SNAPSHOT="true" \ +.PHONY: e2e/multi-az +e2e/multi-az: bin/helm bin/ginkgo TEST_PATH=./tests/e2e/... \ GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" \ + GINKGO_PARALLEL=5 \ ./hack/e2e/run.sh -.PHONY: test-e2e-external -test-e2e-external: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EBS_INSTALL_SNAPSHOT="true" \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ - GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \ +.PHONY: e2e/external +e2e/external: bin/helm bin/kubetest2 COLLECT_METRICS="true" \ ./hack/e2e/run.sh -.PHONY: test-e2e-external-arm64 -test-e2e-external-arm64: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EBS_INSTALL_SNAPSHOT="true" \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ - GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \ - INSTANCE_TYPE="m7g.medium" \ - AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64" \ +.PHONY: e2e/external-arm64 +e2e/external-arm64: bin/helm bin/kubetest2 IMAGE_ARCH="arm64" \ ./hack/e2e/run.sh -.PHONY: test-e2e-external-eks -test-e2e-external-eks: - CLUSTER_TYPE=eksctl \ - HELM_VALUES_FILE="./hack/values_eksctl.yaml" \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EBS_INSTALL_SNAPSHOT="true" \ - EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \ - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ - GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \ - ./hack/e2e/run.sh - -.PHONY: test-e2e-external-eks-windows -test-e2e-external-eks-windows: - CLUSTER_TYPE=eksctl \ +.PHONY: e2e/external-windows +e2e/external-windows: bin/helm bin/kubetest2 WINDOWS=true \ - HELM_VALUES_FILE="./hack/values_eksctl.yaml" \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ - EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \ - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ GINKGO_SKIP="\[Disruptive\]|\[Serial\]|\[LinuxOnly\]|\[Feature:VolumeSnapshotDataSource\]|\(xfs\)|\(ext4\)|\(block volmode\)" \ GINKGO_PARALLEL=15 \ - NODE_OS_DISTRO="windows" \ + EBS_INSTALL_SNAPSHOT="false" \ ./hack/e2e/run.sh -.PHONY: test-e2e-external-kustomize -test-e2e-external-kustomize: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - EBS_INSTALL_SNAPSHOT="true" \ - TEST_PATH=./tests/e2e-kubernetes/... \ - GINKGO_FOCUS="External.Storage" \ - GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \ +.PHONY: e2e/external-kustomize +e2e/external-kustomize: bin/kubetest2 DEPLOY_METHOD="kustomize" \ ./hack/e2e/run.sh -.PHONY: test-helm-chart -test-helm-chart: - AWS_REGION=us-west-2 \ - AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \ - EBS_INSTALL_SNAPSHOT="true" \ +.PHONY: e2e/helm-ct +e2e/helm-ct: bin/helm bin/ct HELM_CT_TEST="true" \ ./hack/e2e/run.sh -.PHONY: verify-vendor -test: verify-vendor -verify: verify-vendor -verify-vendor: - @ echo; echo "### $@:" - @ ./hack/verify-vendor.sh - -.PHONY: verify-kustomize -verify: verify-kustomize -verify-kustomize: - @ echo; echo "### $@:" - @ ./hack/verify-kustomize - -.PHONY: generate-kustomize -generate-kustomize: bin/helm - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-attacher.yaml > ../../deploy/kubernetes/base/clusterrole-attacher.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-csi-node.yaml > ../../deploy/kubernetes/base/clusterrole-csi-node.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-provisioner.yaml > ../../deploy/kubernetes/base/clusterrole-provisioner.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-resizer.yaml > ../../deploy/kubernetes/base/clusterrole-resizer.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-snapshotter.yaml > ../../deploy/kubernetes/base/clusterrole-snapshotter.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-attacher.yaml > ../../deploy/kubernetes/base/clusterrolebinding-attacher.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-csi-node.yaml > ../../deploy/kubernetes/base/clusterrolebinding-csi-node.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-provisioner.yaml > ../../deploy/kubernetes/base/clusterrolebinding-provisioner.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-resizer.yaml > ../../deploy/kubernetes/base/clusterrolebinding-resizer.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-snapshotter.yaml > ../../deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/controller.yaml --api-versions 'snapshot.storage.k8s.io/v1' --set 'controller.userAgentExtra=kustomize' | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/controller.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/csidriver.yaml > ../../deploy/kubernetes/base/csidriver.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/node.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/node.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/poddisruptionbudget-controller.yaml --api-versions 'policy/v1/PodDisruptionBudget' | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/poddisruptionbudget-controller.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/serviceaccount-csi-controller.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/serviceaccount-csi-controller.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/serviceaccount-csi-node.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/serviceaccount-csi-node.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/role-leases.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/role-leases.yaml - cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/rolebinding-leases.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/rolebinding-leases.yaml +## Release scripts +# Targets run as part of performing a release .PHONY: update-truth-sidecars update-truth-sidecars: hack/release-scripts/get-latest-sidecar-images @@ -291,4 +161,98 @@ generate-sidecar-tags: update-truth-sidecars charts/aws-ebs-csi-driver/values.ya ./hack/release-scripts/generate-sidecar-tags .PHONY: update-sidecar-dependencies -update-sidecar-dependencies: update-truth-sidecars generate-sidecar-tags generate-kustomize +update-sidecar-dependencies: update-truth-sidecars generate-sidecar-tags update/kustomize + +## CI aliases +# Targets intended to be executed mostly or only by CI jobs + +.PHONY: all-push-with-a1compat +all-push-with-a1compat: sub-image-linux-arm64-al2 all-image-registry push-manifest + +test-e2e-%: + ./hack/prow-e2e.sh test-e2e-$* + +test-helm-chart: + ./hack/prow-e2e.sh test-helm-chart + +## Builds + +bin: + @mkdir -p $@ + +bin/$(BINARY): $(GO_SOURCES) | bin + CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -mod=readonly -ldflags ${LDFLAGS} -o $@ ./cmd/ + +.PHONY: all-image-registry +all-image-registry: $(addprefix sub-image-,$(ALL_OS_ARCH_OSVERSION)) + +sub-image-%: + $(MAKE) OS=$(call word-hyphen,$*,1) ARCH=$(call word-hyphen,$*,2) OSVERSION=$(call word-hyphen,$*,3) image + +.PHONY: image +image: + docker buildx build \ + --platform=$(OS)/$(ARCH) \ + --progress=plain \ + --target=$(OS)-$(OSVERSION) \ + --output=type=registry \ + -t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \ + --build-arg=GOPROXY=$(GOPROXY) \ + --build-arg=VERSION=$(VERSION) \ + `./hack/provenance.sh` \ + . + +.PHONY: create-manifest +create-manifest: all-image-registry +# sed expression: +# LHS: match 0 or more not space characters +# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS + docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g") + +.PHONY: push-manifest +push-manifest: create-manifest + docker manifest push --purge $(IMAGE):$(TAG) + +## Tools +# Tools necessary to perform other targets + +bin/%: hack/tools/install.sh hack/tools/python-runner.sh + @TOOLS_PATH="$(shell pwd)/bin" ./hack/tools/install.sh $* + +## Updaters +# Automatic generators/formatters for code + +.PHONY: update/gofmt +update/gofmt: + gofmt -s -w . + +.PHONY: update/kustomize +update/kustomize: bin/helm + ./hack/update-kustomize.sh + +.PHONY: update/mockgen +update/mockgen: bin/mockgen + ./hack/update-mockgen.sh + +.PHONY: update/gomod +update/gomod: + go mod tidy + +.PHONY: update/shfmt +update/shfmt: bin/shfmt + ./bin/shfmt -w -i 2 -d . + +## Verifiers +# Linters and similar + +.PHONY: verify/golangci-lint +verify/golangci-lint: bin/golangci-lint + ./bin/golangci-lint run --timeout=10m --verbose + +.PHONY: verify/govet +verify/govet: + go vet $$(go list ./...) + +.PHONY: verify/update +verify/update: bin/helm bin/mockgen + ./hack/verify-update.sh diff --git a/deploy/kubernetes/base/clusterrolebinding-attacher.yaml b/deploy/kubernetes/base/clusterrolebinding-attacher.yaml index 5715d2651b..12405a09b1 100644 --- a/deploy/kubernetes/base/clusterrolebinding-attacher.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-attacher.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-controller-sa - namespace: default roleRef: kind: ClusterRole name: ebs-external-attacher-role diff --git a/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml b/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml index 095db52510..4cc33b7e82 100644 --- a/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-node-sa - namespace: default roleRef: kind: ClusterRole name: ebs-csi-node-role diff --git a/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml b/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml index 3544bc61e2..e95a94b962 100644 --- a/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-controller-sa - namespace: default roleRef: kind: ClusterRole name: ebs-external-provisioner-role diff --git a/deploy/kubernetes/base/clusterrolebinding-resizer.yaml b/deploy/kubernetes/base/clusterrolebinding-resizer.yaml index c80a9a26bf..543086e74c 100644 --- a/deploy/kubernetes/base/clusterrolebinding-resizer.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-resizer.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-controller-sa - namespace: default roleRef: kind: ClusterRole name: ebs-external-resizer-role diff --git a/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml b/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml index 7946414d59..81ed7c2b80 100644 --- a/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml +++ b/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml @@ -9,7 +9,6 @@ metadata: subjects: - kind: ServiceAccount name: ebs-csi-controller-sa - namespace: default roleRef: kind: ClusterRole name: ebs-external-snapshotter-role diff --git a/docs/makefile.md b/docs/makefile.md new file mode 100644 index 0000000000..5314f0c65c --- /dev/null +++ b/docs/makefile.md @@ -0,0 +1,202 @@ +# Use of the EBS CSI Driver `Makefile` + +The EBS CSI Driver comes with a Makefile that can be used to develop, build, test, and release the driver. This file documents Makefile targets, the parameters they support, and common usage scenarios. + +## Prerequisites + +The `Makefile` has the following dependencies: +- `go`: https://go.dev/doc/install +- `python` (may be named `python3`) and `pip`: https://www.python.org/downloads/ +- `jq`: https://github.com/jqlang/jq/releases +- `kubectl`: https://kubernetes.io/docs/tasks/tools/#kubectl +- `git`: https://git-scm.com/downloads +- `docker` and `docker buildx`: https://docs.docker.com/get-docker/ and https://github.com/docker/buildx#installing +- `make` +- Standard POSIX tools (`awk`, `grep`, `cat`, etc) + +All other tools are downloaded for you at runtime. + +## Quickstart Guide + +This guide demonstrates the basic workflow for developing for the EBS CSI Driver. More detailed documentation of the available `make` targets is available below. + +### 1. Local development for the EBS CSI Driver + +If your changes are Helm-only, skip to section 2 (Run E2E tests) below after making your changes. + +During development, use `make` at any time to build a driver binary, and thus discover compiler errors. When your change is ready to be tested, run `make test` to execute the unit test suite for the driver. If you are making a significant change to the driver (such as a bugfix or new feature), please add new unit tests or update existing tests where applicable. + +### 2. Run E2E tests + +To create a `kops` cluster to test the driver against: +```bash +make cluster/create +``` + +If your change affects the Windows implementation of the driver, instead create an `eksctl` cluster with Windows nodes: +```bash +export WINDOWS="true" +# Note: CLUSTER_TYPE must be set for all e2e tests and cluster deletion +# Re-export it if necessary (for example, if running tests in a separate terminal tab) +export CLUSTER_TYPE="eksctl" +make cluster/create +``` + +If you are making a change to the driver, the recommended test suite to run is the external tests: +```bash +# Normal external tests (excluding Windows tests) +make e2e/external +# Instead, if testing Windows +make e2e/external-windows +``` + +If you are making a change to the Helm chart, the recommended test suite to run is the Helm `ct` tests: +```bash +make e2e/helm-ct +``` + +To cleanup your cluster after finishing testing: +```bash +make cluster/delete +``` + +### 3. Before submitting a PR + +Run `make update` to automatically format go source files and re-generate automatically generated files. If `make update` produces any changes, commit them before submitting a PR. + +Run `make verify` to run linters and other similar code checking tools. Fix any issues `make verify` discovers before submitting a PR. + +## Building + +### `make` or `make bin/aws-ebs-csi-driver` or `make bin/aws-ebs-csi-driver.exe` + +Build a binary copy of the EBS CSI Driver for the local platform. This is the default behavior when calling `make` with no target. + +The target OS and/or architecture can be overridden via the `OS` and `ARCH` environment variables (for example, `OS=linux ARCH=arm64 make`) + +### `make image` + +Build and push a single image of the driver based on the local platform (the same overrides as `make` apply, as well as `OSVERSION` to override container OS version). In most cases, `make all-push` is more suitable. Environment variables are accepted to override the `REGISTRY`, `IMAGE` name, and image `TAG`. + +### `make all-push` + +Build and push a multi-arch image of the driver based on the OSes in `ALL_OS`, architectures in `ALL_ARCH_linux`/`ALL_ARCH_windows`, and OS versions in `ALL_OSVERSION_linux`/`ALL_OSVERSION_windows`. Also supports `REGISTRY`, `IMAGE`, and `TAG`. + +## Local Development + +### `make test` + +Run all unit tests with race condition checking enabled. + +### `make test-sanity` + +Run the official [CSI sanity tests](https://github.com/kubernetes-csi/csi-test). _Warning: Currently, 3 of the tests are known to fail incorrectly._ + +### `make verify` + +Performs local verification that other than unit tests (linters, manifest updates, etc) + +### `make update` + +Updates Kustomize manifests, formatting, and tidies `go.mod`. `make verify` will ensure that `make update` was run by checking if it creates a diff. + +## Cluster Management + +### `make cluster/create` + +Creates a cluster for running E2E tests against. There are many parameters that can be provided via environment variables, a full list is available in [`config.sh`](../hack/e2e/config.sh), but the primary parameters are: + +- `CLUSTER_TYPE`: The tool used to create the cluster, either `kops` or `eksctl` - defaults to `kops` +- `CLUSTER_NAME`: The name of the cluster to create - defaults to `ebs-csi-e2e.k8s.local` +- `INSTANCE_TYPE`: The instance type to use for cluster nodes - defaults to `c5.large` +- `AMI_PARAMETER`: The SSM parameter of where to get the AMI for the cluster nodes (`kops` clusters only) - defaults to `/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64` +- `WINDOWS`: Whether or not to create a Windows node group for the cluster (`eksctl` clusters only) - defaults to `false` +- `AWS_REGION`: Which region to create the cluster in - defaults to `us-west-2` +- `AWS_AVAILABILITY_ZONES`: Which AZs to create nodes for the cluster in - defaults to `us-west-2a,us-west-2b,us-west-2c` + +#### Example: Create a default (`kops`) cluster + +```bash +make cluster/create +``` + +#### Example: Create a cluster with only one Availability Zone +```bash +export AWS_AVAILABILITY_ZONES="us-west-2a" +make cluster/create +``` + +#### Example: Create an `eksctl` cluster + +```bash +export CLUSTER_TYPE="eksctl" +make cluster/create +``` + +#### Example: Create a cluster with Graviton nodes for `arm64` testing + +```bash +export INSTANCE_TYPE="m7g.medium" +export AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64" +make cluster/create +``` + +#### Example: Create a cluster with Windows nodes + +```bash +export WINDOWS="true" +export CLUSTER_TYPE="eksctl" +make cluster/create +``` + +### `make cluster/delete` + +Deletes a cluster created by `make cluster/create`. You must pass the same `CLUSTER_TYPE` and `CLUSTER_NAME` as used when creating the cluster. + +## E2E Tests + +Run E2E tests against a cluster created by `make cluster/create`. You must pass the same `CLUSTER_TYPE` and `CLUSTER_NAME` as used when creating the cluster. + +Alternatively, you may run on an externally created cluster by passing `CLUSTER_TYPE` (required to determine which `values.yaml` to deploy) and `KUBECONFIG`. For `kops` clusters, the node IAM role should include the appropriate IAM policies to use the driver (see [the installation docs](./install.md#set-up-driver-permissions)). For `eksctl` clusters, the `ebs-csi-controller-sa` service account should be pre-created and setup to supply an IRSA role with the appropriate policies. + +### `make e2e/external` + +Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md). This is the most comprehensive E2E test, recommended for local development. + +### `make e2e/single-az` + +Run the single-AZ EBS CSI E2E tests. Requires a cluster with only one Availability Zone. + +### `make e2e/multi-az` + +Run the multi-AZ EBS CSI E2E tests. Requires a cluster with at least two Availability Zones. + +### `make e2e/external-arm64` + +Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md) using an ARM64 image of the EBS CSI Driver. Requires a cluster with Graviton nodes. + +### `make e2e/external-windows` + +Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md) with Windows tests enabled. Requires a cluster with Windows nodes. + +### `make e2e/external-kustomize` + +Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md), but using `kustomize` to deploy the driver instead of `helm`. + +### `make e2e/helm-ct` + +Test the EBS CSI Driver Helm chart via the [Helm `chart-testing` tool](https://github.com/helm/chart-testing). + +## Release Scripts + +### `make update-sidecar-dependencies` + +Convenience target to perform all sidecar updates and regenerate the manifests. This is the primary target to use unless more granular control is needed. + +### `make update-truth-sidecars` + +Retrieves the latest sidecar container images and creates or updates `hack/release-scripts/image-digests.yaml`. + +### `make generate-sidecar-tags` + +Updates the Kustomize and Helm sidecar tags with the values from `hack/release-scripts/image-digests.yaml`. diff --git a/hack/.gitignore b/hack/.gitignore deleted file mode 100644 index 3cc5e8f053..0000000000 --- a/hack/.gitignore +++ /dev/null @@ -1 +0,0 @@ -ebs-e2e-test/ diff --git a/hack/e2e/README.md b/hack/e2e/README.md deleted file mode 100644 index 914f671b68..0000000000 --- a/hack/e2e/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Usage - -run.sh will build and push a driver image, create a kops cluster, helm install the driver pointing to the built image, run ginkgo tests, then clean everything up. - -See below for an example. - -KOPS_STATE_FILE is an S3 bucket you have write access to. - -TEST_ID is a token used for idempotency. - -For more details, see the script itself. - -For more examples, see the top-level Makefile. - -``` -TEST_PATH=./tests/e2e-migration/... \ -EBS_CHECK_MIGRATION=true \ -TEST_ID=18512 \ -CLEAN=false \ -KOPS_STATE_FILE=s3://mattwon \ -AWS_REGION=us-west-2 \ -AWS_AVAILABILITY_ZONES=us-west-2a \ -GINKGO_FOCUS=Dynamic.\*xfs.\*should.store.data \ -GINKGO_NODES=1 \ -./hack/e2e/run.sh -``` - -# git read-tree - -Reference: https://stackoverflow.com/questions/23937436/add-subdirectory-of-remote-repo-with-git-subtree - -How to consume this directory by read-treeing the ebs repo: - -``` -git remote add ebs git@github.com:kubernetes-sigs/aws-ebs-csi-driver.git --no-tags -git fetch ebs -git read-tree --prefix=hack/e2e/ -u ebs/master:hack/e2e -``` - -To commit changes and submit them as a PR back to the ebs repo: - -``` -git diff ebs/master:hack/e2e HEAD:hack/e2e > /tmp/hack_e2e.diff -pushd $GOPATH/src/github.com/kubernetes-sigs/aws-ebs-csi-driver -git apply --reject --directory hack/e2e /tmp/hack_e2e.diff -git commit -``` - -To consume newer changes from the ebs repo: - -``` -git fetch ebs -git diff HEAD:hack/e2e ebs/master:hack/e2e > /tmp/hack_e2e.diff -git apply --reject --directory hack/e2e /tmp/hack_e2e.diff -git add hack/e2e -git commit -m "Update hack/e2e" -``` diff --git a/hack/e2e/chart-testing.sh b/hack/e2e/chart-testing.sh deleted file mode 100644 index 8e7011c2e6..0000000000 --- a/hack/e2e/chart-testing.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -uo pipefail - -function ct_install() { - INSTALL_PATH=${1} - CHART_TESTING_VERSION=${2} - if [[ ! -e ${INSTALL_PATH}/chart-testing ]]; then - CHART_TESTING_DOWNLOAD_URL="https://github.com/helm/chart-testing/releases/download/v${CHART_TESTING_VERSION}/chart-testing_${CHART_TESTING_VERSION}_linux_amd64.tar.gz" - curl --silent --location "${CHART_TESTING_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}" - chmod +x "${INSTALL_PATH}"/ct - fi - apt-get update && apt-get install -y yamllint -} diff --git a/hack/e2e/config.sh b/hack/e2e/config.sh new file mode 100644 index 0000000000..f2d1a843f6 --- /dev/null +++ b/hack/e2e/config.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" +TEST_DIR="${BASE_DIR}/csi-test-artifacts" +mkdir -p "${TEST_DIR}" +CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml +KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"} + +export AWS_REGION=${AWS_REGION:-us-west-2} +ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c} +FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1) +NODE_COUNT=${NODE_COUNT:-3} +INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large} +WINDOWS=${WINDOWS:-"false"} + +# kops: must include patch version (e.g. 1.19.1) +# eksctl: mustn't include patch version (e.g. 1.19) +K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-1.29.0} +K8S_VERSION_EKSCTL=${K8S_VERSION_EKSCTL:-1.28} + +EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"true"} +EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.3.2"} + +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +KOPS_BUCKET=${KOPS_BUCKET:-${AWS_ACCOUNT_ID}-ebs-csi-e2e-kops} + +AMI_PARAMETER=${AMI_PARAMETER:-/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64} +AMI_ID=$(aws ssm get-parameters --names ${AMI_PARAMETER} --region ${AWS_REGION} --query 'Parameters[0].Value' --output text) + +CREATE_MISSING_ECR_REPO=${CREATE_MISSING_ECR_REPO:-"true"} +IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/aws-ebs-csi-driver} +IMAGE_TAG=${IMAGE_TAG:-$(md5sum <<<"${CLUSTER_NAME}.${CLUSTER_TYPE}" | awk '{ print $1 }')} +IMAGE_ARCH=${IMAGE_ARCH:-amd64} + +DEPLOY_METHOD=${DEPLOY_METHOD:-"helm"} +HELM_CT_TEST=${HELM_CT_TEST:-"false"} +HELM_EXTRA_FLAGS=${HELM_EXTRA_FLAGS:-} +COLLECT_METRICS=${COLLECT_METRICS:-"false"} + +TEST_PATH=${TEST_PATH:-"./tests/e2e-kubernetes/..."} +GINKGO_FOCUS=${GINKGO_FOCUS:-"External.Storage"} +GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]|\[Serial\]"} +GINKGO_PARALLEL=${GINKGO_PARALLEL:-25} + +# TODO: Left in for now, but look into if this is still necessary and remove if not +EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-"Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV"} diff --git a/hack/e2e/create-cluster.sh b/hack/e2e/create-cluster.sh new file mode 100755 index 0000000000..7171111419 --- /dev/null +++ b/hack/e2e/create-cluster.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script creates a cluster for use of running the e2e tests +# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller +# All other environment variables have default values (see config.sh) but +# many can be overridden on demand if needed + +set -euo pipefail + +BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" +BIN="${BASE_DIR}/../../bin" + +source "${BASE_DIR}/config.sh" +source "${BASE_DIR}/util.sh" +source "${BASE_DIR}/kops/kops.sh" +source "${BASE_DIR}/eksctl/eksctl.sh" + +if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + BUCKET_CHECK=$("${BIN}/aws" s3api head-bucket --region us-east-1 --bucket "${KOPS_BUCKET}" 2>&1 || true) + if grep -q "Forbidden" <<<"${BUCKET_CHECK}"; then + echo "Kops requires a S3 bucket in order to store the state" >&2 + echo "This script is attempting to use a bucket called \`${KOPS_BUCKET}\`" >&2 + echo "That bucket already exists and you do not have access to it" >&2 + echo "You can change the bucket by setting the environment variable \$KOPS_BUCKET" >&2 + exit 1 + fi + if grep -q "Not Found" <<<"${BUCKET_CHECK}"; then + "${BIN}/aws" s3api create-bucket --region us-east-1 --bucket "${KOPS_BUCKET}" --acl private >/dev/null + fi + + kops_create_cluster \ + "$CLUSTER_NAME" \ + "${BIN}/kops" \ + "$ZONES" \ + "$NODE_COUNT" \ + "$INSTANCE_TYPE" \ + "$AMI_ID" \ + "$K8S_VERSION_KOPS" \ + "$CLUSTER_FILE" \ + "$KUBECONFIG" \ + "${BASE_DIR}/kops/patch-cluster.yaml" \ + "${BASE_DIR}/kops/patch-node.yaml" \ + "s3://${KOPS_BUCKET}" +elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + eksctl_create_cluster \ + "$CLUSTER_NAME" \ + "${BIN}/eksctl" \ + "$ZONES" \ + "$INSTANCE_TYPE" \ + "$K8S_VERSION_EKSCTL" \ + "$CLUSTER_FILE" \ + "$KUBECONFIG" \ + "${BASE_DIR}/eksctl/patch.yaml" \ + "$EKSCTL_ADMIN_ROLE" \ + "$WINDOWS" \ + "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml" +else + echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2 + exit 1 +fi diff --git a/hack/e2e/delete-cluster.sh b/hack/e2e/delete-cluster.sh new file mode 100755 index 0000000000..dca150d0c5 --- /dev/null +++ b/hack/e2e/delete-cluster.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script deletes a cluster that was created by `create-cluster.sh` +# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller +# All other environment variables have default values (see config.sh) but +# many can be overridden on demand if needed + +set -euo pipefail + +BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" +BIN="${BASE_DIR}/../../bin" + +source "${BASE_DIR}/config.sh" +source "${BASE_DIR}/util.sh" +source "${BASE_DIR}/kops/kops.sh" +source "${BASE_DIR}/eksctl/eksctl.sh" + +if [[ "${CLUSTER_TYPE}" == "kops" ]]; then + kops_delete_cluster \ + "${BIN}/kops" \ + "${CLUSTER_NAME}" \ + "s3://${KOPS_BUCKET}" +elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then + eksctl_delete_cluster \ + "${BIN}/eksctl" \ + "${CLUSTER_NAME}" +else + echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2 + exit 1 +fi diff --git a/hack/e2e/ecr.sh b/hack/e2e/ecr.sh index dc82f1ae2c..9f5e4f0d25 100644 --- a/hack/e2e/ecr.sh +++ b/hack/e2e/ecr.sh @@ -1,9 +1,20 @@ #!/bin/bash -set -uo pipefail +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -source "${BASE_DIR}"/util.sh +set -euo pipefail function ecr_build_and_push() { REGION=${1} @@ -11,26 +22,27 @@ function ecr_build_and_push() { IMAGE_NAME=${3} IMAGE_TAG=${4} IMAGE_ARCH=${5} - set +e - if docker images --format "{{.Repository}}:{{.Tag}}" | grep "${IMAGE_NAME}:${IMAGE_TAG}"; then - set -e - loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed" + + loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}" + aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com" + + # Only setup buildx builder on Prow, allow local users to use docker cache + if [ -n "${PROW_JOB_ID:-}" ]; then + trap "docker buildx rm ebs-csi-multiarch-builder" EXIT + docker buildx create --bootstrap --use --name ebs-csi-multiarch-builder + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + fi + + export IMAGE="${IMAGE_NAME}" + export TAG="${IMAGE_TAG}" + if [[ "$WINDOWS" == true ]]; then + export ALL_OS="linux windows" + export ALL_OSVERSION_windows="ltsc2022" + export ALL_ARCH_linux="amd64" + export ALL_ARCH_windows="${IMAGE_ARCH}" else - set -e - loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}" - aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com - if [[ "$WINDOWS" == true ]]; then - export DOCKER_CLI_EXPERIMENTAL=enabled - export TAG=${IMAGE_TAG} - export IMAGE=${IMAGE_NAME} - trap "docker buildx rm multiarch-builder" EXIT - docker buildx create --use --name multiarch-builder - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - make all-push - else - IMAGE=${IMAGE_NAME} TAG=${IMAGE_TAG} OS=linux ARCH=${IMAGE_ARCH} OSVERSION=al2023 make image - docker tag "${IMAGE_NAME}":"${IMAGE_TAG}"-linux-${IMAGE_ARCH}-al2023 "${IMAGE_NAME}":"${IMAGE_TAG}" - docker push "${IMAGE_NAME}":"${IMAGE_TAG}" - fi + export ALL_OS="linux" + export ALL_ARCH_linux="${IMAGE_ARCH}" fi + make -j $(nproc) all-push } diff --git a/hack/e2e/eksctl.sh b/hack/e2e/eksctl/eksctl.sh similarity index 59% rename from hack/e2e/eksctl.sh rename to hack/e2e/eksctl/eksctl.sh index 6d448dc84c..4b2ccaf68e 100644 --- a/hack/e2e/eksctl.sh +++ b/hack/e2e/eksctl/eksctl.sh @@ -1,20 +1,27 @@ #!/bin/bash -set -euo pipefail +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script updates the kustomize templates in deploy/kubernetes/base/ by +# running `helm template` and stripping the namespace from the output -function eksctl_install() { - INSTALL_PATH=${1} - EKSCTL_VERSION=${2} - if [[ ! -e ${INSTALL_PATH}/eksctl ]]; then - EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/download/v${EKSCTL_VERSION}/eksctl_$(uname -s)_amd64.tar.gz" - curl --silent --location "${EKSCTL_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}" - chmod +x "${INSTALL_PATH}"/eksctl - fi -} +set -euo pipefail function eksctl_create_cluster() { CLUSTER_NAME=${1} - BIN=${2} + EKSCTL_BIN=${2} ZONES=${3} INSTANCE_TYPE=${4} K8S_VERSION=${5} @@ -27,12 +34,12 @@ function eksctl_create_cluster() { CLUSTER_NAME="${CLUSTER_NAME//./-}" - if eksctl_cluster_exists "${CLUSTER_NAME}" "${BIN}"; then + if eksctl_cluster_exists "${CLUSTER_NAME}" "${EKSCTL_BIN}"; then loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} upgrade cluster -f "${CLUSTER_FILE}" + ${EKSCTL_BIN} upgrade cluster -f "${CLUSTER_FILE}" else loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)" - ${BIN} create cluster \ + ${EKSCTL_BIN} create cluster \ --managed \ --ssh-access=false \ --zones "${ZONES}" \ @@ -41,29 +48,29 @@ function eksctl_create_cluster() { --version="${K8S_VERSION}" \ --disable-pod-imds \ --dry-run \ - "${CLUSTER_NAME}" > "${CLUSTER_FILE}" + "${CLUSTER_NAME}" >"${CLUSTER_FILE}" if test -f "$EKSCTL_PATCH_FILE"; then eksctl_patch_cluster_file "$CLUSTER_FILE" "$EKSCTL_PATCH_FILE" fi loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}" + ${EKSCTL_BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}" fi loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}" loudecho "Getting cluster ${CLUSTER_NAME}" - ${BIN} get cluster "${CLUSTER_NAME}" + ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}" if [[ -n "$EKSCTL_ADMIN_ROLE" ]]; then AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}" loudecho "Granting ${ADMIN_ARN} admin access to the cluster" - ${BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin + ${EKSCTL_BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin fi if [[ "$WINDOWS" == true ]]; then - ${BIN} create nodegroup \ + ${EKSCTL_BIN} create nodegroup \ --managed=true \ --ssh-access=false \ --cluster="${CLUSTER_NAME}" \ @@ -71,7 +78,7 @@ function eksctl_create_cluster() { --instance-types=m5.2xlarge \ -n ng-windows \ -m 3 \ - -M 3 \ + -M 3 kubectl apply --kubeconfig "${KUBECONFIG}" -f "$VPC_CONFIGMAP_FILE" fi @@ -81,9 +88,9 @@ function eksctl_create_cluster() { function eksctl_cluster_exists() { CLUSTER_NAME=${1} - BIN=${2} + EKSCTL_BIN=${2} set +e - if ${BIN} get cluster "${CLUSTER_NAME}"; then + if ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}"; then set -e return 0 else @@ -93,10 +100,13 @@ function eksctl_cluster_exists() { } function eksctl_delete_cluster() { - BIN=${1} + EKSCTL_BIN=${1} CLUSTER_NAME=${2} + + CLUSTER_NAME="${CLUSTER_NAME//./-}" + loudecho "Deleting cluster ${CLUSTER_NAME}" - ${BIN} delete cluster "${CLUSTER_NAME}" + ${EKSCTL_BIN} delete cluster "${CLUSTER_NAME}" } function eksctl_patch_cluster_file() { @@ -112,7 +122,7 @@ function eksctl_patch_cluster_file() { cp "$CLUSTER_FILE" "$CLUSTER_FILE_0" # Patch only the Cluster - kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml > "$CLUSTER_FILE_1" + kubectl patch --kubeconfig "/dev/null" -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml >"$CLUSTER_FILE_1" mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0" # Done patching, overwrite original CLUSTER_FILE diff --git a/hack/eksctl-patch.yaml b/hack/e2e/eksctl/patch.yaml similarity index 100% rename from hack/eksctl-patch.yaml rename to hack/e2e/eksctl/patch.yaml diff --git a/hack/values_eksctl.yaml b/hack/e2e/eksctl/values.yaml similarity index 100% rename from hack/values_eksctl.yaml rename to hack/e2e/eksctl/values.yaml diff --git a/hack/vpc-resource-controller-configmap.yaml b/hack/e2e/eksctl/vpc-resource-controller-configmap.yaml similarity index 100% rename from hack/vpc-resource-controller-configmap.yaml rename to hack/e2e/eksctl/vpc-resource-controller-configmap.yaml diff --git a/hack/e2e/helm.sh b/hack/e2e/helm.sh deleted file mode 100644 index 500441e1ae..0000000000 --- a/hack/e2e/helm.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -uo pipefail - -function helm_install() { - INSTALL_PATH=${1} - if [[ ! -e ${INSTALL_PATH}/helm ]]; then - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 get_helm.sh - export USE_SUDO=false - export HELM_INSTALL_DIR=${INSTALL_PATH} - ./get_helm.sh - rm get_helm.sh - fi -} diff --git a/hack/e2e/kops.sh b/hack/e2e/kops/kops.sh similarity index 64% rename from hack/e2e/kops.sh rename to hack/e2e/kops/kops.sh index b5e5dade43..4f34c3580b 100644 --- a/hack/e2e/kops.sh +++ b/hack/e2e/kops/kops.sh @@ -1,30 +1,27 @@ #!/bin/bash -set -euo pipefail - -OS_ARCH=$(go env GOOS)-amd64 +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script updates the kustomize templates in deploy/kubernetes/base/ by +# running `helm template` and stripping the namespace from the output -BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -source "${BASE_DIR}"/util.sh - -function kops_install() { - INSTALL_PATH=${1} - KOPS_VERSION=${2} - if [[ -e "${INSTALL_PATH}"/kops ]]; then - INSTALLED_KOPS_VERSION=$("${INSTALL_PATH}"/kops version) - if [[ "$INSTALLED_KOPS_VERSION" == *"$KOPS_VERSION"* ]]; then - echo "KOPS $INSTALLED_KOPS_VERSION already installed!" - return - fi - fi - KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH} - curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops - chmod +x "${INSTALL_PATH}"/kops -} +set -euo pipefail function kops_create_cluster() { CLUSTER_NAME=${1} - BIN=${2} + KOPS_BIN=${2} ZONES=${3} NODE_COUNT=${4} INSTANCE_TYPE=${5} @@ -36,12 +33,12 @@ function kops_create_cluster() { KOPS_PATCH_NODE_FILE=${11} KOPS_STATE_FILE=${12} - if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then + if kops_cluster_exists "${CLUSTER_NAME}" "${KOPS_BIN}" "${KOPS_STATE_FILE}"; then loudecho "Replacing cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" + ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" else loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)" - ${BIN} create cluster --state "${KOPS_STATE_FILE}" \ + ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \ --zones "${ZONES}" \ --node-count="${NODE_COUNT}" \ --node-size="${INSTANCE_TYPE}" \ @@ -49,7 +46,7 @@ function kops_create_cluster() { --kubernetes-version="${K8S_VERSION}" \ --dry-run \ -o yaml \ - "${CLUSTER_NAME}" > "${CLUSTER_FILE}" + "${CLUSTER_NAME}" >"${CLUSTER_FILE}" if test -f "$KOPS_PATCH_FILE"; then kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" "" @@ -59,26 +56,26 @@ function kops_create_cluster() { fi loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" + ${KOPS_BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}" fi loudecho "Updating cluster $CLUSTER_NAME with $CLUSTER_FILE" - ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes + ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes loudecho "Exporting cluster ${CLUSTER_NAME} kubecfg to ${KUBECONFIG}" - ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" + ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}" loudecho "Validating cluster ${CLUSTER_NAME}" - ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}" + ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}" return $? } function kops_cluster_exists() { CLUSTER_NAME=${1} - BIN=${2} + KOPS_BIN=${2} KOPS_STATE_FILE=${3} set +e - if ${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then + if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then set -e return 0 else @@ -88,11 +85,11 @@ function kops_cluster_exists() { } function kops_delete_cluster() { - BIN=${1} + KOPS_BIN=${1} CLUSTER_NAME=${2} KOPS_STATE_FILE=${3} loudecho "Deleting cluster ${CLUSTER_NAME}" - ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes + ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes } # TODO switch this to python, work exclusively with yaml, use kops toolbox @@ -119,15 +116,15 @@ function kops_patch_cluster_file() { if [ -n "$ROLE" ]; then FILTER="$FILTER | select(.spec.role==\"$ROLE\")" fi - jq "$FILTER" "$CLUSTER_FILE_JSON" > "$CLUSTER_FILE_0" + jq "$FILTER" "$CLUSTER_FILE_JSON" >"$CLUSTER_FILE_0" # Patch only the json objects - kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json > "$CLUSTER_FILE_1" + kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json >"$CLUSTER_FILE_1" mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0" # Delete the original json objects, add the patched # TODO Cluster must always be first? - jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1" + jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" >"$CLUSTER_FILE_1" mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0" # HACK convert the array of json objects to multiple yaml documents @@ -144,14 +141,14 @@ function kops_patch_cluster_file() { function yaml_to_json() { IN=${1} OUT=${2} - kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s > "$OUT" + kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s >"$OUT" } function json_to_yaml() { IN=${1} OUT=${2} for ((i = 0; i < $(jq length "$IN"); i++)); do - echo "---" >> "$OUT" - jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT" + echo "---" >>"$OUT" + jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >>"$OUT" done } diff --git a/hack/kops-patch.yaml b/hack/e2e/kops/patch-cluster.yaml similarity index 100% rename from hack/kops-patch.yaml rename to hack/e2e/kops/patch-cluster.yaml diff --git a/hack/kops-patch-node.yaml b/hack/e2e/kops/patch-node.yaml similarity index 100% rename from hack/kops-patch-node.yaml rename to hack/e2e/kops/patch-node.yaml diff --git a/hack/values.yaml b/hack/e2e/kops/values.yaml similarity index 100% rename from hack/values.yaml rename to hack/e2e/kops/values.yaml diff --git a/hack/e2e/metrics/metrics.sh b/hack/e2e/metrics/metrics.sh index a01ac8a284..a0faab4d47 100644 --- a/hack/e2e/metrics/metrics.sh +++ b/hack/e2e/metrics/metrics.sh @@ -46,9 +46,9 @@ check_dependencies() { collect_metrics() { log "Collecting metrics in $METRICS_DIR_PATH" mkdir -p "$METRICS_DIR_PATH" - + log "Collecting deployment time" - echo -e "$DEPLOYMENT_TIME" > "$METRICS_DIR_PATH/deployment_time.txt" + echo -e "$DEPLOYMENT_TIME" >"$METRICS_DIR_PATH/deployment_time.txt" log "Collecting resource metrics" install_metrics_server @@ -95,7 +95,7 @@ collect_resource_metrics() { local readonly label="$2" local readonly output_file="$3" - kubectl get PodMetrics --kubeconfig "$KUBECONFIG" -n "${namespace}" -l "${label}" -o yaml > "${output_file}" + kubectl get PodMetrics --kubeconfig "$KUBECONFIG" -n "${namespace}" -l "${label}" -o yaml >"${output_file}" } collect_clusterloader2_metrics() { @@ -112,7 +112,7 @@ collect_clusterloader2_metrics() { --repo-root="$PERF_TESTS_DIR" \ --test-configs="$CLUSTER_LOADER_CONFIG" \ --test-overrides="$CLUSTER_LOADER_OVERRIDE" \ - --report-dir="$METRICS_DIR_PATH" \ + --report-dir="$METRICS_DIR_PATH" local readonly exit_code=$? if [[ ${exit_code} -ne 0 ]]; then @@ -161,9 +161,9 @@ upload_metrics() { }' aws s3api put-bucket-policy \ - --bucket "$SOURCE_BUCKET" \ - --policy "$bucket_policy" \ - --region "$AWS_REGION" + --bucket "$SOURCE_BUCKET" \ + --policy "$bucket_policy" \ + --region "$AWS_REGION" log "Setting lifecycle policy on S3 bucket $SOURCE_BUCKET" local readonly lifecycle_policy='{ diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh index a1866e546f..a8a1bbf8dc 100755 --- a/hack/e2e/run.sh +++ b/hack/e2e/run.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,171 +14,59 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -euo pipefail - -BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") -source "${BASE_DIR}"/ecr.sh -source "${BASE_DIR}"/eksctl.sh -source "${BASE_DIR}"/helm.sh -source "${BASE_DIR}"/kops.sh -source "${BASE_DIR}"/util.sh -source "${BASE_DIR}"/chart-testing.sh -source "${BASE_DIR}"/metrics/metrics.sh - -DRIVER_NAME=${DRIVER_NAME:-aws-ebs-csi-driver} -CONTAINER_NAME=${CONTAINER_NAME:-ebs-plugin} - -TEST_ID=${TEST_ID:-$RANDOM} -CLUSTER_NAME=test-cluster-${TEST_ID}.k8s.local -CLUSTER_TYPE=${CLUSTER_TYPE:-kops} - -TEST_DIR=${BASE_DIR}/csi-test-artifacts -BIN_DIR=${TEST_DIR}/bin -CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml -KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"} - -REGION=${AWS_REGION:-us-west-2} -ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c} -FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1) -NODE_COUNT=${NODE_COUNT:-3} -INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large} - -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}} -IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}} -IMAGE_ARCH=${IMAGE_ARCH:-amd64} - -# kops: must include patch version (e.g. 1.19.1) -# eksctl: mustn't include patch version (e.g. 1.19) -K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-${K8S_VERSION:-1.29.0}} -K8S_VERSION_EKSCTL=${K8S_VERSION_EKSCTL:-${K8S_VERSION:-1.28}} - -KOPS_VERSION=${KOPS_VERSION:-1.29.0-alpha.2} -KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-shared-e2e} -KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml} -KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-./hack/kops-patch-node.yaml} -AMI_PARAMETER=${AMI_PARAMETER:-/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64} -AMI_ID=$(aws ssm get-parameters --names ${AMI_PARAMETER} --region ${REGION} --query 'Parameters[0].Value' --output text) - -EKSCTL_VERSION=${EKSCTL_VERSION:-0.165.0} -EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml} -VPC_CONFIGMAP_FILE=${VPC_CONFIGMAP_FILE:-./hack/vpc-resource-controller-configmap.yaml} -EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-} -# Creates a windows node group. -WINDOWS=${WINDOWS:-"false"} +# This script builds and deploys the EBS CSI Driver and runs e2e tests +# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller +# All other environment variables have default values (see config.sh) but +# many can be overridden on demand if needed -# Valid deploy methods: "helm" (default), "kustomize" -DEPLOY_METHOD=${DEPLOY_METHOD:-"helm"} - -HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml} -HELM_EXTRA_FLAGS=${HELM_EXTRA_FLAGS:-} - -TEST_PATH=${TEST_PATH:-"./tests/e2e/..."} -ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"} -GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-e2e\]"} -GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"} -GINKGO_NODES=${GINKGO_NODES:-4} -GINKGO_PARALLEL=${GINKGO_PARALLEL:-25} -NODE_OS_DISTRO=${NODE_OS_DISTRO:-"linux"} -TEST_EXTRA_FLAGS=${TEST_EXTRA_FLAGS:-} - -EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"false"} -# https://github.com/kubernetes-csi/external-snapshotter -EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.3.2"} +set -euo pipefail -HELM_CT_TEST=${HELM_CT_TEST:-"false"} -# https://github.com/helm/chart-testing -CHART_TESTING_VERSION=${CHART_TESTING_VERSION:-3.8.0} -CLEAN=${CLEAN:-"true"} +BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" +BIN="${BASE_DIR}/../../bin" -COLLECT_METRICS=${COLLECT_METRICS:-"false"} +source "${BASE_DIR}/config.sh" +source "${BASE_DIR}/util.sh" +source "${BASE_DIR}/ecr.sh" +source "${BASE_DIR}/metrics/metrics.sh" -loudecho "Testing in region ${REGION} and zones ${ZONES}" -mkdir -p "${BIN_DIR}" -export PATH=${PATH}:${BIN_DIR} +## Setup if [[ "${CLUSTER_TYPE}" == "kops" ]]; then - loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}" - kops_install "${BIN_DIR}" "${KOPS_VERSION}" - KOPS_BIN=${BIN_DIR}/kops + HELM_VALUES_FILE="${BASE_DIR}/kops/values.yaml" + K8S_VERSION="${K8S_VERSION_KOPS}" elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then - loudecho "Installing eksctl ${EKSCTL_VERSION} to ${BIN_DIR}" - eksctl_install "${BIN_DIR}" "${EKSCTL_VERSION}" - EKSCTL_BIN=${BIN_DIR}/eksctl + HELM_VALUES_FILE="${BASE_DIR}/eksctl/values.yaml" + K8S_VERSION="${K8S_VERSION_EKSCTL}" else - loudecho "${CLUSTER_TYPE} must be kops or eksctl!" + echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2 exit 1 fi -loudecho "Installing helm to ${BIN_DIR}" -helm_install "${BIN_DIR}" -HELM_BIN=${BIN_DIR}/helm - -if [[ "${HELM_CT_TEST}" == true ]]; then - loudecho "Installing chart-testing ${CHART_TESTING_VERSION} to ${BIN_DIR}" - ct_install "${BIN_DIR}" "${CHART_TESTING_VERSION}" - CHART_TESTING_BIN=${BIN_DIR}/ct +if [[ "$WINDOWS" == true ]]; then + NODE_OS_DISTRO="windows" else - loudecho "Installing ginkgo to ${BIN_DIR}" - GINKGO_BIN=${BIN_DIR}/ginkgo - if [[ ! -e ${GINKGO_BIN} ]]; then - pushd /tmp - GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install github.com/onsi/ginkgo/v2/ginkgo@v2.11.0 - popd - ginkgo version - fi - loudecho "Installing kubetest2 to ${BIN_DIR}" - KUBETEST2_BIN=${BIN_DIR}/kubetest2 - if [[ ! -e ${KUBETEST2_BIN} ]]; then - pushd /tmp - GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install sigs.k8s.io/kubetest2/...@latest - popd + NODE_OS_DISTRO="linux" +fi + +## Build image + +if [[ "${CREATE_MISSING_ECR_REPO}" == true ]]; then + REPO_CHECK=$(aws ecr describe-repositories --region "${AWS_REGION}") + if [ $(jq '.repositories | map(.repositoryName) | index("aws-ebs-csi-driver")' <<<"${REPO_CHECK}") == "null" ]; then + aws ecr create-repository --region "${AWS_REGION}" --repository-name aws-ebs-csi-driver >/dev/null fi fi -ecr_build_and_push "${REGION}" \ +ecr_build_and_push "${AWS_REGION}" \ "${AWS_ACCOUNT_ID}" \ "${IMAGE_NAME}" \ "${IMAGE_TAG}" \ "${IMAGE_ARCH}" -if [[ "${CLUSTER_TYPE}" == "kops" ]]; then - kops_create_cluster \ - "$CLUSTER_NAME" \ - "$KOPS_BIN" \ - "$ZONES" \ - "$NODE_COUNT" \ - "$INSTANCE_TYPE" \ - "$AMI_ID" \ - "$K8S_VERSION_KOPS" \ - "$CLUSTER_FILE" \ - "$KUBECONFIG" \ - "$KOPS_PATCH_FILE" \ - "$KOPS_PATCH_NODE_FILE" \ - "$KOPS_STATE_FILE" - if [[ $? -ne 0 ]]; then - exit 1 - fi -elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then - eksctl_create_cluster \ - "$CLUSTER_NAME" \ - "$EKSCTL_BIN" \ - "$ZONES" \ - "$INSTANCE_TYPE" \ - "$K8S_VERSION_EKSCTL" \ - "$CLUSTER_FILE" \ - "$KUBECONFIG" \ - "$EKSCTL_PATCH_FILE" \ - "$EKSCTL_ADMIN_ROLE" \ - "$WINDOWS" \ - "$VPC_CONFIGMAP_FILE" - if [[ $? -ne 0 ]]; then - exit 1 - fi -fi +## Deploy if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then - loudecho "Installing snapshot controller and CRDs" + loudecho "Applying snapshot controller and CRDs" kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml @@ -186,37 +74,20 @@ if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml fi -if [[ "${HELM_CT_TEST}" == true ]]; then - loudecho "Test and lint Helm chart with chart-testing" - if [ -n "${PROW_JOB_ID:-}" ]; then - # Prow-specific setup - # Required becuase chart_testing ALWAYS needs a remote - git remote add ct https://github.com/kubernetes-sigs/aws-ebs-csi-driver.git - git fetch ct "${PULL_BASE_REF}" - export CT_REMOTE="ct" - export CT_TARGET_BRANCH="${PULL_BASE_REF}" - fi - set -x - set +e - export KUBECONFIG="${KUBECONFIG}" - ${CHART_TESTING_BIN} lint-and-install --config ${PWD}/tests/ct-config.yaml --helm-extra-set-args="--set=image.repository=${IMAGE_NAME},image.tag=${IMAGE_TAG},node.tolerateAllTaints=false" - TEST_PASSED=$? - set -e - set +x -else - loudecho "Deploying driver via ${DEPLOY_METHOD}" +if [[ "${HELM_CT_TEST}" != true ]]; then startSec=$(date +'%s') if [[ ${DEPLOY_METHOD} == "helm" ]]; then - HELM_ARGS=(upgrade --install "${DRIVER_NAME}" + HELM_ARGS=(upgrade --install "aws-ebs-csi-driver" --namespace kube-system --set image.repository="${IMAGE_NAME}" --set image.tag="${IMAGE_TAG}" --set node.enableWindows="${WINDOWS}" + --set=controller.k8sTagClusterId="${CLUSTER_NAME}" --timeout 10m0s --wait --kubeconfig "${KUBECONFIG}" - ./charts/"${DRIVER_NAME}") + ./charts/aws-ebs-csi-driver) if [[ -f "$HELM_VALUES_FILE" ]]; then HELM_ARGS+=(-f "${HELM_VALUES_FILE}") fi @@ -225,34 +96,58 @@ else HELM_ARGS+=("${EXPANDED_HELM_EXTRA_FLAGS}") fi set -x - "${HELM_BIN}" "${HELM_ARGS[@]}" + "${BIN}/helm" "${HELM_ARGS[@]}" set +x elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then + set -x kubectl --kubeconfig "${KUBECONFIG}" apply -k "./deploy/kubernetes/overlays/stable" kubectl --kubeconfig "${KUBECONFIG}" --namespace kube-system wait --timeout 10m0s --for "condition=ready" pod -l "app.kubernetes.io/name=aws-ebs-csi-driver" + set +x fi endSec=$(date +'%s') deployTimeSeconds=$(((endSec - startSec) / 1)) loudecho "Driver deployment complete, time used: $deployTimeSeconds seconds" +fi + +## Run tests + +if [[ "${HELM_CT_TEST}" == true ]]; then + loudecho "Test and lint Helm chart with chart-testing" + if [ -n "${PROW_JOB_ID:-}" ]; then + # Prow-specific setup + # Required becuase chart_testing ALWAYS needs a remote + git remote add ct https://github.com/kubernetes-sigs/aws-ebs-csi-driver.git + git fetch ct "${PULL_BASE_REF}" + export CT_REMOTE="ct" + export CT_TARGET_BRANCH="${PULL_BASE_REF}" + fi + set -x + set +e + KUBECONFIG="$KUBECONFIG" PATH="${BIN}:${PATH}" "${BIN}/ct" lint-and-install \ + --config="${BASE_DIR}/../../tests/ct-config.yaml" \ + --helm-extra-set-args="--set=image.repository=${IMAGE_NAME},image.tag=${IMAGE_TAG},node.tolerateAllTaints=false" + TEST_PASSED=$? + set -e + set +x +else loudecho "Testing focus ${GINKGO_FOCUS}" if [[ $TEST_PATH == "./tests/e2e-kubernetes/..." ]]; then - pushd ${PWD}/tests/e2e-kubernetes - packageVersion=$(echo $(cut -d '.' -f 1,2 <<< $K8S_VERSION)) + pushd "${BASE_DIR}/../../tests/e2e-kubernetes" + packageVersion=$(echo $(cut -d '.' -f 1,2 <<<$K8S_VERSION)) set -x set +e - kubetest2 noop \ + "${BIN}/kubetest2" noop \ --run-id="e2e-kubernetes" \ --test=ginkgo \ -- \ --skip-regex="${GINKGO_SKIP}" \ --focus-regex="${GINKGO_FOCUS}" \ - --test-package-version=$(curl -L https://dl.k8s.io/release/stable-$packageVersion.txt) \ + --test-package-version=$(curl -L https://dl.k8s.io/release/stable-${packageVersion}.txt) \ --parallel=${GINKGO_PARALLEL} \ - --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=$KUBECONFIG -node-os-distro=${NODE_OS_DISTRO}" - + --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=${KUBECONFIG} -node-os-distro=${NODE_OS_DISTRO}" TEST_PASSED=$? set -e set +x @@ -260,68 +155,67 @@ else fi if [[ $TEST_PATH == "./tests/e2e/..." ]]; then - eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS" set -x set +e - ${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "${TEST_PATH}" -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${FIRST_ZONE}" "${EXPANDED_TEST_EXTRA_FLAGS}" + "${BIN}/ginkgo" -p -nodes="${GINKGO_PARALLEL}" -v \ + --focus="${GINKGO_FOCUS}" \ + --skip="${GINKGO_SKIP}" \ + "${BASE_DIR}/../../tests/e2e/..." \ + -- \ + -kubeconfig="${KUBECONFIG}" \ + -report-dir="${TEST_DIR}/artifacts" \ + -gce-zone="${FIRST_ZONE}" TEST_PASSED=$? set -e set +x fi - PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name) + PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=aws-ebs-csi-driver,app.kubernetes.io/instance=aws-ebs-csi-driver" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name) while IFS= read -r POD; do - loudecho "Printing pod ${POD} ${CONTAINER_NAME} container logs" + loudecho "Printing pod ${POD} container logs" set +e - kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" \ - --kubeconfig "${KUBECONFIG}" + kubectl logs "${POD}" -n kube-system --all-containers --ignore-errors --kubeconfig "${KUBECONFIG}" set -e - done <<< "${PODS}" + done <<<"${PODS}" fi -OVERALL_TEST_PASSED="${TEST_PASSED}" - if [[ "${COLLECT_METRICS}" == true ]]; then metrics_collector "$KUBECONFIG" \ "$AWS_ACCOUNT_ID" \ "$AWS_REGION" \ "$NODE_OS_DISTRO" \ "$deployTimeSeconds" \ - "$DRIVER_NAME" \ + "aws-ebs-csi-driver" \ "$VERSION" fi -if [[ "${CLEAN}" == true ]]; then - loudecho "Cleaning" +## Cleanup - if [[ "${HELM_CT_TEST}" != true ]]; then - loudecho "Removing driver via ${DEPLOY_METHOD}" - if [[ ${DEPLOY_METHOD} == "helm" ]]; then - ${HELM_BIN} del "${DRIVER_NAME}" \ - --namespace kube-system \ - --kubeconfig "${KUBECONFIG}" - elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then - kubectl --kubeconfig "${KUBECONFIG}" delete -k "./deploy/kubernetes/overlays/stable" - fi +if [[ "${HELM_CT_TEST}" != true ]]; then + loudecho "Removing driver via ${DEPLOY_METHOD}" + if [[ ${DEPLOY_METHOD} == "helm" ]]; then + ${BIN}/helm del "aws-ebs-csi-driver" \ + --namespace kube-system \ + --kubeconfig "${KUBECONFIG}" + elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then + kubectl --kubeconfig "${KUBECONFIG}" delete -k "${BASE_DIR}/../../deploy/kubernetes/overlays/stable" fi +fi - if [[ "${CLUSTER_TYPE}" == "kops" ]]; then - kops_delete_cluster \ - "${KOPS_BIN}" \ - "${CLUSTER_NAME}" \ - "${KOPS_STATE_FILE}" - elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then - eksctl_delete_cluster \ - "${EKSCTL_BIN}" \ - "${CLUSTER_NAME}" - fi -else - loudecho "Not cleaning" +if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then + loudecho "Removing snapshot controller and CRDs" + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml + kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml fi -loudecho "OVERALL_TEST_PASSED: ${OVERALL_TEST_PASSED}" -if [[ $OVERALL_TEST_PASSED -ne 0 ]]; then +## Output result + +loudecho "TEST_PASSED: ${TEST_PASSED}" +if [[ $TEST_PASSED -ne 0 ]]; then loudecho "FAIL!" exit 1 else diff --git a/hack/provenance b/hack/provenance.sh similarity index 96% rename from hack/provenance rename to hack/provenance.sh index c04e7ebeee..fcbc141c89 100755 --- a/hack/provenance +++ b/hack/provenance.sh @@ -26,7 +26,7 @@ # Thus, this script echos back the flag `--provenance=false` if and only # if the local buildx installation supports it. If not, it exits silently. -BUILDX_TEST=`docker buildx build --provenance=false 2>&1` +BUILDX_TEST=$(docker buildx build --provenance=false 2>&1) if [[ "${BUILDX_TEST}" == *"See 'docker buildx build --help'."* ]]; then if [[ "${BUILDX_TEST}" == *"requires exactly 1 argument"* ]] && ! docker buildx inspect | grep -qE "^Driver:\s*docker$"; then echo "--provenance=false" diff --git a/hack/prow-e2e.sh b/hack/prow-e2e.sh new file mode 100755 index 0000000000..57a13b110b --- /dev/null +++ b/hack/prow-e2e.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script runs tests in CI by creating a cluster, running the tests, +# cleaning up (regardless of test success/failure), and passing out the result + +case ${1} in +test-e2e-single-az) + TEST="single-az" + export AWS_AVAILABILITY_ZONES="us-west-2a" + ;; +test-e2e-multi-az) + TEST="multi-az" + ;; +test-e2e-external) + TEST="external" + ;; +test-e2e-external-arm64) + TEST="external-arm64" + export INSTANCE_TYPE="m7g.medium" + export AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64" + ;; +test-e2e-external-eks) + TEST="external" + export CLUSTER_TYPE="eksctl" + ;; +test-e2e-external-eks-windows) + TEST="external-windows" + export CLUSTER_TYPE="eksctl" + export WINDOWS="true" + ;; +test-e2e-external-kustomize) + TEST="external-kustomize" + ;; +test-helm-chart) + TEST="helm-ct" + ;; +*) + echo "Unknown e2e test ${1}" >&2 + exit 1 + ;; +esac + +export CLUSTER_NAME="ebs-csi-e2e-${RANDOM}.k8s.local" +# Use S3 bucket created for CI +export KOPS_BUCKET=${KOPS_BUCKET:-"k8s-kops-csi-shared-e2e"} +# Always use us-west-2 in CI, no matter where the local client is +export AWS_REGION=us-west-2 + +make cluster/create || exit 1 +make e2e/${TEST} +E2E_PASSED=$? +make cluster/delete + +echo "E2E_PASSED: ${E2E_PASSED}" +if [[ $E2E_PASSED -ne 0 ]]; then + echo "FAIL!" + exit 1 +else + echo "SUCCESS!" +fi diff --git a/hack/prow.sh b/hack/prow.sh index a5fd118efc..0cf0d778d9 100755 --- a/hack/prow.sh +++ b/hack/prow.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -42,4 +42,4 @@ loudecho "Push manifest list containing amazon linux and windows based images to export REGISTRY=$REGISTRY_NAME export TAG=$GIT_TAG export VERSION=$PULL_BASE_REF -IMAGE=gcr.io/k8s-staging-provider-aws/aws-ebs-csi-driver make all-push +IMAGE=gcr.io/k8s-staging-provider-aws/aws-ebs-csi-driver make -j $(nproc) all-push-with-a1compat diff --git a/hack/release b/hack/release deleted file mode 100755 index 26ffa1d0f6..0000000000 --- a/hack/release +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/local/bin/python3 - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import hashlib -import json -import os -import requests - -def file_sha512(fileName, repoName): - download(fileName, repoName) - with open(fileName, 'rb') as file: - m = hashlib.sha512() - blob = file.read() - m.update(blob) - print("[{}](https://github.com/{}/archive/{}) | `{}`" - .format(fileName,repoName, fileName,m.hexdigest())) - os.remove(fileName) - -def download(fileName, repoName): - url = 'https://github.com/{}/archive/{}'.format(repoName, fileName) - r = requests.get(url, allow_redirects=True) - open(fileName, 'wb').write(r.content) - -def print_header(repo, version): - # Title - print('# {}'.format(version)) - - # documentation section - print('[Documentation](https://github.com/{}/blob/{}/docs/README.md)\n' - .format(repo,version)) - - # sha512 - print('filename | sha512 hash') - print('--------- | ------------') - file_sha512(version+".zip", repo) - file_sha512(version+".tar.gz", repo) - -class Github: - def __init__(self, user, token): - self._url = 'https://api.github.com' - self._user = user - self._token = token - - def get_commits(self, repo, since, branch): - resp = requests.get('{}/repos/{}/compare/{}...{}'.format(self._url, repo, since, branch), - auth=(self._user, self._token)) - jsonResp = json.loads(resp.content) - return jsonResp['commits'] - - def to_pr_numbers(self, repo, commit): - sha = commit['sha'] - resp = requests.get('{}/repos/{}/commits/{}/pulls'.format(self._url, repo, sha), - headers={'Accept': 'application/vnd.github.groot-preview+json'}, - auth=(self._user, self._token)) - jsonResp = json.loads(resp.content) - ret = [] - for pr in jsonResp: - ret.append(pr['number']) - - return ret - - def get_pr(self, repo, pr_number): - resp = requests.get('{}/repos/{}/pulls/{}'.format(self._url, repo, pr_number), - auth=(self._user, self._token)) - jsonResp = json.loads(resp.content) - return jsonResp - - def print_release_note(self, repo, since, branch): - # remove merge commits - commits = self.get_commits(repo, since, branch) - commits = filter(lambda c: not c['commit']['message'].startswith('Merge pull request'), commits) - pr_numbers = set() - for commit in commits: - numbers = self.to_pr_numbers(repo, commit) - for pr in numbers: - pr_numbers.add(pr) - - # dedupe pr numbers - pr_numbers = sorted(list(pr_numbers)) - - for number in pr_numbers: - pr = self.get_pr(repo, number) - if 'user' in pr: - user = pr['user']['login'] - print('* {} ([#{}]({}), [@{}](https://github.com/{}))'.format(pr['title'], pr['number'], pr['html_url'], user, user)) - -def print_sha(args): - version = args.version - repo = args.repo - print_header(repo, version) - -def print_notes(args): - repo = args.repo - since = args.since - branch = args.branch - user = args.github_user - token = args.github_token - - g = Github(user, token) - g.print_release_note(repo, since, branch) - -if __name__=="__main__": - parser = argparse.ArgumentParser(description='Generate release CHANGELOG') - parser.add_argument('--repo', metavar='repo', type=str, default='kubernetes-sigs/aws-ebs-csi-driver', help='the full github repository name') - parser.add_argument('--github-user', metavar='user', type=str, help='the github user for github api') - parser.add_argument('--github-token', metavar='token', type=str, help='the github token for github api') - - subParsers = parser.add_subparsers(title='subcommands', description='[note|sha]') - - noteParser = subParsers.add_parser('note', help='generate release notes') - noteParser.add_argument('--since', metavar='since', type=str, required=True, help='since version tag, e.g. if releasing v1.3.5 then set this to v1.3.4') - noteParser.add_argument('--branch', metavar='branch', type=str, required=True, help='release branch, e.g. if releasing v1.3.5 then set this to release-1.3') - noteParser.set_defaults(func=print_notes) - - shaParser = subParsers.add_parser('sha', help='generate SHA for released version tag') - shaParser.add_argument('--version', metavar='version', type=str, required=True, help='the version to release') - shaParser.set_defaults(func=print_sha) - - args = parser.parse_args() - args.func(args) diff --git a/hack/release-scripts/generate-release-pr b/hack/release-scripts/generate-release-pr index 89f98f3d49..11c3ceea83 100755 --- a/hack/release-scripts/generate-release-pr +++ b/hack/release-scripts/generate-release-pr @@ -38,7 +38,7 @@ check_dependencies() { if [[ $(uname) = "Darwin" ]]; then if ! command -v "gsed" &>/dev/null; then log "gsed could not be found, please install it." - exit 1 + exit 1 fi SED="gsed" fi @@ -52,13 +52,13 @@ error_handler() { trap 'error_handler ${LINENO} $? "$BASH_COMMAND"' ERR # --- Script -usage () { +usage() { echo "Usage: $0 [PREV_DRIVER_VERSION] [NEW_DRIVER_VERSION]" echo "example: $0 v1.23.1 v1.24.0" exit 1 } -setup_vars () { +setup_vars() { export PREV_DRIVER_VERSION=$1 export NEW_DRIVER_VERSION=$2 @@ -72,18 +72,21 @@ setup_vars () { export CHART_PATH="$ROOT_DIRECTORY/charts/aws-ebs-csi-driver/Chart.yaml" } -parse_args () { +parse_args() { # Confirm 2 parameters [[ $# -ne 2 ]] && usage # Confirm new driver version > prev driver version log "Confirming $1 < $2" - sort -C -V <(echo "$1"; echo "$2") || usage + sort -C -V <( + echo "$1" + echo "$2" + ) || usage setup_vars "$@" } -update_readme () { +update_readme() { log "Updating README.md" # vi macro that adds new driver version 'Container Images' row to README.md vi -s <(echo "gg/## Container Images @@ -91,36 +94,39 @@ update_readme () { jjjjjjp:wq") "$README_PATH" } -update_makefile () { +update_makefile() { log "Updating Makefile" $SED "s/VERSION?=$PREV_DRIVER_VERSION/VERSION?=$NEW_DRIVER_VERSION/g" -i "$MAKEFILE_PATH" } -update_installmd () { +update_installmd() { log "Updating docs/install.md" prev_major_minor_version=$(echo "$PREV_DRIVER_VERSION" | sed 's/v\([0-9]*\.[0-9]*\).*/\1/') new_major_minor_version=$(echo "$NEW_DRIVER_VERSION" | sed 's/v\([0-9]*\.[0-9]*\).*/\1/') $SED "s/?ref=release-$prev_major_minor_version/?ref=release-$new_major_minor_version/g" -i "$INSTALL_MD_PATH" } -update_chart_and_overlays () { +update_chart_and_overlays() { log "Updating helm chart and generates kustomize" prev_minor_patch_version=$(echo "$PREV_DRIVER_VERSION" | sed 's/v[0-9]*\.//') new_minor_patch_version=$(echo "$NEW_DRIVER_VERSION" | sed 's/v[0-9]*\.//') $SED "s/$prev_minor_patch_version$/$new_minor_patch_version/g" -i "$CHART_PATH" - (cd "$ROOT_DIRECTORY"; make generate-kustomize) > "/dev/null" + ( + cd "$ROOT_DIRECTORY" + make generate-kustomize + ) >"/dev/null" } -update_upstream_repo () { +update_upstream_repo() { update_readme update_makefile update_installmd update_chart_and_overlays } -print_rest_of_release_steps () { +print_rest_of_release_steps() { echo "SUCCESS! Before you submit the release PR, you must also: 1. Check that 'git diff' produces what you expected. @@ -128,7 +134,7 @@ Before you submit the release PR, you must also: 3. Update charts/aws-ebs-csi-driver/CHANGELOG.md" } -main () { +main() { check_dependencies parse_args "$@" diff --git a/hack/release-scripts/generate-sidecar-tags b/hack/release-scripts/generate-sidecar-tags index 5062658e41..8143ea12d7 100755 --- a/hack/release-scripts/generate-sidecar-tags +++ b/hack/release-scripts/generate-sidecar-tags @@ -50,7 +50,7 @@ check_dependencies() { if [[ $(uname) = "Darwin" ]]; then if ! command -v "gsed" &>/dev/null; then log "gsed could not be found, please install it." - exit 1 + exit 1 fi SED="gsed" fi @@ -66,7 +66,7 @@ trap 'error_handler ${LINENO} $? "$BASH_COMMAND"' ERR # --- Script trap 'rm $tmp_filename' EXIT -update_gcr_kustomize_sidecar_tag () { +update_gcr_kustomize_sidecar_tag() { sidecar_name=$1 line_above=$2 @@ -75,7 +75,7 @@ update_gcr_kustomize_sidecar_tag () { $SED -i "\|$line_above|{n;s/.*/ newTag: $tag/;}" "$KUSTOMIZE_FILEPATH" } -update_helm_chart_sidecar_tag () { +update_helm_chart_sidecar_tag() { sidecar_name=$1 export TAG @@ -84,7 +84,7 @@ update_helm_chart_sidecar_tag () { yq ".sidecars.$sidecar_name.image.tag = env(TAG)" -i "$HELM_VALUES_FILEPATH" } -generate_gcr_kustomize () { +generate_gcr_kustomize() { update_gcr_kustomize_sidecar_tag "provisioner" "newName: registry.k8s.io/sig-storage/csi-provisioner" update_gcr_kustomize_sidecar_tag "attacher" "newName: registry.k8s.io/sig-storage/csi-attacher" update_gcr_kustomize_sidecar_tag "livenessProbe" "newName: registry.k8s.io/sig-storage/livenessprobe" @@ -95,18 +95,17 @@ generate_gcr_kustomize () { log "Success: All sidecar tags in $KUSTOMIZE_FILEPATH updated" } -generate_helm_sidecars () { - yq '.sidecars | keys | .[]' "$IMAGE_DIGESTS_FILEPATH" > "$tmp_filename" +generate_helm_sidecars() { + yq '.sidecars | keys | .[]' "$IMAGE_DIGESTS_FILEPATH" >"$tmp_filename" - for sidecar in $(cat "$tmp_filename") - do - update_helm_chart_sidecar_tag "$sidecar" - done + for sidecar in $(cat "$tmp_filename"); do + update_helm_chart_sidecar_tag "$sidecar" + done log "Success: All sidecar tags in $HELM_VALUES_FILEPATH updated" } -main () { +main() { check_dependencies generate_gcr_kustomize generate_helm_sidecars diff --git a/hack/release-scripts/get-latest-sidecar-images b/hack/release-scripts/get-latest-sidecar-images index 7a7428a12b..6de372f241 100755 --- a/hack/release-scripts/get-latest-sidecar-images +++ b/hack/release-scripts/get-latest-sidecar-images @@ -52,7 +52,7 @@ trap 'error_handler ${LINENO} $? "$BASH_COMMAND"' ERR # --- Script trap 'rm $tmp_filename' EXIT -generate_image_digests_file () { +generate_image_digests_file() { touch "$OUTPUT_FILEPATH" yq '.sidecars.snapshotter.image = "public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter"' -i "$OUTPUT_FILEPATH" @@ -68,28 +68,27 @@ crane_get_latest_image_tag() { image=$1 export TAG - TAG=$(crane ls "$image" | sed '/latest/d' | sort -V | tail -1) # Get tag for $image with latest semvar + TAG=$(crane ls "$image" | sed '/latest/d' | sort -V | tail -1) # Get tag for $image with latest semvar } -update_sidecars_source_of_truth () { - yq '.sidecars | keys | .[]' "$OUTPUT_FILEPATH" > "$tmp_filename" +update_sidecars_source_of_truth() { + yq '.sidecars | keys | .[]' "$OUTPUT_FILEPATH" >"$tmp_filename" - for sidecar in $(cat "$tmp_filename") - do - log "Updating $sidecar in $OUTPUT_FILEPATH" - image=$(yq ".sidecars.$sidecar.image" "$OUTPUT_FILEPATH") + for sidecar in $(cat "$tmp_filename"); do + log "Updating $sidecar in $OUTPUT_FILEPATH" + image=$(yq ".sidecars.$sidecar.image" "$OUTPUT_FILEPATH") - export TAG - crane_get_latest_image_tag "$image" - yq ".sidecars.$sidecar.tag = env(TAG)" -i "$OUTPUT_FILEPATH" + export TAG + crane_get_latest_image_tag "$image" + yq ".sidecars.$sidecar.tag = env(TAG)" -i "$OUTPUT_FILEPATH" - export DIGEST - DIGEST=$(crane digest "$image:$TAG") - yq ".sidecars.$sidecar.manifestDigest = env(DIGEST)" -i "$OUTPUT_FILEPATH" - done + export DIGEST + DIGEST=$(crane digest "$image:$TAG") + yq ".sidecars.$sidecar.manifestDigest = env(DIGEST)" -i "$OUTPUT_FILEPATH" + done } -main () { +main() { check_dependencies generate_image_digests_file update_sidecars_source_of_truth diff --git a/hack/test-integration.sh b/hack/test-integration.sh deleted file mode 100755 index 906e551dff..0000000000 --- a/hack/test-integration.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -if ! [[ "$0" =~ hack/test-integration.sh ]]; then - echo "must be run from repository root" - exit 127 -fi - -export GO111MODULE=on -go test -c ./tests/integration/... -o bin/integration.test && \ - sudo -E bin/integration.test -test.v -ginkgo.v diff --git a/hack/tools/install.sh b/hack/tools/install.sh new file mode 100755 index 0000000000..9cd72a2557 --- /dev/null +++ b/hack/tools/install.sh @@ -0,0 +1,183 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# https://pypi.org/project/awscli/ +AWSCLI_VERSION="1.31.7" +# https://github.com/helm/chart-testing +CT_VERSION="v3.10.1" +# https://github.com/eksctl-io/eksctl +EKSCTL_VERSION="v0.165.0" +# https://github.com/onsi/ginkgo +GINKGO_VERSION="v2.13.2" +# https://github.com/golangci/golangci-lint +GOLANGCI_LINT_VERSION="v1.54.0" +# https://github.com/helm/helm +HELM_VERSION="v3.13.2" +# https://github.com/kubernetes/kops +KOPS_VERSION="v1.29.0-alpha.3" +# https://github.com/golang/mock +MOCKGEN_VERSION="v1.6.0" +# https://github.com/patrickvane/shfmt +SHFMT_VERSION="v3.7.0" +# https://pypi.org/project/yamale/ +YAMALE_VERSION="4.0.4" +# https://pypi.org/project/yamllint/ +YAMLLINT_VERSION="1.32.0" + +OS="$(go env GOHOSTOS)" +ARCH="$(go env GOHOSTARCH)" + +# Installation helpers + +function install_binary() { + INSTALL_PATH="${1}" + DOWNLOAD_URL="${2}" + BINARY_NAME="${3}" + + curl --location "${DOWNLOAD_URL}" --output "${INSTALL_PATH}/${BINARY_NAME}" + chmod +x "${INSTALL_PATH}/${BINARY_NAME}" +} + +function install_go() { + INSTALL_PATH="${1}" + PACKAGE="${2}" + + export GOBIN="${INSTALL_PATH}" + go install "${PACKAGE}" +} + +function install_pip() { + INSTALL_PATH="${1}" + PACKAGE="${2}" + COMMAND="${3}" + + source "${INSTALL_PATH}/venv/bin/activate" + python3 -m pip install "${PACKAGE}" + cp "$(dirname "${0}")/python-runner.sh" "${INSTALL_PATH}/${COMMAND}" +} + +function install_tar_binary() { + INSTALL_PATH="${1}" + DOWNLOAD_URL="${2}" + BINARY_PATH="${3}" + + BINARY_NAME="$(basename "${BINARY_PATH}")" + + if [ "${DOWNLOAD_URL##*.}" = "gz" ]; then + TAR_EXTRA_FLAGS="-z" + elif [ "${DOWNLOAD_URL##*.}" = "xz" ]; then + TAR_EXTRA_FLAGS="-J" + else + TAR_EXTRA_FLAGS="" + fi + + curl --location "${DOWNLOAD_URL}" | tar "$TAR_EXTRA_FLAGS" --extract --touch --transform "s/.*/${BINARY_NAME}/" -C "${INSTALL_PATH}" "${BINARY_PATH}" + chmod +x "${INSTALL_PATH}/${BINARY_NAME}" +} + +# Tool-specific installers + +function install_aws() { + INSTALL_PATH="${1}" + + install_pip "${INSTALL_PATH}" "awscli==${AWSCLI_VERSION}" "aws" +} + +function install_ct() { + INSTALL_PATH="${1}" + + install_tar_binary "${INSTALL_PATH}" "https://github.com/helm/chart-testing/releases/download/${CT_VERSION}/chart-testing_${CT_VERSION:1}_${OS}_${ARCH}.tar.gz" "ct" + install_pip "${INSTALL_PATH}" "yamale==${YAMALE_VERSION}" "yamale" + install_pip "${INSTALL_PATH}" "yamllint==${YAMLLINT_VERSION}" "yamllint" +} + +function install_eksctl() { + INSTALL_PATH="${1}" + + install_tar_binary "${INSTALL_PATH}" "https://github.com/weaveworks/eksctl/releases/download/${EKSCTL_VERSION}/eksctl_${OS^}_${ARCH}.tar.gz" "eksctl" +} + +function install_ginkgo() { + INSTALL_PATH="${1}" + + install_go "${INSTALL_PATH}" "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}" +} + +function install_golangci-lint() { + INSTALL_PATH="${1}" + + # golangci-lint recommends against installing with `go install`: https://golangci-lint.run/usage/install/#install-from-source + install_tar_binary "${INSTALL_PATH}" "https://github.com/golangci/golangci-lint/releases/download/${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}.tar.gz" "golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}/golangci-lint" +} + +function install_helm() { + INSTALL_PATH="${1}" + + install_tar_binary "${INSTALL_PATH}" "https://get.helm.sh/helm-${HELM_VERSION}-${OS}-${ARCH}.tar.gz" "${OS}-${ARCH}/helm" +} + +function install_kops() { + INSTALL_PATH="${1}" + + install_binary "${INSTALL_PATH}" "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-${OS}-${ARCH}" "kops" +} + +function install_kubetest2() { + INSTALL_PATH="${1}" + + install_go "${INSTALL_PATH}" "sigs.k8s.io/kubetest2/...@latest" +} + +function install_mockgen() { + INSTALL_PATH="${1}" + + install_go "${INSTALL_PATH}" "github.com/golang/mock/mockgen@${MOCKGEN_VERSION}" +} + +function install_shfmt() { + INSTALL_PATH="${1}" + + install_go "${INSTALL_PATH}" "mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION}" +} + +# Utility functions + +function create_environment() { + INSTALL_PATH="${1}" + + if command -v "python3"; then + PYTHON_CMD="python3" + else + PYTHON_CMD="python" + fi + VIRTUAL_ENV_DISABLE_PROMPT=1 "${PYTHON_CMD}" -m venv "${INSTALL_PATH}/venv" +} + +function install_tool() { + INSTALL_PATH="${1}" + TOOL="${2}" + + "install_${TOOL}" "${INSTALL_PATH}" +} + +# Script dispatcher + +if [ ! -d "${TOOLS_PATH}/venv" ]; then + create_environment "${TOOLS_PATH}" +fi +install_tool "${TOOLS_PATH}" "${1}" diff --git a/hack/verify-all b/hack/tools/python-runner.sh similarity index 69% rename from hack/verify-all rename to hack/tools/python-runner.sh index e6cabf8811..67c8434eab 100755 --- a/hack/verify-all +++ b/hack/tools/python-runner.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2019 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,11 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -set -euo pipefail +# This script is used as a stub for python commands installed to bin/ +# It activates the venv inside bin/venv/ and then passes through -PKG_ROOT=$(git rev-parse --show-toplevel) - -${PKG_ROOT}/hack/verify-gofmt -${PKG_ROOT}/hack/verify-govet -${PKG_ROOT}/bin/golangci-lint run --deadline=10m -${PKG_ROOT}/hack/verify-vendor.sh +source "$(dirname "${0}")/venv/bin/activate" +exec "$(basename "${0}")" "$@" diff --git a/hack/update-gofmt b/hack/update-gofmt deleted file mode 100755 index dd016e1762..0000000000 --- a/hack/update-gofmt +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -w diff --git a/hack/update-gomock b/hack/update-gomock deleted file mode 100755 index ee6600dda4..0000000000 --- a/hack/update-gomock +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -mockgen -package cloud -destination=./pkg/cloud/mock_cloud.go -source pkg/cloud/cloud_interface.go -mockgen -package cloud -destination=./pkg/cloud/mock_metadata.go -source pkg/cloud/metadata_interface.go -mockgen -package driver -destination=./pkg/driver/mock_mount.go -source pkg/driver/mount.go -mockgen -package mounter -destination=./pkg/mounter/mock_mount_windows.go -source pkg/mounter/safe_mounter_windows.go - -# Reflection-based mocking for external dependencies -mockgen -package cloud -destination=./pkg/cloud/mock_ec2.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API -mockgen -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface -mockgen -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface -mockgen -package driver -destination=./pkg/driver/mock_k8s_storagev1.go k8s.io/client-go/kubernetes/typed/storage/v1 VolumeAttachmentInterface,StorageV1Interface - -# Fixes "Mounter Type cannot implement 'Mounter' as it has a non-exported method and is defined in a different package" -# See https://github.com/kubernetes/mount-utils/commit/a20fcfb15a701977d086330b47b7efad51eb608e for context. -sed -i '/type MockMounter struct {/a \\tmount_utils.Interface' pkg/driver/mock_mount.go -sed -i '/type MockProxyMounter struct {/a \\tmount.Interface' pkg/mounter/mock_mount_windows.go diff --git a/hack/update-gomod b/hack/update-gomod deleted file mode 100755 index d89c2a7edd..0000000000 --- a/hack/update-gomod +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -set -euo pipefail -set -x - -VERSION=${1#"v"} -if [ -z "$VERSION" ]; then - echo "Must specify version!" - exit 1 -fi -MODS=($( - curl -sS https://raw.githubusercontent.com/kubernetes/kubernetes/v${VERSION}/go.mod | - sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p' -)) -echo $MODS -for MOD in "${MODS[@]}"; do - echo $MOD - V=$( - go mod download -json "${MOD}@kubernetes-${VERSION}" | - sed -n 's|.*"Version": "\(.*\)".*|\1|p' - ) - go mod edit "-replace=${MOD}=${MOD}@${V}" -done diff --git a/hack/update-kustomize.sh b/hack/update-kustomize.sh new file mode 100755 index 0000000000..03f1d5b8dd --- /dev/null +++ b/hack/update-kustomize.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script updates the kustomize templates in deploy/kubernetes/base/ by +# running `helm template` and stripping the namespace from the output + +set -euo pipefail + +BIN="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../bin" +TEMP_DIR=$(mktemp -d) +trap "rm -rf \"${TEMP_DIR}\"" EXIT +cp "deploy/kubernetes/base/kustomization.yaml" "${TEMP_DIR}/kustomization.yaml" + +"${BIN}/helm" template --output-dir "${TEMP_DIR}" --skip-tests --api-versions 'snapshot.storage.k8s.io/v1' --api-versions 'policy/v1/PodDisruptionBudget' --set 'controller.userAgentExtra=kustomize' kustomize charts/aws-ebs-csi-driver >/dev/null +rm -rf "deploy/kubernetes/base" +mv "${TEMP_DIR}/aws-ebs-csi-driver/templates" "deploy/kubernetes/base" + +sed -i '/namespace:/d' deploy/kubernetes/base/* +cp "${TEMP_DIR}/kustomization.yaml" "deploy/kubernetes/base/kustomization.yaml" diff --git a/hack/update-mockgen.sh b/hack/update-mockgen.sh new file mode 100755 index 0000000000..5681f223f9 --- /dev/null +++ b/hack/update-mockgen.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Copyright 2023 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +BIN="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../bin" + +# Source-based mocking for internal interfaces +"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_cloud.go -source pkg/cloud/cloud_interface.go +"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_metadata.go -source pkg/cloud/metadata_interface.go +"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_mount.go -source pkg/driver/mount.go +"${BIN}/mockgen" -package mounter -destination=./pkg/mounter/mock_mount_windows.go -source pkg/mounter/safe_mounter_windows.go + +# Reflection-based mocking for external dependencies +"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_ec2.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API +"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface +"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface +"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_storagev1.go k8s.io/client-go/kubernetes/typed/storage/v1 VolumeAttachmentInterface,StorageV1Interface + +# Fixes "Mounter Type cannot implement 'Mounter' as it has a non-exported method and is defined in a different package" +# See https://github.com/kubernetes/mount-utils/commit/a20fcfb15a701977d086330b47b7efad51eb608e for context. +sed -i '/type MockMounter struct {/a \\tmount_utils.Interface' pkg/driver/mock_mount.go +sed -i '/type MockProxyMounter struct {/a \\tmount.Interface' pkg/mounter/mock_mount_windows.go diff --git a/hack/verify-gofmt b/hack/verify-gofmt deleted file mode 100755 index ae0106d65f..0000000000 --- a/hack/verify-gofmt +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -echo "Verifying gofmt" - -diff=$(find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -d 2>&1) -if [[ -n "${diff}" ]]; then - echo "${diff}" - echo - echo "Please run hack/update-gofmt to fix the issue(s)" - exit 1 -fi -echo "No issue found" diff --git a/hack/verify-govet b/hack/verify-govet deleted file mode 100755 index f33dade338..0000000000 --- a/hack/verify-govet +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail - -echo "Verifying govet" - -go vet $(go list ./... | grep -v vendor) - -echo "Done" diff --git a/hack/verify-kustomize b/hack/verify-update.sh similarity index 50% rename from hack/verify-kustomize rename to hack/verify-update.sh index ad4fb3cda9..fb0c5791ce 100755 --- a/hack/verify-kustomize +++ b/hack/verify-update.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2022 The Kubernetes Authors. +# Copyright 2023 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,23 +14,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This script verifies that `make update` does not need to run +# It does so by creating a temporary copy of the repo and running `make update` +# in the copy, and then checking if it produces a diff to the local copy + set -euo pipefail -echo "Verifying kustomize" -diff=$(git status --porcelain=v1) -if [[ -n "${diff}" ]]; then - echo "${diff}" - echo - echo "Please commit all changes before verifying" +ROOT="$(dirname "${0}")/../" +TEST_DIR=$(mktemp -d) +trap "rm -rf \"${TEST_DIR}\"" EXIT +cp -rf "${ROOT}/." "${TEST_DIR}" + +if ! make -C "${TEST_DIR}" update >/dev/null; then + echo "\`make update\` failed!" exit 1 fi -make generate-kustomize -echo -diff=$(git status --porcelain=v1) -if [[ -n "${diff}" ]]; then - echo "${diff}" - echo - echo "Please run make generate-kustomize to fix the issue(s)" + +if ! diff -x bin -r "${TEST_DIR}" "${ROOT}"; then + echo "Auto-generation/formatting needs to run!" + echo "Run \`make update\` to fix!" exit 1 fi -echo "No issue found" diff --git a/hack/verify-vendor.sh b/hack/verify-vendor.sh deleted file mode 100755 index e849dd46d0..0000000000 --- a/hack/verify-vendor.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -echo "Repo uses 'go mod'." -if ! (set -x; env GO111MODULE=on go mod tidy); then - echo "ERROR: vendor check failed." - exit 1 -elif [ "$(git status --porcelain -- go.mod go.sum | wc -l)" -gt 0 ]; then - echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; - git diff -- go.mod go.sum - exit 1 -elif [ -d vendor ]; then - if ! (set -x; env GO111MODULE=on go mod vendor); then - echo "ERROR: vendor check failed." - exit 1 - elif [ "$(git status --porcelain -- vendor | wc -l)" -gt 0 ]; then - echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':" - git status -- vendor - git diff -- vendor - exit 1 - else - echo "Go dependencies and vendor directory up-to-date." - fi -else - echo "Go dependencies up-to-date." -fi