diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000000..2f526620ac
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,8 @@
+.github/
+.idea/
+bin/
+hack/
+charts/
+deploy/
+docs/
+examples/
diff --git a/.github/workflows/publish-ecr.yaml b/.github/workflows/publish-ecr.yaml
index 1c32b44ee4..e6a3267431 100644
--- a/.github/workflows/publish-ecr.yaml
+++ b/.github/workflows/publish-ecr.yaml
@@ -39,7 +39,7 @@ jobs:
           echo "VERSION=${GITHUB_REF_NAME}" >> $GITHUB_ENV
 
     - name: Build, tag, and push manifest to Amazon ECR
-      run: make -j `nproc` all-push
+      run: make -j `nproc` all-push-with-a1compat
 
   ecr-public:
     name: Push to ECR Public
diff --git a/.gitignore b/.gitignore
index 000ff70b0d..4be608e597 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,3 +33,7 @@ vendor/
 
 # Files used by Makefile when upgrading sidecars
 hack/release-scripts/image-digests.yaml
+
+# E2E artifacts
+_rundir/
+_artifacts/
diff --git a/Dockerfile b/Dockerfile
index 5ea6b1b045..a60b4be999 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,7 +24,7 @@ COPY . .
 ARG TARGETOS
 ARG TARGETARCH
 ARG VERSION
-RUN OS=$TARGETOS ARCH=$TARGETARCH make $TARGETOS/$TARGETARCH
+RUN OS=$TARGETOS ARCH=$TARGETARCH make
 
 FROM public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base-csi-ebs:latest-al23 AS linux-al2023
 COPY --from=builder /go/src/github.com/kubernetes-sigs/aws-ebs-csi-driver/bin/aws-ebs-csi-driver /bin/aws-ebs-csi-driver
diff --git a/Makefile b/Makefile
index 86b25a87f7..da02b4232d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-# Copyright 2019 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,29 +12,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+###
+### This Makefile is documented in docs/makefile.md
+###
+
+## Variables/Functions
+
 VERSION?=v1.26.1
 
 PKG=github.com/kubernetes-sigs/aws-ebs-csi-driver
 GIT_COMMIT?=$(shell git rev-parse HEAD)
 BUILD_DATE?=$(shell date -u -Iseconds)
-
 LDFLAGS?="-X ${PKG}/pkg/driver.driverVersion=${VERSION} -X ${PKG}/pkg/cloud.driverVersion=${VERSION} -X ${PKG}/pkg/driver.gitCommit=${GIT_COMMIT} -X ${PKG}/pkg/driver.buildDate=${BUILD_DATE} -s -w"
 
-GO111MODULE=on
-GOPATH=$(shell go env GOPATH)
-GOOS=$(shell go env GOOS)
-GOBIN=$(shell pwd)/bin
+OS?=$(shell go env GOHOSTOS)
+ARCH?=$(shell go env GOHOSTARCH)
+ifeq ($(OS),windows)
+	BINARY=aws-ebs-csi-driver.exe
+	OSVERSION?=ltsc2022
+else
+	BINARY=aws-ebs-csi-driver
+	OSVERSION?=al2023
+endif
+
+GO_SOURCES=go.mod go.sum $(shell find pkg cmd -type f -name "*.go")
 
 REGISTRY?=gcr.io/k8s-staging-provider-aws
 IMAGE?=$(REGISTRY)/aws-ebs-csi-driver
 TAG?=$(GIT_COMMIT)
 
-OUTPUT_TYPE?=docker
-
-OS?=linux
-ARCH?=amd64
-OSVERSION?=al2023
-
 ALL_OS?=linux windows
 ALL_ARCH_linux?=amd64 arm64
 ALL_OSVERSION_linux?=al2023
@@ -43,244 +49,108 @@ ALL_OS_ARCH_OSVERSION_linux=$(foreach arch, $(ALL_ARCH_linux), $(foreach osversi
 ALL_ARCH_windows?=amd64
 ALL_OSVERSION_windows?=ltsc2019 ltsc2022
 ALL_OS_ARCH_OSVERSION_windows=$(foreach arch, $(ALL_ARCH_windows), $(foreach osversion, ${ALL_OSVERSION_windows}, windows-$(arch)-${osversion}))
-
 ALL_OS_ARCH_OSVERSION=$(foreach os, $(ALL_OS), ${ALL_OS_ARCH_OSVERSION_${os}})
 
+CLUSTER_NAME?=ebs-csi-e2e.k8s.local
+CLUSTER_TYPE?=kops
+WINDOWS?=false
+
 # split words on hyphen, access by 1-index
 word-hyphen = $(word $2,$(subst -, ,$1))
 
 .EXPORT_ALL_VARIABLES:
 
-.PHONY: linux/$(ARCH) bin/aws-ebs-csi-driver
-linux/$(ARCH): bin/aws-ebs-csi-driver
-bin/aws-ebs-csi-driver: | bin
-	CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -mod=mod -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver ./cmd/
-
-.PHONY: windows/$(ARCH) bin/aws-ebs-csi-driver.exe
-windows/$(ARCH): bin/aws-ebs-csi-driver.exe
-bin/aws-ebs-csi-driver.exe: | bin
-	CGO_ENABLED=0 GOOS=windows GOARCH=$(ARCH) go build -mod=mod -ldflags ${LDFLAGS} -o bin/aws-ebs-csi-driver.exe ./cmd/
-
-# Builds all linux images (not windows because it can't be exported with OUTPUT_TYPE=docker)
-.PHONY: all
-all: all-image-docker
-
-# Builds all linux and windows images and pushes them
-.PHONY: all-push
-all-push: all-image-registry push-manifest
-
-.PHONY: push-manifest
-push-manifest: create-manifest
-	docker manifest push --purge $(IMAGE):$(TAG)
-
-.PHONY: create-manifest
-create-manifest: all-image-registry
-# sed expression:
-# LHS: match 0 or more not space characters
-# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS
-	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g")
-
-# Only linux for OUTPUT_TYPE=docker because windows image cannot be exported
-# "Currently, multi-platform images cannot be exported with the docker export type. The most common usecase for multi-platform images is to directly push to a registry (see registry)."
-# https://docs.docker.com/engine/reference/commandline/buildx_build/#output
-.PHONY: all-image-docker
-all-image-docker: $(addprefix sub-image-docker-,$(ALL_OS_ARCH_OSVERSION_linux))
-.PHONY: all-image-registry
-all-image-registry: sub-image-registry-linux-arm64-al2 $(addprefix sub-image-registry-,$(ALL_OS_ARCH_OSVERSION))
-
-sub-image-%:
-	$(MAKE) OUTPUT_TYPE=$(call word-hyphen,$*,1) OS=$(call word-hyphen,$*,2) ARCH=$(call word-hyphen,$*,3) OSVERSION=$(call word-hyphen,$*,4) image
+## Default target
+# When no target is supplied, make runs the first target that does not begin with a .
+# Alias that to building the binary
+.PHONY: default
+default: bin/$(BINARY)
 
-.PHONY: image
-image: .image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION)
-.image-$(TAG)-$(OS)-$(ARCH)-$(OSVERSION):
-	docker buildx build \
-		--platform=$(OS)/$(ARCH) \
-		--progress=plain \
-		--target=$(OS)-$(OSVERSION) \
-		--output=type=$(OUTPUT_TYPE) \
-		-t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \
-		--build-arg=GOPROXY=$(GOPROXY) \
-		--build-arg=VERSION=$(VERSION) \
-		`./hack/provenance` \
-		.
-	touch $@
+## Top level targets
 
 .PHONY: clean
 clean:
-	rm -rf .*image-* bin/
-
-bin /tmp/helm /tmp/kubeval:
-	@mkdir -p $@
+	rm -rf bin/
 
-bin/helm: | /tmp/helm bin
-	@curl -o /tmp/helm/helm.tar.gz -sSL https://get.helm.sh/helm-v3.11.2-${GOOS}-amd64.tar.gz
-	@tar -zxf /tmp/helm/helm.tar.gz -C bin --strip-components=1
-	@rm -rf /tmp/helm/*
+.PHONY: test
+test:
+	go test -v -race ./cmd/... ./pkg/...
 
-bin/kubeval: | /tmp/kubeval bin
-	@curl -o /tmp/kubeval/kubeval.tar.gz -sSL https://github.com/instrumenta/kubeval/releases/download/0.16.1/kubeval-linux-amd64.tar.gz
-	@tar -zxf /tmp/kubeval/kubeval.tar.gz -C bin kubeval
-	@rm -rf /tmp/kubeval/*
+.PHONY: test-sanity
+test-sanity:
+	go test -v -race ./tests/sanity/...
 
-bin/mockgen: | bin
-	go install github.com/golang/mock/mockgen@v1.6.0
+.PHONY: tools
+tools: bin/aws bin/ct bin/eksctl bin/ginkgo bin/golangci-lint bin/helm bin/kops bin/kubetest2 bin/mockgen bin/shfmt
 
-bin/golangci-lint: | bin
-	echo "Installing golangci-lint..."
-	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.54.0
+.PHONY: update
+update: update/gofmt update/kustomize update/mockgen update/gomod update/shfmt
+	@echo "All updates succeeded!"
 
-.PHONY: kubeval
-kubeval: bin/kubeval
-	bin/kubeval -d deploy/kubernetes/base,deploy/kubernetes/cluster,deploy/kubernetes/overlays -i kustomization.yaml,crd_.+\.yaml,controller_add
+.PHONY: verify
+verify: verify/govet verify/golangci-lint verify/update
+	@echo "All verifications passed!"
 
-.PHONY: mockgen
-mockgen: bin/mockgen
-	./hack/update-gomock
+.PHONY: all-push
+all-push: all-image-registry push-manifest
 
-.PHONY: verify
-verify: bin/golangci-lint
-	echo "verifying and linting files ..."
-	./hack/verify-all
-	echo "Congratulations! All Go source files have been linted."
+.PHONY: cluster/create
+cluster/create: bin/kops bin/eksctl
+	./hack/e2e/create-cluster.sh
 
-.PHONY: test
-test:
-	go test -v -race ./cmd/... ./pkg/...
+.PHONY: cluster/delete
+cluster/delete: bin/kops bin/eksctl
+	./hack/e2e/delete-cluster.sh
 
-.PHONY: test-sanity
-test-sanity:
-	go test -v -race ./tests/sanity/...
+## E2E targets
+# Targets to run e2e tests
 
-.PHONY: test-e2e-single-az
-test-e2e-single-az:
-	AWS_REGION=us-west-2 \
+.PHONY: e2e/single-az
+e2e/single-az: bin/helm bin/ginkgo
 	AWS_AVAILABILITY_ZONES=us-west-2a \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME,controller.volumeModificationFeature.enabled=true' \
-	EBS_INSTALL_SNAPSHOT="true" \
 	TEST_PATH=./tests/e2e/... \
 	GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \
-	GINKGO_SKIP="\"sc1\"|\"st1\"" \
+	GINKGO_PARALLEL=5 \
+	HELM_EXTRA_FLAGS="--set=controller.volumeModificationFeature.enabled=true" \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-multi-az
-test-e2e-multi-az:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EBS_INSTALL_SNAPSHOT="true" \
+.PHONY: e2e/multi-az
+e2e/multi-az: bin/helm bin/ginkgo
 	TEST_PATH=./tests/e2e/... \
 	GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" \
+	GINKGO_PARALLEL=5 \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-external
-test-e2e-external:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EBS_INSTALL_SNAPSHOT="true" \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
-	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
+.PHONY: e2e/external
+e2e/external: bin/helm bin/kubetest2
 	COLLECT_METRICS="true" \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-external-arm64
-test-e2e-external-arm64:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EBS_INSTALL_SNAPSHOT="true" \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
-	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
-	INSTANCE_TYPE="m7g.medium" \
-	AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64" \
+.PHONY: e2e/external-arm64
+e2e/external-arm64: bin/helm bin/kubetest2
 	IMAGE_ARCH="arm64" \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-external-eks
-test-e2e-external-eks:
-	CLUSTER_TYPE=eksctl \
-	HELM_VALUES_FILE="./hack/values_eksctl.yaml" \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EBS_INSTALL_SNAPSHOT="true" \
-	EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
-	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
-	./hack/e2e/run.sh
-
-.PHONY: test-e2e-external-eks-windows
-test-e2e-external-eks-windows:
-	CLUSTER_TYPE=eksctl \
+.PHONY: e2e/external-windows
+e2e/external-windows: bin/helm bin/kubetest2
 	WINDOWS=true \
-	HELM_VALUES_FILE="./hack/values_eksctl.yaml" \
-	HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \
-	EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
 	GINKGO_SKIP="\[Disruptive\]|\[Serial\]|\[LinuxOnly\]|\[Feature:VolumeSnapshotDataSource\]|\(xfs\)|\(ext4\)|\(block volmode\)" \
 	GINKGO_PARALLEL=15 \
-	NODE_OS_DISTRO="windows" \
+	EBS_INSTALL_SNAPSHOT="false" \
 	./hack/e2e/run.sh
 
-.PHONY: test-e2e-external-kustomize
-test-e2e-external-kustomize:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	EBS_INSTALL_SNAPSHOT="true" \
-	TEST_PATH=./tests/e2e-kubernetes/... \
-	GINKGO_FOCUS="External.Storage" \
-	GINKGO_SKIP="\[Disruptive\]|\[Serial\]" \
+.PHONY: e2e/external-kustomize
+e2e/external-kustomize: bin/kubetest2
 	DEPLOY_METHOD="kustomize" \
 	./hack/e2e/run.sh
 
-.PHONY: test-helm-chart
-test-helm-chart:
-	AWS_REGION=us-west-2 \
-	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
-	EBS_INSTALL_SNAPSHOT="true" \
+.PHONY: e2e/helm-ct
+e2e/helm-ct: bin/helm bin/ct
 	HELM_CT_TEST="true" \
 	./hack/e2e/run.sh
 
-.PHONY: verify-vendor
-test: verify-vendor
-verify: verify-vendor
-verify-vendor:
-	@ echo; echo "### $@:"
-	@ ./hack/verify-vendor.sh
-
-.PHONY: verify-kustomize
-verify: verify-kustomize
-verify-kustomize:
-	@ echo; echo "### $@:"
-	@ ./hack/verify-kustomize
-
-.PHONY: generate-kustomize
-generate-kustomize: bin/helm
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-attacher.yaml > ../../deploy/kubernetes/base/clusterrole-attacher.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-csi-node.yaml > ../../deploy/kubernetes/base/clusterrole-csi-node.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-provisioner.yaml > ../../deploy/kubernetes/base/clusterrole-provisioner.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-resizer.yaml > ../../deploy/kubernetes/base/clusterrole-resizer.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrole-snapshotter.yaml > ../../deploy/kubernetes/base/clusterrole-snapshotter.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-attacher.yaml > ../../deploy/kubernetes/base/clusterrolebinding-attacher.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-csi-node.yaml > ../../deploy/kubernetes/base/clusterrolebinding-csi-node.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-provisioner.yaml > ../../deploy/kubernetes/base/clusterrolebinding-provisioner.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-resizer.yaml > ../../deploy/kubernetes/base/clusterrolebinding-resizer.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/clusterrolebinding-snapshotter.yaml > ../../deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/controller.yaml --api-versions 'snapshot.storage.k8s.io/v1' --set 'controller.userAgentExtra=kustomize' | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/controller.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/csidriver.yaml > ../../deploy/kubernetes/base/csidriver.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/node.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/node.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/poddisruptionbudget-controller.yaml --api-versions 'policy/v1/PodDisruptionBudget' | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/poddisruptionbudget-controller.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/serviceaccount-csi-controller.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/serviceaccount-csi-controller.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/serviceaccount-csi-node.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/serviceaccount-csi-node.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/role-leases.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/role-leases.yaml
-	cd charts/aws-ebs-csi-driver && ../../bin/helm template kustomize . -s templates/rolebinding-leases.yaml | sed -e "/namespace: /d" > ../../deploy/kubernetes/base/rolebinding-leases.yaml
+## Release scripts
+# Targets run as part of performing a release
 
 .PHONY: update-truth-sidecars
 update-truth-sidecars: hack/release-scripts/get-latest-sidecar-images
@@ -291,4 +161,98 @@ generate-sidecar-tags: update-truth-sidecars charts/aws-ebs-csi-driver/values.ya
 	./hack/release-scripts/generate-sidecar-tags
 
 .PHONY: update-sidecar-dependencies
-update-sidecar-dependencies: update-truth-sidecars generate-sidecar-tags generate-kustomize
+update-sidecar-dependencies: update-truth-sidecars generate-sidecar-tags update/kustomize
+
+## CI aliases
+# Targets intended to be executed mostly or only by CI jobs
+
+.PHONY: all-push-with-a1compat
+all-push-with-a1compat: sub-image-linux-arm64-al2 all-image-registry push-manifest
+
+test-e2e-%:
+	./hack/prow-e2e.sh test-e2e-$*
+
+test-helm-chart:
+	./hack/prow-e2e.sh test-helm-chart
+
+## Builds
+
+bin:
+	@mkdir -p $@
+
+bin/$(BINARY): $(GO_SOURCES) | bin
+	CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -mod=readonly -ldflags ${LDFLAGS} -o $@ ./cmd/
+
+.PHONY: all-image-registry
+all-image-registry: $(addprefix sub-image-,$(ALL_OS_ARCH_OSVERSION))
+
+sub-image-%:
+	$(MAKE) OS=$(call word-hyphen,$*,1) ARCH=$(call word-hyphen,$*,2) OSVERSION=$(call word-hyphen,$*,3) image
+
+.PHONY: image
+image:
+	docker buildx build \
+		--platform=$(OS)/$(ARCH) \
+		--progress=plain \
+		--target=$(OS)-$(OSVERSION) \
+		--output=type=registry \
+		-t=$(IMAGE):$(TAG)-$(OS)-$(ARCH)-$(OSVERSION) \
+		--build-arg=GOPROXY=$(GOPROXY) \
+		--build-arg=VERSION=$(VERSION) \
+		`./hack/provenance.sh` \
+		.
+
+.PHONY: create-manifest
+create-manifest: all-image-registry
+# sed expression:
+# LHS: match 0 or more not space characters
+# RHS: replace with $(IMAGE):$(TAG)-& where & is what was matched on LHS
+	docker manifest create --amend $(IMAGE):$(TAG) $(shell echo $(ALL_OS_ARCH_OSVERSION) | sed -e "s~[^ ]*~$(IMAGE):$(TAG)\-&~g")
+
+.PHONY: push-manifest
+push-manifest: create-manifest
+	docker manifest push --purge $(IMAGE):$(TAG)
+
+## Tools
+# Tools necessary to perform other targets
+
+bin/%: hack/tools/install.sh hack/tools/python-runner.sh
+	@TOOLS_PATH="$(shell pwd)/bin" ./hack/tools/install.sh $*
+
+## Updaters
+# Automatic generators/formatters for code
+
+.PHONY: update/gofmt
+update/gofmt:
+	gofmt -s -w .
+
+.PHONY: update/kustomize
+update/kustomize: bin/helm
+	./hack/update-kustomize.sh
+
+.PHONY: update/mockgen
+update/mockgen: bin/mockgen
+	./hack/update-mockgen.sh
+
+.PHONY: update/gomod
+update/gomod:
+	go mod tidy
+
+.PHONY: update/shfmt
+update/shfmt: bin/shfmt
+	./bin/shfmt -w -i 2 -d .
+
+## Verifiers
+# Linters and similar
+
+.PHONY: verify/golangci-lint
+verify/golangci-lint: bin/golangci-lint
+	./bin/golangci-lint run --timeout=10m --verbose
+
+.PHONY: verify/govet
+verify/govet:
+	go vet $$(go list ./...)
+
+.PHONY: verify/update
+verify/update: bin/helm bin/mockgen
+	./hack/verify-update.sh
diff --git a/deploy/kubernetes/base/clusterrolebinding-attacher.yaml b/deploy/kubernetes/base/clusterrolebinding-attacher.yaml
index 5715d2651b..12405a09b1 100644
--- a/deploy/kubernetes/base/clusterrolebinding-attacher.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-attacher.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-controller-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-external-attacher-role
diff --git a/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml b/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml
index 095db52510..4cc33b7e82 100644
--- a/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-csi-node.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-node-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-csi-node-role
diff --git a/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml b/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml
index 3544bc61e2..e95a94b962 100644
--- a/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-provisioner.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-controller-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-external-provisioner-role
diff --git a/deploy/kubernetes/base/clusterrolebinding-resizer.yaml b/deploy/kubernetes/base/clusterrolebinding-resizer.yaml
index c80a9a26bf..543086e74c 100644
--- a/deploy/kubernetes/base/clusterrolebinding-resizer.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-resizer.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-controller-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-external-resizer-role
diff --git a/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml b/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml
index 7946414d59..81ed7c2b80 100644
--- a/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml
+++ b/deploy/kubernetes/base/clusterrolebinding-snapshotter.yaml
@@ -9,7 +9,6 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: ebs-csi-controller-sa
-    namespace: default
 roleRef:
   kind: ClusterRole
   name: ebs-external-snapshotter-role
diff --git a/docs/makefile.md b/docs/makefile.md
new file mode 100644
index 0000000000..5314f0c65c
--- /dev/null
+++ b/docs/makefile.md
@@ -0,0 +1,202 @@
+# Use of the EBS CSI Driver `Makefile`
+
+The EBS CSI Driver comes with a Makefile that can be used to develop, build, test, and release the driver. This file documents Makefile targets, the parameters they support, and common usage scenarios.
+
+## Prerequisites
+
+The `Makefile` has the following dependencies:
+- `go`: https://go.dev/doc/install
+- `python` (may be named `python3`) and `pip`: https://www.python.org/downloads/
+- `jq`: https://github.com/jqlang/jq/releases
+- `kubectl`: https://kubernetes.io/docs/tasks/tools/#kubectl
+- `git`: https://git-scm.com/downloads
+- `docker` and `docker buildx`: https://docs.docker.com/get-docker/ and https://github.com/docker/buildx#installing
+- `make`
+- Standard POSIX tools (`awk`, `grep`, `cat`, etc)
+
+All other tools are downloaded for you at runtime.
+
+## Quickstart Guide
+
+This guide demonstrates the basic workflow for developing for the EBS CSI Driver. More detailed documentation of the available `make` targets is available below.
+
+### 1. Local development for the EBS CSI Driver
+
+If your changes are Helm-only, skip to section 2 (Run E2E tests) below after making your changes.
+
+During development, use `make` at any time to build a driver binary, and thus discover compiler errors. When your change is ready to be tested, run `make test` to execute the unit test suite for the driver. If you are making a significant change to the driver (such as a bugfix or new feature), please add new unit tests or update existing tests where applicable.
+
+### 2. Run E2E tests
+
+To create a `kops` cluster to test the driver against:
+```bash
+make cluster/create
+```
+
+If your change affects the Windows implementation of the driver, instead create an `eksctl` cluster with Windows nodes:
+```bash
+export WINDOWS="true"
+# Note: CLUSTER_TYPE must be set for all e2e tests and cluster deletion
+# Re-export it if necessary (for example, if running tests in a separate terminal tab)
+export CLUSTER_TYPE="eksctl"
+make cluster/create
+```
+
+If you are making a change to the driver, the recommended test suite to run is the external tests:
+```bash
+# Normal external tests (excluding Windows tests)
+make e2e/external
+# Instead, if testing Windows
+make e2e/external-windows
+```
+
+If you are making a change to the Helm chart, the recommended test suite to run is the Helm `ct` tests:
+```bash
+make e2e/helm-ct
+```
+
+To cleanup your cluster after finishing testing:
+```bash
+make cluster/delete
+```
+
+### 3. Before submitting a PR
+
+Run `make update` to automatically format go source files and re-generate automatically generated files. If `make update` produces any changes, commit them before submitting a PR.
+
+Run `make verify` to run linters and other similar code checking tools. Fix any issues `make verify` discovers before submitting a PR.
+
+## Building
+
+### `make` or `make bin/aws-ebs-csi-driver` or `make bin/aws-ebs-csi-driver.exe`
+
+Build a binary copy of the EBS CSI Driver for the local platform. This is the default behavior when calling `make` with no target.
+
+The target OS and/or architecture can be overridden via the `OS` and `ARCH` environment variables (for example, `OS=linux ARCH=arm64 make`)
+
+### `make image`
+
+Build and push a single image of the driver based on the local platform (the same overrides as `make` apply, as well as `OSVERSION` to override container OS version). In most cases, `make all-push` is more suitable. Environment variables are accepted to override the `REGISTRY`, `IMAGE` name, and image `TAG`.
+
+### `make all-push`
+
+Build and push a multi-arch image of the driver based on the OSes in `ALL_OS`, architectures in `ALL_ARCH_linux`/`ALL_ARCH_windows`, and OS versions in `ALL_OSVERSION_linux`/`ALL_OSVERSION_windows`. Also supports `REGISTRY`, `IMAGE`, and `TAG`.
+
+## Local Development
+
+### `make test`
+
+Run all unit tests with race condition checking enabled.
+
+### `make test-sanity`
+
+Run the official [CSI sanity tests](https://github.com/kubernetes-csi/csi-test). _Warning: Currently, 3 of the tests are known to fail incorrectly._
+
+### `make verify`
+
+Performs local verification that other than unit tests (linters, manifest updates, etc)
+
+### `make update`
+
+Updates Kustomize manifests, formatting, and tidies `go.mod`. `make verify` will ensure that `make update` was run by checking if it creates a diff.
+
+## Cluster Management
+
+### `make cluster/create`
+
+Creates a cluster for running E2E tests against. There are many parameters that can be provided via environment variables, a full list is available in [`config.sh`](../hack/e2e/config.sh), but the primary parameters are:
+
+- `CLUSTER_TYPE`: The tool used to create the cluster, either `kops` or `eksctl` - defaults to `kops`
+- `CLUSTER_NAME`: The name of the cluster to create - defaults to `ebs-csi-e2e.k8s.local`
+- `INSTANCE_TYPE`: The instance type to use for cluster nodes - defaults to `c5.large`
+- `AMI_PARAMETER`: The SSM parameter of where to get the AMI for the cluster nodes (`kops` clusters only) - defaults to `/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64`
+- `WINDOWS`: Whether or not to create a Windows node group for the cluster (`eksctl` clusters only) - defaults to `false`
+- `AWS_REGION`: Which region to create the cluster in - defaults to `us-west-2`
+- `AWS_AVAILABILITY_ZONES`: Which AZs to create nodes for the cluster in - defaults to `us-west-2a,us-west-2b,us-west-2c`
+
+#### Example: Create a default (`kops`) cluster
+
+```bash
+make cluster/create
+```
+
+#### Example: Create a cluster with only one Availability Zone
+```bash
+export AWS_AVAILABILITY_ZONES="us-west-2a"
+make cluster/create
+```
+
+#### Example: Create an `eksctl` cluster
+
+```bash
+export CLUSTER_TYPE="eksctl"
+make cluster/create
+```
+
+#### Example: Create a cluster with Graviton nodes for `arm64` testing
+
+```bash
+export INSTANCE_TYPE="m7g.medium"
+export AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64"
+make cluster/create
+```
+
+#### Example: Create a cluster with Windows nodes
+
+```bash
+export WINDOWS="true"
+export CLUSTER_TYPE="eksctl"
+make cluster/create
+```
+
+### `make cluster/delete`
+
+Deletes a cluster created by `make cluster/create`. You must pass the same `CLUSTER_TYPE` and `CLUSTER_NAME` as used when creating the cluster.
+
+## E2E Tests
+
+Run E2E tests against a cluster created by `make cluster/create`. You must pass the same `CLUSTER_TYPE` and `CLUSTER_NAME` as used when creating the cluster.
+
+Alternatively, you may run on an externally created cluster by passing `CLUSTER_TYPE` (required to determine which `values.yaml` to deploy) and `KUBECONFIG`. For `kops` clusters, the node IAM role should include the appropriate IAM policies to use the driver (see [the installation docs](./install.md#set-up-driver-permissions)). For `eksctl` clusters, the `ebs-csi-controller-sa` service account should be pre-created and setup to supply an IRSA role with the appropriate policies.
+
+### `make e2e/external`
+
+Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md). This is the most comprehensive E2E test, recommended for local development.
+
+### `make e2e/single-az`
+
+Run the single-AZ EBS CSI E2E tests. Requires a cluster with only one Availability Zone.
+
+### `make e2e/multi-az`
+
+Run the multi-AZ EBS CSI E2E tests. Requires a cluster with at least two Availability Zones.
+
+### `make e2e/external-arm64`
+
+Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md) using an ARM64 image of the EBS CSI Driver. Requires a cluster with Graviton nodes.
+
+### `make e2e/external-windows`
+
+Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md) with Windows tests enabled. Requires a cluster with Windows nodes.
+
+### `make e2e/external-kustomize`
+
+Run the Kubernetes upstream [external storage E2E tests](https://github.com/kubernetes/kubernetes/blob/master/test/e2e/README.md), but using `kustomize` to deploy the driver instead of `helm`.
+
+### `make e2e/helm-ct`
+
+Test the EBS CSI Driver Helm chart via the [Helm `chart-testing` tool](https://github.com/helm/chart-testing).
+
+## Release Scripts
+
+### `make update-sidecar-dependencies`
+
+Convenience target to perform all sidecar updates and regenerate the manifests. This is the primary target to use unless more granular control is needed.
+
+### `make update-truth-sidecars`
+
+Retrieves the latest sidecar container images and creates or updates `hack/release-scripts/image-digests.yaml`.
+
+### `make generate-sidecar-tags`
+
+Updates the Kustomize and Helm sidecar tags with the values from `hack/release-scripts/image-digests.yaml`.
diff --git a/hack/.gitignore b/hack/.gitignore
deleted file mode 100644
index 3cc5e8f053..0000000000
--- a/hack/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-ebs-e2e-test/
diff --git a/hack/e2e/README.md b/hack/e2e/README.md
deleted file mode 100644
index 914f671b68..0000000000
--- a/hack/e2e/README.md
+++ /dev/null
@@ -1,57 +0,0 @@
-# Usage
-
-run.sh will build and push a driver image, create a kops cluster, helm install the driver pointing to the built image, run ginkgo tests, then clean everything up.
-
-See below for an example.
-
-KOPS_STATE_FILE is an S3 bucket you have write access to.
-
-TEST_ID is a token used for idempotency.
-
-For more details, see the script itself.
-
-For more examples, see the top-level Makefile.
-
-```
-TEST_PATH=./tests/e2e-migration/... \
-EBS_CHECK_MIGRATION=true \
-TEST_ID=18512 \
-CLEAN=false \
-KOPS_STATE_FILE=s3://mattwon \
-AWS_REGION=us-west-2 \
-AWS_AVAILABILITY_ZONES=us-west-2a \
-GINKGO_FOCUS=Dynamic.\*xfs.\*should.store.data \
-GINKGO_NODES=1 \
-./hack/e2e/run.sh
-```
-
-# git read-tree
-
-Reference: https://stackoverflow.com/questions/23937436/add-subdirectory-of-remote-repo-with-git-subtree
-
-How to consume this directory by read-treeing the ebs repo:
-
-```
-git remote add ebs git@github.com:kubernetes-sigs/aws-ebs-csi-driver.git --no-tags
-git fetch ebs
-git read-tree --prefix=hack/e2e/ -u ebs/master:hack/e2e
-```
-
-To commit changes and submit them as a PR back to the ebs repo:
-
-```
-git diff ebs/master:hack/e2e HEAD:hack/e2e > /tmp/hack_e2e.diff
-pushd $GOPATH/src/github.com/kubernetes-sigs/aws-ebs-csi-driver
-git apply --reject --directory hack/e2e /tmp/hack_e2e.diff
-git commit
-```
-
-To consume newer changes from the ebs repo:
-
-```
-git fetch ebs
-git diff HEAD:hack/e2e ebs/master:hack/e2e > /tmp/hack_e2e.diff
-git apply --reject --directory hack/e2e /tmp/hack_e2e.diff
-git add hack/e2e
-git commit -m "Update hack/e2e"
-```
diff --git a/hack/e2e/chart-testing.sh b/hack/e2e/chart-testing.sh
deleted file mode 100644
index 8e7011c2e6..0000000000
--- a/hack/e2e/chart-testing.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -uo pipefail
-
-function ct_install() {
-  INSTALL_PATH=${1}
-  CHART_TESTING_VERSION=${2}
-  if [[ ! -e ${INSTALL_PATH}/chart-testing ]]; then
-    CHART_TESTING_DOWNLOAD_URL="https://github.com/helm/chart-testing/releases/download/v${CHART_TESTING_VERSION}/chart-testing_${CHART_TESTING_VERSION}_linux_amd64.tar.gz"
-    curl --silent --location "${CHART_TESTING_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}"
-    chmod +x "${INSTALL_PATH}"/ct
-  fi
-  apt-get update && apt-get install -y yamllint
-}
diff --git a/hack/e2e/config.sh b/hack/e2e/config.sh
new file mode 100644
index 0000000000..f2d1a843f6
--- /dev/null
+++ b/hack/e2e/config.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+TEST_DIR="${BASE_DIR}/csi-test-artifacts"
+mkdir -p "${TEST_DIR}"
+CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
+KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"}
+
+export AWS_REGION=${AWS_REGION:-us-west-2}
+ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
+FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1)
+NODE_COUNT=${NODE_COUNT:-3}
+INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large}
+WINDOWS=${WINDOWS:-"false"}
+
+# kops: must include patch version (e.g. 1.19.1)
+# eksctl: mustn't include patch version (e.g. 1.19)
+K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-1.29.0}
+K8S_VERSION_EKSCTL=${K8S_VERSION_EKSCTL:-1.28}
+
+EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"true"}
+EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.3.2"}
+
+AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+KOPS_BUCKET=${KOPS_BUCKET:-${AWS_ACCOUNT_ID}-ebs-csi-e2e-kops}
+
+AMI_PARAMETER=${AMI_PARAMETER:-/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64}
+AMI_ID=$(aws ssm get-parameters --names ${AMI_PARAMETER} --region ${AWS_REGION} --query 'Parameters[0].Value' --output text)
+
+CREATE_MISSING_ECR_REPO=${CREATE_MISSING_ECR_REPO:-"true"}
+IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/aws-ebs-csi-driver}
+IMAGE_TAG=${IMAGE_TAG:-$(md5sum <<<"${CLUSTER_NAME}.${CLUSTER_TYPE}" | awk '{ print $1 }')}
+IMAGE_ARCH=${IMAGE_ARCH:-amd64}
+
+DEPLOY_METHOD=${DEPLOY_METHOD:-"helm"}
+HELM_CT_TEST=${HELM_CT_TEST:-"false"}
+HELM_EXTRA_FLAGS=${HELM_EXTRA_FLAGS:-}
+COLLECT_METRICS=${COLLECT_METRICS:-"false"}
+
+TEST_PATH=${TEST_PATH:-"./tests/e2e-kubernetes/..."}
+GINKGO_FOCUS=${GINKGO_FOCUS:-"External.Storage"}
+GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]|\[Serial\]"}
+GINKGO_PARALLEL=${GINKGO_PARALLEL:-25}
+
+# TODO: Left in for now, but look into if this is still necessary and remove if not
+EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-"Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV"}
diff --git a/hack/e2e/create-cluster.sh b/hack/e2e/create-cluster.sh
new file mode 100755
index 0000000000..7171111419
--- /dev/null
+++ b/hack/e2e/create-cluster.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script creates a cluster for use of running the e2e tests
+# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller
+# All other environment variables have default values (see config.sh) but
+# many can be overridden on demand if needed
+
+set -euo pipefail
+
+BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+BIN="${BASE_DIR}/../../bin"
+
+source "${BASE_DIR}/config.sh"
+source "${BASE_DIR}/util.sh"
+source "${BASE_DIR}/kops/kops.sh"
+source "${BASE_DIR}/eksctl/eksctl.sh"
+
+if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
+  BUCKET_CHECK=$("${BIN}/aws" s3api head-bucket --region us-east-1 --bucket "${KOPS_BUCKET}" 2>&1 || true)
+  if grep -q "Forbidden" <<<"${BUCKET_CHECK}"; then
+    echo "Kops requires a S3 bucket in order to store the state" >&2
+    echo "This script is attempting to use a bucket called \`${KOPS_BUCKET}\`" >&2
+    echo "That bucket already exists and you do not have access to it" >&2
+    echo "You can change the bucket by setting the environment variable \$KOPS_BUCKET" >&2
+    exit 1
+  fi
+  if grep -q "Not Found" <<<"${BUCKET_CHECK}"; then
+    "${BIN}/aws" s3api create-bucket --region us-east-1 --bucket "${KOPS_BUCKET}" --acl private >/dev/null
+  fi
+
+  kops_create_cluster \
+    "$CLUSTER_NAME" \
+    "${BIN}/kops" \
+    "$ZONES" \
+    "$NODE_COUNT" \
+    "$INSTANCE_TYPE" \
+    "$AMI_ID" \
+    "$K8S_VERSION_KOPS" \
+    "$CLUSTER_FILE" \
+    "$KUBECONFIG" \
+    "${BASE_DIR}/kops/patch-cluster.yaml" \
+    "${BASE_DIR}/kops/patch-node.yaml" \
+    "s3://${KOPS_BUCKET}"
+elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
+  eksctl_create_cluster \
+    "$CLUSTER_NAME" \
+    "${BIN}/eksctl" \
+    "$ZONES" \
+    "$INSTANCE_TYPE" \
+    "$K8S_VERSION_EKSCTL" \
+    "$CLUSTER_FILE" \
+    "$KUBECONFIG" \
+    "${BASE_DIR}/eksctl/patch.yaml" \
+    "$EKSCTL_ADMIN_ROLE" \
+    "$WINDOWS" \
+    "${BASE_DIR}/eksctl/vpc-resource-controller-configmap.yaml"
+else
+  echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2
+  exit 1
+fi
diff --git a/hack/e2e/delete-cluster.sh b/hack/e2e/delete-cluster.sh
new file mode 100755
index 0000000000..dca150d0c5
--- /dev/null
+++ b/hack/e2e/delete-cluster.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script deletes a cluster that was created by `create-cluster.sh`
+# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller
+# All other environment variables have default values (see config.sh) but
+# many can be overridden on demand if needed
+
+set -euo pipefail
+
+BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+BIN="${BASE_DIR}/../../bin"
+
+source "${BASE_DIR}/config.sh"
+source "${BASE_DIR}/util.sh"
+source "${BASE_DIR}/kops/kops.sh"
+source "${BASE_DIR}/eksctl/eksctl.sh"
+
+if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
+  kops_delete_cluster \
+    "${BIN}/kops" \
+    "${CLUSTER_NAME}" \
+    "s3://${KOPS_BUCKET}"
+elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
+  eksctl_delete_cluster \
+    "${BIN}/eksctl" \
+    "${CLUSTER_NAME}"
+else
+  echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2
+  exit 1
+fi
diff --git a/hack/e2e/ecr.sh b/hack/e2e/ecr.sh
index dc82f1ae2c..9f5e4f0d25 100644
--- a/hack/e2e/ecr.sh
+++ b/hack/e2e/ecr.sh
@@ -1,9 +1,20 @@
 #!/bin/bash
 
-set -uo pipefail
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
-source "${BASE_DIR}"/util.sh
+set -euo pipefail
 
 function ecr_build_and_push() {
   REGION=${1}
@@ -11,26 +22,27 @@ function ecr_build_and_push() {
   IMAGE_NAME=${3}
   IMAGE_TAG=${4}
   IMAGE_ARCH=${5}
-  set +e
-  if docker images --format "{{.Repository}}:{{.Tag}}" | grep "${IMAGE_NAME}:${IMAGE_TAG}"; then
-    set -e
-    loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
+
+  loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
+  aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com"
+
+  # Only setup buildx builder on Prow, allow local users to use docker cache
+  if [ -n "${PROW_JOB_ID:-}" ]; then
+    trap "docker buildx rm ebs-csi-multiarch-builder" EXIT
+    docker buildx create --bootstrap --use --name ebs-csi-multiarch-builder
+    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+  fi
+
+  export IMAGE="${IMAGE_NAME}"
+  export TAG="${IMAGE_TAG}"
+  if [[ "$WINDOWS" == true ]]; then
+    export ALL_OS="linux windows"
+    export ALL_OSVERSION_windows="ltsc2022"
+    export ALL_ARCH_linux="amd64"
+    export ALL_ARCH_windows="${IMAGE_ARCH}"
   else
-    set -e
-    loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
-    aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com
-    if [[ "$WINDOWS" == true ]]; then
-      export DOCKER_CLI_EXPERIMENTAL=enabled
-      export TAG=${IMAGE_TAG}
-      export IMAGE=${IMAGE_NAME}
-      trap "docker buildx rm multiarch-builder" EXIT
-      docker buildx create --use --name multiarch-builder
-      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-      make all-push
-    else
-      IMAGE=${IMAGE_NAME} TAG=${IMAGE_TAG} OS=linux ARCH=${IMAGE_ARCH} OSVERSION=al2023 make image
-      docker tag "${IMAGE_NAME}":"${IMAGE_TAG}"-linux-${IMAGE_ARCH}-al2023 "${IMAGE_NAME}":"${IMAGE_TAG}"
-      docker push "${IMAGE_NAME}":"${IMAGE_TAG}"
-    fi
+    export ALL_OS="linux"
+    export ALL_ARCH_linux="${IMAGE_ARCH}"
   fi
+  make -j $(nproc) all-push
 }
diff --git a/hack/e2e/eksctl.sh b/hack/e2e/eksctl/eksctl.sh
similarity index 59%
rename from hack/e2e/eksctl.sh
rename to hack/e2e/eksctl/eksctl.sh
index 6d448dc84c..4b2ccaf68e 100644
--- a/hack/e2e/eksctl.sh
+++ b/hack/e2e/eksctl/eksctl.sh
@@ -1,20 +1,27 @@
 #!/bin/bash
 
-set -euo pipefail
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script updates the kustomize templates in deploy/kubernetes/base/ by
+# running `helm template` and stripping the namespace from the output
 
-function eksctl_install() {
-  INSTALL_PATH=${1}
-  EKSCTL_VERSION=${2}
-  if [[ ! -e ${INSTALL_PATH}/eksctl ]]; then
-    EKSCTL_DOWNLOAD_URL="https://github.com/weaveworks/eksctl/releases/download/v${EKSCTL_VERSION}/eksctl_$(uname -s)_amd64.tar.gz"
-    curl --silent --location "${EKSCTL_DOWNLOAD_URL}" | tar xz -C "${INSTALL_PATH}"
-    chmod +x "${INSTALL_PATH}"/eksctl
-  fi
-}
+set -euo pipefail
 
 function eksctl_create_cluster() {
   CLUSTER_NAME=${1}
-  BIN=${2}
+  EKSCTL_BIN=${2}
   ZONES=${3}
   INSTANCE_TYPE=${4}
   K8S_VERSION=${5}
@@ -27,12 +34,12 @@ function eksctl_create_cluster() {
 
   CLUSTER_NAME="${CLUSTER_NAME//./-}"
 
-  if eksctl_cluster_exists "${CLUSTER_NAME}" "${BIN}"; then
+  if eksctl_cluster_exists "${CLUSTER_NAME}" "${EKSCTL_BIN}"; then
     loudecho "Upgrading cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${BIN} upgrade cluster -f "${CLUSTER_FILE}"
+    ${EKSCTL_BIN} upgrade cluster -f "${CLUSTER_FILE}"
   else
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
-    ${BIN} create cluster \
+    ${EKSCTL_BIN} create cluster \
       --managed \
       --ssh-access=false \
       --zones "${ZONES}" \
@@ -41,29 +48,29 @@ function eksctl_create_cluster() {
       --version="${K8S_VERSION}" \
       --disable-pod-imds \
       --dry-run \
-      "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
+      "${CLUSTER_NAME}" >"${CLUSTER_FILE}"
 
     if test -f "$EKSCTL_PATCH_FILE"; then
       eksctl_patch_cluster_file "$CLUSTER_FILE" "$EKSCTL_PATCH_FILE"
     fi
 
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
+    ${EKSCTL_BIN} create cluster -f "${CLUSTER_FILE}" --kubeconfig "${KUBECONFIG}"
   fi
 
   loudecho "Cluster ${CLUSTER_NAME} kubecfg written to ${KUBECONFIG}"
   loudecho "Getting cluster ${CLUSTER_NAME}"
-  ${BIN} get cluster "${CLUSTER_NAME}"
+  ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}"
 
   if [[ -n "$EKSCTL_ADMIN_ROLE" ]]; then
     AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
     ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}"
     loudecho "Granting ${ADMIN_ARN} admin access to the cluster"
-    ${BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin
+    ${EKSCTL_BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin
   fi
 
   if [[ "$WINDOWS" == true ]]; then
-    ${BIN} create nodegroup \
+    ${EKSCTL_BIN} create nodegroup \
       --managed=true \
       --ssh-access=false \
       --cluster="${CLUSTER_NAME}" \
@@ -71,7 +78,7 @@ function eksctl_create_cluster() {
       --instance-types=m5.2xlarge \
       -n ng-windows \
       -m 3 \
-      -M 3 \
+      -M 3
 
     kubectl apply --kubeconfig "${KUBECONFIG}" -f "$VPC_CONFIGMAP_FILE"
   fi
@@ -81,9 +88,9 @@ function eksctl_create_cluster() {
 
 function eksctl_cluster_exists() {
   CLUSTER_NAME=${1}
-  BIN=${2}
+  EKSCTL_BIN=${2}
   set +e
-  if ${BIN} get cluster "${CLUSTER_NAME}"; then
+  if ${EKSCTL_BIN} get cluster "${CLUSTER_NAME}"; then
     set -e
     return 0
   else
@@ -93,10 +100,13 @@ function eksctl_cluster_exists() {
 }
 
 function eksctl_delete_cluster() {
-  BIN=${1}
+  EKSCTL_BIN=${1}
   CLUSTER_NAME=${2}
+
+  CLUSTER_NAME="${CLUSTER_NAME//./-}"
+
   loudecho "Deleting cluster ${CLUSTER_NAME}"
-  ${BIN} delete cluster "${CLUSTER_NAME}"
+  ${EKSCTL_BIN} delete cluster "${CLUSTER_NAME}"
 }
 
 function eksctl_patch_cluster_file() {
@@ -112,7 +122,7 @@ function eksctl_patch_cluster_file() {
   cp "$CLUSTER_FILE" "$CLUSTER_FILE_0"
 
   # Patch only the Cluster
-  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml > "$CLUSTER_FILE_1"
+  kubectl patch --kubeconfig "/dev/null" -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$EKSCTL_PATCH_FILE")" -o yaml >"$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
   # Done patching, overwrite original CLUSTER_FILE
diff --git a/hack/eksctl-patch.yaml b/hack/e2e/eksctl/patch.yaml
similarity index 100%
rename from hack/eksctl-patch.yaml
rename to hack/e2e/eksctl/patch.yaml
diff --git a/hack/values_eksctl.yaml b/hack/e2e/eksctl/values.yaml
similarity index 100%
rename from hack/values_eksctl.yaml
rename to hack/e2e/eksctl/values.yaml
diff --git a/hack/vpc-resource-controller-configmap.yaml b/hack/e2e/eksctl/vpc-resource-controller-configmap.yaml
similarity index 100%
rename from hack/vpc-resource-controller-configmap.yaml
rename to hack/e2e/eksctl/vpc-resource-controller-configmap.yaml
diff --git a/hack/e2e/helm.sh b/hack/e2e/helm.sh
deleted file mode 100644
index 500441e1ae..0000000000
--- a/hack/e2e/helm.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-set -uo pipefail
-
-function helm_install() {
-  INSTALL_PATH=${1}
-  if [[ ! -e ${INSTALL_PATH}/helm ]]; then
-    curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-    chmod 700 get_helm.sh
-    export USE_SUDO=false
-    export HELM_INSTALL_DIR=${INSTALL_PATH}
-    ./get_helm.sh
-    rm get_helm.sh
-  fi
-}
diff --git a/hack/e2e/kops.sh b/hack/e2e/kops/kops.sh
similarity index 64%
rename from hack/e2e/kops.sh
rename to hack/e2e/kops/kops.sh
index b5e5dade43..4f34c3580b 100644
--- a/hack/e2e/kops.sh
+++ b/hack/e2e/kops/kops.sh
@@ -1,30 +1,27 @@
 #!/bin/bash
 
-set -euo pipefail
-
-OS_ARCH=$(go env GOOS)-amd64
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script updates the kustomize templates in deploy/kubernetes/base/ by
+# running `helm template` and stripping the namespace from the output
 
-BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
-source "${BASE_DIR}"/util.sh
-
-function kops_install() {
-  INSTALL_PATH=${1}
-  KOPS_VERSION=${2}
-  if [[ -e "${INSTALL_PATH}"/kops ]]; then
-    INSTALLED_KOPS_VERSION=$("${INSTALL_PATH}"/kops version)
-    if [[ "$INSTALLED_KOPS_VERSION" == *"$KOPS_VERSION"* ]]; then
-      echo "KOPS $INSTALLED_KOPS_VERSION already installed!"
-      return
-    fi
-  fi
-  KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH}
-  curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops
-  chmod +x "${INSTALL_PATH}"/kops
-}
+set -euo pipefail
 
 function kops_create_cluster() {
   CLUSTER_NAME=${1}
-  BIN=${2}
+  KOPS_BIN=${2}
   ZONES=${3}
   NODE_COUNT=${4}
   INSTANCE_TYPE=${5}
@@ -36,12 +33,12 @@ function kops_create_cluster() {
   KOPS_PATCH_NODE_FILE=${11}
   KOPS_STATE_FILE=${12}
 
-  if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
+  if kops_cluster_exists "${CLUSTER_NAME}" "${KOPS_BIN}" "${KOPS_STATE_FILE}"; then
     loudecho "Replacing cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
+    ${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   else
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE (dry run)"
-    ${BIN} create cluster --state "${KOPS_STATE_FILE}" \
+    ${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
       --zones "${ZONES}" \
       --node-count="${NODE_COUNT}" \
       --node-size="${INSTANCE_TYPE}" \
@@ -49,7 +46,7 @@ function kops_create_cluster() {
       --kubernetes-version="${K8S_VERSION}" \
       --dry-run \
       -o yaml \
-      "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
+      "${CLUSTER_NAME}" >"${CLUSTER_FILE}"
 
     if test -f "$KOPS_PATCH_FILE"; then
       kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" ""
@@ -59,26 +56,26 @@ function kops_create_cluster() {
     fi
 
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
-    ${BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
+    ${KOPS_BIN} create --state "${KOPS_STATE_FILE}" -f "${CLUSTER_FILE}"
   fi
 
   loudecho "Updating cluster $CLUSTER_NAME with $CLUSTER_FILE"
-  ${BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
+  ${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes
 
   loudecho "Exporting cluster ${CLUSTER_NAME} kubecfg to ${KUBECONFIG}"
-  ${BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
+  ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
 
   loudecho "Validating cluster ${CLUSTER_NAME}"
-  ${BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}"
+  ${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m --kubeconfig "${KUBECONFIG}"
   return $?
 }
 
 function kops_cluster_exists() {
   CLUSTER_NAME=${1}
-  BIN=${2}
+  KOPS_BIN=${2}
   KOPS_STATE_FILE=${3}
   set +e
-  if ${BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
+  if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
     set -e
     return 0
   else
@@ -88,11 +85,11 @@ function kops_cluster_exists() {
 }
 
 function kops_delete_cluster() {
-  BIN=${1}
+  KOPS_BIN=${1}
   CLUSTER_NAME=${2}
   KOPS_STATE_FILE=${3}
   loudecho "Deleting cluster ${CLUSTER_NAME}"
-  ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
+  ${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
 }
 
 # TODO switch this to python, work exclusively with yaml, use kops toolbox
@@ -119,15 +116,15 @@ function kops_patch_cluster_file() {
   if [ -n "$ROLE" ]; then
     FILTER="$FILTER | select(.spec.role==\"$ROLE\")"
   fi
-  jq "$FILTER" "$CLUSTER_FILE_JSON" > "$CLUSTER_FILE_0"
+  jq "$FILTER" "$CLUSTER_FILE_JSON" >"$CLUSTER_FILE_0"
 
   # Patch only the json objects
-  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json > "$CLUSTER_FILE_1"
+  kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json >"$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
   # Delete the original json objects, add the patched
   # TODO Cluster must always be first?
-  jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1"
+  jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" >"$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
   # HACK convert the array of json objects to multiple yaml documents
@@ -144,14 +141,14 @@ function kops_patch_cluster_file() {
 function yaml_to_json() {
   IN=${1}
   OUT=${2}
-  kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s > "$OUT"
+  kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s >"$OUT"
 }
 
 function json_to_yaml() {
   IN=${1}
   OUT=${2}
   for ((i = 0; i < $(jq length "$IN"); i++)); do
-    echo "---" >> "$OUT"
-    jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT"
+    echo "---" >>"$OUT"
+    jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >>"$OUT"
   done
 }
diff --git a/hack/kops-patch.yaml b/hack/e2e/kops/patch-cluster.yaml
similarity index 100%
rename from hack/kops-patch.yaml
rename to hack/e2e/kops/patch-cluster.yaml
diff --git a/hack/kops-patch-node.yaml b/hack/e2e/kops/patch-node.yaml
similarity index 100%
rename from hack/kops-patch-node.yaml
rename to hack/e2e/kops/patch-node.yaml
diff --git a/hack/values.yaml b/hack/e2e/kops/values.yaml
similarity index 100%
rename from hack/values.yaml
rename to hack/e2e/kops/values.yaml
diff --git a/hack/e2e/metrics/metrics.sh b/hack/e2e/metrics/metrics.sh
index a01ac8a284..a0faab4d47 100644
--- a/hack/e2e/metrics/metrics.sh
+++ b/hack/e2e/metrics/metrics.sh
@@ -46,9 +46,9 @@ check_dependencies() {
 collect_metrics() {
   log "Collecting metrics in $METRICS_DIR_PATH"
   mkdir -p "$METRICS_DIR_PATH"
-  
+
   log "Collecting deployment time"
-  echo -e "$DEPLOYMENT_TIME" > "$METRICS_DIR_PATH/deployment_time.txt"
+  echo -e "$DEPLOYMENT_TIME" >"$METRICS_DIR_PATH/deployment_time.txt"
 
   log "Collecting resource metrics"
   install_metrics_server
@@ -95,7 +95,7 @@ collect_resource_metrics() {
   local readonly label="$2"
   local readonly output_file="$3"
 
-  kubectl get PodMetrics --kubeconfig "$KUBECONFIG" -n "${namespace}" -l "${label}" -o yaml > "${output_file}"
+  kubectl get PodMetrics --kubeconfig "$KUBECONFIG" -n "${namespace}" -l "${label}" -o yaml >"${output_file}"
 }
 
 collect_clusterloader2_metrics() {
@@ -112,7 +112,7 @@ collect_clusterloader2_metrics() {
     --repo-root="$PERF_TESTS_DIR" \
     --test-configs="$CLUSTER_LOADER_CONFIG" \
     --test-overrides="$CLUSTER_LOADER_OVERRIDE" \
-    --report-dir="$METRICS_DIR_PATH" \
+    --report-dir="$METRICS_DIR_PATH"
 
   local readonly exit_code=$?
   if [[ ${exit_code} -ne 0 ]]; then
@@ -161,9 +161,9 @@ upload_metrics() {
   }'
 
   aws s3api put-bucket-policy \
-  --bucket "$SOURCE_BUCKET" \
-  --policy "$bucket_policy" \
-  --region "$AWS_REGION"
+    --bucket "$SOURCE_BUCKET" \
+    --policy "$bucket_policy" \
+    --region "$AWS_REGION"
 
   log "Setting lifecycle policy on S3 bucket $SOURCE_BUCKET"
   local readonly lifecycle_policy='{
diff --git a/hack/e2e/run.sh b/hack/e2e/run.sh
index a1866e546f..a8a1bbf8dc 100755
--- a/hack/e2e/run.sh
+++ b/hack/e2e/run.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2019 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,171 +14,59 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -euo pipefail
-
-BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
-source "${BASE_DIR}"/ecr.sh
-source "${BASE_DIR}"/eksctl.sh
-source "${BASE_DIR}"/helm.sh
-source "${BASE_DIR}"/kops.sh
-source "${BASE_DIR}"/util.sh
-source "${BASE_DIR}"/chart-testing.sh
-source "${BASE_DIR}"/metrics/metrics.sh
-
-DRIVER_NAME=${DRIVER_NAME:-aws-ebs-csi-driver}
-CONTAINER_NAME=${CONTAINER_NAME:-ebs-plugin}
-
-TEST_ID=${TEST_ID:-$RANDOM}
-CLUSTER_NAME=test-cluster-${TEST_ID}.k8s.local
-CLUSTER_TYPE=${CLUSTER_TYPE:-kops}
-
-TEST_DIR=${BASE_DIR}/csi-test-artifacts
-BIN_DIR=${TEST_DIR}/bin
-CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
-KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"}
-
-REGION=${AWS_REGION:-us-west-2}
-ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
-FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1)
-NODE_COUNT=${NODE_COUNT:-3}
-INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large}
-
-AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
-IMAGE_NAME=${IMAGE_NAME:-${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com/${DRIVER_NAME}}
-IMAGE_TAG=${IMAGE_TAG:-${TEST_ID}}
-IMAGE_ARCH=${IMAGE_ARCH:-amd64}
-
-# kops: must include patch version (e.g. 1.19.1)
-# eksctl: mustn't include patch version (e.g. 1.19)
-K8S_VERSION_KOPS=${K8S_VERSION_KOPS:-${K8S_VERSION:-1.29.0}}
-K8S_VERSION_EKSCTL=${K8S_VERSION_EKSCTL:-${K8S_VERSION:-1.28}}
-
-KOPS_VERSION=${KOPS_VERSION:-1.29.0-alpha.2}
-KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-shared-e2e}
-KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml}
-KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-./hack/kops-patch-node.yaml}
-AMI_PARAMETER=${AMI_PARAMETER:-/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64}
-AMI_ID=$(aws ssm get-parameters --names ${AMI_PARAMETER} --region ${REGION} --query 'Parameters[0].Value' --output text)
-
-EKSCTL_VERSION=${EKSCTL_VERSION:-0.165.0}
-EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml}
-VPC_CONFIGMAP_FILE=${VPC_CONFIGMAP_FILE:-./hack/vpc-resource-controller-configmap.yaml}
-EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-}
-# Creates a windows node group.
-WINDOWS=${WINDOWS:-"false"}
+# This script builds and deploys the EBS CSI Driver and runs e2e tests
+# CLUSTER_NAME and CLUSTER_TYPE are expected to be specified by the caller
+# All other environment variables have default values (see config.sh) but
+# many can be overridden on demand if needed
 
-# Valid deploy methods: "helm" (default), "kustomize"
-DEPLOY_METHOD=${DEPLOY_METHOD:-"helm"}
-
-HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml}
-HELM_EXTRA_FLAGS=${HELM_EXTRA_FLAGS:-}
-
-TEST_PATH=${TEST_PATH:-"./tests/e2e/..."}
-ARTIFACTS=${ARTIFACTS:-"${TEST_DIR}/artifacts"}
-GINKGO_FOCUS=${GINKGO_FOCUS:-"\[ebs-csi-e2e\]"}
-GINKGO_SKIP=${GINKGO_SKIP:-"\[Disruptive\]"}
-GINKGO_NODES=${GINKGO_NODES:-4}
-GINKGO_PARALLEL=${GINKGO_PARALLEL:-25}
-NODE_OS_DISTRO=${NODE_OS_DISTRO:-"linux"}
-TEST_EXTRA_FLAGS=${TEST_EXTRA_FLAGS:-}
-
-EBS_INSTALL_SNAPSHOT=${EBS_INSTALL_SNAPSHOT:-"false"}
-# https://github.com/kubernetes-csi/external-snapshotter
-EBS_INSTALL_SNAPSHOT_VERSION=${EBS_INSTALL_SNAPSHOT_VERSION:-"v6.3.2"}
+set -euo pipefail
 
-HELM_CT_TEST=${HELM_CT_TEST:-"false"}
-# https://github.com/helm/chart-testing
-CHART_TESTING_VERSION=${CHART_TESTING_VERSION:-3.8.0}
-CLEAN=${CLEAN:-"true"}
+BASE_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
+BIN="${BASE_DIR}/../../bin"
 
-COLLECT_METRICS=${COLLECT_METRICS:-"false"}
+source "${BASE_DIR}/config.sh"
+source "${BASE_DIR}/util.sh"
+source "${BASE_DIR}/ecr.sh"
+source "${BASE_DIR}/metrics/metrics.sh"
 
-loudecho "Testing in region ${REGION} and zones ${ZONES}"
-mkdir -p "${BIN_DIR}"
-export PATH=${PATH}:${BIN_DIR}
+## Setup
 
 if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
-  loudecho "Installing kops ${KOPS_VERSION} to ${BIN_DIR}"
-  kops_install "${BIN_DIR}" "${KOPS_VERSION}"
-  KOPS_BIN=${BIN_DIR}/kops
+  HELM_VALUES_FILE="${BASE_DIR}/kops/values.yaml"
+  K8S_VERSION="${K8S_VERSION_KOPS}"
 elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
-  loudecho "Installing eksctl ${EKSCTL_VERSION} to ${BIN_DIR}"
-  eksctl_install "${BIN_DIR}" "${EKSCTL_VERSION}"
-  EKSCTL_BIN=${BIN_DIR}/eksctl
+  HELM_VALUES_FILE="${BASE_DIR}/eksctl/values.yaml"
+  K8S_VERSION="${K8S_VERSION_EKSCTL}"
 else
-  loudecho "${CLUSTER_TYPE} must be kops or eksctl!"
+  echo "Cluster type ${CLUSTER_TYPE} is invalid, must be kops or eksctl" >&2
   exit 1
 fi
 
-loudecho "Installing helm to ${BIN_DIR}"
-helm_install "${BIN_DIR}"
-HELM_BIN=${BIN_DIR}/helm
-
-if [[ "${HELM_CT_TEST}" == true ]]; then
-  loudecho "Installing chart-testing ${CHART_TESTING_VERSION} to ${BIN_DIR}"
-  ct_install "${BIN_DIR}" "${CHART_TESTING_VERSION}"
-  CHART_TESTING_BIN=${BIN_DIR}/ct
+if [[ "$WINDOWS" == true ]]; then
+  NODE_OS_DISTRO="windows"
 else
-  loudecho "Installing ginkgo to ${BIN_DIR}"
-  GINKGO_BIN=${BIN_DIR}/ginkgo
-  if [[ ! -e ${GINKGO_BIN} ]]; then
-    pushd /tmp
-    GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install github.com/onsi/ginkgo/v2/ginkgo@v2.11.0
-    popd
-    ginkgo version
-  fi
-  loudecho "Installing kubetest2 to ${BIN_DIR}"
-  KUBETEST2_BIN=${BIN_DIR}/kubetest2
-  if [[ ! -e ${KUBETEST2_BIN} ]]; then
-    pushd /tmp
-    GOPATH=${TEST_DIR} GOBIN=${BIN_DIR} go install sigs.k8s.io/kubetest2/...@latest
-    popd
+  NODE_OS_DISTRO="linux"
+fi
+
+## Build image
+
+if [[ "${CREATE_MISSING_ECR_REPO}" == true ]]; then
+  REPO_CHECK=$(aws ecr describe-repositories --region "${AWS_REGION}")
+  if [ $(jq '.repositories | map(.repositoryName) | index("aws-ebs-csi-driver")' <<<"${REPO_CHECK}") == "null" ]; then
+    aws ecr create-repository --region "${AWS_REGION}" --repository-name aws-ebs-csi-driver >/dev/null
   fi
 fi
 
-ecr_build_and_push "${REGION}" \
+ecr_build_and_push "${AWS_REGION}" \
   "${AWS_ACCOUNT_ID}" \
   "${IMAGE_NAME}" \
   "${IMAGE_TAG}" \
   "${IMAGE_ARCH}"
 
-if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
-  kops_create_cluster \
-    "$CLUSTER_NAME" \
-    "$KOPS_BIN" \
-    "$ZONES" \
-    "$NODE_COUNT" \
-    "$INSTANCE_TYPE" \
-    "$AMI_ID" \
-    "$K8S_VERSION_KOPS" \
-    "$CLUSTER_FILE" \
-    "$KUBECONFIG" \
-    "$KOPS_PATCH_FILE" \
-    "$KOPS_PATCH_NODE_FILE" \
-    "$KOPS_STATE_FILE"
-  if [[ $? -ne 0 ]]; then
-    exit 1
-  fi
-elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
-  eksctl_create_cluster \
-    "$CLUSTER_NAME" \
-    "$EKSCTL_BIN" \
-    "$ZONES" \
-    "$INSTANCE_TYPE" \
-    "$K8S_VERSION_EKSCTL" \
-    "$CLUSTER_FILE" \
-    "$KUBECONFIG" \
-    "$EKSCTL_PATCH_FILE" \
-    "$EKSCTL_ADMIN_ROLE" \
-    "$WINDOWS" \
-    "$VPC_CONFIGMAP_FILE"
-  if [[ $? -ne 0 ]]; then
-    exit 1
-  fi
-fi
+## Deploy
 
 if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then
-  loudecho "Installing snapshot controller and CRDs"
+  loudecho "Applying snapshot controller and CRDs"
   kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
   kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml
   kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
@@ -186,37 +74,20 @@ if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then
   kubectl apply --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
 fi
 
-if [[ "${HELM_CT_TEST}" == true ]]; then
-  loudecho "Test and lint Helm chart with chart-testing"
-  if [ -n "${PROW_JOB_ID:-}" ]; then
-    # Prow-specific setup
-    # Required becuase chart_testing ALWAYS needs a remote
-    git remote add ct https://github.com/kubernetes-sigs/aws-ebs-csi-driver.git
-    git fetch ct "${PULL_BASE_REF}"
-    export CT_REMOTE="ct"
-    export CT_TARGET_BRANCH="${PULL_BASE_REF}"
-  fi
-  set -x
-  set +e
-  export KUBECONFIG="${KUBECONFIG}"
-  ${CHART_TESTING_BIN} lint-and-install --config ${PWD}/tests/ct-config.yaml --helm-extra-set-args="--set=image.repository=${IMAGE_NAME},image.tag=${IMAGE_TAG},node.tolerateAllTaints=false"
-  TEST_PASSED=$?
-  set -e
-  set +x
-else
-  loudecho "Deploying driver via ${DEPLOY_METHOD}"
+if [[ "${HELM_CT_TEST}" != true ]]; then
   startSec=$(date +'%s')
 
   if [[ ${DEPLOY_METHOD} == "helm" ]]; then
-    HELM_ARGS=(upgrade --install "${DRIVER_NAME}"
+    HELM_ARGS=(upgrade --install "aws-ebs-csi-driver"
       --namespace kube-system
       --set image.repository="${IMAGE_NAME}"
       --set image.tag="${IMAGE_TAG}"
       --set node.enableWindows="${WINDOWS}"
+      --set=controller.k8sTagClusterId="${CLUSTER_NAME}"
       --timeout 10m0s
       --wait
       --kubeconfig "${KUBECONFIG}"
-      ./charts/"${DRIVER_NAME}")
+      ./charts/aws-ebs-csi-driver)
     if [[ -f "$HELM_VALUES_FILE" ]]; then
       HELM_ARGS+=(-f "${HELM_VALUES_FILE}")
     fi
@@ -225,34 +96,58 @@ else
       HELM_ARGS+=("${EXPANDED_HELM_EXTRA_FLAGS}")
     fi
     set -x
-    "${HELM_BIN}" "${HELM_ARGS[@]}"
+    "${BIN}/helm" "${HELM_ARGS[@]}"
     set +x
   elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then
+    set -x
     kubectl --kubeconfig "${KUBECONFIG}" apply -k "./deploy/kubernetes/overlays/stable"
     kubectl --kubeconfig "${KUBECONFIG}" --namespace kube-system wait --timeout 10m0s --for "condition=ready" pod -l "app.kubernetes.io/name=aws-ebs-csi-driver"
+    set +x
   fi
 
   endSec=$(date +'%s')
   deployTimeSeconds=$(((endSec - startSec) / 1))
   loudecho "Driver deployment complete, time used: $deployTimeSeconds seconds"
+fi
+
+## Run tests
+
+if [[ "${HELM_CT_TEST}" == true ]]; then
+  loudecho "Test and lint Helm chart with chart-testing"
+  if [ -n "${PROW_JOB_ID:-}" ]; then
+    # Prow-specific setup
+    # Required becuase chart_testing ALWAYS needs a remote
+    git remote add ct https://github.com/kubernetes-sigs/aws-ebs-csi-driver.git
+    git fetch ct "${PULL_BASE_REF}"
+    export CT_REMOTE="ct"
+    export CT_TARGET_BRANCH="${PULL_BASE_REF}"
+  fi
+  set -x
+  set +e
+  KUBECONFIG="$KUBECONFIG" PATH="${BIN}:${PATH}" "${BIN}/ct" lint-and-install \
+    --config="${BASE_DIR}/../../tests/ct-config.yaml" \
+    --helm-extra-set-args="--set=image.repository=${IMAGE_NAME},image.tag=${IMAGE_TAG},node.tolerateAllTaints=false"
+  TEST_PASSED=$?
+  set -e
+  set +x
+else
   loudecho "Testing focus ${GINKGO_FOCUS}"
 
   if [[ $TEST_PATH == "./tests/e2e-kubernetes/..." ]]; then
-    pushd ${PWD}/tests/e2e-kubernetes
-    packageVersion=$(echo $(cut -d '.' -f 1,2 <<< $K8S_VERSION))
+    pushd "${BASE_DIR}/../../tests/e2e-kubernetes"
+    packageVersion=$(echo $(cut -d '.' -f 1,2 <<<$K8S_VERSION))
 
     set -x
     set +e
-    kubetest2 noop \
+    "${BIN}/kubetest2" noop \
       --run-id="e2e-kubernetes" \
       --test=ginkgo \
       -- \
       --skip-regex="${GINKGO_SKIP}" \
       --focus-regex="${GINKGO_FOCUS}" \
-      --test-package-version=$(curl -L https://dl.k8s.io/release/stable-$packageVersion.txt) \
+      --test-package-version=$(curl -L https://dl.k8s.io/release/stable-${packageVersion}.txt) \
       --parallel=${GINKGO_PARALLEL} \
-      --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=$KUBECONFIG -node-os-distro=${NODE_OS_DISTRO}"
-
+      --test-args="-storage.testdriver=${PWD}/manifests.yaml -kubeconfig=${KUBECONFIG} -node-os-distro=${NODE_OS_DISTRO}"
     TEST_PASSED=$?
     set -e
     set +x
@@ -260,68 +155,67 @@ else
   fi
 
   if [[ $TEST_PATH == "./tests/e2e/..." ]]; then
-    eval "EXPANDED_TEST_EXTRA_FLAGS=$TEST_EXTRA_FLAGS"
     set -x
     set +e
-    ${GINKGO_BIN} -p -nodes="${GINKGO_NODES}" -v --focus="${GINKGO_FOCUS}" --skip="${GINKGO_SKIP}" "${TEST_PATH}" -- -kubeconfig="${KUBECONFIG}" -report-dir="${ARTIFACTS}" -gce-zone="${FIRST_ZONE}" "${EXPANDED_TEST_EXTRA_FLAGS}"
+    "${BIN}/ginkgo" -p -nodes="${GINKGO_PARALLEL}" -v \
+      --focus="${GINKGO_FOCUS}" \
+      --skip="${GINKGO_SKIP}" \
+      "${BASE_DIR}/../../tests/e2e/..." \
+      -- \
+      -kubeconfig="${KUBECONFIG}" \
+      -report-dir="${TEST_DIR}/artifacts" \
+      -gce-zone="${FIRST_ZONE}"
     TEST_PASSED=$?
     set -e
     set +x
   fi
 
-  PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=${DRIVER_NAME},app.kubernetes.io/instance=${DRIVER_NAME}" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name)
+  PODS=$(kubectl get pod -n kube-system -l "app.kubernetes.io/name=aws-ebs-csi-driver,app.kubernetes.io/instance=aws-ebs-csi-driver" -o json --kubeconfig "${KUBECONFIG}" | jq -r .items[].metadata.name)
 
   while IFS= read -r POD; do
-    loudecho "Printing pod ${POD} ${CONTAINER_NAME} container logs"
+    loudecho "Printing pod ${POD} container logs"
     set +e
-    kubectl logs "${POD}" -n kube-system "${CONTAINER_NAME}" \
-      --kubeconfig "${KUBECONFIG}"
+    kubectl logs "${POD}" -n kube-system --all-containers --ignore-errors --kubeconfig "${KUBECONFIG}"
     set -e
-  done <<< "${PODS}"
+  done <<<"${PODS}"
 fi
 
-OVERALL_TEST_PASSED="${TEST_PASSED}"
-
 if [[ "${COLLECT_METRICS}" == true ]]; then
   metrics_collector "$KUBECONFIG" \
     "$AWS_ACCOUNT_ID" \
     "$AWS_REGION" \
     "$NODE_OS_DISTRO" \
     "$deployTimeSeconds" \
-    "$DRIVER_NAME" \
+    "aws-ebs-csi-driver" \
     "$VERSION"
 fi
 
-if [[ "${CLEAN}" == true ]]; then
-  loudecho "Cleaning"
+## Cleanup
 
-  if [[ "${HELM_CT_TEST}" != true ]]; then
-    loudecho "Removing driver via ${DEPLOY_METHOD}"
-    if [[ ${DEPLOY_METHOD} == "helm" ]]; then
-      ${HELM_BIN} del "${DRIVER_NAME}" \
-        --namespace kube-system \
-        --kubeconfig "${KUBECONFIG}"
-    elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then
-      kubectl --kubeconfig "${KUBECONFIG}" delete -k "./deploy/kubernetes/overlays/stable"
-    fi
+if [[ "${HELM_CT_TEST}" != true ]]; then
+  loudecho "Removing driver via ${DEPLOY_METHOD}"
+  if [[ ${DEPLOY_METHOD} == "helm" ]]; then
+    ${BIN}/helm del "aws-ebs-csi-driver" \
+      --namespace kube-system \
+      --kubeconfig "${KUBECONFIG}"
+  elif [[ ${DEPLOY_METHOD} == "kustomize" ]]; then
+    kubectl --kubeconfig "${KUBECONFIG}" delete -k "${BASE_DIR}/../../deploy/kubernetes/overlays/stable"
   fi
+fi
 
-  if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
-    kops_delete_cluster \
-      "${KOPS_BIN}" \
-      "${CLUSTER_NAME}" \
-      "${KOPS_STATE_FILE}"
-  elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
-    eksctl_delete_cluster \
-      "${EKSCTL_BIN}" \
-      "${CLUSTER_NAME}"
-  fi
-else
-  loudecho "Not cleaning"
+if [[ "${EBS_INSTALL_SNAPSHOT}" == true ]]; then
+  loudecho "Removing snapshot controller and CRDs"
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
+  kubectl delete --kubeconfig "${KUBECONFIG}" -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${EBS_INSTALL_SNAPSHOT_VERSION}"/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml
 fi
 
-loudecho "OVERALL_TEST_PASSED: ${OVERALL_TEST_PASSED}"
-if [[ $OVERALL_TEST_PASSED -ne 0 ]]; then
+## Output result
+
+loudecho "TEST_PASSED: ${TEST_PASSED}"
+if [[ $TEST_PASSED -ne 0 ]]; then
   loudecho "FAIL!"
   exit 1
 else
diff --git a/hack/provenance b/hack/provenance.sh
similarity index 96%
rename from hack/provenance
rename to hack/provenance.sh
index c04e7ebeee..fcbc141c89 100755
--- a/hack/provenance
+++ b/hack/provenance.sh
@@ -26,7 +26,7 @@
 # Thus, this script echos back the flag `--provenance=false` if and only
 # if the local buildx installation supports it. If not, it exits silently.
 
-BUILDX_TEST=`docker buildx build --provenance=false 2>&1`
+BUILDX_TEST=$(docker buildx build --provenance=false 2>&1)
 if [[ "${BUILDX_TEST}" == *"See 'docker buildx build --help'."* ]]; then
   if [[ "${BUILDX_TEST}" == *"requires exactly 1 argument"* ]] && ! docker buildx inspect | grep -qE "^Driver:\s*docker$"; then
     echo "--provenance=false"
diff --git a/hack/prow-e2e.sh b/hack/prow-e2e.sh
new file mode 100755
index 0000000000..57a13b110b
--- /dev/null
+++ b/hack/prow-e2e.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script runs tests in CI by creating a cluster, running the tests,
+# cleaning up (regardless of test success/failure), and passing out the result
+
+case ${1} in
+test-e2e-single-az)
+  TEST="single-az"
+  export AWS_AVAILABILITY_ZONES="us-west-2a"
+  ;;
+test-e2e-multi-az)
+  TEST="multi-az"
+  ;;
+test-e2e-external)
+  TEST="external"
+  ;;
+test-e2e-external-arm64)
+  TEST="external-arm64"
+  export INSTANCE_TYPE="m7g.medium"
+  export AMI_PARAMETER="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64"
+  ;;
+test-e2e-external-eks)
+  TEST="external"
+  export CLUSTER_TYPE="eksctl"
+  ;;
+test-e2e-external-eks-windows)
+  TEST="external-windows"
+  export CLUSTER_TYPE="eksctl"
+  export WINDOWS="true"
+  ;;
+test-e2e-external-kustomize)
+  TEST="external-kustomize"
+  ;;
+test-helm-chart)
+  TEST="helm-ct"
+  ;;
+*)
+  echo "Unknown e2e test ${1}" >&2
+  exit 1
+  ;;
+esac
+
+export CLUSTER_NAME="ebs-csi-e2e-${RANDOM}.k8s.local"
+# Use S3 bucket created for CI
+export KOPS_BUCKET=${KOPS_BUCKET:-"k8s-kops-csi-shared-e2e"}
+# Always use us-west-2 in CI, no matter where the local client is
+export AWS_REGION=us-west-2
+
+make cluster/create || exit 1
+make e2e/${TEST}
+E2E_PASSED=$?
+make cluster/delete
+
+echo "E2E_PASSED: ${E2E_PASSED}"
+if [[ $E2E_PASSED -ne 0 ]]; then
+  echo "FAIL!"
+  exit 1
+else
+  echo "SUCCESS!"
+fi
diff --git a/hack/prow.sh b/hack/prow.sh
index a5fd118efc..0cf0d778d9 100755
--- a/hack/prow.sh
+++ b/hack/prow.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2019 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -42,4 +42,4 @@ loudecho "Push manifest list containing amazon linux and windows based images to
 export REGISTRY=$REGISTRY_NAME
 export TAG=$GIT_TAG
 export VERSION=$PULL_BASE_REF
-IMAGE=gcr.io/k8s-staging-provider-aws/aws-ebs-csi-driver make all-push
+IMAGE=gcr.io/k8s-staging-provider-aws/aws-ebs-csi-driver make -j $(nproc) all-push-with-a1compat
diff --git a/hack/release b/hack/release
deleted file mode 100755
index 26ffa1d0f6..0000000000
--- a/hack/release
+++ /dev/null
@@ -1,134 +0,0 @@
-#!/usr/local/bin/python3
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import hashlib
-import json
-import os
-import requests
-
-def file_sha512(fileName, repoName):
-    download(fileName, repoName)
-    with open(fileName, 'rb') as file:
-        m = hashlib.sha512()
-        blob = file.read()
-        m.update(blob)
-        print("[{}](https://github.com/{}/archive/{}) | `{}`"
-                .format(fileName,repoName, fileName,m.hexdigest()))
-    os.remove(fileName)
-
-def download(fileName, repoName):
-    url = 'https://github.com/{}/archive/{}'.format(repoName, fileName)
-    r = requests.get(url, allow_redirects=True)
-    open(fileName, 'wb').write(r.content)
-
-def print_header(repo, version):
-    # Title
-    print('# {}'.format(version))
-
-    # documentation section
-    print('[Documentation](https://github.com/{}/blob/{}/docs/README.md)\n'
-            .format(repo,version))
-
-    # sha512
-    print('filename  | sha512 hash')
-    print('--------- | ------------')
-    file_sha512(version+".zip", repo)
-    file_sha512(version+".tar.gz", repo)
-
-class Github:
-    def __init__(self, user, token):
-        self._url = 'https://api.github.com'
-        self._user = user
-        self._token = token
-
-    def get_commits(self, repo, since, branch):
-        resp = requests.get('{}/repos/{}/compare/{}...{}'.format(self._url, repo, since, branch),
-                auth=(self._user, self._token))
-        jsonResp = json.loads(resp.content)
-        return jsonResp['commits']
-
-    def to_pr_numbers(self, repo, commit):
-        sha = commit['sha']
-        resp = requests.get('{}/repos/{}/commits/{}/pulls'.format(self._url, repo, sha),
-                headers={'Accept': 'application/vnd.github.groot-preview+json'},
-                auth=(self._user, self._token))
-        jsonResp = json.loads(resp.content)
-        ret = []
-        for pr in jsonResp:
-            ret.append(pr['number'])
-
-        return ret
-
-    def get_pr(self, repo, pr_number):
-        resp = requests.get('{}/repos/{}/pulls/{}'.format(self._url, repo, pr_number),
-                auth=(self._user, self._token))
-        jsonResp = json.loads(resp.content)
-        return jsonResp
-
-    def print_release_note(self, repo, since, branch):
-        # remove merge commits
-        commits = self.get_commits(repo, since, branch)
-        commits = filter(lambda c: not c['commit']['message'].startswith('Merge pull request'), commits)
-        pr_numbers = set()
-        for commit in commits:
-            numbers = self.to_pr_numbers(repo, commit)
-            for pr in numbers:
-                pr_numbers.add(pr)
-
-        # dedupe pr numbers
-        pr_numbers = sorted(list(pr_numbers))
-
-        for number in pr_numbers:
-            pr = self.get_pr(repo, number)
-            if 'user' in pr:
-                user = pr['user']['login']
-                print('* {} ([#{}]({}), [@{}](https://github.com/{}))'.format(pr['title'], pr['number'], pr['html_url'], user, user))
-
-def print_sha(args):
-    version = args.version
-    repo = args.repo
-    print_header(repo, version)
-
-def print_notes(args):
-    repo = args.repo
-    since = args.since
-    branch = args.branch
-    user = args.github_user
-    token = args.github_token
-
-    g = Github(user, token)
-    g.print_release_note(repo, since, branch)
-
-if __name__=="__main__":
-    parser = argparse.ArgumentParser(description='Generate release CHANGELOG')
-    parser.add_argument('--repo', metavar='repo', type=str, default='kubernetes-sigs/aws-ebs-csi-driver', help='the full github repository name')
-    parser.add_argument('--github-user', metavar='user', type=str, help='the github user for github api')
-    parser.add_argument('--github-token', metavar='token', type=str, help='the github token for github api')
-
-    subParsers = parser.add_subparsers(title='subcommands', description='[note|sha]')
-
-    noteParser = subParsers.add_parser('note', help='generate release notes')
-    noteParser.add_argument('--since', metavar='since', type=str, required=True, help='since version tag, e.g. if releasing v1.3.5 then set this to v1.3.4')
-    noteParser.add_argument('--branch', metavar='branch', type=str, required=True, help='release branch, e.g. if releasing v1.3.5 then set this to release-1.3')
-    noteParser.set_defaults(func=print_notes)
-
-    shaParser = subParsers.add_parser('sha', help='generate SHA for released version tag')
-    shaParser.add_argument('--version', metavar='version', type=str, required=True, help='the version to release')
-    shaParser.set_defaults(func=print_sha)
-
-    args = parser.parse_args()
-    args.func(args)
diff --git a/hack/release-scripts/generate-release-pr b/hack/release-scripts/generate-release-pr
index 89f98f3d49..11c3ceea83 100755
--- a/hack/release-scripts/generate-release-pr
+++ b/hack/release-scripts/generate-release-pr
@@ -38,7 +38,7 @@ check_dependencies() {
   if [[ $(uname) = "Darwin" ]]; then
     if ! command -v "gsed" &>/dev/null; then
       log "gsed could not be found, please install it."
-        exit 1
+      exit 1
     fi
     SED="gsed"
   fi
@@ -52,13 +52,13 @@ error_handler() {
 trap 'error_handler ${LINENO} $? "$BASH_COMMAND"' ERR
 
 # --- Script
-usage () {
+usage() {
   echo "Usage: $0 [PREV_DRIVER_VERSION] [NEW_DRIVER_VERSION]"
   echo "example: $0 v1.23.1 v1.24.0"
   exit 1
 }
 
-setup_vars () {
+setup_vars() {
   export PREV_DRIVER_VERSION=$1
   export NEW_DRIVER_VERSION=$2
 
@@ -72,18 +72,21 @@ setup_vars () {
   export CHART_PATH="$ROOT_DIRECTORY/charts/aws-ebs-csi-driver/Chart.yaml"
 }
 
-parse_args () {
+parse_args() {
   # Confirm 2 parameters
   [[ $# -ne 2 ]] && usage
 
   # Confirm new driver version > prev driver version
   log "Confirming $1 < $2"
-  sort -C -V <(echo "$1"; echo "$2") || usage
+  sort -C -V <(
+    echo "$1"
+    echo "$2"
+  ) || usage
 
   setup_vars "$@"
 }
 
-update_readme () {
+update_readme() {
   log "Updating README.md"
   # vi macro that adds new driver version 'Container Images' row to README.md
   vi -s <(echo "gg/## Container Images
@@ -91,36 +94,39 @@ update_readme () {
   jjjjjjp:wq") "$README_PATH"
 }
 
-update_makefile () {
+update_makefile() {
   log "Updating Makefile"
   $SED "s/VERSION?=$PREV_DRIVER_VERSION/VERSION?=$NEW_DRIVER_VERSION/g" -i "$MAKEFILE_PATH"
 }
 
-update_installmd () {
+update_installmd() {
   log "Updating docs/install.md"
   prev_major_minor_version=$(echo "$PREV_DRIVER_VERSION" | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')
   new_major_minor_version=$(echo "$NEW_DRIVER_VERSION" | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')
   $SED "s/?ref=release-$prev_major_minor_version/?ref=release-$new_major_minor_version/g" -i "$INSTALL_MD_PATH"
 }
 
-update_chart_and_overlays () {
+update_chart_and_overlays() {
   log "Updating helm chart and generates kustomize"
   prev_minor_patch_version=$(echo "$PREV_DRIVER_VERSION" | sed 's/v[0-9]*\.//')
   new_minor_patch_version=$(echo "$NEW_DRIVER_VERSION" | sed 's/v[0-9]*\.//')
 
   $SED "s/$prev_minor_patch_version$/$new_minor_patch_version/g" -i "$CHART_PATH"
 
-  (cd "$ROOT_DIRECTORY"; make generate-kustomize) > "/dev/null"
+  (
+    cd "$ROOT_DIRECTORY"
+    make generate-kustomize
+  ) >"/dev/null"
 }
 
-update_upstream_repo () {
+update_upstream_repo() {
   update_readme
   update_makefile
   update_installmd
   update_chart_and_overlays
 }
 
-print_rest_of_release_steps () {
+print_rest_of_release_steps() {
   echo "SUCCESS!
 Before you submit the release PR, you must also:
   1. Check that 'git diff' produces what you expected.
@@ -128,7 +134,7 @@ Before you submit the release PR, you must also:
   3. Update charts/aws-ebs-csi-driver/CHANGELOG.md"
 }
 
-main () {
+main() {
   check_dependencies
   parse_args "$@"
 
diff --git a/hack/release-scripts/generate-sidecar-tags b/hack/release-scripts/generate-sidecar-tags
index 5062658e41..8143ea12d7 100755
--- a/hack/release-scripts/generate-sidecar-tags
+++ b/hack/release-scripts/generate-sidecar-tags
@@ -50,7 +50,7 @@ check_dependencies() {
   if [[ $(uname) = "Darwin" ]]; then
     if ! command -v "gsed" &>/dev/null; then
       log "gsed could not be found, please install it."
-        exit 1
+      exit 1
     fi
     SED="gsed"
   fi
@@ -66,7 +66,7 @@ trap 'error_handler ${LINENO} $? "$BASH_COMMAND"' ERR
 # --- Script
 trap 'rm $tmp_filename' EXIT
 
-update_gcr_kustomize_sidecar_tag () {
+update_gcr_kustomize_sidecar_tag() {
   sidecar_name=$1
   line_above=$2
 
@@ -75,7 +75,7 @@ update_gcr_kustomize_sidecar_tag () {
   $SED -i "\|$line_above|{n;s/.*/    newTag: $tag/;}" "$KUSTOMIZE_FILEPATH"
 }
 
-update_helm_chart_sidecar_tag () {
+update_helm_chart_sidecar_tag() {
   sidecar_name=$1
 
   export TAG
@@ -84,7 +84,7 @@ update_helm_chart_sidecar_tag () {
   yq ".sidecars.$sidecar_name.image.tag = env(TAG)" -i "$HELM_VALUES_FILEPATH"
 }
 
-generate_gcr_kustomize () {
+generate_gcr_kustomize() {
   update_gcr_kustomize_sidecar_tag "provisioner" "newName: registry.k8s.io/sig-storage/csi-provisioner"
   update_gcr_kustomize_sidecar_tag "attacher" "newName: registry.k8s.io/sig-storage/csi-attacher"
   update_gcr_kustomize_sidecar_tag "livenessProbe" "newName: registry.k8s.io/sig-storage/livenessprobe"
@@ -95,18 +95,17 @@ generate_gcr_kustomize () {
   log "Success: All sidecar tags in $KUSTOMIZE_FILEPATH updated"
 }
 
-generate_helm_sidecars () {
-  yq '.sidecars | keys | .[]' "$IMAGE_DIGESTS_FILEPATH" > "$tmp_filename"
+generate_helm_sidecars() {
+  yq '.sidecars | keys | .[]' "$IMAGE_DIGESTS_FILEPATH" >"$tmp_filename"
 
-  for sidecar in $(cat "$tmp_filename")
-     do
-       update_helm_chart_sidecar_tag "$sidecar"
-     done
+  for sidecar in $(cat "$tmp_filename"); do
+    update_helm_chart_sidecar_tag "$sidecar"
+  done
 
   log "Success: All sidecar tags in $HELM_VALUES_FILEPATH updated"
 }
 
-main () {
+main() {
   check_dependencies
   generate_gcr_kustomize
   generate_helm_sidecars
diff --git a/hack/release-scripts/get-latest-sidecar-images b/hack/release-scripts/get-latest-sidecar-images
index 7a7428a12b..6de372f241 100755
--- a/hack/release-scripts/get-latest-sidecar-images
+++ b/hack/release-scripts/get-latest-sidecar-images
@@ -52,7 +52,7 @@ trap 'error_handler ${LINENO} $? "$BASH_COMMAND"' ERR
 # --- Script
 trap 'rm $tmp_filename' EXIT
 
-generate_image_digests_file () {
+generate_image_digests_file() {
   touch "$OUTPUT_FILEPATH"
 
   yq '.sidecars.snapshotter.image = "public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter"' -i "$OUTPUT_FILEPATH"
@@ -68,28 +68,27 @@ crane_get_latest_image_tag() {
   image=$1
 
   export TAG
-  TAG=$(crane ls "$image" | sed '/latest/d' | sort -V | tail -1)  # Get tag for $image with latest semvar
+  TAG=$(crane ls "$image" | sed '/latest/d' | sort -V | tail -1) # Get tag for $image with latest semvar
 }
 
-update_sidecars_source_of_truth () {
-  yq '.sidecars | keys | .[]' "$OUTPUT_FILEPATH" > "$tmp_filename"
+update_sidecars_source_of_truth() {
+  yq '.sidecars | keys | .[]' "$OUTPUT_FILEPATH" >"$tmp_filename"
 
-  for sidecar in $(cat "$tmp_filename")
-     do
-       log "Updating $sidecar in $OUTPUT_FILEPATH"
-       image=$(yq ".sidecars.$sidecar.image" "$OUTPUT_FILEPATH")
+  for sidecar in $(cat "$tmp_filename"); do
+    log "Updating $sidecar in $OUTPUT_FILEPATH"
+    image=$(yq ".sidecars.$sidecar.image" "$OUTPUT_FILEPATH")
 
-       export TAG
-       crane_get_latest_image_tag "$image"
-       yq ".sidecars.$sidecar.tag = env(TAG)" -i "$OUTPUT_FILEPATH"
+    export TAG
+    crane_get_latest_image_tag "$image"
+    yq ".sidecars.$sidecar.tag = env(TAG)" -i "$OUTPUT_FILEPATH"
 
-       export DIGEST
-       DIGEST=$(crane digest "$image:$TAG")
-       yq ".sidecars.$sidecar.manifestDigest = env(DIGEST)" -i "$OUTPUT_FILEPATH"
-     done
+    export DIGEST
+    DIGEST=$(crane digest "$image:$TAG")
+    yq ".sidecars.$sidecar.manifestDigest = env(DIGEST)" -i "$OUTPUT_FILEPATH"
+  done
 }
 
-main () {
+main() {
   check_dependencies
   generate_image_digests_file
   update_sidecars_source_of_truth
diff --git a/hack/test-integration.sh b/hack/test-integration.sh
deleted file mode 100755
index 906e551dff..0000000000
--- a/hack/test-integration.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-if ! [[ "$0" =~ hack/test-integration.sh ]]; then
-  echo "must be run from repository root"
-  exit 127
-fi
-
-export GO111MODULE=on
-go test -c ./tests/integration/... -o bin/integration.test && \
-  sudo -E bin/integration.test -test.v -ginkgo.v
diff --git a/hack/tools/install.sh b/hack/tools/install.sh
new file mode 100755
index 0000000000..9cd72a2557
--- /dev/null
+++ b/hack/tools/install.sh
@@ -0,0 +1,183 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# https://pypi.org/project/awscli/
+AWSCLI_VERSION="1.31.7"
+# https://github.com/helm/chart-testing
+CT_VERSION="v3.10.1"
+# https://github.com/eksctl-io/eksctl
+EKSCTL_VERSION="v0.165.0"
+# https://github.com/onsi/ginkgo
+GINKGO_VERSION="v2.13.2"
+# https://github.com/golangci/golangci-lint
+GOLANGCI_LINT_VERSION="v1.54.0"
+# https://github.com/helm/helm
+HELM_VERSION="v3.13.2"
+# https://github.com/kubernetes/kops
+KOPS_VERSION="v1.29.0-alpha.3"
+# https://github.com/golang/mock
+MOCKGEN_VERSION="v1.6.0"
+# https://github.com/patrickvane/shfmt
+SHFMT_VERSION="v3.7.0"
+# https://pypi.org/project/yamale/
+YAMALE_VERSION="4.0.4"
+# https://pypi.org/project/yamllint/
+YAMLLINT_VERSION="1.32.0"
+
+OS="$(go env GOHOSTOS)"
+ARCH="$(go env GOHOSTARCH)"
+
+# Installation helpers
+
+function install_binary() {
+  INSTALL_PATH="${1}"
+  DOWNLOAD_URL="${2}"
+  BINARY_NAME="${3}"
+
+  curl --location "${DOWNLOAD_URL}" --output "${INSTALL_PATH}/${BINARY_NAME}"
+  chmod +x "${INSTALL_PATH}/${BINARY_NAME}"
+}
+
+function install_go() {
+  INSTALL_PATH="${1}"
+  PACKAGE="${2}"
+
+  export GOBIN="${INSTALL_PATH}"
+  go install "${PACKAGE}"
+}
+
+function install_pip() {
+  INSTALL_PATH="${1}"
+  PACKAGE="${2}"
+  COMMAND="${3}"
+
+  source "${INSTALL_PATH}/venv/bin/activate"
+  python3 -m pip install "${PACKAGE}"
+  cp "$(dirname "${0}")/python-runner.sh" "${INSTALL_PATH}/${COMMAND}"
+}
+
+function install_tar_binary() {
+  INSTALL_PATH="${1}"
+  DOWNLOAD_URL="${2}"
+  BINARY_PATH="${3}"
+
+  BINARY_NAME="$(basename "${BINARY_PATH}")"
+
+  if [ "${DOWNLOAD_URL##*.}" = "gz" ]; then
+    TAR_EXTRA_FLAGS="-z"
+  elif [ "${DOWNLOAD_URL##*.}" = "xz" ]; then
+    TAR_EXTRA_FLAGS="-J"
+  else
+    TAR_EXTRA_FLAGS=""
+  fi
+
+  curl --location "${DOWNLOAD_URL}" | tar "$TAR_EXTRA_FLAGS" --extract --touch --transform "s/.*/${BINARY_NAME}/" -C "${INSTALL_PATH}" "${BINARY_PATH}"
+  chmod +x "${INSTALL_PATH}/${BINARY_NAME}"
+}
+
+# Tool-specific installers
+
+function install_aws() {
+  INSTALL_PATH="${1}"
+
+  install_pip "${INSTALL_PATH}" "awscli==${AWSCLI_VERSION}" "aws"
+}
+
+function install_ct() {
+  INSTALL_PATH="${1}"
+
+  install_tar_binary "${INSTALL_PATH}" "https://github.com/helm/chart-testing/releases/download/${CT_VERSION}/chart-testing_${CT_VERSION:1}_${OS}_${ARCH}.tar.gz" "ct"
+  install_pip "${INSTALL_PATH}" "yamale==${YAMALE_VERSION}" "yamale"
+  install_pip "${INSTALL_PATH}" "yamllint==${YAMLLINT_VERSION}" "yamllint"
+}
+
+function install_eksctl() {
+  INSTALL_PATH="${1}"
+
+  install_tar_binary "${INSTALL_PATH}" "https://github.com/weaveworks/eksctl/releases/download/${EKSCTL_VERSION}/eksctl_${OS^}_${ARCH}.tar.gz" "eksctl"
+}
+
+function install_ginkgo() {
+  INSTALL_PATH="${1}"
+
+  install_go "${INSTALL_PATH}" "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
+}
+
+function install_golangci-lint() {
+  INSTALL_PATH="${1}"
+
+  # golangci-lint recommends against installing with `go install`: https://golangci-lint.run/usage/install/#install-from-source
+  install_tar_binary "${INSTALL_PATH}" "https://github.com/golangci/golangci-lint/releases/download/${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}.tar.gz" "golangci-lint-${GOLANGCI_LINT_VERSION:1}-${OS}-${ARCH}/golangci-lint"
+}
+
+function install_helm() {
+  INSTALL_PATH="${1}"
+
+  install_tar_binary "${INSTALL_PATH}" "https://get.helm.sh/helm-${HELM_VERSION}-${OS}-${ARCH}.tar.gz" "${OS}-${ARCH}/helm"
+}
+
+function install_kops() {
+  INSTALL_PATH="${1}"
+
+  install_binary "${INSTALL_PATH}" "https://github.com/kubernetes/kops/releases/download/${KOPS_VERSION}/kops-${OS}-${ARCH}" "kops"
+}
+
+function install_kubetest2() {
+  INSTALL_PATH="${1}"
+
+  install_go "${INSTALL_PATH}" "sigs.k8s.io/kubetest2/...@latest"
+}
+
+function install_mockgen() {
+  INSTALL_PATH="${1}"
+
+  install_go "${INSTALL_PATH}" "github.com/golang/mock/mockgen@${MOCKGEN_VERSION}"
+}
+
+function install_shfmt() {
+  INSTALL_PATH="${1}"
+
+  install_go "${INSTALL_PATH}" "mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION}"
+}
+
+# Utility functions
+
+function create_environment() {
+  INSTALL_PATH="${1}"
+
+  if command -v "python3"; then
+    PYTHON_CMD="python3"
+  else
+    PYTHON_CMD="python"
+  fi
+  VIRTUAL_ENV_DISABLE_PROMPT=1 "${PYTHON_CMD}" -m venv "${INSTALL_PATH}/venv"
+}
+
+function install_tool() {
+  INSTALL_PATH="${1}"
+  TOOL="${2}"
+
+  "install_${TOOL}" "${INSTALL_PATH}"
+}
+
+# Script dispatcher
+
+if [ ! -d "${TOOLS_PATH}/venv" ]; then
+  create_environment "${TOOLS_PATH}"
+fi
+install_tool "${TOOLS_PATH}" "${1}"
diff --git a/hack/verify-all b/hack/tools/python-runner.sh
similarity index 69%
rename from hack/verify-all
rename to hack/tools/python-runner.sh
index e6cabf8811..67c8434eab 100755
--- a/hack/verify-all
+++ b/hack/tools/python-runner.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2019 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,11 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -euo pipefail
+# This script is used as a stub for python commands installed to bin/
+# It activates the venv inside bin/venv/ and then passes through
 
-PKG_ROOT=$(git rev-parse --show-toplevel)
-
-${PKG_ROOT}/hack/verify-gofmt
-${PKG_ROOT}/hack/verify-govet
-${PKG_ROOT}/bin/golangci-lint run --deadline=10m
-${PKG_ROOT}/hack/verify-vendor.sh
+source "$(dirname "${0}")/venv/bin/activate"
+exec "$(basename "${0}")" "$@"
diff --git a/hack/update-gofmt b/hack/update-gofmt
deleted file mode 100755
index dd016e1762..0000000000
--- a/hack/update-gofmt
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -w
diff --git a/hack/update-gomock b/hack/update-gomock
deleted file mode 100755
index ee6600dda4..0000000000
--- a/hack/update-gomock
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-mockgen -package cloud -destination=./pkg/cloud/mock_cloud.go -source pkg/cloud/cloud_interface.go
-mockgen -package cloud -destination=./pkg/cloud/mock_metadata.go -source pkg/cloud/metadata_interface.go
-mockgen -package driver -destination=./pkg/driver/mock_mount.go -source pkg/driver/mount.go
-mockgen -package mounter -destination=./pkg/mounter/mock_mount_windows.go -source pkg/mounter/safe_mounter_windows.go
-
-# Reflection-based mocking for external dependencies
-mockgen -package cloud -destination=./pkg/cloud/mock_ec2.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API
-mockgen -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface
-mockgen -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface
-mockgen -package driver -destination=./pkg/driver/mock_k8s_storagev1.go k8s.io/client-go/kubernetes/typed/storage/v1 VolumeAttachmentInterface,StorageV1Interface
-
-# Fixes "Mounter Type cannot implement 'Mounter' as it has a non-exported method and is defined in a different package"
-# See https://github.com/kubernetes/mount-utils/commit/a20fcfb15a701977d086330b47b7efad51eb608e for context.
-sed -i '/type MockMounter struct {/a \\tmount_utils.Interface' pkg/driver/mock_mount.go
-sed -i '/type MockProxyMounter struct {/a \\tmount.Interface' pkg/mounter/mock_mount_windows.go
diff --git a/hack/update-gomod b/hack/update-gomod
deleted file mode 100755
index d89c2a7edd..0000000000
--- a/hack/update-gomod
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-set -euo pipefail
-set -x
-
-VERSION=${1#"v"}
-if [ -z "$VERSION" ]; then
-    echo "Must specify version!"
-    exit 1
-fi
-MODS=($(
-    curl -sS https://raw.githubusercontent.com/kubernetes/kubernetes/v${VERSION}/go.mod |
-    sed -n 's|.*k8s.io/\(.*\) => ./staging/src/k8s.io/.*|k8s.io/\1|p'
-))
-echo $MODS
-for MOD in "${MODS[@]}"; do
-    echo $MOD
-    V=$(
-        go mod download -json "${MOD}@kubernetes-${VERSION}" |
-        sed -n 's|.*"Version": "\(.*\)".*|\1|p'
-    )
-    go mod edit "-replace=${MOD}=${MOD}@${V}"
-done
diff --git a/hack/update-kustomize.sh b/hack/update-kustomize.sh
new file mode 100755
index 0000000000..03f1d5b8dd
--- /dev/null
+++ b/hack/update-kustomize.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script updates the kustomize templates in deploy/kubernetes/base/ by
+# running `helm template` and stripping the namespace from the output
+
+set -euo pipefail
+
+BIN="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../bin"
+TEMP_DIR=$(mktemp -d)
+trap "rm -rf \"${TEMP_DIR}\"" EXIT
+cp "deploy/kubernetes/base/kustomization.yaml" "${TEMP_DIR}/kustomization.yaml"
+
+"${BIN}/helm" template --output-dir "${TEMP_DIR}" --skip-tests --api-versions 'snapshot.storage.k8s.io/v1' --api-versions 'policy/v1/PodDisruptionBudget' --set 'controller.userAgentExtra=kustomize' kustomize charts/aws-ebs-csi-driver >/dev/null
+rm -rf "deploy/kubernetes/base"
+mv "${TEMP_DIR}/aws-ebs-csi-driver/templates" "deploy/kubernetes/base"
+
+sed -i '/namespace:/d' deploy/kubernetes/base/*
+cp "${TEMP_DIR}/kustomization.yaml" "deploy/kubernetes/base/kustomization.yaml"
diff --git a/hack/update-mockgen.sh b/hack/update-mockgen.sh
new file mode 100755
index 0000000000..5681f223f9
--- /dev/null
+++ b/hack/update-mockgen.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+BIN="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../bin"
+
+# Source-based mocking for internal interfaces
+"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_cloud.go -source pkg/cloud/cloud_interface.go
+"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_metadata.go -source pkg/cloud/metadata_interface.go
+"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_mount.go -source pkg/driver/mount.go
+"${BIN}/mockgen" -package mounter -destination=./pkg/mounter/mock_mount_windows.go -source pkg/mounter/safe_mounter_windows.go
+
+# Reflection-based mocking for external dependencies
+"${BIN}/mockgen" -package cloud -destination=./pkg/cloud/mock_ec2.go github.com/aws/aws-sdk-go/service/ec2/ec2iface EC2API
+"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_client.go -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface
+"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_corev1.go k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface
+"${BIN}/mockgen" -package driver -destination=./pkg/driver/mock_k8s_storagev1.go k8s.io/client-go/kubernetes/typed/storage/v1 VolumeAttachmentInterface,StorageV1Interface
+
+# Fixes "Mounter Type cannot implement 'Mounter' as it has a non-exported method and is defined in a different package"
+# See https://github.com/kubernetes/mount-utils/commit/a20fcfb15a701977d086330b47b7efad51eb608e for context.
+sed -i '/type MockMounter struct {/a \\tmount_utils.Interface' pkg/driver/mock_mount.go
+sed -i '/type MockProxyMounter struct {/a \\tmount.Interface' pkg/mounter/mock_mount_windows.go
diff --git a/hack/verify-gofmt b/hack/verify-gofmt
deleted file mode 100755
index ae0106d65f..0000000000
--- a/hack/verify-gofmt
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-echo "Verifying gofmt"
-
-diff=$(find . -name "*.go" | grep -v "\/vendor\/" | xargs gofmt -s -d 2>&1)
-if [[ -n "${diff}" ]]; then
-  echo "${diff}"
-  echo
-  echo "Please run hack/update-gofmt to fix the issue(s)"
-  exit 1
-fi
-echo "No issue found"
diff --git a/hack/verify-govet b/hack/verify-govet
deleted file mode 100755
index f33dade338..0000000000
--- a/hack/verify-govet
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -euo pipefail
-
-echo "Verifying govet"
-
-go vet $(go list ./... | grep -v vendor)
-
-echo "Done"
diff --git a/hack/verify-kustomize b/hack/verify-update.sh
similarity index 50%
rename from hack/verify-kustomize
rename to hack/verify-update.sh
index ad4fb3cda9..fb0c5791ce 100755
--- a/hack/verify-kustomize
+++ b/hack/verify-update.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2022 The Kubernetes Authors.
+# Copyright 2023 The Kubernetes Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,23 +14,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This script verifies that `make update` does not need to run
+# It does so by creating a temporary copy of the repo and running `make update`
+# in the copy, and then checking if it produces a diff to the local copy
+
 set -euo pipefail
 
-echo "Verifying kustomize"
-diff=$(git status --porcelain=v1)
-if [[ -n "${diff}" ]]; then
-  echo "${diff}"
-  echo
-  echo "Please commit all changes before verifying"
+ROOT="$(dirname "${0}")/../"
+TEST_DIR=$(mktemp -d)
+trap "rm -rf \"${TEST_DIR}\"" EXIT
+cp -rf "${ROOT}/." "${TEST_DIR}"
+
+if ! make -C "${TEST_DIR}" update >/dev/null; then
+  echo "\`make update\` failed!"
   exit 1
 fi
-make generate-kustomize
-echo
-diff=$(git status --porcelain=v1)
-if [[ -n "${diff}" ]]; then
-  echo "${diff}"
-  echo
-  echo "Please run make generate-kustomize to fix the issue(s)"
+
+if ! diff -x bin -r "${TEST_DIR}" "${ROOT}"; then
+  echo "Auto-generation/formatting needs to run!"
+  echo "Run \`make update\` to fix!"
   exit 1
 fi
-echo "No issue found"
diff --git a/hack/verify-vendor.sh b/hack/verify-vendor.sh
deleted file mode 100755
index e849dd46d0..0000000000
--- a/hack/verify-vendor.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2019 The Kubernetes Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-echo "Repo uses 'go mod'."
-if ! (set -x; env GO111MODULE=on go mod tidy); then
-    echo "ERROR: vendor check failed."
-    exit 1
-elif [ "$(git status --porcelain -- go.mod go.sum | wc -l)" -gt 0 ]; then
-    echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':";
-    git diff -- go.mod go.sum
-    exit 1
-elif [ -d vendor ]; then
-    if ! (set -x; env GO111MODULE=on go mod vendor); then
-	echo "ERROR: vendor check failed."
-	exit 1
-    elif [ "$(git status --porcelain -- vendor | wc -l)" -gt 0 ]; then
-	echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"
-	git status -- vendor
-	git diff -- vendor
-	exit 1
-    else
-	echo "Go dependencies and vendor directory up-to-date."
-    fi
-else
-    echo "Go dependencies up-to-date."
-fi