From 0133a6226cd86f5445462b589ec7b902206f97ea Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 19 May 2022 03:31:31 +0200 Subject: [PATCH] build-sys: Fix `make clobber` Get it to actually clean up all the things that it should. Signed-off-by: Luke Shumaker --- Makefile | 4 ++++ _cxx/envoy.mk | 8 ++++---- build-aux/check.mk | 9 +++++++++ build-aux/docker.mk | 3 +-- build-aux/generate.mk | 11 ++++++++++- build-aux/lint.mk | 2 ++ build-aux/main.mk | 26 +++++++++++++++++++++++++- builder/builder.mk | 8 +++++++- 8 files changed, 62 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 4215b3fc8e7..a4d71be24a7 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,10 @@ BUILD_ARCH ?= linux/amd64 # Bootstrapping the build env ifneq ($(MAKECMDGOALS),$(OSS_HOME)/build-aux/go-version.txt) + ifeq ($(filter shortest-stem,$(.FEATURES)),) + $(info [make] MAKE_VERSION=$(MAKE_VERSION)) + $(error A version of Make that supports the 'shortest-stem' feature, which should be 3.82 or later) + endif $(_prelude.go.ensure) ifneq ($(filter $(shell go env GOROOT),$(subst :, ,$(shell go env GOPATH))),) $(error Your $$GOPATH (where *your* Go stuff goes) and $$GOROOT (where Go *itself* is installed) are both set to the same directory ($(shell go env GOROOT)); it is remarkable that it has not blown up catastrophically before now) diff --git a/_cxx/envoy.mk b/_cxx/envoy.mk index ba3422cff88..0940c59307e 100644 --- a/_cxx/envoy.mk +++ b/_cxx/envoy.mk @@ -291,7 +291,7 @@ update-base: $(OSS_HOME)/docker/base-envoy/envoy-static $(OSS_HOME)/docker/base- @PS4=; set -ex; { \ if [ '$(ENVOY_COMMIT)' != '-' ] && docker pull $(ENVOY_FULL_DOCKER_TAG); then \ echo 'Already up-to-date: $(ENVOY_FULL_DOCKER_TAG)'; \ - ENVOY_VERSION_OUTPUT=$$(docker run -it --entrypoint envoy-static $(ENVOY_FULL_DOCKER_TAG) --version | grep "version:"); \ + ENVOY_VERSION_OUTPUT=$$(docker run --rm -it --entrypoint envoy-static $(ENVOY_FULL_DOCKER_TAG) --version | grep "version:"); \ ENVOY_VERSION_EXPECTED="envoy-static .*version:.* $(ENVOY_COMMIT)/.*"; \ if ! echo "$$ENVOY_VERSION_OUTPUT" | grep "$$ENVOY_VERSION_EXPECTED"; then \ { set +x; } &>/dev/null; \ @@ -309,7 +309,7 @@ update-base: $(OSS_HOME)/docker/base-envoy/envoy-static $(OSS_HOME)/docker/base- fi; \ docker build --build-arg=base=$$(cat $(OSS_HOME)/_cxx/envoy-build-image.txt) -f $(OSS_HOME)/docker/base-envoy/Dockerfile -t $(ENVOY_FULL_DOCKER_TAG) $(OSS_HOME)/docker/base-envoy; \ if [ '$(ENVOY_COMMIT)' != '-' ]; then \ - ENVOY_VERSION_OUTPUT=$$(docker run -it --entrypoint envoy-static $(ENVOY_FULL_DOCKER_TAG) --version | grep "version:"); \ + ENVOY_VERSION_OUTPUT=$$(docker run --rm -it --entrypoint envoy-static $(ENVOY_FULL_DOCKER_TAG) --version | grep "version:"); \ ENVOY_VERSION_EXPECTED="envoy-static .*version:.* $(ENVOY_COMMIT)/.*"; \ if ! echo "$$ENVOY_VERSION_OUTPUT" | grep "$$ENVOY_VERSION_EXPECTED"; then \ { set +x; } &>/dev/null; \ @@ -326,7 +326,7 @@ update-base: $(OSS_HOME)/docker/base-envoy/envoy-static $(OSS_HOME)/docker/base- @PS4=; set -ex; { \ if [ '$(ENVOY_COMMIT)' != '-' ] && docker pull $(ENVOY_DOCKER_TAG); then \ echo 'Already up-to-date: $(ENVOY_DOCKER_TAG)'; \ - ENVOY_VERSION_OUTPUT=$$(docker run -it --entrypoint envoy-static-stripped $(ENVOY_DOCKER_TAG) --version | grep "version:"); \ + ENVOY_VERSION_OUTPUT=$$(docker run --rm -it --entrypoint envoy-static-stripped $(ENVOY_DOCKER_TAG) --version | grep "version:"); \ ENVOY_VERSION_EXPECTED="envoy-static-stripped .*version:.* $(ENVOY_COMMIT)/.*"; \ if ! echo "$$ENVOY_VERSION_OUTPUT" | grep "$$ENVOY_VERSION_EXPECTED"; then \ { set +x; } &>/dev/null; \ @@ -344,7 +344,7 @@ update-base: $(OSS_HOME)/docker/base-envoy/envoy-static $(OSS_HOME)/docker/base- fi; \ docker build -f $(OSS_HOME)/docker/base-envoy/Dockerfile.stripped -t $(ENVOY_DOCKER_TAG) $(OSS_HOME)/docker/base-envoy; \ if [ '$(ENVOY_COMMIT)' != '-' ]; then \ - ENVOY_VERSION_OUTPUT=$$(docker run -it --entrypoint envoy-static-stripped $(ENVOY_DOCKER_TAG) --version | grep "version:"); \ + ENVOY_VERSION_OUTPUT=$$(docker run --rm -it --entrypoint envoy-static-stripped $(ENVOY_DOCKER_TAG) --version | grep "version:"); \ ENVOY_VERSION_EXPECTED="envoy-static-stripped .*version:.* $(ENVOY_COMMIT)/.*"; \ if ! echo "$$ENVOY_VERSION_OUTPUT" | grep "$$ENVOY_VERSION_EXPECTED"; then \ { set +x; } &>/dev/null; \ diff --git a/build-aux/check.mk b/build-aux/check.mk index cf8deed5769..bccd2877b1d 100644 --- a/build-aux/check.mk +++ b/build-aux/check.mk @@ -16,6 +16,7 @@ push-pytest-images: docker/kat-server.docker.push.remote test_svcs = auth shadow stats $(foreach svc,$(test_svcs),docker/.test-$(svc).docker.stamp): docker/.%.docker.stamp: docker/%/Dockerfile FORCE docker build --iidfile=$@ $( $@ +.clean/is-dir += docker/kat-server +docker/kat-server.img.tar.clean: docker/kat-server.clean # # Helm tests @@ -77,3 +80,9 @@ endif $(tools/kubectl) --kubeconfig=$(DEV_KUBECONFIG) --namespace=emissary-system wait --timeout=90s --for=condition=available Deployments/emissary-apiext cd $(chart_dir) && KUBECONFIG=$(DEV_KUBECONFIG) $(abspath $(tools/ct)) install --config=./ct.yaml .PHONY: test-chart + +# +# Other + +.clean/is-dir += .pytest_cache +clean: .pytest_cache.clean .coverage.clean diff --git a/build-aux/docker.mk b/build-aux/docker.mk index 9a9bfe337b6..be25e17575a 100644 --- a/build-aux/docker.mk +++ b/build-aux/docker.mk @@ -220,8 +220,7 @@ docker.LOCALHOST = $(if $(filter darwin,$(GOHOSTOS)),host.docker.internal,localh %.docker.clean: $(addprefix %.docker.clean.,$(_docker.clean.groups)) if [ -e $*.docker ]; then docker image rm "$$(cat $*.docker)" || true; fi -# It "shouldn't" need the ".*" suffix, but it makes it easier to hook in with things like a .stamp file - rm -f $*.docker $*.docker.* + rm -f $*.docker $(*D)/.$(*F).docker.stamp .PHONY: %.docker.clean # Evaluate _docker.tag.rule with _docker.tag.group=TAG_GROUPNAME for diff --git a/build-aux/generate.mk b/build-aux/generate.mk index 1ef9aec9f4b..be5ce7c017a 100644 --- a/build-aux/generate.mk +++ b/build-aux/generate.mk @@ -366,12 +366,14 @@ $(OSS_HOME)/k8s-config/%/helm-expanded.yaml: \ $(OSS_HOME)/k8s-config/%/values.yaml \ $(boguschart_dir) helm template --namespace=$(helm.namespace.$*) --values=$(@D)/values.yaml $(or $(helm.name.$*),$*) $(boguschart_dir) >$@ - $(OSS_HOME)/k8s-config/%/output.yaml: \ $(OSS_HOME)/k8s-config/%/helm-expanded.yaml \ $(OSS_HOME)/k8s-config/%/require.yaml \ $(tools/filter-yaml) $(tools/filter-yaml) $(filter %/helm-expanded.yaml,$^) $(filter %/require.yaml,$^) >$@ +k8s-config.clean: + rm -f k8s-config/*/helm-expanded.yaml k8s-config/*/output.yaml +clean: k8s-config.clean $(OSS_HOME)/manifests/emissary/%.yaml.in: $(OSS_HOME)/k8s-config/%/output.yaml cp $< $@ @@ -390,14 +392,21 @@ $(OSS_HOME)/python/tests/integration/manifests/rbac_namespace_scope.yaml: $(OSS_ $(OSS_HOME)/build-aux/pip-show.txt: docker/base-pip.docker.tag.local docker run --rm "$$(cat docker/base-pip.docker)" sh -c 'pip freeze --exclude-editable | cut -d= -f1 | xargs pip show' > $@ +clean: build-aux/pip-show.txt.clean $(OSS_HOME)/build-aux/go-version.txt: docker/base-python/Dockerfile sed -En 's,.*https://dl\.google\.com/go/go([0-9a-z.-]*)\.linux-amd64\.tar\.gz.*,\1,p' < $< > $@ +clean: build-aux/go-version.txt.clean + $(OSS_HOME)/build-aux/py-version.txt: docker/base-python/Dockerfile { grep -o 'python3=\S*' | cut -d= -f2; } < $< > $@ +clean: build-aux/py-version.txt.clean $(OSS_HOME)/build-aux/go1%.src.tar.gz: curl -o $@ --fail -L https://dl.google.com/go/$(@F) +build-aux/go.src.tar.gz.clean: + rm -f build-aux/go1*.src.tar.gz +clobber: build-aux/go.src.tar.gz.clean $(OSS_HOME)/DEPENDENCIES.md: $(tools/go-mkopensource) $(tools/py-mkopensource) $(OSS_HOME)/build-aux/go-version.txt $(OSS_HOME)/build-aux/pip-show.txt $(MAKE) $(OSS_HOME)/build-aux/go$$(cat $(OSS_HOME)/build-aux/go-version.txt).src.tar.gz diff --git a/build-aux/lint.mk b/build-aux/lint.mk index 49329f2334e..064c23455f4 100644 --- a/build-aux/lint.mk +++ b/build-aux/lint.mk @@ -22,6 +22,8 @@ lint: exit $$r; \ } .PHONY: lint +.clean/is-dir += .mypy_cache +clean: .dmypy.json.clean .mypy_cache.clean golint: $(tools/golangci-lint) @PS4=; set -x; r=0; { \ diff --git a/build-aux/main.mk b/build-aux/main.mk index a0a587cf74b..8cc957725fd 100644 --- a/build-aux/main.mk +++ b/build-aux/main.mk @@ -8,8 +8,9 @@ include build-aux/var.mk .PHONY: %.clean # Also provide a default *.clean implementation +.clean/is-dir ?= %.clean: - rm -f $* + rm -f $(if $(filter $*,$(.clean/is-dir)),-r) $* # For files that should only-maybe update when the rule runs, put ".stamp" on # the left-side of the ":", and just go ahead and update it within the rule. @@ -29,6 +30,10 @@ _ocibuild-images += kat-server $(foreach img,$(_ocibuild-images),docker/.$(img).docker.stamp): docker/.%.docker.stamp: docker/%.img.tar docker load < $< docker inspect $$(bsdtar xfO $< manifest.json|jq -r '.[0].RepoTags[0]') --format='{{.Id}}' > $@ +clean: $(foreach img,$(_ocibuild-images),docker/$(img).img.tar.clean) + +%.img.tar.clean: %.docker.clean + rm -f $*.img.tar $(*D)/.$(*F).img.tar.stamp $(*D)/$(*F).*.layer.tar # # Specific rules @@ -44,6 +49,7 @@ $(foreach img,$(_ocibuild-images),docker/.$(img).docker.stamp): docker/.%.docker # that don't need Emissary-specif stuff. docker/.base.img.tar.stamp: FORCE $(tools/crane) docker/base-python/Dockerfile $(tools/crane) pull $(shell gawk '$$1 == "FROM" { print $$2; quit; }' < docker/base-python/Dockerfile) $@ || test -e $@ +clobber: docker/base.img.tar.clean # base-python: Base OS, plus some Emissary-specific setup of # low-level/expensive pieces of the Python environment. This does NOT @@ -59,6 +65,7 @@ docker/.base.img.tar.stamp: FORCE $(tools/crane) docker/base-python/Dockerfile # (`apk add`, libc-specific compilation...). docker/.base-python.docker.stamp: FORCE docker/base-python/Dockerfile docker/base-python.docker.gen docker/base-python.docker.gen >$@ +clobber: docker/base-python.docker.clean # base-pip: base-python, but with requirements.txt installed. # @@ -73,19 +80,23 @@ docker/.base-python.docker.stamp: FORCE docker/base-python/Dockerfile docker/bas # $(tools/py-list-deps) --include-dev python/ | $(tools/write-ifchanged) $@ python/requirements.in: $(tools/py-list-deps) $(tools/write-ifchanged) FORCE set -o pipefail; $(tools/py-list-deps) --no-include-dev python/ | $(tools/write-ifchanged) $@ +clean: python/requirements.in.clean python/.requirements.txt.stamp: python/requirements.in docker/base-python.docker.tag.local # The --interactive is so that stdin gets passed through; otherwise Docker closes stdin. set -ex -o pipefail; { \ docker run --platform="$(BUILD_ARCH)" --rm --interactive "$$(cat docker/base-python.docker)" sh -c 'tar xf - && find ~/.cache/pip -name "maturin-*.whl" -exec pip install --no-deps {} + >&2 && pip-compile --allow-unsafe --no-build-isolation -q >&2 && cat requirements.txt' \ < <(bsdtar -cf - -C $(@D) requirements.in requirements.txt) \ > $@; } +clean: python/.requirements.txt.stamp.clean python/requirements.txt: python/%: python/.%.stamp $(tools/copy-ifchanged) $(tools/copy-ifchanged) $< $@ .PRECIOUS: python/requirements.txt docker/base-pip/requirements.txt: python/requirements.txt $(tools/copy-ifchanged) $(tools/copy-ifchanged) $< $@ +clean: docker/base-pip/requirements.txt.clean docker/.base-pip.docker.stamp: docker/.%.docker.stamp: docker/%/Dockerfile docker/%/requirements.txt docker/base-python.docker.tag.local docker build --platform="$(BUILD_ARCH)" --build-arg=from="$$(sed -n 2p docker/base-python.docker.tag.local)" --iidfile=$@ $($(dst)$(NL))) + +# +# Destructive rules + +clobber: clean + +.clean/is-dir += build-output +clean: build-output.clean + +.clean/is-dir += python/ambassador.egg-info +python.clean: %.clean: python/ambassador.egg-info.clean + find $* -name __pycache__ -exec rm -rf -- {} + +clean: python.clean diff --git a/builder/builder.mk b/builder/builder.mk index 30e177b2b48..ce8e3d80996 100644 --- a/builder/builder.mk +++ b/builder/builder.mk @@ -205,6 +205,7 @@ python/ambassador.version: $(tools/write-ifchanged) FORCE echo $(patsubst v%,%,$(VERSION)); \ git rev-parse HEAD; \ } | $(tools/write-ifchanged) $@ +clean: python/ambassador.version.clean # Give Make a hint about which pattern rules to apply. Honestly, I'm # not sure why Make isn't figuring it out on its own, but it isn't. @@ -224,6 +225,8 @@ docker/.base-envoy.docker.stamp: FORCE fi; \ echo $(ENVOY_DOCKER_TAG) >$@; \ } +clobber: docker/base-envoy.docker.clean + docker/.$(LCNAME).docker.stamp: %/.$(LCNAME).docker.stamp: %/base.docker.tag.local %/base-envoy.docker.tag.local %/base-pip.docker.tag.local python/ambassador.version $(BUILDER_HOME)/Dockerfile $(OSS_HOME)/build-aux/py-version.txt $(tools/dsum) FORCE @printf "${CYN}==> ${GRN}Building image ${BLU}$(LCNAME)${END}\n" @printf " ${BLU}base=$$(sed -n 2p $*/base.docker.tag.local)${END}\n" @@ -237,6 +240,7 @@ docker/.$(LCNAME).docker.stamp: %/.$(LCNAME).docker.stamp: %/base.docker.tag.loc --build-arg=builderbase="$$(sed -n 2p $*/base-pip.docker.tag.local)" \ --build-arg=py_version="$$(cat build-aux/py-version.txt)" \ --iidfile=$@; } +clean: docker/$(LCNAME).docker.clean REPO=$(BUILDER_NAME) @@ -341,7 +345,7 @@ build-output/bin/envoy: docker/base-envoy.docker.tag.local mkdir -p $(@D) { \ echo '#!/bin/bash'; \ - echo "docker run -v $(OSS_HOME):$(OSS_HOME) -v /var/:/var/ -v /tmp/:/tmp/ -t --entrypoint /usr/local/bin/envoy-static-stripped $$(cat docker/base-envoy.docker) \"\$$@\""; \ + echo "docker run --rm -v $(OSS_HOME):$(OSS_HOME) -v /var/:/var/ -v /tmp/:/tmp/ -t --entrypoint /usr/local/bin/envoy-static-stripped $$(cat docker/base-envoy.docker) \"\$$@\""; \ } > $@ chmod +x $@ @@ -374,6 +378,8 @@ $(OSS_HOME)/venv: python/requirements.txt python/requirements-dev.txt $@/bin/pip3 install -r python/requirements.txt $@/bin/pip3 install -r python/requirements-dev.txt $@/bin/pip3 install -e $(OSS_HOME)/python +.clean/is-dir += venv +clobber: venv.clean GOTEST_ARGS ?= -race -count=1 -timeout 30m GOTEST_ARGS += -parallel=150 # The ./pkg/envoy-control-plane/cache/v{2,3}/ tests require high parallelism to reliably work