From 66b37fc8649c441017d8528ad2de4b5eca650fdd Mon Sep 17 00:00:00 2001 From: Povilas Versockas Date: Mon, 11 May 2020 06:50:53 +0300 Subject: [PATCH] vpa: add multiarch builds --- vertical-pod-autoscaler/builder/Dockerfile | 3 ++ .../pkg/admission-controller/.gitignore | 5 ++ .../pkg/admission-controller/Dockerfile | 4 +- .../pkg/admission-controller/Makefile | 53 +++++++++++++++---- .../pkg/recommender/.gitignore | 5 ++ .../pkg/recommender/Dockerfile | 3 +- .../pkg/recommender/Makefile | 51 ++++++++++++++---- .../input/spec/spec_client_test_util.go | 6 ++- .../pkg/updater/.gitignore | 5 ++ .../pkg/updater/Dockerfile | 3 +- vertical-pod-autoscaler/pkg/updater/Makefile | 52 ++++++++++++++---- 11 files changed, 154 insertions(+), 36 deletions(-) diff --git a/vertical-pod-autoscaler/builder/Dockerfile b/vertical-pod-autoscaler/builder/Dockerfile index f778ed9e5278..3e33d48dd66a 100644 --- a/vertical-pod-autoscaler/builder/Dockerfile +++ b/vertical-pod-autoscaler/builder/Dockerfile @@ -18,6 +18,9 @@ LABEL maintainer="Beata Skiba " ENV GOPATH /gopath/ ENV PATH $GOPATH/bin:$PATH +ARG GOARCH +ARG LDFLAGS + RUN go version RUN go get github.com/tools/godep RUN godep version diff --git a/vertical-pod-autoscaler/pkg/admission-controller/.gitignore b/vertical-pod-autoscaler/pkg/admission-controller/.gitignore index 5c528967c904..03f13774a440 100644 --- a/vertical-pod-autoscaler/pkg/admission-controller/.gitignore +++ b/vertical-pod-autoscaler/pkg/admission-controller/.gitignore @@ -1,2 +1,7 @@ # Admission Controller binary admission-controller +admission-controller-amd64 +admission-controller-arm64 +admission-controller-arm +admission-controller-ppc64le +admission-controller-s390x diff --git a/vertical-pod-autoscaler/pkg/admission-controller/Dockerfile b/vertical-pod-autoscaler/pkg/admission-controller/Dockerfile index 768d04316c6b..51da41ff2c99 100644 --- a/vertical-pod-autoscaler/pkg/admission-controller/Dockerfile +++ b/vertical-pod-autoscaler/pkg/admission-controller/Dockerfile @@ -14,8 +14,8 @@ FROM gcr.io/distroless/static:latest MAINTAINER Tomasz Kulczynski "tkulczynski@google.com" - -copy admission-controller / +ARG ARCH +copy admission-controller-$ARCH /admission-controller ENTRYPOINT ["/admission-controller"] CMD ["--v=4", "--stderrthreshold=info"] diff --git a/vertical-pod-autoscaler/pkg/admission-controller/Makefile b/vertical-pod-autoscaler/pkg/admission-controller/Makefile index fae6cb29b2c1..756563e5e765 100644 --- a/vertical-pod-autoscaler/pkg/admission-controller/Makefile +++ b/vertical-pod-autoscaler/pkg/admission-controller/Makefile @@ -8,6 +8,10 @@ ENVVAR=CGO_ENABLED=0 $(TEST_ENVVAR) GOOS?=linux COMPONENT=admission-controller FULL_COMPONENT=vpa-${COMPONENT} +ARCH?=amd64 + +ALL_ARCHITECTURES=amd64 arm arm64 ppc64le s390x +export DOCKER_CLI_EXPERIMENTAL=enabled build: clean $(ENVVAR) GOOS=$(GOOS) go build ./... @@ -16,13 +20,21 @@ build: clean build-binary: clean $(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT} -build-binary-with-vendor: clean - $(ENVVAR) GOOS=$(GOOS) go build -mod vendor -o ${COMPONENT} +.PHONY: build-binary-with-vendor +build-binary-with-vendor: $(addprefix build-binary-with-vendor-,$(ALL_ARCHITECTURES)) clean + +.PHONY: build-binary-with-vendor-* +build-binary-with-vendor-%: + $(ENVVAR) GOARCH=$* GOOS=$(GOOS) go build -mod vendor -o ${COMPONENT}-$* test-unit: clean build $(TEST_ENVVAR) go test --test.short -race ./... $(FLAGS) -docker-build: +.PHONY: docker-build +docker-build: $(addprefix docker-build-,$(ALL_ARCHITECTURES)) + +.PHONY: docker-build-* +docker-build-%: ifndef REGISTRY ERR = $(error REGISTRY is undefined) $(ERR) @@ -31,9 +43,16 @@ ifndef TAG ERR = $(error TAG is undefined) $(ERR) endif - docker build --pull -t ${REGISTRY}/${FULL_COMPONENT}:${TAG} . + docker build --pull -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} --build-arg ARCH=$* . + +.PHONY: docker-push +docker-push: $(addprefix sub-push-,$(ALL_ARCHITECTURES)) push-multi-arch; + +.PHONY: sub-push-* +sub-push-%: docker-build-% do-push-% ; -docker-push: +.PHONY: do-push-* +do-push-%: ifndef REGISTRY ERR = $(error REGISTRY is undefined) $(ERR) @@ -42,19 +61,33 @@ ifndef TAG ERR = $(error TAG is undefined) $(ERR) endif - docker push ${REGISTRY}/${FULL_COMPONENT}:${TAG} + docker push ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} + +.PHONY: push-multi-arch +push-multi-arch: + docker manifest create --amend $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(shell echo $(ALL_ARCHITECTURES) | sed -e "s~[^ ]*~$(REGISTRY)/${FULL_COMPONENT}\-&:$(TAG)~g") + @for arch in $(ALL_ARCHITECTURES); do docker manifest annotate --arch $${arch} $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(REGISTRY)/${FULL_COMPONENT}-$${arch}:${TAG}; done + docker manifest push --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG) docker-builder: docker build -t vpa-autoscaling-builder ../../builder -build-in-docker: clean docker-builder - docker run -v `pwd`/../..:/gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler && make build-binary-with-vendor -C pkg/admission-controller' +.PHONY: build-in-docker +build-in-docker: $(addprefix build-in-docker-,$(ALL_ARCHITECTURES)) + +.PHONY: build-in-docker-* +build-in-docker-%: clean docker-builder + docker run -v `pwd`/../..:/gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler && make build-binary-with-vendor-$* -C pkg/admission-controller' + +.PHONY: release release: build-in-docker docker-build docker-push @echo "Full in-docker release ${FULL_COMPONENT}:${TAG} completed" -clean: - rm -f ${COMPONENT} +clean: $(addprefix clean-,$(ALL_ARCHITECTURES)) + +clean-%: + rm -f ${COMPONENT}-$* format: test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -d {} + | tee /dev/stderr)" || \ diff --git a/vertical-pod-autoscaler/pkg/recommender/.gitignore b/vertical-pod-autoscaler/pkg/recommender/.gitignore index 7e45a7511de5..fe7fcf3d0850 100644 --- a/vertical-pod-autoscaler/pkg/recommender/.gitignore +++ b/vertical-pod-autoscaler/pkg/recommender/.gitignore @@ -1,2 +1,7 @@ # Recommender binary recommender +recommender-amd64 +recommender-arm64 +recommender-arm +recommender-ppc64le +recommender-s390x diff --git a/vertical-pod-autoscaler/pkg/recommender/Dockerfile b/vertical-pod-autoscaler/pkg/recommender/Dockerfile index 53a065d55566..6e5d172824b8 100644 --- a/vertical-pod-autoscaler/pkg/recommender/Dockerfile +++ b/vertical-pod-autoscaler/pkg/recommender/Dockerfile @@ -15,7 +15,8 @@ FROM gcr.io/distroless/static:latest MAINTAINER Krzysztof Grygiel "kgrygiel@google.com" -COPY recommender / +ARG ARCH +COPY recommender-$ARCH /recommender ENTRYPOINT ["/recommender"] CMD ["--v=4", "--stderrthreshold=info", "--prometheus-address=http://prometheus.monitoring.svc"] diff --git a/vertical-pod-autoscaler/pkg/recommender/Makefile b/vertical-pod-autoscaler/pkg/recommender/Makefile index aafea98093eb..f921fe5a82f6 100644 --- a/vertical-pod-autoscaler/pkg/recommender/Makefile +++ b/vertical-pod-autoscaler/pkg/recommender/Makefile @@ -8,6 +8,10 @@ ENVVAR=CGO_ENABLED=0 $(TEST_ENVVAR) GOOS?=linux COMPONENT=recommender FULL_COMPONENT=vpa-${COMPONENT} +ARCH?=amd64 + +ALL_ARCHITECTURES=amd64 arm arm64 ppc64le s390x +export DOCKER_CLI_EXPERIMENTAL=enabled build: clean $(ENVVAR) GOOS=$(GOOS) go build ./... @@ -16,13 +20,21 @@ build: clean build-binary: clean $(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT} -build-binary-with-vendor: clean - $(ENVVAR) GOOS=$(GOOS) go build -mod vendor -o ${COMPONENT} +.PHONY: build-binary-with-vendor +build-binary-with-vendor: $(addprefix build-binary-with-vendor-,$(ALL_ARCHITECTURES)) clean + +.PHONY: build-binary-with-vendor-* +build-binary-with-vendor-%: + $(ENVVAR) GOARCH=$* GOOS=$(GOOS) go build -mod vendor -o ${COMPONENT}-$* test-unit: clean build $(TEST_ENVVAR) go test --test.short -race ./... $(FLAGS) -docker-build: +.PHONY: docker-build +docker-build: $(addprefix docker-build-,$(ALL_ARCHITECTURES)) + +.PHONY: docker-build-* +docker-build-%: ifndef REGISTRY ERR = $(error REGISTRY is undefined) $(ERR) @@ -31,9 +43,16 @@ ifndef TAG ERR = $(error TAG is undefined) $(ERR) endif - docker build --pull -t ${REGISTRY}/${FULL_COMPONENT}:${TAG} . + docker build --pull -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} --build-arg ARCH=$* . + +.PHONY: docker-push +docker-push: $(addprefix sub-push-,$(ALL_ARCHITECTURES)) push-multi-arch; -docker-push: +.PHONY: sub-push-* +sub-push-%: docker-build-% do-push-% ; + +.PHONY: do-push-* +do-push-%: ifndef REGISTRY ERR = $(error REGISTRY is undefined) $(ERR) @@ -42,19 +61,31 @@ ifndef TAG ERR = $(error TAG is undefined) $(ERR) endif - docker push ${REGISTRY}/${FULL_COMPONENT}:${TAG} + docker push ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} + +.PHONY: push-multi-arch +push-multi-arch: + docker manifest create --amend $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(shell echo $(ALL_ARCHITECTURES) | sed -e "s~[^ ]*~$(REGISTRY)/${FULL_COMPONENT}\-&:$(TAG)~g") + @for arch in $(ALL_ARCHITECTURES); do docker manifest annotate --arch $${arch} $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(REGISTRY)/${FULL_COMPONENT}-$${arch}:${TAG}; done + docker manifest push --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG) docker-builder: docker build -t vpa-autoscaling-builder ../../builder -build-in-docker: clean docker-builder - docker run -v `pwd`/../..:/gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler && make build-binary-with-vendor -C pkg/recommender' +.PHONY: build-in-docker +build-in-docker: $(addprefix build-in-docker-,$(ALL_ARCHITECTURES)) + +.PHONY: build-in-docker-* +build-in-docker-%: clean docker-builder + docker run -v `pwd`/../..:/gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler && make build-binary-with-vendor-$* -C pkg/recommender' release: build-in-docker docker-build docker-push @echo "Full in-docker release ${FULL_COMPONENT}:${TAG} completed" -clean: - rm -f ${COMPONENT} +clean: $(addprefix clean-,$(ALL_ARCHITECTURES)) + +clean-%: + rm -f ${COMPONENT}-$* format: test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -d {} + | tee /dev/stderr)" || \ diff --git a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go index bb92feae2b02..6118579a7e9d 100644 --- a/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go +++ b/vertical-pod-autoscaler/pkg/recommender/input/spec/spec_client_test_util.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/mock" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -58,6 +58,7 @@ spec: memory: "1024Mi" cpu: "1000m" ` + const pod2Yaml = ` apiVersion: v1 kind: Pod @@ -121,7 +122,8 @@ func newSpecClientTestCase() *specClientTestCase { podYamls: []string{pod1Yaml, pod2Yaml}, } } -func newTestContainerSpec(podID model.PodID, containerName string, milicores int, memory int) BasicContainerSpec { + +func newTestContainerSpec(podID model.PodID, containerName string, milicores int, memory int64) BasicContainerSpec { containerID := model.ContainerID{ PodID: podID, ContainerName: containerName, diff --git a/vertical-pod-autoscaler/pkg/updater/.gitignore b/vertical-pod-autoscaler/pkg/updater/.gitignore index 9cb108ed9cf9..3834e7718699 100644 --- a/vertical-pod-autoscaler/pkg/updater/.gitignore +++ b/vertical-pod-autoscaler/pkg/updater/.gitignore @@ -1,2 +1,7 @@ # Updater binary updater +updater-amd64 +updater-arm64 +updater-arm +updater-ppc64le +updater-s390x diff --git a/vertical-pod-autoscaler/pkg/updater/Dockerfile b/vertical-pod-autoscaler/pkg/updater/Dockerfile index 8aad9977a01b..631584bc5e04 100644 --- a/vertical-pod-autoscaler/pkg/updater/Dockerfile +++ b/vertical-pod-autoscaler/pkg/updater/Dockerfile @@ -16,7 +16,8 @@ FROM gcr.io/distroless/static:latest MAINTAINER Marcin Wielgus "mwielgus@google.com" -COPY updater / +ARG ARCH +COPY updater-$ARCH /updater ENTRYPOINT ["/updater"] CMD ["--v=4", "--stderrthreshold=info"] diff --git a/vertical-pod-autoscaler/pkg/updater/Makefile b/vertical-pod-autoscaler/pkg/updater/Makefile index d3edb1772906..918d5dae633b 100644 --- a/vertical-pod-autoscaler/pkg/updater/Makefile +++ b/vertical-pod-autoscaler/pkg/updater/Makefile @@ -8,6 +8,10 @@ ENVVAR=CGO_ENABLED=0 $(TEST_ENVVAR) GOOS?=linux COMPONENT=updater FULL_COMPONENT=vpa-${COMPONENT} +ARCH?=amd64 + +ALL_ARCHITECTURES=amd64 arm arm64 ppc64le s390x +export DOCKER_CLI_EXPERIMENTAL=enabled build: clean $(ENVVAR) GOOS=$(GOOS) go build ./... @@ -16,13 +20,21 @@ build: clean build-binary: clean $(ENVVAR) GOOS=$(GOOS) go build -o ${COMPONENT} -build-binary-with-vendor: clean - $(ENVVAR) GOOS=$(GOOS) go build -mod vendor -o ${COMPONENT} +.PHONY: build-binary-with-vendor +build-binary-with-vendor: $(addprefix build-binary-with-vendor-,$(ALL_ARCHITECTURES)) clean + +.PHONY: build-binary-with-vendor-* +build-binary-with-vendor-%: + $(ENVVAR) GOARCH=$* GOOS=$(GOOS) go build -mod vendor -o ${COMPONENT}-$* test-unit: clean build $(TEST_ENVVAR) go test --test.short -race ./... $(FLAGS) -docker-build: +.PHONY: docker-build +docker-build: $(addprefix docker-build-,$(ALL_ARCHITECTURES)) + +.PHONY: docker-build-* +docker-build-%: ifndef REGISTRY ERR = $(error REGISTRY is undefined) $(ERR) @@ -31,9 +43,16 @@ ifndef TAG ERR = $(error TAG is undefined) $(ERR) endif - docker build --pull -t ${REGISTRY}/${FULL_COMPONENT}:${TAG} . + docker build --pull -t ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} --build-arg ARCH=$* . + +.PHONY: docker-push +docker-push: $(addprefix sub-push-,$(ALL_ARCHITECTURES)) push-multi-arch; -docker-push: +.PHONY: sub-push-* +sub-push-%: docker-build-% do-push-% ; + +.PHONY: do-push-* +do-push-%: ifndef REGISTRY ERR = $(error REGISTRY is undefined) $(ERR) @@ -42,19 +61,32 @@ ifndef TAG ERR = $(error TAG is undefined) $(ERR) endif - docker push ${REGISTRY}/${FULL_COMPONENT}:${TAG} + docker push ${REGISTRY}/${FULL_COMPONENT}-$*:${TAG} + +.PHONY: push-multi-arch +push-multi-arch: + docker manifest create --amend $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(shell echo $(ALL_ARCHITECTURES) | sed -e "s~[^ ]*~$(REGISTRY)/${FULL_COMPONENT}\-&:$(TAG)~g") + @for arch in $(ALL_ARCHITECTURES); do docker manifest annotate --arch $${arch} $(REGISTRY)/${FULL_COMPONENT}:$(TAG) $(REGISTRY)/${FULL_COMPONENT}-$${arch}:${TAG}; done + docker manifest push --purge $(REGISTRY)/${FULL_COMPONENT}:$(TAG) docker-builder: docker build -t vpa-autoscaling-builder ../../builder -build-in-docker: clean docker-builder - docker run -v `pwd`/../..:/gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler && make build-binary-with-vendor -C pkg/updater' +.PHONY: build-in-docker +build-in-docker: $(addprefix build-in-docker-,$(ALL_ARCHITECTURES)) +.PHONY: build-in-docker-* +build-in-docker-%: clean docker-builder + docker run -v `pwd`/../..:/gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler vpa-autoscaling-builder:latest bash -c 'cd /gopath/src/k8s.io/autoscaler/vertical-pod-autoscaler && make build-binary-with-vendor-$* -C pkg/updater' + +.PHONY: release release: build-in-docker docker-build docker-push @echo "Full in-docker release ${FULL_COMPONENT}:${TAG} completed" -clean: - rm -f ${COMPONENT} +clean: $(addprefix clean-,$(ALL_ARCHITECTURES)) + +clean-%: + rm -f ${COMPONENT}-$* format: test -z "$$(find . -path ./vendor -prune -type f -o -name '*.go' -exec gofmt -s -d {} + | tee /dev/stderr)" || \