Change Dockerfiles (and make targets) to use cross-compile.

Continue supporting ARCH env for backward compat, but introduce {BUILDARCH, TARGETARCH}.
kubernetes-sigs · Mar 10, 2024 · 204e317 · 204e317
1 parent 8ff71dc
commit 204e317
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 37 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -36,10 +36,10 @@ jobs:
       run: |
         mkdir _output
 
-        docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master -f artifacts/images/agent-build.Dockerfile --build-arg GO_TOOLCHAIN=${{ env.GO_TOOLCHAIN }} --build-arg GO_VERSION=${{ env.GO_VERSION }} --build-arg BASEIMAGE=${{ env.BASEIMAGE }} .
-        docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar
-        docker build -t gcr.io/k8s-staging-kas-network-proxy/proxy-server:master -f artifacts/images/server-build.Dockerfile --build-arg GO_TOOLCHAIN=${{ env.GO_TOOLCHAIN }} --build-arg GO_VERSION=${{ env.GO_VERSION }} --build-arg BASEIMAGE=${{ env.BASEIMAGE }} .
-        docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > _output/konnectivity-server.tar
+        make docker-build/proxy-agent-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
+        docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local > _output/konnectivity-agent.tar
+        make docker-build/proxy-server-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
+        docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local > _output/konnectivity-server.tar
 
     - uses: actions/upload-artifact@v2
       with:
@@ -150,8 +150,8 @@ jobs:
         # preload konnectivity images
         docker load --input konnectivity-server.tar
         docker load --input konnectivity-agent.tar
-        /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server:master --name ${{ env.KIND_CLUSTER_NAME}}
-        /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master --name ${{ env.KIND_CLUSTER_NAME}}
+        /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local --name ${{ env.KIND_CLUSTER_NAME}}
+        /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local --name ${{ env.KIND_CLUSTER_NAME}}
         kubectl apply -f examples/kind/konnectivity-server.yaml
         kubectl apply -f examples/kind/konnectivity-agent-ds.yaml
 

diff --git a/Makefile b/Makefile
@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# ARCH is still supported, but prefer BUILDARCH and TARGETARCH.
 ARCH ?= amd64
+BUILDARCH ?= amd64
+TARGETARCH ?= $(ARCH)
 ARCH_LIST ?= amd64 arm arm64 ppc64le s390x
 RELEASE_ARCH_LIST = amd64 arm64
 # The output type could either be docker (local), or registry.
@@ -201,53 +204,53 @@ docker-push: docker-push/proxy-agent docker-push/proxy-server
 .PHONY: docker-build/proxy-agent
 docker-build/proxy-agent: cmd/agent/main.go proto/agent/agent.pb.go buildx-setup
 	@[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
-	echo "Building proxy-agent for ${ARCH}"
-	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/agent-build.Dockerfile -t ${AGENT_FULL_IMAGE}-$(ARCH):${TAG}
+	echo "Building proxy-agent with ${BUILDARCH} for ${TARGETARCH}"
+	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/agent-build.Dockerfile -t ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-push/proxy-agent
 docker-push/proxy-agent: docker-build/proxy-agent
 	@[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
-	${DOCKER_CMD} push ${AGENT_FULL_IMAGE}-$(ARCH):${TAG}
+	${DOCKER_CMD} push ${AGENT_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-build/proxy-server
 docker-build/proxy-server: cmd/server/main.go proto/agent/agent.pb.go buildx-setup
 	@[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
-	echo "Building proxy-server for ${ARCH}"
-	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/server-build.Dockerfile -t ${SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+	echo "Building proxy-server with ${BUILDARCH} for ${TARGETARCH}"
+	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/server-build.Dockerfile -t ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-push/proxy-server
 docker-push/proxy-server: docker-build/proxy-server
 	@[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
-	${DOCKER_CMD} push ${SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+	${DOCKER_CMD} push ${SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-build/proxy-test-client
 docker-build/proxy-test-client: cmd/test-client/main.go proto/agent/agent.pb.go buildx-setup
 	@[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
-	echo "Building proxy-test-client for ${ARCH}"
-	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-client-build.Dockerfile -t ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG}
+	echo "Building proxy-test-client with ${BUILDARCH} for ${TARGETARCH}"
+	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-client-build.Dockerfile -t ${TEST_CLIENT_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-push/proxy-test-client
 docker-push/proxy-test-client: docker-build/proxy-test-client
 	@[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
-	${DOCKER_CMD} push ${TEST_CLIENT_FULL_IMAGE}-$(ARCH):${TAG}
+	${DOCKER_CMD} push ${TEST_CLIENT_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-build/http-test-server
 docker-build/http-test-server: cmd/test-server/main.go buildx-setup
 	@[ "${TAG}" ] || ( echo "TAG is not set"; exit 1 )
-	echo "Building http-test-server for ${ARCH}"
-	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(ARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg ARCH=$(ARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-server-build.Dockerfile -t ${TEST_SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+	echo "Building http-test-server with ${BUILDARCH} for ${TARGETARCH}"
+	${DOCKER_CMD} buildx build . --pull --output=type=$(OUTPUT_TYPE) --platform linux/$(TARGETARCH) --build-arg GO_TOOLCHAIN=$(GO_TOOLCHAIN) --build-arg GO_VERSION=$(GO_VERSION) --build-arg BUILDARCH=$(BUILDARCH) --build-arg TARGETARCH=$(TARGETARCH) --build-arg BASEIMAGE=$(BASEIMAGE) -f artifacts/images/test-server-build.Dockerfile -t ${TEST_SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 .PHONY: docker-push/http-test-server
 docker-push/http-test-server: docker-build/http-test-server
 	@[ "${DOCKER_CMD}" ] || ( echo "DOCKER_CMD is not set"; exit 1 )
-	${DOCKER_CMD} push ${TEST_SERVER_FULL_IMAGE}-$(ARCH):${TAG}
+	${DOCKER_CMD} push ${TEST_SERVER_FULL_IMAGE}-$(TARGETARCH):${TAG}
 
 ## --------------------------------------
 ## Docker — All ARCH
 ## --------------------------------------
 
 # As `docker buildx` is time and resource consuming, if not necessary, building specific arch images,
-# like `make docker-build-arch-aamd64`, is recommended.
+# like `make docker-build-arch-amd64`, is recommended.
 .PHONY: docker-build-all
 docker-build-all: $(addprefix docker-build-arch-,$(ARCH_LIST))
 
@@ -261,16 +264,16 @@ docker-build-arch-%:
 	$(MAKE) docker-build/proxy-server-$*
 
 docker-build/proxy-agent-%:
-	$(MAKE) ARCH=$* docker-build/proxy-agent
+	$(MAKE) TARGETARCH=$* docker-build/proxy-agent
 
 docker-push/proxy-agent-%:
-	$(MAKE) ARCH=$* docker-push/proxy-agent
+	$(MAKE) TARGETARCH=$* docker-push/proxy-agent
 
 docker-build/proxy-server-%:
-	$(MAKE) ARCH=$* docker-build/proxy-server
+	$(MAKE) TARGETARCH=$* docker-build/proxy-server
 
 docker-push/proxy-server-%:
-	$(MAKE) ARCH=$* docker-push/proxy-server
+	$(MAKE) TARGETARCH=$* docker-push/proxy-server
 
 .PHONY: docker-push-manifest/proxy-agent
 docker-push-manifest/proxy-agent: ## Push the fat manifest docker image.

diff --git a/artifacts/images/agent-build.Dockerfile b/artifacts/images/agent-build.Dockerfile
@@ -1,10 +1,11 @@
 # Build the proxy-agent binary
 
+ARG BUILDARCH
 ARG GO_TOOLCHAIN
 ARG GO_VERSION
 ARG BASEIMAGE
 
-FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder
+FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -26,8 +27,8 @@ COPY proto/  proto/
 
 
 # Build
-ARG ARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-agent sigs.k8s.io/apiserver-network-proxy/cmd/agent
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-agent sigs.k8s.io/apiserver-network-proxy/cmd/agent
 
 FROM ${BASEIMAGE}
 

diff --git a/artifacts/images/server-build.Dockerfile b/artifacts/images/server-build.Dockerfile
@@ -1,10 +1,11 @@
 # Build the proxy-server binary
 
+ARG BUILDARCH
 ARG GO_TOOLCHAIN
 ARG GO_VERSION
 ARG BASEIMAGE
 
-FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder
+FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -25,8 +26,8 @@ COPY cmd/    cmd/
 COPY proto/  proto/
 
 # Build
-ARG ARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-server sigs.k8s.io/apiserver-network-proxy/cmd/server
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-server sigs.k8s.io/apiserver-network-proxy/cmd/server
 
 FROM ${BASEIMAGE}
 

diff --git a/artifacts/images/test-client-build.Dockerfile b/artifacts/images/test-client-build.Dockerfile
@@ -1,10 +1,11 @@
 # Build the client binary
 
+ARG BUILDARCH
 ARG GO_TOOLCHAIN
 ARG GO_VERSION
 ARG BASEIMAGE
 
-FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder
+FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -25,8 +26,8 @@ COPY cmd/    cmd/
 COPY proto/  proto/
 
 # Build
-ARG ARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-test-client sigs.k8s.io/apiserver-network-proxy/cmd/test-client
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o proxy-test-client sigs.k8s.io/apiserver-network-proxy/cmd/test-client
 
 FROM ${BASEIMAGE}
 

diff --git a/artifacts/images/test-server-build.Dockerfile b/artifacts/images/test-server-build.Dockerfile
@@ -1,10 +1,11 @@
 # Build the http test server binary
 
+ARG BUILDARCH
 ARG GO_TOOLCHAIN
 ARG GO_VERSION
 ARG BASEIMAGE
 
-FROM ${GO_TOOLCHAIN}:${GO_VERSION} as builder
+FROM --platform=linux/${BUILDARCH} ${GO_TOOLCHAIN}:${GO_VERSION} as builder
 
 # Copy in the go src
 WORKDIR /go/src/sigs.k8s.io/apiserver-network-proxy
@@ -24,8 +25,8 @@ COPY pkg/    pkg/
 COPY cmd/    cmd/
 
 # Build
-ARG ARCH
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o http-test-server sigs.k8s.io/apiserver-network-proxy/cmd/test-server
+ARG TARGETARCH
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -mod=vendor -v -a -ldflags '-extldflags "-static"' -o http-test-server sigs.k8s.io/apiserver-network-proxy/cmd/test-server
 
 FROM ${BASEIMAGE}
 

diff --git a/examples/kind/konnectivity-agent-ds.yaml b/examples/kind/konnectivity-agent-ds.yaml
@@ -36,7 +36,8 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: konnectivity-agent-container
-        image: gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master
+        image: gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local
+        imagePullPolicy: Never
         resources:
           requests:
             cpu: 50m

diff --git a/examples/kind/konnectivity-server.yaml b/examples/kind/konnectivity-server.yaml
@@ -55,7 +55,8 @@ spec:
       hostNetwork: true
       containers:
       - name: konnectivity-server-container
-        image: gcr.io/k8s-staging-kas-network-proxy/proxy-server:master
+        image: gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local
+        imagePullPolicy: Never
         resources:
           requests:
             cpu: 1m