From ebc9b7b067f39306ea7b750d6e2624d2406c44a6 Mon Sep 17 00:00:00 2001 From: David Gageot Date: Sat, 1 Feb 2020 07:41:28 +0100 Subject: [PATCH] Restore pull request #3433 from dgageot/faster-skaffold-builder Signed-off-by: David Gageot --- Makefile | 32 +++++++--- deploy/cloudbuild-release.yaml | 22 ++----- deploy/cloudbuild.yaml | 22 ++----- deploy/skaffold/Dockerfile | 105 ++---------------------------- deploy/skaffold/Dockerfile.deps | 110 ++++++++++++++++++++++++++++++++ hack/check-skaffold-builder.sh | 29 +++++++++ hack/checks.sh | 1 + hack/skaffold-deps-sha1.sh | 21 ++++++ 8 files changed, 201 insertions(+), 141 deletions(-) create mode 100644 deploy/skaffold/Dockerfile.deps create mode 100755 hack/check-skaffold-builder.sh create mode 100755 hack/skaffold-deps-sha1.sh diff --git a/Makefile b/Makefile index 647108848a0..22671cd4905 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,7 @@ GO_LDFLAGS_darwin =" $(GO_LDFLAGS) -extldflags \"$(LDFLAGS_darwin)\"" GO_LDFLAGS_linux =" $(GO_LDFLAGS) -extldflags \"$(LDFLAGS_linux)\"" GO_FILES := $(shell find . -type f -name '*.go' -not -path "./vendor/*") +DEPS_DIGEST := $(shell ./hack/skaffold-deps-sha1.sh) $(BUILD_DIR)/$(PROJECT): $(BUILD_DIR)/$(PROJECT)-$(GOOS)-$(GOARCH) cp $(BUILD_DIR)/$(PROJECT)-$(GOOS)-$(GOARCH) $@ @@ -73,11 +74,14 @@ $(BUILD_DIR)/$(PROJECT)-$(GOOS)-$(GOARCH): generate-statik $(GO_FILES) $(BUILD_D GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=1 go build -tags $(GO_BUILD_TAGS_$(GOOS)) -ldflags $(GO_LDFLAGS_$(GOOS)) -gcflags $(GO_GCFLAGS) -asmflags $(GO_ASMFLAGS) -o $@ $(BUILD_PACKAGE) $(BUILD_DIR)/$(PROJECT)-%-$(GOARCH): generate-statik $(GO_FILES) $(BUILD_DIR) - docker build --build-arg PROJECT=$(REPOPATH) \ + docker build \ + --build-arg PROJECT=$(REPOPATH) \ --build-arg TARGETS=$*/$(GOARCH) \ --build-arg FLAG_LDFLAGS=$(GO_LDFLAGS_$(*)) \ --build-arg FLAG_TAGS=$(GO_BUILD_TAGS_$(*)) \ - -f deploy/cross/Dockerfile -t skaffold/cross . + -f deploy/cross/Dockerfile \ + -t skaffold/cross \ + . docker run --rm --entrypoint sh skaffold/cross -c "cat /build/skaffold*" > $@ %.sha256: % @@ -133,11 +137,12 @@ endif .PHONY: release release: cross $(BUILD_DIR)/VERSION docker build \ - -f deploy/skaffold/Dockerfile \ - --cache-from gcr.io/$(GCP_PROJECT)/skaffold-builder \ --build-arg VERSION=$(VERSION) \ + -f deploy/skaffold/Dockerfile \ + --target release \ -t gcr.io/$(GCP_PROJECT)/skaffold:latest \ - -t gcr.io/$(GCP_PROJECT)/skaffold:$(VERSION) . + -t gcr.io/$(GCP_PROJECT)/skaffold:$(VERSION) \ + . gsutil -m cp $(BUILD_DIR)/$(PROJECT)-* $(GSC_RELEASE_PATH)/ gsutil -m cp $(BUILD_DIR)/VERSION $(GSC_RELEASE_PATH)/VERSION gsutil -m cp -r $(GSC_RELEASE_PATH)/* $(GSC_RELEASE_LATEST) @@ -146,9 +151,10 @@ release: cross $(BUILD_DIR)/VERSION release-build: cross docker build \ -f deploy/skaffold/Dockerfile \ - --cache-from gcr.io/$(GCP_PROJECT)/skaffold-builder \ + --target release \ -t gcr.io/$(GCP_PROJECT)/skaffold:edge \ - -t gcr.io/$(GCP_PROJECT)/skaffold:$(COMMIT) . + -t gcr.io/$(GCP_PROJECT)/skaffold:$(COMMIT) \ + . gsutil -m cp $(BUILD_DIR)/$(PROJECT)-* $(GSC_BUILD_PATH)/ gsutil -m cp -r $(GSC_BUILD_PATH)/* $(GSC_BUILD_LATEST) @@ -156,12 +162,22 @@ release-build: cross clean: rm -rf $(BUILD_DIR) hack/bin +.PHONY: build_deps +build_deps: + docker build \ + -f deploy/skaffold/Dockerfile.deps \ + -t gcr.io/$(GCP_PROJECT)/build_deps:$(DEPS_DIGEST) \ + deploy/skaffold + docker push gcr.io/$(GCP_PROJECT)/build_deps:$(DEPS_DIGEST) + @./hack/check-skaffold-builder.sh + .PHONY: skaffold-builder skaffold-builder: time docker build \ -f deploy/skaffold/Dockerfile \ --target builder \ - -t gcr.io/$(GCP_PROJECT)/skaffold-builder . + -t gcr.io/$(GCP_PROJECT)/skaffold-builder \ + . .PHONY: integration-in-kind integration-in-kind: skaffold-builder diff --git a/deploy/cloudbuild-release.yaml b/deploy/cloudbuild-release.yaml index 94541bbd508..0564619dce2 100644 --- a/deploy/cloudbuild-release.yaml +++ b/deploy/cloudbuild-release.yaml @@ -1,17 +1,6 @@ # using default substitutions, provided by Google Cloud Build # see: https://cloud.google.com/container-builder/docs/configuring-builds/substitute-variable-values#using_default_substitutions steps: - -# Download the latest version of the builder image - - name: 'gcr.io/cloud-builders/docker' - entrypoint: 'bash' - args: - - '-c' - - | - docker pull gcr.io/$PROJECT_ID/skaffold-builder:latest || exit 0 -# until https://github.com/GoogleCloudPlatform/cloud-builders/issues/253 is fixed - -# Rebuild the builder image if necessary - name: 'gcr.io/cloud-builders/docker' args: - 'build' @@ -19,10 +8,6 @@ steps: - 'gcr.io/$PROJECT_ID/skaffold-builder:latest' - '-f' - 'deploy/skaffold/Dockerfile' - - '--cache-from' - - 'gcr.io/$PROJECT_ID/skaffold-builder:latest' - - '--target' - - 'builder' - '.' # Do the go build & push the results to GCS @@ -35,8 +20,13 @@ steps: - 'RELEASE_BUCKET=$_RELEASE_BUCKET' - 'GCP_PROJECT=$PROJECT_ID' +# Check that skaffold is in the image + - name: 'gcr.io/$PROJECT_ID/skaffold:latest' + args: + - 'skaffold' + - 'version' + images: -- 'gcr.io/$PROJECT_ID/skaffold-builder:latest' - 'gcr.io/$PROJECT_ID/skaffold:latest' - 'gcr.io/$PROJECT_ID/skaffold:$TAG_NAME' diff --git a/deploy/cloudbuild.yaml b/deploy/cloudbuild.yaml index 3111bd999ed..379ff01e8be 100644 --- a/deploy/cloudbuild.yaml +++ b/deploy/cloudbuild.yaml @@ -1,17 +1,6 @@ # using default substitutions, provided by Google Cloud Build # see: https://cloud.google.com/container-builder/docs/configuring-builds/substitute-variable-values#using_default_substitutions steps: - -# Download the latest version of the builder image - - name: 'gcr.io/cloud-builders/docker' - entrypoint: 'bash' - args: - - '-c' - - | - docker pull gcr.io/$PROJECT_ID/skaffold-builder:latest || exit 0 -# until https://github.com/GoogleCloudPlatform/cloud-builders/issues/253 is fixed - -# Rebuild the builder image if necessary - name: 'gcr.io/cloud-builders/docker' args: - 'build' @@ -19,10 +8,6 @@ steps: - 'gcr.io/$PROJECT_ID/skaffold-builder:latest' - '-f' - 'deploy/skaffold/Dockerfile' - - '--cache-from' - - 'gcr.io/$PROJECT_ID/skaffold-builder:latest' - - '--target' - - 'builder' - '.' # Do the go build & push the results to GCS @@ -34,8 +19,13 @@ steps: - 'RELEASE_BUCKET=$_RELEASE_BUCKET' - 'GCP_PROJECT=$PROJECT_ID' +# Check that skaffold is in the image + - name: 'gcr.io/$PROJECT_ID/skaffold:edge' + args: + - 'skaffold' + - 'version' + images: -- 'gcr.io/$PROJECT_ID/skaffold-builder:latest' - 'gcr.io/$PROJECT_ID/skaffold:edge' - 'gcr.io/$PROJECT_ID/skaffold:$COMMIT_SHA' diff --git a/deploy/skaffold/Dockerfile b/deploy/skaffold/Dockerfile index c272b7f0fb5..8906341c250 100644 --- a/deploy/skaffold/Dockerfile +++ b/deploy/skaffold/Dockerfile @@ -12,109 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Download kubectl -FROM alpine:3.10 as download-kubectl -ENV KUBECTL_VERSION v1.12.8 -ENV KUBECTL_URL https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -RUN wget -O kubectl "${KUBECTL_URL}" -RUN chmod +x kubectl - -# Download helm -FROM alpine:3.10 as download-helm -ENV HELM_VERSION v2.12.0 -ENV HELM_URL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz -RUN wget -O helm.tar.gz "${HELM_URL}" -RUN tar -xvf helm.tar.gz --strip-components 1 - -# Download kustomize -FROM alpine:3.10 as download-kustomize -ENV KUSTOMIZE_VERSION 2.1.0 -ENV KUSTOMIZE_URL https://github.com/kubernetes-sigs/kustomize/releases/download/v${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64 -RUN wget -O kustomize "${KUSTOMIZE_URL}" -RUN chmod +x kustomize - -# Download kompose -FROM alpine:3.10 as download-kompose -ENV KOMPOSE_VERSION v1.19.0 -ENV KOMPOSE_URL https://github.com/kubernetes/kompose/releases/download/${KOMPOSE_VERSION}/kompose-linux-amd64 -RUN wget -O kompose "${KOMPOSE_URL}" -RUN chmod +x kompose - -# Download container-structure-test -FROM alpine:3.10 as download-container-structure-test -ENV CONTAINER_STRUCTURE_TEST_VERSION v1.5.0 -ENV CONTAINER_STRUCTURE_TEST_URL https://storage.googleapis.com/container-structure-test/${CONTAINER_STRUCTURE_TEST_VERSION}/container-structure-test-linux-amd64 -RUN wget -O container-structure-test "${CONTAINER_STRUCTURE_TEST_URL}" -RUN chmod +x container-structure-test - -# Download kind -FROM alpine:3.10 as download-kind -ENV KIND_VERSION v0.7.0 -ENV KIND_URL https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64 -RUN wget -O kind "${KIND_URL}" -RUN chmod +x kind - -# Download gcloud -FROM alpine:3.10 as download-gcloud -ENV GCLOUD_VERSION 274.0.0 -ENV GCLOUD_URL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz -RUN wget -O gcloud.tar.gz "${GCLOUD_URL}" -RUN tar -zxf gcloud.tar.gz - -# Download bazel -FROM alpine:3.10 as download-bazel -ENV BAZEL_VERSION 2.0.0 -ENV BAZEL_URL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-x86_64 -RUN wget -O bazel "${BAZEL_URL}" -RUN chmod +x bazel - -FROM gcr.io/gcp-runtimes/ubuntu_16_0_4 as runtime_deps - -RUN apt-get update && \ - apt-get install --no-install-recommends --no-install-suggests -y \ - git python unzip && \ - rm -rf /var/lib/apt/lists/* - -COPY --from=docker:18.09.6 /usr/local/bin/docker /usr/local/bin/ -COPY --from=download-kubectl kubectl /usr/local/bin/ -COPY --from=download-helm helm /usr/local/bin/ -COPY --from=download-kustomize kustomize /usr/local/bin/ -COPY --from=download-kompose kompose /usr/local/bin/ -COPY --from=download-container-structure-test container-structure-test /usr/local/bin/ -COPY --from=download-bazel bazel /usr/local/bin/ -COPY --from=download-gcloud google-cloud-sdk/ /google-cloud-sdk/ -COPY --from=download-kind kind /usr/local/bin/ - -# Finish installation of bazel -RUN bazel version - -# Finish installation of gcloud -RUN CLOUDSDK_PYTHON="python2.7" /google-cloud-sdk/install.sh \ - --usage-reporting=false \ - --bash-completion=false \ - --disable-installation-options -ENV PATH=$PATH:/google-cloud-sdk/bin -RUN gcloud auth configure-docker - -FROM runtime_deps as builder -RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \ - build-essential \ - python-setuptools \ - lsb-release \ - openjdk-8-jdk \ - software-properties-common \ - jq \ - apt-transport-https && \ - rm -rf /var/lib/apt/lists/* -COPY --from=golang:1.13 /usr/local/go /usr/local/go -ENV PATH /usr/local/go/bin:/root/go/bin:$PATH +# This base image has to be updated manually after running `make build_deps` +FROM gcr.io/k8s-skaffold/build_deps:0c78248757f1753c4c64a791160d5e2fdd35f157 as builder WORKDIR /skaffold COPY . . -FROM builder as build-skaffold +FROM builder as release ARG VERSION RUN make clean && make out/skaffold-linux-amd64 VERSION=$VERSION && mv out/skaffold-linux-amd64 /usr/bin/skaffold - -FROM runtime_deps as distribution -COPY --from=build-skaffold /usr/bin/skaffold /usr/bin/skaffold -RUN skaffold credits -d /THIRD_PARTY_NOTICES \ No newline at end of file +RUN skaffold credits -d /THIRD_PARTY_NOTICES diff --git a/deploy/skaffold/Dockerfile.deps b/deploy/skaffold/Dockerfile.deps new file mode 100644 index 00000000000..99c572f5274 --- /dev/null +++ b/deploy/skaffold/Dockerfile.deps @@ -0,0 +1,110 @@ +# Copyright 2019 The Skaffold Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Download kubectl +FROM alpine:3.10 as download-kubectl +ENV KUBECTL_VERSION v1.14.10 +ENV KUBECTL_URL https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl +RUN wget -O kubectl "${KUBECTL_URL}" +RUN chmod +x kubectl + +# Download helm +FROM alpine:3.10 as download-helm +ENV HELM_VERSION v2.12.0 +ENV HELM_URL https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz +RUN wget -O helm.tar.gz "${HELM_URL}" +RUN tar -xvf helm.tar.gz --strip-components 1 + +# Download kustomize +FROM alpine:3.10 as download-kustomize +ENV KUSTOMIZE_VERSION 3.5.4 +ENV KUSTOMIZE_URL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz +RUN wget -O kustomize.tar.gz "${KUSTOMIZE_URL}" +RUN tar -xvf kustomize.tar.gz + +# Download kompose +FROM alpine:3.10 as download-kompose +ENV KOMPOSE_VERSION v1.20.0 +ENV KOMPOSE_URL https://github.com/kubernetes/kompose/releases/download/${KOMPOSE_VERSION}/kompose-linux-amd64 +RUN wget -O kompose "${KOMPOSE_URL}" +RUN chmod +x kompose + +# Download container-structure-test +FROM alpine:3.10 as download-container-structure-test +ENV CONTAINER_STRUCTURE_TEST_VERSION v1.8.0 +ENV CONTAINER_STRUCTURE_TEST_URL https://storage.googleapis.com/container-structure-test/${CONTAINER_STRUCTURE_TEST_VERSION}/container-structure-test-linux-amd64 +RUN wget -O container-structure-test "${CONTAINER_STRUCTURE_TEST_URL}" +RUN chmod +x container-structure-test + +# Download kind +FROM alpine:3.10 as download-kind +ENV KIND_VERSION v0.7.0 +ENV KIND_URL https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64 +RUN wget -O kind "${KIND_URL}" +RUN chmod +x kind + +# Download gcloud +FROM alpine:3.10 as download-gcloud +ENV GCLOUD_VERSION 276.0.0 +ENV GCLOUD_URL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz +RUN wget -O gcloud.tar.gz "${GCLOUD_URL}" +RUN tar -zxf gcloud.tar.gz + +# Download bazel +FROM alpine:3.10 as download-bazel +ENV BAZEL_VERSION 2.0.0 +ENV BAZEL_URL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-x86_64 +RUN wget -O bazel "${BAZEL_URL}" +RUN chmod +x bazel + +FROM gcr.io/gcp-runtimes/ubuntu_16_0_4 as runtime_deps + +RUN apt-get update && \ + apt-get install --no-install-recommends --no-install-suggests -y \ + git python unzip && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=docker:18.09.6 /usr/local/bin/docker /usr/local/bin/ +COPY --from=download-kubectl kubectl /usr/local/bin/ +COPY --from=download-helm helm /usr/local/bin/ +COPY --from=download-kustomize kustomize /usr/local/bin/ +COPY --from=download-kompose kompose /usr/local/bin/ +COPY --from=download-container-structure-test container-structure-test /usr/local/bin/ +COPY --from=download-bazel bazel /usr/local/bin/ +COPY --from=download-gcloud google-cloud-sdk/ /google-cloud-sdk/ +COPY --from=download-kind kind /usr/local/bin/ + +# Finish installation of bazel +RUN bazel version + +# Finish installation of gcloud +RUN CLOUDSDK_PYTHON="python2.7" /google-cloud-sdk/install.sh \ + --usage-reporting=false \ + --bash-completion=false \ + --disable-installation-options +ENV PATH=$PATH:/google-cloud-sdk/bin +RUN gcloud auth configure-docker + +FROM runtime_deps +RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y \ + build-essential \ + python-setuptools \ + lsb-release \ + openjdk-8-jdk \ + software-properties-common \ + jq \ + apt-transport-https && \ + rm -rf /var/lib/apt/lists/* +COPY --from=golang:1.13 /usr/local/go /usr/local/go +ENV PATH /usr/local/go/bin:/root/go/bin:$PATH diff --git a/hack/check-skaffold-builder.sh b/hack/check-skaffold-builder.sh new file mode 100755 index 00000000000..6404eee17f8 --- /dev/null +++ b/hack/check-skaffold-builder.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Skaffold Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +DOCKERFILE=deploy/skaffold/Dockerfile +SHA1=$("${DIR}/skaffold-deps-sha1.sh") +UPDATED_DOCKERFILE=$(sed -E "s/FROM (.*):.* as (.*)/FROM \1:${SHA1} as \2/" deploy/skaffold/Dockerfile) +echo "${UPDATED_DOCKERFILE}" > "${DOCKERFILE}" + +if [[ ! -z $(git status --porcelain -- "${DOCKERFILE}") ]]; then + echo "Please commit ${DOCKERFILE}" + exit 1 +fi \ No newline at end of file diff --git a/hack/checks.sh b/hack/checks.sh index 0d289dbdcef..487923fb968 100755 --- a/hack/checks.sh +++ b/hack/checks.sh @@ -21,6 +21,7 @@ RESET='\033[0m' echo "Running validation scripts..." scripts=( "hack/check-schema-changes.sh" + "hack/check-skaffold-builder.sh" "hack/boilerplate.sh" "hack/gofmt.sh" "hack/pedantic-imports.sh" diff --git a/hack/skaffold-deps-sha1.sh b/hack/skaffold-deps-sha1.sh new file mode 100755 index 00000000000..07ca6cb55b1 --- /dev/null +++ b/hack/skaffold-deps-sha1.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Skaffold Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e -o pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +openssl sha1 -r "${DIR}/../deploy/skaffold/Dockerfile.deps" | cut -d' ' -f1