Skip to content

Commit

Permalink
Merge pull request #1710 from justaugustus/krel-ci-build
Browse files Browse the repository at this point in the history
k8s-ci-builder: Clean up image building configs
  • Loading branch information
k8s-ci-robot authored Nov 10, 2020
2 parents f1d09d3 + d83512a commit 65be49f
Show file tree
Hide file tree
Showing 15 changed files with 552 additions and 45 deletions.
4 changes: 2 additions & 2 deletions cmd/krel/cmd/anago/push.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ func init() {
)

pushCmd.PersistentFlags().StringVar(
&pushOpts.DockerRegistry,
&pushOpts.Registry,
"container-registry",
"",
"Container image registry to be used",
Expand Down Expand Up @@ -177,7 +177,7 @@ func runPushRelease(
// In an official nomock release, we want to ensure that container images
// have been promoted from staging to production, so we do the image
// manifest validation against production instead of staging.
targetRegistry := opts.DockerRegistry
targetRegistry := opts.Registry
if targetRegistry == release.GCRIOPathStaging {
targetRegistry = release.GCRIOPathProd
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/krel/cmd/ci_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ func init() {

// TODO: Switch to "--registry" once CI no longer uses it
ciBuildCmd.PersistentFlags().StringVar(
&ciBuildOpts.DockerRegistry,
"docker-registry",
&ciBuildOpts.Registry,
"registry",
"",
"If set, push docker images to specified registry/project",
)
Expand Down
4 changes: 2 additions & 2 deletions cmd/krel/cmd/push.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,8 @@ func init() {

// TODO: Switch to "--registry" once CI no longer uses it
pushBuildCmd.PersistentFlags().StringVar(
&pushBuildOpts.DockerRegistry,
"docker-registry",
&pushBuildOpts.Registry,
"registry",
"",
"If set, push docker images to specified registry/project",
)
Expand Down
6 changes: 4 additions & 2 deletions docs/krel/push.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ Push Kubernetes release artifacts to Google Cloud Storage (GCS)
- [Summary](#summary)
- [Installation](#installation)
- [Usage](#usage)
- [Important notes](#important-notes)
- [Command line flags](#command-line-flags)
- [Examples](#examples)
- [Important Notes](#important-notes)

## Summary

Expand All @@ -31,7 +33,7 @@ In `--ci` mode, 'push' runs in mock mode by default. Use `--nomock` to do a real
--bucket string Specify an alternate bucket for pushes (normally 'devel' or 'ci') (default "devel")
--buildDir string Specify an alternate build directory (defaults to '_output') (default "_output")
--ci Used when called from Jenkins (for ci runs)
--docker-registry string If set, push docker images to specified registry/project
--registry string If set, push docker images to specified registry/project
--extra-publish-file string Used when need to upload additional version file to GCS. The path is relative and is append to a GCS path. (--ci only)
--gcs-suffix string Specify a suffix to append to the upload destination on GCS
-h, --help help for push
Expand Down
114 changes: 107 additions & 7 deletions images/releng/k8s-ci-builder/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

ARG GO_VERSION
ARG OLD_BAZEL_VERSION
FROM golang:${GO_VERSION} as builder

WORKDIR /go/src/k8s.io/release
Expand All @@ -23,15 +24,114 @@ RUN ./compile-release-tools

### Production image

FROM gcr.io/k8s-testimages/krte:latest-master
# Includes tools used for building Kubernetes in CI
#
# NOTE: we attempt to avoid unnecessary tools and image layers while
# supporting kubernetes builds, kind installation, etc.

WORKDIR /
COPY --from=builder /go/bin/* ./
FROM launcher.gcr.io/google/bazel:${OLD_BAZEL_VERSION} as old-bazel
FROM debian:buster

# arg that specifies the image name (for debugging)
ARG IMAGE_ARG
# arg that specifies the bazel version to install
ARG BAZEL_VERSION
ARG GO_VERSION

# add envs:
# - so we can debug with the image name:tag
# - with the bazel version
# - adding gsutil etc. to path (where we will install them)
# - disabling prompts when installing gsutil etc.
# - hinting that we are in a docker container
ENV IMAGE=${IMAGE_ARG} \
BAZEL_VERSION=${BAZEL_VERSION} \
GOPATH=/home/prow/go \
PATH=/home/prow/go/bin:/usr/local/go/bin:/google-cloud-sdk/bin:${PATH} \
CLOUDSDK_CORE_DISABLE_PROMPTS=1 \
CONTAINER=docker

ARG DEBIAN_FRONTEND=noninteractive
# copy in image utility scripts
COPY ["images/releng/k8s-ci-builder/wrapper.sh", \
"images/releng/k8s-ci-builder/create_bazel_cache_rcs.sh", \
"images/releng/k8s-ci-builder/install-bazel.sh", \
"/usr/local/bin/"]

RUN apt-get -q update \
&& apt-get install -qqy \
jq
# Install tools needed to:
# - install docker
# - build kubernetes (dockerized, or with bazel)
#
# TODO: the `sed` is a bit of a hack, look into alternatives.
# Why this exists: `docker service start` on debian runs a `cgroupfs_mount` method,
# We're already inside docker though so we can be sure these are already mounted.
# Trying to remount these makes for a very noisy error block in the beginning of
# the pod logs, so we just comment out the call to it... :shrug:
RUN echo "Installing Packages ..." \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
apt-transport-https \
build-essential \
ca-certificates \
curl \
file \
git \
gnupg2 \
jq \
kmod \
lsb-release \
mercurial \
openssh-client \
pkg-config \
procps \
python \
python-dev \
python-pip \
rsync \
software-properties-common \
unzip \
&& rm -rf /var/lib/apt/lists/* \
&& echo "Installing Go ..." \
&& export GO_TARBALL="go${GO_VERSION}.linux-amd64.tar.gz"\
&& curl -fsSL "https://storage.googleapis.com/golang/${GO_TARBALL}" --output "${GO_TARBALL}" \
&& tar xzf "${GO_TARBALL}" -C /usr/local \
&& rm "${GO_TARBALL}"\
&& mkdir -p "${GOPATH}/bin" \
&& echo "Installing Bazel ..." \
&& install-bazel.sh \
&& echo "Installing gcloud SDK, kubectl ..." \
&& curl -fsSL https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz --output google-cloud-sdk.tar.gz \
&& tar xzf google-cloud-sdk.tar.gz -C / \
&& rm google-cloud-sdk.tar.gz \
&& /google-cloud-sdk/install.sh \
--disable-installation-options \
--bash-completion=false \
--path-update=false \
--usage-reporting=false \
&& gcloud components install kubectl \
&& gcloud components install alpha \
&& gcloud components install beta \
&& echo "Installing Docker ..." \
&& curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | apt-key add - \
&& add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \
$(lsb_release -cs) stable" \
&& apt-get update \
&& apt-get install -y --no-install-recommends docker-ce \
&& rm -rf /var/lib/apt/lists/* \
&& sed -i 's/cgroupfs_mount$/#cgroupfs_mount\n/' /etc/init.d/docker \
&& echo "Ensuring Legacy Iptables ..." \
&& update-alternatives --set iptables /usr/sbin/iptables-legacy \
&& update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy

ARG OLD_BAZEL_VERSION
COPY --from=old-bazel \
/usr/local/lib/bazel/bin/bazel-real /usr/local/lib/bazel/bin/bazel-${OLD_BAZEL_VERSION}

# Copy in release tools from kubernetes/release
WORKDIR /
COPY --from=builder /go/bin/* ./

# entrypoint is our wrapper script, in Prow you will need to explicitly re-specify this
ENTRYPOINT ["wrapper.sh"]
# volume for docker in docker, use an emptyDir in Prow
VOLUME ["/var/lib/docker"]
64 changes: 64 additions & 0 deletions images/releng/k8s-ci-builder/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# set default shell
SHELL=/bin/bash -o pipefail

REGISTRY ?= gcr.io/k8s-staging-releng
IMGNAME = k8s-ci-builder
CONFIG ?= default

IMAGE = $(REGISTRY)/$(IMGNAME)

TAG ?= $(shell git describe --tags --always --dirty)

# Build args
GO_VERSION ?= 1.15.3
BAZEL_VERSION ?= 3.4.1
OLD_BAZEL_VERSION ?= 2.2.0

BUILD_ARGS = --build-arg=GO_VERSION=$(GO_VERSION) \
--build-arg=BAZEL_VERSION=$(BAZEL_VERSION) \
--build-arg=OLD_BAZEL_VERSION=$(OLD_BAZEL_VERSION)

# Ensure support for 'docker buildx' and 'docker manifest' commands
export DOCKER_CLI_EXPERIMENTAL=enabled

# build with buildx
# https://github.com/docker/buildx/issues/59
.PHONY: build
build: init-docker-buildx
echo "Building $(IMGNAME)..."
docker buildx build \
--load \
--progress plain \
--platform linux/amd64 \
--tag $(IMAGE):$(CONFIG) \
--tag $(IMAGE):$(TAG)-$(CONFIG) \
--tag $(IMAGE):latest-$(CONFIG) \
$(BUILD_ARGS) \
-f $(CURDIR)/Dockerfile \
../../../.

.PHONY: push
push: build
echo "Pushing $(IMGNAME) tags"
docker push $(IMAGE):$(CONFIG)
docker push $(IMAGE):$(TAG)-$(CONFIG)
docker push $(IMAGE):latest-$(CONFIG)

# enable buildx
.PHONY: init-docker-buildx
init-docker-buildx:
./../../../hack/init-buildx.sh
55 changes: 35 additions & 20 deletions images/releng/k8s-ci-builder/cloudbuild.yaml
Original file line number Diff line number Diff line change
@@ -1,38 +1,53 @@
# See https://cloud.google.com/cloud-build/docs/build-config
# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md for more details on image pushing process

# this must be specified in seconds. If omitted, defaults to 600s (10 mins)
timeout: 1200s

# this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
# or any new substitutions added in the future.
options:
substitution_option: ALLOW_LOOSE
machineType: 'N1_HIGHCPU_8'

steps:
- name: gcr.io/cloud-builders/docker
id: build
dir: images/releng/k8s-ci-builder
# TODO: Update image version
- name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200824-5d057db'
entrypoint: 'bash'
dir: ./images/releng/k8s-ci-builder
env:
- DOCKER_CLI_EXPERIMENTAL=enabled
- REGISTRY=gcr.io/$PROJECT_ID
- HOME=/root
- TAG=${_GIT_TAG}
- PULL_BASE_REF=${_PULL_BASE_REF}
- CONFIG=${_CONFIG}
- GO_VERSION=${_GO_VERSION}
- BAZEL_VERSION=${_BAZEL_VERSION}
- OLD_BAZEL_VERSION=${_OLD_BAZEL_VERSION}
args:
- build
- --tag=gcr.io/$PROJECT_ID/k8s-ci-builder:${_GIT_TAG}-${_CONFIG}
- --tag=gcr.io/$PROJECT_ID/k8s-ci-builder:latest-${_CONFIG}
- --tag=gcr.io/$PROJECT_ID/k8s-ci-builder:${_KUBE_CROSS_VERSION}
- --build-arg=GO_VERSION=${_GO_VERSION}
- --build-arg=KUBE_CROSS_VERSION=${_KUBE_CROSS_VERSION}
- -f ./Dockerfile
- ../../../.
- '-c'
- |
gcloud auth configure-docker \
&& make push
substitutions:
# _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
# can be used as a substitution
_GIT_TAG: '12345'
_PULL_BASE_REF: 'dev'
_CONFIG: 'cross0.0'
_CONFIG: 'config'
_GO_VERSION: '0.0.0'
_KUBE_CROSS_VERSION: 'v0.0.0-0'

images:
- 'gcr.io/$PROJECT_ID/k8s-ci-builder:${_GIT_TAG}-${_CONFIG}'
- 'gcr.io/$PROJECT_ID/k8s-ci-builder:latest-${_CONFIG}'
- 'gcr.io/$PROJECT_ID/k8s-ci-builder:${_KUBE_CROSS_VERSION}'
_BAZEL_VERSION: '0.0.0'
_OLD_BAZEL_VERSION: '0.0.0'

tags:
- 'k8s-ci-builder'
- ${_GIT_TAG}
- ${_PULL_BASE_REF}
- ${_CONFIG}
- ${_GO_VERSION}
- ${_KUBE_CROSS_VERSION}

images:
- 'gcr.io/$PROJECT_ID/k8s-ci-builder:${_CONFIG}'
- 'gcr.io/$PROJECT_ID/k8s-ci-builder:${_GIT_TAG}-${_CONFIG}'
- 'gcr.io/$PROJECT_ID/k8s-ci-builder:latest-${_CONFIG}'
Loading

0 comments on commit 65be49f

Please sign in to comment.