Add arm64 support to docker image creating tasks

This changes the x86_64 image in nightly and final build steps to be prefixed
with "amd64-", and adds an arm64 image which is prefixed with "arm64-".
The unprefixed image, which was previously the x86_64 image, is now a
multi-architecture image which points to the two prefixed images.

This change also splits build_docker_image into build_docker_image_arm64
and build_docker_image_x86_64. The Build and Test Docker TeamCity builds
have been split accordingly, into arm64 and x86_64 variants for each.
These builds do not build/test the combined image, only the individual
platforms' images.

Part of CRDB-3143

Release note (general change): Cockroach Docker images are now multi
architecture manifests supporting the x86_64 (amd64) and arm64
architectures.

build/release/teamcity-publish-redhat-release.sh

@@ -47,23 +47,34 @@ tc_start_block "Configure docker"
 docker_login_with_redhat
 tc_end_block "Configure docker"
 
-tc_start_block "Rebuild docker image"
-sed \
-  -e "s,@repository@,${dockerhub_repository},g" \
-  -e "s,@tag@,${build_name},g" \
-  build/deploy-redhat/Dockerfile.in > build/deploy-redhat/Dockerfile
+tc_start_block "Rebuild per-arch docker images"
+declare -a combined_image_args
+for docker_arch in "${docker_archs[@]}"; do
+  cp --recursive build/deploy-redhat "build/deploy-redhat-${docker_arch}"
 
-cat build/deploy-redhat/Dockerfile
+  sed \
+    -e "s,@repository@,${dockerhub_repository},g" \
+    -e "s,@tag@,${docker_arch}-${build_name},g" \
+    "build/deploy-redhat-${docker_arch}/Dockerfile.in" > "build/deploy-redhat-${docker_arch}/Dockerfile"
 
-docker build --no-cache \
-  --label release=$rhel_release \
-  --tag=${rhel_repository}:${build_name} \
-  build/deploy-redhat
-tc_end_block "Rebuild docker image"
+  cat build/deploy-redhat-${docker_arch}/Dockerfile
 
-tc_start_block "Push RedHat docker image"
-retry docker push "${rhel_repository}:${build_name}"
-tc_end_block "Push RedHat docker image"
+  build_docker_tag="${rhel_repository}:${docker_arch}-${build_name}"
+  docker build --no-cache \
+    --label=release=$rhel_release \
+    --platform="linux/${docker_arch}" \
+    --tag="${build_docker_tag}" \
+    --pull \
+    "build/deploy-redhat-${docker_arch}"
+
+  combined_image_args=( "${docker_tags[@]}" --amend "${build_docker_tag}" )
+done
+tc_end_block "Rebuild per-arch docker images"
+
+tc_start_block "Push multiarch RedHat docker image"
+docker manifest create "${rhel_repository}:${build_name}" "${combined_image_args[@]}"
+retry docker manifest push "${rhel_repository}:${build_name}"
+tc_end_block "Push multiarch RedHat docker image"
 
 tc_start_block "Run preflight"
 mkdir -p artifacts

build/release/teamcity-support.sh

@@ -4,6 +4,18 @@
 root="$(dirname $(dirname $(cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )))"
 source "$root/build/teamcity-common-support.sh"
 
+declare -a platform_names=( "amd64::amd64" "aarch64::arm64" )
+declare -a tarball_archs=( "${platform_names[@]%%::*}" ) 
+declare -a docker_archs=( "${platform_names[@]##*::}" )
+
+function tarball_arch_from_platform_name() {
+  echo "${1%%::*}"
+}
+
+function docker_arch_from_platform_name() {
+  echo "${1##*::}"
+}
+
 remove_files_on_exit() {
   rm -rf ~/.docker
   common_support_remove_files_on_exit

build/teamcity/cockroach/ci/builds/build_docker_image.sh

@@ -1,5 +1,11 @@
 #!/usr/bin/env bash
-set -euo pipefail
+set -euxo pipefail
+
+# The first and only parameter is the name of the architecture the image is being built for.
+# This should be the format Docker expects for the platform flag minus linux/ - i.e., either
+# amd64 or arm64. Old TC configs will run this file directly and not supply a platform, so
+# default to amd64.
+build_arch=${1:-amd64}
 
 dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
 source "$dir/teamcity-support.sh"  # For $root
@@ -11,7 +17,7 @@ chmod o+rwx "${artifacts}"
 
 tc_start_block "Copy cockroach binary and dependency files to build/deploy"
 
-# Get the cockroach binary from Build (Linux x86_64)
+# Get the cockroach binary from Build (Linux ${arch})
 # Artifacts rules:
 # bazel-bin/pkg/cmd/cockroach/cockroach_/cockroach=>upstream_artifacts
 # bazel-bin/c-deps/libgeos/lib/libgeos.so=>upstream_artifacts
@@ -33,11 +39,19 @@ docker_image_tar_name="cockroach-docker-image.tar"
 
 docker_tag="cockroachdb/cockroach-ci"
 
+# We have to always pull here because this runner may have been used to build
+# a different architecture's docker image. If that's the case, the cache will
+# return the cached version of the UBI base image (which will be for the wrong
+# architecture), then build will use it and fail because it's for the wrong
+# architecture. The cache is really stupid, in other words.
+
 docker build \
   --no-cache \
+  --platform=linux/${build_arch} \
   --tag="$docker_tag" \
   --memory 30g \
   --memory-swap -1 \
+  --pull \
   build/deploy
 
 docker save "$docker_tag" | gzip > "${artifacts}/${docker_image_tar_name}".gz

build/teamcity/cockroach/ci/builds/build_docker_image_arm64.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+source "$(dirname "${0}")/build_docker_image.sh" arm64

build/teamcity/cockroach/ci/builds/build_docker_image_x86_64.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+source "$(dirname "${0}")/build_docker_image.sh" amd64

build/teamcity/internal/release/process/make-and-publish-build-artifacts.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -euo pipefail
+set -euxo pipefail
 
 dir="$(dirname $(dirname $(dirname $(dirname $(dirname "${0}")))))"
 source "$dir/release/teamcity-support.sh"
@@ -65,20 +65,46 @@ $BAZEL_BIN/pkg/cmd/publish-provisional-artifacts/publish-provisional-artifacts_/
 EOF
 tc_end_block "Compile and publish S3 artifacts"
 
-tc_start_block "Make and push docker image"
+tc_start_block "Make and push multiarch docker images"
 configure_docker_creds
 docker_login_with_google
 
-# TODO: update publish-provisional-artifacts with option to leave one or more cockroach binaries in the local filesystem
-# NB: tar usually stops reading as soon as it sees an empty block but that makes
-# curl unhappy, so passing `i` will cause it to read to the end.
-curl -f -s -S -o- "https://${bucket}.s3.amazonaws.com/cockroach-${build_name}.linux-amd64.tgz" | tar ixfz - --strip-components 1
-cp cockroach lib/libgeos.so lib/libgeos_c.so build/deploy
-cp -r licenses build/deploy/
+combined_docker_tag="${gcr_repository}:${build_name}"
+declare -a combined_image_args
+
+for platform_name in "${platform_names[@]}"; do
+  tarball_arch="$(tarball_arch_from_platform_name "$platform_name")"
+  docker_arch="$(docker_arch_from_platform_name "$platform_name")"
+  # TODO: update publish-provisional-artifacts with option to leave one or more cockroach binaries in the local filesystem
+  # NB: tar usually stops reading as soon as it sees an empty block but that makes
+  # curl unhappy, so passing `--ignore-zeros` will cause it to read to the end.
+  cp --recursive "build/deploy" "build/deploy-${docker_arch}"
+  curl \
+    --fail \
+    --silent \
+    --show-error \
+    --output /dev/stdout \
+    --url "https://${bucket}.s3.amazonaws.com/cockroach-${build_name}.linux-${tarball_arch}.tgz" \
+    | tar \
+    --directory="build/deploy-${docker_arch}" \
+    --extract \
+    --file=/dev/stdin \
+    --ungzip \
+    --ignore-zeros \
+    --strip-components=1
+  cp --recursive licenses "build/deploy-${docker_arch}"
+
+  build_docker_tag="${gcr_repository}:${docker_arch}-${build_name}"
+  docker build --no-cache --pull --platform "linux/${docker_arch}" --tag="${build_docker_tag}" "build/deploy-${docker_arch}"
+  docker push "${build_docker_tag}"
+  combined_image_args=( "${docker_tags[@]}" --amend "${build_docker_tag}" )
+done
+
+docker manifest create "${combined_docker_tag}" "${docker_amends[@]}"
+docker manifest push "${combined_docker_tag}"
+tc_end_block "Make and push multiarch docker images"
+
 
-docker build --no-cache --tag="${gcr_repository}:${build_name}" build/deploy
-docker push "${gcr_repository}:${build_name}"
-tc_end_block "Make and push docker image"
 
 # Make finding the tag name easy.
 cat << EOF

build/teamcity/internal/release/process/publish-cockroach-release.sh

@@ -95,26 +95,63 @@ EOF
 tc_end_block "Make and publish release S3 artifacts"
 
 
-tc_start_block "Make and push docker images"
+tc_start_block "Make and push multiarch docker images"
 configure_docker_creds
 docker_login_with_google
 docker_login
 
-# TODO: update publish-provisional-artifacts with option to leave one or more cockroach binaries in the local filesystem?
-curl -f -s -S -o- "https://${s3_download_hostname}/cockroach-${build_name}.linux-amd64.tgz" | tar ixfz - --strip-components 1
-cp cockroach lib/libgeos.so lib/libgeos_c.so build/deploy
-cp -r licenses build/deploy/
+declare -a latest_images
+declare -a latest_images_by_branch
+declare -a arch_specific_images
+declare -a combined_image_args
+
+for platform_name in "${platform_names[@]}"; do
+  tarball_arch="$(tarball_arch_from_platform_name "$platform_name")"
+  docker_arch="$(docker_arch_from_platform_name "$platform_name")"
+  # TODO: update publish-provisional-artifacts with option to leave one or more cockroach binaries in the local filesystem
+  # NB: tar usually stops reading as soon as it sees an empty block but that makes
+  # curl unhappy, so passing `--ignore-zeros` will cause it to read to the end.
+  cp --recursive "build/deploy" "build/deploy-${docker_arch}"
+  curl \
+    --fail \
+    --silent \
+    --show-error \
+    --output /dev/stdout \
+    --url "https://${bucket}.s3.amazonaws.com/cockroach-${build_name}.linux-${tarball_arch}.tgz" \
+    | tar \
+    --directory="build/deploy-${docker_arch}" \
+    --extract \
+    --file=/dev/stdin \
+    --ungzip \
+    --ignore-zeros \
+    --strip-components=1
+  cp --recursive licenses "build/deploy-${docker_arch}"
+
+  build_docker_tag="${gcr_repository}:${docker_arch}-${build_name}"
+
+  docker build \
+    --label version="$version" \
+    --no-cache \
+    --pull \
+    --platform="linux/${docker_arch}" \
+    --tag="${dockerhub_repository}:${docker_arch}-"{"${build_name}",latest,latest-"${release_branch}"} \
+    --tag="${gcr_repository}:${docker_arch}-${build_name}" \
+    "build/deploy-${docker_arch}"
+  docker push "${dockerhub_repository}:${docker_arch}-${build_name}"
+  docker push "${gcr_repository}:${docker_arch}-${build_name}"
+
+  arch_specific_images=( "${arch_specific_images[@]}" "${build_docker_tag}" )
+  combined_image_args=( "${docker_tags[@]}" --amend "${build_docker_tag}" )
+  latest_images=("${latest_images[@]}" "${dockerhub_repository}:${docker-arch}-latest")
+  latest_images_by_branch=("${latest_images_by_branch[@]}" "${dockerhub_repository}:${docker_arch}-latest-${release_branch}")
+done
 
-docker build \
-  --label version=$version \
-  --no-cache \
-  --tag=${dockerhub_repository}:{"$build_name",latest,latest-"${release_branch}"} \
-  --tag=${gcr_repository}:${build_name} \
-  build/deploy
+docker manifest create "${combined_docker_tag}" "${docker_amends[@]}"
+docker manifest push "${combined_docker_tag}"
 
-docker push "${dockerhub_repository}:${build_name}"
-docker push "${gcr_repository}:${build_name}"
-tc_end_block "Make and push docker images"
+docker manifest create "${dockerhub_repository}:latest" "${docker_amends[@]}"
+docker manifest create "${dockerhub_repository}:latest-${release_branch}" "${docker_amends[@]}"
+tc_end_block "Make and push multiarch docker images"
 
 
 tc_start_block "Push release tag to GitHub"
@@ -152,37 +189,40 @@ tc_end_block "Publish S3 binaries and archive as latest"
 
 tc_start_block "Tag docker image as latest-RELEASE_BRANCH"
 if [[ -z "$PRE_RELEASE" ]]; then
-  docker push "${dockerhub_repository}:latest-${release_branch}"
+  docker push "${latest_images_by_branch[@]}"
+  docker manifest push "${dockerhub_repository}:latest-${release_branch}"
 else
-  echo "The ${dockerhub_repository}:latest-${release_branch} docker image tag was _not_ pushed."
+  echo "The ${dockerhub_repository}:latest-${release_branch} docker image tags were _not_ pushed."
 fi
-tc_end_block "Tag docker image as latest-RELEASE_BRANCH"
+tc_end_block "Tag docker images as latest-RELEASE_BRANCH"
 
 
-tc_start_block "Tag docker image as latest"
+tc_start_block "Tag docker images as latest"
 # Only push the "latest" tag for our most recent release branch and for the
 # latest unstable release
 # https://github.com/cockroachdb/cockroach/issues/41067
 # https://github.com/cockroachdb/cockroach/issues/48309
 if [[ -n "${PUBLISH_LATEST}" || -n "${PRE_RELEASE}" ]]; then
-  docker push "${dockerhub_repository}:latest"
+  docker push "${latest_images[@]}"
+  docker manifest push "${dockerhub_repository}:latest"
 else
-  echo "The ${dockerhub_repository}:latest docker image tag was _not_ pushed."
+  echo "The ${dockerhub_repository}:latest docker image tags were _not_ pushed."
 fi
-tc_end_block "Tag docker image as latest"
+tc_end_block "Tag docker images as latest"
 
 
 tc_start_block "Verify docker images"
 
 images=(
   "${dockerhub_repository}:${build_name}"
   "${gcr_repository}:${build_name}"
+  "${arch_specific_images[@]}"
 )
 if [[ -z "$PRE_RELEASE" ]]; then
-  images+=("${dockerhub_repository}:latest-${release_branch}")
+  images+=("${dockerhub_repository}:latest-${release_branch}" "${latest_images_by_branch[@]}")
 fi
 if [[ -n "${PUBLISH_LATEST}" || -n "${PRE_RELEASE}" ]]; then
-  images+=("${dockerhub_repository}:latest")
+  images+=("${dockerhub_repository}:latest" "${latest_images[@]}")
 fi
 
 error=0