Enable creation of multi-arch docker images

.github/workflows/ci.yml

@@ -281,3 +281,15 @@ jobs:
         run: go mod tidy
       - shell: bash
         run: .github/workflows/check-clean-branch.sh
+  test_build_image:
+    name: Image build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install qemu (required for cross-platform builds)
+        run: |
+          sudo apt update
+          sudo apt -y install qemu qemu-user-static
+      - name: Check image build
+        shell: bash
+        run: bash -x scripts/tests.build_image.sh

.github/workflows/publish_docker_image.yml

@@ -14,9 +14,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Publish image to DockerHub
+      - name: Install qemu (required for cross-platform builds)
+        run: |
+          sudo apt update
+          sudo apt -y install qemu qemu-user-static
+          sudo systemctl restart docker
+      - name: Create multiplatform docker builder
+        run: docker buildx create --use
+      - name: Build and publish images to DockerHub
         env:
           DOCKER_USERNAME: ${{ secrets.docker_username }}
           DOCKER_PASS: ${{ secrets.docker_pass }}
-          DOCKER_REPO: ${{ secrets.docker_repo }}
-        run: .github/workflows/publish_image.sh
+          DOCKER_IMAGE: ${{ secrets.docker_repo }}
+        run: scripts/build_image.sh

Dockerfile

@@ -6,9 +6,28 @@
 # README.md
 # go.mod
 # ============= Compilation Stage ================
-FROM golang:1.21.9-bullseye AS builder
+# Always use the native platform to ensure fast builds
+FROM --platform=$BUILDPLATFORM golang:1.21.9-bullseye AS builder
 
 WORKDIR /build
+
+ARG TARGETPLATFORM
+ARG BUILDPLATFORM
+
+# Configure a cross-compiler if the target platform differs from the build platform.
+#
+# build_env.sh is used to capture the environmental changes required by the build step since RUN
+# environment state is not otherwise persistent.
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ] && [ "$BUILDPLATFORM" != "linux/arm64" ]; then \
+    apt-get update && apt-get install -y gcc-aarch64-linux-gnu && \
+    echo "export CC=aarch64-linux-gnu-gcc" > ./build_env.sh \
+    ; elif [ "$TARGETPLATFORM" = "linux/amd64" ] && [ "$BUILDPLATFORM" != "linux/amd64" ]; then \
+    apt-get update && apt-get install -y gcc-x86-64-linux-gnu && \
+    echo "export CC=x86_64-linux-gnu-gcc" > ./build_env.sh \
+    ; else \
+    echo "export CC=gcc" > ./build_env.sh \
+    ; fi
+
 # Copy and download avalanche dependencies using go mod
 COPY go.mod .
 COPY go.sum .
@@ -17,15 +36,28 @@ RUN go mod download
 # Copy the code into the container
 COPY . .
 
-# Build avalanchego
+# Ensure pre-existing builds are not available for inclusion in the final image
+RUN [ -d ./build ] && rm -rf ./build/* || true
+
+# Build avalanchego. The build environment is configured with build_env.sh from the step
+# enabling cross-compilation.
 ARG RACE_FLAG=""
-RUN ./scripts/build.sh ${RACE_FLAG}
+RUN . ./build_env.sh && \
+    echo "{CC=$CC, TARGETPLATFORM=$TARGETPLATFORM, BUILDPLATFORM=$BUILDPLATFORM}" && \
+    export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) && \
+    ./scripts/build.sh ${RACE_FLAG}
+
+# Create this directory in the builder to avoid requiring anything to be executed in the
+# potentially emulated execution container.
+RUN mkdir -p /avalanchego/build
 
 # ============= Cleanup Stage ================
+# Commands executed in this stage may be emulated (i.e. very slow) if TARGETPLATFORM and
+# BUILDPLATFORM have different arches.
 FROM debian:11-slim AS execution
 
 # Maintain compatibility with previous images
-RUN mkdir -p /avalanchego/build
+COPY --from=builder /avalanchego/build /avalanchego/build
 WORKDIR /avalanchego/build
 
 # Copy the executables into the container

scripts/build_image.sh

@@ -2,6 +2,26 @@
 
 set -euo pipefail
 
+# e.g.,
+# ./scripts/build_image.sh                                           # Build local single-arch image
+# DOCKER_IMAGE=myavalanchego ./scripts/build_image.sh                # Build local single arch image with a custom image name
+# DOCKER_IMAGE=avaplatform/avalanchego ./scripts/build_image.sh      # Build and push multi-arch image to docker hub
+# DOCKER_IMAGE=localhost:5001/avalanchego ./scripts/build_image.sh   # Build and push multi-arch image to private registry
+# DOCKER_IMAGE=localhost:5001/myavalanchego ./scripts/build_image.sh # Build and push multi-arch image to private registry with a custom image name
+
+# Multi-arch builds require Docker Buildx and QEMU. buildx should be enabled by
+# default in the verson of docker included with Ubuntu 22.04, and qemu can be
+# installed as follows:
+#
+#  sudo apt-get install qemu qemu-user-static
+#
+# After installing qemu, it will also be necessary to start a new builder that can
+# support multiplatform builds:
+#
+#  docker buildx create --use
+#
+# Reference: https://docs.docker.com/buildx/working-with-buildx/
+
 # Directory above this script
 AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
 
@@ -13,14 +33,57 @@ if [[ $current_branch == *"-race" ]]; then
   exit 1
 fi
 
-# WARNING: this will use the most recent commit even if there are un-committed changes present
-full_commit_hash="$(git --git-dir="$AVALANCHE_PATH/.git" rev-parse HEAD)"
-commit_hash="${full_commit_hash::8}"
+# The published name should be 'avaplatform/avalanchego', but to avoid unintentional
+# pushes it is defaulted to 'avalanchego' (without a repo or registry name) which can
+# only be used to create local images.
+DOCKER_IMAGE=${DOCKER_IMAGE:-"avalanchego"}
+
+# buildx (BuildKit) improves the speed and UI of builds over the legacy builder and
+# simplifies creation of multi-arch images.
+#
+# Reference: https://docs.docker.com/build/buildkit/
+DOCKER_CMD="docker buildx build"
+
+if [[ "${DOCKER_IMAGE}" == *"/"* ]]; then
+  # Build a multi-arch image since the image name includes a slash which indicates
+  # the use of a registry e.g.
+  #
+  #  - dockerhub: [repo]/[image name]:[tag]
+  #  - private registry: [private registry hostname]/[image name]:[tag]
+  #
+  # A registry is required to build a multi-arch image since a multi-arch image is
+  # not really an image at all. A multi-arch image (also called a manifest) is
+  # basically a list of arch-specific images available from the same registry that
+  # hosts the manifest. Manifests are not supported for local images.
+  #
+  # Reference: https://docs.docker.com/build/building/multi-platform/
+  PLATFORMS="${PLATFORMS:-linux/amd64,linux/arm64}"
+  DOCKER_CMD="${DOCKER_CMD} --push --platform=${PLATFORMS}"
+
+  # A populated DOCKER_USERNAME env var triggers login
+  if [[ -n "${DOCKER_USERNAME:-}" ]]; then
+    echo "$DOCKER_PASS" | docker login --username "$DOCKER_USERNAME" --password-stdin
+  fi
+else
+  # Build a single-arch image since the image name does not include a slash which
+  # indicates that a registry is not available.
+  #
+  # Building a single-arch image with buildx and having the resulting image show up
+  # in the local store of docker images (ala 'docker build') requires explicitly
+  # loading it from the buildx store with '--load'.
+  DOCKER_CMD="${DOCKER_CMD} --load"
+fi
+
+echo "Building Docker Image with tags: $DOCKER_IMAGE:$commit_hash , $DOCKER_IMAGE:$current_branch"
+${DOCKER_CMD} -t "$DOCKER_IMAGE:$commit_hash" -t "$DOCKER_IMAGE:$current_branch" \
+              "$AVALANCHE_PATH" -f "$AVALANCHE_PATH/Dockerfile"
 
-echo "Building Docker Image with tags: $avalanchego_dockerhub_repo:$commit_hash , $avalanchego_dockerhub_repo:$current_branch"
-docker build -t "$avalanchego_dockerhub_repo:$commit_hash" \
-        -t "$avalanchego_dockerhub_repo:$current_branch" "$AVALANCHE_PATH" -f "$AVALANCHE_PATH/Dockerfile"
+echo "Building Docker Image with tags: $DOCKER_IMAGE:$commit_hash-race , $DOCKER_IMAGE:$current_branch-race"
+${DOCKER_CMD} --build-arg="RACE_FLAG=-r" -t "$DOCKER_IMAGE:$commit_hash-race" -t "$DOCKER_IMAGE:$current_branch-race" \
+              "$AVALANCHE_PATH" -f "$AVALANCHE_PATH/Dockerfile"
 
-echo "Building Docker Image with tags: $avalanchego_dockerhub_repo:$commit_hash-race , $avalanchego_dockerhub_repo:$current_branch-race"
-docker build --build-arg="RACE_FLAG=-r" -t "$avalanchego_dockerhub_repo:$commit_hash-race" \
-        -t "$avalanchego_dockerhub_repo:$current_branch-race" "$AVALANCHE_PATH" -f "$AVALANCHE_PATH/Dockerfile"
+# Only tag the latest image for the master branch when images are pushed to a registry
+if [[ "${DOCKER_IMAGE}" == *"/"* && $current_branch == "master" ]]; then
+  echo "Tagging current avalanchego images as $DOCKER_IMAGE:latest"
+  docker buildx imagetools create -t "$DOCKER_IMAGE:latest" "$DOCKER_IMAGE:$commit_hash"
+fi

scripts/constants.sh

@@ -13,14 +13,18 @@ AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd ) # Direct
 # Where AvalancheGo binary goes
 avalanchego_path="$AVALANCHE_PATH/build/avalanchego"
 
-# Avalabs docker hub
-# avaplatform/avalanchego - defaults to local as to avoid unintentional pushes
-# You should probably set it - export DOCKER_REPO='avaplatform/avalanchego'
-avalanchego_dockerhub_repo=${DOCKER_REPO:-"avalanchego"}
-
-# Current branch
+# Current branch  (shared between image build and its test script)
 # TODO: fix "fatal: No names found, cannot describe anything" in github CI
 current_branch=$(git symbolic-ref -q --short HEAD || git describe --tags --exact-match || true)
+# Supply a default branch when one is not discovered
+if [[ -z $current_branch ]]; then
+  current_branch=ci_dummy
+fi
+
+# Current commit (shared between image build and its test script)
+# WARNING: this will use the most recent commit even if there are un-committed changes present
+full_commit_hash="$(git --git-dir="$AVALANCHE_PATH/.git" rev-parse HEAD)"
+commit_hash="${full_commit_hash::8}"
 
 git_commit=${AVALANCHEGO_COMMIT:-$( git rev-list -1 HEAD )}
 

scripts/tests.build_image.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# This test script is intended to execute successfully on a ubuntu 22.04 host with either the
+# amd64 or arm64 arches. Recent docker (with buildx support) and qemu are required. See
+# build_image.sh for more details.
+
+# TODO(marun) Perform more extensive validation (e.g. e2e testing) against one or more images
+
+# Directory above this script
+AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
+
+source "$AVALANCHE_PATH"/scripts/constants.sh
+
+build_and_test() {
+  local image_name=$1
+
+  DOCKER_IMAGE="$image_name" ./scripts/build_image.sh
+
+  echo "listing images"
+  docker images
+
+  local host_arch
+  host_arch="$(go env GOARCH)"
+
+  if [[ "$image_name" == *"/"* ]]; then
+    # Test all arches if testing a multi-arch image
+    local arches=("amd64" "arm64")
+  else
+    # Test only the host platform for single arch builds
+    local arches=("$host_arch")
+  fi
+
+  # Check all of the images expected to have been built
+  local target_images=(
+    "$image_name:$commit_hash"
+    "$image_name:$current_branch"
+    "$image_name:$commit_hash-race"
+    "$image_name:$current_branch-race"
+  )
+
+  for arch in "${arches[@]}"; do
+    for target_image in "${target_images[@]}"; do
+      if [[ "$host_arch" == "amd64" && "$arch" == "arm64" && "$target_image" =~ "-race" ]]; then
+        # Error reported when trying to sanity check this configuration in github ci:
+        #
+        #   FATAL: ThreadSanitizer: unsupported VMA range
+        #   FATAL: Found 39 - Supported 48
+        #
+        echo "skipping sanity check for $target_image"
+        echo "image is for arm64 and binary is compiled with race detection"
+        echo "amd64 github workers are known to run kernels incompatible with these images"
+      else
+        echo "checking sanity of image $target_image for $arch by running 'avalanchego --version'"
+        docker run  -t --rm --platform "linux/$arch" "$target_image" /avalanchego/build/avalanchego --version
+      fi
+    done
+  done
+}
+
+echo "checking build of single-arch images"
+build_and_test avalanchego
+
+echo "starting local docker registry to allow verification of multi-arch image builds"
+REGISTRY_CONTAINER_ID="$(docker run --rm -d -P registry:2)"
+REGISTRY_PORT="$(docker port "$REGISTRY_CONTAINER_ID" 5000/tcp | grep -v "::" | awk -F: '{print $NF}')"
+
+echo "starting docker builder that supports multiplatform builds"
+# - creating a new builder enables multiplatform builds
+# - '--driver-opt network=host' enables the builder to use the local registry
+docker buildx create --use --name ci-builder --driver-opt network=host
+
+# Ensure registry and builder cleanup on teardown
+function cleanup {
+  echo "stopping local docker registry"
+  docker stop "${REGISTRY_CONTAINER_ID}"
+  echo "removing multiplatform builder"
+  docker buildx rm ci-builder
+}
+trap cleanup EXIT
+
+echo "checking build of multi-arch images"
+build_and_test "localhost:${REGISTRY_PORT}/avalanchego"