[antithesis] Refactor image build for reuse by other repos

.github/workflows/publish_antithesis_images.yml

@@ -35,12 +35,12 @@ jobs:
       run: bash -x ./scripts/build_antithesis_images.sh
       env:
         IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
-        TAG: ${{ github.event.inputs.image_tag || 'latest' }}
+        IMAGE_TAG: ${{ github.event.inputs.image_tag || 'latest' }}
         TEST_SETUP: avalanchego
 
     - name: Build and push images for xsvm test setup
       run: bash -x ./scripts/build_antithesis_images.sh
       env:
         IMAGE_PREFIX: ${{ env.REGISTRY }}/${{ env.REPOSITORY }}
-        TAG: ${{ github.event.inputs.image_tag || 'latest' }}
+        IMAGE_TAG: ${{ github.event.inputs.image_tag || 'latest' }}
         TEST_SETUP: xsvm

scripts/build_antithesis_images.sh

@@ -5,142 +5,85 @@ set -euo pipefail
 # Builds docker images for antithesis testing.
 
 # e.g.,
-# TEST_SETUP=avalanchego ./scripts/build_antithesis_images.sh                                    # Build local images for avalanchego
-# TEST_SETUP=avalanchego NODE_ONLY=1 ./scripts/build_antithesis_images.sh                        # Build only a local node image for avalanchego
-# TEST_SETUP=xsvm ./scripts/build_antithesis_images.sh                                           # Build local images for xsvm
-# TEST_SETUP=xsvm IMAGE_PREFIX=<registry>/<repo> TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag
+# TEST_SETUP=avalanchego ./scripts/build_antithesis_images.sh                                          # Build local images for avalanchego
+# TEST_SETUP=avalanchego NODE_ONLY=1 ./scripts/build_antithesis_images.sh                              # Build only a local node image for avalanchego
+# TEST_SETUP=xsvm ./scripts/build_antithesis_images.sh                                                 # Build local images for xsvm
+# TEST_SETUP=xsvm IMAGE_PREFIX=<registry>/<repo> IMAGE_TAG=latest ./scripts/build_antithesis_images.sh # Specify a prefix to enable image push and use a specific tag
+
+TEST_SETUP="${TEST_SETUP:-}"
+if [[ "${TEST_SETUP}" != "avalanchego" && "${TEST_SETUP}" != "xsvm" ]]; then
+  echo "TEST_SETUP must be set. Valid values are 'avalanchego' or 'xsvm'"
+  exit 255
+fi
 
 # Directory above this script
 AVALANCHE_PATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )"; cd .. && pwd )
 
+# Import common functions used to build images for antithesis test setups
+source "${AVALANCHE_PATH}"/scripts/lib_build_antithesis_images.sh
+
 # Specifying an image prefix will ensure the image is pushed after build
 IMAGE_PREFIX="${IMAGE_PREFIX:-}"
 
-TAG="${TAG:-}"
-if [[ -z "${TAG}" ]]; then
+IMAGE_TAG="${IMAGE_TAG:-}"
+if [[ -z "${IMAGE_TAG}" ]]; then
   # Default to tagging with the commit hash
   source "${AVALANCHE_PATH}"/scripts/constants.sh
-  TAG="${commit_hash}"
+  IMAGE_TAG="${commit_hash}"
 fi
 
 # The dockerfiles don't specify the golang version to minimize the changes required to bump
 # the version. Instead, the golang version is provided as an argument.
 GO_VERSION="$(go list -m -f '{{.GoVersion}}')"
 
-function build_images {
+# Helper to simplify calling build_builder_image for test setups in this repo
+function build_builder_image_for_avalanchego {
+  echo "Building builder image"
+  build_antithesis_builder_image "${GO_VERSION}" "antithesis-avalanchego-builder:${IMAGE_TAG}" "${AVALANCHE_PATH}" "${AVALANCHE_PATH}"
+}
+
+# Helper to simplify calling build_antithesis_images for test setups in this repo
+function build_antithesis_images_for_avalanchego {
   local test_setup=$1
-  local uninstrumented_node_dockerfile=$2
-  local image_prefix=$3
+  local image_prefix=$2
+  local uninstrumented_node_dockerfile=$3
   local node_only=${4:-}
 
-  # Define image names
-  local base_image_name="antithesis-${test_setup}"
-  if [[ -n "${image_prefix}" ]]; then
-    base_image_name="${image_prefix}/${base_image_name}"
-  fi
-  local node_image_name="${base_image_name}-node:${TAG}"
-  local workload_image_name="${base_image_name}-workload:${TAG}"
-  local config_image_name="${base_image_name}-config:${TAG}"
-  # The same builder image is used to build node and workload images for all test
-  # setups. It is not intended to be pushed.
-  local builder_image_name="antithesis-avalanchego-builder:${TAG}"
-
-  # Define dockerfiles
-  local base_dockerfile="${AVALANCHE_PATH}/tests/antithesis/${test_setup}/Dockerfile"
-  local builder_dockerfile="${base_dockerfile}.builder-instrumented"
-  local node_dockerfile="${base_dockerfile}.node"
-  # Working directory for instrumented builds
-  local builder_workdir="/avalanchego_instrumented/customer"
-  if [[ "$(go env GOARCH)" == "arm64" ]]; then
-    # Antithesis instrumentation is only supported on amd64. On apple silicon (arm64),
-    # uninstrumented Dockerfiles will be used to enable local test development.
-    builder_dockerfile="${base_dockerfile}.builder-uninstrumented"
-    node_dockerfile="${uninstrumented_node_dockerfile}"
-    # Working directory for uninstrumented builds
-    builder_workdir="/build"
-  fi
-
-  # Define default build command
-  local docker_cmd="docker buildx build\
- --build-arg GO_VERSION=${GO_VERSION}\
- --build-arg NODE_IMAGE=${node_image_name}\
- --build-arg BUILDER_IMAGE=${builder_image_name}\
- --build-arg BUILDER_WORKDIR=${builder_workdir}\
- --build-arg TAG=${TAG}"
-
-  if [[ "${test_setup}" == "xsvm" ]]; then
-    # The xsvm node image is built on the avalanchego node image, which is assumed to have already been
-    # built. The image name doesn't include the image prefix because it is not intended to be pushed.
-    docker_cmd="${docker_cmd} --build-arg AVALANCHEGO_NODE_IMAGE=antithesis-avalanchego-node:${TAG}"
-  fi
-
-  if [[ "${test_setup}" == "avalanchego" ]]; then
-    # Build the image that enables compiling golang binaries for the node and workload
-    # image builds. The builder image is intended to enable building instrumented binaries
-    # if built on amd64 and non-instrumented binaries if built on arm64.
-    #
-    # The builder image is not intended to be pushed so it needs to be built in advance of
-    # adding `--push` to docker_cmd. Since it is never prefixed with `[registry]/[repo]`,
-    # attempting to push will result in an error like `unauthorized: access token has
-    # insufficient scopes`.
-    ${docker_cmd} -t "${builder_image_name}" -f "${builder_dockerfile}" "${AVALANCHE_PATH}"
-  fi
-
-  if [[ -n "${image_prefix}" && -z "${node_only}" ]]; then
-    # Push images with an image prefix since the prefix defines a
-    # registry location, and only if building all images. When
-    # building just the node image the image is only intended to be
-    # used locally.
-    docker_cmd="${docker_cmd} --push"
-  fi
-
-  # Build node image first to allow the workload image to use it.
-  ${docker_cmd} -t "${node_image_name}" -f "${node_dockerfile}" "${AVALANCHE_PATH}"
-
-  if [[ -n "${node_only}" ]]; then
-    # Skip building the config and workload images. Supports building the avalanchego
-    # node image as the base image for the xsvm node image.
-    return
-  fi
-
-  TARGET_PATH="${AVALANCHE_PATH}/build/antithesis/${test_setup}"
-  if [[ -d "${TARGET_PATH}" ]]; then
-    # Ensure the target path is empty before generating the compose config
-    rm -r "${TARGET_PATH:?}"
+  if [[ -z "${node_only}" ]]; then
+    echo "Building node image for ${test_setup}"
+  else
+    echo "Building images for ${test_setup}"
   fi
+  build_antithesis_images "${GO_VERSION}" "${image_prefix}" "antithesis-${test_setup}" "${IMAGE_TAG}" "${IMAGE_TAG}" \
+                          "${AVALANCHE_PATH}/tests/antithesis/${test_setup}/Dockerfile" "${uninstrumented_node_dockerfile}" \
+                          "${AVALANCHE_PATH}" "${node_only}"
+}
 
-  # Define the env vars for the compose config generation
-  COMPOSE_ENV="TARGET_PATH=${TARGET_PATH} IMAGE_TAG=${TAG}"
+if [[ "${TEST_SETUP}" == "avalanchego" ]]; then
+  build_builder_image_for_avalanchego
 
-  if [[ "${test_setup}" == "xsvm" ]]; then
-    # Ensure avalanchego and xsvm binaries are available to create an initial db state that includes subnets.
-    "${AVALANCHE_PATH}"/scripts/build.sh
-    "${AVALANCHE_PATH}"/scripts/build_xsvm.sh
-    COMPOSE_ENV="${COMPOSE_ENV} AVALANCHEGO_PATH=${AVALANCHE_PATH}/build/avalanchego AVALANCHEGO_PLUGIN_DIR=${HOME}/.avalanchego/plugins"
-  fi
+  echo "Generating compose configuration for ${TEST_SETUP}"
+  gen_antithesis_compose_config "${IMAGE_TAG}" "${AVALANCHE_PATH}/tests/antithesis/avalanchego/gencomposeconfig" \
+                                "${AVALANCHE_PATH}/build/antithesis/avalanchego"
 
-  # Generate compose config for copying into the config image
-  # shellcheck disable=SC2086
-  env ${COMPOSE_ENV} go run "${AVALANCHE_PATH}/tests/antithesis/${test_setup}/gencomposeconfig"
+  build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/Dockerfile" "${NODE_ONLY:-}"
+else
+  build_builder_image_for_avalanchego
 
-  # Build the config image
-  ${docker_cmd} -t "${config_image_name}" -f "${base_dockerfile}.config" "${AVALANCHE_PATH}"
+  # Only build the avalanchego node image to use as the base for the xsvm image. Provide an empty
+  # image prefix (the 1st argument) to prevent the image from being pushed
+  NODE_ONLY=1
+  build_antithesis_images_for_avalanchego avalanchego "" "${AVALANCHE_PATH}/Dockerfile" "${NODE_ONLY}"
 
-  # Build the workload image
-  ${docker_cmd} -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${AVALANCHE_PATH}"
-}
+  # Ensure avalanchego and xsvm binaries are available to create an initial db state that includes subnets.
+  echo "Building binaries required for configuring the ${TEST_SETUP} test setup"
+  "${AVALANCHE_PATH}"/scripts/build.sh
+  "${AVALANCHE_PATH}"/scripts/build_xsvm.sh
 
-TEST_SETUP="${TEST_SETUP:-}"
-if [[ "${TEST_SETUP}" == "avalanchego" ]]; then
-  build_images avalanchego "${AVALANCHE_PATH}/Dockerfile" "${IMAGE_PREFIX}" "${NODE_ONLY:-}"
-elif [[ "${TEST_SETUP}" == "xsvm" ]]; then
-  # Only build the node image to use as the base for the xsvm image. Provide an empty
-  # image prefix (the 3rd argument) to prevent the image from being pushed
-  NODE_ONLY=1
-  build_images avalanchego "${AVALANCHE_PATH}/Dockerfile" "" "${NODE_ONLY}"
+  echo "Generating compose configuration for ${TEST_SETUP}"
+  gen_antithesis_compose_config "${IMAGE_TAG}" "${AVALANCHE_PATH}/tests/antithesis/xsvm/gencomposeconfig" \
+                                "${AVALANCHE_PATH}/build/antithesis/xsvm" \
+                                "AVALANCHEGO_PATH=${AVALANCHE_PATH}/build/avalanchego AVALANCHEGO_PLUGIN_DIR=${HOME}/.avalanchego/plugins"
 
-  build_images xsvm "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile" "${IMAGE_PREFIX}"
-else
-  echo "TEST_SETUP must be set. Valid values are 'avalanchego' or 'xsvm'"
-  exit 255
+  build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile"
 fi

scripts/lib_build_antithesis_images.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# This script defines helper functions to enable building images for antithesis test setups. It is
+# intended to be reusable by repos other than avalanchego so almost all inputs are accepted as parameters
+# rather than being discovered from the environment.
+#
+# Since this file only defines functions, it is intended to be sourced rather than executed.
+
+# Build the image that enables compiling golang binaries for the node and workload image
+# builds. The builder image is intended to enable building instrumented binaries if built
+# on amd64 and non-instrumented binaries if built on arm64.
+function build_antithesis_builder_image {
+  local go_version=$1
+  local image_name=$2
+  local avalanchego_path=$3
+  local target_path=$4
+
+  local base_dockerfile="${avalanchego_path}/tests/antithesis/Dockerfile"
+  local builder_dockerfile="${base_dockerfile}.builder-instrumented"
+  if [[ "$(go env GOARCH)" == "arm64" ]]; then
+    # Antithesis instrumentation is only supported on amd64. On apple silicon (arm64),
+    # an uninstrumented Dockerfile will be used to enable local test development.
+    builder_dockerfile="${base_dockerfile}.builder-uninstrumented"
+  fi
+
+  docker buildx build --build-arg GO_VERSION="${go_version}" -t "${image_name}" -f "${builder_dockerfile}" "${target_path}"
+}
+
+# Build the antithesis node, workload, and config images.
+function build_antithesis_images {
+  local go_version=$1
+  local image_prefix=$2
+  local base_image_name=$3
+  local image_tag=$4
+  local node_image_tag=$5
+  local base_dockerfile=$6
+  local uninstrumented_node_dockerfile=$7
+  local target_path=$8
+  local node_only=${9:-}
+
+  # Define image names
+  if [[ -n "${image_prefix}" ]]; then
+    base_image_name="${image_prefix}/${base_image_name}"
+  fi
+  local node_image_name="${base_image_name}-node:${image_tag}"
+  local workload_image_name="${base_image_name}-workload:${image_tag}"
+  local config_image_name="${base_image_name}-config:${image_tag}"
+
+  # Define dockerfiles
+  local node_dockerfile="${base_dockerfile}.node"
+  # Working directory for instrumented builds
+  local builder_workdir="/instrumented/customer"
+  if [[ "$(go env GOARCH)" == "arm64" ]]; then
+    # Antithesis instrumentation is only supported on amd64. On apple silicon (arm64),
+    # uninstrumented Dockerfiles will be used to enable local test development.
+    node_dockerfile="${uninstrumented_node_dockerfile}"
+    # Working directory for uninstrumented builds
+    builder_workdir="/build"
+  fi
+
+  # Define default build command
+  local docker_cmd="docker buildx build\
+ --build-arg GO_VERSION=${go_version}\
+ --build-arg BUILDER_IMAGE_TAG=${image_tag}\
+ --build-arg BUILDER_WORKDIR=${builder_workdir}\
+ --build-arg AVALANCHEGO_NODE_IMAGE=antithesis-avalanchego-node:${node_image_tag}"
+
+  if [[ -n "${image_prefix}" && -z "${node_only}" ]]; then
+    # Push images with an image prefix since the prefix defines a registry location, and only if building
+    # all images. When building just the node image the image is only intended to be used locally.
+    docker_cmd="${docker_cmd} --push"
+  fi
+
+  # Build node image first to allow the workload image to use it.
+  ${docker_cmd} -t "${node_image_name}" -f "${node_dockerfile}" "${target_path}"
+
+  if [[ -n "${node_only}" ]]; then
+    # Skip building the config and workload images. Supports building the avalanchego node image as the
+    # base image for a VM node image.
+    return
+  fi
+
+  # Build the config image
+  ${docker_cmd} -t "${config_image_name}" -f "${base_dockerfile}.config" "${target_path}"
+
+  # Build the workload image
+  ${docker_cmd} -t "${workload_image_name}" -f "${base_dockerfile}.workload" "${target_path}"
+}
+
+# Generate the docker compose configuration for the antithesis config image.
+function gen_antithesis_compose_config {
+  local image_tag=$1
+  local exe_path=$2
+  local target_path=$3
+  local extra_compose_args=${4:-}
+
+  if [[ -d "${target_path}" ]]; then
+    # Ensure the target path is empty before generating the compose config
+    rm -r "${target_path:?}"
+  fi
+
+  # Define the env vars for the compose config generation
+  local compose_env="TARGET_PATH=${target_path} IMAGE_TAG=${image_tag} ${extra_compose_args}"
+
+  # Generate compose config for copying into the config image
+  # shellcheck disable=SC2086
+  env ${compose_env} go run "${exe_path}"
+}

scripts/lib_test_antithesis_images.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Validates the compose configuration of the antithesis config image
+# identified by IMAGE_NAME and IMAGE_TAG by:
+#
+#   1. Extracting the docker compose configuration from the image
+#   2. Running the workload and its target network without error for a minute
+#   3. Stopping the workload and its target network
+#
+# This script is intended to be sourced rather than executed directly.
+
+if [[ -z "${IMAGE_NAME:-}" || -z "${IMAGE_TAG:-}" ]]; then
+  echo "IMAGE_NAME and IMAGE_TAG must be set"
+  exit 1
+fi
+
+# Create a container from the config image to extract compose configuration from
+CONTAINER_NAME="tmp-${IMAGE_NAME}"
+docker create --name "${CONTAINER_NAME}" "${IMAGE_NAME}:${IMAGE_TAG}" /bin/true
+
+# Create a temporary directory to write the compose configuration to
+TMPDIR="$(mktemp -d)"
+echo "using temporary directory ${TMPDIR} as the docker compose path"
+
+COMPOSE_FILE="${TMPDIR}/docker-compose.yml"
+COMPOSE_CMD="docker compose -f ${COMPOSE_FILE}"
+
+# Ensure cleanup
+function cleanup {
+  echo "removing temporary container"
+  docker rm "${CONTAINER_NAME}"
+  echo "stopping and removing the docker compose project"
+  ${COMPOSE_CMD} down --volumes
+  if [[ -z "${DEBUG:-}" ]]; then
+    echo "removing temporary dir"
+    rm -rf "${TMPDIR}"
+  fi
+}
+trap cleanup EXIT
+
+# Copy the docker-compose.yml file out of the container
+docker cp "${CONTAINER_NAME}":/docker-compose.yml "${COMPOSE_FILE}"
+
+# Copy the volume paths out of the container
+docker cp "${CONTAINER_NAME}":/volumes "${TMPDIR}/"
+
+# Run the docker compose project for 30 seconds without error. Local
+# network bootstrap is ~6s, but github workers can be much slower.
+${COMPOSE_CMD} up -d
+sleep 30
+if ${COMPOSE_CMD} ps -q | xargs docker inspect -f '{{ .State.Status }}' | grep -v 'running'; then
+  echo "An error occurred."
+  exit 1
+fi
+
+echo "Successfully invoked the antithesis test setup configured by ${IMAGE_NAME}:${IMAGE_TAG}"