From 289cc6a97964dbfcb038c88ee596713b356e3687 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 23 Aug 2022 14:48:27 -0400 Subject: [PATCH] Use base_image/cloud.yml as release version SoT Fixes: #135 Previously, a maintainer integrating a new OS release was required to update multiple files. Reduce this burden by encoding the release-version in base-image YAML and retrieving it as needed. This also has the added side-benefit of allowing the release version to be tagged on the images themselves for reference. Signed-off-by: Chris Evich --- Makefile | 37 ++++++++++++++++++++----------------- base_images/cloud.yml | 19 +++++++++++++------ cache_images/cloud.yml | 25 ++++++++++++++++++------- get_base_release.sh | 21 +++++++++++++++++++++ image_builder/gce.yml | 4 ++++ podman/fedora_release | 1 - podman/prior-fedora_release | 1 - podman/ubuntu_release | 1 - 8 files changed, 76 insertions(+), 33 deletions(-) create mode 100644 get_base_release.sh delete mode 100644 podman/fedora_release delete mode 100644 podman/prior-fedora_release delete mode 100644 podman/ubuntu_release diff --git a/Makefile b/Makefile index a7c7cda2..179886fb 100644 --- a/Makefile +++ b/Makefile @@ -156,6 +156,13 @@ $(_TEMPDIR)/user-data: $(_TEMPDIR) $(_TEMPDIR)/cidata.ssh.pub $(_TEMPDIR)/cidata .PHONY: cidata cidata: $(_TEMPDIR)/user-data $(_TEMPDIR)/meta-data +override _fedora_release = $(shell bash get_base_release.sh FEDORA) +override _prior_fedora_release = $(shell bash get_base_release.sh PRIOR_FEDORA) +override _ubuntu_release = $(shell bash get_base_release.sh UBUNTU) +define build_podman_container + $(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(call err_if_empty,_$(1)_release) +endef + # First argument is the path to the template JSON, second # argument is the path to AWS_SHARED_CREDENTIALS_FILE # when required. N/B: GAC_FILEPATH is always required. @@ -167,6 +174,9 @@ define packer_build $(PACKER_INSTALL_DIR)/packer build \ -force \ -var TEMPDIR="$(_TEMPDIR)" \ + -var FEDORA_RELEASE="$(call err_if_empty,_fedora_release)" \ + -var PRIOR_FEDORA_RELEASE="$(call err_if_empty,_prior_fedora_release)" \ + -var UBUNTU_RELEASE="$(call err_if_empty,_ubuntu_release)" \ $(if $(PACKER_BUILDS),-only=$(PACKER_BUILDS)) \ $(if $(IMG_SFX),-var IMG_SFX=$(IMG_SFX)) \ $(if $(DEBUG_NESTED_VM),-var TTYDEV=$(shell tty),-var TTYDEV=/dev/null) \ @@ -176,8 +186,8 @@ endef .PHONY: image_builder image_builder: image_builder/manifest.json ## Create image-building image and import into GCE (needed for making all other images) -image_builder/manifest.json: image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer - $(call packer_build,$<,) +image_builder/manifest.json: base_images/cloud.json image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer + $(call packer_build,image_builder/gce.json,) # Note: We assume this repo is checked out somewhere under the caller's # home-dir for bind-mounting purposes. Otherwise possibly necessary @@ -210,26 +220,19 @@ base_images/manifest.json: base_images/cloud.json $(wildcard base_images/*.sh) c .PHONY: cache_images cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE. Optionally, set PACKER_BUILDS= to select builder(s). -cache_images/manifest.json: cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer +cache_images/manifest.json: base_images/cloud.json cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer $(call packer_build,cache_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE)) -override _fedora_podman_release := $(file < podman/fedora_release) -override _prior-fedora_podman_release := $(file < podman/prior-fedora_release) -override _ubuntu_podman_release := $(file < podman/ubuntu_release) -define build_podman_container - $(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(_$(1)_release) -endef - .PHONY: fedora_podman -fedora_podman: ## Build Fedora podman development container +fedora_podman: base_images/cloud.json ## Build Fedora podman development container $(call build_podman_container,$@,fedora) .PHONY: prior-fedora_podman -prior-fedora_podman: ## Build Prior-Fedora podman development container +prior-fedora_podman: base_images/cloud.json ## Build Prior-Fedora podman development container $(call build_podman_container,$@,prior-fedora) .PHONY: ubuntu_podman -ubuntu_podman: ## Build Ubuntu podman development container +ubuntu_podman: base_images/cloud.json ## Build Ubuntu podman development container $(call build_podman_container,$@,ubuntu) $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/% @@ -246,10 +249,10 @@ $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_i .PHONY: skopeo_cidev skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar ## Build Skopeo development and CI container -$(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora +$(_TEMPDIR)/skopeo_cidev.tar: base_images/cloud.json podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \ --security-opt seccomp=unconfined \ - --build-arg=BASE_TAG=$(_fedora_podman_release) \ + --build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \ -v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \ skopeo_cidev rm -f $@ @@ -257,10 +260,10 @@ $(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $( .PHONY: ccia ccia: $(_TEMPDIR)/ccia.tar ## Build the Cirrus-CI Artifacts container image -$(_TEMPDIR)/ccia.tar: ccia/Containerfile +$(_TEMPDIR)/ccia.tar: ccia/Containerfile base_images/cloud.json podman build -t ccia:$(call err_if_empty,IMG_SFX) \ --security-opt seccomp=unconfined \ - --build-arg=BASE_TAG=$(_fedora_podman_release) \ + --build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \ ccia rm -f $@ podman save --quiet -o $@ ccia:$(IMG_SFX) diff --git a/base_images/cloud.yml b/base_images/cloud.yml index 995ca284..1e439ac9 100644 --- a/base_images/cloud.yml +++ b/base_images/cloud.yml @@ -14,11 +14,11 @@ variables: # Empty value means it must be passed in on command-line # Naming suffix for images to prevent clashes IMG_SFX: - # BIG-FAT-WARNING: When updating the image names and/or URLs below, - # ensure the distro version numbers contained in the `podman/*_release` - # files exactly match. These represent the container base-image tags - # to build from - just as the sources below are the base-images to - # start from building VM images. + # This data is also used when building cache and container images. + # It is assumed to match all the other OS variables below. + UBUNTU_RELEASE: 22.04 + FEDORA_RELEASE: 36 + PRIOR_FEDORA_RELEASE: 35 # Upstream source for Ubuntu image to duplicate (prevents expiration). # Use the most recent image based on this family name. @@ -61,6 +61,7 @@ builders: src: '{{user `UBUNTU_BASE_FAMILY`}}' stage: 'base' arch: 'x86_64' + release: 'ubuntu-{{user `UBUNTU_RELEASE`}}' # Gotcha: https://www.packer.io/docs/builders/googlecompute#gotchas ssh_username: 'packer' temporary_key_pair_type: ed25519 @@ -144,6 +145,7 @@ builders: Name: 'fedora-aws-b{{user `IMG_SFX`}}' src: '{{user `FEDORAPROJECT_AMI`}}' automation: 'true' + release: 'fedora-{{user `FEDORA_RELEASE`}}' run_tags: *awstags run_volume_tags: *awstags snapshot_tags: *awstags @@ -213,14 +215,19 @@ post-processors: image_family: '{{build_name}}-base' # Can't save the url in an image_label image_description: '{{user `FEDORA_IMAGE_URL`}}' - image_labels: &importlabels + image_labels: <<: *imgcpylabels src: 'fedoraproject' + release: '{{user `FEDORA_RELEASE`}}' - <<: *gcp_import only: ['prior-fedora'] image_name: "prior-fedora-b{{user `IMG_SFX`}}" image_family: '{{build_name}}-base' image_description: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' + image_labels: + <<: *imgcpylabels + src: 'fedoraproject' + release: '{{user `PRIOR_FEDORA_RELEASE`}}' # This is critical, especially for the aws builders. # Producing the cache-images from these base images # needs to lookup the runtime-produced AMI ID. diff --git a/cache_images/cloud.yml b/cache_images/cloud.yml index c2390817..9b8f9e51 100644 --- a/cache_images/cloud.yml +++ b/cache_images/cloud.yml @@ -13,6 +13,10 @@ variables: # Empty value means it must be passed in on command-line # Required path to service account credentials file GAC_FILEPATH: "{{env `GAC_FILEPATH`}}" + # Makefile sets these from values in base_images/cloud.yml + UBUNTU_RELEASE: + FEDORA_RELEASE: + PRIOR_FEDORA_RELEASE: builders: - &gce_hosted_image @@ -28,9 +32,10 @@ builders: zone: 'us-central1-a' disk_size: 20 # REQUIRED: Runtime allocation > this value disable_default_service_account: true - labels: # For the VM + labels: &gce_labels # For the VM sfx: '{{user `IMG_SFX`}}' src: '{{ build_name }}-b{{user `IMG_SFX` }}' + release: 'ubuntu-{{user `FEDORA_RELEASE`}}' stage: cache ssh_username: packer # arbitrary, packer will create & setup w/ temp. keypair ssh_pty: 'true' @@ -41,15 +46,20 @@ builders: - <<: *gce_hosted_image name: 'fedora' + labels: &fedora_gce_labels + <<: *gce_labels + release: 'fedora-{{user `FEDORA_RELEASE`}}' - <<: *gce_hosted_image name: 'prior-fedora' + labels: *fedora_gce_labels - &aux_fed_img <<: *gce_hosted_image name: 'build-push' source_image: 'fedora-b{{user `IMG_SFX`}}' source_image_family: 'fedora-base' + labels: *fedora_gce_labels - <<: *aux_fed_img name: 'fedora-podman-py' @@ -98,7 +108,7 @@ builders: volume_type: 'gp2' delete_on_termination: true # These are critical and used by security-polciy to enforce instance launch limits. - tags: &tags + tags: &ami_tags # EC2 expects "Name" tag to be capitalized Name: '{{build_name}}-c{{user `IMG_SFX`}}' sfx: '{{user `IMG_SFX`}}' @@ -106,9 +116,10 @@ builders: automation: 'true' stage: 'cache' arch: 'x86_64' - run_tags: *tags - run_volume_tags: *tags - snapshot_tags: *tags + release: 'fedora-{{user `FEDORA_RELEASE`}}' + run_tags: *ami_tags + run_volume_tags: *ami_tags + snapshot_tags: *ami_tags # Also required to make AMI private ami_users: - *accountid @@ -129,7 +140,7 @@ builders: name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}' instance_type: 't4g.medium' # arm64 type tags: &netavark_tags - <<: *tags + <<: *ami_tags Name: '{{build_name}}-c{{user `IMG_SFX`}}' arch: 'arm64' run_tags: *netavark_tags @@ -147,7 +158,7 @@ builders: name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}' instance_type: 't4g.medium' # arm64 type tags: &podman_tags - <<: *tags + <<: *ami_tags Name: '{{build_name}}-c{{user `IMG_SFX`}}' arch: 'arm64' run_tags: *podman_tags diff --git a/get_base_release.sh b/get_base_release.sh new file mode 100644 index 00000000..65bfb2c0 --- /dev/null +++ b/get_base_release.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# This script is intended to be called by the Makefile only. +# Any other use may produce unexpected results. It expects +# to be called with the name of a supported OS in all upper-case. +# The value of the corresponding _RELEASE variable will be +# extracted from base_images/cloud.json and printed to stdout. + +set -eo pipefail + +SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") +REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH") +CLOUD_JSON="$REPO_DIRPATH/base_images/cloud.json" + +# shellcheck source=./lib.sh +source "$REPO_DIRPATH/lib.sh" + +[[ -r "$CLOUD_JSON" ]] || die "Cannot read from '$CLOUD_JSON'" + +jq -r -e ".variables.${1}_RELEASE" $CLOUD_JSON diff --git a/image_builder/gce.yml b/image_builder/gce.yml index b521e2bd..716e89ab 100644 --- a/image_builder/gce.yml +++ b/image_builder/gce.yml @@ -14,6 +14,10 @@ variables: # N/B: There are length/character limitations in GCE for image names IMG_SFX: '{{ timestamp }}' + # These aren't used, but are required to be present. + UBUNTU_RELEASE: + FEDORA_RELEASE: + PRIOR_FEDORA_RELEASE: builders: - name: 'image-builder' diff --git a/podman/fedora_release b/podman/fedora_release deleted file mode 100644 index 7facc899..00000000 --- a/podman/fedora_release +++ /dev/null @@ -1 +0,0 @@ -36 diff --git a/podman/prior-fedora_release b/podman/prior-fedora_release deleted file mode 100644 index 8f92bfdd..00000000 --- a/podman/prior-fedora_release +++ /dev/null @@ -1 +0,0 @@ -35 diff --git a/podman/ubuntu_release b/podman/ubuntu_release deleted file mode 100644 index dcdf6284..00000000 --- a/podman/ubuntu_release +++ /dev/null @@ -1 +0,0 @@ -22.04