Use base_image/cloud.yml as release version SoT

Fixes: #135

Previously, a maintainer integrating a new OS release was required to
update multiple files.  Reduce this burden by encoding the
release-version in base-image YAML and retrieving it as needed.  This
also has the added side-benefit of allowing the release version to be
tagged on the images themselves for reference.

Signed-off-by: Chris Evich <[email protected]>

Makefile

@@ -156,6 +156,13 @@ $(_TEMPDIR)/user-data: $(_TEMPDIR) $(_TEMPDIR)/cidata.ssh.pub $(_TEMPDIR)/cidata
 .PHONY: cidata
 cidata: $(_TEMPDIR)/user-data $(_TEMPDIR)/meta-data
 
+override _fedora_release = $(shell bash get_base_release.sh FEDORA)
+override _prior_fedora_release = $(shell bash get_base_release.sh PRIOR_FEDORA)
+override _ubuntu_release = $(shell bash get_base_release.sh UBUNTU)
+define build_podman_container
+	$(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(call err_if_empty,_$(1)_release)
+endef
+
 # First argument is the path to the template JSON, second
 # argument is the path to AWS_SHARED_CREDENTIALS_FILE
 # when required.  N/B: GAC_FILEPATH is always required.
@@ -167,6 +174,9 @@ define packer_build
 			$(PACKER_INSTALL_DIR)/packer build \
 			-force \
 			-var TEMPDIR="$(_TEMPDIR)" \
+			-var FEDORA_RELEASE="$(call err_if_empty,_fedora_release)" \
+			-var PRIOR_FEDORA_RELEASE="$(call err_if_empty,_prior_fedora_release)" \
+			-var UBUNTU_RELEASE="$(call err_if_empty,_ubuntu_release)" \
 			$(if $(PACKER_BUILDS),-only=$(PACKER_BUILDS)) \
 			$(if $(IMG_SFX),-var IMG_SFX=$(IMG_SFX)) \
 			$(if $(DEBUG_NESTED_VM),-var TTYDEV=$(shell tty),-var TTYDEV=/dev/null) \
@@ -176,8 +186,8 @@ endef
 
 .PHONY: image_builder
 image_builder: image_builder/manifest.json ## Create image-building image and import into GCE (needed for making all other images)
-image_builder/manifest.json: image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer
-	$(call packer_build,$<,)
+image_builder/manifest.json: base_images/cloud.json image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer
+	$(call packer_build,image_builder/gce.json,)
 
 # Note: We assume this repo is checked out somewhere under the caller's
 # home-dir for bind-mounting purposes.  Otherwise possibly necessary
@@ -210,26 +220,19 @@ base_images/manifest.json: base_images/cloud.json $(wildcard base_images/*.sh) c
 
 .PHONY: cache_images
 cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE.  Optionally, set PACKER_BUILDS=<csv> to select builder(s).
-cache_images/manifest.json: cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
+cache_images/manifest.json: base_images/cloud.json cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
 	$(call packer_build,cache_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE))
 
-override _fedora_podman_release := $(file < podman/fedora_release)
-override _prior-fedora_podman_release := $(file < podman/prior-fedora_release)
-override _ubuntu_podman_release := $(file < podman/ubuntu_release)
-define build_podman_container
-	$(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(_$(1)_release)
-endef
-
 .PHONY: fedora_podman
-fedora_podman:  ## Build Fedora podman development container
+fedora_podman:  base_images/cloud.json  ## Build Fedora podman development container
 	$(call build_podman_container,$@,fedora)
 
 .PHONY: prior-fedora_podman
-prior-fedora_podman:  ## Build Prior-Fedora podman development container
+prior-fedora_podman:  base_images/cloud.json  ## Build Prior-Fedora podman development container
 	$(call build_podman_container,$@,prior-fedora)
 
 .PHONY: ubuntu_podman
-ubuntu_podman:  ## Build Ubuntu podman development container
+ubuntu_podman:  base_images/cloud.json  ## Build Ubuntu podman development container
 	$(call build_podman_container,$@,ubuntu)
 
 $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/%
@@ -249,7 +252,7 @@ skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar  ## Build Skopeo development and CI c
 $(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora
 	podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \
 		--security-opt seccomp=unconfined \
-		--build-arg=BASE_TAG=$(_fedora_podman_release) \
+		--build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \
 		-v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \
 		skopeo_cidev
 	rm -f $@
@@ -260,7 +263,7 @@ ccia: $(_TEMPDIR)/ccia.tar  ## Build the Cirrus-CI Artifacts container image
 $(_TEMPDIR)/ccia.tar: ccia/Containerfile
 	podman build -t ccia:$(call err_if_empty,IMG_SFX) \
 		--security-opt seccomp=unconfined \
-		--build-arg=BASE_TAG=$(_fedora_podman_release) \
+		--build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \
 		ccia
 	rm -f $@
 	podman save --quiet -o $@ ccia:$(IMG_SFX)

base_images/cloud.yml

@@ -14,11 +14,11 @@ variables:  # Empty value means it must be passed in on command-line
     # Naming suffix for images to prevent clashes
     IMG_SFX:
 
-    # BIG-FAT-WARNING:  When updating the image names and/or URLs below,
-    # ensure the distro version numbers contained in the `podman/*_release`
-    # files exactly match.  These represent the container base-image tags
-    # to build from - just as the sources below are the base-images to
-    # start from building VM images.
+    # This data is also used when building cache and container images.
+    # It is assumed to match all the other OS variables below.
+    UBUNTU_RELEASE: 22.04
+    FEDORA_RELEASE: 36
+    PRIOR_FEDORA_RELEASE: 35
 
     # Upstream source for Ubuntu image to duplicate (prevents expiration).
     # Use the most recent image based on this family name.
@@ -61,6 +61,7 @@ builders:
         src: '{{user `UBUNTU_BASE_FAMILY`}}'
         stage: 'base'
         arch: 'x86_64'
+        release: 'ubuntu-{{user `UBUNTU_RELEASE`}}'
       # Gotcha: https://www.packer.io/docs/builders/googlecompute#gotchas
       ssh_username: 'packer'
       temporary_key_pair_type: ed25519
@@ -144,6 +145,7 @@ builders:
         Name: 'fedora-aws-b{{user `IMG_SFX`}}'
         src: '{{user `FEDORAPROJECT_AMI`}}'
         automation: 'true'
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
       run_tags: *awstags
       run_volume_tags: *awstags
       snapshot_tags: *awstags
@@ -213,14 +215,19 @@ post-processors:
         image_family: '{{build_name}}-base'
         # Can't save the url in an image_label
         image_description: '{{user `FEDORA_IMAGE_URL`}}'
-        image_labels: &importlabels
+        image_labels:
           <<: *imgcpylabels
           src: 'fedoraproject'
+          release: '{{user `FEDORA_RELEASE`}}'
       - <<: *gcp_import
         only: ['prior-fedora']
         image_name: "prior-fedora-b{{user `IMG_SFX`}}"
         image_family: '{{build_name}}-base'
         image_description: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
+        image_labels:
+          <<: *imgcpylabels
+          src: 'fedoraproject'
+          release: '{{user `PRIOR_FEDORA_RELEASE`}}'
       # This is critical, especially for the aws builders.
       # Producing the cache-images from these base images
       # needs to lookup the runtime-produced AMI ID.

cache_images/cloud.yml

@@ -13,6 +13,10 @@ variables:  # Empty value means it must be passed in on command-line
     # Required path to service account credentials file
     GAC_FILEPATH: "{{env `GAC_FILEPATH`}}"
 
+    # Makefile sets these from values in base_images/cloud.yml
+    UBUNTU_RELEASE:
+    FEDORA_RELEASE:
+    PRIOR_FEDORA_RELEASE:
 
 builders:
     - &gce_hosted_image
@@ -28,9 +32,10 @@ builders:
       zone: 'us-central1-a'
       disk_size: 20  # REQUIRED: Runtime allocation > this value
       disable_default_service_account: true
-      labels:  # For the VM
+      labels: &gce_labels # For the VM
         sfx: '{{user `IMG_SFX`}}'
         src: '{{ build_name }}-b{{user `IMG_SFX` }}'
+        release: 'ubuntu-{{user `FEDORA_RELEASE`}}'
         stage: cache
       ssh_username: packer  # arbitrary, packer will create & setup w/ temp. keypair
       ssh_pty: 'true'
@@ -41,15 +46,20 @@ builders:
 
     - <<: *gce_hosted_image
       name: 'fedora'
+      labels: &fedora_gce_labels
+        <<: *gce_labels
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
 
     - <<: *gce_hosted_image
       name: 'prior-fedora'
+      labels: *fedora_gce_labels
 
     - &aux_fed_img
       <<: *gce_hosted_image
       name: 'build-push'
       source_image: 'fedora-b{{user `IMG_SFX`}}'
       source_image_family: 'fedora-base'
+      labels: *fedora_gce_labels
 
     - <<: *aux_fed_img
       name: 'fedora-podman-py'
@@ -98,17 +108,18 @@ builders:
           volume_type: 'gp2'
           delete_on_termination: true
       # These are critical and used by security-polciy to enforce instance launch limits.
-      tags: &tags
+      tags: &ami_tags
         # EC2 expects "Name" tag to be capitalized
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         sfx: '{{user `IMG_SFX`}}'
         src: '{{.SourceAMI}}'  # Generated AMI ID looked up at runtime
         automation: 'true'
         stage: 'cache'
         arch: 'x86_64'
-      run_tags: *tags
-      run_volume_tags: *tags
-      snapshot_tags: *tags
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
+      run_tags: *ami_tags
+      run_volume_tags: *ami_tags
+      snapshot_tags: *ami_tags
       # Also required to make AMI private
       ami_users:
         - *accountid
@@ -129,7 +140,7 @@ builders:
             name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'
       instance_type: 't4g.medium'  # arm64 type
       tags: &netavark_tags
-        <<: *tags
+        <<: *ami_tags
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         arch: 'arm64'
       run_tags: *netavark_tags
@@ -147,7 +158,7 @@ builders:
             name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'
       instance_type: 't4g.medium'  # arm64 type
       tags: &podman_tags
-        <<: *tags
+        <<: *ami_tags
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         arch: 'arm64'
       run_tags: *podman_tags

get_base_release.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# This script is intended to be called by the Makefile only.
+# Any other use may produce unexpected results.  It expects
+# to be called with the name of a supported OS in all upper-case.
+# The value of the corresponding <NAME>_RELEASE variable will be
+# extracted from base_images/cloud.json and printed to stdout.
+
+set -eo pipefail
+
+SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}")
+SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH")
+REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH")
+CLOUD_JSON="$REPO_DIRPATH/base_images/cloud.json"
+
+# shellcheck source=./lib.sh
+source "$REPO_DIRPATH/lib.sh"
+
+[[ -r "$CLOUD_JSON" ]] || die "Cannot read from '$CLOUD_JSON'"
+
+jq -r -e ".variables.${1}_RELEASE" $CLOUD_JSON

image_builder/gce.yml

@@ -14,6 +14,10 @@ variables:
     # N/B: There are length/character limitations in GCE for image names
     IMG_SFX: '{{ timestamp }}'
 
+    # These aren't used, but are required to be present.
+    UBUNTU_RELEASE:
+    FEDORA_RELEASE:
+    PRIOR_FEDORA_RELEASE:
 
 builders:
     - name: 'image-builder'