Use base_image/cloud.yml as release version SoT

Fixes: #135

Previously, a maintainer integrating a new OS release was required to
update multiple files.  Reduce this burden by encoding the
release-version in base-image YAML and retrieving it as needed.  This
also has the added side-benefit of allowing the release version to be
tagged on the images themselves for reference.

Signed-off-by: Chris Evich <[email protected]>

Makefile

@@ -159,6 +159,13 @@ $(_TEMPDIR)/user-data: $(_TEMPDIR) $(_TEMPDIR)/cidata.ssh.pub $(_TEMPDIR)/cidata
 .PHONY: cidata
 cidata: $(_TEMPDIR)/user-data $(_TEMPDIR)/meta-data
 
+override _fedora_release = $(shell bash get_base_release.sh FEDORA)
+override _prior_fedora_release = $(shell bash get_base_release.sh PRIOR_FEDORA)
+override _ubuntu_release = $(shell bash get_base_release.sh UBUNTU)
+define build_podman_container
+	$(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(2)
+endef
+
 # First argument is the path to the template JSON, second
 # argument is the path to AWS_SHARED_CREDENTIALS_FILE
 # when required.  N/B: GAC_FILEPATH is always required.
@@ -170,6 +177,9 @@ define packer_build
 			$(PACKER_INSTALL_DIR)/packer build \
 			-force \
 			-var TEMPDIR="$(_TEMPDIR)" \
+			-var FEDORA_RELEASE="$(call err_if_empty,_fedora_release)" \
+			-var PRIOR_FEDORA_RELEASE="$(call err_if_empty,_prior_fedora_release)" \
+			-var UBUNTU_RELEASE="$(call err_if_empty,_ubuntu_release)" \
 			$(if $(PACKER_BUILDS),-only=$(PACKER_BUILDS)) \
 			$(if $(IMG_SFX),-var IMG_SFX=$(IMG_SFX)) \
 			$(if $(DEBUG_NESTED_VM),-var TTYDEV=$(shell tty),-var TTYDEV=/dev/null) \
@@ -179,8 +189,8 @@ endef
 
 .PHONY: image_builder
 image_builder: image_builder/manifest.json ## Create image-building image and import into GCE (needed for making all other images)
-image_builder/manifest.json: image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer
-	$(call packer_build,$<,)
+image_builder/manifest.json: base_images/cloud.json image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer
+	$(call packer_build,image_builder/gce.json,)
 
 # Note: We assume this repo is checked out somewhere under the caller's
 # home-dir for bind-mounting purposes.  Otherwise possibly necessary
@@ -213,27 +223,20 @@ base_images/manifest.json: base_images/cloud.json $(wildcard base_images/*.sh) c
 
 .PHONY: cache_images
 cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE.  Optionally, set PACKER_BUILDS=<csv> to select builder(s).
-cache_images/manifest.json: cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
+cache_images/manifest.json: base_images/cloud.json cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
 	$(call packer_build,cache_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE))
 
-override _fedora_podman_release := $(file < podman/fedora_release)
-override _prior-fedora_podman_release := $(file < podman/prior-fedora_release)
-override _ubuntu_podman_release := $(file < podman/ubuntu_release)
-define build_podman_container
-	$(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(_$(1)_release)
-endef
-
 .PHONY: fedora_podman
-fedora_podman:  ## Build Fedora podman development container
-	$(call build_podman_container,$@,fedora)
+fedora_podman:  base_images/cloud.json  ## Build Fedora podman development container
+	$(call build_podman_container,$@,$(call err_if_empty,_fedora_release))
 
 .PHONY: prior-fedora_podman
-prior-fedora_podman:  ## Build Prior-Fedora podman development container
-	$(call build_podman_container,$@,prior-fedora)
+prior-fedora_podman:  base_images/cloud.json  ## Build Prior-Fedora podman development container
+	$(call build_podman_container,$@,$(call err_if_empty,_prior_fedora_release))
 
 .PHONY: ubuntu_podman
-ubuntu_podman:  ## Build Ubuntu podman development container
-	$(call build_podman_container,$@,ubuntu)
+ubuntu_podman:  base_images/cloud.json  ## Build Ubuntu podman development container
+	$(call build_podman_container,$@,$(call err_if_empty,_ubuntu_release))
 
 $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/%
 	podman build -t $*_podman:$(call err_if_empty,IMG_SFX) \
@@ -249,21 +252,21 @@ $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_i
 
 .PHONY: skopeo_cidev
 skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar  ## Build Skopeo development and CI container
-$(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora
+$(_TEMPDIR)/skopeo_cidev.tar: base_images/cloud.json $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora
 	podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \
 		--security-opt seccomp=unconfined \
-		--build-arg=BASE_TAG=$(_fedora_podman_release) \
+		--build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \
 		-v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \
 		skopeo_cidev
 	rm -f $@
 	podman save --quiet -o $@ skopeo_cidev:$(IMG_SFX)
 
 .PHONY: ccia
 ccia: $(_TEMPDIR)/ccia.tar  ## Build the Cirrus-CI Artifacts container image
-$(_TEMPDIR)/ccia.tar: ccia/Containerfile
+$(_TEMPDIR)/ccia.tar: ccia/Containerfile base_images/cloud.json
 	podman build -t ccia:$(call err_if_empty,IMG_SFX) \
 		--security-opt seccomp=unconfined \
-		--build-arg=BASE_TAG=$(_fedora_podman_release) \
+		--build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \
 		ccia
 	rm -f $@
 	podman save --quiet -o $@ ccia:$(IMG_SFX)

base_images/cloud.yml

@@ -17,11 +17,11 @@ variables:  # Empty value means it must be passed in on command-line
     # Allows providing handy cross-reference to the build log
     CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}"
 
-    # BIG-FAT-WARNING:  When updating the image names and/or URLs below,
-    # ensure the distro version numbers contained in the `podman/*_release`
-    # files exactly match.  These represent the container base-image tags
-    # to build from - just as the sources below are the base-images to
-    # start from building VM images.
+    # This data is also used when building cache and container images.
+    # It is assumed to match all the other OS variables below.
+    UBUNTU_RELEASE: "22.04"
+    FEDORA_RELEASE: "36"
+    PRIOR_FEDORA_RELEASE: "35"
 
     # Upstream source for Ubuntu image to duplicate (prevents expiration).
     # Use the most recent image based on this family name.
@@ -64,6 +64,7 @@ builders:
         src: '{{user `UBUNTU_BASE_FAMILY`}}'
         stage: 'base'
         arch: 'x86_64'
+        release: 'ubuntu-{{user `UBUNTU_RELEASE`}}'
       # Gotcha: https://www.packer.io/docs/builders/googlecompute#gotchas
       ssh_username: 'packer'
       temporary_key_pair_type: ed25519
@@ -147,6 +148,7 @@ builders:
         Name: 'fedora-aws-b{{user `IMG_SFX`}}'
         src: '{{user `FEDORAPROJECT_AMI`}}'
         automation: 'true'
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
       run_tags: *awstags
       run_volume_tags: *awstags
       snapshot_tags: *awstags
@@ -216,14 +218,19 @@ post-processors:
         image_family: '{{build_name}}-base'
         # Can't save the url in an image_label
         image_description: '{{user `FEDORA_IMAGE_URL`}}'
-        image_labels: &importlabels
+        image_labels:
           <<: *imgcpylabels
           src: 'fedoraproject'
+          release: '{{user `FEDORA_RELEASE`}}'
       - <<: *gcp_import
         only: ['prior-fedora']
         image_name: "prior-fedora-b{{user `IMG_SFX`}}"
         image_family: '{{build_name}}-base'
         image_description: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
+        image_labels:
+          <<: *imgcpylabels
+          src: 'fedoraproject'
+          release: '{{user `PRIOR_FEDORA_RELEASE`}}'
       # This is critical, especially for the aws builders.
       # Producing the cache-images from these base images
       # needs to lookup the runtime-produced AMI ID.

cache_images/cloud.yml

@@ -16,6 +16,11 @@ variables:  # Empty value means it must be passed in on command-line
     # Allows providing handy cross-reference to the build log
     CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}"
 
+    # Makefile sets these from values in base_images/cloud.yml
+    UBUNTU_RELEASE:
+    FEDORA_RELEASE:
+    PRIOR_FEDORA_RELEASE:
+
 builders:
     - &gce_hosted_image
       name: 'ubuntu'
@@ -30,9 +35,10 @@ builders:
       zone: 'us-central1-a'
       disk_size: 20  # REQUIRED: Runtime allocation > this value
       disable_default_service_account: true
-      labels:  # For the VM
+      labels: &gce_labels # For the VM
         sfx: '{{user `IMG_SFX`}}'
         src: '{{ build_name }}-b{{user `IMG_SFX` }}'
+        release: 'ubuntu-{{user `FEDORA_RELEASE`}}'
         stage: cache
       ssh_username: packer  # arbitrary, packer will create & setup w/ temp. keypair
       ssh_pty: 'true'
@@ -43,15 +49,20 @@ builders:
 
     - <<: *gce_hosted_image
       name: 'fedora'
+      labels: &fedora_gce_labels
+        <<: *gce_labels
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
 
     - <<: *gce_hosted_image
       name: 'prior-fedora'
+      labels: *fedora_gce_labels
 
     - &aux_fed_img
       <<: *gce_hosted_image
       name: 'build-push'
       source_image: 'fedora-b{{user `IMG_SFX`}}'
       source_image_family: 'fedora-base'
+      labels: *fedora_gce_labels
 
     - <<: *aux_fed_img
       name: 'fedora-podman-py'
@@ -100,17 +111,18 @@ builders:
           volume_type: 'gp2'
           delete_on_termination: true
       # These are critical and used by security-polciy to enforce instance launch limits.
-      tags: &tags
+      tags: &ami_tags
         # EC2 expects "Name" tag to be capitalized
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         sfx: '{{user `IMG_SFX`}}'
         src: '{{.SourceAMI}}'  # Generated AMI ID looked up at runtime
         automation: 'true'
         stage: 'cache'
         arch: 'x86_64'
-      run_tags: *tags
-      run_volume_tags: *tags
-      snapshot_tags: *tags
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
+      run_tags: *ami_tags
+      run_volume_tags: *ami_tags
+      snapshot_tags: *ami_tags
       # Also required to make AMI private
       ami_users:
         - *accountid
@@ -131,7 +143,7 @@ builders:
             name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'
       instance_type: 't4g.medium'  # arm64 type
       tags: &netavark_tags
-        <<: *tags
+        <<: *ami_tags
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         arch: 'arm64'
       run_tags: *netavark_tags
@@ -149,7 +161,7 @@ builders:
             name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'
       instance_type: 't4g.medium'  # arm64 type
       tags: &podman_tags
-        <<: *tags
+        <<: *ami_tags
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         arch: 'arm64'
       run_tags: *podman_tags

get_base_release.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# This script is intended to be called by the Makefile only.
+# Any other use may produce unexpected results.  It expects
+# to be called with the name of a supported OS in all upper-case.
+# The value of the corresponding <NAME>_RELEASE variable will be
+# extracted from base_images/cloud.json and printed to stdout.
+
+set -eo pipefail
+
+SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}")
+SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH")
+REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH")
+CLOUD_JSON="$REPO_DIRPATH/base_images/cloud.json"
+
+# shellcheck source=./lib.sh
+source "$REPO_DIRPATH/lib.sh"
+
+[[ -r "$CLOUD_JSON" ]] || die "Cannot read from '$CLOUD_JSON'"
+
+jq -r -e ".variables.${1}_RELEASE" $CLOUD_JSON

image_builder/gce.yml

@@ -17,6 +17,10 @@ variables:
     # Allows providing handy cross-reference to the build log
     CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}"
 
+    # These aren't used, but are required to be present.
+    UBUNTU_RELEASE:
+    FEDORA_RELEASE:
+    PRIOR_FEDORA_RELEASE:
 
 builders:
     - name: 'image-builder'