Use base_image/cloud.yml as release version SoT

Fixes: #135 Previously, a maintainer integrating a new OS release was required to update multiple files. Reduce this burden by encoding the release-version in base-image YAML and retrieving it as needed. This also has the added side-benefit of allowing the release version to be tagged on the images themselves for reference. Signed-off-by: Chris Evich <[email protected]>
containers · Aug 25, 2022 · a6e1030 · a6e1030
1 parent 480ff5d
commit a6e1030
Show file tree

Hide file tree

Showing 8 changed files with 86 additions and 42 deletions.
diff --git a/Makefile b/Makefile
@@ -159,6 +159,13 @@ $(_TEMPDIR)/user-data: $(_TEMPDIR) $(_TEMPDIR)/cidata.ssh.pub $(_TEMPDIR)/cidata
 .PHONY: cidata
 cidata: $(_TEMPDIR)/user-data $(_TEMPDIR)/meta-data
 
+override _fedora_release = $(shell bash get_base_release.sh FEDORA)
+override _prior_fedora_release = $(shell bash get_base_release.sh PRIOR_FEDORA)
+override _ubuntu_release = $(shell bash get_base_release.sh UBUNTU)
+define build_podman_container
+	$(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(2)
+endef
+
 # First argument is the path to the template JSON, second
 # argument is the path to AWS_SHARED_CREDENTIALS_FILE
 # when required.  N/B: GAC_FILEPATH is always required.
@@ -170,6 +177,9 @@ define packer_build
 			$(PACKER_INSTALL_DIR)/packer build \
 			-force \
 			-var TEMPDIR="$(_TEMPDIR)" \
+			-var FEDORA_RELEASE="$(call err_if_empty,_fedora_release)" \
+			-var PRIOR_FEDORA_RELEASE="$(call err_if_empty,_prior_fedora_release)" \
+			-var UBUNTU_RELEASE="$(call err_if_empty,_ubuntu_release)" \
 			$(if $(PACKER_BUILDS),-only=$(PACKER_BUILDS)) \
 			$(if $(IMG_SFX),-var IMG_SFX=$(IMG_SFX)) \
 			$(if $(DEBUG_NESTED_VM),-var TTYDEV=$(shell tty),-var TTYDEV=/dev/null) \
@@ -179,8 +189,8 @@ endef
 
 .PHONY: image_builder
 image_builder: image_builder/manifest.json ## Create image-building image and import into GCE (needed for making all other images)
-image_builder/manifest.json: image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer
-	$(call packer_build,$<,)
+image_builder/manifest.json: base_images/cloud.json image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer
+	$(call packer_build,image_builder/gce.json,)
 
 # Note: We assume this repo is checked out somewhere under the caller's
 # home-dir for bind-mounting purposes.  Otherwise possibly necessary
@@ -213,27 +223,20 @@ base_images/manifest.json: base_images/cloud.json $(wildcard base_images/*.sh) c
 
 .PHONY: cache_images
 cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE.  Optionally, set PACKER_BUILDS=<csv> to select builder(s).
-cache_images/manifest.json: cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
+cache_images/manifest.json: base_images/cloud.json cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
 	$(call packer_build,cache_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE))
 
-override _fedora_podman_release := $(file < podman/fedora_release)
-override _prior-fedora_podman_release := $(file < podman/prior-fedora_release)
-override _ubuntu_podman_release := $(file < podman/ubuntu_release)
-define build_podman_container
-	$(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(_$(1)_release)
-endef
-
 .PHONY: fedora_podman
-fedora_podman:  ## Build Fedora podman development container
-	$(call build_podman_container,$@,fedora)
+fedora_podman:  base_images/cloud.json  ## Build Fedora podman development container
+	$(call build_podman_container,$@,$(call err_if_empty,_fedora_release))
 
 .PHONY: prior-fedora_podman
-prior-fedora_podman:  ## Build Prior-Fedora podman development container
-	$(call build_podman_container,$@,prior-fedora)
+prior-fedora_podman:  base_images/cloud.json  ## Build Prior-Fedora podman development container
+	$(call build_podman_container,$@,$(call err_if_empty,_prior_fedora_release))
 
 .PHONY: ubuntu_podman
-ubuntu_podman:  ## Build Ubuntu podman development container
-	$(call build_podman_container,$@,ubuntu)
+ubuntu_podman:  base_images/cloud.json  ## Build Ubuntu podman development container
+	$(call build_podman_container,$@,$(call err_if_empty,_ubuntu_release))
 
 $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/%
 	podman build -t $*_podman:$(call err_if_empty,IMG_SFX) \
@@ -249,21 +252,21 @@ $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_i
 
 .PHONY: skopeo_cidev
 skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar  ## Build Skopeo development and CI container
-$(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora
+$(_TEMPDIR)/skopeo_cidev.tar: base_images/cloud.json $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora
 	podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \
 		--security-opt seccomp=unconfined \
-		--build-arg=BASE_TAG=$(_fedora_podman_release) \
+		--build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \
 		-v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \
 		skopeo_cidev
 	rm -f $@
 	podman save --quiet -o $@ skopeo_cidev:$(IMG_SFX)
 
 .PHONY: ccia
 ccia: $(_TEMPDIR)/ccia.tar  ## Build the Cirrus-CI Artifacts container image
-$(_TEMPDIR)/ccia.tar: ccia/Containerfile
+$(_TEMPDIR)/ccia.tar: ccia/Containerfile base_images/cloud.json
 	podman build -t ccia:$(call err_if_empty,IMG_SFX) \
 		--security-opt seccomp=unconfined \
-		--build-arg=BASE_TAG=$(_fedora_podman_release) \
+		--build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \
 		ccia
 	rm -f $@
 	podman save --quiet -o $@ ccia:$(IMG_SFX)

diff --git a/base_images/cloud.yml b/base_images/cloud.yml
@@ -17,11 +17,11 @@ variables:  # Empty value means it must be passed in on command-line
     # Allows providing handy cross-reference to the build log
     CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}"
 
-    # BIG-FAT-WARNING:  When updating the image names and/or URLs below,
-    # ensure the distro version numbers contained in the `podman/*_release`
-    # files exactly match.  These represent the container base-image tags
-    # to build from - just as the sources below are the base-images to
-    # start from building VM images.
+    # This data is also used when building cache and container images.
+    # It is assumed to match all the other OS variables below.
+    UBUNTU_RELEASE: "22.04"
+    FEDORA_RELEASE: "36"
+    PRIOR_FEDORA_RELEASE: "35"
 
     # Upstream source for Ubuntu image to duplicate (prevents expiration).
     # Use the most recent image based on this family name.
@@ -50,7 +50,7 @@ builders:
       # Prefix IMG_SFX with "b" so this is never confused with a cache_image name
       image_name: '{{build_name}}-b{{user `IMG_SFX`}}'
       image_family: '{{build_name}}-base'
-      image_description: 'Built in https://cirrus-ci.com/build/{{user `IMG_SFX`}}'
+      image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       source_image_family: '{{user `UBUNTU_BASE_FAMILY`}}'
       source_image_project_id: 'ubuntu-os-cloud'
       project_id: '{{user `GCP_PROJECT_ID`}}'
@@ -64,6 +64,7 @@ builders:
         src: '{{user `UBUNTU_BASE_FAMILY`}}'
         stage: 'base'
         arch: 'x86_64'
+        release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}'
       # Gotcha: https://www.packer.io/docs/builders/googlecompute#gotchas
       ssh_username: 'packer'
       temporary_key_pair_type: ed25519
@@ -132,7 +133,7 @@ builders:
       security_group_id: "sg-042c75677872ef81c"
       # Prefix IMG_SFX with "b" so this is never confused with a cache_image
       ami_name: 'fedora-aws-b{{user `IMG_SFX`}}'
-      ami_description: 'Built in https://cirrus-ci.com/build/{{user `IMG_SFX`}}'
+      ami_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       ebs_optimized: true
       launch_block_device_mappings:
         - device_name: '/dev/sda1'
@@ -147,6 +148,7 @@ builders:
         Name: 'fedora-aws-b{{user `IMG_SFX`}}'
         src: '{{user `FEDORAPROJECT_AMI`}}'
         automation: 'true'
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
       run_tags: *awstags
       run_volume_tags: *awstags
       snapshot_tags: *awstags
@@ -214,16 +216,19 @@ post-processors:
         gcs_object_name: '{{build_name}}-{{user `IMG_SFX`}}.tar.gz'
         image_name: "fedora-b{{user `IMG_SFX`}}"
         image_family: '{{build_name}}-base'
-        # Can't save the url in an image_label
-        image_description: '{{user `FEDORA_IMAGE_URL`}}'
-        image_labels: &importlabels
+        image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
+        image_labels:
           <<: *imgcpylabels
           src: 'fedoraproject'
+          release: 'fedora-{{user `FEDORA_RELEASE`}}'
       - <<: *gcp_import
         only: ['prior-fedora']
         image_name: "prior-fedora-b{{user `IMG_SFX`}}"
         image_family: '{{build_name}}-base'
-        image_description: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
+        image_labels:
+          <<: *imgcpylabels
+          src: 'fedoraproject'
+          release: 'fedora-{{user `PRIOR_FEDORA_RELEASE`}}'
       # This is critical, especially for the aws builders.
       # Producing the cache-images from these base images
       # needs to lookup the runtime-produced AMI ID.

diff --git a/cache_images/cloud.yml b/cache_images/cloud.yml
@@ -16,6 +16,11 @@ variables:  # Empty value means it must be passed in on command-line
     # Allows providing handy cross-reference to the build log
     CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}"
 
+    # Makefile sets these from values in base_images/cloud.yml
+    UBUNTU_RELEASE:
+    FEDORA_RELEASE:
+    PRIOR_FEDORA_RELEASE:
+
 builders:
     - &gce_hosted_image
       name: 'ubuntu'
@@ -25,14 +30,16 @@ builders:
       # Prefix IMG_SFX with "c" so this is never confused with a base_image name
       image_name: '{{ build_name }}-c{{user `IMG_SFX`}}'
       image_family: '{{ build_name }}-cache'
+      image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       project_id: '{{user `GCP_PROJECT_ID`}}'
       account_file: '{{user `GAC_FILEPATH`}}'
       zone: 'us-central1-a'
       disk_size: 20  # REQUIRED: Runtime allocation > this value
       disable_default_service_account: true
-      labels:  # For the VM
+      labels: &gce_labels # For the VM
         sfx: '{{user `IMG_SFX`}}'
         src: '{{ build_name }}-b{{user `IMG_SFX` }}'
+        release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}'
         stage: cache
       ssh_username: packer  # arbitrary, packer will create & setup w/ temp. keypair
       ssh_pty: 'true'
@@ -43,15 +50,20 @@ builders:
 
     - <<: *gce_hosted_image
       name: 'fedora'
+      labels: &fedora_gce_labels
+        <<: *gce_labels
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
 
     - <<: *gce_hosted_image
       name: 'prior-fedora'
+      labels: *fedora_gce_labels
 
     - &aux_fed_img
       <<: *gce_hosted_image
       name: 'build-push'
       source_image: 'fedora-b{{user `IMG_SFX`}}'
       source_image_family: 'fedora-base'
+      labels: *fedora_gce_labels
 
     - <<: *aux_fed_img
       name: 'fedora-podman-py'
@@ -92,25 +104,26 @@ builders:
       security_group_id: "sg-042c75677872ef81c"
       # Prefix IMG_SFX with "b" so this is never confused with a cache_image
       ami_name: '{{build_name}}-c{{user `IMG_SFX`}}'
-      ami_description: 'Built in https://cirrus-ci.com/build/{{user `IMG_SFX`}}'
+      ami_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       ebs_optimized: true
       launch_block_device_mappings:
         - device_name: '/dev/sda1'
           volume_size: 200
           volume_type: 'gp2'
           delete_on_termination: true
       # These are critical and used by security-polciy to enforce instance launch limits.
-      tags: &tags
+      tags: &ami_tags
         # EC2 expects "Name" tag to be capitalized
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         sfx: '{{user `IMG_SFX`}}'
         src: '{{.SourceAMI}}'  # Generated AMI ID looked up at runtime
         automation: 'true'
         stage: 'cache'
         arch: 'x86_64'
-      run_tags: *tags
-      run_volume_tags: *tags
-      snapshot_tags: *tags
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
+      run_tags: *ami_tags
+      run_volume_tags: *ami_tags
+      snapshot_tags: *ami_tags
       # Also required to make AMI private
       ami_users:
         - *accountid
@@ -131,7 +144,7 @@ builders:
             name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'
       instance_type: 't4g.medium'  # arm64 type
       tags: &netavark_tags
-        <<: *tags
+        <<: *ami_tags
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         arch: 'arm64'
       run_tags: *netavark_tags
@@ -149,7 +162,7 @@ builders:
             name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'
       instance_type: 't4g.medium'  # arm64 type
       tags: &podman_tags
-        <<: *tags
+        <<: *ami_tags
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         arch: 'arm64'
       run_tags: *podman_tags

diff --git a/get_base_release.sh b/get_base_release.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# This script is intended to be called by the Makefile only.
+# Any other use may produce unexpected results.  It expects
+# to be called with the name of a supported OS in all upper-case.
+# The value of the corresponding <NAME>_RELEASE variable will be
+# extracted from base_images/cloud.json and printed to stdout.
+
+set -eo pipefail
+
+SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}")
+SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH")
+REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH")
+CLOUD_JSON="$REPO_DIRPATH/base_images/cloud.json"
+
+# shellcheck source=./lib.sh
+source "$REPO_DIRPATH/lib.sh"
+
+[[ -r "$CLOUD_JSON" ]] || die "Cannot read from '$CLOUD_JSON'"
+
+jq -r -e ".variables.${1}_RELEASE" $CLOUD_JSON
diff --git a/image_builder/gce.yml b/image_builder/gce.yml
@@ -17,6 +17,10 @@ variables:
     # Allows providing handy cross-reference to the build log
     CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}"
 
+    # These aren't used, but are required to be present.
+    UBUNTU_RELEASE:
+    FEDORA_RELEASE:
+    PRIOR_FEDORA_RELEASE:
 
 builders:
     - name: 'image-builder'
@@ -25,6 +29,7 @@ builders:
       source_image_family: 'centos-stream-8'
       image_name: '{{build_name}}-{{user `IMG_SFX`}}'  # output image name
       image_family: '{{build_name}}'  # output family name for image
+      image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       project_id: '{{user `GCP_PROJECT_ID`}}'
       account_file: '{{user `GAC_FILEPATH`}}'
       zone: 'us-central1-a'

diff --git a/podman/fedora_release b/podman/fedora_release
diff --git a/podman/prior-fedora_release b/podman/prior-fedora_release
diff --git a/podman/ubuntu_release b/podman/ubuntu_release