Get release and image info from a single-source-of-truth

Fixes: #135 Previously, a maintainer integrating a new OS release was required to update multiple files. Reduce this burden by encoding release-versions and other key details into a simple YAML file. Craft a helper script for the `Makefile` to use in retrieving values as required. Having a single source of truth for these details makes it easy for multiple tools to access and maintainers to fiddle with with as needed. Signed-off-by: Chris Evich <[email protected]>
containers · Aug 25, 2022 · eba99e1 · eba99e1
1 parent 480ff5d
commit eba99e1
Show file tree

Hide file tree

Showing 11 changed files with 153 additions and 74 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -59,6 +59,7 @@ image_builder_task:
       PACKER_BUILDS: 'image-builder'
       # Google Application Credentials (JSON) with access to create VM images
       GAC_JSON: ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9]
+      AWS_SHARED_CREDENTIALS_FILE: notused
 
     script: "ci/make_image_builder.sh"
     manifest_artifacts:

diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,3 @@
 */*.json
+image_info.json
 /.cache
diff --git a/Makefile b/Makefile
@@ -13,6 +13,9 @@ export_full = $(eval export $(if $(call err_if_empty,$(1)),$(1)))
 # Evaluate to the value of $(1) if $(CI) is the literal string "true", else $(2)
 if_ci_else = $(if $(findstring true,$(CI)),$(1),$(2))
 
+# Represent value for key $(1) from image_info.json or empty on error.
+get_image_info = $(shell bash get_image_info.sh "$(1)")
+
 ##### Important Paths and variables #####
 
 # Most targets require possession of service-account credentials (JSON file)
@@ -74,6 +77,22 @@ override _HLPFMT = "%-20s %s\n"
 # N/B: There are length/character limitations in GCE for image names
 IMG_SFX ?=
 
+# Env. vars needed by packer
+export CHECKPOINT_DISABLE = 1  # Disable hashicorp phone-home
+export PACKER_CACHE_DIR = $(call err_if_empty,_TEMPDIR)
+
+# Vars from image_info.json for various targets
+override _fedora_release = $(call get_image_info,fedora.release)
+override _fedora_image_url = $(call get_image_info,fedora.image_url)
+override _fedora_csum_url = $(call get_image_info,fedora.csum_url)
+override _fedora_ami = $(call get_image_info,fedora.ami)
+override _fedora_arm64_ami = $(call get_image_info,fedora.arm64_ami)
+override _prior_fedora_release = $(call get_image_info,prior_fedora.release)
+override _prior_fedora_image = $(call get_image_info,prior_fedora.image_url)
+override _prior_fedora_csum = $(call get_image_info,prior_fedora.csum_url)
+override _ubuntu_release = $(call get_image_info,ubuntu.release)
+override _ubuntu_base_family = $(call get_image_info,ubuntu.family)
+
 ##### Targets #####
 
 # N/B: The double-# after targets is gawk'd out as the target description
@@ -159,28 +178,38 @@ $(_TEMPDIR)/user-data: $(_TEMPDIR) $(_TEMPDIR)/cidata.ssh.pub $(_TEMPDIR)/cidata
 .PHONY: cidata
 cidata: $(_TEMPDIR)/user-data $(_TEMPDIR)/meta-data
 
-# First argument is the path to the template JSON, second
-# argument is the path to AWS_SHARED_CREDENTIALS_FILE
-# when required.  N/B: GAC_FILEPATH is always required.
+define build_podman_container
+	$(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(2)
+endef
+
+# First argument is the path to the template JSON
 define packer_build
-	env PACKER_CACHE_DIR="$(_TEMPDIR)" \
-		AWS_SHARED_CREDENTIALS_FILE="$(2)" \
+	env AWS_SHARED_CREDENTIALS_FILE="$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE)" \
 		GAC_FILEPATH="$(call err_if_empty,GAC_FILEPATH)" \
-		CHECKPOINT_DISABLE=1 \
+		FEDORA_RELEASE="$(call err_if_empty,_fedora_release)" \
+		FEDORA_IMAGE_URL="$(call err_if_empty,_fedora_image_url)" \
+		FEDORA_CSUM_URL="$(call err_if_empty,_fedora_csum_url)" \
+		FEDORA_AMI="$(call err_if_empty,_fedora_ami)" \
+		FEDORA_ARM64_AMI="$(call err_if_empty,_fedora_arm64_ami)" \
+		PRIOR_FEDORA_RELEASE="$(call err_if_empty,_prior_fedora_release)" \
+		PRIOR_FEDORA_IMAGE="$(call err_if_empty,_prior_fedora_image)" \
+		PRIOR_FEDORA_CSUM="$(call err_if_empty,_prior_fedora_csum)" \
+		UBUNTU_RELEASE="$(call err_if_empty,_ubuntu_release)" \
+		UBUNTU_BASE_FAMILY="$(call err_if_empty,_ubuntu_base_family)" \
 			$(PACKER_INSTALL_DIR)/packer build \
 			-force \
 			-var TEMPDIR="$(_TEMPDIR)" \
+			-var IMG_SFX="$(call err_if_empty,IMG_SFX)" \
 			$(if $(PACKER_BUILDS),-only=$(PACKER_BUILDS)) \
-			$(if $(IMG_SFX),-var IMG_SFX=$(IMG_SFX)) \
 			$(if $(DEBUG_NESTED_VM),-var TTYDEV=$(shell tty),-var TTYDEV=/dev/null) \
 			$(if $(PACKER_BUILD_ARGS),$(PACKER_BUILD_ARGS)) \
 			$(1)
 endef
 
 .PHONY: image_builder
 image_builder: image_builder/manifest.json ## Create image-building image and import into GCE (needed for making all other images)
-image_builder/manifest.json: image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer
-	$(call packer_build,$<,)
+image_builder/manifest.json: image_info.json image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer
+	$(call packer_build,image_builder/gce.json)
 
 # Note: We assume this repo is checked out somewhere under the caller's
 # home-dir for bind-mounting purposes.  Otherwise possibly necessary
@@ -208,32 +237,25 @@ $(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR)/.cache/centos $(wildcard image_
 # This needs to run in a virt/nested-virt capable environment
 base_images: base_images/manifest.json ## Create, prepare, and import base-level images into GCE.  Optionally, set PACKER_BUILDS=<csv> to select builder(s).
 
-base_images/manifest.json: base_images/cloud.json $(wildcard base_images/*.sh) cidata $(_TEMPDIR)/cidata.ssh $(PACKER_INSTALL_DIR)/packer
-	$(call packer_build,base_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE))
+base_images/manifest.json: image_info.json base_images/cloud.json $(wildcard base_images/*.sh) cidata $(_TEMPDIR)/cidata.ssh $(PACKER_INSTALL_DIR)/packer
+	$(call packer_build,base_images/cloud.json)
 
 .PHONY: cache_images
 cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE.  Optionally, set PACKER_BUILDS=<csv> to select builder(s).
-cache_images/manifest.json: cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
-	$(call packer_build,cache_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE))
-
-override _fedora_podman_release := $(file < podman/fedora_release)
-override _prior-fedora_podman_release := $(file < podman/prior-fedora_release)
-override _ubuntu_podman_release := $(file < podman/ubuntu_release)
-define build_podman_container
-	$(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(_$(1)_release)
-endef
+cache_images/manifest.json: image_info.json cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
+	$(call packer_build,cache_images/cloud.json)
 
 .PHONY: fedora_podman
-fedora_podman:  ## Build Fedora podman development container
-	$(call build_podman_container,$@,fedora)
+fedora_podman:  image_info.json  ## Build Fedora podman development container
+	$(call build_podman_container,$@,$(call err_if_empty,_fedora_release))
 
 .PHONY: prior-fedora_podman
-prior-fedora_podman:  ## Build Prior-Fedora podman development container
-	$(call build_podman_container,$@,prior-fedora)
+prior-fedora_podman:  image_info.json  ## Build Prior-Fedora podman development container
+	$(call build_podman_container,$@,$(call err_if_empty,_prior_fedora_release))
 
 .PHONY: ubuntu_podman
-ubuntu_podman:  ## Build Ubuntu podman development container
-	$(call build_podman_container,$@,ubuntu)
+ubuntu_podman:  image_info.json  ## Build Ubuntu podman development container
+	$(call build_podman_container,$@,$(call err_if_empty,_ubuntu_release))
 
 $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/%
 	podman build -t $*_podman:$(call err_if_empty,IMG_SFX) \
@@ -249,21 +271,21 @@ $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_i
 
 .PHONY: skopeo_cidev
 skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar  ## Build Skopeo development and CI container
-$(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora
+$(_TEMPDIR)/skopeo_cidev.tar: image_info.json $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora
 	podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \
 		--security-opt seccomp=unconfined \
-		--build-arg=BASE_TAG=$(_fedora_podman_release) \
+		--build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \
 		-v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \
 		skopeo_cidev
 	rm -f $@
 	podman save --quiet -o $@ skopeo_cidev:$(IMG_SFX)
 
 .PHONY: ccia
 ccia: $(_TEMPDIR)/ccia.tar  ## Build the Cirrus-CI Artifacts container image
-$(_TEMPDIR)/ccia.tar: ccia/Containerfile
+$(_TEMPDIR)/ccia.tar: image_info.json ccia/Containerfile
 	podman build -t ccia:$(call err_if_empty,IMG_SFX) \
 		--security-opt seccomp=unconfined \
-		--build-arg=BASE_TAG=$(_fedora_podman_release) \
+		--build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \
 		ccia
 	rm -f $@
 	podman save --quiet -o $@ ccia:$(IMG_SFX)
@@ -309,6 +331,7 @@ $(_TEMPDIR)/get_ci_vm.tar: lib.sh get_ci_vm/Containerfile get_ci_vm/entrypoint.s
 .PHONY: clean
 clean: ## Remove all generated files referenced in this Makefile
 	-rm -rf $(_TEMPDIR)
+	-rm -f image_info.json
 	-rm -f image_builder/*.json
 	-rm -f base_images/{*.json,cidata*,*-data}
 	-rm -f ci_debug.tar
diff --git a/base_images/cloud.yml b/base_images/cloud.yml
@@ -17,31 +17,19 @@ variables:  # Empty value means it must be passed in on command-line
     # Allows providing handy cross-reference to the build log
     CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}"
 
-    # BIG-FAT-WARNING:  When updating the image names and/or URLs below,
-    # ensure the distro version numbers contained in the `podman/*_release`
-    # files exactly match.  These represent the container base-image tags
-    # to build from - just as the sources below are the base-images to
-    # start from building VM images.
+    # See image_info.yml
+    FEDORA_RELEASE: "{{env `FEDORA_RELEASE`}}"
+    FEDORA_IMAGE_URL: "{{env `FEDORA_IMAGE_URL`}}"
+    FEDORA_CSUM_URL: "{{env `FEDORA_CSUM_URL`}}"
+    FEDORA_AMI: "{{env `FEDORA_AMI`}}"
+    FEDORA_ARM64_AMI: "{{env `FEDORA_ARM64_AMI`}}"
 
-    # Upstream source for Ubuntu image to duplicate (prevents expiration).
-    # Use the most recent image based on this family name.
-    UBUNTU_BASE_FAMILY: 'ubuntu-2204-lts'
+    PRIOR_FEDORA_RELEASE: "{{env `PRIOR_FEDORA_RELEASE`}}"
+    PRIOR_FEDORA_IMAGE_URL: "{{env `PRIOR_FEDORA_IMAGE_URL`}}"
+    PRIOR_FEDORA_CSUM: "{{env `PRIOR_FEDORA_CSUM`}}"
 
-    # AWS base image in 'US East (N. Virginia)' region
-    # from https://alt.fedoraproject.org/cloud
-    FEDORAPROJECT_AMI: "ami-08b7bda26f4071b80"
-    FEDORAPROJECT_AARCH64_AMI: "ami-01925eb0821988986"
-
-    # Latest Fedora release (qcow and CHECKSUM) download URLs
-    # N/B: There are Fedora-Cloud...GCP.tar.gz images available, however
-    #      as of this comment, they lack the cloud-init package which is
-    #      required by GCP for startup resizing of the rootfs.
-    FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-36-1.5-x86_64-CHECKSUM"
-    FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-Base-36-1.5.x86_64.qcow2"
-
-    # Prior Fedora release
-    PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-35-1.2-x86_64-CHECKSUM"
-    PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-Base-35-1.2.x86_64.qcow2"
+    UBUNTU_RELEASE: "{{env `UBUNTU_RELEASE` | clean_resource_name}}"
+    UBUNTU_BASE_FAMILY: "{{env `UBUNTU_BASE_FAMILY`}}"
 
 
 builders:
@@ -50,7 +38,7 @@ builders:
       # Prefix IMG_SFX with "b" so this is never confused with a cache_image name
       image_name: '{{build_name}}-b{{user `IMG_SFX`}}'
       image_family: '{{build_name}}-base'
-      image_description: 'Built in https://cirrus-ci.com/build/{{user `IMG_SFX`}}'
+      image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       source_image_family: '{{user `UBUNTU_BASE_FAMILY`}}'
       source_image_project_id: 'ubuntu-os-cloud'
       project_id: '{{user `GCP_PROJECT_ID`}}'
@@ -64,6 +52,7 @@ builders:
         src: '{{user `UBUNTU_BASE_FAMILY`}}'
         stage: 'base'
         arch: 'x86_64'
+        release: 'ubuntu-{{user `UBUNTU_RELEASE`}}'
       # Gotcha: https://www.packer.io/docs/builders/googlecompute#gotchas
       ssh_username: 'packer'
       temporary_key_pair_type: ed25519
@@ -119,7 +108,7 @@ builders:
     - &fedora-aws
       name: 'fedora-aws'
       type: 'amazon-ebs'
-      source_ami: '{{user `FEDORAPROJECT_AMI`}}'
+      source_ami: '{{user `FEDORA_AMI`}}'
       instance_type: 'm5zn.metal'
       # In case of packer problem or ungraceful exit, don't wait for shutdown.
       # This doesn't always work properly, sometimes leaving EC2 instances in
@@ -132,7 +121,7 @@ builders:
       security_group_id: "sg-042c75677872ef81c"
       # Prefix IMG_SFX with "b" so this is never confused with a cache_image
       ami_name: 'fedora-aws-b{{user `IMG_SFX`}}'
-      ami_description: 'Built in https://cirrus-ci.com/build/{{user `IMG_SFX`}}'
+      ami_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       ebs_optimized: true
       launch_block_device_mappings:
         - device_name: '/dev/sda1'
@@ -145,8 +134,9 @@ builders:
         <<: *imgcpylabels
         # EC2 expects "Name" to be capitalized
         Name: 'fedora-aws-b{{user `IMG_SFX`}}'
-        src: '{{user `FEDORAPROJECT_AMI`}}'
+        src: '{{user `FEDORA_AMI`}}'
         automation: 'true'
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
       run_tags: *awstags
       run_volume_tags: *awstags
       snapshot_tags: *awstags
@@ -164,12 +154,12 @@ builders:
 
     - <<: *fedora-aws
       name: 'fedora-aws-arm64'
-      source_ami: '{{user `FEDORAPROJECT_AARCH64_AMI`}}'
+      source_ami: '{{user `FEDORA_ARM64_AMI`}}'
       instance_type: 't4g.medium'  # arm64 type
       ami_name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'  # must be unique
       tags: &awsarm64tags
         <<: *awstags
-        src: '{{user `FEDORAPROJECT_AARCH64_AMI`}}'
+        src: '{{user `FEDORA_ARM64_AMI`}}'
         arch: 'arm64'
       run_tags: *awsarm64tags
       run_volume_tags: *awsarm64tags
@@ -214,16 +204,19 @@ post-processors:
         gcs_object_name: '{{build_name}}-{{user `IMG_SFX`}}.tar.gz'
         image_name: "fedora-b{{user `IMG_SFX`}}"
         image_family: '{{build_name}}-base'
-        # Can't save the url in an image_label
-        image_description: '{{user `FEDORA_IMAGE_URL`}}'
-        image_labels: &importlabels
+        image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
+        image_labels:
           <<: *imgcpylabels
           src: 'fedoraproject'
+          release: 'fedora-{{user `FEDORA_RELEASE`}}'
       - <<: *gcp_import
         only: ['prior-fedora']
         image_name: "prior-fedora-b{{user `IMG_SFX`}}"
         image_family: '{{build_name}}-base'
-        image_description: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
+        image_labels:
+          <<: *imgcpylabels
+          src: 'fedoraproject'
+          release: 'fedora-{{user `PRIOR_FEDORA_RELEASE`}}'
       # This is critical, especially for the aws builders.
       # Producing the cache-images from these base images
       # needs to lookup the runtime-produced AMI ID.

diff --git a/cache_images/cloud.yml b/cache_images/cloud.yml
@@ -16,6 +16,11 @@ variables:  # Empty value means it must be passed in on command-line
     # Allows providing handy cross-reference to the build log
     CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}"
 
+    # See image_info.yml
+    FEDORA_RELEASE: "{{env `FEDORA_RELEASE`}}"
+    PRIOR_FEDORA_RELEASE: "{{env `PRIOR_FEDORA_RELEASE`}}"
+    UBUNTU_RELEASE: "{{env `UBUNTU_RELEASE` | clean_resource_name}}"
+
 builders:
     - &gce_hosted_image
       name: 'ubuntu'
@@ -25,14 +30,16 @@ builders:
       # Prefix IMG_SFX with "c" so this is never confused with a base_image name
       image_name: '{{ build_name }}-c{{user `IMG_SFX`}}'
       image_family: '{{ build_name }}-cache'
+      image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       project_id: '{{user `GCP_PROJECT_ID`}}'
       account_file: '{{user `GAC_FILEPATH`}}'
       zone: 'us-central1-a'
       disk_size: 20  # REQUIRED: Runtime allocation > this value
       disable_default_service_account: true
-      labels:  # For the VM
+      labels: &gce_labels # For the VM
         sfx: '{{user `IMG_SFX`}}'
         src: '{{ build_name }}-b{{user `IMG_SFX` }}'
+        release: 'ubuntu-{{user `UBUNTU_RELEASE`}}'
         stage: cache
       ssh_username: packer  # arbitrary, packer will create & setup w/ temp. keypair
       ssh_pty: 'true'
@@ -43,15 +50,20 @@ builders:
 
     - <<: *gce_hosted_image
       name: 'fedora'
+      labels: &fedora_gce_labels
+        <<: *gce_labels
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
 
     - <<: *gce_hosted_image
       name: 'prior-fedora'
+      labels: *fedora_gce_labels
 
     - &aux_fed_img
       <<: *gce_hosted_image
       name: 'build-push'
       source_image: 'fedora-b{{user `IMG_SFX`}}'
       source_image_family: 'fedora-base'
+      labels: *fedora_gce_labels
 
     - <<: *aux_fed_img
       name: 'fedora-podman-py'
@@ -92,25 +104,26 @@ builders:
       security_group_id: "sg-042c75677872ef81c"
       # Prefix IMG_SFX with "b" so this is never confused with a cache_image
       ami_name: '{{build_name}}-c{{user `IMG_SFX`}}'
-      ami_description: 'Built in https://cirrus-ci.com/build/{{user `IMG_SFX`}}'
+      ami_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       ebs_optimized: true
       launch_block_device_mappings:
         - device_name: '/dev/sda1'
           volume_size: 200
           volume_type: 'gp2'
           delete_on_termination: true
       # These are critical and used by security-polciy to enforce instance launch limits.
-      tags: &tags
+      tags: &ami_tags
         # EC2 expects "Name" tag to be capitalized
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         sfx: '{{user `IMG_SFX`}}'
         src: '{{.SourceAMI}}'  # Generated AMI ID looked up at runtime
         automation: 'true'
         stage: 'cache'
         arch: 'x86_64'
-      run_tags: *tags
-      run_volume_tags: *tags
-      snapshot_tags: *tags
+        release: 'fedora-{{user `FEDORA_RELEASE`}}'
+      run_tags: *ami_tags
+      run_volume_tags: *ami_tags
+      snapshot_tags: *ami_tags
       # Also required to make AMI private
       ami_users:
         - *accountid
@@ -131,7 +144,7 @@ builders:
             name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'
       instance_type: 't4g.medium'  # arm64 type
       tags: &netavark_tags
-        <<: *tags
+        <<: *ami_tags
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         arch: 'arm64'
       run_tags: *netavark_tags
@@ -149,7 +162,7 @@ builders:
             name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'
       instance_type: 't4g.medium'  # arm64 type
       tags: &podman_tags
-        <<: *tags
+        <<: *ami_tags
         Name: '{{build_name}}-c{{user `IMG_SFX`}}'
         arch: 'arm64'
       run_tags: *podman_tags