Add support for importing generic cloud images

***DEPENDS ON:*** containers#178 Downstream CI needs dictate early testing on new Fedora releases at the Beta stage or earlier. Unfortunately at the time of this commit, the Fedora cloud-sig does not provide ready-made beta images in AWS EC2. Add a new image-build stage called `import_images`, that takes care of bringing the generic F37 cloud images into AWS. Update the `base_images` stage such that it will locate and use the imported images. Unify all the build-stage wrapper scripts into a single/common `ci/make.sh` wrapper. Finally, make a few semi-related comment/readability improvements in various places. Signed-off-by: Chris Evich <[email protected]>
cevich · Sep 8, 2022 · 0304a50 · 0304a50
1 parent 7647593
commit 0304a50
Show file tree

Hide file tree

Showing 8 changed files with 245 additions and 107 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -9,7 +9,7 @@ env:
     # No need to go crazy, but grab enough to cover most PRs
     CIRRUS_CLONE_DEPTH: 50
     # Version of packer to use when building images
-    PACKER_VERSION: &PACKER_VERSION "1.8.0"
+    PACKER_VERSION: &PACKER_VERSION "1.8.3"
     # Unique suffix label to use for all images produced by _this_ run (build)
     IMG_SFX: "${CIRRUS_BUILD_ID}"
 
@@ -32,6 +32,7 @@ validate_task:
         - "make clean"
         - "make help"
         - "make image_builder/gce.json"
+        - "make import_images/cloud.json"
         - "make base_images/cloud.json"
         - "make cache_images/cloud.json"
 
@@ -49,7 +50,7 @@ image_builder_task:
     auto_cancellation: $CI != "true"
     stateful: true
     timeout_in: 40m
-    container:
+    container: &image_builder_container
         dockerfile: "image_builder/Containerfile"
         cpu: 2
         memory: "2G"
@@ -60,8 +61,7 @@ image_builder_task:
       # Google Application Credentials (JSON) with access to create VM images
       GAC_JSON: ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9]
       AWS_SHARED_CREDENTIALS_FILE: notused
-
-    script: "ci/make_image_builder.sh"
+    script: "ci/make.sh image_builder"
     manifest_artifacts:
         path: image_builder/manifest.json
         type: application/json
@@ -133,14 +133,43 @@ tooling_images_task:
         fingerprint_key: "tooling-cache-version-1"
 
 
+import_images_task:
+    name: "Import upstream Fedora images"
+    alias: "import_images"
+    only_if: *is_pr
+    skip: *ci_docs_tooling
+    depends_on:
+        - container_images
+        - image_builder
+    # Packer needs time to clean up partially created VM images
+    auto_cancellation: $CI != "true"
+    stateful: true
+    timeout_in: 30m
+    container: *image_builder_container
+    matrix:
+        - &import_image
+          name: "${PACKER_BUILDS} Import Image"
+          env:
+            PACKER_BUILDS: "fedora-aws"
+        - <<: *import_image
+          env:
+            PACKER_BUILDS: "fedora-aws-arm64"
+    env:
+      AWS_INI: &aws_ini ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930]
+      GAC_JSON: &gac_json ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9]
+    script: "ci/make.sh import_images"
+    manifest_artifacts:
+        path: import_images/manifest.json
+        type: application/json
+
+
 base_images_task:
     name: "Build VM Base-images"
     alias: "base_images"
     only_if: *is_pr
     skip: *ci_docs_tooling
     depends_on:
-        - container_images
-        - image_builder
+        - import_images
     # Packer needs time to clean up partially created VM images
     auto_cancellation: $CI != "true"
     stateful: true
@@ -170,9 +199,9 @@ base_images_task:
           env:
             PACKER_BUILDS: "ubuntu"
     env:
-      GAC_JSON: ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9]
-      AWS_INI: ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930]
-    script: "ci/make_base_images.sh"
+      GAC_JSON: *gac_json
+      AWS_INI: *aws_ini
+    script: "ci/make.sh base_images"
     manifest_artifacts:
         path: base_images/manifest.json
         type: application/json
@@ -226,9 +255,9 @@ cache_images_task:
           env:
             PACKER_BUILDS: "ubuntu"
     env:
-      GAC_JSON: ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9]
-      AWS_INI: ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930]
-    script: "ci/make_cache_images.sh"
+      GAC_JSON: *gac_json
+      AWS_INI: *aws_ini
+    script: "ci/make.sh cache_images"
     manifest_artifacts:
         path: cache_images/manifest.json
         type: application/json
@@ -460,6 +489,7 @@ success_task:
         - image_builder
         - container_images
         - tooling_images
+        - import_images
         - base_images
         - cache_images
         - test_imgts

diff --git a/Makefile b/Makefile
@@ -17,15 +17,17 @@ if_ci_else = $(if $(findstring true,$(CI)),$(1),$(2))
 
 export CENTOS_STREAM_RELEASE = 8
 
-export FEDORA_RELEASE = 36
-export FEDORA_IMAGE_URL = https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-Base-36-1.5.x86_64.qcow2
-export FEDORA_CSUM_URL = https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-36-1.5-x86_64-CHECKSUM
-export FEDORA_AMI = ami-08b7bda26f4071b80
-export FEDORA_ARM64_AMI = ami-01925eb0821988986
+# Ref: https://dl.fedoraproject.org/pub/fedora/linux/
 
-export PRIOR_FEDORA_RELEASE = 35
-export PRIOR_FEDORA_IMAGE_URL = https://dl.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-Base-35-1.2.x86_64.qcow2
-export PRIOR_FEDORA_CSUM_URL = https://dl.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-35-1.2-x86_64-CHECKSUM
+export FEDORA_RELEASE = 37
+export FEDORA_IMAGE_URL = https://dl.fedoraproject.org/pub/fedora/linux/development/37/Cloud/x86_64/images/Fedora-Cloud-Base-37-20220908.n.0.x86_64.qcow2
+export FEDORA_CSUM_URL = https://dl.fedoraproject.org/pub/fedora/linux/development/37/Cloud/x86_64/images/Fedora-Cloud-37-x86_64-20220908.n.0-CHECKSUM
+export FEDORA_ARM64_IMAGE_URL = https://dl.fedoraproject.org/pub/fedora/linux/development/37/Cloud/aarch64/images/Fedora-Cloud-Base-37-20220908.n.0.aarch64.qcow2
+export FEDORA_ARM64_CSUM_URL = https://dl.fedoraproject.org/pub/fedora/linux/development/37/Cloud/aarch64/images/Fedora-Cloud-37-aarch64-20220908.n.0-CHECKSUM
+
+export PRIOR_FEDORA_RELEASE = 36
+export PRIOR_FEDORA_IMAGE_URL = https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-Base-36-1.5.x86_64.qcow2
+export PRIOR_FEDORA_CSUM_URL = https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-36-1.5-x86_64-CHECKSUM
 
 export UBUNTU_RELEASE = 22.04
 export UBUNTU_BASE_FAMILY = ubuntu-2204-lts
@@ -226,15 +228,21 @@ image_builder_debug: $(_TEMPDIR)/image_builder_debug.tar ## Build and enter cont
 $(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR)/.cache/centos $(wildcard image_builder/*)
 	$(call podman_build,$@,image_builder_debug,image_builder,centos)
 
+.PHONY: import_images
+import_images: import_images/manifest.json ## Import generic Fedora cloud images into AWS EC2.
+
+import_images/manifest.json: import_images/cloud.json $(PACKER_INSTALL_DIR)/packer
+	$(call packer_build,import_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE))
+
 .PHONY: base_images
 # This needs to run in a virt/nested-virt capable environment
-base_images: base_images/manifest.json ## Create, prepare, and import base-level images into GCE.  Optionally, set PACKER_BUILDS=<csv> to select builder(s).
+base_images: base_images/manifest.json ## Create, prepare, and import base-level images into GCE.
 
 base_images/manifest.json: base_images/cloud.json $(wildcard base_images/*.sh) cidata $(_TEMPDIR)/cidata.ssh $(PACKER_INSTALL_DIR)/packer
 	$(call packer_build,base_images/cloud.json)
 
 .PHONY: cache_images
-cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE.  Optionally, set PACKER_BUILDS=<csv> to select builder(s).
+cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE.
 cache_images/manifest.json: cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
 	$(call packer_build,cache_images/cloud.json)
 
@@ -327,5 +335,5 @@ $(_TEMPDIR)/get_ci_vm.tar: lib.sh get_ci_vm/Containerfile get_ci_vm/entrypoint.s
 clean: ## Remove all generated files referenced in this Makefile
 	-rm -rf $(_TEMPDIR)
 	-rm -f image_builder/*.json
-	-rm -f base_images/{*.json,cidata*,*-data}
+	-rm -f *_images/{*.json,cidata*,*-data}
 	-rm -f ci_debug.tar
diff --git a/base_images/cloud.yml b/base_images/cloud.yml
@@ -21,8 +21,6 @@ variables:  # Empty value means it must be passed in on command-line
     FEDORA_RELEASE: "{{env `FEDORA_RELEASE`}}"
     FEDORA_IMAGE_URL: "{{env `FEDORA_IMAGE_URL`}}"
     FEDORA_CSUM_URL: "{{env `FEDORA_CSUM_URL`}}"
-    FEDORA_AMI: "{{env `FEDORA_AMI`}}"
-    FEDORA_ARM64_AMI: "{{env `FEDORA_ARM64_AMI`}}"
 
     PRIOR_FEDORA_RELEASE: "{{env `PRIOR_FEDORA_RELEASE`}}"
     PRIOR_FEDORA_IMAGE_URL: "{{env `PRIOR_FEDORA_IMAGE_URL`}}"
@@ -108,7 +106,23 @@ builders:
     - &fedora-aws
       name: 'fedora-aws'
       type: 'amazon-ebs'
-      source_ami: '{{user `FEDORA_AMI`}}'
+      source_ami_filter:  # Will fail if >1 or no AMI found
+        owners:
+            # Docs are wrong, specifying the Account ID required to make AMIs private.
+            # The Account ID is hard-coded here out of expediency, since passing in
+            # more packer args from the command-line (in Makefile) is non-trivial.
+            - &accountid '449134212816'
+        # It's necessary to 'search' for the base-image by these criteria.  If
+        # more than one image is found, Packer will fail the build (and display
+        # the conflicting AMI IDs).
+        filters: &ami_filters
+            architecture: 'x86_64'
+            image-type: 'machine'
+            is-public: 'false'
+            name: '{{build_name}}-i{{user `IMG_SFX`}}'
+            root-device-type: 'ebs'
+            state: 'available'
+            virtualization-type: 'hvm'
       instance_type: 'm5zn.metal'
       # In case of packer problem or ungraceful exit, don't wait for shutdown.
       # This doesn't always work properly, sometimes leaving EC2 instances in
@@ -120,7 +134,7 @@ builders:
       # Required for network access, must be the 'default' group used by Cirrus-CI
       security_group_id: "sg-042c75677872ef81c"
       # Prefix IMG_SFX with "b" so this is never confused with a cache_image
-      ami_name: 'fedora-aws-b{{user `IMG_SFX`}}'
+      ami_name: &ami_name '{{build_name}}-b{{user `IMG_SFX`}}'
       ami_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}'
       ebs_optimized: true
       launch_block_device_mappings:
@@ -133,19 +147,17 @@ builders:
       tags: &awstags
         <<: *imgcpylabels
         # EC2 expects "Name" to be capitalized
-        Name: 'fedora-aws-b{{user `IMG_SFX`}}'
-        src: '{{user `FEDORA_AMI`}}'
+        Name: *ami_name
+        src: '{{.SourceAMI}}'
         automation: 'true'
         release: 'fedora-{{user `FEDORA_RELEASE`}}'
       run_tags: *awstags
       run_volume_tags: *awstags
       snapshot_tags: *awstags
-      # Docs are wrong, specifying the Account ID required to make AMIs private.
       # This is necessary for security - The CI service accounts are not permitted
-      # to use AMI's from any other account, including public ones.  The Account
-      # ID is hard-coded here out of expediency, since passing in more packer args
-      # from the command-line (in Makefile) is non-trivial.
-      ami_users: ["449134212816"]
+      # to use AMI's from any other account, including public ones.
+      ami_users:
+        - *accountid
       ssh_username: 'fedora'
       ssh_clear_authorized_keys: true
       # N/B: Required Packer >= 1.8.0
@@ -154,12 +166,15 @@ builders:
 
     - <<: *fedora-aws
       name: 'fedora-aws-arm64'
-      source_ami: '{{user `FEDORA_ARM64_AMI`}}'
+      source_ami_filter:
+        owners:
+            - *accountid
+        filters:
+            <<: *ami_filters
+            architecture: 'arm64'
       instance_type: 't4g.medium'  # arm64 type
-      ami_name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}'  # must be unique
       tags: &awsarm64tags
         <<: *awstags
-        src: '{{user `FEDORA_ARM64_AMI`}}'
         arch: 'arm64'
       run_tags: *awsarm64tags
       run_volume_tags: *awsarm64tags
@@ -213,6 +228,7 @@ post-processors:
         only: ['prior-fedora']
         image_name: "prior-fedora-b{{user `IMG_SFX`}}"
         image_family: '{{build_name}}-base'
+        image_description: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
         image_labels:
           <<: *imgcpylabels
           src: 'fedoraproject'

diff --git a/ci/make_base_images.sh → ci/make.sh b/ci/make_base_images.sh → ci/make.sh
@@ -1,9 +1,12 @@
 #!/bin/bash
 
+set -eo pipefail
+
 # This script is intended to be used by Cirrus-CI, from the VM
-# built by the 'image_builder' makefile target in this repo.  Use
-# of this script in any other context/environment is unlikely to
-# function as intended.
+# built by the 'image_builder' Makefile target in this repo.
+# It's purpose is simply to verify & configure the runtime
+# environment from data provided by CI, and call the make
+# with the first argument passed to this script.
 
 SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}")
 SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH")
@@ -17,17 +20,23 @@ if [[ -z "$CI" ]] || [[ "$CI" != "true" ]] || [[ "$CIRRUS_CI" != "$CI" ]]; then
     die "Unexpected \$CI=$CI and/or \$CIRRUS_CI=$CIRRUS_CI"
 elif [[ -z "$IMG_SFX" ]] || [[ -z "$PACKER_BUILDS" ]]; then
     die "Required non-empty values for \$IMG_SFX=$IMG_SFX and \$PACKER_BUILDS=$PACKER_BUILDS"
+elif [[ -z "$1" ]]; then
+    die "Build stage name is required as the first argument"
 fi
 
 if skip_on_pr_label; then
     exit 0  # skip build
 fi
 
 set_gac_filepath
-set_aws_filepath
 
-set -exo pipefail
+# Not all builds need this.
+if [[ -n "$AWS_INI" ]]; then
+    set_aws_filepath
+fi
+
+set -x
 cd "$REPO_DIRPATH"
 export IMG_SFX=$IMG_SFX
 export PACKER_BUILDS=$PACKER_BUILDS
-make base_images
+make ${1}
diff --git a/ci/make_cache_images.sh b/ci/make_cache_images.sh
diff --git a/ci/make_image_builder.sh b/ci/make_image_builder.sh