Add support for building AWS EC2 AMIs

.cirrus.yml

@@ -9,7 +9,7 @@ env:
     # No need to go crazy, but grab enough to cover most PRs
     CIRRUS_CLONE_DEPTH: 50
     # Version of packer to use when building images
-    PACKER_VERSION: &PACKER_VERSION "1.7.0"
+    PACKER_VERSION: &PACKER_VERSION "1.8.0"
     # Unique suffix label to use for all images produced by _this_ run (build)
     IMG_SFX: "${CIRRUS_BUILD_ID}"
 
@@ -32,8 +32,8 @@ validate_task:
         - "make clean"
         - "make help"
         - "make image_builder/gce.json"
-        - "make base_images/gce.json"
-        - "make cache_images/gce.json"
+        - "make base_images/cloud.json"
+        - "make cache_images/cloud.json"
 
 
 image_builder_task:
@@ -138,10 +138,11 @@ base_images_task:
     skip: *ci_docs_tooling
     depends_on:
         - container_images
+        - image_builder
     # Packer needs time to clean up partially created VM images
     auto_cancellation: $CI != "true"
     stateful: true
-    timeout_in: 30m
+    timeout_in: 45m
     # Cannot use a container for this task, virt required for fedora image conversion
     gce_instance:
         <<: *ibi_vm
@@ -157,11 +158,18 @@ base_images_task:
         - <<: *base_image
           env:
             PACKER_BUILDS: "prior-fedora"
+        - <<: *base_image
+          env:
+            PACKER_BUILDS: "fedora-aws"
+        - <<: *base_image
+          env:
+            PACKER_BUILDS: "fedora-aws-arm64"
         - <<: *base_image
           env:
             PACKER_BUILDS: "ubuntu"
     env:
       GAC_JSON: ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9]
+      AWS_INI: ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930]
     script: "ci/make_base_images.sh"
     manifest_artifacts:
         path: base_images/manifest.json
@@ -174,11 +182,12 @@ cache_images_task:
     only_if: *is_pr
     skip: *ci_docs_tooling
     depends_on:
+        - image_builder
         - base_images
     # Packer needs time to clean up partially created VM images
     auto_cancellation: $CI != "true"
     stateful: true
-    timeout_in: 40m
+    timeout_in: 45m
     container:
         dockerfile: "image_builder/Containerfile"
         cpu: 2
@@ -201,12 +210,19 @@ cache_images_task:
             PACKER_BUILDS: "fedora-podman-py"
         - <<: *cache_image
           env:
-            PACKER_BUILDS: "ubuntu"
+            PACKER_BUILDS: "fedora-aws"
+        - <<: *cache_image
+          env:
+            PACKER_BUILDS: "fedora-netavark-aws"
         - <<: *cache_image
           env:
             PACKER_BUILDS: "build-push"
+        - <<: *cache_image
+          env:
+            PACKER_BUILDS: "ubuntu"
     env:
       GAC_JSON: ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9]
+      AWS_INI: ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930]
     script: "ci/make_cache_images.sh"
     manifest_artifacts:
         path: cache_images/manifest.json
@@ -220,6 +236,7 @@ imgts_task:
     only_if: *is_pr
     skip: *ci_docs_tooling
     depends_on:
+        - base_images
         - cache_images
     container:
         image: 'quay.io/libpod/imgts:c$IMG_SFX'

Makefile

@@ -20,6 +20,10 @@ if_ci_else = $(if $(findstring true,$(CI)),$(1),$(2))
 # VM images, and storage objects.
 export GAC_FILEPATH
 
+# Ditto for AWS credentials (INI file) with access to create VMs and images.
+# ref: https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-where
+export AWS_SHARED_CREDENTIALS_FILE
+
 PACKER_LOG ?=
 # Uncomment the following to enable additional logging from packer.
 #override PACKER_LOG := 1
@@ -82,11 +86,13 @@ ci_debug: $(_TEMPDIR)/ci_debug.tar ## Build and enter container for local develo
 	/usr/bin/podman run -it --rm \
 		--security-opt label=disable \
 		-v $(_MKFILE_DIR):$(_MKFILE_DIR) -w $(_MKFILE_DIR) \
-		-v $(_TEMPDIR):$(_TEMPDIR):Z \
-		-v $(call err_if_empty,GAC_FILEPATH):$(GAC_FILEPATH):Z \
+		-v $(_TEMPDIR):$(_TEMPDIR) \
+		-v $(call err_if_empty,GAC_FILEPATH):$(GAC_FILEPATH) \
+		-v $(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE):$(AWS_SHARED_CREDENTIALS_FILE) \
 		-e PACKER_INSTALL_DIR=/usr/local/bin \
 		-e PACKER_VERSION=$(call err_if_empty,PACKER_VERSION) \
-		-e GAC_FILEPATH=$(call err_if_empty,GAC_FILEPATH) \
+		-e GAC_FILEPATH=$(GAC_FILEPATH) \
+		-e AWS_SHARED_CREDENTIALS_FILE=$(AWS_SHARED_CREDENTIALS_FILE) \
 		-e TEMPDIR=$(_TEMPDIR) \
 		docker-archive:$<
 
@@ -102,7 +108,7 @@ define podman_build
 	podman save --quiet -o $(1) $(2)
 endef
 
-$(_TEMPDIR)/ci_debug.tar: $(_TEMPDIR)/.cache/fedora ci/Containerfile ci/install_packages.txt ci/install_packages.sh lib.sh
+$(_TEMPDIR)/ci_debug.tar: $(_TEMPDIR)/.cache/fedora $(wildcard ci/*)
 	$(call podman_build,$@,ci_debug,ci,fedora)
 
 $(_TEMPDIR):
@@ -150,13 +156,17 @@ $(_TEMPDIR)/user-data: $(_TEMPDIR) $(_TEMPDIR)/cidata.ssh.pub $(_TEMPDIR)/cidata
 .PHONY: cidata
 cidata: $(_TEMPDIR)/user-data $(_TEMPDIR)/meta-data
 
+# First argument is the path to the template JSON, second
+# argument is the path to AWS_SHARED_CREDENTIALS_FILE
+# when required.  N/B: GAC_FILEPATH is always required.
 define packer_build
 	env PACKER_CACHE_DIR="$(_TEMPDIR)" \
+		AWS_SHARED_CREDENTIALS_FILE="$(2)" \
+		GAC_FILEPATH="$(call err_if_empty,GAC_FILEPATH)" \
 		CHECKPOINT_DISABLE=1 \
 			$(PACKER_INSTALL_DIR)/packer build \
 			-force \
 			-var TEMPDIR="$(_TEMPDIR)" \
-			-var GAC_FILEPATH="$(call err_if_empty,GAC_FILEPATH)" \
 			$(if $(PACKER_BUILDS),-only=$(PACKER_BUILDS)) \
 			$(if $(IMG_SFX),-var IMG_SFX=$(IMG_SFX)) \
 			$(if $(DEBUG_NESTED_VM),-var TTYDEV=$(shell tty),-var TTYDEV=/dev/null) \
@@ -167,7 +177,7 @@ endef
 .PHONY: image_builder
 image_builder: image_builder/manifest.json ## Create image-building image and import into GCE (needed for making all other images)
 image_builder/manifest.json: image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer
-	$(call packer_build,$<)
+	$(call packer_build,$<,)
 
 # Note: We assume this repo is checked out somewhere under the caller's
 # home-dir for bind-mounting purposes.  Otherwise possibly necessary
@@ -177,29 +187,31 @@ image_builder/manifest.json: image_builder/gce.json image_builder/setup.sh lib.s
 image_builder_debug: $(_TEMPDIR)/image_builder_debug.tar ## Build and enter container for local development/debugging of targets requiring packer + virtualization
 	/usr/bin/podman run -it --rm \
 		--security-opt label=disable -v $$HOME:$$HOME -w $(_MKFILE_DIR) \
-		-v $(_TEMPDIR):$(_TEMPDIR):Z \
-		-v $(call err_if_empty,GAC_FILEPATH):$(GAC_FILEPATH):Z \
+		-v $(_TEMPDIR):$(_TEMPDIR) \
+		-v $(call err_if_empty,GAC_FILEPATH):$(GAC_FILEPATH) \
+		-v $(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE):$(AWS_SHARED_CREDENTIALS_FILE) \
 		-v /dev/kvm:/dev/kvm \
 		-e PACKER_INSTALL_DIR=/usr/local/bin \
 		-e PACKER_VERSION=$(call err_if_empty,PACKER_VERSION) \
 		-e IMG_SFX=$(call err_if_empty,IMG_SFX) \
-		-e GAC_FILEPATH=$(call err_if_empty,GAC_FILEPATH) \
+		-e GAC_FILEPATH=$(GAC_FILEPATH) \
+		-e AWS_SHARED_CREDENTIALS_FILE=$(AWS_SHARED_CREDENTIALS_FILE) \
 		docker-archive:$<
 
-$(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR)/.cache/centos image_builder/Containerfile image_builder/install_packages.txt ci/install_packages.sh lib.sh
+$(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR)/.cache/centos $(wildcard image_builder/*)
 	$(call podman_build,$@,image_builder_debug,image_builder,centos)
 
 .PHONY: base_images
 # This needs to run in a virt/nested-virt capable environment
 base_images: base_images/manifest.json ## Create, prepare, and import base-level images into GCE.  Optionally, set PACKER_BUILDS=<csv> to select builder(s).
 
-base_images/manifest.json: base_images/gce.json base_images/fedora_base-setup.sh cidata $(_TEMPDIR)/cidata.ssh $(PACKER_INSTALL_DIR)/packer
-	$(call packer_build,$<)
+base_images/manifest.json: base_images/cloud.json $(wildcard base_images/*.sh) cidata $(_TEMPDIR)/cidata.ssh $(PACKER_INSTALL_DIR)/packer
+	$(call packer_build,base_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE))
 
 .PHONY: cache_images
 cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE.  Optionally, set PACKER_BUILDS=<csv> to select builder(s).
-cache_images/manifest.json: cache_images/gce.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
-	$(call packer_build,$<)
+cache_images/manifest.json: cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer
+	$(call packer_build,cache_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE))
 
 override _fedora_podman_release := $(file < podman/fedora_release)
 override _prior-fedora_podman_release := $(file < podman/prior-fedora_release)