From 2bd0e5eb471336a57e4b464fa67507e88821e158 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 17 Jan 2023 12:25:56 -0500 Subject: [PATCH] Human-readable date-based image suffixes Historically, the ID used to identify a set of images was taken from the Cirrus-CI build ID. This was done to make auditing easier, since one can easily retrieve the build logs using the ID. However, there are many disadvantages to using the build id: * It's not human-readable, making it difficult to ascertain exactly when the images were built. * It's not guaranteed to be incremental, and therefore cannot be utilized as a "version". * It doesn't convey helpful information like when it was produced, or which release of Fedora is included in the set. For these and other reasons, switch to a simple date-based image suffix encoded in a repository file. This value may also include the versions of Fedora, Prior-Fedora, and Ubuntu for easy reference. Signed-off-by: Chris Evich --- .cirrus.star | 11 ++++++++++ .cirrus.yml | 11 +++------- IMG_SFX | 1 + Makefile | 57 +++++++++++++++++++++++++++--------------------- README.md | 59 ++++++++++++++++++++++---------------------------- ci/validate.sh | 32 +++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 66 deletions(-) create mode 100644 .cirrus.star create mode 100644 IMG_SFX create mode 100755 ci/validate.sh diff --git a/.cirrus.star b/.cirrus.star new file mode 100644 index 00000000..9a96f2a9 --- /dev/null +++ b/.cirrus.star @@ -0,0 +1,11 @@ +# +# Lang. ref: https://github.com/bazelbuild/starlark/blob/master/spec.md#contents +# Impl. ref: https://cirrus-ci.org/guide/programming-tasks/ +load("cirrus", "fs") + +def main(): + return { + "env": { + "IMG_SFX": fs.read("IMG_SFX").strip() + }, + } diff --git a/.cirrus.yml b/.cirrus.yml index 6feec235..d6006201 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,8 +10,8 @@ env: CIRRUS_CLONE_DEPTH: 50 # Version of packer to use when building images PACKER_VERSION: &PACKER_VERSION "1.8.3" - # Unique suffix label to use for all images produced by _this_ run (build) - IMG_SFX: "${CIRRUS_BUILD_ID}" + #IMG_SFX = + gcp_credentials: ENCRYPTED[823fdbc2fee3c27fa054ba1e9cfca084829b5e71572f1703a28e0746b1a924ee5860193f931adce197d40bf89e7027fe] @@ -29,12 +29,7 @@ validate_task: PACKER_VERSION: *PACKER_VERSION script: - "ci/shellcheck.sh" - - "make clean" - - "make help" - - "make image_builder/gce.json" - - "make base_images/cloud.json" - - "make cache_images/cloud.json" - - "make win_images/win-server-wsl.json" + - "ci/validate.sh" image_builder_task: diff --git a/IMG_SFX b/IMG_SFX new file mode 100644 index 00000000..7c997017 --- /dev/null +++ b/IMG_SFX @@ -0,0 +1 @@ +230119-131309-f37-f36-u2204 diff --git a/Makefile b/Makefile index 24ef76a7..663a5907 100644 --- a/Makefile +++ b/Makefile @@ -94,9 +94,8 @@ override _PACKER_URL := https://releases.hashicorp.com/packer/$(strip $(call err # Align each line properly to the header override _HLPFMT = "%-20s %s\n" -# Suffix used to identify images produce by _this_ execution -# N/B: There are length/character limitations in GCE for image names -IMG_SFX ?= +# Suffix value for any images built from this make execution +_IMG_SFX ?= $(file $@,$(shell date +%y%m%d)-$(shell date +%H%M%S)-f$(FEDORA_RELEASE)-f$(PRIOR_FEDORA_RELEASE)-u$(subst .,,$(UBUNTU_RELEASE))) + @echo "$(file $(4) endef -$(_TEMPDIR)/fedora-aws-$(IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-$(IMG_SFX).snapshot_id import_images/register.json.in +$(_TEMPDIR)/fedora-aws-$(_IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-$(_IMG_SFX).snapshot_id import_images/register.json.in $(call _register_sed,fedora-aws,x86_64,$(file <$<),$@) -$(_TEMPDIR)/fedora-aws-arm64-$(IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-arm64-$(IMG_SFX).snapshot_id import_images/register.json.in +$(_TEMPDIR)/fedora-aws-arm64-$(_IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-arm64-$(_IMG_SFX).snapshot_id import_images/register.json.in $(call _register_sed,fedora-aws-arm64,arm64,$(file <$<),$@) # Avoid multiple registrations for the same image @@ -333,9 +340,9 @@ $(_TEMPDIR)/%.ami.json: $(_TEMPDIR)/%.ami.id $(_TEMPDIR)/%.ami.name | tee $@ .PHONY: import_images -import_images: $(_TEMPDIR)/fedora-aws-$(IMG_SFX).ami.json $(_TEMPDIR)/fedora-aws-arm64-$(IMG_SFX).ami.json import_images/manifest.json.in ## Import generic Fedora cloud images into AWS EC2. +import_images: $(_TEMPDIR)/fedora-aws-$(_IMG_SFX).ami.json $(_TEMPDIR)/fedora-aws-arm64-$(_IMG_SFX).ami.json import_images/manifest.json.in ## Import generic Fedora cloud images into AWS EC2. sed -r \ - -e 's/@@@IMG_SFX@@@/$(IMG_SFX)/' \ + -e 's/@@@IMG_SFX@@@/$(_IMG_SFX)/' \ -e 's/@@@CIRRUS_TASK_ID@@@/$(CIRRUS_TASK_ID)/' \ import_images/manifest.json.in \ > import_images/manifest.json @@ -343,7 +350,7 @@ import_images: $(_TEMPDIR)/fedora-aws-$(IMG_SFX).ami.json $(_TEMPDIR)/fedora-aws @echo "############################################################" @echo "Please update Makefile value:" @echo "" - @echo " FEDORA_IMPORT_IMG_SFX = $(IMG_SFX)" + @echo " FEDORA_IMPORT_IMG_SFX = $(_IMG_SFX)" @echo "############################################################" .PHONY: base_images @@ -377,7 +384,7 @@ ubuntu_podman: ## Build Ubuntu podman development container $(call build_podman_container,$@,$(UBUNTU_RELEASE)) $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/% - podman build -t $*_podman:$(call err_if_empty,IMG_SFX) \ + podman build -t $*_podman:$(call err_if_empty,_IMG_SFX) \ --security-opt seccomp=unconfined \ --build-arg=BASE_NAME=$(subst prior-,,$*) \ --build-arg=BASE_TAG=$(call err_if_empty,BASE_TAG) \ @@ -386,40 +393,40 @@ $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_i -v $(_TEMPDIR)/.cache/$*:/var/cache/apt:Z \ -f podman/Containerfile . rm -f $@ - podman save --quiet -o $@ $*_podman:$(IMG_SFX) + podman save --quiet -o $@ $*_podman:$(_IMG_SFX) .PHONY: skopeo_cidev skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar ## Build Skopeo development and CI container $(_TEMPDIR)/skopeo_cidev.tar: $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora - podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \ + podman build -t skopeo_cidev:$(call err_if_empty,_IMG_SFX) \ --security-opt seccomp=unconfined \ --build-arg=BASE_TAG=$(FEDORA_RELEASE) \ -v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \ skopeo_cidev rm -f $@ - podman save --quiet -o $@ skopeo_cidev:$(IMG_SFX) + podman save --quiet -o $@ skopeo_cidev:$(_IMG_SFX) # TODO: Temporarily force F36 due to: # https://github.com/aio-libs/aiohttp/issues/6600 .PHONY: ccia ccia: $(_TEMPDIR)/ccia.tar ## Build the Cirrus-CI Artifacts container image $(_TEMPDIR)/ccia.tar: ccia/Containerfile - podman build -t ccia:$(call err_if_empty,IMG_SFX) \ + podman build -t ccia:$(call err_if_empty,_IMG_SFX) \ --security-opt seccomp=unconfined \ --build-arg=BASE_TAG=36 \ ccia rm -f $@ - podman save --quiet -o $@ ccia:$(IMG_SFX) + podman save --quiet -o $@ ccia:$(_IMG_SFX) .PHONY: imgts imgts: $(_TEMPDIR)/imgts.tar ## Build the VM image time-stamping container image $(_TEMPDIR)/imgts.tar: imgts/Containerfile imgts/entrypoint.sh imgts/google-cloud-sdk.repo imgts/lib_entrypoint.sh $(_TEMPDIR)/.cache/centos - $(call podman_build,$@,imgts:$(call err_if_empty,IMG_SFX),imgts,centos) + $(call podman_build,$@,imgts:$(call err_if_empty,_IMG_SFX),imgts,centos) define imgts_base_podman_build podman load -i $(_TEMPDIR)/imgts.tar - podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest - $(call podman_build,$@,$(1):$(call err_if_empty,IMG_SFX),$(1),centos) + podman tag imgts:$(call err_if_empty,_IMG_SFX) imgts:latest + $(call podman_build,$@,$(1):$(call err_if_empty,_IMG_SFX),$(1),centos) endef .PHONY: imgobsolete @@ -445,9 +452,9 @@ $(_TEMPDIR)/orphanvms.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh orphanv .PHONY: .get_ci_vm get_ci_vm: $(_TEMPDIR)/get_ci_vm.tar ## Build the get_ci_vm container image $(_TEMPDIR)/get_ci_vm.tar: lib.sh get_ci_vm/Containerfile get_ci_vm/entrypoint.sh get_ci_vm/setup.sh $(_TEMPDIR) - podman build -t get_ci_vm:$(call err_if_empty,IMG_SFX) -f get_ci_vm/Containerfile . + podman build -t get_ci_vm:$(call err_if_empty,_IMG_SFX) -f get_ci_vm/Containerfile . rm -f $@ - podman save --quiet -o $@ get_ci_vm:$(IMG_SFX) + podman save --quiet -o $@ get_ci_vm:$(_IMG_SFX) .PHONY: clean clean: ## Remove all generated files referenced in this Makefile diff --git a/README.md b/README.md index 9a8ee462..b2e88dd0 100644 --- a/README.md +++ b/README.md @@ -145,23 +145,25 @@ see step 4 below. ### Process: ### -1. After you make your script changes, push to a PR. They will be - validated and linted before VM image production begins. +1. Whether or not you made any script changes, before pushing to a PR run + `make IMG_SFX` and commit the change. This helps avoid overwriting + existing images - potentially already in use by CI. The 'validate' + CI task will asserts every PR makes changes to this file. -2. The name of all output GCE images will share a common suffix (*image ID*). +2. The name of all output images will share a common `IMG_SFX` value Assuming a successful image-build, a [github-action](.github/workflows/pr_image_id.yml) - will post the new *image ID* as a comment in the PR. If this automation - breaks, you may need to [figure the ID out the hard + will post the types and names of all built images as a comment in the PR. + If this automation breaks, you may need to [figure the ID out the hard way](README.md#Looking-up-an-image-ID). For AWS EC2 images, every one will have a unique AMI ID assigned. You'll need to [look these up separately](README.md#Looking-up-an-image-ID) - until the github action is updated. + until the github action is fixed. 3. Go over to whatever other containers/repository needed the image update. Open the `.cirrus.yml` file, and find the 'env' line referencing the *image ID* and/or *AMI*. It will likely be named `IMAGE_SUFFIX:` or something - similar. Paste in the *image ID* or *AMI*. + similar. Paste in the *image ID* (it should begin with the letter `c`). 4. Open up a PR with this change, and push it. Once all tests pass and you're satisfied with the image changes, ask somebody to review/approve both @@ -173,30 +175,27 @@ see step 4 below. ### Looking up an image ID: ### -A GCE *image ID* is simply big number prefixed by the letter 'c'. You may -need to look it up in a PR for example, if +A *image ID* is simply the `IMG_SFX` value prefixed by a letter. The letter +`b` represents base-images, and 'c' represents cache-images. You may +need to look the value up if (for example), [the automated comment posting github-action](.github/workflows/pr_image_id.yml) -fails. For AWS EC2 images, you'll need to look up the AMI ID (string) for each -cache-image produced. +failed. -1. In a PR, find and click the build task for the image you're interested in. - Near the top of the Cirrus-CI WebUI, will be a section labeled 'Artifacts'. +1. In any PR, find and click the *details* link for one of the build tasks. + Near the top of the *Cirrus-CI WebUI*, will be a section labeled 'Artifacts'. -2. Click the `manifest` artifact +2. Click the `manifest` artifact. -3. Click the `cache_images` folder +3. Click the nested folder name (if there is one). 4. Click the `manifest.json` file, it should open in your browser window. -5. For *GCE images* look at the `artifact_id` field. It will end in a - `c`. This is the ID for - this one, specific image. **Every AWS image will have a unique AMI ID** - (unlike the shared ID for GCE images). +5. For *GCE images* look at the `artifact_id` field. It will contain the + full *image id* value in the form `c`. +6. For *AWS EC2 images* look under the `custom_data` field for the `IMG_SFX` + value. If this was a base-image, simply prefix the value with a `b`, or + a `c` for a cache-image. ## The image-builder image (overview step 1) @@ -404,26 +403,20 @@ credential files and ensure correct account configuration. Having these files stored *in your home directory* on your laptop/workstation, the process of producing images proceeds as follows: -1. Invent some unique identity suffix for your images. It may contain (***only***) - lowercase letters, numbers and dashes; nothing else. Some suggestions - of useful values would be your name and today's date. If you manage to screw - this up somehow, stern errors will be presented without causing any real harm. - -2. Ensure you have podman installed, and lots of available network and CPU +1. Ensure you have podman installed, and lots of available network and CPU resources (i.e. turn off YouTube, shut down background VMs and other hungry tasks). Build the image-builder container image, by executing ``` make image_builder_debug GAC_FILEPATH= \ - AWS_SHARED_CREDENTIALS_FILE= \ - IMG_SFX= + AWS_SHARED_CREDENTIALS_FILE= ``` -3. You will be dropped into a debugging container, inside a volume-mount of +2. You will be dropped into a debugging container, inside a volume-mount of the repository root. This container is practically identical to the VM produced and used in *overview step 1*. If changes are made, the container image should be re-built to reflect them. -4. If you wish to build only a subset of available images, list the names +3. If you wish to build only a subset of available images, list the names you want as comma-separated values of the `PACKER_BUILDS` variable. Be sure you *export* this variable so that `make` has access to it. For example, `export PACKER_BUILDS=ubuntu,prior-fedora`. diff --git a/ci/validate.sh b/ci/validate.sh new file mode 100755 index 00000000..1d21e55e --- /dev/null +++ b/ci/validate.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# This script is intended to be run by Cirrus-CI to validate PR +# content prior to building any images. It should not be run +# under any other context. + +set -eo pipefail + +SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") +REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") + +# shellcheck source=./lib.sh +source "$REPO_DIRPATH/lib.sh" + +# die() will add a reference to this file and line number. +[[ "$CIRRUS_CI" == "true" ]] || \ + die "This script is only/ever intended to be run by Cirrus-CI." + +for target in image_builder/gce.json base_images/cloud.json \ + cache_images/cloud.json win_images/win-server-wsl.json; do + if ! make $target; then + die "Running 'make $target' failed, please validate input YAML files." + fi +done + +# Variable is defined by Cirrus-CI at runtime +# shellcheck disable=SC2154 +if ! git diff --name-only ${CIRRUS_BASE_SHA}..HEAD | grep -q IMG_SFX; then + die "Every PR must include an updated IMG_SFX file. +Simply run 'make IMG_SFX', commit the result, and re-push." +fi