diff --git a/.cirrus.star b/.cirrus.star new file mode 100644 index 00000000..9a96f2a9 --- /dev/null +++ b/.cirrus.star @@ -0,0 +1,11 @@ +# +# Lang. ref: https://github.com/bazelbuild/starlark/blob/master/spec.md#contents +# Impl. ref: https://cirrus-ci.org/guide/programming-tasks/ +load("cirrus", "fs") + +def main(): + return { + "env": { + "IMG_SFX": fs.read("IMG_SFX").strip() + }, + } diff --git a/.cirrus.yml b/.cirrus.yml index 6feec235..e2e9071d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,8 +10,8 @@ env: CIRRUS_CLONE_DEPTH: 50 # Version of packer to use when building images PACKER_VERSION: &PACKER_VERSION "1.8.3" - # Unique suffix label to use for all images produced by _this_ run (build) - IMG_SFX: "${CIRRUS_BUILD_ID}" + #IMG_SFX = + gcp_credentials: ENCRYPTED[823fdbc2fee3c27fa054ba1e9cfca084829b5e71572f1703a28e0746b1a924ee5860193f931adce197d40bf89e7027fe] @@ -35,6 +35,9 @@ validate_task: - "make base_images/cloud.json" - "make cache_images/cloud.json" - "make win_images/win-server-wsl.json" + # Every PR should contain an update to the IMG_SFX file. + # Simply run `make IMG_SFX` and commit the result. + - "git diff --name-only ${CIRRUS_BASE_SHA}..HEAD | grep -q IMG_SFX" image_builder_task: diff --git a/IMG_SFX b/IMG_SFX new file mode 100644 index 00000000..0d81597e --- /dev/null +++ b/IMG_SFX @@ -0,0 +1 @@ +230118v0-f37p36u2204 diff --git a/Makefile b/Makefile index 24ef76a7..263b53fb 100644 --- a/Makefile +++ b/Makefile @@ -94,9 +94,8 @@ override _PACKER_URL := https://releases.hashicorp.com/packer/$(strip $(call err # Align each line properly to the header override _HLPFMT = "%-20s %s\n" -# Suffix used to identify images produce by _this_ execution -# N/B: There are length/character limitations in GCE for image names -IMG_SFX ?= +# Suffix value for any images built from this make execution +_IMG_SFX ?= $(file $@,$(shell date +%y%m%d)v0-f$(FEDORA_RELEASE)p$(PRIOR_FEDORA_RELEASE)u$(subst .,,$(UBUNTU_RELEASE))) + .PHONY: ci_debug ci_debug: $(_TEMPDIR)/ci_debug.tar ## Build and enter container for local development/debugging of container-based Cirrus-CI tasks /usr/bin/podman run -it --rm \ @@ -208,7 +212,7 @@ define packer_build $(PACKER_INSTALL_DIR)/packer build \ -force \ -var TEMPDIR="$(_TEMPDIR)" \ - -var IMG_SFX="$(call err_if_empty,IMG_SFX)" \ + -var IMG_SFX="$(call err_if_empty,_IMG_SFX)" \ $(if $(PACKER_BUILDS),-only=$(PACKER_BUILDS)) \ $(if $(DEBUG_NESTED_VM),-var TTYDEV=$(shell tty),-var TTYDEV=/dev/null) \ $(if $(PACKER_BUILD_ARGS),$(PACKER_BUILD_ARGS)) \ @@ -234,7 +238,7 @@ image_builder_debug: $(_TEMPDIR)/image_builder_debug.tar ## Build and enter cont -v /dev/kvm:/dev/kvm \ -e PACKER_INSTALL_DIR=/usr/local/bin \ -e PACKER_VERSION=$(call err_if_empty,PACKER_VERSION) \ - -e IMG_SFX=$(call err_if_empty,IMG_SFX) \ + -e IMG_SFX=$(call err_if_empty,_IMG_SFX) \ -e GAC_FILEPATH=$(GAC_FILEPATH) \ -e AWS_SHARED_CREDENTIALS_FILE=$(AWS_SHARED_CREDENTIALS_FILE) \ docker-archive:$< @@ -244,14 +248,14 @@ $(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR)/.cache/centos $(wildcard image_ # Avoid re-downloading unnecessarily # Ref: https://www.gnu.org/software/make/manual/html_node/Special-Targets.html#Special-Targets -.PRECIOUS: $(_TEMPDIR)/fedora-aws-$(IMG_SFX).$(IMPORT_FORMAT) -$(_TEMPDIR)/fedora-aws-$(IMG_SFX).$(IMPORT_FORMAT): $(_TEMPDIR) +.PRECIOUS: $(_TEMPDIR)/fedora-aws-$(_IMG_SFX).$(IMPORT_FORMAT) +$(_TEMPDIR)/fedora-aws-$(_IMG_SFX).$(IMPORT_FORMAT): $(_TEMPDIR) bash import_images/handle_image.sh \ $@ \ $(call err_if_empty,FEDORA_IMAGE_URL) \ $(call err_if_empty,FEDORA_CSUM_URL) -$(_TEMPDIR)/fedora-aws-arm64-$(IMG_SFX).$(IMPORT_FORMAT): $(_TEMPDIR) +$(_TEMPDIR)/fedora-aws-arm64-$(_IMG_SFX).$(IMPORT_FORMAT): $(_TEMPDIR) bash import_images/handle_image.sh \ $@ \ $(call err_if_empty,FEDORA_ARM64_IMAGE_URL) \ @@ -298,17 +302,17 @@ $(_TEMPDIR)/%.snapshot_id: $(_TEMPDIR)/%.import_task_id define _register_sed sed -r \ -e 's/@@@NAME@@@/$(1)/' \ - -e 's/@@@IMG_SFX@@@/$(IMG_SFX)/' \ + -e 's/@@@IMG_SFX@@@/$(_IMG_SFX)/' \ -e 's/@@@ARCH@@@/$(2)/' \ -e 's/@@@SNAPSHOT_ID@@@/$(3)/' \ import_images/register.json.in \ > $(4) endef -$(_TEMPDIR)/fedora-aws-$(IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-$(IMG_SFX).snapshot_id import_images/register.json.in +$(_TEMPDIR)/fedora-aws-$(_IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-$(_IMG_SFX).snapshot_id import_images/register.json.in $(call _register_sed,fedora-aws,x86_64,$(file <$<),$@) -$(_TEMPDIR)/fedora-aws-arm64-$(IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-arm64-$(IMG_SFX).snapshot_id import_images/register.json.in +$(_TEMPDIR)/fedora-aws-arm64-$(_IMG_SFX).register.json: $(_TEMPDIR)/fedora-aws-arm64-$(_IMG_SFX).snapshot_id import_images/register.json.in $(call _register_sed,fedora-aws-arm64,arm64,$(file <$<),$@) # Avoid multiple registrations for the same image @@ -333,9 +337,9 @@ $(_TEMPDIR)/%.ami.json: $(_TEMPDIR)/%.ami.id $(_TEMPDIR)/%.ami.name | tee $@ .PHONY: import_images -import_images: $(_TEMPDIR)/fedora-aws-$(IMG_SFX).ami.json $(_TEMPDIR)/fedora-aws-arm64-$(IMG_SFX).ami.json import_images/manifest.json.in ## Import generic Fedora cloud images into AWS EC2. +import_images: $(_TEMPDIR)/fedora-aws-$(_IMG_SFX).ami.json $(_TEMPDIR)/fedora-aws-arm64-$(_IMG_SFX).ami.json import_images/manifest.json.in ## Import generic Fedora cloud images into AWS EC2. sed -r \ - -e 's/@@@IMG_SFX@@@/$(IMG_SFX)/' \ + -e 's/@@@IMG_SFX@@@/$(_IMG_SFX)/' \ -e 's/@@@CIRRUS_TASK_ID@@@/$(CIRRUS_TASK_ID)/' \ import_images/manifest.json.in \ > import_images/manifest.json @@ -343,7 +347,7 @@ import_images: $(_TEMPDIR)/fedora-aws-$(IMG_SFX).ami.json $(_TEMPDIR)/fedora-aws @echo "############################################################" @echo "Please update Makefile value:" @echo "" - @echo " FEDORA_IMPORT_IMG_SFX = $(IMG_SFX)" + @echo " FEDORA_IMPORT_IMG_SFX = $(_IMG_SFX)" @echo "############################################################" .PHONY: base_images @@ -377,7 +381,7 @@ ubuntu_podman: ## Build Ubuntu podman development container $(call build_podman_container,$@,$(UBUNTU_RELEASE)) $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/% - podman build -t $*_podman:$(call err_if_empty,IMG_SFX) \ + podman build -t $*_podman:$(call err_if_empty,_IMG_SFX) \ --security-opt seccomp=unconfined \ --build-arg=BASE_NAME=$(subst prior-,,$*) \ --build-arg=BASE_TAG=$(call err_if_empty,BASE_TAG) \ @@ -386,40 +390,40 @@ $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_i -v $(_TEMPDIR)/.cache/$*:/var/cache/apt:Z \ -f podman/Containerfile . rm -f $@ - podman save --quiet -o $@ $*_podman:$(IMG_SFX) + podman save --quiet -o $@ $*_podman:$(_IMG_SFX) .PHONY: skopeo_cidev skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar ## Build Skopeo development and CI container $(_TEMPDIR)/skopeo_cidev.tar: $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora - podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \ + podman build -t skopeo_cidev:$(call err_if_empty,_IMG_SFX) \ --security-opt seccomp=unconfined \ --build-arg=BASE_TAG=$(FEDORA_RELEASE) \ -v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \ skopeo_cidev rm -f $@ - podman save --quiet -o $@ skopeo_cidev:$(IMG_SFX) + podman save --quiet -o $@ skopeo_cidev:$(_IMG_SFX) # TODO: Temporarily force F36 due to: # https://github.com/aio-libs/aiohttp/issues/6600 .PHONY: ccia ccia: $(_TEMPDIR)/ccia.tar ## Build the Cirrus-CI Artifacts container image $(_TEMPDIR)/ccia.tar: ccia/Containerfile - podman build -t ccia:$(call err_if_empty,IMG_SFX) \ + podman build -t ccia:$(call err_if_empty,_IMG_SFX) \ --security-opt seccomp=unconfined \ --build-arg=BASE_TAG=36 \ ccia rm -f $@ - podman save --quiet -o $@ ccia:$(IMG_SFX) + podman save --quiet -o $@ ccia:$(_IMG_SFX) .PHONY: imgts imgts: $(_TEMPDIR)/imgts.tar ## Build the VM image time-stamping container image $(_TEMPDIR)/imgts.tar: imgts/Containerfile imgts/entrypoint.sh imgts/google-cloud-sdk.repo imgts/lib_entrypoint.sh $(_TEMPDIR)/.cache/centos - $(call podman_build,$@,imgts:$(call err_if_empty,IMG_SFX),imgts,centos) + $(call podman_build,$@,imgts:$(call err_if_empty,_IMG_SFX),imgts,centos) define imgts_base_podman_build podman load -i $(_TEMPDIR)/imgts.tar - podman tag imgts:$(call err_if_empty,IMG_SFX) imgts:latest - $(call podman_build,$@,$(1):$(call err_if_empty,IMG_SFX),$(1),centos) + podman tag imgts:$(call err_if_empty,_IMG_SFX) imgts:latest + $(call podman_build,$@,$(1):$(call err_if_empty,_IMG_SFX),$(1),centos) endef .PHONY: imgobsolete @@ -445,9 +449,9 @@ $(_TEMPDIR)/orphanvms.tar: $(_TEMPDIR)/imgts.tar imgts/lib_entrypoint.sh orphanv .PHONY: .get_ci_vm get_ci_vm: $(_TEMPDIR)/get_ci_vm.tar ## Build the get_ci_vm container image $(_TEMPDIR)/get_ci_vm.tar: lib.sh get_ci_vm/Containerfile get_ci_vm/entrypoint.sh get_ci_vm/setup.sh $(_TEMPDIR) - podman build -t get_ci_vm:$(call err_if_empty,IMG_SFX) -f get_ci_vm/Containerfile . + podman build -t get_ci_vm:$(call err_if_empty,_IMG_SFX) -f get_ci_vm/Containerfile . rm -f $@ - podman save --quiet -o $@ get_ci_vm:$(IMG_SFX) + podman save --quiet -o $@ get_ci_vm:$(_IMG_SFX) .PHONY: clean clean: ## Remove all generated files referenced in this Makefile diff --git a/README.md b/README.md index 9a8ee462..6c78c41a 100644 --- a/README.md +++ b/README.md @@ -145,23 +145,25 @@ see step 4 below. ### Process: ### -1. After you make your script changes, push to a PR. They will be - validated and linted before VM image production begins. +1. Whether or not you made any script changes, run `make IMG_SFX`. Before pushing + to a PR, confirm there are no other existing build PRs that could clash + with the date-based `IMG_SFX` value. If there are, increment the `v0` value + in the file before pushing to a new PR. -2. The name of all output GCE images will share a common suffix (*image ID*). +2. The name of all output images will share a common `IMG_SFX` value Assuming a successful image-build, a [github-action](.github/workflows/pr_image_id.yml) - will post the new *image ID* as a comment in the PR. If this automation - breaks, you may need to [figure the ID out the hard + will post the types and names of all built images as a comment in the PR. + If this automation breaks, you may need to [figure the ID out the hard way](README.md#Looking-up-an-image-ID). For AWS EC2 images, every one will have a unique AMI ID assigned. You'll need to [look these up separately](README.md#Looking-up-an-image-ID) - until the github action is updated. + until the github action is fixed. 3. Go over to whatever other containers/repository needed the image update. Open the `.cirrus.yml` file, and find the 'env' line referencing the *image ID* and/or *AMI*. It will likely be named `IMAGE_SUFFIX:` or something - similar. Paste in the *image ID* or *AMI*. + similar. Paste in the *image ID* (it should begin with the letter `c`). 4. Open up a PR with this change, and push it. Once all tests pass and you're satisfied with the image changes, ask somebody to review/approve both @@ -173,30 +175,27 @@ see step 4 below. ### Looking up an image ID: ### -A GCE *image ID* is simply big number prefixed by the letter 'c'. You may -need to look it up in a PR for example, if +A *image ID* is simply the `IMG_SFX` value prefixed by a letter. The letter +`b` represents base-images, and 'c' represents cache-images. You may +need to look the value up if (for example), [the automated comment posting github-action](.github/workflows/pr_image_id.yml) -fails. For AWS EC2 images, you'll need to look up the AMI ID (string) for each -cache-image produced. +failed. -1. In a PR, find and click the build task for the image you're interested in. - Near the top of the Cirrus-CI WebUI, will be a section labeled 'Artifacts'. +1. In any PR, find and click the *details* link for one of the build tasks. + Near the top of the *Cirrus-CI WebUI*, will be a section labeled 'Artifacts'. -2. Click the `manifest` artifact +2. Click the `manifest` artifact. -3. Click the `cache_images` folder +3. Click the nested folder name (if there is one). 4. Click the `manifest.json` file, it should open in your browser window. -5. For *GCE images* look at the `artifact_id` field. It will end in a - `c`. This is the ID for - this one, specific image. **Every AWS image will have a unique AMI ID** - (unlike the shared ID for GCE images). +5. For *GCE images* look at the `artifact_id` field. It will contain the + full *image id* value in the form `c`. +6. For *AWS EC2 images* look under the `custom_data` field for the `IMG_SFX` + value. If this was a base-image, simply prefix the value with a `b`, or + a `c` for a cache-image. ## The image-builder image (overview step 1) @@ -404,26 +403,20 @@ credential files and ensure correct account configuration. Having these files stored *in your home directory* on your laptop/workstation, the process of producing images proceeds as follows: -1. Invent some unique identity suffix for your images. It may contain (***only***) - lowercase letters, numbers and dashes; nothing else. Some suggestions - of useful values would be your name and today's date. If you manage to screw - this up somehow, stern errors will be presented without causing any real harm. - -2. Ensure you have podman installed, and lots of available network and CPU +1. Ensure you have podman installed, and lots of available network and CPU resources (i.e. turn off YouTube, shut down background VMs and other hungry tasks). Build the image-builder container image, by executing ``` make image_builder_debug GAC_FILEPATH= \ - AWS_SHARED_CREDENTIALS_FILE= \ - IMG_SFX= + AWS_SHARED_CREDENTIALS_FILE= ``` -3. You will be dropped into a debugging container, inside a volume-mount of +2. You will be dropped into a debugging container, inside a volume-mount of the repository root. This container is practically identical to the VM produced and used in *overview step 1*. If changes are made, the container image should be re-built to reflect them. -4. If you wish to build only a subset of available images, list the names +3. If you wish to build only a subset of available images, list the names you want as comma-separated values of the `PACKER_BUILDS` variable. Be sure you *export* this variable so that `make` has access to it. For example, `export PACKER_BUILDS=ubuntu,prior-fedora`.