diff --git a/Makefile b/Makefile index ec21c39c..3ea9a2c6 100644 --- a/Makefile +++ b/Makefile @@ -159,6 +159,13 @@ $(_TEMPDIR)/user-data: $(_TEMPDIR) $(_TEMPDIR)/cidata.ssh.pub $(_TEMPDIR)/cidata .PHONY: cidata cidata: $(_TEMPDIR)/user-data $(_TEMPDIR)/meta-data +override _fedora_release = $(shell bash get_base_release.sh FEDORA) +override _prior_fedora_release = $(shell bash get_base_release.sh PRIOR_FEDORA) +override _ubuntu_release = $(shell bash get_base_release.sh UBUNTU) +define build_podman_container + $(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(2) +endef + # First argument is the path to the template JSON, second # argument is the path to AWS_SHARED_CREDENTIALS_FILE # when required. N/B: GAC_FILEPATH is always required. @@ -170,6 +177,9 @@ define packer_build $(PACKER_INSTALL_DIR)/packer build \ -force \ -var TEMPDIR="$(_TEMPDIR)" \ + -var FEDORA_RELEASE="$(call err_if_empty,_fedora_release)" \ + -var PRIOR_FEDORA_RELEASE="$(call err_if_empty,_prior_fedora_release)" \ + -var UBUNTU_RELEASE="$(call err_if_empty,_ubuntu_release)" \ $(if $(PACKER_BUILDS),-only=$(PACKER_BUILDS)) \ $(if $(IMG_SFX),-var IMG_SFX=$(IMG_SFX)) \ $(if $(DEBUG_NESTED_VM),-var TTYDEV=$(shell tty),-var TTYDEV=/dev/null) \ @@ -179,8 +189,8 @@ endef .PHONY: image_builder image_builder: image_builder/manifest.json ## Create image-building image and import into GCE (needed for making all other images) -image_builder/manifest.json: image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer - $(call packer_build,$<,) +image_builder/manifest.json: base_images/cloud.json image_builder/gce.json image_builder/setup.sh lib.sh systemd_banish.sh $(PACKER_INSTALL_DIR)/packer + $(call packer_build,image_builder/gce.json,) # Note: We assume this repo is checked out somewhere under the caller's # home-dir for bind-mounting purposes. Otherwise possibly necessary @@ -213,27 +223,20 @@ base_images/manifest.json: base_images/cloud.json $(wildcard base_images/*.sh) c .PHONY: cache_images cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE. Optionally, set PACKER_BUILDS= to select builder(s). -cache_images/manifest.json: cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer +cache_images/manifest.json: base_images/cloud.json cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer $(call packer_build,cache_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE)) -override _fedora_podman_release := $(file < podman/fedora_release) -override _prior-fedora_podman_release := $(file < podman/prior-fedora_release) -override _ubuntu_podman_release := $(file < podman/ubuntu_release) -define build_podman_container - $(MAKE) $(_TEMPDIR)/$(1).tar BASE_TAG=$(_$(1)_release) -endef - .PHONY: fedora_podman -fedora_podman: ## Build Fedora podman development container - $(call build_podman_container,$@,fedora) +fedora_podman: base_images/cloud.json ## Build Fedora podman development container + $(call build_podman_container,$@,$(call err_if_empty,_fedora_release)) .PHONY: prior-fedora_podman -prior-fedora_podman: ## Build Prior-Fedora podman development container - $(call build_podman_container,$@,prior-fedora) +prior-fedora_podman: base_images/cloud.json ## Build Prior-Fedora podman development container + $(call build_podman_container,$@,$(call err_if_empty,_prior_fedora_release)) .PHONY: ubuntu_podman -ubuntu_podman: ## Build Ubuntu podman development container - $(call build_podman_container,$@,ubuntu) +ubuntu_podman: base_images/cloud.json ## Build Ubuntu podman development container + $(call build_podman_container,$@,$(call err_if_empty,_ubuntu_release)) $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_images/*.sh) $(wildcard cache_images/*.sh) $(_TEMPDIR)/.cache/% podman build -t $*_podman:$(call err_if_empty,IMG_SFX) \ @@ -249,10 +252,10 @@ $(_TEMPDIR)/%_podman.tar: podman/Containerfile podman/setup.sh $(wildcard base_i .PHONY: skopeo_cidev skopeo_cidev: $(_TEMPDIR)/skopeo_cidev.tar ## Build Skopeo development and CI container -$(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora +$(_TEMPDIR)/skopeo_cidev.tar: base_images/cloud.json $(wildcard skopeo_base/*) $(_TEMPDIR)/.cache/fedora podman build -t skopeo_cidev:$(call err_if_empty,IMG_SFX) \ --security-opt seccomp=unconfined \ - --build-arg=BASE_TAG=$(_fedora_podman_release) \ + --build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \ -v $(_TEMPDIR)/.cache/fedora:/var/cache/dnf:Z \ skopeo_cidev rm -f $@ @@ -260,10 +263,10 @@ $(_TEMPDIR)/skopeo_cidev.tar: podman/fedora_release $(wildcard skopeo_base/*) $( .PHONY: ccia ccia: $(_TEMPDIR)/ccia.tar ## Build the Cirrus-CI Artifacts container image -$(_TEMPDIR)/ccia.tar: ccia/Containerfile +$(_TEMPDIR)/ccia.tar: ccia/Containerfile base_images/cloud.json podman build -t ccia:$(call err_if_empty,IMG_SFX) \ --security-opt seccomp=unconfined \ - --build-arg=BASE_TAG=$(_fedora_podman_release) \ + --build-arg=BASE_TAG=$(call err_if_empty,_fedora_release) \ ccia rm -f $@ podman save --quiet -o $@ ccia:$(IMG_SFX) diff --git a/base_images/cloud.yml b/base_images/cloud.yml index 5c9cda31..aa2b42bc 100644 --- a/base_images/cloud.yml +++ b/base_images/cloud.yml @@ -17,11 +17,11 @@ variables: # Empty value means it must be passed in on command-line # Allows providing handy cross-reference to the build log CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}" - # BIG-FAT-WARNING: When updating the image names and/or URLs below, - # ensure the distro version numbers contained in the `podman/*_release` - # files exactly match. These represent the container base-image tags - # to build from - just as the sources below are the base-images to - # start from building VM images. + # This data is also used when building cache and container images. + # It is assumed to match all the other OS variables below. + UBUNTU_RELEASE: "22.04" + FEDORA_RELEASE: "36" + PRIOR_FEDORA_RELEASE: "35" # Upstream source for Ubuntu image to duplicate (prevents expiration). # Use the most recent image based on this family name. @@ -50,7 +50,7 @@ builders: # Prefix IMG_SFX with "b" so this is never confused with a cache_image name image_name: '{{build_name}}-b{{user `IMG_SFX`}}' image_family: '{{build_name}}-base' - image_description: 'Built in https://cirrus-ci.com/build/{{user `IMG_SFX`}}' + image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}' source_image_family: '{{user `UBUNTU_BASE_FAMILY`}}' source_image_project_id: 'ubuntu-os-cloud' project_id: '{{user `GCP_PROJECT_ID`}}' @@ -64,6 +64,7 @@ builders: src: '{{user `UBUNTU_BASE_FAMILY`}}' stage: 'base' arch: 'x86_64' + release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}' # Gotcha: https://www.packer.io/docs/builders/googlecompute#gotchas ssh_username: 'packer' temporary_key_pair_type: ed25519 @@ -132,7 +133,7 @@ builders: security_group_id: "sg-042c75677872ef81c" # Prefix IMG_SFX with "b" so this is never confused with a cache_image ami_name: 'fedora-aws-b{{user `IMG_SFX`}}' - ami_description: 'Built in https://cirrus-ci.com/build/{{user `IMG_SFX`}}' + ami_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}' ebs_optimized: true launch_block_device_mappings: - device_name: '/dev/sda1' @@ -147,6 +148,7 @@ builders: Name: 'fedora-aws-b{{user `IMG_SFX`}}' src: '{{user `FEDORAPROJECT_AMI`}}' automation: 'true' + release: 'fedora-{{user `FEDORA_RELEASE`}}' run_tags: *awstags run_volume_tags: *awstags snapshot_tags: *awstags @@ -214,16 +216,19 @@ post-processors: gcs_object_name: '{{build_name}}-{{user `IMG_SFX`}}.tar.gz' image_name: "fedora-b{{user `IMG_SFX`}}" image_family: '{{build_name}}-base' - # Can't save the url in an image_label - image_description: '{{user `FEDORA_IMAGE_URL`}}' - image_labels: &importlabels + image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}' + image_labels: <<: *imgcpylabels src: 'fedoraproject' + release: 'fedora-{{user `FEDORA_RELEASE`}}' - <<: *gcp_import only: ['prior-fedora'] image_name: "prior-fedora-b{{user `IMG_SFX`}}" image_family: '{{build_name}}-base' - image_description: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' + image_labels: + <<: *imgcpylabels + src: 'fedoraproject' + release: 'fedora-{{user `PRIOR_FEDORA_RELEASE`}}' # This is critical, especially for the aws builders. # Producing the cache-images from these base images # needs to lookup the runtime-produced AMI ID. diff --git a/cache_images/cloud.yml b/cache_images/cloud.yml index f51c9189..e8dc54bf 100644 --- a/cache_images/cloud.yml +++ b/cache_images/cloud.yml @@ -16,6 +16,11 @@ variables: # Empty value means it must be passed in on command-line # Allows providing handy cross-reference to the build log CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}" + # Makefile sets these from values in base_images/cloud.yml + UBUNTU_RELEASE: + FEDORA_RELEASE: + PRIOR_FEDORA_RELEASE: + builders: - &gce_hosted_image name: 'ubuntu' @@ -25,14 +30,16 @@ builders: # Prefix IMG_SFX with "c" so this is never confused with a base_image name image_name: '{{ build_name }}-c{{user `IMG_SFX`}}' image_family: '{{ build_name }}-cache' + image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}' project_id: '{{user `GCP_PROJECT_ID`}}' account_file: '{{user `GAC_FILEPATH`}}' zone: 'us-central1-a' disk_size: 20 # REQUIRED: Runtime allocation > this value disable_default_service_account: true - labels: # For the VM + labels: &gce_labels # For the VM sfx: '{{user `IMG_SFX`}}' src: '{{ build_name }}-b{{user `IMG_SFX` }}' + release: 'ubuntu-{{user `UBUNTU_RELEASE` | clean_resource_name}}' stage: cache ssh_username: packer # arbitrary, packer will create & setup w/ temp. keypair ssh_pty: 'true' @@ -43,15 +50,20 @@ builders: - <<: *gce_hosted_image name: 'fedora' + labels: &fedora_gce_labels + <<: *gce_labels + release: 'fedora-{{user `FEDORA_RELEASE`}}' - <<: *gce_hosted_image name: 'prior-fedora' + labels: *fedora_gce_labels - &aux_fed_img <<: *gce_hosted_image name: 'build-push' source_image: 'fedora-b{{user `IMG_SFX`}}' source_image_family: 'fedora-base' + labels: *fedora_gce_labels - <<: *aux_fed_img name: 'fedora-podman-py' @@ -92,7 +104,7 @@ builders: security_group_id: "sg-042c75677872ef81c" # Prefix IMG_SFX with "b" so this is never confused with a cache_image ami_name: '{{build_name}}-c{{user `IMG_SFX`}}' - ami_description: 'Built in https://cirrus-ci.com/build/{{user `IMG_SFX`}}' + ami_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}' ebs_optimized: true launch_block_device_mappings: - device_name: '/dev/sda1' @@ -100,7 +112,7 @@ builders: volume_type: 'gp2' delete_on_termination: true # These are critical and used by security-polciy to enforce instance launch limits. - tags: &tags + tags: &ami_tags # EC2 expects "Name" tag to be capitalized Name: '{{build_name}}-c{{user `IMG_SFX`}}' sfx: '{{user `IMG_SFX`}}' @@ -108,9 +120,10 @@ builders: automation: 'true' stage: 'cache' arch: 'x86_64' - run_tags: *tags - run_volume_tags: *tags - snapshot_tags: *tags + release: 'fedora-{{user `FEDORA_RELEASE`}}' + run_tags: *ami_tags + run_volume_tags: *ami_tags + snapshot_tags: *ami_tags # Also required to make AMI private ami_users: - *accountid @@ -131,7 +144,7 @@ builders: name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}' instance_type: 't4g.medium' # arm64 type tags: &netavark_tags - <<: *tags + <<: *ami_tags Name: '{{build_name}}-c{{user `IMG_SFX`}}' arch: 'arm64' run_tags: *netavark_tags @@ -149,7 +162,7 @@ builders: name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}' instance_type: 't4g.medium' # arm64 type tags: &podman_tags - <<: *tags + <<: *ami_tags Name: '{{build_name}}-c{{user `IMG_SFX`}}' arch: 'arm64' run_tags: *podman_tags diff --git a/get_base_release.sh b/get_base_release.sh new file mode 100644 index 00000000..65bfb2c0 --- /dev/null +++ b/get_base_release.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# This script is intended to be called by the Makefile only. +# Any other use may produce unexpected results. It expects +# to be called with the name of a supported OS in all upper-case. +# The value of the corresponding _RELEASE variable will be +# extracted from base_images/cloud.json and printed to stdout. + +set -eo pipefail + +SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") +REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH") +CLOUD_JSON="$REPO_DIRPATH/base_images/cloud.json" + +# shellcheck source=./lib.sh +source "$REPO_DIRPATH/lib.sh" + +[[ -r "$CLOUD_JSON" ]] || die "Cannot read from '$CLOUD_JSON'" + +jq -r -e ".variables.${1}_RELEASE" $CLOUD_JSON diff --git a/image_builder/gce.yml b/image_builder/gce.yml index 63efdbef..113d4da8 100644 --- a/image_builder/gce.yml +++ b/image_builder/gce.yml @@ -17,6 +17,10 @@ variables: # Allows providing handy cross-reference to the build log CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}" + # These aren't used, but are required to be present. + UBUNTU_RELEASE: + FEDORA_RELEASE: + PRIOR_FEDORA_RELEASE: builders: - name: 'image-builder' @@ -25,6 +29,7 @@ builders: source_image_family: 'centos-stream-8' image_name: '{{build_name}}-{{user `IMG_SFX`}}' # output image name image_family: '{{build_name}}' # output family name for image + image_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}' project_id: '{{user `GCP_PROJECT_ID`}}' account_file: '{{user `GAC_FILEPATH`}}' zone: 'us-central1-a' diff --git a/podman/fedora_release b/podman/fedora_release deleted file mode 100644 index 7facc899..00000000 --- a/podman/fedora_release +++ /dev/null @@ -1 +0,0 @@ -36 diff --git a/podman/prior-fedora_release b/podman/prior-fedora_release deleted file mode 100644 index 8f92bfdd..00000000 --- a/podman/prior-fedora_release +++ /dev/null @@ -1 +0,0 @@ -35 diff --git a/podman/ubuntu_release b/podman/ubuntu_release deleted file mode 100644 index dcdf6284..00000000 --- a/podman/ubuntu_release +++ /dev/null @@ -1 +0,0 @@ -22.04