diff --git a/.cirrus.yml b/.cirrus.yml index 5766900a..0c616f31 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -32,6 +32,7 @@ validate_task: - "make clean" - "make help" - "make image_builder/gce.json" + - "make import_images/cloud.json" - "make base_images/cloud.json" - "make cache_images/cloud.json" @@ -49,7 +50,7 @@ image_builder_task: auto_cancellation: $CI != "true" stateful: true timeout_in: 40m - container: + container: &image_builder_container dockerfile: "image_builder/Containerfile" cpu: 2 memory: "2G" @@ -59,8 +60,7 @@ image_builder_task: PACKER_BUILDS: 'image-builder' # Google Application Credentials (JSON) with access to create VM images GAC_JSON: ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9] - - script: "ci/make_image_builder.sh" + script: "ci/make.sh image_builder" manifest_artifacts: path: image_builder/manifest.json type: application/json @@ -132,14 +132,43 @@ tooling_images_task: fingerprint_key: "tooling-cache-version-1" +import_images_task: + name: "Build VM import-images" + alias: "import_images" + only_if: *is_pr + skip: *ci_docs_tooling + depends_on: + - container_images + - image_builder + # Packer needs time to clean up partially created VM images + auto_cancellation: $CI != "true" + stateful: true + timeout_in: 30m + container: *image_builder_container + matrix: + - &import_image + name: "${PACKER_BUILDS} Import Image" + env: + PACKER_BUILDS: "fedora-aws" + - <<: *import_image + env: + PACKER_BUILDS: "fedora-aws-arm64" + env: + AWS_INI: &awsini ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930] + GAC_JSON: &gacjson ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9] + script: "ci/make.sh import_images" + manifest_artifacts: &manifest_artifacts + path: import_images/manifest.json + type: application/json + + base_images_task: name: "Build VM Base-images" alias: "base_images" only_if: *is_pr skip: *ci_docs_tooling depends_on: - - container_images - - image_builder + - import_images # Packer needs time to clean up partially created VM images auto_cancellation: $CI != "true" stateful: true @@ -169,12 +198,10 @@ base_images_task: env: PACKER_BUILDS: "ubuntu" env: - GAC_JSON: ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9] - AWS_INI: ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930] - script: "ci/make_base_images.sh" - manifest_artifacts: - path: base_images/manifest.json - type: application/json + GAC_JSON: *gacjson + AWS_INI: *awsini + script: "ci/make.sh base_images" + manifest_artifacts: *manifest_artifacts cache_images_task: @@ -227,10 +254,8 @@ cache_images_task: env: GAC_JSON: ENCRYPTED[7fba7fb26ab568ae39f799ab58a476123206576b0135b3d1019117c6d682391370c801e149f29324ff4b50133012aed9] AWS_INI: ENCRYPTED[4cd69097cd29a9899e51acf3bbacceeb83cb5c907d272ca1e2a8ccd515b03f2368a0680870c0d120fc32bc578bb0a930] - script: "ci/make_cache_images.sh" - manifest_artifacts: - path: cache_images/manifest.json - type: application/json + script: "ci/make.sh cache_images" + manifest_artifacts: *manifest_artifacts # Test metadata addition to images (built or not) to ensure container functions @@ -458,6 +483,7 @@ success_task: - image_builder - container_images - tooling_images + - import_images - base_images - cache_images - test_imgts diff --git a/Makefile b/Makefile index 3ea9a2c6..d2955265 100644 --- a/Makefile +++ b/Makefile @@ -214,15 +214,21 @@ image_builder_debug: $(_TEMPDIR)/image_builder_debug.tar ## Build and enter cont $(_TEMPDIR)/image_builder_debug.tar: $(_TEMPDIR)/.cache/centos $(wildcard image_builder/*) $(call podman_build,$@,image_builder_debug,image_builder,centos) +.PHONY: import_images +import_images: import_images/manifest.json ## Import generic Fedora cloud images into AWS EC2. + +import_images/manifest.json: import_images/cloud.json + $(call packer_build,import_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE)) + .PHONY: base_images # This needs to run in a virt/nested-virt capable environment -base_images: base_images/manifest.json ## Create, prepare, and import base-level images into GCE. Optionally, set PACKER_BUILDS= to select builder(s). +base_images: base_images/manifest.json ## Create, prepare, and import base-level images into GCE. base_images/manifest.json: base_images/cloud.json $(wildcard base_images/*.sh) cidata $(_TEMPDIR)/cidata.ssh $(PACKER_INSTALL_DIR)/packer $(call packer_build,base_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE)) .PHONY: cache_images -cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE. Optionally, set PACKER_BUILDS= to select builder(s). +cache_images: cache_images/manifest.json ## Create, prepare, and import top-level images into GCE. cache_images/manifest.json: base_images/cloud.json cache_images/cloud.json $(wildcard cache_images/*.sh) $(PACKER_INSTALL_DIR)/packer $(call packer_build,cache_images/cloud.json,$(call err_if_empty,AWS_SHARED_CREDENTIALS_FILE)) @@ -313,5 +319,5 @@ $(_TEMPDIR)/get_ci_vm.tar: lib.sh get_ci_vm/Containerfile get_ci_vm/entrypoint.s clean: ## Remove all generated files referenced in this Makefile -rm -rf $(_TEMPDIR) -rm -f image_builder/*.json - -rm -f base_images/{*.json,cidata*,*-data} + -rm -f *_images/{*.json,cidata*,*-data} -rm -f ci_debug.tar diff --git a/base_images/cloud.yml b/base_images/cloud.yml index aa2b42bc..9fe758b0 100644 --- a/base_images/cloud.yml +++ b/base_images/cloud.yml @@ -27,21 +27,16 @@ variables: # Empty value means it must be passed in on command-line # Use the most recent image based on this family name. UBUNTU_BASE_FAMILY: 'ubuntu-2204-lts' - # AWS base image in 'US East (N. Virginia)' region - # from https://alt.fedoraproject.org/cloud - FEDORAPROJECT_AMI: "ami-08b7bda26f4071b80" - FEDORAPROJECT_AARCH64_AMI: "ami-01925eb0821988986" - # Latest Fedora release (qcow and CHECKSUM) download URLs # N/B: There are Fedora-Cloud...GCP.tar.gz images available, however # as of this comment, they lack the cloud-init package which is # required by GCP for startup resizing of the rootfs. - FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-36-1.5-x86_64-CHECKSUM" - FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-Base-36-1.5.x86_64.qcow2" + FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/development/37/Cloud/x86_64/images/Fedora-Cloud-37-x86_64-20220824.n.2-CHECKSUM" + FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/development/37/Cloud/x86_64/images/Fedora-Cloud-Base-37-20220824.n.2.x86_64.qcow2" # Prior Fedora release - PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-35-1.2-x86_64-CHECKSUM" - PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-Base-35-1.2.x86_64.qcow2" + PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-36-1.5-x86_64-CHECKSUM" + PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/36/Cloud/x86_64/images/Fedora-Cloud-Base-36-1.5.x86_64.qcow2" builders: @@ -120,8 +115,24 @@ builders: - &fedora-aws name: 'fedora-aws' type: 'amazon-ebs' - source_ami: '{{user `FEDORAPROJECT_AMI`}}' instance_type: 'm5zn.metal' + source_ami_filter: # Will fail if >1 or no AMI found + owners: + # Docs are wrong, specifying the Account ID required to make AMIs private. + # The Account ID is hard-coded here out of expediency, since passing in + # more packer args from the command-line (in Makefile) is non-trivial. + - &accountid '449134212816' + # It's necessary to 'search' for the base-image by these criteria. If + # more than one image is found, Packer will fail the build (and display + # the conflicting AMI IDs). + filters: &ami_filters + architecture: 'x86_64' + image-type: 'machine' + is-public: 'false' + name: '{{build_name}}-i{{user `IMG_SFX`}}' + root-device-type: 'ebs' + state: 'available' + virtualization-type: 'hvm' # In case of packer problem or ungraceful exit, don't wait for shutdown. # This doesn't always work properly, sometimes leaving EC2 instances in # a 'stopped' instead of terminated state :( @@ -132,7 +143,7 @@ builders: # Required for network access, must be the 'default' group used by Cirrus-CI security_group_id: "sg-042c75677872ef81c" # Prefix IMG_SFX with "b" so this is never confused with a cache_image - ami_name: 'fedora-aws-b{{user `IMG_SFX`}}' + ami_name: &ami_name '{{build_name}}-b{{user `IMG_SFX`}}' ami_description: 'Built in https://cirrus-ci.com/task/{{user `CIRRUS_TASK_ID`}}' ebs_optimized: true launch_block_device_mappings: @@ -145,19 +156,17 @@ builders: tags: &awstags <<: *imgcpylabels # EC2 expects "Name" to be capitalized - Name: 'fedora-aws-b{{user `IMG_SFX`}}' + Name: *ami_name src: '{{user `FEDORAPROJECT_AMI`}}' automation: 'true' release: 'fedora-{{user `FEDORA_RELEASE`}}' run_tags: *awstags run_volume_tags: *awstags snapshot_tags: *awstags - # Docs are wrong, specifying the Account ID required to make AMIs private. # This is necessary for security - The CI service accounts are not permitted - # to use AMI's from any other account, including public ones. The Account - # ID is hard-coded here out of expediency, since passing in more packer args - # from the command-line (in Makefile) is non-trivial. - ami_users: ["449134212816"] + # to use AMI's from any other account, including public ones. + ami_users: + - *accountid ssh_username: 'fedora' ssh_clear_authorized_keys: true # N/B: Required Packer >= 1.8.0 @@ -166,9 +175,13 @@ builders: - <<: *fedora-aws name: 'fedora-aws-arm64' - source_ami: '{{user `FEDORAPROJECT_AARCH64_AMI`}}' + source_ami_filter: + owners: + - *accountid + filters: + <<: *ami_filters + architecture: 'arm64' instance_type: 't4g.medium' # arm64 type - ami_name: 'fedora-aws-arm64-b{{user `IMG_SFX`}}' # must be unique tags: &awsarm64tags <<: *awstags src: '{{user `FEDORAPROJECT_AARCH64_AMI`}}' @@ -225,6 +238,7 @@ post-processors: only: ['prior-fedora'] image_name: "prior-fedora-b{{user `IMG_SFX`}}" image_family: '{{build_name}}-base' + image_description: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' image_labels: <<: *imgcpylabels src: 'fedoraproject' diff --git a/ci/make_base_images.sh b/ci/make.sh similarity index 58% rename from ci/make_base_images.sh rename to ci/make.sh index 582c2c4e..65758567 100755 --- a/ci/make_base_images.sh +++ b/ci/make.sh @@ -1,9 +1,12 @@ #!/bin/bash +set -eo pipefail + # This script is intended to be used by Cirrus-CI, from the VM -# built by the 'image_builder' makefile target in this repo. Use -# of this script in any other context/environment is unlikely to -# function as intended. +# built by the 'image_builder' Makefile target in this repo. +# It's purpose is simply to verify & configure the runtime +# environment from data provided by CI, and call the make +# with the first argument passed to this script. SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") @@ -17,17 +20,25 @@ if [[ -z "$CI" ]] || [[ "$CI" != "true" ]] || [[ "$CIRRUS_CI" != "$CI" ]]; then die "Unexpected \$CI=$CI and/or \$CIRRUS_CI=$CIRRUS_CI" elif [[ -z "$IMG_SFX" ]] || [[ -z "$PACKER_BUILDS" ]]; then die "Required non-empty values for \$IMG_SFX=$IMG_SFX and \$PACKER_BUILDS=$PACKER_BUILDS" +elif [[ -z "$1" ]]; then + die "Build stage name is required as the first argument" fi if skip_on_pr_label; then exit 0 # skip build fi -set_gac_filepath -set_aws_filepath +# In case we're running under -x, don't expose secret values +if [[ "${#GAC_JSON}" -gt 0 ]]; then + set_gac_filepath +fi + +if [[ "${#AWS_INI}" -gt 0 ]]; then + set_aws_filepath +fi -set -exo pipefail +set -x cd "$REPO_DIRPATH" export IMG_SFX=$IMG_SFX export PACKER_BUILDS=$PACKER_BUILDS -make base_images +make ${1} diff --git a/ci/make_cache_images.sh b/ci/make_cache_images.sh deleted file mode 100755 index 45bbbc79..00000000 --- a/ci/make_cache_images.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -# This script is intended to be used by Cirrus-CI, from the container -# built by the ContainerFile in this directory. Use of this script -# in any other context/environment is unlikely to function as intended. - -SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") -SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") -REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") - -# shellcheck source=./lib.sh -source "$REPO_DIRPATH/lib.sh" - -# shellcheck disable=SC2154 -if [[ -z "$CI" ]] || [[ "$CI" != "true" ]] || [[ "$CIRRUS_CI" != "$CI" ]]; then - die "Unexpected \$CI=$CI and/or \$CIRRUS_CI=$CIRRUS_CI" -elif [[ -z "$IMG_SFX" ]] || [[ -z "$PACKER_BUILDS" ]]; then - die "Required non-empty values for \$IMG_SFX=$IMG_SFX and \$PACKER_BUILDS=$PACKER_BUILDS" -fi - -if skip_on_pr_label; then - exit 0 # skip build -fi - -set_gac_filepath -set_aws_filepath - -set -x -cd "$REPO_DIRPATH" -export IMG_SFX=$IMG_SFX -export PACKER_BUILDS=$PACKER_BUILDS -make cache_images diff --git a/ci/make_image_builder.sh b/ci/make_image_builder.sh deleted file mode 100755 index 4d3438c5..00000000 --- a/ci/make_image_builder.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# This script is intended to be used by Cirrus-CI, from the container -# built by the ContainerFile in this directory. Use of this script -# in any other context/environment is unlikely to function as intended. - -SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") -SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") -REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") - -# shellcheck source=./lib.sh -source "$REPO_DIRPATH/lib.sh" - -# shellcheck disable=SC2154 -if [[ -z "$CI" ]] || [[ "$CI" != "true" ]] || [[ "$CIRRUS_CI" != "$CI" ]]; then - die "Unexpected \$CI=$CI and/or \$CIRRUS_CI=$CIRRUS_CI" -elif [[ -z "$IMG_SFX" ]]; then - die "Required non-empty values for \$IMG_SFX=$IMG_SFX" -fi - -if skip_on_pr_label; then - exit 0 # skip build -fi - -set_gac_filepath - -set -exo pipefail -cd "$REPO_DIRPATH" -export IMG_SFX=$IMG_SFX -make image_builder diff --git a/import_images/cloud.yml b/import_images/cloud.yml new file mode 100644 index 00000000..2e53da50 --- /dev/null +++ b/import_images/cloud.yml @@ -0,0 +1,86 @@ +--- + +# This Packer template is used to import images into AWS EC2. Since +# this must happen in a `post-processor`, it must occur before the +# regular base_images/cloud.yml template is used. + +variables: # Empty value means it must be passed in on command-line + # Required for 'make clean' support and not clobbering a memory-backed /tmp + TEMPDIR: + # Naming suffix for images to prevent clashes + IMG_SFX: + + # Allows providing handy cross-reference to the build log + CIRRUS_TASK_ID: "{{env `CIRRUS_TASK_ID`}}" + + # BIG-FAT-WARNING: When updating the image names and/or URLs below, + # ensure the distro version numbers contained in the `podman/*_release` + # files exactly match. These represent the container base-image tags + # to build from - just as the sources below are the base-images to + # start from building VM images. + + # Latest Fedora release (qcow2 format) download URLs + FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/development/37/Cloud/x86_64/images/Fedora-Cloud-Base-37-20220824.n.2.x86_64.qcow2" + FEDORA_ARM64_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/development/37/Cloud/aarch64/images/Fedora-Cloud-Base-37-20220823.n.0.aarch64.qcow2" + +builders: + - name: 'fedora-aws' + type: null + + - name: 'fedora-aws-arm64' + type: null + +provisioners: + - only: ['fedora-aws'] + type: "shell-local" + inline: + - 'set -e' + - 'curl --location "{{user `FEDORA_IMAGE_URL`}}" -o {{user `TEMPDIR`}}/{{build_name}}.qcow2' + + - only: ['fedora-aws-arm64'] + type: "shell-local" + inline: + - 'set -e' + - 'curl --location "{{user `FEDORA_ARM64_IMAGE_URL`}}" -o {{user `TEMPDIR`}}/{{build_name}}.qcow2' + + - type: "shell-local" + inline: + - 'set -e' + - 'qemu-img convert {{user `TEMPDIR`}}/{{build_name}}.qcow2 -O raw {{user `TEMPDIR`}}/{{build_name}}.raw' + +post-processors: + # Must be double-nested to guarantee execution order + - - only: ['fedora-aws', 'fedora-aws-arm64'] + type: 'artifice' + keep_input_artifact: false + files: ['{{user `TEMPDIR`}}/{{build_name}}.raw'] + - &aws + only: ['fedora-aws'] + type: 'amazon-import' + region: 'us-east-1' + s3_bucket_name: packer-image-import + ami_description: 'Imported by https://cirrus-ci.com/build/{{user `IMG_SFX`}}' + ami_users: ["449134212816"] + ami_name: &ami_name "{{build_name}}-i{{user `IMG_SFX`}}" + boot_mode: "uefi" + format: "raw" + keep_input_artifact: false + tags: &aws_tags + Name: *ami_name + sfx: '{{user `IMG_SFX`}}' + automation: 'true' + stage: 'import' + arch: 'x86_64' + - <<: *aws + only: ['fedora-aws-arm64'] + architecture: 'arm64' + tags: + <<: *aws_tags + arch: 'arm64' + - type: 'manifest' + output: 'import_images/manifest.json' # Collected by Cirrus-CI + strip_path: true + custom_data: + IMG_SFX: '{{ user `IMG_SFX` }}' + STAGE: import + TASK: '{{user `CIRRUS_TASK_ID`}}'