From 0dcadb61290b93f5146ddeb4f0e0051751630f93 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 19 Jul 2022 10:39:30 -0400 Subject: [PATCH 1/3] Rework CCIA container to not depend on volume-mounts This is required for use in a github action. In that environment there is no possibility to create or pass volumes into containers. Update tests and documentation accordingly. Signed-off-by: Chris Evich --- ccia/Containerfile | 2 -- ccia/README.md | 19 ++++++++----------- ccia/test.sh | 6 +++--- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/ccia/Containerfile b/ccia/Containerfile index a39ee2db..77573dda 100644 --- a/ccia/Containerfile +++ b/ccia/Containerfile @@ -29,5 +29,3 @@ COPY --from=builder /usr/share/automation /usr/share/automation COPY --from=builder /etc/automation_environment /etc/automation_environment ENTRYPOINT ["/usr/share/automation/bin/cirrus-ci_artifacts"] -VOLUME ["/data"] -WORKDIR /data diff --git a/ccia/README.md b/ccia/README.md index dd50d767..c85747df 100644 --- a/ccia/README.md +++ b/ccia/README.md @@ -13,21 +13,18 @@ previously cached. For example: ## Usage -It is recommended that you first create a volume to store any downloaded -artifacts. +It is recommended that you run the container with a `--workdir` set, and +a volume mounted at that location. In this way, any downloaded +artifacts will be accessable after the container exits -`podman volume create ccia` - -Knowing a Cirrus-CI build ID, you can download all artifacts (or select +Knowing a Cirrus-CI build ID, the container will download all artifacts (or select [a subset using a regex](https://github.com/containers/automation/tree/main/cirrus-ci_artifacts#usage). -The artifacts subdirectory will be written into the `/data` volume, so -be sure it's mounted. The `--verbose` option is shown for illustrative -purposes, it's not required. +The `--verbose` option shown below is not required. ``` BID= -podman run -it --rm -v ccia:/data ccia $BID --verbose -DATA="$(podman volume inspect ccia | jq -r '.[].Mountpoint')/$BID" -ls -laR $DATA +mkdir -p /tmp/artifacts +podman run -it --rm -v /tmp/artifacts:/data -w /data ccia $BID --verbose +ls -laR /tmp/artifacts ``` diff --git a/ccia/test.sh b/ccia/test.sh index 2de0b8d8..2635c163 100755 --- a/ccia/test.sh +++ b/ccia/test.sh @@ -17,7 +17,7 @@ ooe.sh microdnf install -y coreutils jq echo "Confirming current build task manifests can be downloaded." ( set -x - cd /data + cd /tmp/ # shellcheck disable=SC2154 $CCIABIN --verbose $CIRRUS_BUILD_ID '.*/manifest.json' ) @@ -25,8 +25,8 @@ echo "Confirming current build task manifests can be downloaded." echo "Confirming any downloaded manifests can be parsed into a build list" ( set -x - cd /data - find ./ -type f -name 'manifest.json' -print0 | \ + cd /tmp + find ./$CIRRUS_BUILD_ID -type f -name 'manifest.json' -print0 | \ xargs --null jq -e '.builds[]' | \ jq -e -s '.' | \ jq -e '{"builds": .}' From f09310bde26529851b4b04c70e04cacb5345dff7 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 19 Jul 2022 10:35:51 -0400 Subject: [PATCH 2/3] Fix ccia test when no images are built For a `[CI:DOCS]` PR, no VM images will be produced. This will cause the ccia container test script to fail. Fix this by providing some fall-back dummy data for the tests to run against. Signed-off-by: Chris Evich --- .../manifest/image_builder/manifest.json | 16 ++++++++++++++ .../manifest/cache_images/manifest.json | 17 ++++++++++++++ .../manifest/base_images/manifest.json | 22 +++++++++++++++++++ .../manifest/cache_images/manifest.json | 17 ++++++++++++++ .../manifest/base_images/manifest.json | 17 ++++++++++++++ .../manifest/cache_images/manifest.json | 17 ++++++++++++++ .../manifest/base_images/manifest.json | 17 ++++++++++++++ .../manifest/cache_images/manifest.json | 17 ++++++++++++++ .../manifest/cache_images/manifest.json | 17 ++++++++++++++ .../manifest/cache_images/manifest.json | 17 ++++++++++++++ .../manifest/cache_images/manifest.json | 17 ++++++++++++++ .../manifest/base_images/manifest.json | 17 ++++++++++++++ .../manifest/cache_images/manifest.json | 17 ++++++++++++++ ccia/test.sh | 9 +++++++- 14 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 ccia/fake_manifests/Image-builder image/manifest/image_builder/manifest.json create mode 100644 ccia/fake_manifests/build-push Cache Image/manifest/cache_images/manifest.json create mode 100644 ccia/fake_manifests/fedora Base Image/manifest/base_images/manifest.json create mode 100644 ccia/fake_manifests/fedora Cache Image/manifest/cache_images/manifest.json create mode 100644 ccia/fake_manifests/fedora-aws Base Image/manifest/base_images/manifest.json create mode 100644 ccia/fake_manifests/fedora-aws Cache Image/manifest/cache_images/manifest.json create mode 100644 ccia/fake_manifests/fedora-aws-arm64 Base Image/manifest/base_images/manifest.json create mode 100644 ccia/fake_manifests/fedora-netavark Cache Image/manifest/cache_images/manifest.json create mode 100644 ccia/fake_manifests/fedora-netavark-aws-arm64 Cache Image/manifest/cache_images/manifest.json create mode 100644 ccia/fake_manifests/fedora-podman-aws-arm64 Cache Image/manifest/cache_images/manifest.json create mode 100644 ccia/fake_manifests/fedora-podman-py Cache Image/manifest/cache_images/manifest.json create mode 100644 ccia/fake_manifests/ubuntu Base Image/manifest/base_images/manifest.json create mode 100644 ccia/fake_manifests/ubuntu Cache Image/manifest/cache_images/manifest.json diff --git a/ccia/fake_manifests/Image-builder image/manifest/image_builder/manifest.json b/ccia/fake_manifests/Image-builder image/manifest/image_builder/manifest.json new file mode 100644 index 00000000..00d261f3 --- /dev/null +++ b/ccia/fake_manifests/Image-builder image/manifest/image_builder/manifest.json @@ -0,0 +1,16 @@ +{ + "builds": [ + { + "name": "image-builder", + "builder_type": "googlecompute", + "build_time": 1658173915, + "files": null, + "artifact_id": "image-builder-5419329914142720", + "packer_run_uuid": "243ae2b1-d4b4-4917-9883-a96c516a2c39", + "custom_data": { + "IMG_SFX": "5419329914142720" + } + } + ], + "last_run_uuid": "243ae2b1-d4b4-4917-9883-a96c516a2c39" +} diff --git a/ccia/fake_manifests/build-push Cache Image/manifest/cache_images/manifest.json b/ccia/fake_manifests/build-push Cache Image/manifest/cache_images/manifest.json new file mode 100644 index 00000000..4b81ba92 --- /dev/null +++ b/ccia/fake_manifests/build-push Cache Image/manifest/cache_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "build-push", + "builder_type": "googlecompute", + "build_time": 1658175996, + "files": null, + "artifact_id": "build-push-c5419329914142720", + "packer_run_uuid": "250b8705-ce4d-7844-7181-f1181dd7e04c", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "cache" + } + } + ], + "last_run_uuid": "250b8705-ce4d-7844-7181-f1181dd7e04c" +} diff --git a/ccia/fake_manifests/fedora Base Image/manifest/base_images/manifest.json b/ccia/fake_manifests/fedora Base Image/manifest/base_images/manifest.json new file mode 100644 index 00000000..34ff65bc --- /dev/null +++ b/ccia/fake_manifests/fedora Base Image/manifest/base_images/manifest.json @@ -0,0 +1,22 @@ +{ + "builds": [ + { + "name": "fedora", + "builder_type": "qemu", + "build_time": 1658175535, + "files": [ + { + "name": "fedora-b5419329914142720", + "size": 0 + } + ], + "artifact_id": "", + "packer_run_uuid": "b1ec41bd-c395-45d6-96cf-f03a1ff2c894", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "base" + } + } + ], + "last_run_uuid": "b1ec41bd-c395-45d6-96cf-f03a1ff2c894" +} diff --git a/ccia/fake_manifests/fedora Cache Image/manifest/cache_images/manifest.json b/ccia/fake_manifests/fedora Cache Image/manifest/cache_images/manifest.json new file mode 100644 index 00000000..fe087181 --- /dev/null +++ b/ccia/fake_manifests/fedora Cache Image/manifest/cache_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "fedora", + "builder_type": "googlecompute", + "build_time": 1658176163, + "files": null, + "artifact_id": "fedora-c5419329914142720", + "packer_run_uuid": "30833ff6-05df-ee1e-4378-57991d592136", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "cache" + } + } + ], + "last_run_uuid": "30833ff6-05df-ee1e-4378-57991d592136" +} diff --git a/ccia/fake_manifests/fedora-aws Base Image/manifest/base_images/manifest.json b/ccia/fake_manifests/fedora-aws Base Image/manifest/base_images/manifest.json new file mode 100644 index 00000000..e8d787fa --- /dev/null +++ b/ccia/fake_manifests/fedora-aws Base Image/manifest/base_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "fedora-aws", + "builder_type": "amazon-ebs", + "build_time": 1658175765, + "files": null, + "artifact_id": "us-east-1:ami-000448bd70242ba3c", + "packer_run_uuid": "193dbe11-9c6b-e0b0-efc9-dc0e8fbf98dc", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "base" + } + } + ], + "last_run_uuid": "193dbe11-9c6b-e0b0-efc9-dc0e8fbf98dc" +} diff --git a/ccia/fake_manifests/fedora-aws Cache Image/manifest/cache_images/manifest.json b/ccia/fake_manifests/fedora-aws Cache Image/manifest/cache_images/manifest.json new file mode 100644 index 00000000..bb48b06a --- /dev/null +++ b/ccia/fake_manifests/fedora-aws Cache Image/manifest/cache_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "fedora-aws", + "builder_type": "amazon-ebs", + "build_time": 1658176592, + "files": null, + "artifact_id": "us-east-1:ami-0442ccd2bb66504b7", + "packer_run_uuid": "df4c911b-80a3-27ee-a513-4b6e29c1c906", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "cache" + } + } + ], + "last_run_uuid": "df4c911b-80a3-27ee-a513-4b6e29c1c906" +} diff --git a/ccia/fake_manifests/fedora-aws-arm64 Base Image/manifest/base_images/manifest.json b/ccia/fake_manifests/fedora-aws-arm64 Base Image/manifest/base_images/manifest.json new file mode 100644 index 00000000..18b97b24 --- /dev/null +++ b/ccia/fake_manifests/fedora-aws-arm64 Base Image/manifest/base_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "fedora-aws-arm64", + "builder_type": "amazon-ebs", + "build_time": 1658175464, + "files": null, + "artifact_id": "us-east-1:ami-0f5f268182775a8c2", + "packer_run_uuid": "e4a389da-e1dc-35db-ef32-361e890e4b30", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "base" + } + } + ], + "last_run_uuid": "e4a389da-e1dc-35db-ef32-361e890e4b30" +} diff --git a/ccia/fake_manifests/fedora-netavark Cache Image/manifest/cache_images/manifest.json b/ccia/fake_manifests/fedora-netavark Cache Image/manifest/cache_images/manifest.json new file mode 100644 index 00000000..cae58d58 --- /dev/null +++ b/ccia/fake_manifests/fedora-netavark Cache Image/manifest/cache_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "fedora-netavark", + "builder_type": "googlecompute", + "build_time": 1658176148, + "files": null, + "artifact_id": "fedora-netavark-c5419329914142720", + "packer_run_uuid": "d95c8118-3970-4a73-d348-692a5a3371a3", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "cache" + } + } + ], + "last_run_uuid": "d95c8118-3970-4a73-d348-692a5a3371a3" +} diff --git a/ccia/fake_manifests/fedora-netavark-aws-arm64 Cache Image/manifest/cache_images/manifest.json b/ccia/fake_manifests/fedora-netavark-aws-arm64 Cache Image/manifest/cache_images/manifest.json new file mode 100644 index 00000000..3949434f --- /dev/null +++ b/ccia/fake_manifests/fedora-netavark-aws-arm64 Cache Image/manifest/cache_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "fedora-netavark-aws-arm64", + "builder_type": "amazon-ebs", + "build_time": 1658176335, + "files": null, + "artifact_id": "us-east-1:ami-07a339e76f84afa7b", + "packer_run_uuid": "b8a09332-800a-09c2-ba0e-2564e6e52f76", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "cache" + } + } + ], + "last_run_uuid": "b8a09332-800a-09c2-ba0e-2564e6e52f76" +} diff --git a/ccia/fake_manifests/fedora-podman-aws-arm64 Cache Image/manifest/cache_images/manifest.json b/ccia/fake_manifests/fedora-podman-aws-arm64 Cache Image/manifest/cache_images/manifest.json new file mode 100644 index 00000000..e2e1dcb9 --- /dev/null +++ b/ccia/fake_manifests/fedora-podman-aws-arm64 Cache Image/manifest/cache_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "fedora-podman-aws-arm64", + "builder_type": "amazon-ebs", + "build_time": 1658176346, + "files": null, + "artifact_id": "us-east-1:ami-051a5e8dad587bf22", + "packer_run_uuid": "7e742dec-035c-6b95-8793-c464b2a6ac0f", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "cache" + } + } + ], + "last_run_uuid": "7e742dec-035c-6b95-8793-c464b2a6ac0f" +} diff --git a/ccia/fake_manifests/fedora-podman-py Cache Image/manifest/cache_images/manifest.json b/ccia/fake_manifests/fedora-podman-py Cache Image/manifest/cache_images/manifest.json new file mode 100644 index 00000000..b06e0d53 --- /dev/null +++ b/ccia/fake_manifests/fedora-podman-py Cache Image/manifest/cache_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "fedora-podman-py", + "builder_type": "googlecompute", + "build_time": 1658176090, + "files": null, + "artifact_id": "fedora-podman-py-c5419329914142720", + "packer_run_uuid": "e5b1e6ab-37a5-a695-624d-47bf0060b272", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "cache" + } + } + ], + "last_run_uuid": "e5b1e6ab-37a5-a695-624d-47bf0060b272" +} diff --git a/ccia/fake_manifests/ubuntu Base Image/manifest/base_images/manifest.json b/ccia/fake_manifests/ubuntu Base Image/manifest/base_images/manifest.json new file mode 100644 index 00000000..711a1ca2 --- /dev/null +++ b/ccia/fake_manifests/ubuntu Base Image/manifest/base_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "ubuntu", + "builder_type": "googlecompute", + "build_time": 1658175167, + "files": null, + "artifact_id": "ubuntu-b5419329914142720", + "packer_run_uuid": "238ce64e-cb7d-4c1b-38ff-3e0eb9e3939a", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "base" + } + } + ], + "last_run_uuid": "238ce64e-cb7d-4c1b-38ff-3e0eb9e3939a" +} diff --git a/ccia/fake_manifests/ubuntu Cache Image/manifest/cache_images/manifest.json b/ccia/fake_manifests/ubuntu Cache Image/manifest/cache_images/manifest.json new file mode 100644 index 00000000..9260fd8d --- /dev/null +++ b/ccia/fake_manifests/ubuntu Cache Image/manifest/cache_images/manifest.json @@ -0,0 +1,17 @@ +{ + "builds": [ + { + "name": "ubuntu", + "builder_type": "googlecompute", + "build_time": 1658176053, + "files": null, + "artifact_id": "ubuntu-c5419329914142720", + "packer_run_uuid": "553ec5c0-7b09-e06d-2837-b272204696d1", + "custom_data": { + "IMG_SFX": "5419329914142720", + "STAGE": "cache" + } + } + ], + "last_run_uuid": "553ec5c0-7b09-e06d-2837-b272204696d1" +} diff --git a/ccia/test.sh b/ccia/test.sh index 2635c163..fd480a0b 100755 --- a/ccia/test.sh +++ b/ccia/test.sh @@ -14,14 +14,21 @@ req_env_vars CIRRUS_CI CIRRUS_BUILD_ID CIRRUS_WORKING_DIR echo "Installing test tooling" ooe.sh microdnf install -y coreutils jq +cd /tmp/ + echo "Confirming current build task manifests can be downloaded." ( set -x - cd /tmp/ # shellcheck disable=SC2154 $CCIABIN --verbose $CIRRUS_BUILD_ID '.*/manifest.json' ) +# It's possible the PR did not produce any manifest.json files +if ! dled=$(find ./$CIRRUS_BUILD_ID -name manifest.json | wc -l) || ((dled==0)); then + mkdir -p ./$CIRRUS_BUILD_ID + cp -a $SCRIPT_DIRPATH/fake_manifests/* ./$CIRRUS_BUILD_ID +fi + echo "Confirming any downloaded manifests can be parsed into a build list" ( set -x From db774205a15787c553ea4bb274e6597738cf09a6 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Mon, 18 Jul 2022 12:18:30 -0400 Subject: [PATCH 3/3] Print names & IDs of all built images This is intended as a help to developers updating images in other repo's CI systems. Esp. WRT AWS AMI ID's being utterly unreadable by humans. Use a bit of `jq` and markdown magic to produce a nice looking table of image names and IDs. Signed-off-by: Chris Evich --- .github/workflows/pr_image_id.yml | 86 +++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pr_image_id.yml b/.github/workflows/pr_image_id.yml index e0a09338..b93a5895 100644 --- a/.github/workflows/pr_image_id.yml +++ b/.github/workflows/pr_image_id.yml @@ -53,13 +53,89 @@ jobs: printf "\n::set-output name=is_pr::%s\n" "false" fi - - name: Add image id comment to pull request - if: steps.retro.outputs.is_pr == 'true' + - if: steps.retro.outputs.is_pr == 'true' + name: Retrieve and process any manifest artifacts + # Use the CCIA image produce by the `Build Tooling images` + # task of the PR we're looking at. This allows testing + # of changes to the CCIA container before merging into `main` + # (where this workflow runs from). + run: | + podman run --rm -v $PWD:/data -w /data \ + quay.io/libpod/ccia:${{ steps.retro.outputs.bid }} \ + --verbose "${{ steps.retro.outputs.bid }}" ".*/manifest.json" + + - if: steps.retro.outputs.is_pr == 'true' + name: Count the number of manifest.json files downloaded + id: manifests + run: | + dled=$(find ./${{ steps.retro.outputs.bid }} -name manifest.json | wc -l) + printf "\n::set-output name=count::%s\n" "$dled" + + - if: steps.manifests.outputs.count > 0 + name: Extract packer builder names and artifact IDs + env: + FLTR: >- + {"name": .builds[].name, + "id": .builds[].artifact_id | ltrimstr("us-east-1:"), + "stage": .builds[].custom_data.STAGE} | + select(.stage == "cache") + run: | + find ./${{ steps.retro.outputs.bid }} \ + -type f -name 'manifest.json' -print0 | \ + xargs --null jq -e -c "$FLTR" | + jq -e -s '.' > built_cache_images.json + + - if: steps.manifests.outputs.count > 0 + name: Debug built_cache_images.json contents + run: | + jq --color-output . built_cache_images.json + + - if: steps.manifests.outputs.count > 0 + id: body + name: Format PR-comment body + shell: python + # Setting multi-line values of step outputs is problematic, + # setting a future (global) env. var. value is the only + # reasonable option. This tricky bit of python sets the + # global $BUILT_CACHE_IMAGES env. var for subsequent workflow + # steps via the file referenced in $GITHUB_ENV. The + # $BUILT_CACHE_IMAGES contents are the markdown to post as + # a PR comment. + run: | + import sys, os, json + l = [] + with open("built_cache_images.json") as f: + for b in json.load(f): # list of builds + if b.get("id") is not None: + l.append(f'|**{b["name"]}**|`{b["id"]}`|\n') + l.sort(key=str.lower) + with open(os.environ["GITHUB_ENV"], "a") as e: + e.write(("BUILT_CACHE_IMAGES< 0 + name: Debug PR comment markdown + # Use a here-document to display to avoid any + # problems with passing special-characters into echo + # The quoted-EOD prevents any shell interpretation. + run: | + cat <<"EOD" + ${{ env.BUILT_CACHE_IMAGES }} + EOD + + - if: steps.manifests.outputs.count > 0 + name: Post PR comment with image name/id table uses: jungwinter/comment@v1 with: issue_number: '${{ steps.retro.outputs.prn }}' type: 'create' token: '${{ secrets.GITHUB_TOKEN }}' - body: >- - [Cirrus CI build](https://cirrus-ci.com/build/${{ steps.retro.outputs.bid }}) - successful. Image ID `c${{ steps.retro.outputs.bid }}` ready for use. + body: | + ${{ env.BUILT_CACHE_IMAGES }}