Merge pull request #153 from cevich/fix_image_id_bot

[CI:TOOLING] Print names & IDs of all built images

.github/workflows/pr_image_id.yml

@@ -53,13 +53,89 @@ jobs:
                       printf "\n::set-output name=is_pr::%s\n" "false"
                   fi
 
-            - name: Add image id comment to pull request
-              if: steps.retro.outputs.is_pr == 'true'
+            - if: steps.retro.outputs.is_pr == 'true'
+              name: Retrieve and process any manifest artifacts
+              # Use the CCIA image produce by the `Build Tooling images`
+              # task of the PR we're looking at.  This allows testing
+              # of changes to the CCIA container before merging into `main`
+              # (where this workflow runs from).
+              run: |
+                podman run --rm -v $PWD:/data -w /data \
+                    quay.io/libpod/ccia:${{ steps.retro.outputs.bid }} \
+                    --verbose "${{ steps.retro.outputs.bid }}" ".*/manifest.json"
+
+            - if: steps.retro.outputs.is_pr == 'true'
+              name: Count the number of manifest.json files downloaded
+              id: manifests
+              run: |
+                dled=$(find ./${{ steps.retro.outputs.bid }} -name manifest.json | wc -l)
+                printf "\n::set-output name=count::%s\n" "$dled"
+
+            - if: steps.manifests.outputs.count > 0
+              name: Extract packer builder names and artifact IDs
+              env:
+                FLTR: >-
+                    {"name": .builds[].name,
+                     "id": .builds[].artifact_id | ltrimstr("us-east-1:"),
+                     "stage": .builds[].custom_data.STAGE} |
+                    select(.stage == "cache")
+              run: |
+                find ./${{ steps.retro.outputs.bid }} \
+                    -type f -name 'manifest.json' -print0 | \
+                xargs --null jq -e -c "$FLTR" |
+                    jq -e -s '.' > built_cache_images.json
+
+            - if: steps.manifests.outputs.count > 0
+              name: Debug built_cache_images.json contents
+              run: |
+                jq --color-output . built_cache_images.json
+
+            - if: steps.manifests.outputs.count > 0
+              id: body
+              name: Format PR-comment body
+              shell: python
+              # Setting multi-line values of step outputs is problematic,
+              # setting a future (global) env. var. value is the only
+              # reasonable option.  This tricky bit of python sets the
+              # global $BUILT_CACHE_IMAGES env. var for subsequent workflow
+              # steps via the file referenced in $GITHUB_ENV.  The
+              # $BUILT_CACHE_IMAGES contents are the markdown to post as
+              # a PR comment.
+              run: |
+                import sys, os, json
+                l = []
+                with open("built_cache_images.json") as f:
+                  for b in json.load(f):  # list of builds
+                    if b.get("id") is not None:
+                      l.append(f'|**{b["name"]}**|`{b["id"]}`|\n')
+                l.sort(key=str.lower)
+                with open(os.environ["GITHUB_ENV"], "a") as e:
+                  e.write(("BUILT_CACHE_IMAGES<<EOF\n"
+                             "[Cirrus CI build](https://cirrus-ci.com/build/${{ steps.retro.outputs.bid }})"
+                             " successful. [Found built image names and"
+                             " IDs](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}):\n"
+                             "\n"
+                             "|Name|ID|\n"
+                             "|---|---|\n"))
+                  e.writelines(l)
+                  e.write("EOF\n")
+
+            - if: steps.manifests.outputs.count > 0
+              name: Debug PR comment markdown
+              # Use a here-document to display to avoid any
+              # problems with passing special-characters into echo
+              # The quoted-EOD prevents any shell interpretation.
+              run: |
+                cat <<"EOD"
+                ${{ env.BUILT_CACHE_IMAGES }}
+                EOD
+
+            - if: steps.manifests.outputs.count > 0
+              name: Post PR comment with image name/id table
               uses: jungwinter/comment@v1
               with:
                   issue_number: '${{ steps.retro.outputs.prn }}'
                   type: 'create'
                   token: '${{ secrets.GITHUB_TOKEN }}'
-                  body: >-
-                    [Cirrus CI build](https://cirrus-ci.com/build/${{ steps.retro.outputs.bid }})
-                    successful. Image ID `c${{ steps.retro.outputs.bid }}` ready for use.
+                  body: |
+                    ${{ env.BUILT_CACHE_IMAGES }}

ccia/Containerfile

@@ -29,5 +29,3 @@ COPY --from=builder /usr/share/automation /usr/share/automation
 COPY --from=builder /etc/automation_environment /etc/automation_environment
 
 ENTRYPOINT ["/usr/share/automation/bin/cirrus-ci_artifacts"]
-VOLUME ["/data"]
-WORKDIR /data

ccia/README.md

@@ -13,21 +13,18 @@ previously cached.  For example:
 
 ## Usage
 
-It is recommended that you first create a volume to store any downloaded
-artifacts.
+It is recommended that you run the container with a `--workdir` set, and
+a volume mounted at that location.  In this way, any downloaded
+artifacts will be accessable after the container exits
 
-`podman volume create ccia`
-
-Knowing a Cirrus-CI build ID, you can download all artifacts (or select
+Knowing a Cirrus-CI build ID, the container will download all artifacts (or select
 [a subset using a
 regex](https://github.com/containers/automation/tree/main/cirrus-ci_artifacts#usage).
-The artifacts subdirectory will be written into the `/data` volume, so
-be sure it's mounted.  The `--verbose` option is shown for illustrative
-purposes, it's not required.
+The `--verbose` option shown below is not required.
 
 ```
 BID=<Cirrus Build ID>
-podman run -it --rm -v ccia:/data ccia $BID --verbose
-DATA="$(podman volume inspect ccia | jq -r '.[].Mountpoint')/$BID"
-ls -laR $DATA
+mkdir -p /tmp/artifacts
+podman run -it --rm -v /tmp/artifacts:/data -w /data ccia $BID --verbose
+ls -laR /tmp/artifacts
 ```

ccia/fake_manifests/Image-builder image/manifest/image_builder/manifest.json

@@ -0,0 +1,16 @@
+{
+  "builds": [
+    {
+      "name": "image-builder",
+      "builder_type": "googlecompute",
+      "build_time": 1658173915,
+      "files": null,
+      "artifact_id": "image-builder-5419329914142720",
+      "packer_run_uuid": "243ae2b1-d4b4-4917-9883-a96c516a2c39",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720"
+      }
+    }
+  ],
+  "last_run_uuid": "243ae2b1-d4b4-4917-9883-a96c516a2c39"
+}

ccia/fake_manifests/build-push Cache Image/manifest/cache_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "build-push",
+      "builder_type": "googlecompute",
+      "build_time": 1658175996,
+      "files": null,
+      "artifact_id": "build-push-c5419329914142720",
+      "packer_run_uuid": "250b8705-ce4d-7844-7181-f1181dd7e04c",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "cache"
+      }
+    }
+  ],
+  "last_run_uuid": "250b8705-ce4d-7844-7181-f1181dd7e04c"
+}

ccia/fake_manifests/fedora Base Image/manifest/base_images/manifest.json

@@ -0,0 +1,22 @@
+{
+  "builds": [
+    {
+      "name": "fedora",
+      "builder_type": "qemu",
+      "build_time": 1658175535,
+      "files": [
+        {
+          "name": "fedora-b5419329914142720",
+          "size": 0
+        }
+      ],
+      "artifact_id": "",
+      "packer_run_uuid": "b1ec41bd-c395-45d6-96cf-f03a1ff2c894",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "base"
+      }
+    }
+  ],
+  "last_run_uuid": "b1ec41bd-c395-45d6-96cf-f03a1ff2c894"
+}

ccia/fake_manifests/fedora Cache Image/manifest/cache_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "fedora",
+      "builder_type": "googlecompute",
+      "build_time": 1658176163,
+      "files": null,
+      "artifact_id": "fedora-c5419329914142720",
+      "packer_run_uuid": "30833ff6-05df-ee1e-4378-57991d592136",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "cache"
+      }
+    }
+  ],
+  "last_run_uuid": "30833ff6-05df-ee1e-4378-57991d592136"
+}

ccia/fake_manifests/fedora-aws Base Image/manifest/base_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "fedora-aws",
+      "builder_type": "amazon-ebs",
+      "build_time": 1658175765,
+      "files": null,
+      "artifact_id": "us-east-1:ami-000448bd70242ba3c",
+      "packer_run_uuid": "193dbe11-9c6b-e0b0-efc9-dc0e8fbf98dc",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "base"
+      }
+    }
+  ],
+  "last_run_uuid": "193dbe11-9c6b-e0b0-efc9-dc0e8fbf98dc"
+}

ccia/fake_manifests/fedora-aws Cache Image/manifest/cache_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "fedora-aws",
+      "builder_type": "amazon-ebs",
+      "build_time": 1658176592,
+      "files": null,
+      "artifact_id": "us-east-1:ami-0442ccd2bb66504b7",
+      "packer_run_uuid": "df4c911b-80a3-27ee-a513-4b6e29c1c906",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "cache"
+      }
+    }
+  ],
+  "last_run_uuid": "df4c911b-80a3-27ee-a513-4b6e29c1c906"
+}

ccia/fake_manifests/fedora-aws-arm64 Base Image/manifest/base_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "fedora-aws-arm64",
+      "builder_type": "amazon-ebs",
+      "build_time": 1658175464,
+      "files": null,
+      "artifact_id": "us-east-1:ami-0f5f268182775a8c2",
+      "packer_run_uuid": "e4a389da-e1dc-35db-ef32-361e890e4b30",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "base"
+      }
+    }
+  ],
+  "last_run_uuid": "e4a389da-e1dc-35db-ef32-361e890e4b30"
+}

ccia/fake_manifests/fedora-netavark Cache Image/manifest/cache_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "fedora-netavark",
+      "builder_type": "googlecompute",
+      "build_time": 1658176148,
+      "files": null,
+      "artifact_id": "fedora-netavark-c5419329914142720",
+      "packer_run_uuid": "d95c8118-3970-4a73-d348-692a5a3371a3",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "cache"
+      }
+    }
+  ],
+  "last_run_uuid": "d95c8118-3970-4a73-d348-692a5a3371a3"
+}

ccia/fake_manifests/fedora-netavark-aws-arm64 Cache Image/manifest/cache_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "fedora-netavark-aws-arm64",
+      "builder_type": "amazon-ebs",
+      "build_time": 1658176335,
+      "files": null,
+      "artifact_id": "us-east-1:ami-07a339e76f84afa7b",
+      "packer_run_uuid": "b8a09332-800a-09c2-ba0e-2564e6e52f76",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "cache"
+      }
+    }
+  ],
+  "last_run_uuid": "b8a09332-800a-09c2-ba0e-2564e6e52f76"
+}

ccia/fake_manifests/fedora-podman-aws-arm64 Cache Image/manifest/cache_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "fedora-podman-aws-arm64",
+      "builder_type": "amazon-ebs",
+      "build_time": 1658176346,
+      "files": null,
+      "artifact_id": "us-east-1:ami-051a5e8dad587bf22",
+      "packer_run_uuid": "7e742dec-035c-6b95-8793-c464b2a6ac0f",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "cache"
+      }
+    }
+  ],
+  "last_run_uuid": "7e742dec-035c-6b95-8793-c464b2a6ac0f"
+}

ccia/fake_manifests/fedora-podman-py Cache Image/manifest/cache_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "fedora-podman-py",
+      "builder_type": "googlecompute",
+      "build_time": 1658176090,
+      "files": null,
+      "artifact_id": "fedora-podman-py-c5419329914142720",
+      "packer_run_uuid": "e5b1e6ab-37a5-a695-624d-47bf0060b272",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "cache"
+      }
+    }
+  ],
+  "last_run_uuid": "e5b1e6ab-37a5-a695-624d-47bf0060b272"
+}

ccia/fake_manifests/ubuntu Base Image/manifest/base_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "ubuntu",
+      "builder_type": "googlecompute",
+      "build_time": 1658175167,
+      "files": null,
+      "artifact_id": "ubuntu-b5419329914142720",
+      "packer_run_uuid": "238ce64e-cb7d-4c1b-38ff-3e0eb9e3939a",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "base"
+      }
+    }
+  ],
+  "last_run_uuid": "238ce64e-cb7d-4c1b-38ff-3e0eb9e3939a"
+}

ccia/fake_manifests/ubuntu Cache Image/manifest/cache_images/manifest.json

@@ -0,0 +1,17 @@
+{
+  "builds": [
+    {
+      "name": "ubuntu",
+      "builder_type": "googlecompute",
+      "build_time": 1658176053,
+      "files": null,
+      "artifact_id": "ubuntu-c5419329914142720",
+      "packer_run_uuid": "553ec5c0-7b09-e06d-2837-b272204696d1",
+      "custom_data": {
+        "IMG_SFX": "5419329914142720",
+        "STAGE": "cache"
+      }
+    }
+  ],
+  "last_run_uuid": "553ec5c0-7b09-e06d-2837-b272204696d1"
+}

ccia/test.sh

@@ -14,19 +14,26 @@ req_env_vars CIRRUS_CI CIRRUS_BUILD_ID CIRRUS_WORKING_DIR
 echo "Installing test tooling"
 ooe.sh microdnf install -y coreutils jq
 
+cd /tmp/
+
 echo "Confirming current build task manifests can be downloaded."
 (
     set -x
-    cd /data
     # shellcheck disable=SC2154
     $CCIABIN --verbose $CIRRUS_BUILD_ID '.*/manifest.json'
 )
 
+# It's possible the PR did not produce any manifest.json files
+if ! dled=$(find ./$CIRRUS_BUILD_ID -name manifest.json | wc -l) || ((dled==0)); then
+    mkdir -p ./$CIRRUS_BUILD_ID
+    cp -a $SCRIPT_DIRPATH/fake_manifests/* ./$CIRRUS_BUILD_ID
+fi
+
 echo "Confirming any downloaded manifests can be parsed into a build list"
 (
     set -x
-    cd /data
-    find ./ -type f -name 'manifest.json' -print0 | \
+    cd /tmp
+    find ./$CIRRUS_BUILD_ID -type f -name 'manifest.json' -print0 | \
         xargs --null jq -e '.builds[]' | \
         jq -e -s '.' | \
         jq -e '{"builds": .}'