Help preserve release-branch CI VM Images

For release-branches, CI VM images must be retained long-term since they
are difficult/impossible to rebuild.  A number of times, often due to
human error, these images have been accidentally lost.

Update automation tooling such that these images may be specially marked.
Upon encountering a permanently marked image, ensure it is never
deprecated or removed.  When found as deprecated, issue a loud error that
will be delivered to the podman-monitor list.

Operational mechanism: A "meta" job should be executing regularly via
cirrus-cron on every single important release-branch. This job uses the
imgts container image.  Therefore, the branch/tag/pr it is running for
may be retrieved and used to determine if the image should be marked as
permanent.  Otherwise, manually updating all images in this way is
possible but would be labor intensive.

Signed-off-by: Chris Evich <[email protected]>

imgobsolete/entrypoint.sh

@@ -39,10 +39,20 @@ $GCLOUD compute images list --format="$FORMAT" --filter="$FILTER" | \
         count_image
         reason=""
         created_ymd=$(date --date=$creationTimestamp --iso-8601=date)
+        permanent=$(egrep --only-matching --max-count=1 --ignore-case 'permanent=true' <<< $labels || true)
         last_used=$(egrep --only-matching --max-count=1 'last-used=[[:digit:]]+' <<< $labels || true)
 
         LABELSFX="labels: '$labels'"
 
+        # Any image (manually) marked with a `permanent=true` label should be
+        # retained forever.  Typically this will be due to it's use by CI in
+        # a release-branch.  The images `repo-ref` and `build-id` labels should
+        # provide clues as to where it's required (may be multiple repos.)
+        if [[ -n "$permanent" ]]; then
+            msg "Retaining forever $name | $labels"
+            continue
+        fi
+
         # No label was set
         if [[ -z "$last_used" ]]
         then # image lacks any tracking labels

imgprune/entrypoint.sh

@@ -39,8 +39,11 @@ $GCLOUD compute images list --show-deprecated \
     do
         count_image
         reason=""
+        permanent=$(egrep --only-matching --max-count=1 --ignore-case 'permanent=true' <<< $labels || true)
+        [[ -z "$permanent" ]] || \
+            die 1 "Refusing to delete a deprecated image labeled permanent=true.  Please use gcloud utility to set image active, then research the cause of deprecation."
         [[ "$dep_state" == "OBSOLETE" ]] || \
-            die 1 "Error: Unexpected depreciation-state encountered for $name: $dep_state; labels: $labels"
+            die 1 "Unexpected depreciation-state encountered for $name: $dep_state; labels: $labels"
         reason="Obsolete as of $del_date; labels: $labels"
         echo "$name $reason" >> $TODELETE
     done

imgts/entrypoint.sh

@@ -16,12 +16,12 @@ gcloud_init
 
 # These must be defined by the cirrus-ci job using the container
 # shellcheck disable=SC2154
-ARGS="
-    --update-labels=last-used=$(date +%s)
-    --update-labels=build-id=$BUILDID
-    --update-labels=repo-ref=$REPOREF
-    --update-labels=project=$GCPPROJECT
-"
+ARGS=(\
+    "--update-labels=last-used=$(date +%s)"
+    "--update-labels=build-id=$BUILDID"
+    "--update-labels=repo-ref=$REPOREF"
+    "--update-labels=project=$GCPPROJECT"
+)
 
 # Must be defined by the cirrus-ci job using the container
 # shellcheck disable=SC2154
@@ -37,11 +37,69 @@ ERRIMGS=''
 # It's possible for multiple simultaneous label updates to clash
 CLASHMSG='Labels fingerprint either invalid or resource labels have changed'
 
+# In an effort to avoid unintentional deletion of release-branch VM images
+# this image-use context must be detected.  This function accepts a single
+# argument: the Cirrus-CI build ID.  It attempts to determine if that build
+# occured on a non-main branch, and if so will return zero.  Otherwise,
+# it will return non-zero for executions on behalf of all PRs or tags.
+is_release_branch_image(){
+    local buildId api query result fltrpfx branch tag
+    buildId=$1
+    api="https://api.github.com/graphql"
+    query="{
+        \"query\": \"query {
+            build(id: $buildId) {
+                branch
+                tag
+                pullRequest
+              }\"
+        }"
+
+    # It's possible for an image to be missing it's build ID label.
+    # For example, the first time imgts operates on a new image.
+    if ((${#buildId}<12)); then
+        warn 0 "Empty/invalid BuildId value found on image, ignoring: '$buildId'"
+        return 1
+    fi
+
+    fltrpfx=".data.build"
+    result=$(curl --silent --location \
+             --request POST --data @- --url "$api" <<<"$query") \
+             || \
+             die 3 "Error communicating with GraphQL API $api: $result"
+
+    # Best effort: It's possible GraphQL query errored, is invalid,
+    # or the build is so old, there is no record of it.  Issue a
+    # warning and move on.
+    if ! jq -e "$fltrpfx" <<<"$result" &> /dev/null; then
+        warn 0 "Received unexpected reply: $result"
+        return 1
+    fi
+
+    branch=$(jq --raw-output "${fltrpfx}.branch" <<<"$result")
+    tag=$(jq --raw-output "${fltrpfx}.tag" <<<"$result")
+    # Cirrus-CI sets `branch=pull/#` for pull-requests.
+    if [[ -z "$tag" && "$branch" != "main" ]] && [[ ! "$branch" =~ pull ]];
+    then
+        msg "Found image for build $buildId which should be retained forever"
+        return 0
+    fi
+
+    # Ignore image last used by a tag or pull-request
+    return 1
+}
+
+unset SET_PERM
+if is_release_branch_image $BUILDID; then
+    ARGS+=("--update-labels=permanent=true")
+    SET_PERM=1
+fi
+
 # Must be defined by the cirrus-ci job using the container
 # shellcheck disable=SC2154
 for image in $IMGNAMES
 do
-    if ! OUTPUT=$($GCLOUD compute images update "$image" $ARGS 2>&1); then
+    if ! OUTPUT=$($GCLOUD compute images update "$image" "${ARGS[@]}" 2>&1); then
         echo "$OUTPUT" > /dev/stderr
         if grep -iq "$CLASHMSG" <<<"$OUTPUT"; then
             # Updating the 'last-used' label is most important.
@@ -52,7 +110,11 @@ do
         msg "Detected update error for '$image'" > /dev/stderr
         ERRIMGS="$ERRIMGS $image"
     else
-        echo "$OUTPUT" > /dev/stderr
+        # Display the URI to the updated image for reference
+        (
+        echo "$OUTPUT"
+        if ((SET_PERM)); then echo "    IMAGE MARKED FOR PERMANENT RETENTION"; fi
+        ) > /dev/stderr
     fi
 done