Implement helper for processing specific failed CI jobs

cirrus-ci_asr/README.md

@@ -0,0 +1,46 @@
+# Description
+
+This is a quickly hacked-together script which examines a Cirrus-CI
+build and prints out task IDs and names based on their status.  Additionally,
+it will specifically detect and list task IDs which have exhibited
+an "CI agent stopped responding!" condition using the status code
+`CIASR`.
+
+The output format is very simple: Each line is composed of the
+task status (all caps) followed by a comma-separated list
+of task IDs, a colon, and quoted task name.
+
+# Installation
+
+Install the python3 module requirements using pip3:
+(Note: These go into `$HOME/.local/lib/python<version>`)
+
+```
+$ pip3 install --user --requirement ./requirements.txt
+```
+
+# Usage
+
+Simply execute the script, providing as arguments:
+
+1. The *user* component of a github repository
+2. The *name* component of a github repository
+3. The *commit SHA* for the target Cirrus-CI build
+
+# Example: Build monitoring
+
+```
+$ watch -n 5 ./cirrus-ci_asr.py containers podman 5d1f8dcea1401854291932d11bea6aa6920a5682
+
+CREATED 6720901876023296:"int podman fedora-32 root host",4521878620471296:"int remote fedora-32 root host",5647778527313920:"int podman fedora-32 rootless host",5084828573892608:"sys podman fedora-32 root host",6210728480735232:"sys remote fedora-32 root host",4803353597181952:"sys podman fedora-32 rootless host"
+TRIGGERED
+SCHEDULED
+EXECUTING 5595001969180672:"Build for fedora-32"
+ABORTED
+FAILED
+COMPLETED 5032052015759360:"Ext. services",6157951922601984:"Smoke Test"
+SKIPPED
+PAUSED
+CIASR
+(updates every 5 seconds)
+```

cirrus-ci_asr/cirrus-ci_asr.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+
+"""Print list of agent-stopped-responding task IDs and status-keyed task IDs"""
+
+import sys
+from collections import namedtuple
+from pprint import pprint
+
+# Ref: https://gql.readthedocs.io/en/latest/index.html
+# pip3 install --user --requirement ./requirements.txt
+# (and/or in a python virtual environment)
+from gql import gql, Client
+from gql.transport.requests import RequestsHTTPTransport
+
+CIRRUS_CI_STATUSES = (
+    "CREATED",
+    "TRIGGERED",
+    "SCHEDULED",
+    "EXECUTING",
+    "ABORTED",
+    "FAILED",
+    "COMPLETED",
+    "SKIPPED",
+    "PAUSED"
+)
+
+def get_raw_builds(client, owner, repo, sha):
+    """Retrieve list of builds for the specified owner/repo @ commit SHA"""
+    # Generated using https://cirrus-ci.com/explorer
+    query = gql('''
+        query buildBySha($owner: String!, $repo: String!, $sha: String!) {
+          searchBuilds(repositoryOwner: $owner, repositoryName: $repo, SHA: $sha) {
+            id
+            buildCreatedTimestamp
+          }
+        }
+    ''')
+    query_vars = dict(owner=owner, repo=repo, sha=sha)
+    result = client.execute(query, variable_values=query_vars)
+    if "searchBuilds" in result and len(result["searchBuilds"]):
+        return result["searchBuilds"]
+    else:
+        raise RuntimeError(f"No Cirrus-CI builds found for {owner}/{repo} commit {sha}")
+
+
+def latest_build_id(raw_builds):
+    """Return the build_id of the most recent build among raw_builds"""
+    latest_ts = 0
+    latest_bid = 0
+    for build in raw_builds:
+        bts = build["buildCreatedTimestamp"]
+        if bts > latest_ts:
+            latest_ts = bts
+            latest_bid = build["id"]
+    if latest_bid:
+        return latest_bid
+    raise RuntimeError(f"Empty raw_builds list")
+
+
+def get_raw_tasks(client, build_id):
+    """Retrieve raw GraphQL task list from a build"""
+    query = gql('''
+        query tasksByBuildID($build_id: ID!) {
+          build(id: $build_id) {
+            tasks {
+              name
+              id
+              status
+              notifications {
+                level
+                message
+              }
+              automaticReRun
+              previousRuns {
+                id
+              }
+            }
+          }
+        }
+    ''')
+    query_vars = dict(build_id=build_id)
+    result = client.execute(query, variable_values=query_vars)
+    if "build" in result and result["build"]:
+        result = result["build"]
+        if "tasks" in result and len(result["tasks"]):
+            return result["tasks"]
+        else:
+            raise RuntimeError(f"No tasks found for build with id {build_id}")
+    else:
+        raise RuntimeError(f"No Cirrus-CI build found with id {build_id}")
+
+
+def status_tid_names(raw_tasks, status):
+    """Return dictionary of task IDs to task names with specified status"""
+    return dict([(task["id"], task["name"])
+                for task in raw_tasks
+                if task["status"] == status])
+
+
+def notif_tids(raw_tasks, reason):
+    """Return dictionary of task IDs to task names which match notification reason"""
+    result={}
+    for task in raw_tasks:
+        for notif in task["notifications"]:
+            if reason in notif["message"]:
+                result[task["id"]] = task["name"]
+    return result
+
+
+def output_tids(keyword, tid_names):
+    """Write line of space separated list of task ID:"name" prefixed by a keyword"""
+    sys.stdout.write(f'{keyword} ')
+    tasks=[f'{tid}:"{name}"' for tid, name in tid_names.items()]
+    sys.stdout.write(",".join(tasks))
+    sys.stdout.write("\n")
+
+
+if __name__ == "__main__":
+    # Ref: https://cirrus-ci.org/api/
+    cirrus_graphql_url = "https://api.cirrus-ci.com/graphql"
+    cirrus_graphql_xport = RequestsHTTPTransport(
+        url=cirrus_graphql_url,
+        verify=True,
+        retries=3)
+    client = Client(transport=cirrus_graphql_xport, fetch_schema_from_transport=True)
+
+    try:
+        raw_builds = get_raw_builds(client, sys.argv[1], sys.argv[2], sys.argv[3])
+    except IndexError as xcpt:
+        print(f"Error: argument {xcpt}\n\nUsage: {sys.argv[0]} <user> <repo> <sha>")
+        sys.exit(1)
+
+    raw_tasks = get_raw_tasks(client, latest_build_id(raw_builds))
+    for cci_status in CIRRUS_CI_STATUSES:
+        output_tids(cci_status, status_tid_names(raw_tasks, cci_status))
+    output_tids("CIASR", notif_tids(raw_tasks, "CI agent stopped responding!"))

cirrus-ci_asr/requirements.txt

@@ -0,0 +1,9 @@
+aiohttp==3.6.2
+async-timeout==3.0.1
+attrs==20.2.0
+certifi==2020.6.20
+gql==3.0.0a3
+multidict==4.7.6
+requests==2.24.0
+websockets==8.1
+yarl==1.5.1

cirrus-ci_asr/test/run_all_tests.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Stupid-simple, very basic "can it run" test
+
+set -e
+
+if [[ "$CIRRUS_CI" != "true" ]]; then
+    echo -e "\nSkipping: Test must be executed under Cirrus-CI\n"
+    exit 0
+fi
+
+cd "$(dirname ${BASH_SOURCE[0]})/../"
+pip3 install --user --requirement ./requirements.txt
+echo "Testing cirrus-ci_asr.py $CIRRUS_REPO_OWNER $CIRRUS_REPO_NAME $CIRRUS_CHANGE_IN_REPO"
+./cirrus-ci_asr.py $CIRRUS_REPO_OWNER $CIRRUS_REPO_NAME $CIRRUS_CHANGE_IN_REPO