ci: find branchs in chrono order

.gitlab/config/generate_pipelines.py

@@ -5,11 +5,11 @@
 #
 # generate_pipeline.py
 #
-#  Created: May 19, 2023
 #   Author: Vicente Adolfo Bolea Sanchez <[email protected]>
 
 from datetime import datetime
 import argparse
+import itertools
 import requests
 import time
 import re
@@ -18,18 +18,43 @@
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 
-def is_date_after(date, days):
-    deadline_sec = int(time.time()) - (days * 86400)
-    utc_dt = datetime.strptime(date, '%Y-%m-%dT%H:%M:%SZ')
-    timestamp_sec = (utc_dt - datetime(1970, 1, 1)).total_seconds()
-    return timestamp_sec > deadline_sec
-
-
-def request_dict(url):
+def request_as_dict(url):
     r = requests.get(url + '?per_page=100', verify=False)
     return r.json()
 
 
+def add_timestamp(branch):
+    date_str = branch['commit']['committed_date']
+    # We ignore the TZ since Gitlab/GitHub always reports in UTC
+    branch['dt'] = int(
+        datetime.strptime(date_str.split(".")[0],
+                          '%Y-%m-%dT%H:%M:%S').timestamp())
+    return branch
+
+
+def is_recent(branch):
+    deadline_sec = int(time.time()) - (args.days * 86400)
+    return branch['dt'] > deadline_sec
+
+
+def has_no_status(branch):
+    gh_commit_sha = branch['commit']['id']
+    # Backported branches use the merge head
+    if re.fullmatch(r'^pr\d+_.*$', branch['name']):
+        gh_commit_sha = branch['commit']['parent_ids'][1]
+
+    # Query GitHub for the status of this commit
+    commit = request_as_dict(gh_url + '/commits/' + gh_commit_sha + '/status')
+    if commit is None or 'sha' not in commit:
+        return False
+
+    for status in commit['statuses']:
+        if status['context'] == args.gh_context:
+            return False
+
+    return True
+
+
 parser = argparse.ArgumentParser(
     prog='generate_pipeline.py',
     description='Generate Dynamic pipelines for Gitlab')
@@ -39,12 +64,12 @@ def request_dict(url):
 parser.add_argument(
     '-n', '--gh-name', required=True,
     help='Full name of the GitHub project. Ex: ornladios/ADIOS2')
-parser.add_argument(
-    '-c', '--gh-context', default='OLCF Crusher (Frontier)',
-    help='Name of the status in GitHub (A.K.A context)')
 parser.add_argument(
     '-p', '--project_id', required=True,
     help='Gitlab internal project ID of the project.')
+parser.add_argument(
+    '-c', '--gh-context', default='OLCF Crusher (Frontier)',
+    help='Name of the status in GitHub (A.K.A context)')
 parser.add_argument(
     '-d', '--days', type=int, default=1,
     help='How many days back to search for commits')
@@ -57,34 +82,21 @@ def request_dict(url):
 args = parser.parse_args()
 
 
-with open(args.template_file, "r") as fd:
+gl_url = args.gl_url + '/api/v4/projects/' + str(args.project_id)
+gh_url = 'https://api.github.com/repos/' + args.gh_name
+
+with open(args.template_file, 'r') as fd:
     template_str = fd.read()
-    gl_url = args.gl_url + "/api/v4/projects/" + str(args.project_id)
-    gh_url = 'https://api.github.com/repos/' + args.gh_name
-    branches = request_dict(gl_url + "/repository/branches")
-    num_pipeline = 0
+
+    branches = request_as_dict(gl_url + '/repository/branches')
+    branches = map(add_timestamp, branches)
+    branches = filter(is_recent, branches)
+    branches = filter(has_no_status, branches)
+
+    # Select the arg.max most least recent branches
+    branches = sorted(branches, key=lambda x: x['dt'])
+    branches = itertools.islice(branches, args.max)
+
     for branch in branches:
-        # Convert to ISO 8601 date format.
-        date_stamp = branch['commit']['committed_date'].split('.')[0] + "Z"
-        if num_pipeline < args.max and is_date_after(date_stamp, args.days):
-            commit_sha = branch['commit']['id']
-            # Backported branches use the merge head
-            gh_commit_sha = commit_sha
-            if re.fullmatch(r'^pr\d+_.*$', branch['name']):
-                gh_commit_sha = branch['commit']['parent_ids'][1]
-
-            # Quit if GitHub does not have the commit
-            if 'sha' not in request_dict(gh_url + "/commits/" + gh_commit_sha):
-                continue
-
-            # Query GitHub for the status of this commit
-            commit = request_dict(gh_url + "/commits/" +
-                                  gh_commit_sha + "/status")
-            status_found = False
-            for status in commit['statuses']:
-                if status['context'] == args.gh_context:
-                    status_found = True
-            if not status_found:
-                num_pipeline += 1
-                print(template_str.format(
-                    branch=branch['name'], commit=commit_sha))
+        print(template_str.format(
+            branch=branch['name'], commit=branch['commit']['id']))

.gitlab/gitlab-ci-crusher.yml

@@ -206,7 +206,7 @@ generate_pipelines:
   variables:
     CUSTOM_CI_BUILDS_DIR: "/lustre/orion/csc303/scratch/vbolea/ci/adios2/runtime"
   script:
-    - .gitlab/config/generate_pipelines.py -u "https://code.olcf.ornl.gov/" -p 78 -n ornladios/ADIOS2 -f .gitlab/config/dynamic_pipeline.yml.in > generated_pipelines.yml
+    - python3.10 .gitlab/config/generate_pipelines.py -u "https://code.olcf.ornl.gov/" -p 78 -n ornladios/ADIOS2 -f .gitlab/config/dynamic_pipeline.yml.in > generated_pipelines.yml
   artifacts:
     paths:
       - generated_pipelines.yml