From fcea566efd8d753aeaf8daec50de7825146dd0da Mon Sep 17 00:00:00 2001
From: Jonathan Lebon <jonathan@jlebon.com>
Date: Thu, 23 Feb 2023 15:41:29 -0500
Subject: [PATCH] jobs/build: don't try to complete ongoing builds

Since 973bcf9 ("jobs/build: rerun `build-arch` if previous build is
incomplete"), there is a race possible where the `build` job rerun logic
could kick in before the `release` jobs initially triggered for that
build has finished. We don't want to queue builds in that case.

Gate the rerun logic on whether the multi-arch locks and release lock
are taken.

It's theoretically possible but highly unlikely that we probe the lock
status before the previous `release` job takes it. Ideally, we would
have a way to directly take the lock and "transfer" its ownership to the
job we trigger. Anyway, if that somehow happens, it would result in the
job being run twice, which is safe.
---
 jobs/build.Jenkinsfile | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/jobs/build.Jenkinsfile b/jobs/build.Jenkinsfile
index 02d2cb359..a446c253a 100644
--- a/jobs/build.Jenkinsfile
+++ b/jobs/build.Jenkinsfile
@@ -265,9 +265,10 @@ lock(resource: "build-${params.STREAM}") {
             // Nothing changed since the latest build. Check if it's missing
             // some arches and retrigger `build-arch` only for the missing
             // arches, and the follow-up `release` job. Match the exact src
-            // config commit that was used. Skip if not uploading since it's
-            // required for multi-arch.
-            if (uploading) {
+            // config commit that was used. But only do this if there isn't
+            // already outstanding work in progress for that build ID. Skip if
+            // not uploading since it's required for multi-arch.
+            if (uploading && !buildid_has_work_pending(buildID, additional_arches)) {
                 def builds = readJSON file: "builds/builds.json"
                 assert buildID == builds.builds[0].id
                 def missing_arches = additional_arches - builds.builds[0].arches
@@ -542,3 +543,14 @@ def run_release_job(buildID) {
         ]
     }
 }
+
+def buildid_has_work_pending(buildID, arches) {
+    def locked = true
+    // these locks match the ones in the release job 
+    def locks = arches.collect{[resource: "release-${buildID}-${it}"]}
+    lock(resource: "release-${params.STREAM}", extra: locks, skipIfLocked: true) {
+        // NB: `return` here wouldn't actually return from the function
+        locked = false
+    }
+    return locked
+}