From 9cb9194d29f2606d7aa1d63b5d6791a50be73315 Mon Sep 17 00:00:00 2001
From: mrmr1993 <mr_1993@hotmail.co.uk>
Date: Wed, 11 Nov 2020 02:29:28 +0000
Subject: [PATCH 1/2] Add script to remove deb-s3 lockfile if it is > 5 mins
 old

---
 scripts/clear-deb-s3-lockfile.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100755 scripts/clear-deb-s3-lockfile.sh

diff --git a/scripts/clear-deb-s3-lockfile.sh b/scripts/clear-deb-s3-lockfile.sh
new file mode 100755
index 00000000000..7afc2bbd2ca
--- /dev/null
+++ b/scripts/clear-deb-s3-lockfile.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+S3_LOCKFILE_DATE="$(aws s3 ls s3://packages.o1test.net/dists/unstable/main/binary-/lockfile | awk '{print $1 " " $2}')"
+if [[ "$S3_LOCKFILE_DATE" == "" ]]; then
+    echo "Could not get the lockfile timestamp from the S3 bucket. Have you set your AWS credentials correctly?"
+    exit 1
+fi
+S3_LOCKFILE_SECONDS=$(date -d "$S3_LOCKFILE_DATE" +%s)
+NOW_SECONDS=$(date -d +%s)
+TIME_DIFF=$(($NOW_SECONDS - $S3_LOCKFILE_SECONDS))
+if [[ $TIME_DIFF > 300 ]]; then
+    echo "Lockfile has been held for > 5 mins. The deb-s3 instance is likely to have died. Deleting lockfile.."
+    aws s3 rm s3://packages.o1test.net/dists/unstable/main/binary-/lockfile
+    echo "Lockfile deleted"
+else
+    echo "Lockfile is younger than 5 mins. There may be a deb-s3 instance actively using it. Refusing to delete."
+    exit 1
+fi

From f295de751fa1d7eb970b33f73f914808ef4f7a72 Mon Sep 17 00:00:00 2001
From: mrmr1993 <mr_1993@hotmail.co.uk>
Date: Wed, 11 Nov 2020 02:40:20 +0000
Subject: [PATCH 2/2] Use clear-deb-s3-lockfile.sh script to try releasing the
 lockfile

---
 buildkite/scripts/publish-deb.sh          | 6 +++++-
 scripts/archive/build-release-archives.sh | 6 +++++-
 scripts/publish-deb.sh                    | 6 +++++-
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh
index e896560daf6..b039fd1684d 100755
--- a/buildkite/scripts/publish-deb.sh
+++ b/buildkite/scripts/publish-deb.sh
@@ -34,7 +34,11 @@ esac
 
 echo "Publishing debs: ${DEBS}"
 set -x
-${DEBS3} --codename "${CODENAME}" "${DEBS}"
+# Upload the deb files to s3.
+# If this fails, attempt to remove the lockfile and retry.
+${DEBS3} --codename "${CODENAME}" "${DEBS}" \
+|| (  scripts/clear-deb-s3-lockfile.sh \
+   && ${DEBS3} --codename "${CODENAME}" "${DEBS}")
 set +x
 echo "Exporting Variables: "
 
diff --git a/scripts/archive/build-release-archives.sh b/scripts/archive/build-release-archives.sh
index 37970a93b71..baa9e08f45d 100755
--- a/scripts/archive/build-release-archives.sh
+++ b/scripts/archive/build-release-archives.sh
@@ -87,7 +87,11 @@ else
     echo "Publishing debs:"
     ls coda-*.deb
     set -x
-    ${DEBS3} --codename ${CODENAME} --component main coda-*.deb
+    # Upload the deb files to s3.
+    # If this fails, attempt to remove the lockfile and retry.
+    ${DEBS3} --codename ${CODENAME} --component main coda-*.deb \
+    || (  scripts/clear-deb-s3-lockfile.sh \
+       && ${DEBS3} --codename main coda-*.deb)
 fi
 
 ###
diff --git a/scripts/publish-deb.sh b/scripts/publish-deb.sh
index fa6c0b76e05..0f1c6d1922e 100755
--- a/scripts/publish-deb.sh
+++ b/scripts/publish-deb.sh
@@ -63,7 +63,11 @@ case "$CIRCLE_JOB" in
     build-artifacts--testnet_postake_medium_curves | FORCED)
         echo "Publishing debs: ${DEBS}"
         set -x
-        ${DEBS3} --codename "${CODENAME}" "${DEBS}"
+        # Upload the deb files to s3.
+        # If this fails, attempt to remove the lockfile and retry.
+        ${DEBS3} --codename "${CODENAME}" "${DEBS}" \
+        || (  scripts/clear-deb-s3-lockfile.sh \
+           && ${DEBS3} --codename "${CODENAME}" "${DEBS}")
         echo "Exporting Variables: "
         # Export Variables for Downstream Steps
         echo "export CODA_DEB_REPO=$CODENAME" >> /tmp/DOCKER_DEPLOY_ENV