diff --git a/buildkite/scripts/publish-deb.sh b/buildkite/scripts/publish-deb.sh index edfb95300aa..00ff843fe08 100755 --- a/buildkite/scripts/publish-deb.sh +++ b/buildkite/scripts/publish-deb.sh @@ -34,7 +34,11 @@ esac echo "Publishing debs: ${DEBS}" set -x -${DEBS3} --codename "${CODENAME}" "${DEBS}" +# Upload the deb files to s3. +# If this fails, attempt to remove the lockfile and retry. +${DEBS3} --codename "${CODENAME}" "${DEBS}" \ +|| ( scripts/clear-deb-s3-lockfile.sh \ + && ${DEBS3} --codename "${CODENAME}" "${DEBS}") set +x echo "Exporting Variables: " diff --git a/scripts/archive/build-release-archives.sh b/scripts/archive/build-release-archives.sh index dde522bbbea..47ca4e1dc7f 100755 --- a/scripts/archive/build-release-archives.sh +++ b/scripts/archive/build-release-archives.sh @@ -87,7 +87,11 @@ else echo "Publishing debs:" ls mina-*.deb set -x - ${DEBS3} --codename ${CODENAME} --component main mina-*.deb + # Upload the deb files to s3. + # If this fails, attempt to remove the lockfile and retry. + ${DEBS3} --codename ${CODENAME} --component main mina-*.deb \ + || ( scripts/clear-deb-s3-lockfile.sh \ + && ${DEBS3} --codename main mina-*.deb) fi ### diff --git a/scripts/clear-deb-s3-lockfile.sh b/scripts/clear-deb-s3-lockfile.sh new file mode 100755 index 00000000000..7afc2bbd2ca --- /dev/null +++ b/scripts/clear-deb-s3-lockfile.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +S3_LOCKFILE_DATE="$(aws s3 ls s3://packages.o1test.net/dists/unstable/main/binary-/lockfile | awk '{print $1 " " $2}')" +if [[ "$S3_LOCKFILE_DATE" == "" ]]; then + echo "Could not get the lockfile timestamp from the S3 bucket. Have you set your AWS credentials correctly?" + exit 1 +fi +S3_LOCKFILE_SECONDS=$(date -d "$S3_LOCKFILE_DATE" +%s) +NOW_SECONDS=$(date -d +%s) +TIME_DIFF=$(($NOW_SECONDS - $S3_LOCKFILE_SECONDS)) +if [[ $TIME_DIFF > 300 ]]; then + echo "Lockfile has been held for > 5 mins. The deb-s3 instance is likely to have died. Deleting lockfile.." + aws s3 rm s3://packages.o1test.net/dists/unstable/main/binary-/lockfile + echo "Lockfile deleted" +else + echo "Lockfile is younger than 5 mins. There may be a deb-s3 instance actively using it. Refusing to delete." + exit 1 +fi diff --git a/scripts/publish-deb.sh b/scripts/publish-deb.sh index dcdd62dfac4..d4eacf74eb9 100755 --- a/scripts/publish-deb.sh +++ b/scripts/publish-deb.sh @@ -66,7 +66,11 @@ case "$CIRCLE_JOB" in build-artifacts--testnet_postake_medium_curves | FORCED) echo "Publishing debs: ${DEBS}" set -x - ${DEBS3} --codename "${CODENAME}" "${DEBS}" + # Upload the deb files to s3. + # If this fails, attempt to remove the lockfile and retry. + ${DEBS3} --codename "${CODENAME}" "${DEBS}" \ + || ( scripts/clear-deb-s3-lockfile.sh \ + && ${DEBS3} --codename "${CODENAME}" "${DEBS}") echo "Exporting Variables: " # Export Variables for Downstream Steps echo "export CODA_DEB_REPO=$CODENAME" >> /tmp/DOCKER_DEPLOY_ENV