From f2a81a80fae9292b89ea32abf432a925d74e11af Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 6 Nov 2023 11:23:54 +0000 Subject: [PATCH 1/3] Handle network hangs on spot instance requests. --- build-system/scripts/request_spot | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 9e35b9b769a..1360f9afff7 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -110,11 +110,11 @@ done # Wait till ssh port is open. >&2 echo "Waiting for SSH at $IP..." -for I in {1..60}; do - if nc -z $IP 22; then - echo $IP - exit 0 +SECONDS=0 +while ! nc -z -w1 $IP 22; do + if (( SECONDS >= 60 )); then + >&2 echo "Timeout: SSH port 22 at $IP did not become available after 60 seconds." + exit 1 fi - sleep 1 -done -exit 1 +fi +echo $IP \ No newline at end of file From 91ca48de5b7e42a68d3b865abf7892be437db872 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 6 Nov 2023 11:24:14 +0000 Subject: [PATCH 2/3] [ci rebuild end-to-end] From 2a092f986e08941e757fa054d24253b9e1b91bbf Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Mon, 6 Nov 2023 11:54:25 +0000 Subject: [PATCH 3/3] [ci rebuild end-to-end] --- build-system/scripts/request_spot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-system/scripts/request_spot b/build-system/scripts/request_spot index 1360f9afff7..3eef009d8ab 100755 --- a/build-system/scripts/request_spot +++ b/build-system/scripts/request_spot @@ -116,5 +116,5 @@ while ! nc -z -w1 $IP 22; do >&2 echo "Timeout: SSH port 22 at $IP did not become available after 60 seconds." exit 1 fi -fi +done echo $IP \ No newline at end of file