From 2e967f6793779dccc139a3c3fccfbdc0424f2462 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 22 Mar 2018 16:52:37 -0700 Subject: [PATCH 1/5] add timeout to retry func, deal with curl + tar --- parts/k8s/kubernetesagentcustomdata.yml | 10 ++++----- parts/k8s/kubernetesjumpboxcustomdata.yml | 2 +- parts/k8s/kubernetesmastercustomdata.yml | 16 +++++++------- parts/k8s/kubernetesmastercustomscript.sh | 22 ++++++++++++++++--- ...kubernetesmastergenerateproxycertscript.sh | 2 +- parts/k8s/kubernetesprovisionsource.sh | 4 ++-- 6 files changed, 36 insertions(+), 20 deletions(-) diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index 2dfc780a99..ed655da6be 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -184,25 +184,25 @@ runcmd: - set -x - . /opt/azure/containers/provision_source.sh - apt_get_update() { for i in $(seq 1 100); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0 ] && break || sleep 1; done; echo Executed apt-get update $i times; } -- retrycmd_if_failure 120 1 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53 -- retrycmd_if_failure 120 1 nc -zw1 aptdocker.azureedge.net 443 +- retrycmd_if_failure 120 1 5 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53 +- retrycmd_if_failure 120 1 5 nc -zw1 aptdocker.azureedge.net 443 - apt-mark hold walinuxagent{{GetKubernetesAgentPreprovisionYaml .}} - echo `date`,`hostname`, preaptupdate>>/opt/m - apt_get_update - echo `date`,`hostname`, postaptupdate>>/opt/m -- retrycmd_if_failure 5 10 apt-get install -y apt-transport-https ca-certificates nfs-common +- retrycmd_if_failure 5 10 120 apt-get install -y apt-transport-https ca-certificates nfs-common - echo `date`,`hostname`, aptinstall>>/opt/m - systemctl enable rpcbind - systemctl enable rpc-statd - systemctl start rpcbind - systemctl start rpc-statd - echo `date`,`hostname`, predockerinstall>>/opt/m -- retrycmd_if_failure_no_stats 180 1 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg +- retrycmd_if_failure_no_stats 180 1 5 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg - cat /tmp/aptdocker.gpg | apt-key add - - echo "deb {{WrapAsVariable "dockerEngineDownloadRepo"}} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list - "echo \"Package: docker-engine\nPin: version {{WrapAsVariable "dockerEngineVersion"}}\nPin-Priority: 550\n\" > /etc/apt/preferences.d/docker.pref" - apt_get_update -- retrycmd_if_failure 20 10 apt-get install -y ebtables docker-engine +- retrycmd_if_failure 20 10 120 apt-get install -y ebtables docker-engine - touch /opt/azure/containers/dockerinstall.complete - echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf - systemctl daemon-reload diff --git a/parts/k8s/kubernetesjumpboxcustomdata.yml b/parts/k8s/kubernetesjumpboxcustomdata.yml index c0b06d9a63..9d58a680f5 100644 --- a/parts/k8s/kubernetesjumpboxcustomdata.yml +++ b/parts/k8s/kubernetesjumpboxcustomdata.yml @@ -17,6 +17,6 @@ write_files: runcmd: - . /opt/azure/containers/provision_source.sh -- retrycmd_if_failure 10 5 curl -LO https://storage.googleapis.com/kubernetes-release/release/v{{.OrchestratorProfile.OrchestratorVersion}}/bin/linux/amd64/kubectl +- retrycmd_if_failure 10 5 10 curl -LO https://storage.googleapis.com/kubernetes-release/release/v{{.OrchestratorProfile.OrchestratorVersion}}/bin/linux/amd64/kubectl - chmod +x ./kubectl - sudo mv ./kubectl /usr/local/bin/kubectl \ No newline at end of file diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index b8518afe21..7e611e10f1 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -319,7 +319,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER source /opt/azure/containers/provision_source.sh ETCD_VER=v{{WrapAsVariable "etcdVersion"}} DOWNLOAD_URL={{WrapAsVariable "etcdDownloadURLBase"}} - retrycmd_if_failure 5 5 curl --retry 5 --retry-delay 10 --retry-max-time 30 -L ${DOWNLOAD_URL}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz + retrycmd_if_failure 5 5 15 curl --retry 5 --retry-delay 10 --retry-max-time 15 -L ${DOWNLOAD_URL}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /usr/bin/ --strip-components=1 systemctl daemon-reload systemctl enable etcd.service @@ -343,7 +343,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER sudo /bin/sed -i s/Restart=on-failure/Restart=always/g /lib/systemd/system/etcd-member.service systemctl daemon-reload systemctl restart etcd-member - retrycmd_if_failure 5 5 curl --retry 5 --retry-delay 10 --retry-max-time 30 --max-time 60 http://127.0.0.1:2379/v2/machines + retrycmd_if_failure 5 5 10 curl --retry 5 --retry-delay 10 --retry-max-time 10 --max-time 60 http://127.0.0.1:2379/v2/machines mkdir -p /etc/kubernetes/manifests usermod -aG docker {{WrapAsVariable "username"}} @@ -371,8 +371,8 @@ runcmd: - . /opt/azure/containers/provision_source.sh - ensure_etcd_ready() { for i in $(seq 1 1800); do if [ -e /opt/azure/containers/certs.ready ]; then break; fi; sleep 1; done } - apt_get_update() { for i in $(seq 1 100); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0 ] && break || sleep 1; done; echo Executed apt-get update $i times; } -- retrycmd_if_failure 120 1 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53 -- retrycmd_if_failure 120 1 nc -zw1 aptdocker.azureedge.net 443 +- retrycmd_if_failure 120 1 5 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53 +- retrycmd_if_failure 120 1 5 nc -zw1 aptdocker.azureedge.net 443 - ensure_etcd_ready - /opt/azure/containers/setup-etcd.sh > /opt/azure/containers/setup-etcd.log 2>&1 - apt-mark hold walinuxagent {{GetKubernetesMasterPreprovisionYaml}} @@ -384,15 +384,15 @@ runcmd: - systemctl restart etcd - MEMBER="$(sudo etcdctl member list | grep -E {{WrapAsVerbatim "variables('masterVMNames')[copyIndex(variables('masterOffset'))]"}} | cut -d{{WrapAsVariable "singleQuote"}}:{{WrapAsVariable "singleQuote"}} -f 1)" - sudo etcdctl member update ${MEMBER} {{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}} -- retrycmd_if_failure 5 5 curl --cacert /etc/kubernetes/certs/ca.crt --cert /etc/kubernetes/certs/etcdclient.crt --key /etc/kubernetes/certs/etcdclient.key --retry 5 --retry-delay 10 --retry-max-time 30 --max-time 60 "{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}"/v2/machines +- retrycmd_if_failure 5 5 10 curl --cacert /etc/kubernetes/certs/ca.crt --cert /etc/kubernetes/certs/etcdclient.crt --key /etc/kubernetes/certs/etcdclient.key --retry 5 --retry-delay 10 --retry-max-time 10 --max-time 60 "{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}"/v2/machines - apt_get_update -- retrycmd_if_failure 5 10 apt-get install -y apt-transport-https ca-certificates -- retrycmd_if_failure_no_stats 180 1 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg +- retrycmd_if_failure 5 10 120 apt-get install -y apt-transport-https ca-certificates +- retrycmd_if_failure_no_stats 180 1 5 curl -fsSL https://aptdocker.azureedge.net/gpg > /tmp/aptdocker.gpg - cat /tmp/aptdocker.gpg | apt-key add - - echo "deb {{WrapAsVariable "dockerEngineDownloadRepo"}} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list - "echo \"Package: docker-engine\nPin: version {{WrapAsVariable "dockerEngineVersion"}}\nPin-Priority: 550\n\" > /etc/apt/preferences.d/docker.pref" - apt_get_update -- retrycmd_if_failure 20 10 apt-get install -y ebtables docker-engine +- retrycmd_if_failure 20 10 120 apt-get install -y ebtables docker-engine - touch /opt/azure/containers/dockerinstall.complete - echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf - systemctl daemon-reload diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index 91e0ee6bea..e3719471d2 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -216,11 +216,27 @@ function configAzureNetworkPolicy() { # Mirror from https://github.com/Azure/azure-container-networking/releases/tag/$AZURE_PLUGIN_VER/azure-vnet-cni-linux-amd64-$AZURE_PLUGIN_VER.tgz AZURE_CNI_TGZ_TMP=/tmp/azure_cni.tgz - retrycmd_if_failure_no_stats 180 1 curl -fsSL ${VNET_CNI_PLUGINS_URL} > $AZURE_CNI_TGZ_TMP + for i in {1..600}; do + tar -tzf $AZURE_CNI_TGZ_TMP + if [ $? -eq 0 ] + then + break + fi + retrycmd_if_failure_no_stats 180 1 10 curl -fsSL ${VNET_CNI_PLUGINS_URL} > $AZURE_CNI_TGZ_TMP + sleep 1 + done tar -xzf $AZURE_CNI_TGZ_TMP -C $CNI_BIN_DIR # Mirror from https://github.com/containernetworking/cni/releases/download/$CNI_RELEASE_VER/cni-amd64-$CNI_RELEASE_VERSION.tgz CONTAINERNETWORKING_CNI_TGZ_TMP=/tmp/containernetworking_cni.tgz - retrycmd_if_failure_no_stats 180 1 curl -fsSL ${CNI_PLUGINS_URL} > $CONTAINERNETWORKING_CNI_TGZ_TMP + for i in {1..600}; do + tar -tzf $CONTAINERNETWORKING_CNI_TGZ_TMP + if [ $? -eq 0 ] + then + break + fi + retrycmd_if_failure_no_stats 180 1 10 curl -fsSL ${CNI_PLUGINS_URL} > $CONTAINERNETWORKING_CNI_TGZ_TMP + sleep 1 + done tar -xzf $CONTAINERNETWORKING_CNI_TGZ_TMP -C $CNI_BIN_DIR ./loopback ./portmap chown -R root:root $CNI_BIN_DIR chmod -R 755 $CNI_BIN_DIR @@ -482,7 +498,7 @@ function ensureDocker() { } function ensureKubelet() { - retrycmd_if_failure 100 1 docker pull $HYPERKUBE_URL + retrycmd_if_failure 100 1 60 docker pull $HYPERKUBE_URL systemctlEnableAndCheck kubelet # only start if a reboot is not required if ! $REBOOTREQUIRED; then diff --git a/parts/k8s/kubernetesmastergenerateproxycertscript.sh b/parts/k8s/kubernetesmastergenerateproxycertscript.sh index 228edf604d..f3905c6674 100644 --- a/parts/k8s/kubernetesmastergenerateproxycertscript.sh +++ b/parts/k8s/kubernetesmastergenerateproxycertscript.sh @@ -45,7 +45,7 @@ write_certs_to_disk_with_retry() { } # block until all etcd is ready -retrycmd_if_failure etcdctl cluster-health +retrycmd_if_failure 100 5 10 etcdctl cluster-health # Make etcd keys, adding a leading whitespace because etcd won't accept a val that begins with a '-' (hyphen)! if etcdctl mk $ETCD_REQUESTHEADER_CLIENT_CA " $(cat ${PROXY_CRT})"; then etcdctl mk $ETCD_PROXY_KEY " $(cat ${PROXY_CLIENT_KEY})" diff --git a/parts/k8s/kubernetesprovisionsource.sh b/parts/k8s/kubernetesprovisionsource.sh index bfea93d11e..a16a9bf0f8 100644 --- a/parts/k8s/kubernetesprovisionsource.sh +++ b/parts/k8s/kubernetesprovisionsource.sh @@ -1,4 +1,4 @@ #!/bin/sh -retrycmd_if_failure() { retries=$1; wait=$2; shift && shift; for i in $(seq 1 $retries); do ${@}; [ $? -eq 0 ] && break || sleep $wait; done; echo Executed \"$@\" $i times; } -retrycmd_if_failure_no_stats() { retries=$1; wait=$2; shift && shift; for i in $(seq 1 $retries); do ${@}; [ $? -eq 0 ] && break || sleep $wait; done; } \ No newline at end of file +retrycmd_if_failure() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; echo Executed \"$@\" $i times; } +retrycmd_if_failure_no_stats() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; } \ No newline at end of file From 9e51072d1c9295f12bb2d8fdb8aa8ce02a905f95 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 22 Mar 2018 17:18:51 -0700 Subject: [PATCH 2/5] remove comments --- parts/k8s/kubernetesmastercustomscript.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index e3719471d2..6c2ed89892 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -206,15 +206,10 @@ function setDockerOpts () { function configAzureNetworkPolicy() { CNI_CONFIG_DIR=/etc/cni/net.d mkdir -p $CNI_CONFIG_DIR - chown -R root:root $CNI_CONFIG_DIR chmod 755 $CNI_CONFIG_DIR - - # Download Azure VNET CNI plugins. CNI_BIN_DIR=/opt/cni/bin mkdir -p $CNI_BIN_DIR - - # Mirror from https://github.com/Azure/azure-container-networking/releases/tag/$AZURE_PLUGIN_VER/azure-vnet-cni-linux-amd64-$AZURE_PLUGIN_VER.tgz AZURE_CNI_TGZ_TMP=/tmp/azure_cni.tgz for i in {1..600}; do tar -tzf $AZURE_CNI_TGZ_TMP @@ -226,7 +221,6 @@ function configAzureNetworkPolicy() { sleep 1 done tar -xzf $AZURE_CNI_TGZ_TMP -C $CNI_BIN_DIR - # Mirror from https://github.com/containernetworking/cni/releases/download/$CNI_RELEASE_VER/cni-amd64-$CNI_RELEASE_VERSION.tgz CONTAINERNETWORKING_CNI_TGZ_TMP=/tmp/containernetworking_cni.tgz for i in {1..600}; do tar -tzf $CONTAINERNETWORKING_CNI_TGZ_TMP @@ -240,15 +234,9 @@ function configAzureNetworkPolicy() { tar -xzf $CONTAINERNETWORKING_CNI_TGZ_TMP -C $CNI_BIN_DIR ./loopback ./portmap chown -R root:root $CNI_BIN_DIR chmod -R 755 $CNI_BIN_DIR - - # Copy config file mv $CNI_BIN_DIR/10-azure.conflist $CNI_CONFIG_DIR/ chmod 600 $CNI_CONFIG_DIR/10-azure.conflist - - # Dump ebtables rules. /sbin/ebtables -t nat --list - - # Enable CNI. configCNINetworkPolicy } From 4032c4eae7dee2331bb1ba731b5804d0f1b2261d Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 23 Mar 2018 09:37:55 -0700 Subject: [PATCH 3/5] generalize tarball download retry --- parts/k8s/kubernetesmastercustomscript.sh | 20 ++------------------ parts/k8s/kubernetesprovisionsource.sh | 3 ++- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index 6c2ed89892..72d2baae8c 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -211,26 +211,10 @@ function configAzureNetworkPolicy() { CNI_BIN_DIR=/opt/cni/bin mkdir -p $CNI_BIN_DIR AZURE_CNI_TGZ_TMP=/tmp/azure_cni.tgz - for i in {1..600}; do - tar -tzf $AZURE_CNI_TGZ_TMP - if [ $? -eq 0 ] - then - break - fi - retrycmd_if_failure_no_stats 180 1 10 curl -fsSL ${VNET_CNI_PLUGINS_URL} > $AZURE_CNI_TGZ_TMP - sleep 1 - done + retrycmd_get_tarball 60 1 $AZURE_CNI_TGZ_TMP ${VNET_CNI_PLUGINS_URL} tar -xzf $AZURE_CNI_TGZ_TMP -C $CNI_BIN_DIR CONTAINERNETWORKING_CNI_TGZ_TMP=/tmp/containernetworking_cni.tgz - for i in {1..600}; do - tar -tzf $CONTAINERNETWORKING_CNI_TGZ_TMP - if [ $? -eq 0 ] - then - break - fi - retrycmd_if_failure_no_stats 180 1 10 curl -fsSL ${CNI_PLUGINS_URL} > $CONTAINERNETWORKING_CNI_TGZ_TMP - sleep 1 - done + retrycmd_get_tarball 60 1 $CONTAINERNETWORKING_CNI_TGZ_TMP ${CNI_PLUGINS_URL} tar -xzf $CONTAINERNETWORKING_CNI_TGZ_TMP -C $CNI_BIN_DIR ./loopback ./portmap chown -R root:root $CNI_BIN_DIR chmod -R 755 $CNI_BIN_DIR diff --git a/parts/k8s/kubernetesprovisionsource.sh b/parts/k8s/kubernetesprovisionsource.sh index a16a9bf0f8..50e0233710 100644 --- a/parts/k8s/kubernetesprovisionsource.sh +++ b/parts/k8s/kubernetesprovisionsource.sh @@ -1,4 +1,5 @@ #!/bin/sh retrycmd_if_failure() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; echo Executed \"$@\" $i times; } -retrycmd_if_failure_no_stats() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; } \ No newline at end of file +retrycmd_if_failure_no_stats() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; } +retrycmd_get_tarball() { retries=$1; wait=$2; tarball=$3; url=$4 shift && shift && shift && shift; for i in $(seq 1 $retries); do tar -tzf $tarball; [ $? -eq 0 ] && break || retrycmd_if_failure_no_stats $retries 1 10 curl -fsSL $url > $tarball; sleep $wait; done; } \ No newline at end of file From fcc3650a98f18790f643b91e3a6cb858e7ae39e3 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 23 Mar 2018 10:26:17 -0700 Subject: [PATCH 4/5] removing unnecessary shifts in retrycmd_get_tarball --- parts/k8s/kubernetesprovisionsource.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parts/k8s/kubernetesprovisionsource.sh b/parts/k8s/kubernetesprovisionsource.sh index 50e0233710..5550f9a915 100644 --- a/parts/k8s/kubernetesprovisionsource.sh +++ b/parts/k8s/kubernetesprovisionsource.sh @@ -2,4 +2,4 @@ retrycmd_if_failure() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; echo Executed \"$@\" $i times; } retrycmd_if_failure_no_stats() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; } -retrycmd_get_tarball() { retries=$1; wait=$2; tarball=$3; url=$4 shift && shift && shift && shift; for i in $(seq 1 $retries); do tar -tzf $tarball; [ $? -eq 0 ] && break || retrycmd_if_failure_no_stats $retries 1 10 curl -fsSL $url > $tarball; sleep $wait; done; } \ No newline at end of file +retrycmd_get_tarball() { retries=$1; wait=$2; tarball=$3; url=$4; for i in $(seq 1 $retries); do tar -tzf $tarball; [ $? -eq 0 ] && break || retrycmd_if_failure_no_stats $retries 1 10 curl -fsSL $url > $tarball; sleep $wait; done; } \ No newline at end of file From b863f3b81b37d3705e7586c6642ad12951a3328d Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 23 Mar 2018 11:08:13 -0700 Subject: [PATCH 5/5] enforce common retry_get_tarball pattern, use -o --- parts/k8s/kubernetesmastercustomdata.yml | 2 +- parts/k8s/kubernetesmastercustomscript.sh | 19 ++++--------------- parts/k8s/kubernetesprovisionsource.sh | 2 +- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 7e611e10f1..c3d8edddee 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -319,7 +319,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER source /opt/azure/containers/provision_source.sh ETCD_VER=v{{WrapAsVariable "etcdVersion"}} DOWNLOAD_URL={{WrapAsVariable "etcdDownloadURLBase"}} - retrycmd_if_failure 5 5 15 curl --retry 5 --retry-delay 10 --retry-max-time 15 -L ${DOWNLOAD_URL}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz + retrycmd_get_tarball 60 1 /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz ${DOWNLOAD_URL}/etcd-${ETCD_VER}-linux-amd64.tar.gz tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /usr/bin/ --strip-components=1 systemctl daemon-reload systemctl enable etcd.service diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh index 72d2baae8c..48267aec5d 100644 --- a/parts/k8s/kubernetesmastercustomscript.sh +++ b/parts/k8s/kubernetesmastercustomscript.sh @@ -280,29 +280,18 @@ function installClearContainersRuntime() { function installGo() { export GO_SRC=/usr/local/go export GOPATH="${HOME}/.go" - - # Remove any old version of Go if [[ -d "$GO_SRC" ]]; then rm -rf "$GO_SRC" fi - - # Remove any old GOPATH if [[ -d "$GOPATH" ]]; then rm -rf "$GOPATH" fi - # Get the latest Go version - GO_VERSION=$(curl --retry 5 --retry-delay 10 --retry-max-time 30 -sSL "https://golang.org/VERSION?m=text") - - echo "Installing Go version $GO_VERSION..." - - # subshell - ( - curl --retry 5 --retry-delay 10 --retry-max-time 30 -sSL "https://storage.googleapis.com/golang/${GO_VERSION}.linux-amd64.tar.gz" | sudo tar -v -C /usr/local -xz - ) + retrycmd_if_failure_no_stats 180 1 5 curl -fsSL https://golang.org/VERSION?m=text > /tmp/gover.txt + GO_VERSION=$(cat /tmp/gover.txt) + retrycmd_get_tarball 60 1 /tmp/golang.tgz https://storage.googleapis.com/golang/${GO_VERSION}.linux-amd64.tar.gz + tar -v -C /usr/local -xzf /tmp/golang.tgz - # Set GOPATH and update PATH - echo "Setting GOPATH and updating PATH" export PATH="${GO_SRC}/bin:${PATH}:${GOPATH}/bin" } diff --git a/parts/k8s/kubernetesprovisionsource.sh b/parts/k8s/kubernetesprovisionsource.sh index 5550f9a915..f1a9d7d945 100644 --- a/parts/k8s/kubernetesprovisionsource.sh +++ b/parts/k8s/kubernetesprovisionsource.sh @@ -2,4 +2,4 @@ retrycmd_if_failure() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; echo Executed \"$@\" $i times; } retrycmd_if_failure_no_stats() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; } -retrycmd_get_tarball() { retries=$1; wait=$2; tarball=$3; url=$4; for i in $(seq 1 $retries); do tar -tzf $tarball; [ $? -eq 0 ] && break || retrycmd_if_failure_no_stats $retries 1 10 curl -fsSL $url > $tarball; sleep $wait; done; } \ No newline at end of file +retrycmd_get_tarball() { retries=$1; wait=$2; tarball=$3; url=$4; for i in $(seq 1 $retries); do tar -tzf $tarball; [ $? -eq 0 ] && break || retrycmd_if_failure_no_stats $retries 1 10 curl -fsSL $url -o $tarball; sleep $wait; done; } \ No newline at end of file