From 07dd5f34f3b1cf3a36e858b813318e0456e58f53 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 9 Apr 2018 14:27:31 -0700 Subject: [PATCH] wait_for_file, rationalize docker install --- parts/k8s/kubernetesagentcustomdata.yml | 6 +--- parts/k8s/kubernetescustomscript.sh | 40 +++--------------------- parts/k8s/kubernetesmastercustomdata.yml | 8 ++--- parts/k8s/kubernetesprovisionsource.sh | 2 +- 4 files changed, 8 insertions(+), 48 deletions(-) diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index 45a13a8aa4..2160182a94 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -181,8 +181,6 @@ runcmd: # the first arg is the number of retries, the second arg is the wait duration between two retries and the rest of the args are the cmd to run - set -x - . /opt/azure/containers/provision_source.sh -- retrycmd_if_failure 120 1 5 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53 -- retrycmd_if_failure 120 1 5 nc -zw1 aptdocker.azureedge.net 443 - retrycmd_if_failure 20 5 5 apt-mark hold walinuxagent{{GetKubernetesAgentPreprovisionYaml .}} - echo `date`,`hostname`, preaptupdate>>/opt/m - apt_get_update @@ -200,11 +198,9 @@ runcmd: - "echo \"Package: docker-engine\nPin: version {{WrapAsVariable "dockerEngineVersion"}}\nPin-Priority: 550\n\" > /etc/apt/preferences.d/docker.pref" - apt_get_update - retrycmd_if_failure 20 10 120 apt-get install -y ebtables docker-engine -- touch /opt/azure/containers/dockerinstall.complete - echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf -- systemctl daemon-reload +- touch /opt/azure/containers/dockerinstall.complete - echo `date`,`hostname`, postdockerinstall>>/opt/m -- retrycmd_if_failure 100 1 10 systemctl daemon-reload && systemctl restart docker - mkdir -p /etc/kubernetes/manifests - usermod -aG docker {{WrapAsVariable "username"}} {{if IsNSeriesSKU .}} diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 96c1a8178c..456d5e3a9c 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -23,25 +23,13 @@ fi ensureRunCommandCompleted() { echo "waiting for runcmd to finish" - for i in {1..900}; do - if [ -e /opt/azure/containers/runcmd.complete ]; then - echo "runcmd finished, took $i seconds" - break - fi - sleep 1 - done + wait_for_file 900 1 /opt/azure/containers/runcmd.complete } ensureDockerInstallCompleted() { echo "waiting for docker install to finish" - for i in {1..900}; do - if [ -e /opt/azure/containers/dockerinstall.complete ]; then - echo "docker install finished, took $i seconds" - break - fi - sleep 1 - done + wait_for_file 3600 1 /opt/azure/containers/dockerinstall.complete } echo `date`,`hostname`, startscript>>/opt/m @@ -170,21 +158,7 @@ function ensureFilepath() { if $REBOOTREQUIRED; then return fi - found=1 - for i in {1..600}; do - if [ -e $1 ] - then - found=0 - echo "$1 is present, took $i seconds to verify" - break - fi - sleep 1 - done - if [ $found -ne 0 ] - then - echo "$1 is not present after $i seconds of trying to verify" - exit 1 - fi + wait_for_file 600 1 $1 } function setKubeletOpts () { @@ -362,13 +336,7 @@ function ensureK8s() { k8sHealthy=1 nodesActive=1 nodesReady=1 - for i in {1..600}; do - if [ -e $KUBECTL ] - then - break - fi - sleep 1 - done + wait_for_file 600 1 $KUBECTL for i in {1..600}; do $KUBECTL 2>/dev/null cluster-info if [ "$?" = "0" ] diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index 9c9a1d7fdb..1eac1a880f 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -372,9 +372,7 @@ runcmd: # the first arg is the number of retries, the second arg is the wait duration between two retries and the rest of the args are the cmd to run - set -x - . /opt/azure/containers/provision_source.sh -- retrycmd_if_failure 120 1 5 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \ -f 2) 53 -- retrycmd_if_failure 120 1 5 nc -zw1 aptdocker.azureedge.net 443 -- ensure_etcd_ready +- wait_for_file 1800 1 /opt/azure/containers/certs.ready - /opt/azure/containers/setup-etcd.sh > /opt/azure/containers/setup-etcd.log 2>&1 - retrycmd_if_failure 20 5 5 apt-mark hold walinuxagent {{GetKubernetesMasterPreprovisionYaml}} - /bin/echo DAEMON_ARGS=--name "{{WrapAsVerbatim "variables('masterVMNames')[copyIndex(variables('masterOffset'))]"}}" --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file={{WrapAsVerbatim "variables('etcdPeerCertFilepath')[copyIndex(variables('masterOffset'))]"}} --peer-key-file={{WrapAsVerbatim "variables('etcdPeerKeyFilepath')[copyIndex(variables('masterOffset'))]"}} --initial-advertise-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-client-urls "{{WrapAsVerbatim "concat(variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))], ',https://127.0.0.1:', variables('masterEtcdClientPort'))"}}" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster {{WrapAsVerbatim "variables('masterEtcdClusterStates')[div(variables('masterCount'), 2)]"}} --data-dir "/var/lib/etcddisk" --initial-cluster-state "new" | tee -a /etc/default/etcd @@ -394,10 +392,8 @@ runcmd: - "echo \"Package: docker-engine\nPin: version {{WrapAsVariable "dockerEngineVersion"}}\nPin-Priority: 550\n\" > /etc/apt/preferences.d/docker.pref" - apt_get_update - retrycmd_if_failure 20 10 120 apt-get install -y ebtables docker-engine -- touch /opt/azure/containers/dockerinstall.complete - echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf -- systemctl daemon-reload -- retrycmd_if_failure 100 1 10 systemctl daemon-reload && systemctl restart docker +- touch /opt/azure/containers/dockerinstall.complete - mkdir -p /etc/kubernetes/manifests - usermod -aG docker {{WrapAsVariable "username"}} - /usr/lib/apt/apt.systemd.daily diff --git a/parts/k8s/kubernetesprovisionsource.sh b/parts/k8s/kubernetesprovisionsource.sh index c94aca8cd2..72b294cfaf 100644 --- a/parts/k8s/kubernetesprovisionsource.sh +++ b/parts/k8s/kubernetesprovisionsource.sh @@ -3,5 +3,5 @@ retrycmd_if_failure() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; echo Executed \"$@\" $i times; } retrycmd_if_failure_no_stats() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0 ] && break || sleep $wait; done; } retrycmd_get_tarball() { retries=$1; wait=$2; tarball=$3; url=$4; for i in $(seq 1 $retries); do tar -tzf $tarball; [ $? -eq 0 ] && break || retrycmd_if_failure_no_stats $retries 1 10 curl -fsSL $url -o $tarball; sleep $wait; done; } -ensure_etcd_ready() { for i in $(seq 1 1800); do if [ -e /opt/azure/containers/certs.ready ]; then break; fi; sleep 1; done } +wait_for_file() { retries=$1; wait=$2; filepath=$3; for i in $(seq 1 $retries); do if [ -f $filepath ]; then break; fi; sleep $wait; done } apt_get_update() { for i in $(seq 1 100); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0 ] && break || sleep 1; done; echo Executed apt-get update $i times; }