ensure docker installs before ensure docker runs (Azure#2305)

tesharp · Mar 15, 2018 · 04c2b7a · 04c2b7a
1 parent 021a079
commit 04c2b7a
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 4 deletions.
diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml
@@ -170,7 +170,7 @@ runcmd:
 - set -x
 - retrycmd_if_failure() { retries=$1; wait=$2; shift && shift; for i in $(seq 1 $retries); do ${@}; [ $? -eq 0  ] && break || sleep $wait; done; echo Executed \"$@\" $i times; }
 - retrycmd_if_failure_no_stats() { retries=$1; wait=$2; shift && shift; for i in $(seq 1 $retries); do ${@}; [ $? -eq 0  ] && break || sleep $wait; done; }
-- apt_get_update() { for i in $(seq 1 20); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0  ] && break || sleep 1; done; echo Executed apt-get update $i times; }
+- apt_get_update() { for i in $(seq 1 100); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0  ] && break || sleep 1; done; echo Executed apt-get update $i times; }
 - retrycmd_if_failure 120 1 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \  -f 2) 53
 - retrycmd_if_failure 120 1 nc -zw1 aptdocker.azureedge.net 443
 - apt-mark hold walinuxagent{{GetKubernetesAgentPreprovisionYaml .}}
@@ -189,7 +189,8 @@ runcmd:
 - echo "deb {{WrapAsVariable "dockerEngineDownloadRepo"}} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list
 - "echo \"Package: docker-engine\nPin: version {{WrapAsVariable "dockerEngineVersion"}}\nPin-Priority: 550\n\" > /etc/apt/preferences.d/docker.pref"
 - apt_get_update
-- retrycmd_if_failure 5 10 apt-get install -y ebtables docker-engine
+- retrycmd_if_failure 20 10 apt-get install -y ebtables docker-engine
+- touch /opt/azure/containers/dockerinstall.complete
 - echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf
 - systemctl daemon-reload
 - echo `date`,`hostname`, postdockerinstall>>/opt/m

diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml
@@ -347,7 +347,7 @@ runcmd:
 - retrycmd_if_failure() { retries=$1; wait=$2; shift && shift; for i in $(seq 1 $retries); do ${@}; [ $? -eq 0  ] && break || sleep $wait; done; echo Executed \"$@\" $i times; }
 - retrycmd_if_failure_no_stats() { retries=$1; wait=$2; shift && shift; for i in $(seq 1 $retries); do ${@}; [ $? -eq 0  ] && break || sleep $wait; done; }
 - ensure_etcd_ready() { for i in $(seq 1 1800); do if [ -e /opt/azure/containers/certs.ready ]; then break; fi; sleep 1; done }
-- apt_get_update() { for i in $(seq 1 20); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0  ] && break || sleep 1; done; echo Executed apt-get update $i times; }
+- apt_get_update() { for i in $(seq 1 100); do apt-get update 2>&1 | grep -x "[WE]:.*"; [ $? -ne 0  ] && break || sleep 1; done; echo Executed apt-get update $i times; }
 - retrycmd_if_failure 120 1 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \  -f 2) 53
 - retrycmd_if_failure 120 1 nc -zw1 aptdocker.azureedge.net 443
 - ensure_etcd_ready
@@ -369,7 +369,8 @@ runcmd:
 - echo "deb {{WrapAsVariable "dockerEngineDownloadRepo"}} ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list
 - "echo \"Package: docker-engine\nPin: version {{WrapAsVariable "dockerEngineVersion"}}\nPin-Priority: 550\n\" > /etc/apt/preferences.d/docker.pref"
 - apt_get_update
-- retrycmd_if_failure 5 10 apt-get install -y ebtables docker-engine
+- retrycmd_if_failure 20 10 apt-get install -y ebtables docker-engine
+- touch /opt/azure/containers/dockerinstall.complete
 - echo "ExecStartPost=/sbin/iptables -P FORWARD ACCEPT" >> /etc/systemd/system/docker.service.d/exec_start.conf
 - systemctl daemon-reload
 - systemctl restart docker

diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh
@@ -67,6 +67,20 @@ ensureRunCommandCompleted()
     done
 }
 
+# cloudinit runcmd and the extension will run in parallel, this is to ensure
+# runcmd finishes
+ensureDockerInstallCompleted()
+{
+    echo "waiting for docker install to finish"
+    for i in {1..900}; do
+        if [ -e /opt/azure/containers/dockerinstall.complete ]; then
+            echo "docker install finished, took $i seconds"
+            break
+        fi
+        sleep 1
+    done
+}
+
 echo `date`,`hostname`, startscript>>/opt/m
 
 # A delay to start the kubernetes processes is necessary
@@ -691,6 +705,7 @@ fi
 
 # master and node
 echo `date`,`hostname`, EnsureDockerStart>>/opt/m
+ensureDockerInstallCompleted
 ensureDocker
 echo `date`,`hostname`, configNetworkPolicyStart>>/opt/m
 configNetworkPolicy