retry apt-mark hold and apt-mark unhold (#2625)

parts/k8s/kubernetesagentcustomdata.yml

@@ -183,7 +183,7 @@ runcmd:
 - . /opt/azure/containers/provision_source.sh
 - retrycmd_if_failure 120 1 5 nc -zuw1 $(grep nameserver /etc/resolv.conf | cut -d \  -f 2) 53
 - retrycmd_if_failure 120 1 5 nc -zw1 aptdocker.azureedge.net 443
-- apt-mark hold walinuxagent{{GetKubernetesAgentPreprovisionYaml .}}
+- retrycmd_if_failure 20 5 5 apt-mark hold walinuxagent{{GetKubernetesAgentPreprovisionYaml .}}
 - echo `date`,`hostname`, preaptupdate>>/opt/m
 - apt_get_update
 - echo `date`,`hostname`, postaptupdate>>/opt/m
@@ -213,7 +213,7 @@ runcmd:
 - echo `date`,`hostname`, PRE-APT-SYSTEMD-DAILY>>/opt/m
 - /usr/lib/apt/apt.systemd.daily
 - echo `date`,`hostname`, POST-APT-SYSTEMD-DAILY>>/opt/m
-- apt-mark unhold walinuxagent
+- retrycmd_if_failure 20 5 5 apt-mark unhold walinuxagent
 - mkdir -p /opt/azure/containers && touch /opt/azure/containers/runcmd.complete
 - echo `date`,`hostname`, endruncmd>>/opt/m
 {{end}}

parts/k8s/kubernetescustomscript.sh

@@ -533,7 +533,7 @@ fi
 
 if [[ $OS == $UBUNTU_OS_NAME ]]; then
 	# make sure walinuxagent doesn't get updated in the middle of running this script
-	apt-mark hold walinuxagent
+	retrycmd_if_failure 20 5 5 apt-mark hold walinuxagent
 fi
 
 echo `date`,`hostname`, EnsureDockerStart>>/opt/m
@@ -578,7 +578,7 @@ if [[ $OS == $UBUNTU_OS_NAME ]]; then
     echo 2dd1ce17-079e-403c-b352-a1921ee207ee > /sys/bus/vmbus/drivers/hv_util/unbind
     sed -i "13i\echo 2dd1ce17-079e-403c-b352-a1921ee207ee > /sys/bus/vmbus/drivers/hv_util/unbind\n" /etc/rc.local
 
-    apt-mark unhold walinuxagent
+    retrycmd_if_failure 20 5 5 apt-mark unhold walinuxagent
 fi
 
 echo "Install complete successfully"

parts/k8s/kubernetesmastercustomdata.yml

@@ -376,7 +376,7 @@ runcmd:
 - retrycmd_if_failure 120 1 5 nc -zw1 aptdocker.azureedge.net 443
 - ensure_etcd_ready
 - /opt/azure/containers/setup-etcd.sh > /opt/azure/containers/setup-etcd.log 2>&1
-- apt-mark hold walinuxagent {{GetKubernetesMasterPreprovisionYaml}}
+- retrycmd_if_failure 20 5 5 apt-mark hold walinuxagent {{GetKubernetesMasterPreprovisionYaml}}
 - /bin/echo DAEMON_ARGS=--name "{{WrapAsVerbatim "variables('masterVMNames')[copyIndex(variables('masterOffset'))]"}}" --peer-client-cert-auth --peer-trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --peer-cert-file={{WrapAsVerbatim "variables('etcdPeerCertFilepath')[copyIndex(variables('masterOffset'))]"}} --peer-key-file={{WrapAsVerbatim "variables('etcdPeerKeyFilepath')[copyIndex(variables('masterOffset'))]"}} --initial-advertise-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-peer-urls "{{WrapAsVerbatim "variables('masterEtcdPeerURLs')[copyIndex(variables('masterOffset'))]"}}" --client-cert-auth --trusted-ca-file={{WrapAsVariable "etcdCaFilepath"}} --cert-file={{WrapAsVariable "etcdServerCertFilepath"}} --key-file={{WrapAsVariable "etcdServerKeyFilepath"}} --advertise-client-urls "{{WrapAsVerbatim "variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))]"}}" --listen-client-urls "{{WrapAsVerbatim "concat(variables('masterEtcdClientURLs')[copyIndex(variables('masterOffset'))], ',https://127.0.0.1:', variables('masterEtcdClientPort'))"}}" --initial-cluster-token "k8s-etcd-cluster" --initial-cluster {{WrapAsVerbatim "variables('masterEtcdClusterStates')[div(variables('masterCount'), 2)]"}} --data-dir "/var/lib/etcddisk" --initial-cluster-state "new" | tee -a /etc/default/etcd
 - /opt/azure/containers/mountetcd.sh
 - /bin/chown -R etcd:etcd /var/lib/etcddisk
@@ -404,6 +404,6 @@ runcmd:
 {{if EnableAggregatedAPIs}}
 - bash /etc/kubernetes/generate-proxy-certs.sh
 {{end}}
-- apt-mark unhold walinuxagent
+- retrycmd_if_failure 20 5 5 apt-mark unhold walinuxagent
 - touch /opt/azure/containers/runcmd.complete
 {{end}}