DCOS: added retry logic to download and install packages (#3009)

* added retry logic to download and install packages

* added TestGenerateIpList test

parts/dcos/bstrap/bootstrapcustomdata.yml

@@ -1,5 +1,3 @@
-#cloud-config
-
 bootcmd:
 - bash -c "if [ ! -f /var/lib/sdb-gpt ];then echo DCOS-5890;parted -s /dev/sdb mklabel
   gpt;touch /var/lib/sdb-gpt;fi"
@@ -76,6 +74,10 @@ MASTER_IP_LIST
   owner: root
   path: /opt/azure/dcos/init_bootstrap.sh
   permissions: '0755'
+- content: 'PROVISION_SOURCE_STR'
+  path: /opt/azure/containers/provision_source.sh
+  permissions: "0744"
+  owner: "root"
 - content: 'PROVISION_STR'
   path: /opt/azure/containers/provision.sh
   permissions: "0744"

parts/dcos/bstrap/bootstrapprovision.sh

@@ -1,26 +1,23 @@
-#!/bin/bash
-
-source /opt/azure/dcos/environment
-
-retrycmd_if_failure() { retries=$1; wait=$2; timeout=$3; shift && shift && shift; for i in $(seq 1 $retries); do timeout $timeout ${@}; [ $? -eq 0  ] && break || sleep $wait; done; echo Executed \"$@\" $i times; }
-
-TMPDIR="/tmp/dcos"
-mkdir -p $TMPDIR
-
-# default dc/os component download address (Azure CDN)
-LIBLTDL_DOWNLOAD_URL=https://dcos-mirror.azureedge.net/pkg/libltdl7_2.4.6-0.1_amd64.deb
-DOCKER_CE_DOWNLOAD_URL=https://dcos-mirror.azureedge.net/pkg/docker-ce_17.09.0~ce-0~ubuntu_amd64.deb
-
-case $DCOS_ENVIRONMENT in
-    # because of Chinese GreatWall Firewall, the default packages on Azure CDN is blocked. So the following Chinese local mirror url should be used instead.
-    AzureChinaCloud)
-        LIBLTDL_DOWNLOAD_URL=http://acsengine.blob.core.chinacloudapi.cn/dcos/libltdl7_2.4.6-0.1_amd64.deb
-        DOCKER_CE_DOWNLOAD_URL=http://mirror.kaiyuanshe.cn/docker-ce/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_17.09.0~ce-0~ubuntu_amd64.deb
-    ;;
-esac
-
-curl -fLsSv --retry 20 -Y 100000 -y 60 -o $TMPDIR/1.deb $LIBLTDL_DOWNLOAD_URL &
-curl -fLsSv --retry 20 -Y 100000 -y 60 -o $TMPDIR/2.deb $DOCKER_CE_DOWNLOAD_URL &
-wait
-
-retrycmd_if_failure 10 10 120 dpkg -i $TMPDIR/{1,2}.deb
+#!/bin/bash
+
+source /opt/azure/containers/provision_source.sh
+source /opt/azure/dcos/environment
+
+# default dc/os component download address (Azure CDN)
+LIBLTDL_DOWNLOAD_URL=https://dcos-mirror.azureedge.net/pkg/libltdl7_2.4.6-0.1_amd64.deb
+DOCKER_CE_DOWNLOAD_URL=https://dcos-mirror.azureedge.net/pkg/docker-ce_17.09.0~ce-0~ubuntu_amd64.deb
+
+case $DCOS_ENVIRONMENT in
+    # because of Chinese GreatWall Firewall, the default packages on Azure CDN is blocked. So the following Chinese local mirror url should be used instead.
+    AzureChinaCloud)
+        LIBLTDL_DOWNLOAD_URL=http://acsengine.blob.core.chinacloudapi.cn/dcos/libltdl7_2.4.6-0.1_amd64.deb
+        DOCKER_CE_DOWNLOAD_URL=http://mirror.kaiyuanshe.cn/docker-ce/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce_17.09.0~ce-0~ubuntu_amd64.deb
+    ;;
+esac
+
+for url in $LIBLTDL_DOWNLOAD_URL $DOCKER_CE_DOWNLOAD_URL; do
+  retry_get_install_deb 10 10 120 $url
+  if [ $? -ne 0  ]; then
+    exit 1
+  fi
+done

parts/dcos/bstrap/dcos1.11.0.customdata.t

@@ -1,88 +1,92 @@
-bootcmd:
-- bash -c "if [ ! -f /var/lib/sdb-gpt ];then echo DCOS-5890;parted -s /dev/sdb mklabel
-  gpt;touch /var/lib/sdb-gpt;fi"
-disk_setup:
-  ephemeral0:
-    layout:
-    - 50
-    - 50
-    overwrite: true
-    table_type: gpt
-fs_setup:
-- device: ephemeral0.1
-  filesystem: ext4
-  overwrite: true
-- device: ephemeral0.2
-  filesystem: ext4
-  overwrite: true
-mounts:
-- - ephemeral0.1
-  - /var/lib/mesos
-- - ephemeral0.2
-  - /var/lib/docker
-runcmd: PREPROVISION_EXTENSION
-- ln -s /bin/rm /usr/bin/rm
-- ln -s /bin/mkdir /usr/bin/mkdir
-- ln -s /bin/tar /usr/bin/tar
-- ln -s /bin/ln /usr/bin/ln
-- ln -s /bin/cp /usr/bin/cp
-- ln -s /bin/systemctl /usr/bin/systemctl
-- ln -s /bin/mount /usr/bin/mount
-- ln -s /bin/bash /usr/bin/bash
-- ln -s /usr/sbin/useradd /usr/bin/useradd
-- systemctl disable --now resolvconf.service
-- systemctl mask --now lxc-net.service
-- systemctl disable --now unscd.service
-- systemctl stop --now unscd.service
-- /opt/azure/containers/provision.sh
-- bash /tmp/dcos/dcos_install.sh ROLENAME
-- bash /opt/azure/dcos/diagnostics_fix.sh
-write_files:
-- content: |
-    [Service]
-    Restart=always
-    StartLimitInterval=0
-    RestartSec=15
-    ExecStartPre=-/sbin/ip link del docker0
-    ExecStart=
-    ExecStart=/usr/bin/dockerd --storage-driver=overlay
-  path: /etc/systemd/system/docker.service.d/execstart.conf
-  permissions: '0644'
-- content: |
-    [Unit]
-    PartOf=docker.service
-    [Socket]
-    ListenStream=/var/run/docker.sock
-    SocketMode=0660
-    SocketUser=root
-    SocketGroup=docker
-    ListenStream=2375
-    BindIPv6Only=both
-    [Install]
-    WantedBy=sockets.target
-  path: /etc/systemd/system/docker.socket
-  permissions: '0644'
-- content: |
-    DCOS_ENVIRONMENT={{{targetEnvironment}}}
-  owner: root
-  path: /opt/azure/dcos/environment
-  permissions: '0644'
-- content: |
-    #!/bin/bash
-
-    for f in /opt/mesosphere/packages/dcos-config--setup_*/etc/dcos-diagnostics-runner-config.json; do
-      if [ -e $f ]; then
-        sed -i.bak "99 s/1s/10s/" $f
-      fi
-    done
-  owner: root
-  path: /opt/azure/dcos/diagnostics_fix.sh
-  permissions: '0744'
-- path: /var/lib/dcos/mesos-slave-common
-  content: 'ATTRIBUTES_STR'
-  permissions: "0644"
-  owner: "root"
-- content: 'PROVISION_STR'
-  path: /opt/azure/containers/provision.sh
-  permissions: "0744"
-  owner: "root"
+bootcmd:
+- bash -c "if [ ! -f /var/lib/sdb-gpt ];then echo DCOS-5890;parted -s /dev/sdb mklabel
+  gpt;touch /var/lib/sdb-gpt;fi"
+disk_setup:
+  ephemeral0:
+    layout:
+    - 50
+    - 50
+    overwrite: true
+    table_type: gpt
+fs_setup:
+- device: ephemeral0.1
+  filesystem: ext4
+  overwrite: true
+- device: ephemeral0.2
+  filesystem: ext4
+  overwrite: true
+mounts:
+- - ephemeral0.1
+  - /var/lib/mesos
+- - ephemeral0.2
+  - /var/lib/docker
+runcmd: PREPROVISION_EXTENSION
+- ln -s /bin/rm /usr/bin/rm
+- ln -s /bin/mkdir /usr/bin/mkdir
+- ln -s /bin/tar /usr/bin/tar
+- ln -s /bin/ln /usr/bin/ln
+- ln -s /bin/cp /usr/bin/cp
+- ln -s /bin/systemctl /usr/bin/systemctl
+- ln -s /bin/mount /usr/bin/mount
+- ln -s /bin/bash /usr/bin/bash
+- ln -s /usr/sbin/useradd /usr/bin/useradd
+- systemctl disable --now resolvconf.service
+- systemctl mask --now lxc-net.service
+- systemctl disable --now unscd.service
+- systemctl stop --now unscd.service
+- /opt/azure/containers/provision.sh
+- bash /tmp/dcos/dcos_install.sh ROLENAME
+- bash /opt/azure/dcos/diagnostics_fix.sh
+write_files:
+- content: |
+    [Service]
+    Restart=always
+    StartLimitInterval=0
+    RestartSec=15
+    ExecStartPre=-/sbin/ip link del docker0
+    ExecStart=
+    ExecStart=/usr/bin/dockerd --storage-driver=overlay
+  path: /etc/systemd/system/docker.service.d/execstart.conf
+  permissions: '0644'
+- content: |
+    [Unit]
+    PartOf=docker.service
+    [Socket]
+    ListenStream=/var/run/docker.sock
+    SocketMode=0660
+    SocketUser=root
+    SocketGroup=docker
+    ListenStream=2375
+    BindIPv6Only=both
+    [Install]
+    WantedBy=sockets.target
+  path: /etc/systemd/system/docker.socket
+  permissions: '0644'
+- content: |
+    DCOS_ENVIRONMENT={{{targetEnvironment}}}
+  owner: root
+  path: /opt/azure/dcos/environment
+  permissions: '0644'
+- content: |
+    #!/bin/bash
+
+    for f in /opt/mesosphere/packages/dcos-config--setup_*/etc/dcos-diagnostics-runner-config.json; do
+      if [ -e $f ]; then
+        sed -i.bak "99 s/1s/10s/" $f
+      fi
+    done
+  owner: root
+  path: /opt/azure/dcos/diagnostics_fix.sh
+  permissions: '0744'
+- path: /var/lib/dcos/mesos-slave-common
+  content: 'ATTRIBUTES_STR'
+  permissions: "0644"
+  owner: "root"
+- content: 'PROVISION_SOURCE_STR'
+  path: /opt/azure/containers/provision_source.sh
+  permissions: "0744"
+  owner: "root"
+- content: 'PROVISION_STR'
+  path: /opt/azure/containers/provision.sh
+  permissions: "0744"
+  owner: "root"