delay docker and kubelet health monitors for 30 mins (#4050)

Azure · Oct 17, 2018 · fdfbd1a · fdfbd1a
1 parent 20ff962
commit fdfbd1a
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 8 deletions.
diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml
@@ -43,6 +43,17 @@ write_files:
   content: !!binary |
     {{WrapAsVariable "healthMonitorScript"}}
 
+- path: "/etc/systemd/system/kubelet-monitor.timer"
+  permissions: "0644"
+  owner: "root"
+  content: |
+    [Unit]
+    Description=a timer that delays kubelet-monitor from starting too soon after boot
+    [Timer]
+    OnBootSec=30min
+    [Install]
+    WantedBy=multi-user.target
+
 - path: "/etc/systemd/system/kubelet-monitor.service"
   permissions: "0644"
   owner: "root"
@@ -55,6 +66,15 @@ write_files:
     RestartSec=10
     RemainAfterExit=yes
     ExecStart=/usr/local/bin/health-monitor.sh kubelet
+
+- path: "/etc/systemd/system/docker-monitor.timer"
+  permissions: "0644"
+  owner: "root"
+  content: |
+    [Unit]
+    Description=a timer that delays docker-monitor from starting too soon after boot
+    [Timer]
+    OnBootSec=30min
     [Install]
     WantedBy=multi-user.target
 
@@ -70,8 +90,6 @@ write_files:
     RestartSec=10
     RemainAfterExit=yes
     ExecStart=/usr/local/bin/health-monitor.sh container-runtime
-    [Install]
-    WantedBy=multi-user.target
 
 {{if .KubernetesConfig.RequiresDocker}}
     {{if not .IsCoreOS}}

diff --git a/parts/k8s/kubernetesconfigs.sh b/parts/k8s/kubernetesconfigs.sh
@@ -245,9 +245,12 @@ function ensureDocker() {
     DOCKER_JSON_FILE=/etc/docker/daemon.json
     wait_for_file 1200 1 $DOCKER_JSON_FILE || exit $ERR_FILE_WATCH_TIMEOUT
     systemctlEnableAndStart docker
+    # Delay start of docker-monitor for 30 mins after booting
+    DOCKER_MONITOR_SYSTEMD_TIMER_FILE=/etc/systemd/system/docker-monitor.timer
+    wait_for_file 1200 1 $DOCKER_MONITOR_SYSTEMD_TIMER_FILE || exit $ERR_FILE_WATCH_TIMEOUT
     DOCKER_MONITOR_SYSTEMD_FILE=/etc/systemd/system/docker-monitor.service
     wait_for_file 1200 1 $DOCKER_MONITOR_SYSTEMD_FILE || exit $ERR_FILE_WATCH_TIMEOUT
-    systemctlEnableAndStart docker-monitor || exit $ERR_SYSTEMCTL_START_FAIL
+    systemctlEnableAndStart docker-monitor.timer || exit $ERR_SYSTEMCTL_START_FAIL
 }
 function ensureKMS() {
     systemctlEnableAndStart kms || exit $ERR_SYSTEMCTL_START_FAIL
@@ -261,9 +264,12 @@ function ensureKubelet() {
     KUBELET_RUNTIME_CONFIG_SCRIPT_FILE=/opt/azure/containers/kubelet.sh
     wait_for_file 1200 1 $KUBELET_RUNTIME_CONFIG_SCRIPT_FILE || exit $ERR_FILE_WATCH_TIMEOUT
     systemctlEnableAndStart kubelet || exit $ERR_KUBELET_START_FAIL
+    # Delay start of kubelet-monitor for 30 mins after booting
+    KUBELET_MONITOR_SYSTEMD_TIMER_FILE=/etc/systemd/system/kubelet-monitor.timer
+    wait_for_file 1200 1 $KUBELET_MONITOR_SYSTEMD_TIMER_FILE || exit $ERR_FILE_WATCH_TIMEOUT
     KUBELET_MONITOR_SYSTEMD_FILE=/etc/systemd/system/kubelet-monitor.service
     wait_for_file 1200 1 $KUBELET_MONITOR_SYSTEMD_FILE || exit $ERR_FILE_WATCH_TIMEOUT
-    systemctlEnableAndStart kubelet-monitor || exit $ERR_SYSTEMCTL_START_FAIL
+    systemctlEnableAndStart kubelet-monitor.timer || exit $ERR_SYSTEMCTL_START_FAIL
 }
 
 function ensureJournal(){

diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml
@@ -49,6 +49,17 @@ write_files:
   content: !!binary |
     {{WrapAsVariable "healthMonitorScript"}}
 
+- path: "/etc/systemd/system/kubelet-monitor.timer"
+  permissions: "0644"
+  owner: "root"
+  content: |
+    [Unit]
+    Description=a timer that delays kubelet-monitor from starting too soon after boot
+    [Timer]
+    OnBootSec=30min
+    [Install]
+    WantedBy=multi-user.target
+
 - path: "/etc/systemd/system/kubelet-monitor.service"
   permissions: "0644"
   owner: "root"
@@ -61,6 +72,15 @@ write_files:
     RestartSec=10
     RemainAfterExit=yes
     ExecStart=/usr/local/bin/health-monitor.sh kubelet
+
+- path: "/etc/systemd/system/docker-monitor.timer"
+  permissions: "0644"
+  owner: "root"
+  content: |
+    [Unit]
+    Description=a timer that delays docker-monitor from starting too soon after boot
+    [Timer]
+    OnBootSec=30min
     [Install]
     WantedBy=multi-user.target
 
@@ -76,8 +96,6 @@ write_files:
     RestartSec=10
     RemainAfterExit=yes
     ExecStart=/usr/local/bin/health-monitor.sh container-runtime
-    [Install]
-    WantedBy=multi-user.target
 
 {{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}}
     {{if not .MasterProfile.IsCoreOS}}

diff --git a/parts/k8s/kubernetesmastercustomdatavmss.yml b/parts/k8s/kubernetesmastercustomdatavmss.yml
@@ -49,6 +49,17 @@ write_files:
   content: !!binary |
     {{WrapAsVariable "healthMonitorScript"}}
 
+- path: "/etc/systemd/system/kubelet-monitor.timer"
+  permissions: "0644"
+  owner: "root"
+  content: |
+    [Unit]
+    Description=a timer that delays kubelet-monitor from starting too soon after boot
+    [Timer]
+    OnBootSec=30min
+    [Install]
+    WantedBy=multi-user.target
+
 - path: "/etc/systemd/system/kubelet-monitor.service"
   permissions: "0644"
   owner: "root"
@@ -61,6 +72,15 @@ write_files:
     RestartSec=10
     RemainAfterExit=yes
     ExecStart=/usr/local/bin/health-monitor.sh kubelet
+
+- path: "/etc/systemd/system/docker-monitor.timer"
+  permissions: "0644"
+  owner: "root"
+  content: |
+    [Unit]
+    Description=a timer that delays docker-monitor from starting too soon after boot
+    [Timer]
+    OnBootSec=30min
     [Install]
     WantedBy=multi-user.target
 
@@ -76,8 +96,6 @@ write_files:
     RestartSec=10
     RemainAfterExit=yes
     ExecStart=/usr/local/bin/health-monitor.sh container-runtime
-    [Install]
-    WantedBy=multi-user.target
 
 {{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}}
     {{if not .MasterProfile.IsCoreOS}}