From fdfbd1a0b0b842f55ed72cedb57dab8b48ad3b5f Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Tue, 16 Oct 2018 17:08:35 -0700 Subject: [PATCH] delay docker and kubelet health monitors for 30 mins (#4050) --- parts/k8s/kubernetesagentcustomdata.yml | 22 ++++++++++++++++++-- parts/k8s/kubernetesconfigs.sh | 10 +++++++-- parts/k8s/kubernetesmastercustomdata.yml | 22 ++++++++++++++++++-- parts/k8s/kubernetesmastercustomdatavmss.yml | 22 ++++++++++++++++++-- 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml index 70c4587f3e..2b6fdb4e07 100644 --- a/parts/k8s/kubernetesagentcustomdata.yml +++ b/parts/k8s/kubernetesagentcustomdata.yml @@ -43,6 +43,17 @@ write_files: content: !!binary | {{WrapAsVariable "healthMonitorScript"}} +- path: "/etc/systemd/system/kubelet-monitor.timer" + permissions: "0644" + owner: "root" + content: | + [Unit] + Description=a timer that delays kubelet-monitor from starting too soon after boot + [Timer] + OnBootSec=30min + [Install] + WantedBy=multi-user.target + - path: "/etc/systemd/system/kubelet-monitor.service" permissions: "0644" owner: "root" @@ -55,6 +66,15 @@ write_files: RestartSec=10 RemainAfterExit=yes ExecStart=/usr/local/bin/health-monitor.sh kubelet + +- path: "/etc/systemd/system/docker-monitor.timer" + permissions: "0644" + owner: "root" + content: | + [Unit] + Description=a timer that delays docker-monitor from starting too soon after boot + [Timer] + OnBootSec=30min [Install] WantedBy=multi-user.target @@ -70,8 +90,6 @@ write_files: RestartSec=10 RemainAfterExit=yes ExecStart=/usr/local/bin/health-monitor.sh container-runtime - [Install] - WantedBy=multi-user.target {{if .KubernetesConfig.RequiresDocker}} {{if not .IsCoreOS}} diff --git a/parts/k8s/kubernetesconfigs.sh b/parts/k8s/kubernetesconfigs.sh index dc6c0b292d..4f9c7a8f1c 100755 --- a/parts/k8s/kubernetesconfigs.sh +++ b/parts/k8s/kubernetesconfigs.sh @@ -245,9 +245,12 @@ function ensureDocker() { DOCKER_JSON_FILE=/etc/docker/daemon.json wait_for_file 1200 1 $DOCKER_JSON_FILE || exit $ERR_FILE_WATCH_TIMEOUT systemctlEnableAndStart docker + # Delay start of docker-monitor for 30 mins after booting + DOCKER_MONITOR_SYSTEMD_TIMER_FILE=/etc/systemd/system/docker-monitor.timer + wait_for_file 1200 1 $DOCKER_MONITOR_SYSTEMD_TIMER_FILE || exit $ERR_FILE_WATCH_TIMEOUT DOCKER_MONITOR_SYSTEMD_FILE=/etc/systemd/system/docker-monitor.service wait_for_file 1200 1 $DOCKER_MONITOR_SYSTEMD_FILE || exit $ERR_FILE_WATCH_TIMEOUT - systemctlEnableAndStart docker-monitor || exit $ERR_SYSTEMCTL_START_FAIL + systemctlEnableAndStart docker-monitor.timer || exit $ERR_SYSTEMCTL_START_FAIL } function ensureKMS() { systemctlEnableAndStart kms || exit $ERR_SYSTEMCTL_START_FAIL @@ -261,9 +264,12 @@ function ensureKubelet() { KUBELET_RUNTIME_CONFIG_SCRIPT_FILE=/opt/azure/containers/kubelet.sh wait_for_file 1200 1 $KUBELET_RUNTIME_CONFIG_SCRIPT_FILE || exit $ERR_FILE_WATCH_TIMEOUT systemctlEnableAndStart kubelet || exit $ERR_KUBELET_START_FAIL + # Delay start of kubelet-monitor for 30 mins after booting + KUBELET_MONITOR_SYSTEMD_TIMER_FILE=/etc/systemd/system/kubelet-monitor.timer + wait_for_file 1200 1 $KUBELET_MONITOR_SYSTEMD_TIMER_FILE || exit $ERR_FILE_WATCH_TIMEOUT KUBELET_MONITOR_SYSTEMD_FILE=/etc/systemd/system/kubelet-monitor.service wait_for_file 1200 1 $KUBELET_MONITOR_SYSTEMD_FILE || exit $ERR_FILE_WATCH_TIMEOUT - systemctlEnableAndStart kubelet-monitor || exit $ERR_SYSTEMCTL_START_FAIL + systemctlEnableAndStart kubelet-monitor.timer || exit $ERR_SYSTEMCTL_START_FAIL } function ensureJournal(){ diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml index c0e68720d7..9329f59c73 100644 --- a/parts/k8s/kubernetesmastercustomdata.yml +++ b/parts/k8s/kubernetesmastercustomdata.yml @@ -49,6 +49,17 @@ write_files: content: !!binary | {{WrapAsVariable "healthMonitorScript"}} +- path: "/etc/systemd/system/kubelet-monitor.timer" + permissions: "0644" + owner: "root" + content: | + [Unit] + Description=a timer that delays kubelet-monitor from starting too soon after boot + [Timer] + OnBootSec=30min + [Install] + WantedBy=multi-user.target + - path: "/etc/systemd/system/kubelet-monitor.service" permissions: "0644" owner: "root" @@ -61,6 +72,15 @@ write_files: RestartSec=10 RemainAfterExit=yes ExecStart=/usr/local/bin/health-monitor.sh kubelet + +- path: "/etc/systemd/system/docker-monitor.timer" + permissions: "0644" + owner: "root" + content: | + [Unit] + Description=a timer that delays docker-monitor from starting too soon after boot + [Timer] + OnBootSec=30min [Install] WantedBy=multi-user.target @@ -76,8 +96,6 @@ write_files: RestartSec=10 RemainAfterExit=yes ExecStart=/usr/local/bin/health-monitor.sh container-runtime - [Install] - WantedBy=multi-user.target {{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}} {{if not .MasterProfile.IsCoreOS}} diff --git a/parts/k8s/kubernetesmastercustomdatavmss.yml b/parts/k8s/kubernetesmastercustomdatavmss.yml index 6af3827d21..b5dc24e472 100644 --- a/parts/k8s/kubernetesmastercustomdatavmss.yml +++ b/parts/k8s/kubernetesmastercustomdatavmss.yml @@ -49,6 +49,17 @@ write_files: content: !!binary | {{WrapAsVariable "healthMonitorScript"}} +- path: "/etc/systemd/system/kubelet-monitor.timer" + permissions: "0644" + owner: "root" + content: | + [Unit] + Description=a timer that delays kubelet-monitor from starting too soon after boot + [Timer] + OnBootSec=30min + [Install] + WantedBy=multi-user.target + - path: "/etc/systemd/system/kubelet-monitor.service" permissions: "0644" owner: "root" @@ -61,6 +72,15 @@ write_files: RestartSec=10 RemainAfterExit=yes ExecStart=/usr/local/bin/health-monitor.sh kubelet + +- path: "/etc/systemd/system/docker-monitor.timer" + permissions: "0644" + owner: "root" + content: | + [Unit] + Description=a timer that delays docker-monitor from starting too soon after boot + [Timer] + OnBootSec=30min [Install] WantedBy=multi-user.target @@ -76,8 +96,6 @@ write_files: RestartSec=10 RemainAfterExit=yes ExecStart=/usr/local/bin/health-monitor.sh container-runtime - [Install] - WantedBy=multi-user.target {{if .OrchestratorProfile.KubernetesConfig.RequiresDocker}} {{if not .MasterProfile.IsCoreOS}}