[Services] Restart Platform-monitor service upon unexpected critical …

…process exit. (#3689) Signed-off-by: Yong Zhao <[email protected]>
sonic-net · Nov 5, 2019 · 4fa3a1e · 4fa3a1e
1 parent d85777b
commit 4fa3a1e
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 0 deletions.
diff --git a/dockers/docker-platform-monitor/Dockerfile.j2 b/dockers/docker-platform-monitor/Dockerfile.j2
@@ -57,5 +57,7 @@ RUN apt-get purge -y           \
 COPY ["docker_init.sh", "lm-sensors.sh", "/usr/bin/"]
 COPY ["docker-pmon.supervisord.conf.j2", "start.sh.j2", "/usr/share/sonic/templates/"]
 COPY ["ssd_tools/*", "/usr/bin/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 
 ENTRYPOINT ["/usr/bin/docker_init.sh"]
diff --git a/dockers/docker-platform-monitor/critical_processes b/dockers/docker-platform-monitor/critical_processes
@@ -0,0 +1,5 @@
+fancontrol
+ledd
+xcvrd
+psud
+syseepromd
diff --git a/dockers/docker-platform-monitor/docker-pmon.supervisord.conf.j2 b/dockers/docker-platform-monitor/docker-pmon.supervisord.conf.j2
@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1

diff --git a/files/build_templates/pmon.service.j2 b/files/build_templates/pmon.service.j2
@@ -6,12 +6,16 @@ After=updategraph.service
 After=syncd.service
 {% endif %}
 Before=ntp-config.service
+StartLimitIntervalSec=1200
+StartLimitBurst=3
 
 [Service]
 User={{ sonicadmin_user }}
 ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+Restart=always
+RestartSec=30
 
 [Install]
 WantedBy=multi-user.target
diff --git a/rules/docker-platform-monitor.mk b/rules/docker-platform-monitor.mk
@@ -44,3 +44,4 @@ $(DOCKER_PLATFORM_MONITOR)_BASE_IMAGE_FILES += cmd_wrapper:/usr/bin/sensors
 $(DOCKER_PLATFORM_MONITOR)_BASE_IMAGE_FILES += cmd_wrapper:/usr/sbin/smartctl
 $(DOCKER_PLATFORM_MONITOR)_BASE_IMAGE_FILES += cmd_wrapper:/usr/sbin/iSmart
 $(DOCKER_PLATFORM_MONITOR)_BASE_IMAGE_FILES += cmd_wrapper:/usr/sbin/SmartCmd
+$(DOCKER_PLATFORM_MONITOR)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)