From 3c5b6e09c1bdcb65a2c4cf17f61c1be6b6e27ba8 Mon Sep 17 00:00:00 2001 From: Rajendra Dendukuri Date: Thu, 19 Sep 2019 11:49:24 -0700 Subject: [PATCH 1/2] SONiC core dump utility - Install systemd-coredump in base o/s - Remove existing simple coredump facility - Enable persistent journald to store coredump history - Minimal default coredump configuration - Added a coredump-config service to generate coredump configuration Core files generated by kernel are created by the host o/s and also stored on host o/s. This applies for processes running inside container as well. Containers do not have access to journal as well as core files. Containers are supposed to have limited access to host o/s and core files and journal may contain sensitive information. Toimprove debugging of crashes inside a container following changes are made: - when INSTALL_DEBUG_TOOLS=y is set in the build. systemd-coredump tool is installed in all containers beside gdb - When SONIC_DEBUGGING_ON=y is set in the build, /var/log/journal and /var/lib/systemd/coredump are mapped inside container To inspect a core file, from a container shell, issue below commands docker exec -ti /bin/bash --- build_debian.sh | 4 ++-- files/build_templates/docker_image_ctl.j2 | 5 +++++ .../build_templates/sonic_debian_extension.j2 | 18 ++++++++++++++++++ .../coredump/coredump-config.service | 11 +++++++++++ files/image_config/coredump/coredump-config.sh | 14 ++++++++++++++ .../coredump.conf.d/00-sonic-coredump.conf | 5 +++++ .../journald.conf.d/00-sonic-journald.conf | 5 +++++ rules/docker-base-stretch.mk | 4 +++- slave.mk | 1 + 9 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 files/image_config/coredump/coredump-config.service create mode 100755 files/image_config/coredump/coredump-config.sh create mode 100644 files/image_config/coredump/coredump.conf.d/00-sonic-coredump.conf create mode 100644 files/image_config/journald/journald.conf.d/00-sonic-journald.conf diff --git a/build_debian.sh b/build_debian.sh index 5c7a9344f43c..ce3a8740184e 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -346,9 +346,7 @@ sudo cp files/image_config/monit/monitrc $FILESYSTEM_ROOT/etc/monit/ sudo chmod 600 $FILESYSTEM_ROOT/etc/monit/monitrc ## Config sysctl -sudo mkdir -p $FILESYSTEM_ROOT/var/core sudo augtool --autosave " -set /files/etc/sysctl.conf/kernel.core_pattern '|/usr/bin/coredump-compress %e %t %p' set /files/etc/sysctl.conf/kernel.softlockup_panic 1 set /files/etc/sysctl.conf/kernel.panic 10 @@ -429,6 +427,8 @@ sudo cp files/dhcp/dhclient.conf $FILESYSTEM_ROOT/etc/dhcp/ if [ -f files/image_config/ntp/ntp ]; then sudo cp ./files/image_config/ntp/ntp $FILESYSTEM_ROOT/etc/init.d/ fi +## Configure application core dump handler +sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get install -y systemd-coredump ## Version file sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index 718730f813ef..b15788161248 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -227,6 +227,11 @@ start() { {%- else %} --tmpfs /tmp \ {%- endif %} +{%- endif %} +{%- if sonic_debugging_on == "y" %} + -v /var/log/journal:/var/log/journal:ro \ + -v /var/lib/systemd/coredump:/var/lib/systemd/coredump:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ {%- endif %} -v /var/run/redis:/var/run/redis:rw \ -v /usr/share/sonic/device/$PLATFORM:/usr/share/sonic/platform:ro \ diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index bcdecaa71766..e917962b248d 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -240,6 +240,24 @@ sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd.service $FILESYSTEM_ROOT/etc/systemd/s echo "caclmgrd.service" | sudo tee -a $GENERATED_SERVICE_FILE sudo cp $IMAGE_CONFIGS/caclmgrd/caclmgrd $FILESYSTEM_ROOT/usr/bin/ +# Allow systemd-coredump to perform cleanup of core files and not tmpfiles.d +sudo sed -i "/\/var\/lib\/systemd\/coredump/d" $FILESYSTEM_ROOT/usr/lib/tmpfiles.d/systemd.conf + +# Customize systemd-coredump configuration +sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/coredump.conf.d +sudo cp $IMAGE_CONFIGS/coredump/coredump.conf.d/00-sonic-coredump.conf $FILESYSTEM_ROOT/etc/systemd/coredump.conf.d +# Setup service to configure coredump service +sudo cp $IMAGE_CONFIGS/coredump/coredump-config.service $FILESYSTEM_ROOT/etc/systemd/system/ +sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable coredump-config.service +sudo cp $IMAGE_CONFIGS/coredump/coredump-config.sh $FILESYSTEM_ROOT/usr/bin/ + +## Enable persistent journal to store coredump history +sudo mkdir -p $FILESYSTEM_ROOT/etc/systemd/journald.conf.d/ +sudo cp files/image_config/journald/journald.conf.d/00-sonic-journald.conf $FILESYSTEM_ROOT/etc/systemd/journald.conf.d/ + +## Shortcut to access core files +sudo ln -sf /var/lib/systemd/coredump $FILESYSTEM_ROOT/var/core + # Copy process-reboot-cause service files sudo cp $IMAGE_CONFIGS/process-reboot-cause/process-reboot-cause.service $FILESYSTEM_ROOT/etc/systemd/system/ echo "process-reboot-cause.service" | sudo tee -a $GENERATED_SERVICE_FILE diff --git a/files/image_config/coredump/coredump-config.service b/files/image_config/coredump/coredump-config.service new file mode 100644 index 000000000000..9aa5206ee333 --- /dev/null +++ b/files/image_config/coredump/coredump-config.service @@ -0,0 +1,11 @@ +[Unit] +Description=Update coredump configuration +Requires=updategraph.service +After=updategraph.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/coredump-config.sh + +[Install] +WantedBy=multi-user.target diff --git a/files/image_config/coredump/coredump-config.sh b/files/image_config/coredump/coredump-config.sh new file mode 100755 index 000000000000..ff08c4468577 --- /dev/null +++ b/files/image_config/coredump/coredump-config.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +DISABLE_COREDUMP_CONF="/etc/sysctl.d/50-disable-coredump.conf" + +if [ "$(redis-cli -n 4 HGET "COREDUMP|config" "enabled")" = "false" ] ; then + echo "kernel.core_pattern=" > ${DISABLE_COREDUMP_CONF} +else + rm -f ${DISABLE_COREDUMP_CONF} +fi + +# Read sysctl conf files again +systemctl restart systemd-sysctl + +exit 0 diff --git a/files/image_config/coredump/coredump.conf.d/00-sonic-coredump.conf b/files/image_config/coredump/coredump.conf.d/00-sonic-coredump.conf new file mode 100644 index 000000000000..5b08f7a0d185 --- /dev/null +++ b/files/image_config/coredump/coredump.conf.d/00-sonic-coredump.conf @@ -0,0 +1,5 @@ +[Coredump] +Storage=external +Compress=yes +ProcessSizeMax=2G +ExternalSizeMax=2G diff --git a/files/image_config/journald/journald.conf.d/00-sonic-journald.conf b/files/image_config/journald/journald.conf.d/00-sonic-journald.conf new file mode 100644 index 000000000000..1055746ad8c4 --- /dev/null +++ b/files/image_config/journald/journald.conf.d/00-sonic-journald.conf @@ -0,0 +1,5 @@ +[Journal] +Storage=persistent +SystemMaxUse=256M +RuntimeMaxUse=356M +MaxLevelStore=crit diff --git a/rules/docker-base-stretch.mk b/rules/docker-base-stretch.mk index 55b7fd9f8661..6405e0d9e3fb 100644 --- a/rules/docker-base-stretch.mk +++ b/rules/docker-base-stretch.mk @@ -11,7 +11,9 @@ VIM = vim OPENSSH = openssh-client SSHPASS = sshpass STRACE = strace -$(DOCKER_BASE_STRETCH)_DBG_IMAGE_PACKAGES += $(GDB) $(GDBSERVER) $(VIM) $(OPENSSH) $(SSHPASS) $(STRACE) +SYSTEMD_COREDUMP = systemd-coredump +$(DOCKER_BASE_STRETCH)_DBG_IMAGE_PACKAGES += $(GDB) $(GDBSERVER) $(VIM) $(OPENSSH) $(SSHPASS) $(STRACE) \ + $(SYSTEMD_COREDUMP) SONIC_DOCKER_IMAGES += $(DOCKER_BASE_STRETCH) SONIC_STRETCH_DOCKERS += $(DOCKER_BASE_STRETCH) diff --git a/slave.mk b/slave.mk index dc13e43f62b2..4841a1a7fa30 100644 --- a/slave.mk +++ b/slave.mk @@ -637,6 +637,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export sonic_asic_platform="$(patsubst %-$(CONFIGURED_ARCH),%,$(CONFIGURED_PLATFORM))" export enable_organization_extensions="$(ENABLE_ORGANIZATION_EXTENSIONS)" export enable_dhcp_graph_service="$(ENABLE_DHCP_GRAPH_SERVICE)" + export sonic_debugging_on="$(SONIC_DEBUGGING_ON)" export shutdown_bgp_on_start="$(SHUTDOWN_BGP_ON_START)" export enable_pfcwd_on_start="$(ENABLE_PFCWD_ON_START)" export installer_debs="$(addprefix $(STRETCH_DEBS_PATH)/,$($*_INSTALLS))" From 130a7d4a4c062763677d777e304387a3648a436e Mon Sep 17 00:00:00 2001 From: Rajendra Dendukuri Date: Fri, 15 Nov 2019 14:30:39 -0500 Subject: [PATCH 2/2] Use install_debug_image=y as the condition to map coredump directory into a container --- files/build_templates/docker_image_ctl.j2 | 8 +++----- slave.mk | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index b15788161248..ee6ff5fd3eee 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -211,6 +211,9 @@ start() { docker create {{docker_image_run_opt}} \ {%- if install_debug_image == "y" %} -v /src:/src:ro -v /debug:/debug:rw \ + -v /var/log/journal:/var/log/journal:ro \ + -v /var/lib/systemd/coredump:/var/lib/systemd/coredump:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ {%- endif %} {%- if '--log-driver=json-file' in docker_image_run_opt or '--log-driver' not in docker_image_run_opt %} --log-opt max-size=2M --log-opt max-file=5 \ @@ -227,11 +230,6 @@ start() { {%- else %} --tmpfs /tmp \ {%- endif %} -{%- endif %} -{%- if sonic_debugging_on == "y" %} - -v /var/log/journal:/var/log/journal:ro \ - -v /var/lib/systemd/coredump:/var/lib/systemd/coredump:ro \ - -v /etc/machine-id:/etc/machine-id:ro \ {%- endif %} -v /var/run/redis:/var/run/redis:rw \ -v /usr/share/sonic/device/$PLATFORM:/usr/share/sonic/platform:ro \ diff --git a/slave.mk b/slave.mk index 4841a1a7fa30..dc13e43f62b2 100644 --- a/slave.mk +++ b/slave.mk @@ -637,7 +637,6 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export sonic_asic_platform="$(patsubst %-$(CONFIGURED_ARCH),%,$(CONFIGURED_PLATFORM))" export enable_organization_extensions="$(ENABLE_ORGANIZATION_EXTENSIONS)" export enable_dhcp_graph_service="$(ENABLE_DHCP_GRAPH_SERVICE)" - export sonic_debugging_on="$(SONIC_DEBUGGING_ON)" export shutdown_bgp_on_start="$(SHUTDOWN_BGP_ON_START)" export enable_pfcwd_on_start="$(ENABLE_PFCWD_ON_START)" export installer_debs="$(addprefix $(STRETCH_DEBS_PATH)/,$($*_INSTALLS))"