From 346789e8fdf475d4aba75d0d48f88147cd0b1de1 Mon Sep 17 00:00:00 2001 From: Jake Hunsaker Date: Fri, 18 Oct 2024 15:22:30 -0400 Subject: [PATCH] [coredump] Refactor info and dump collection Refactor the plugin to adjust how `coredumpctl info` and coredump file collection is handled. Plugin will now collect the compressed coredump file for coredump entries for which coredumpctl reports that the file is present, for the first X number of coredump files where X is the new `dumps` plugin option. A second new plugin option, `executable`, can be used to specific a regex string to match coredump entries against to determine if they should be collected or not. The default behavior is to collect for all entries. A symlink for coredump file collections will be dropped in the plugin directory to aide in associating dump files with coredumpctl entries, as it may not always be obvious for end users based on filenames alone. Signed-off-by: Jake Hunsaker --- sos/report/plugins/coredump.py | 85 ++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/sos/report/plugins/coredump.py b/sos/report/plugins/coredump.py index 1db1794d4f..cecd538845 100644 --- a/sos/report/plugins/coredump.py +++ b/sos/report/plugins/coredump.py @@ -8,20 +8,50 @@ # # See the LICENSE file in the source distribution for further information. +import os +import re from sos.report.plugins import Plugin, IndependentPlugin, PluginOpt class Coredump(Plugin, IndependentPlugin): + """ + This plugin collects from the systemd-coredump service, which is leveraged + via the coredumpctl command. - short_desc = 'Retrieve coredump information' + Beyond basic configuration of the service, the plugin will also attempt to + collect information on available coredumps. For any coredumps reported by + coredumpctl for which the core is present, this plugin will collect that + coredump file. The number of dump files collects is controlled by the + `dumps` option, defaulting to 5. Note that core collection is performed + based on the most recent cores generated. The summary information provided + by `coredumpctl info` is also collected. + + The coredump files will be at their canonical location in the sos archive, + usually under var/lib/systemd/coredump. Symlinks are dropped in the plugin + directory to assist in identifying which coredump aligns to which entry in + the provided `coredumpctl list` output collection. + + Users may leverage the `executable` option to control which coredumps the + plugin will collect. This option takes a case-insensitive python regex + string. If provided, only coredump entries for which the EXE field in + `coredumpctl list` output matches will be collected. + + The dump files collected are compressed, and users should be aware that + when inflated these files can be orders of magnitude larger than their + collected sizes. + """ + + short_desc = 'systemd-coredump related information and dump files' plugin_name = "coredump" profiles = ('system', 'debug') packages = ('systemd-udev', 'systemd-coredump') option_list = [ - PluginOpt("detailed", default=False, - desc="collect detailed information for every report") + PluginOpt("dumps", default=5, desc="number of dump files to collect"), + PluginOpt("executable", default='', + desc=("only collect info and dump output for executables " + "matching this regex")) ] def setup(self): @@ -29,15 +59,50 @@ def setup(self): "/etc/systemd/coredump.conf", "/etc/systemd/coredump.conf.d/", "/run/systemd/coredump.conf.d/", - "/usr/lib/systemd/coredump.conf.d/" + "/usr/lib/systemd/coredump.conf.d/", + "/usr/lib/systemd/systemd-coredump" ]) - self.add_cmd_output("coredumpctl dump") + coredump_list = self.collect_cmd_output("coredumpctl list --reverse") + if coredump_list['status'] != 0: + return - coredump_list = self.collect_cmd_output("coredumpctl list") - if self.get_option("detailed") and coredump_list['status'] == 0: - for line in coredump_list["output"].splitlines()[1:]: - self.add_cmd_output("coredumpctl info " - f"{line.split()[4]}") + cores_collected = 0 + for line in coredump_list["output"].splitlines()[1:]: + cdump = line.split() + pid = cdump[4] + exe = cdump[-2] + if regex := self.get_option("executable"): + if not re.search(regex, exe, re.I): + continue + cinfo = self.collect_cmd_output(f"coredumpctl info {pid}") + if cinfo['status'] != 0: + continue + res = cinfo['output'] + if cores_collected < self.get_option("dumps"): + core = re.search(r"(^\s*Storage:(.*)(\(present\)))", res, re.M) + try: + core_path = core.groups()[1].strip() + self.add_copy_spec(core_path, tailit=False, sizelimit=100) + plugpath = self.path_join( + self.commons['cmddir'], + self.name(), + f"coredump_{pid}_{self._mangle_command(exe)}" + ) + # use os.path instead of the plugin wrapper here so that + # we always get the real path of the archive on disk + arcpath = os.path.join(self.archive.get_archive_path(), + core_path) + linkpath = os.path.relpath(arcpath, plugpath) + linkpath = linkpath.replace('../', '', 1) + self.archive.add_link(linkpath, plugpath) + cores_collected += 1 + except AttributeError: + # no match on the re.search() + pass + except Exception as err: + self._log_info( + f"Could not collect coredump for {pid} : {err}" + ) # vim: set et ts=4 sw=4 :