From 9a7726bd38bb0d05c71cc13fba4b87d6df44cc6f Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Fri, 26 Jul 2024 11:15:20 -0700 Subject: [PATCH 01/14] debuginfo: mark OL8 UEK6 aarch64 as compatible The OL8 UEK6 x86_64 builds used a broken CTF toolchain for a long time, resulting in broken CTF. However, the toolchain used to build aarch64 was newer and not affected by the bug. Mark it as compatible so we can use drgn with CTF on aarch64 for this release. Signed-off-by: Stephen Brennan --- drgn_tools/debuginfo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drgn_tools/debuginfo.py b/drgn_tools/debuginfo.py index 1c30995d..b7eda953 100644 --- a/drgn_tools/debuginfo.py +++ b/drgn_tools/debuginfo.py @@ -680,10 +680,12 @@ def get( # For OL8, UEK6, the CTF generation process produced buggy data. The # data was fixed starting in 5.4.17-2136.323.1: all prior versions are - # fully broken. + # fully broken. This is specific to x86_64: the aarch64 build used a + # different toolchain which was not affected. if ( kver.ol_version == 8 and kver.uek_version == 6 + and kver.arch == "x86_64" and kver.release_tuple < (2136, 323, 1) ): return cls.NO From 932f2db57dee3cfa4b9ff25ddce78d7e275c96dd Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Mon, 12 Aug 2024 10:51:48 -0700 Subject: [PATCH 02/14] debuginfo: cache "using_ctf" so we know module debuginfo is loaded For module debuginfo, we use a heuristic to detect whether debuginfo is loaded. This can fail and it's also extra overhead. When we load CTF, we know that we've got debuginfo for all modules. Cache this fact and use it to special case the module_has_debuginfo() check. This fixes test failures which happen due to a failed check. The test failures result in rather cryptic errors due to missing type fields. EG: AttributeError: 'struct files_struct' has no member 'fdt' These get caused because when CTF is loaded, the failed check will result in the debuginfo code loading a DWARF module debuginfo file, which can cause conflicts in the type lookups. Signed-off-by: Stephen Brennan --- drgn_tools/cli.py | 1 + drgn_tools/corelens.py | 1 + drgn_tools/module.py | 2 ++ tests/conftest.py | 1 + 4 files changed, 5 insertions(+) diff --git a/drgn_tools/cli.py b/drgn_tools/cli.py index c36b3de9..c6809320 100644 --- a/drgn_tools/cli.py +++ b/drgn_tools/cli.py @@ -90,6 +90,7 @@ def _set_debuginfo( ctf_path = _get_ctf_path(release, args) if ctf_path and HAVE_CTF: load_ctf(prog, ctf_path) + prog.cache["using_ctf"] = True return "CTF", ctf_path elif ctf_path: problems.append( diff --git a/drgn_tools/corelens.py b/drgn_tools/corelens.py index 9518e732..f485d7db 100644 --- a/drgn_tools/corelens.py +++ b/drgn_tools/corelens.py @@ -430,6 +430,7 @@ def _load_prog_and_debuginfo(args: argparse.Namespace) -> Tuple[Program, bool]: path = args.ctf or f"/lib/modules/{release}/kernel/vmlinux.ctfa" if os.path.isfile(path) and _check_ctf_compat(release, args.vmcore): load_ctf(prog, path) + prog.cache["using_ctf"] = True return prog, True except ModuleNotFoundError: pass diff --git a/drgn_tools/module.py b/drgn_tools/module.py index 81e5e5a3..b99e3426 100644 --- a/drgn_tools/module.py +++ b/drgn_tools/module.py @@ -473,6 +473,8 @@ def module_has_debuginfo(module: Object) -> bool: imperfect heuristic. In the future, drgn will have an API which lets us enumerate modules directly query whether it has loaded debuginfo. """ + if module.prog_.cache.get("using_ctf"): + return True fst = _first_kallsyms_symbol(module) if not fst: return False diff --git a/tests/conftest.py b/tests/conftest.py index 525a66e0..d1a646e3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,7 @@ def prog() -> drgn.Program: from drgn.helpers.linux.ctf import load_ctf load_ctf(p, path=CTF_FILE) + p.cache["using_ctf"] = True except ModuleNotFoundError: raise Exception("CTF is not supported, cannot run CTF test") elif DEBUGINFO: From d298b7559152c8152d71c0a937f28ae09008a267 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Mon, 12 Aug 2024 11:13:36 -0700 Subject: [PATCH 03/14] tests: mm: don't fail test over missing variable Signed-off-by: Stephen Brennan --- tests/test_mm.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_mm.py b/tests/test_mm.py index 97648541..4a05d917 100644 --- a/tests/test_mm.py +++ b/tests/test_mm.py @@ -59,8 +59,17 @@ def test_AddrKind_categorize_data(prog: drgn.Program) -> None: assert mm.AddrKind.categorize(prog, rodata) == mm.AddrKind.RODATA # kernel/panic.c: uninitialized - bss = prog.symbol("panic_on_taint").address - assert mm.AddrKind.categorize(prog, bss) == mm.AddrKind.BSS + try: + bss = prog.symbol("panic_on_taint").address + except LookupError: + # panic_on_taint was added in 5.8, 77cb8f12fc6e9 ("kernel: add + # panic_on_taint"), which was backported to stable kernels. Don't fail + # the test if it's not found. However, some vmcores in our test suite + # are incredibly old UEK versions from before these stable backports. + # Don't fail the tests just because this variable is not found. + pass + else: + assert mm.AddrKind.categorize(prog, bss) == mm.AddrKind.BSS # percpu pcpu = prog.symbol("runqueues").address From 84f28a551dede16dbddfca2c80b8c0ebbe87a4b9 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Mon, 12 Aug 2024 11:14:00 -0700 Subject: [PATCH 04/14] tests: fix the requirements to allow drgn 0.0.27 Signed-off-by: Stephen Brennan --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 47cb3c3d..e26bb498 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ -drgn>=0.0.25,<0.0.27 +drgn>=0.0.25,<0.0.28 pytest<7.1 From ba567b0621db039d9f7cf8c78021778eca0a411e Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Mon, 12 Aug 2024 11:15:44 -0700 Subject: [PATCH 05/14] lsmod: fix typo Signed-off-by: Stephen Brennan --- drgn_tools/lsmod.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drgn_tools/lsmod.py b/drgn_tools/lsmod.py index 0b15d7ee..f760d5ec 100644 --- a/drgn_tools/lsmod.py +++ b/drgn_tools/lsmod.py @@ -74,9 +74,7 @@ def print_module_summary(prog: Program) -> None: """Print a list of module details and dependencies""" # List all loaded modules table_value = [] - table_value.append( - ["MODULE", "NAME", "SIZE", "REF", "DEPENDENDENT MODULES"] - ) + table_value.append(["MODULE", "NAME", "SIZE", "REF", "DEPENDENT MODULES"]) for mod in KernelModule.all(prog): dep_mod = [] for depuse in for_each_module_use(mod.obj.source_list.address_of_()): From 65aea4648961f4152510686001ee330f65de13ca Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Thu, 25 Jul 2024 18:01:39 -0700 Subject: [PATCH 06/14] bt_has_any: fall back to symbol name when indexing We currently use the "frame.name" which may not be present when using CTF. Instead, we should fallback to the symbol name. The frame_name() function is not appropriate, since this may include additional information, such as the module containing the symbol. So introduce a new helper, func_name() for this case. Signed-off-by: Stephen Brennan --- drgn_tools/bt.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/drgn_tools/bt.py b/drgn_tools/bt.py index 518a7755..bf45e778 100644 --- a/drgn_tools/bt.py +++ b/drgn_tools/bt.py @@ -37,6 +37,15 @@ ) +def func_name(prog: drgn.Program, frame: drgn.StackFrame) -> t.Optional[str]: + if frame.name: + return frame.name + try: + return frame.symbol().name + except LookupError: + return None + + def frame_name(prog: drgn.Program, frame: drgn.StackFrame) -> str: """Return a suitable name for a stack frame""" # Looking up the module for an address is currently a bit inefficient, since @@ -473,7 +482,10 @@ def _index_functions(prog: drgn.Program) -> t.Dict[str, t.Set[int]]: try: frames = bt_frames(task) for frame in frames: - func_to_pids[frame.name].add(pid) + name = func_name(prog, frame) + if not name: + continue + func_to_pids[name].add(pid) except FaultError: # FaultError: catch unusual unwinding issues pass @@ -496,7 +508,8 @@ def _indexed_bt_has_any( for pid in pids: task = find_task(prog, pid) for frame in bt_frames(task): - if frame.name in funcs: + name = func_name(prog, frame) + if name in funcs: result.append((task, frame)) return result @@ -532,7 +545,7 @@ def bt_has_any( try: frames = bt_frames(task) for frame in frames: - if frame.name in funcs: + if func_name(prog, frame) in funcs: frame_list.append((task, frame)) return frame_list @@ -546,7 +559,7 @@ def bt_has_any( try: frames = bt_frames(task) for frame in frames: - if frame.name in funcs: + if func_name(prog, frame) in funcs: frame_list.append((task, frame)) except (FaultError, ValueError): # FaultError: catch unusual unwinding issues From c4c2117bb4771e1bf8cc3791a80a0860c8a19b65 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Fri, 26 Jul 2024 11:33:23 -0700 Subject: [PATCH 07/14] bt: use frame pointer when formatting aarch64 stack frames aarch64 doesn't guarantee provide the value of SP when doing a frame pointer based unwind. Don't fail on this: instead fall back to FP. Fixes internal issue tracked as LSE-374. Signed-off-by: Stephen Brennan --- drgn_tools/bt.py | 28 ++++++++++++++++++++++++++-- tests/test_bt.py | 5 ----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/drgn_tools/bt.py b/drgn_tools/bt.py index bf45e778..7349b523 100644 --- a/drgn_tools/bt.py +++ b/drgn_tools/bt.py @@ -5,6 +5,7 @@ import typing as t import drgn +from drgn import Architecture from drgn import FaultError from drgn import Object from drgn import Program @@ -159,6 +160,10 @@ def expand_traces(trace: drgn.StackTrace) -> t.List[drgn.StackTrace]: # We should continue appending traces so long as (a) we can find a pt_regs, # and (b) the stack pointer for that pt_regs is different than the stack # pointer for the current stack. + # + # NOTE: aarch64 does not guarantee having SP if we're unwinding with frame + # pointers. However, trace[0] always has SP, because we generally have a + # full register set to start the trace. Thus, this should be safe to test. while pt_regs is not None and pt_regs.sp.value_() != trace[0].sp: # Interrupted user address. if ( @@ -288,9 +293,28 @@ def print_frames( :param start_idx: Where to start counting the frame indices from :param indent: How many spaces to indent the output """ + # On aarch64 without DWARF, it seems we're not guaranteed to have the stack + # pointer, or the frame pointer. Fallback to FP, then NULL, here so we don't + # crash during unwinds. + if prog.platform.arch == Architecture.AARCH64: + + def get_sp(frame: drgn.StackFrame) -> int: + try: + return frame.sp + except LookupError: + try: + return frame.register("fp") + except LookupError: + return 0 + + else: + + def get_sp(frame: drgn.StackFrame) -> int: + return frame.sp + pfx = " " * indent for i, frame in enumerate(trace): - sp = frame.sp # drgn 0.0.22 + sp = get_sp(frame) intr = "!" if frame.interrupted else " " try: pc = hex(frame.pc) @@ -313,7 +337,7 @@ def print_frames( # with a different stack pointer than the previous. That is: only # when we reach the frame for a non-inline function. Also, only # output registers when we have show_vars=True. - if i == len(trace) - 1 or trace[i].sp != trace[i + 1].sp: + if i == len(trace) - 1 or sp != get_sp(trace[i + 1]): registers = frame.registers() regnames = list(registers.keys()) # This formats the registers in three columns. diff --git a/tests/test_bt.py b/tests/test_bt.py index 3159fc6b..be77c3db 100644 --- a/tests/test_bt.py +++ b/tests/test_bt.py @@ -8,11 +8,6 @@ @pytest.mark.skip_vmcore("*-uek4") def test_bt_smoke(prog, request, debuginfo_type): - if ( - debuginfo_type == "ctf" - and prog.platform.arch == drgn.Architecture.AARCH64 - ): - pytest.xfail("still unsupported for unwinding with aarch64 + CTF") if prog.flags & drgn.ProgramFlags.IS_LIVE: thread = prog.thread(1) else: From f55e095322b7715e1d558db53c3590903687acef Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Fri, 26 Jul 2024 11:34:57 -0700 Subject: [PATCH 08/14] bt: gracefully handle FaultError for variable values There's no reason to be confident that a variable value will be valid when we're formatting it in the stack trace. It is possible that it will contain stale values that may result in FaultErrors. Be sure to catch this and simply print a message saying the FaultError happened. This ensures bt does not crash in these exceptional cases. Signed-off-by: Stephen Brennan --- drgn_tools/bt.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drgn_tools/bt.py b/drgn_tools/bt.py index 7349b523..2755809a 100644 --- a/drgn_tools/bt.py +++ b/drgn_tools/bt.py @@ -373,7 +373,13 @@ def get_sp(frame: drgn.StackFrame) -> int: raise if val.absent_ and not show_absent: continue - val_str = val.format_(dereference=False).replace("\n", "\n ") + + try: + val_str = val.format_(dereference=False).replace( + "\n", "\n " + ) + except FaultError: + val_str = "(FaultError occurred while formatting!)" print(pfx + " " * 5 + f"{local} = {val_str}") From 62e4b754215111663ae64ce1a14f6632dbe250dd Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Fri, 26 Jul 2024 11:40:02 -0700 Subject: [PATCH 09/14] bt_has_any: remove redundant code Signed-off-by: Stephen Brennan --- drgn_tools/bt.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/drgn_tools/bt.py b/drgn_tools/bt.py index 2755809a..86bfaf2d 100644 --- a/drgn_tools/bt.py +++ b/drgn_tools/bt.py @@ -571,21 +571,9 @@ def bt_has_any( return _indexed_bt_has_any(prog, funcs) frame_list = [] - if task is not None: - try: - frames = bt_frames(task) - for frame in frames: - if func_name(prog, frame) in funcs: - frame_list.append((task, frame)) - - return frame_list + tasks = [task] if task is not None else for_each_task(prog) - except (FaultError, ValueError): - # FaultError: catch unusual unwinding issues - # ValueError: catch "cannot unwind stack of running task" - pass - - for task in for_each_task(prog): + for task in tasks: try: frames = bt_frames(task) for frame in frames: From caba4ef32879ec976906595802ff77bf62783e05 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Mon, 12 Aug 2024 11:10:48 -0700 Subject: [PATCH 10/14] tests: bt: get rid of the UEK4 exception There's no reason to exempt UEK4 from this test: it's not that drgn can't do stack traces on UEK4, it's really that "crashed_thread()" doesn't work on UEK4 vmcores. Let's work around that so we can still smoke test the actual bt operation. Signed-off-by: Stephen Brennan --- tests/test_bt.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/test_bt.py b/tests/test_bt.py index be77c3db..b6daa77c 100644 --- a/tests/test_bt.py +++ b/tests/test_bt.py @@ -1,17 +1,28 @@ -# Copyright (c) 2023, Oracle and/or its affiliates. +# Copyright (c) 2024, Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ import drgn -import pytest +from drgn.helpers.linux import cpu_curr from drgn_tools import bt -@pytest.mark.skip_vmcore("*-uek4") -def test_bt_smoke(prog, request, debuginfo_type): +def test_bt_smoke(prog): if prog.flags & drgn.ProgramFlags.IS_LIVE: thread = prog.thread(1) else: - thread = prog.crashed_thread() + try: + thread = prog.crashed_thread() + except Exception: + # On x86_64 uek4, the sysrq does not actually trigger a panic, it + # triggers a NULL pointer dereference, which triggers an "oops", and + # that directly calls into the kexec code without ever calling + # panic(). Thus, panic_cpu == -1, and prog.crashing_cpu() page + # faults because it tries to index the wrong per-cpu variables. + # To handle this, use the x86_64-specific "crashing_cpu" variable. + # Note that on some drgn versions we get "FaultError", others we get + # "Exception". So we just catch Exception here. + pid = cpu_curr(prog, prog["crashing_cpu"]).pid.value_() + thread = prog.thread(pid) print("===== STACK TRACE [show_vars=False] =====") bt.bt(thread, show_vars=False) From 84721a531caa209542f600c6d597c23494e5b44b Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Tue, 13 Aug 2024 16:56:38 -0700 Subject: [PATCH 11/14] bt_has_any: disregard constprop / isra Signed-off-by: Stephen Brennan --- drgn_tools/bt.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drgn_tools/bt.py b/drgn_tools/bt.py index 86bfaf2d..3dae7c5d 100644 --- a/drgn_tools/bt.py +++ b/drgn_tools/bt.py @@ -42,7 +42,13 @@ def func_name(prog: drgn.Program, frame: drgn.StackFrame) -> t.Optional[str]: if frame.name: return frame.name try: - return frame.symbol().name + sym = frame.symbol().name + if ".isra." in sym: + return sym[: sym.index(".isra.")] + elif ".constprop." in sym: + return sym[: sym.index(".constprop.")] + else: + return sym except LookupError: return None From 5d8b674bb52e36aa790278a8aec9e1517c5d71d4 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Tue, 13 Aug 2024 16:56:55 -0700 Subject: [PATCH 12/14] bt_has_any: add one_per_task argument Signed-off-by: Stephen Brennan --- drgn_tools/bt.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drgn_tools/bt.py b/drgn_tools/bt.py index 3dae7c5d..8cf8b954 100644 --- a/drgn_tools/bt.py +++ b/drgn_tools/bt.py @@ -531,6 +531,7 @@ def _index_functions(prog: drgn.Program) -> t.Dict[str, t.Set[int]]: def _indexed_bt_has_any( prog: drgn.Program, funcs: t.List[str], + one_per_task: bool = False, ) -> t.List[t.Tuple[drgn.Object, drgn.StackFrame]]: index = prog.cache.get("drgn_tools.bt._index_functions") if index is None: @@ -547,6 +548,9 @@ def _indexed_bt_has_any( name = func_name(prog, frame) if name in funcs: result.append((task, frame)) + if one_per_task: + # don't return any more results for this task + break return result @@ -554,6 +558,7 @@ def bt_has_any( prog: drgn.Program, funcs: t.List[str], task: t.Optional[drgn.Object] = None, + one_per_task: bool = False, ) -> t.List[t.Tuple[drgn.Object, drgn.StackFrame]]: """ Search for tasks whose stack contains the given functions @@ -585,6 +590,8 @@ def bt_has_any( for frame in frames: if func_name(prog, frame) in funcs: frame_list.append((task, frame)) + if one_per_task: + break except (FaultError, ValueError): # FaultError: catch unusual unwinding issues # ValueError: catch "cannot unwind stack of running task" From 1eb1c94e017aac8e4e9c424b72c5c7268b782422 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Thu, 25 Jul 2024 18:05:06 -0700 Subject: [PATCH 13/14] lock: add fallback mechanism for CTF or DWARF absent objects For mutexes, locks, and semaphores, we are able to reliably test whether any given pointer actually corresponds to the correct lock. We simply pretend that it is valid, and check to see whether the current task is present on the list of waiters. Imran initially implemented the is_task_blocked_on_lock() function with this great idea. We don't know where lock pointer will actually be on the stack: that's what DWARF would tell us. We could hard-code the stack offsets we've observed in the past, but this is not very maintainable. It takes a lot of code to implement and it takes a lot of time & resources to check every kernel. Plus, we would need to check each new kernel as they are released. The alternative, as suggested by Junxiao, is much simpler: just check every stack offset from the top of the stack to the mutex/sem lock function. We can eliminate many addresses by the fact that they may not be kernel memory addresses. For the remaining addresses, so long as we are careful (validating that we are really following a linked list, and not going into a loop), there should be no problem testing them to see if they are really locks. This commit implements the approach. Signed-off-by: Stephen Brennan Co-authored-by: Imran Khan Suggested-by: Junxiao Bi --- drgn_tools/lock.py | 101 ++++++++++++----------------- drgn_tools/locking.py | 144 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 63 deletions(-) diff --git a/drgn_tools/lock.py b/drgn_tools/lock.py index 89dd8f19..21eee8a8 100755 --- a/drgn_tools/lock.py +++ b/drgn_tools/lock.py @@ -32,7 +32,6 @@ from typing import Set from typing import Tuple -import drgn from drgn import Object from drgn import Program from drgn import StackFrame @@ -47,6 +46,7 @@ from drgn_tools.locking import _RWSEM_READER_SHIFT from drgn_tools.locking import for_each_mutex_waiter from drgn_tools.locking import for_each_rwsem_waiter +from drgn_tools.locking import get_lock_from_frame from drgn_tools.locking import get_rwsem_owner from drgn_tools.locking import get_rwsem_spinners_info from drgn_tools.locking import mutex_owner @@ -120,32 +120,31 @@ def scan_mutex_lock( if pid is not None: wtask = find_task(prog, pid) - frame_list = bt_has_any(prog, ["__mutex_lock"]) + frame_list = bt_has_any( + prog, + [ + "__mutex_lock", + "__mutex_lock_interruptible_slowpath", + "__mutex_lock_slowpath", + "__mutex_lock_killable_slowpath", + ], + one_per_task=True, + ) if not frame_list: return seen_mutexes: Set[int] = set() - warned_absent = False for task, frame in frame_list: - try: - mutex = frame["lock"] - mutex_addr = mutex.value_() - except drgn.ObjectAbsentError: - if not warned_absent: - print( - "warning: failed to get mutex from stack frame" - "- information is incomplete" - ) - warned_absent = True + mutex = get_lock_from_frame(prog, task, frame, "mutex", "lock") + if not mutex: continue - - struct_owner = mutex_owner(prog, mutex) - + mutex_addr = mutex.value_() if mutex_addr in seen_mutexes: continue seen_mutexes.add(mutex_addr) + struct_owner = mutex_owner(prog, mutex) index = 0 print(f"Mutex: 0x{mutex_addr:x}") print( @@ -154,10 +153,9 @@ def scan_mutex_lock( "PID :", struct_owner.pid.value_(), ) - print("") if stack: - bt(struct_owner.pid) - print("") + bt(struct_owner) + print("") print( "Mutex WAITERS (Index, cpu, comm, pid, state, wait time (d hr:min:sec:ms)):" @@ -183,31 +181,23 @@ def scan_mutex_lock( def show_sem_lock( prog: Program, frame_list, - seen_sems, stack: bool, time: Optional[int] = None, pid: Optional[int] = None, ) -> None: """Show semaphore details""" - warned_absent = False wtask = None if pid is not None: wtask = find_task(prog, pid) + seen_sems: Set[int] = set() + for task, frame in frame_list: - try: - sem = frame["sem"] - semaddr = sem.value_() - except drgn.ObjectAbsentError: - if not warned_absent: - print( - "warning: failed to get semaphore from stack frame" - "- information is incomplete" - ) - warned_absent = True + sem = get_lock_from_frame(prog, task, frame, "semaphore", "sem") + if not sem: continue - + semaddr = sem.value_() if semaddr in seen_sems: continue seen_sems.add(semaddr) @@ -238,31 +228,23 @@ def show_sem_lock( def show_rwsem_lock( prog: Program, frame_list: List[Tuple[Object, StackFrame]], - seen_rwsems: Set[int], stack: bool, time: Optional[int] = None, pid: Optional[int] = None, ) -> None: """Show rw_semaphore details""" - warned_absent = False wtask = None if pid is not None: wtask = find_task(prog, pid) + seen_rwsems: Set[int] = set() + for task, frame in frame_list: - try: - rwsem = frame["sem"] - rwsemaddr = rwsem.value_() - except drgn.ObjectAbsentError: - if not warned_absent: - print( - "warning: failed to get rwsemaphore from stack frame" - "- information is incomplete" - ) - warned_absent = True + rwsem = get_lock_from_frame(prog, task, frame, "rw_semaphore", "sem") + if not rwsem: continue - + rwsemaddr = rwsem.value_() if rwsemaddr in seen_rwsems: continue seen_rwsems.add(rwsemaddr) @@ -281,10 +263,9 @@ def show_rwsem_lock( print( f"Writer owner ({owner_task.type_.type_name()})0x{owner_task.value_():x}: (pid){owner_task.pid.value_()}" ) - print("") if stack: - bt(owner_task.pid) - print("") + bt(owner_task) + print("") elif owner_type == RwsemStateCode.READER_OWNED: # For reader owned rwsems, we can get number of readers in newer kernels( >= v5.3.1). # So try to retrieve that info. @@ -331,7 +312,6 @@ def scan_sem_lock( if pid is not None: wtask = find_task(prog, pid) - seen_sems: Set[int] = set() functions = [ "__down", "__down_common", @@ -339,9 +319,9 @@ def scan_sem_lock( "__down_killable", "__down_timeout", ] - frame_list = bt_has_any(prog, functions, wtask) + frame_list = bt_has_any(prog, functions, wtask, one_per_task=True) if frame_list: - show_sem_lock(prog, frame_list, seen_sems, stack, time, pid) + show_sem_lock(prog, frame_list, stack, time, pid) def scan_rwsem_lock( @@ -355,16 +335,19 @@ def scan_rwsem_lock( if pid is not None: wtask = find_task(prog, pid) - seen_rwsems: Set[int] = set() functions = [ "__rwsem_down_write_failed_common", "__rwsem_down_read_failed_common", + "rwsem_down_write_failed", + "rwsem_down_write_failed_killable", "rwsem_down_write_slowpath", + "rwsem_down_read_failed", + "rwsem_down_read_failed_killable", "rwsem_down_read_slowpath", ] - frame_list = bt_has_any(prog, functions, wtask) + frame_list = bt_has_any(prog, functions, wtask, one_per_task=True) if frame_list: - show_rwsem_lock(prog, frame_list, seen_rwsems, stack, time, pid) + show_rwsem_lock(prog, frame_list, stack, time, pid) def scan_lock( @@ -374,16 +357,13 @@ def scan_lock( pid: Optional[int] = None, ) -> None: """Scan tasks for Mutex and Semaphore""" - print("Scanning Mutexes...") - print("") + print("Scanning Mutexes...\n") scan_mutex_lock(prog, stack, time, pid) - print("Scanning Semaphores...") - print("") + print("Scanning Semaphores...\n") scan_sem_lock(prog, stack, time, pid) - print("Scanning RWSemaphores...") - print("") + print("Scanning RWSemaphores...\n") scan_rwsem_lock(prog, stack, time, pid) @@ -391,7 +371,6 @@ class Locking(CorelensModule): """Display active mutex and semaphores and their waiters""" name = "lock" - need_dwarf = True def add_args(self, parser: argparse.ArgumentParser) -> None: parser.add_argument( diff --git a/drgn_tools/locking.py b/drgn_tools/locking.py index 59b3472d..ee84b58a 100644 --- a/drgn_tools/locking.py +++ b/drgn_tools/locking.py @@ -5,21 +5,29 @@ """ import enum from typing import Iterable +from typing import Optional from typing import Tuple import drgn +from drgn import Architecture from drgn import cast +from drgn import FaultError +from drgn import IntegerLike from drgn import NULL from drgn import Object from drgn import Program +from drgn import StackFrame +from drgn.helpers import ValidationError from drgn.helpers.linux.list import list_empty from drgn.helpers.linux.list import list_for_each_entry +from drgn.helpers.linux.list import validate_list_for_each_entry from drgn.helpers.linux.percpu import per_cpu from drgn.helpers.linux.sched import cpu_curr from drgn.helpers.linux.sched import task_cpu from drgn.helpers.linux.sched import task_state_to_char from drgn_tools.bt import bt +from drgn_tools.mm import AddrKind from drgn_tools.table import FixedTable from drgn_tools.task import get_current_run_time from drgn_tools.task import task_lastrun2now @@ -179,7 +187,7 @@ def show_lock_waiter( :param stacktrace: true to dump stack trace of the waiter :returns: None """ - prefix = "[%d] " % index + prefix = "[%d]" % index ncpu = task_cpu(task) print( "%12s: %-4s %-4d %-16s %-8d %-6s %-16s" @@ -194,8 +202,8 @@ def show_lock_waiter( ) ) if stacktrace: + bt(task, indent=12) print("") - bt(task) def for_each_rwsem_waiter(prog: Program, rwsem: Object) -> Iterable[Object]: @@ -226,6 +234,49 @@ def for_each_mutex_waiter(prog: Program, mutex: Object) -> Iterable[Object]: yield waiter.task +def for_each_rwsem_waiter_careful( + prog: Program, rwsem: Object +) -> Iterable[Object]: + """ + List task waiting on the rw semaphore + + :param prog: drgn program + :param rwsem: ``struct rw_semaphore *`` + :returns: ``struct task_struct *`` + """ + seen = set() + for waiter in validate_list_for_each_entry( + prog.type("struct rwsem_waiter"), rwsem.wait_list.address_of_(), "list" + ): + addr = waiter.value_() + if addr in seen: + raise ValidationError("circular list") + seen.add(addr) + yield waiter.task + + +def for_each_mutex_waiter_careful( + prog: Program, + mutex: Object, +) -> Iterable[Object]: + """ + List task waiting on the mutex + + :param prog: drgn program + :param mutex: ``struct mutex *`` + :returns: ``struct task_struct *`` + """ + seen = set() + for waiter in validate_list_for_each_entry( + prog.type("struct mutex_waiter"), mutex.wait_list.address_of_(), "list" + ): + addr = waiter.value_() + if addr in seen: + raise ValidationError("circular list") + seen.add(addr) + yield waiter.task + + ###################################### # rwsem ###################################### @@ -519,3 +570,92 @@ def get_rwsem_info(rwsem: Object, callstack: int = 0) -> None: print("There are no waiters") else: get_rwsem_waiters_info(rwsem, callstack) + + +def is_task_blocked_on_lock( + pid: IntegerLike, lock_type: str, lock: Object +) -> bool: + """ + Check if a task is blocked on a given lock or not + :param pid: PID of task + :param var_name: variable name (sem, or mutex) + :param lock: ``struct mutex *`` or ``struct semaphore *`` or ``struct rw_semaphore *`` + :returns: True if task is blocked on given lock, False otherwise. + """ + + try: + if lock_type == "semaphore" or lock_type == "rw_semaphore": + return pid in [ + waiter.pid.value_() + for waiter in for_each_rwsem_waiter_careful(lock.prog_, lock) + ] + elif lock_type == "mutex": + return pid in [ + waiter.pid.value_() + for waiter in for_each_mutex_waiter_careful(lock.prog_, lock) + ] + else: + return False + except (FaultError, ValidationError): + return False + + +def get_lock_from_frame( + prog: Program, task: Object, frame: StackFrame, kind: str, var: str +) -> Optional[Object]: + """ + Given a stack frame, try to get the relevant lock out of it. + + :param task: the ``struct task_struct *`` of the task + :param frame: the stack frame in question + :param kind: the kind of lock (mutex, rw_semaphore, semaphore) + :param var: the variable name within the stack frame + :returns: an object of the appropriate type for the lock kind. If it could + not be found, returns None + """ + # Try to use DWARF CFI to get the variable value. This is the most + # straightforward method to get the lock variable. + try: + lock = frame[var] + if not lock.absent_: + return lock + except (drgn.ObjectAbsentError, KeyError): + pass + + # Fall back to a brute force method which works shockingly well. Simply scan + # from the top of the stack down to the frame of the lock. If the address is + # a kernel address, we'll test it (carefully) to see whether it's a valid + # lock, and if so, we'll return it. + # + # First, we need to get the start and end address to search. This is + # arch-specific. + if prog.platform.arch == Architecture.X86_64: + candidates = range(task.thread.sp, frame.sp, 8) + elif prog.platform.arch == Architecture.AARCH64: + candidates = range( + task.thread.cpu_context.sp, + # We can't rely on SP being available. We need the top of this + # function's frame, which is the *previous* frame's frame pointer, + # which we get by simply dereferencing the fp register. + prog.read_u64(frame.register("fp")), + 8, + ) + + tp = prog.type(f"struct {kind} *") + pid = task.pid.value_() + for addr in candidates: + value = prog.read_u64(addr) + akind = AddrKind.categorize(prog, value) + if akind not in ( + AddrKind.DIRECT_MAP, + AddrKind.DATA, + AddrKind.BSS, + AddrKind.PERCPU, # it would be weird, but not impossible + AddrKind.VMALLOC, + AddrKind.MODULE, + ): + continue + lock = Object(prog, tp, value=value) + if is_task_blocked_on_lock(pid, kind, lock): + return lock + return None From 6d07880655d4607621cbcd1caae943204b4e2007 Mon Sep 17 00:00:00 2001 From: Stephen Brennan Date: Mon, 12 Aug 2024 11:16:18 -0700 Subject: [PATCH 14/14] tests: lock: add proper tests for lock fallbacks We've generated vmcores for every combination of OL, UEK, and architecture. Additionally, for combinations where there are multiple offsets, we've generated a vmcore on the oldest and newest version. Each vmcore is generated after first loading the "lockmod" kernel module, so that we can test loading the lock value out of the stack frame. The test is written to test the fallback on DWARF (in case the stack frame lookup fails) and CTF. Signed-off-by: Stephen Brennan --- tests/test_lock.py | 79 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/tests/test_lock.py b/tests/test_lock.py index 361146bc..69f356d4 100644 --- a/tests/test_lock.py +++ b/tests/test_lock.py @@ -1,7 +1,84 @@ -# Copyright (c) 2023, Oracle and/or its affiliates. +# Copyright (c) 2024, Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +import pytest +from drgn.helpers.linux import for_each_task + from drgn_tools import lock +from drgn_tools import locking +from drgn_tools.bt import func_name +# the rwsem code does not support UEK4, no reason to add support +@pytest.mark.skip_vmcore("*uek4*") def test_locks(prog): lock.scan_lock(prog, stack=True) + + +@pytest.mark.skip_live +@pytest.mark.vmcore("*lockmod*") +def test_with_lockmod(prog, debuginfo_type): + lockmod_threads = [] + for task in for_each_task(prog): + if task.comm.string_().startswith(b"lockmod"): + lockmod_threads.append(task) + + if not lockmod_threads: + pytest.skip("no lockmod kernel module found") + + for task in lockmod_threads: + print(f"PID {task.pid.value_()} COMM {task.comm.string_().decode()}") + comm = task.comm.string_() + if b"owner" in comm: + # this owns the locks + continue + + if b"mutex" in comm: + kind = "mutex" + var = "lock" + func_substr = "mutex_lock" + elif b"rwsem" in comm: + kind = "rw_semaphore" + var = "sem" + func_substr = "rwsem" + else: + kind = "semaphore" + var = "sem" + func_substr = "down" + + # There can be multiple frames which may contain the lock, we will need + # to try all of them. + trace = prog.stack_trace(task) + frames = [] + for frame in trace: + fn = func_name(prog, frame) + if fn and func_substr in fn: + frames.append(frame) + if not frames: + pytest.fail("could not find relevant stack frame in lockmod") + + # Test 1: if DWARF debuginfo is present, then this will try to use the + # variable name to access the lock. Otherwise, for CTF we will fall back + # to using the stack offsets. + for frame in frames: + value = locking.get_lock_from_frame(prog, task, frame, kind, var) + if value is not None: + break + else: + pytest.fail(f"Could not find lock using {debuginfo_type}") + + if debuginfo_type == "ctf": + # The second test is redundant, skip it. + continue + + # Test 2: if DWARF debuginfo is present, we can actually give a fake + # variable name! This will force the code to fall back to the stack + # offsets, which should still work. This essentially simulates the + # possibility of a DWARF unwind where we get an absent object. + for frame in frames: + value = locking.get_lock_from_frame( + prog, task, frame, kind, "invalid variable name" + ) + if value is not None: + break + else: + pytest.fail("Could not find lock using fallback method")