diff --git a/libbpf-tools/klockstat.bpf.c b/libbpf-tools/klockstat.bpf.c index d003859f2d1b..fdc57a0c4532 100644 --- a/libbpf-tools/klockstat.bpf.c +++ b/libbpf-tools/klockstat.bpf.c @@ -62,6 +62,13 @@ struct { __type(value, struct lock_stat); } stat_map SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, MAX_ENTRIES); + __type(key, u32); + __type(value, void *); +} locks SEC(".maps"); + static bool tracing_task(u64 task_id) { u32 tgid = task_id >> 32; @@ -98,8 +105,8 @@ static void lock_contended(void *ctx, void *lock) * Note: if you make major changes to this bpf program, double check * that you aren't skipping too many frames. */ - li->stack_id = bpf_get_stackid(ctx, &stack_map, - 4 | BPF_F_FAST_STACK_CMP); + li->stack_id = bpf_get_stackid(ctx, &stack_map, 4 | BPF_F_FAST_STACK_CMP); + /* Legit failures include EEXIST */ if (li->stack_id < 0) return; @@ -405,4 +412,331 @@ int BPF_PROG(up_write, struct rw_semaphore *lock) return 0; } +SEC("kprobe/mutex_lock") +int BPF_KPROBE(kprobe_mutex_lock, struct mutex *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + lock_contended(ctx, lock); + return 0; +} + +SEC("kretprobe/mutex_lock") +int BPF_KRETPROBE(kprobe_mutex_lock_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + lock_acquired(*lock); + return 0; +} + +SEC("kprobe/mutex_trylock") +int BPF_KPROBE(kprobe_mutex_trylock, struct mutex *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + return 0; +} + +SEC("kretprobe/mutex_trylock") +int BPF_KRETPROBE(kprobe_mutex_trylock_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + if (ret) { + lock_contended(ctx, *lock); + lock_acquired(*lock); + } + return 0; +} + +SEC("kprobe/mutex_lock_interruptible") +int BPF_KPROBE(kprobe_mutex_lock_interruptible, struct mutex *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + lock_contended(ctx, lock); + return 0; +} + +SEC("kretprobe/mutex_lock_interruptible") +int BPF_KRETPROBE(kprobe_mutex_lock_interruptible_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + if (ret) + lock_aborted(*lock); + else + lock_acquired(*lock); + return 0; +} + +SEC("kprobe/mutex_lock_killable") +int BPF_KPROBE(kprobe_mutex_lock_killable, struct mutex *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + lock_contended(ctx, lock); + return 0; +} + +SEC("kretprobe/mutex_lock_killable") +int BPF_KRETPROBE(kprobe_mutex_lock_killable_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + if (ret) + lock_aborted(*lock); + else + lock_acquired(*lock); + return 0; +} + +SEC("kprobe/mutex_unlock") +int BPF_KPROBE(kprobe_mutex_unlock, struct mutex *lock) +{ + lock_released(lock); + return 0; +} + +SEC("kprobe/down_read") +int BPF_KPROBE(kprobe_down_read, struct rw_semaphore *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + lock_contended(ctx, lock); + return 0; +} + +SEC("kretprobe/down_read") +int BPF_KRETPROBE(kprobe_down_read_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + lock_acquired(*lock); + return 0; +} + +SEC("kprobe/down_read_trylock") +int BPF_KPROBE(kprobe_down_read_trylock, struct rw_semaphore *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + return 0; +} + +SEC("kretprobe/down_read_trylock") +int BPF_KRETPROBE(kprobe_down_read_trylock_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + if (ret == 1) { + lock_contended(ctx, *lock); + lock_acquired(*lock); + } + return 0; +} + +SEC("kprobe/down_read_interruptible") +int BPF_KPROBE(kprobe_down_read_interruptible, struct rw_semaphore *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + lock_contended(ctx, lock); + return 0; +} + +SEC("kretprobe/down_read_interruptible") +int BPF_KRETPROBE(kprobe_down_read_interruptible_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + if (ret) + lock_aborted(*lock); + else + lock_acquired(*lock); + return 0; +} + +SEC("kprobe/down_read_killable") +int BPF_KPROBE(kprobe_down_read_killable, struct rw_semaphore *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + lock_contended(ctx, lock); + return 0; +} + +SEC("kretprobe/down_read_killable") +int BPF_KRETPROBE(kprobe_down_read_killable_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + if (ret) + lock_aborted(*lock); + else + lock_acquired(*lock); + return 0; +} + +SEC("kprobe/up_read") +int BPF_KPROBE(kprobe_up_read, struct rw_semaphore *lock) +{ + lock_released(lock); + return 0; +} + +SEC("kprobe/down_write") +int BPF_KPROBE(kprobe_down_write, struct rw_semaphore *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + lock_contended(ctx, lock); + return 0; +} + +SEC("kretprobe/down_write") +int BPF_KRETPROBE(kprobe_down_write_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + lock_acquired(*lock); + return 0; +} + +SEC("kprobe/down_write_trylock") +int BPF_KPROBE(kprobe_down_write_trylock, struct rw_semaphore *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + return 0; +} + +SEC("kretprobe/down_write_trylock") +int BPF_KRETPROBE(kprobe_down_write_trylock_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + if (ret == 1) { + lock_contended(ctx, *lock); + lock_acquired(*lock); + } + return 0; +} + +SEC("kprobe/down_write_killable") +int BPF_KPROBE(kprobe_down_write_killable, struct rw_semaphore *lock) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + + bpf_map_update_elem(&locks, &tid, &lock, BPF_ANY); + lock_contended(ctx, lock); + return 0; +} + +SEC("kretprobe/down_write_killable") +int BPF_KRETPROBE(kprobe_down_write_killable_exit, long ret) +{ + u32 tid = (u32)bpf_get_current_pid_tgid(); + void **lock; + + lock = bpf_map_lookup_elem(&locks, &tid); + if (!lock) + return 0; + + bpf_map_delete_elem(&locks, &tid); + + if (ret) + lock_aborted(*lock); + else + lock_acquired(*lock); + return 0; +} + +SEC("kprobe/up_write") +int BPF_KPROBE(kprobe_up_write, struct rw_semaphore *lock) +{ + lock_released(lock); + return 0; +} + char LICENSE[] SEC("license") = "GPL"; diff --git a/libbpf-tools/klockstat.c b/libbpf-tools/klockstat.c index e3252f9af41c..0580f6999fd7 100644 --- a/libbpf-tools/klockstat.c +++ b/libbpf-tools/klockstat.c @@ -114,7 +114,8 @@ static const struct argp_option opts[] = { {}, }; -static void *parse_lock_addr(const char *lock_name) { +static void *parse_lock_addr(const char *lock_name) +{ unsigned long lock_addr; return sscanf(lock_name, "0x%lx", &lock_addr) ? (void*)lock_addr : NULL; @@ -127,7 +128,7 @@ static void *get_lock_addr(struct ksyms *ksyms, const char *lock_name) return ksym ? (void*)ksym->addr : parse_lock_addr(lock_name); } -const char *get_lock_name(struct ksyms *ksyms, unsigned long addr) +static const char *get_lock_name(struct ksyms *ksyms, unsigned long addr) { const struct ksym *ksym = ksyms__map_addr(ksyms, addr); @@ -267,10 +268,10 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) env->interval = env->duration; env->iterations = env->duration / env->interval; } - if (env->per_thread && env->nr_stack_entries != 1) { + if (env->per_thread && env->nr_stack_entries != 1) { warn("--per-thread and --stacks cannot be used together\n"); argp_usage(state); - } + } break; default: return ARGP_ERR_UNKNOWN; @@ -597,6 +598,100 @@ static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va return vfprintf(stderr, format, args); } +static void enable_fentry(struct klockstat_bpf *obj) +{ + bool debug_lock; + + bpf_program__set_autoload(obj->progs.kprobe_mutex_lock, false); + bpf_program__set_autoload(obj->progs.kprobe_mutex_lock_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_mutex_trylock, false); + bpf_program__set_autoload(obj->progs.kprobe_mutex_trylock_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_mutex_lock_interruptible, false); + bpf_program__set_autoload(obj->progs.kprobe_mutex_lock_interruptible_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_mutex_lock_killable, false); + bpf_program__set_autoload(obj->progs.kprobe_mutex_lock_killable_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_mutex_unlock, false); + + bpf_program__set_autoload(obj->progs.kprobe_down_read, false); + bpf_program__set_autoload(obj->progs.kprobe_down_read_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_down_read_trylock, false); + bpf_program__set_autoload(obj->progs.kprobe_down_read_trylock_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_down_read_interruptible, false); + bpf_program__set_autoload(obj->progs.kprobe_down_read_interruptible_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_down_read_killable, false); + bpf_program__set_autoload(obj->progs.kprobe_down_read_killable_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_up_read, false); + bpf_program__set_autoload(obj->progs.kprobe_down_write, false); + bpf_program__set_autoload(obj->progs.kprobe_down_write_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_down_write_trylock, false); + bpf_program__set_autoload(obj->progs.kprobe_down_write_trylock_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_down_write_killable, false); + bpf_program__set_autoload(obj->progs.kprobe_down_write_killable_exit, false); + bpf_program__set_autoload(obj->progs.kprobe_up_write, false); + + /* CONFIG_DEBUG_LOCK_ALLOC is on */ + debug_lock = fentry_can_attach("mutex_lock_nested", NULL); + if (!debug_lock) + return; + + bpf_program__set_attach_target(obj->progs.mutex_lock, 0, + "mutex_lock_nested"); + bpf_program__set_attach_target(obj->progs.mutex_lock_exit, 0, + "mutex_lock_nested"); + bpf_program__set_attach_target(obj->progs.mutex_lock_interruptible, 0, + "mutex_lock_interruptible_nested"); + bpf_program__set_attach_target(obj->progs.mutex_lock_interruptible_exit, 0, + "mutex_lock_interruptible_nested"); + bpf_program__set_attach_target(obj->progs.mutex_lock_killable, 0, + "mutex_lock_killable_nested"); + bpf_program__set_attach_target(obj->progs.mutex_lock_killable_exit, 0, + "mutex_lock_killable_nested"); + + bpf_program__set_attach_target(obj->progs.down_read, 0, + "down_read_nested"); + bpf_program__set_attach_target(obj->progs.down_read_exit, 0, + "down_read_nested"); + bpf_program__set_attach_target(obj->progs.down_read_killable, 0, + "down_read_killable_nested"); + bpf_program__set_attach_target(obj->progs.down_read_killable_exit, 0, + "down_read_killable_nested"); + bpf_program__set_attach_target(obj->progs.down_write, 0, + "down_write_nested"); + bpf_program__set_attach_target(obj->progs.down_write_exit, 0, + "down_write_nested"); + bpf_program__set_attach_target(obj->progs.down_write_killable, 0, + "down_write_killable_nested"); + bpf_program__set_attach_target(obj->progs.down_write_killable_exit, 0, + "down_write_killable_nested"); +} + +static void enable_kprobes(struct klockstat_bpf *obj) +{ + bpf_program__set_autoload(obj->progs.mutex_lock, false); + bpf_program__set_autoload(obj->progs.mutex_lock_exit, false); + bpf_program__set_autoload(obj->progs.mutex_trylock_exit, false); + bpf_program__set_autoload(obj->progs.mutex_lock_interruptible, false); + bpf_program__set_autoload(obj->progs.mutex_lock_interruptible_exit, false); + bpf_program__set_autoload(obj->progs.mutex_lock_killable, false); + bpf_program__set_autoload(obj->progs.mutex_lock_killable_exit, false); + bpf_program__set_autoload(obj->progs.mutex_unlock, false); + + bpf_program__set_autoload(obj->progs.down_read, false); + bpf_program__set_autoload(obj->progs.down_read_exit, false); + bpf_program__set_autoload(obj->progs.down_read_trylock_exit, false); + bpf_program__set_autoload(obj->progs.down_read_interruptible, false); + bpf_program__set_autoload(obj->progs.down_read_interruptible_exit, false); + bpf_program__set_autoload(obj->progs.down_read_killable, false); + bpf_program__set_autoload(obj->progs.down_read_killable_exit, false); + bpf_program__set_autoload(obj->progs.up_read, false); + bpf_program__set_autoload(obj->progs.down_write, false); + bpf_program__set_autoload(obj->progs.down_write_exit, false); + bpf_program__set_autoload(obj->progs.down_write_trylock_exit, false); + bpf_program__set_autoload(obj->progs.down_write_killable, false); + bpf_program__set_autoload(obj->progs.down_write_killable_exit, false); + bpf_program__set_autoload(obj->progs.up_write, false); +} + int main(int argc, char **argv) { static const struct argp argp = { @@ -649,40 +744,11 @@ int main(int argc, char **argv) obj->rodata->targ_lock = lock_addr; obj->rodata->per_thread = env.per_thread; - if (fentry_can_attach("mutex_lock_nested", NULL)) { - bpf_program__set_attach_target(obj->progs.mutex_lock, 0, - "mutex_lock_nested"); - bpf_program__set_attach_target(obj->progs.mutex_lock_exit, 0, - "mutex_lock_nested"); - bpf_program__set_attach_target(obj->progs.mutex_lock_interruptible, 0, - "mutex_lock_interruptible_nested"); - bpf_program__set_attach_target(obj->progs.mutex_lock_interruptible_exit, 0, - "mutex_lock_interruptible_nested"); - bpf_program__set_attach_target(obj->progs.mutex_lock_killable, 0, - "mutex_lock_killable_nested"); - bpf_program__set_attach_target(obj->progs.mutex_lock_killable_exit, 0, - "mutex_lock_killable_nested"); - } - - if (fentry_can_attach("down_read_nested", NULL)) { - bpf_program__set_attach_target(obj->progs.down_read, 0, - "down_read_nested"); - bpf_program__set_attach_target(obj->progs.down_read_exit, 0, - "down_read_nested"); - bpf_program__set_attach_target(obj->progs.down_read_killable, 0, - "down_read_killable_nested"); - bpf_program__set_attach_target(obj->progs.down_read_killable_exit, 0, - "down_read_killable_nested"); - - bpf_program__set_attach_target(obj->progs.down_write, 0, - "down_write_nested"); - bpf_program__set_attach_target(obj->progs.down_write_exit, 0, - "down_write_nested"); - bpf_program__set_attach_target(obj->progs.down_write_killable, 0, - "down_write_killable_nested"); - bpf_program__set_attach_target(obj->progs.down_write_killable_exit, 0, - "down_write_killable_nested"); - } + if (fentry_can_attach("mutex_lock", NULL) || + fentry_can_attach("mutex_lock_nested", NULL)) + enable_fentry(obj); + else + enable_kprobes(obj); err = klockstat_bpf__load(obj); if (err) { diff --git a/libbpf-tools/klockstat.h b/libbpf-tools/klockstat.h index d95e43a6af3b..f9f9bec1d22b 100644 --- a/libbpf-tools/klockstat.h +++ b/libbpf-tools/klockstat.h @@ -17,7 +17,7 @@ struct lock_stat { __u64 hld_max_time; __u64 hld_max_id; __u64 hld_max_lock_ptr; - char hld_max_comm[TASK_COMM_LEN]; + char hld_max_comm[TASK_COMM_LEN]; }; #endif /*__KLOCKSTAT_H */ diff --git a/libbpf-tools/trace_helpers.c b/libbpf-tools/trace_helpers.c index 3f9185870189..8c5a53627f56 100644 --- a/libbpf-tools/trace_helpers.c +++ b/libbpf-tools/trace_helpers.c @@ -1008,10 +1008,8 @@ static bool fentry_try_attach(int id) prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACING, "test", "GPL", insns, sizeof(insns) / sizeof(struct bpf_insn), &opts); - if (prog_fd < 0) { - fprintf(stderr, "failed to try attaching to fentry: %s\n", error); + if (prog_fd < 0) return false; - } attach_fd = bpf_raw_tracepoint_open(NULL, prog_fd); if (attach_fd >= 0)