diff --git a/libbpf-tools/biosnoop.bpf.c b/libbpf-tools/biosnoop.bpf.c index e31d0325608b..a07b15e41541 100644 --- a/libbpf-tools/biosnoop.bpf.c +++ b/libbpf-tools/biosnoop.bpf.c @@ -75,6 +75,24 @@ int BPF_PROG(blk_account_io_start, struct request *rq) return trace_pid(rq); } +SEC("kprobe/blk_account_io_start") +int BPF_KPROBE(kprobe_blk_account_io_start, struct request *rq) +{ + if (filter_cg && !bpf_current_task_under_cgroup(&cgroup_map, 0)) + return 0; + + return trace_pid(rq); +} + +SEC("kprobe/__blk_account_io_start") +int BPF_KPROBE(kprobe___blk_account_io_start, struct request *rq) +{ + if (filter_cg && !bpf_current_task_under_cgroup(&cgroup_map, 0)) + return 0; + + return trace_pid(rq); +} + SEC("kprobe/blk_account_io_merge_bio") int BPF_KPROBE(blk_account_io_merge_bio, struct request *rq) { diff --git a/libbpf-tools/biosnoop.c b/libbpf-tools/biosnoop.c index da306bf7acff..dd37d3b85b40 100644 --- a/libbpf-tools/biosnoop.c +++ b/libbpf-tools/biosnoop.c @@ -197,6 +197,8 @@ int main(int argc, char **argv) int err; int idx, cg_map_fd; int cgfd = -1; + bool blk_account_io_start_symbol; + bool blk_account_io_start_fentry; err = argp_parse(&argp, argc, argv, 0, NULL, NULL); if (err) @@ -229,12 +231,32 @@ int main(int argc, char **argv) obj->rodata->targ_queued = env.queued; obj->rodata->filter_cg = env.cg; - if (fentry_can_attach("blk_account_io_start", NULL)) - bpf_program__set_attach_target(obj->progs.blk_account_io_start, 0, - "blk_account_io_start"); - else - bpf_program__set_attach_target(obj->progs.blk_account_io_start, 0, - "__blk_account_io_start"); + + if (kprobe_exists("blk_account_io_start")) { + if (fentry_can_attach("blk_account_io_start", NULL)) { + bpf_program__set_attach_target(obj->progs.blk_account_io_start, 0, + "blk_account_io_start"); + bpf_program__set_autoload(obj->progs.kprobe_blk_account_io_start, false); + blk_account_io_start_fentry = true; + } else { + bpf_program__set_autoload(obj->progs.blk_account_io_start, false); + blk_account_io_start_fentry = false; + } + bpf_program__set_autoload(obj->progs.kprobe___blk_account_io_start, false); + blk_account_io_start_symbol = true; + } else { + if (fentry_can_attach("__blk_account_io_start", NULL)) { + bpf_program__set_attach_target(obj->progs.blk_account_io_start, 0, + "__blk_account_io_start"); + bpf_program__set_autoload(obj->progs.kprobe___blk_account_io_start, false); + blk_account_io_start_fentry = true; + } else { + bpf_program__set_autoload(obj->progs.blk_account_io_start, false); + blk_account_io_start_fentry = false; + } + bpf_program__set_autoload(obj->progs.kprobe_blk_account_io_start, false); + blk_account_io_start_symbol = false; + } err = biosnoop_bpf__load(obj); if (err) { @@ -257,12 +279,24 @@ int main(int argc, char **argv) } } - obj->links.blk_account_io_start = bpf_program__attach(obj->progs.blk_account_io_start); - if (!obj->links.blk_account_io_start) { - err = -errno; - fprintf(stderr, "failed to attach blk_account_io_start: %s\n", - strerror(-err)); - goto cleanup; + + if (blk_account_io_start_fentry) + obj->links.blk_account_io_start = bpf_program__attach( + obj->progs.blk_account_io_start); + else + if (blk_account_io_start_symbol) + obj->links.kprobe_blk_account_io_start = bpf_program__attach( + obj->progs.kprobe_blk_account_io_start); + else + obj->links.kprobe___blk_account_io_start = bpf_program__attach( + obj->progs.kprobe___blk_account_io_start); + if (!obj->links.blk_account_io_start && + !obj->links.kprobe_blk_account_io_start && + !obj->links.kprobe___blk_account_io_start) { + err = -errno; + fprintf(stderr, "failed to attach blk_account_io_start: %s\n", + strerror(-err)); + goto cleanup; } ksyms = ksyms__load(); if (!ksyms) { diff --git a/libbpf-tools/biostacks.bpf.c b/libbpf-tools/biostacks.bpf.c index c3950910481c..3b1cf2103a46 100644 --- a/libbpf-tools/biostacks.bpf.c +++ b/libbpf-tools/biostacks.bpf.c @@ -67,20 +67,8 @@ int trace_start(void *ctx, struct request *rq, bool merge_bio) return 0; } -SEC("fentry/blk_account_io_start") -int BPF_PROG(blk_account_io_start, struct request *rq) -{ - return trace_start(ctx, rq, false); -} - -SEC("kprobe/blk_account_io_merge_bio") -int BPF_KPROBE(blk_account_io_merge_bio, struct request *rq) -{ - return trace_start(ctx, rq, true); -} - -SEC("fentry/blk_account_io_done") -int BPF_PROG(blk_account_io_done, struct request *rq) +static __always_inline +int trace_done(void *ctx, struct request *rq) { u64 slot, ts = bpf_ktime_get_ns(); struct internal_rqinfo *i_rqinfop; @@ -110,4 +98,47 @@ int BPF_PROG(blk_account_io_done, struct request *rq) return 0; } + +SEC("fentry/blk_account_io_start") +int BPF_PROG(blk_account_io_start, struct request *rq) +{ + return trace_start(ctx, rq, false); +} + +SEC("fentry/blk_account_io_done") +int BPF_PROG(blk_account_io_done, struct request *rq) +{ + return trace_done(ctx, rq); +} + +SEC("kprobe/blk_account_io_start") +int BPF_KPROBE(kprobe_blk_account_io_start, struct request *rq) +{ + return trace_start(ctx, rq, false); +} + +SEC("kprobe/blk_account_io_done") +int BPF_KPROBE(kprobe_blk_account_io_done, struct request *rq) +{ + return trace_done(ctx, rq); +} + +SEC("kprobe/__blk_account_io_start") +int BPF_KPROBE(kprobe___blk_account_io_start, struct request *rq) +{ + return trace_start(ctx, rq, false); +} + +SEC("kprobe/__blk_account_io_done") +int BPF_KPROBE(kprobe___blk_account_io_done, struct request *rq) +{ + return trace_done(ctx, rq); +} + +SEC("kprobe/blk_account_io_merge_bio") +int BPF_KPROBE(blk_account_io_merge_bio, struct request *rq) +{ + return trace_start(ctx, rq, true); +} + char LICENSE[] SEC("license") = "GPL"; diff --git a/libbpf-tools/biostacks.c b/libbpf-tools/biostacks.c index e1878d1f73e5..899af9ab8e34 100644 --- a/libbpf-tools/biostacks.c +++ b/libbpf-tools/biostacks.c @@ -172,52 +172,56 @@ int main(int argc, char **argv) obj->rodata->targ_ms = env.milliseconds; - if (fentry_can_attach("blk_account_io_start", NULL)) { - bpf_program__set_attach_target(obj->progs.blk_account_io_start, 0, - "blk_account_io_start"); - bpf_program__set_attach_target(obj->progs.blk_account_io_done, 0, - "blk_account_io_done"); + if (kprobe_exists("blk_account_io_start")) { + if (fentry_can_attach("blk_account_io_start", NULL)) { + bpf_program__set_attach_target(obj->progs.blk_account_io_start, 0, + "blk_account_io_start"); + bpf_program__set_attach_target(obj->progs.blk_account_io_done, 0, + "blk_account_io_done"); + bpf_program__set_autoload(obj->progs.kprobe_blk_account_io_start, false); + bpf_program__set_autoload(obj->progs.kprobe_blk_account_io_done, false); + } else { + bpf_program__set_autoload(obj->progs.blk_account_io_start, false); + bpf_program__set_autoload(obj->progs.blk_account_io_done, false); + } + bpf_program__set_autoload(obj->progs.kprobe___blk_account_io_start, false); + bpf_program__set_autoload(obj->progs.kprobe___blk_account_io_done, false); } else { - bpf_program__set_attach_target(obj->progs.blk_account_io_start, 0, - "__blk_account_io_start"); - bpf_program__set_attach_target(obj->progs.blk_account_io_done, 0, - "__blk_account_io_done"); + if (fentry_can_attach("__blk_account_io_start", NULL)) { + bpf_program__set_attach_target(obj->progs.blk_account_io_start, 0, + "__blk_account_io_start"); + bpf_program__set_attach_target(obj->progs.blk_account_io_done, 0, + "__blk_account_io_done"); + bpf_program__set_autoload(obj->progs.kprobe___blk_account_io_start, false); + bpf_program__set_autoload(obj->progs.kprobe___blk_account_io_done, false); + } else { + bpf_program__set_autoload(obj->progs.blk_account_io_start, false); + bpf_program__set_autoload(obj->progs.blk_account_io_done, false); + } + bpf_program__set_autoload(obj->progs.kprobe_blk_account_io_start, false); + bpf_program__set_autoload(obj->progs.kprobe_blk_account_io_done, false); } + if (!kprobe_exists("blk_account_io_merge_bio")) + bpf_program__set_autoload(obj->progs.blk_account_io_merge_bio, false); + err = biostacks_bpf__load(obj); if (err) { fprintf(stderr, "failed to load BPF object: %d\n", err); goto cleanup; } - obj->links.blk_account_io_start = bpf_program__attach(obj->progs.blk_account_io_start); - if (!obj->links.blk_account_io_start) { - err = -errno; - fprintf(stderr, "failed to attach blk_account_io_start: %s\n", strerror(-err)); + err = biostacks_bpf__attach(obj); + if (err) { + fprintf(stderr, "failed to attach BPF programs\n"); goto cleanup; } + ksyms = ksyms__load(); if (!ksyms) { fprintf(stderr, "failed to load kallsyms\n"); goto cleanup; } - if (ksyms__get_symbol(ksyms, "blk_account_io_merge_bio")) { - obj->links.blk_account_io_merge_bio = - bpf_program__attach(obj->progs.blk_account_io_merge_bio); - if (!obj->links.blk_account_io_merge_bio) { - err = -errno; - fprintf(stderr, "failed to attach blk_account_io_merge_bio: %s\n", - strerror(-err)); - goto cleanup; - } - } - obj->links.blk_account_io_done = bpf_program__attach(obj->progs.blk_account_io_done); - if (!obj->links.blk_account_io_done) { - err = -errno; - fprintf(stderr, "failed to attach blk_account_io_done: %s\n", - strerror(-err)); - goto cleanup; - } signal(SIGINT, sig_handler); diff --git a/libbpf-tools/readahead.bpf.c b/libbpf-tools/readahead.bpf.c index 89863e67d60d..0b6a3526097e 100644 --- a/libbpf-tools/readahead.bpf.c +++ b/libbpf-tools/readahead.bpf.c @@ -24,18 +24,8 @@ struct { struct hist hist = {}; -SEC("fentry/do_page_cache_ra") -int BPF_PROG(do_page_cache_ra) -{ - u32 pid = bpf_get_current_pid_tgid(); - u64 one = 1; - - bpf_map_update_elem(&in_readahead, &pid, &one, 0); - return 0; -} - -SEC("fexit/__page_cache_alloc") -int BPF_PROG(page_cache_alloc_ret, gfp_t gfp, struct page *ret) +static __always_inline +int do_page_cache_alloc_ret(struct page *ret) { u32 pid = bpf_get_current_pid_tgid(); u64 ts; @@ -51,17 +41,8 @@ int BPF_PROG(page_cache_alloc_ret, gfp_t gfp, struct page *ret) return 0; } -SEC("fexit/do_page_cache_ra") -int BPF_PROG(do_page_cache_ra_ret) -{ - u32 pid = bpf_get_current_pid_tgid(); - - bpf_map_delete_elem(&in_readahead, &pid); - return 0; -} - -SEC("fentry/mark_page_accessed") -int BPF_PROG(mark_page_accessed, struct page *page) +static __always_inline +int do_mark_page_accessed(struct page *page) { u64 *tsp, slot, ts = bpf_ktime_get_ns(); s64 delta; @@ -84,4 +65,83 @@ int BPF_PROG(mark_page_accessed, struct page *page) return 0; } +static __always_inline +int do_page_cache_ra(void) +{ + u32 pid = bpf_get_current_pid_tgid(); + u64 one = 1; + + bpf_map_update_elem(&in_readahead, &pid, &one, 0); + return 0; +} + +static __always_inline +int do_page_cache_ra_ret(void) +{ + u32 pid = bpf_get_current_pid_tgid(); + + bpf_map_delete_elem(&in_readahead, &pid); + return 0; +} + +SEC("fentry/do_page_cache_ra") +int BPF_PROG(fentry_do_page_cache_ra) +{ + return do_page_cache_ra(); +} + +SEC("fexit/__page_cache_alloc") +int BPF_PROG(fexit_page_cache_alloc, gfp_t gfp, struct page *ret) +{ + return do_page_cache_alloc_ret(ret); +} + +SEC("fexit/do_page_cache_ra") +int BPF_PROG(fexit_do_page_cache_ra) +{ + return do_page_cache_ra_ret(); +} + +SEC("fentry/mark_page_accessed") +int BPF_PROG(fentry_mark_page_accessed, struct page *page) +{ + return do_mark_page_accessed(page); +} + +SEC("kprobe/do_page_cache_ra") +int BPF_KPROBE(kprobe_do_page_cache_ra) +{ + return do_page_cache_ra(); +} + +SEC("kretprobe/do_page_cache_ra") +int BPF_KRETPROBE(kretprobe_do_page_cache_ra) +{ + return do_page_cache_ra_ret(); +} + +SEC("kprobe/__do_page_cache_readahead") +int BPF_KPROBE(kprobe___do_page_cache_readahead) +{ + return do_page_cache_ra(); +} + +SEC("kretprobe/__do_page_cache_readahead") +int BPF_KRETPROBE(kretprobe___do_page_cache_readahead) +{ + return do_page_cache_ra_ret(); +} + +SEC("kretprobe/__page_cache_alloc") +int BPF_KRETPROBE(kretprobe_page_cache_alloc, struct page *ret) +{ + return do_page_cache_alloc_ret(ret); +} + +SEC("kprobe/mark_page_accessed") +int BPF_KPROBE(kprobe_mark_page_accessed, struct page *page) +{ + return do_mark_page_accessed(page); +} + char LICENSE[] SEC("license") = "GPL"; diff --git a/libbpf-tools/readahead.c b/libbpf-tools/readahead.c index 13ea4e6c1502..860131fd7d5b 100644 --- a/libbpf-tools/readahead.c +++ b/libbpf-tools/readahead.c @@ -76,24 +76,6 @@ static void sig_handler(int sig) exiting = true; } -static int readahead__set_attach_target(struct bpf_program *prog) -{ - int err; - - err = bpf_program__set_attach_target(prog, 0, "do_page_cache_ra"); - if (!err) - return 0; - - err = bpf_program__set_attach_target(prog, 0, - "__do_page_cache_readahead"); - if (!err) - return 0; - - fprintf(stderr, "failed to set attach target for %s: %s\n", - bpf_program__name(prog), strerror(-err)); - return err; -} - int main(int argc, char **argv) { static const struct argp argp = { @@ -121,13 +103,45 @@ int main(int argc, char **argv) * Starting from v5.10-rc1 (8238287), __do_page_cache_readahead has * renamed to do_page_cache_ra. So we specify the function dynamically. */ - err = readahead__set_attach_target(obj->progs.do_page_cache_ra); - if (err) - goto cleanup; - err = readahead__set_attach_target(obj->progs.do_page_cache_ra_ret); - if (err) - goto cleanup; + if (kprobe_exists("do_page_cache_ra")) { + if (fentry_can_attach("do_page_cache_ra", NULL)) { + bpf_program__set_attach_target(obj->progs.fentry_do_page_cache_ra, 0, + "do_page_cache_ra"); + bpf_program__set_attach_target(obj->progs.fexit_do_page_cache_ra, 0, + "do_page_cache_ra"); + bpf_program__set_autoload(obj->progs.kprobe_do_page_cache_ra, false); + bpf_program__set_autoload(obj->progs.kretprobe_do_page_cache_ra, false); + } else { + bpf_program__set_autoload(obj->progs.fentry_do_page_cache_ra, false); + bpf_program__set_autoload(obj->progs.fexit_do_page_cache_ra, false); + } + bpf_program__set_autoload(obj->progs.kprobe___do_page_cache_readahead, false); + bpf_program__set_autoload(obj->progs.kretprobe___do_page_cache_readahead, false); + } else { + if (fentry_can_attach("__do_page_cache_readahead", NULL)) { + bpf_program__set_attach_target(obj->progs.fentry_do_page_cache_ra, 0, + "__do_page_cache_readahead"); + bpf_program__set_attach_target(obj->progs.fexit_do_page_cache_ra, 0, + "__do_page_cache_readahead"); + bpf_program__set_autoload(obj->progs.kprobe___do_page_cache_readahead, false); + bpf_program__set_autoload(obj->progs.kretprobe___do_page_cache_readahead, false); + } else { + bpf_program__set_autoload(obj->progs.fentry_do_page_cache_ra, false); + bpf_program__set_autoload(obj->progs.fexit_do_page_cache_ra, false); + } + bpf_program__set_autoload(obj->progs.kprobe_do_page_cache_ra, false); + bpf_program__set_autoload(obj->progs.kretprobe_do_page_cache_ra, false); + } + + if (fentry_can_attach("page_cache_alloc", NULL)) + bpf_program__set_autoload(obj->progs.kretprobe_page_cache_alloc, false); + else + bpf_program__set_autoload(obj->progs.fexit_page_cache_alloc, false); + if (fentry_can_attach("mark_page_accessed", NULL)) + bpf_program__set_autoload(obj->progs.kprobe_mark_page_accessed, false); + else + bpf_program__set_autoload(obj->progs.fentry_mark_page_accessed, false); err = readahead_bpf__load(obj); if (err) { fprintf(stderr, "failed to load BPF object\n");