From 3d72a8e8c1ed2269b530de109d052c5cea8bac39 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Wed, 4 Dec 2024 12:56:42 -0800 Subject: [PATCH] add library infrastructure and turn allocator into a library --- .../scx => lib/include/lib}/sdt_task.h | 22 +++++++++++++- lib/meson.build | 29 +++++++++++++++++++ .../sdt_task_impl.bpf.h => lib/sdt_task.bpf.c | 13 +++++---- meson-scripts/bpftool_build_skel | 8 ++++- meson-scripts/compile_scx_lib | 11 +++++++ meson.build | 26 +++++++++++++++-- scheds/c/scx_nest.bpf.c | 2 +- scheds/c/scx_sdt.bpf.c | 7 +++-- scheds/c/scx_sdt.c | 1 - scheds/include/lib | 1 + scheds/include/scx/common.bpf.h | 10 +++---- scheds/include/scx/common.h | 1 + scheds/meson.build | 6 ---- 13 files changed, 110 insertions(+), 27 deletions(-) rename {scheds/include/scx => lib/include/lib}/sdt_task.h (78%) create mode 100644 lib/meson.build rename scheds/include/scx/sdt_task_impl.bpf.h => lib/sdt_task.bpf.c (98%) create mode 100755 meson-scripts/compile_scx_lib create mode 120000 scheds/include/lib diff --git a/scheds/include/scx/sdt_task.h b/lib/include/lib/sdt_task.h similarity index 78% rename from scheds/include/scx/sdt_task.h rename to lib/include/lib/sdt_task.h index 290b93534..c81739046 100644 --- a/scheds/include/scx/sdt_task.h +++ b/lib/include/lib/sdt_task.h @@ -5,7 +5,7 @@ * Copyright (c) 2024 Emil Tsalapatis */ #pragma once -#include "bpf_arena_common.h" +#include #ifndef div_round_up #define div_round_up(a, b) (((a) + (b) - 1) / (b)) @@ -90,3 +90,23 @@ struct sdt_stats { __u64 active_allocs; __u64 arena_pages_used; }; + + +#ifdef __BPF__ + +void __arena *sdt_task_data(struct task_struct *p); +int sdt_task_init(__u64 data_size); +void __arena *sdt_task_alloc(struct task_struct *p); +void sdt_task_free(struct task_struct *p); + +/* + * The verifier does not support returning non-scalar values between BPF + * programs, even though returning arena pointers is both safe and valid. + * This macro typecasts the returned arena pointer on behalf of the caller. + */ +#define SDT_TASK_RETRIEVE(_p) ((void __arena *)sdt_task_retrieve(_p)) +#define SDT_TASK_ALLOC(_p) ((void __arena *)sdt_task_alloc(_p)) +/* For uniformity. */ +#define SDT_TASK_FREE(_p) (sdt_task_free(_p)) + +#endif /* __BPF__ */ diff --git a/lib/meson.build b/lib/meson.build new file mode 100644 index 000000000..eb8061dda --- /dev/null +++ b/lib/meson.build @@ -0,0 +1,29 @@ +libs = ['sdt_task'] + +objs = [] + +foreach lib : libs + bpf_o = gen_bpf_o.process(lib + '.bpf.c', extra_args: bpf_includes) + tgt = custom_target(lib, + output: lib + '.bpf.o', + input: lib + '.bpf.c', + command: [bpf_clang, bpf_base_cflags, bpf_includes, '-target', 'bpf', + libbpf_c_headers, '-c', '@INPUT@', '-o', '@OUTPUT@'], + depends: [libbpf], + build_by_default: true, + build_always_stale: true, + ) + + objs += [tgt] +endforeach + +custom_target(scx_lib_name, + output: scx_lib_name + '.bpf.o', + command: [compile_scx_lib, bpftool_exe_path, '@OUTPUT@'], + depends: objs, + build_by_default: true, + build_always_stale: true, +) + +# Install include/lib as a lib/ subdir of our headers +install_subdir(join_paths(meson.current_source_dir(), 'include/scx'), install_dir: 'include', install_tag: 'devel') diff --git a/scheds/include/scx/sdt_task_impl.bpf.h b/lib/sdt_task.bpf.c similarity index 98% rename from scheds/include/scx/sdt_task_impl.bpf.h rename to lib/sdt_task.bpf.c index 4eb555b55..720bc7f0b 100644 --- a/scheds/include/scx/sdt_task_impl.bpf.h +++ b/lib/sdt_task.bpf.c @@ -4,9 +4,9 @@ * Copyright (c) 2024 Tejun Heo * Copyright (c) 2024 Emil Tsalapatis */ -#pragma once -#include "sdt_task.h" +#include +#include #define SDT_TASK_FN_ATTRS inline __attribute__((unused, always_inline)) @@ -269,7 +269,7 @@ static SDT_TASK_FN_ATTRS int sdt_pool_set_size(struct sdt_task_pool __arena *poo } /* initialize the whole thing, maybe misnomer */ -static SDT_TASK_FN_ATTRS int sdt_task_init(__u64 data_size) +__hidden SDT_TASK_FN_ATTRS int sdt_task_init(__u64 data_size) { int ret; @@ -429,7 +429,7 @@ static SDT_TASK_FN_ATTRS void sdt_task_free_idx(__u64 idx) return; } -static SDT_TASK_FN_ATTRS +__hidden SDT_TASK_FN_ATTRS void __arena *sdt_task_data(struct task_struct *p) { struct sdt_task_data __arena *data; @@ -447,7 +447,8 @@ void __arena *sdt_task_data(struct task_struct *p) } -static SDT_TASK_FN_ATTRS void sdt_task_free(struct task_struct *p) +__hidden SDT_TASK_FN_ATTRS +void sdt_task_free(struct task_struct *p) { struct sdt_task_map_val *mval; @@ -539,7 +540,7 @@ int sdt_task_find_empty(struct sdt_task_desc __arena *desc, return ret; } -static SDT_TASK_FN_ATTRS +__hidden SDT_TASK_FN_ATTRS void __arena *sdt_task_alloc(struct task_struct *p) { struct sdt_alloc_stack __arena *stack = prealloc_stack; diff --git a/meson-scripts/bpftool_build_skel b/meson-scripts/bpftool_build_skel index a3a651805..f204a5947 100755 --- a/meson-scripts/bpftool_build_skel +++ b/meson-scripts/bpftool_build_skel @@ -6,12 +6,18 @@ bpftool="$1" input="$2" skel="$3" subskel="$4" +lib="$5" stem="${input%.o}" name="${input%.bpf.o}" name="${name##*/}" -"$bpftool" gen object "$stem".l1o "$input" +if [ `basename $lib` == $name ]; +then + "$bpftool" gen object "$stem".l1o "$input" +else + "$bpftool" gen object "$stem".l1o "$input" "$lib".bpf.o +fi "$bpftool" gen object "$stem".l2o "$stem".l1o "$bpftool" gen object "$stem".l3o "$stem".l2o cmp "$stem".l2o "$stem".l3o diff --git a/meson-scripts/compile_scx_lib b/meson-scripts/compile_scx_lib new file mode 100755 index 000000000..47398f5fc --- /dev/null +++ b/meson-scripts/compile_scx_lib @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +bpftool="$1" +output="$2" +dir="$(dirname $output)" + +libs=`find $dir -type f -name *.bpf.o | grep -v lib.bpf.o` + +bpftool gen object "$output" $libs diff --git a/meson.build b/meson.build index d8a338d07..1a2bc5ae1 100644 --- a/meson.build +++ b/meson.build @@ -47,6 +47,11 @@ fetch_bpftool = find_program(join_paths(meson.current_source_dir(), 'meson-scripts/fetch_bpftool')) build_bpftool = find_program(join_paths(meson.current_source_dir(), 'meson-scripts/build_bpftool')) +compile_scx_lib = find_program(join_paths(meson.current_source_dir(), + 'meson-scripts/compile_scx_lib')) + +scx_lib_name = 'lib' +scx_lib_path = join_paths(meson.current_build_dir(), 'lib/', scx_lib_name) bpf_clang_ver = run_command(get_clang_ver, bpf_clang, check: true).stdout().strip() if bpf_clang_ver == '' @@ -128,7 +133,7 @@ if should_build_libbpf endforeach message('Fetching libbpf repo') - libbpf_commit = '686f600bca59e107af4040d0838ca2b02c14ff50' + libbpf_commit = 'c5f22aca0f3aa855daa159b2777472b35e721804' run_command(fetch_libbpf, meson.current_build_dir(), libbpf_commit, check: true) make_jobs = 1 @@ -184,7 +189,7 @@ endif if should_build_bpftool message('Fetching bpftool repo') - bpftool_commit = '77a72987353fcae8ce330fd87d4c7afb7677a169' + bpftool_commit = '183e7010387d1fc9f08051426e9a9fbd5f8d409e' run_command(fetch_bpftool, meson.current_build_dir(), bpftool_commit, check: true) bpftool_target = custom_target('bpftool_target', @@ -269,7 +274,20 @@ gen_bpf_o = generator(bpf_clang, gen_bpf_skel = generator(bpftool_build_skel, output: ['@BASENAME@.skel.h','@BASENAME@.subskel.h' ], depends: [libbpf, bpftool_target], - arguments: [bpftool_exe_path, '@INPUT@', '@OUTPUT0@', '@OUTPUT1@']) + arguments: [bpftool_exe_path, '@INPUT@', '@OUTPUT0@', '@OUTPUT1@', scx_lib_path]) + + +# BPF compilation uses the gen_bpf_o generator. The following should be +# passed in as extra_args. +bpf_includes = ['-I', join_paths(meson.current_source_dir(), 'scheds/include'), + '-I', join_paths(meson.current_source_dir(), 'scheds/include/arch/' + arch_dict[cpu]), + '-I', join_paths(meson.current_source_dir(), 'scheds/include/bpf-compat'), + '-I', join_paths(meson.current_source_dir(), 'lib/include'),] + + +# Common include paths for user C compilation. The following should be +# passed in as executable::include_directories. +subdir('lib') # # For rust sub-projects. @@ -445,6 +463,8 @@ if enable_stress endif endif +thread_dep = dependency('threads') + subdir('scheds') systemd = dependency('systemd', required: get_option('systemd')) diff --git a/scheds/c/scx_nest.bpf.c b/scheds/c/scx_nest.bpf.c index 0766413ab..78359dac8 100644 --- a/scheds/c/scx_nest.bpf.c +++ b/scheds/c/scx_nest.bpf.c @@ -24,6 +24,7 @@ * Copyright (c) 2023 Tejun Heo */ #include +#include #include "scx_nest.h" @@ -42,7 +43,6 @@ enum { }; #define CLOCK_BOOTTIME 7 -#define NUMA_NO_NODE -1 const volatile u64 p_remove_ns = 2 * NSEC_PER_MSEC; const volatile u64 r_max = 5; diff --git a/scheds/c/scx_sdt.bpf.c b/scheds/c/scx_sdt.bpf.c index 4c0ded725..ea2e449dc 100644 --- a/scheds/c/scx_sdt.bpf.c +++ b/scheds/c/scx_sdt.bpf.c @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include -#include +#include +#include #include "scx_sdt.h" @@ -11,7 +12,7 @@ UEI_DEFINE(uei); #define SHARED_DSQ 0 #define DEFINE_SDT_STAT(metric) \ -static SDT_TASK_FN_ATTRS void \ +static inline void \ stat_inc_##metric(struct scx_stats __arena *stats) \ { \ cast_kern(stats); \ @@ -25,7 +26,7 @@ DEFINE_SDT_STAT(exit); DEFINE_SDT_STAT(select_idle_cpu); DEFINE_SDT_STAT(select_busy_cpu); -static SDT_TASK_FN_ATTRS void +static inline void scx_stat_global_update(struct scx_stats __arena *stats) { cast_kern(stats); diff --git a/scheds/c/scx_sdt.c b/scheds/c/scx_sdt.c index 87a0d6eff..45798154b 100644 --- a/scheds/c/scx_sdt.c +++ b/scheds/c/scx_sdt.c @@ -11,7 +11,6 @@ #include #include #include -#include #include "scx_sdt.bpf.skel.h" const char help_fmt[] = diff --git a/scheds/include/lib b/scheds/include/lib new file mode 120000 index 000000000..66f9e6741 --- /dev/null +++ b/scheds/include/lib @@ -0,0 +1 @@ +../../lib/include/lib \ No newline at end of file diff --git a/scheds/include/scx/common.bpf.h b/scheds/include/scx/common.bpf.h index 6a51bdf72..93376654a 100644 --- a/scheds/include/scx/common.bpf.h +++ b/scheds/include/scx/common.bpf.h @@ -386,22 +386,22 @@ void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym; } #define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu) -/// Provides iterator for possible and online cpus. +/// Provides iterator for possible and online cpus. /// /// # Example /// /// ``` /// static inline void example_use() { /// int *cpu; -/// +/// /// for_each_possible_cpu(cpu){ /// bpf_printk("CPU %d is possible", *cpu); /// } -/// +/// /// for_each_online_cpu(cpu){ /// bpf_printk("CPU %d is online", *cpu); /// } -/// } +/// } /// ``` def_iter_struct(possible); def_iter_new(possible); @@ -418,7 +418,7 @@ def_iter_destroy(online); /* * Access a cpumask in read-only mode (typically to check bits). */ -const struct cpumask *cast_mask(struct bpf_cpumask *mask) { +const inline struct cpumask *cast_mask(struct bpf_cpumask *mask) { return (const struct cpumask *)mask; } diff --git a/scheds/include/scx/common.h b/scheds/include/scx/common.h index 1f3a0bd96..2f79578c7 100644 --- a/scheds/include/scx/common.h +++ b/scheds/include/scx/common.h @@ -72,5 +72,6 @@ typedef int64_t s64; #include "user_exit_info.h" #include "compat.h" #include "enums.h" +#include #endif /* __SCHED_EXT_COMMON_H */ diff --git a/scheds/meson.build b/scheds/meson.build index 4144227a7..65fa412c6 100644 --- a/scheds/meson.build +++ b/scheds/meson.build @@ -1,9 +1,3 @@ -# BPF compilation uses the gen_bpf_o generator. The following should be -# passed in as extra_args. -bpf_includes = ['-I', join_paths(meson.current_source_dir(), 'include'), - '-I', join_paths(meson.current_source_dir(), 'include/arch/' + arch_dict[cpu]), - '-I', join_paths(meson.current_source_dir(), 'include/bpf-compat'),] - # Common include paths for user C compilation. The following should be # passed in as executable::include_directories. user_c_includes = include_directories('include')