add library infrastructure and turn allocator into a library

scheds/include/scx/sdt_task.h → lib/include/lib/sdt_task.h

@@ -5,7 +5,7 @@
  * Copyright (c) 2024 Emil Tsalapatis <[email protected]>
  */
 #pragma once
-#include "bpf_arena_common.h"
+#include <scx/bpf_arena_common.h>
 
 #ifndef div_round_up
 #define div_round_up(a, b) (((a) + (b) - 1) / (b))
@@ -90,3 +90,23 @@ struct sdt_stats {
 	__u64	active_allocs;
 	__u64	arena_pages_used;
 };
+
+
+#ifdef __BPF__
+
+void __arena *sdt_task_data(struct task_struct *p);
+int sdt_task_init(__u64 data_size);
+void __arena *sdt_task_alloc(struct task_struct *p);
+void sdt_task_free(struct task_struct *p);
+
+/*
+ * The verifier does not support returning non-scalar values between BPF
+ * programs, even though returning arena pointers is both safe and valid.
+ * This macro typecasts the returned arena pointer on behalf of the caller.
+ */
+#define SDT_TASK_RETRIEVE(_p) ((void __arena *)sdt_task_retrieve(_p))
+#define SDT_TASK_ALLOC(_p) ((void __arena *)sdt_task_alloc(_p))
+/* For uniformity. */
+#define SDT_TASK_FREE(_p) (sdt_task_free(_p))
+
+#endif /* __BPF__ */

lib/meson.build

@@ -0,0 +1,29 @@
+libs = ['sdt_task']
+
+objs = []
+
+foreach lib : libs
+  bpf_o = gen_bpf_o.process(lib + '.bpf.c', extra_args: bpf_includes)
+  tgt = custom_target(lib,
+    output: lib + '.bpf.o',
+    input: lib + '.bpf.c',
+    command: [bpf_clang, bpf_base_cflags, bpf_includes, '-target', 'bpf',
+        libbpf_c_headers, '-c', '@INPUT@', '-o', '@OUTPUT@'],
+    depends: [libbpf],
+    build_by_default: true,
+    build_always_stale: true,
+  )
+
+  objs += [tgt]
+endforeach
+
+custom_target(scx_lib_name,
+  output: scx_lib_name + '.bpf.o',
+  command: [compile_scx_lib, bpftool_exe_path, '@OUTPUT@'],
+  depends: objs,
+  build_by_default: true,
+  build_always_stale: true,
+)
+
+# Install include/lib as a lib/ subdir of our headers
+install_subdir(join_paths(meson.current_source_dir(), 'include/scx'), install_dir: 'include', install_tag: 'devel')

scheds/include/scx/sdt_task_impl.bpf.h → lib/sdt_task.bpf.c

@@ -4,9 +4,9 @@
  * Copyright (c) 2024 Tejun Heo <[email protected]>
  * Copyright (c) 2024 Emil Tsalapatis <[email protected]>
  */
-#pragma once
 
-#include "sdt_task.h"
+#include <scx/common.bpf.h>
+#include <lib/sdt_task.h>
 
 #define SDT_TASK_FN_ATTRS	inline __attribute__((unused, always_inline))
 
@@ -269,7 +269,7 @@ static SDT_TASK_FN_ATTRS int sdt_pool_set_size(struct sdt_task_pool __arena *poo
 }
 
 /* initialize the whole thing, maybe misnomer */
-static SDT_TASK_FN_ATTRS int sdt_task_init(__u64 data_size)
+__hidden SDT_TASK_FN_ATTRS int sdt_task_init(__u64 data_size)
 {
 	int ret;
 
@@ -429,7 +429,7 @@ static SDT_TASK_FN_ATTRS void sdt_task_free_idx(__u64 idx)
 	return;
 }
 
-static SDT_TASK_FN_ATTRS
+__hidden SDT_TASK_FN_ATTRS
 void __arena *sdt_task_data(struct task_struct *p)
 {
 	struct sdt_task_data __arena *data;
@@ -447,7 +447,8 @@ void __arena *sdt_task_data(struct task_struct *p)
 }
 
 
-static SDT_TASK_FN_ATTRS void sdt_task_free(struct task_struct *p)
+__hidden SDT_TASK_FN_ATTRS
+void sdt_task_free(struct task_struct *p)
 {
 	struct sdt_task_map_val *mval;
 
@@ -539,7 +540,7 @@ int sdt_task_find_empty(struct sdt_task_desc __arena *desc,
 	return ret;
 }
 
-static SDT_TASK_FN_ATTRS
+__hidden SDT_TASK_FN_ATTRS
 void __arena *sdt_task_alloc(struct task_struct *p)
 {
 	struct sdt_alloc_stack __arena *stack = prealloc_stack;

meson-scripts/bpftool_build_skel

@@ -6,12 +6,18 @@ bpftool="$1"
 input="$2"
 skel="$3"
 subskel="$4"
+lib="$5"
 
 stem="${input%.o}"
 name="${input%.bpf.o}"
 name="${name##*/}"
 
-"$bpftool" gen object "$stem".l1o "$input"
+if [ `basename $lib` == $name ];
+then
+	"$bpftool" gen object "$stem".l1o "$input"
+else
+	"$bpftool" gen object "$stem".l1o "$input" "$lib".bpf.o
+fi
 "$bpftool" gen object "$stem".l2o "$stem".l1o
 "$bpftool" gen object "$stem".l3o "$stem".l2o
 cmp "$stem".l2o "$stem".l3o

meson-scripts/compile_scx_lib

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+bpftool="$1"
+output="$2"
+dir="$(dirname $output)"
+
+libs=`find $dir -type f -name *.bpf.o | grep -v lib.bpf.o`
+
+bpftool gen object "$output" $libs

meson.build

@@ -47,6 +47,11 @@ fetch_bpftool = find_program(join_paths(meson.current_source_dir(),
                                       'meson-scripts/fetch_bpftool'))
 build_bpftool = find_program(join_paths(meson.current_source_dir(),
                                       'meson-scripts/build_bpftool'))
+compile_scx_lib = find_program(join_paths(meson.current_source_dir(),
+                                      'meson-scripts/compile_scx_lib'))
+
+scx_lib_name = 'lib'
+scx_lib_path = join_paths(meson.current_build_dir(), 'lib/', scx_lib_name)
 
 bpf_clang_ver = run_command(get_clang_ver, bpf_clang, check: true).stdout().strip()
 if bpf_clang_ver == ''
@@ -128,7 +133,7 @@ if should_build_libbpf
   endforeach
 
   message('Fetching libbpf repo')
-  libbpf_commit = '686f600bca59e107af4040d0838ca2b02c14ff50'
+  libbpf_commit = 'c5f22aca0f3aa855daa159b2777472b35e721804'
   run_command(fetch_libbpf, meson.current_build_dir(), libbpf_commit, check: true)
 
   make_jobs = 1
@@ -184,7 +189,7 @@ endif
 
 if should_build_bpftool
   message('Fetching bpftool repo')
-  bpftool_commit = '77a72987353fcae8ce330fd87d4c7afb7677a169'
+  bpftool_commit = '183e7010387d1fc9f08051426e9a9fbd5f8d409e'
   run_command(fetch_bpftool, meson.current_build_dir(), bpftool_commit, check: true)
 
   bpftool_target = custom_target('bpftool_target',
@@ -269,7 +274,20 @@ gen_bpf_o = generator(bpf_clang,
 gen_bpf_skel = generator(bpftool_build_skel,
                          output: ['@[email protected]','@[email protected]' ],
                          depends: [libbpf, bpftool_target],
-                         arguments: [bpftool_exe_path, '@INPUT@', '@OUTPUT0@', '@OUTPUT1@'])
+                         arguments: [bpftool_exe_path, '@INPUT@', '@OUTPUT0@', '@OUTPUT1@', scx_lib_path])
+
+
+# BPF compilation uses the gen_bpf_o generator. The following should be
+# passed in as extra_args.
+bpf_includes = ['-I', join_paths(meson.current_source_dir(), 'scheds/include'),
+                '-I', join_paths(meson.current_source_dir(), 'scheds/include/arch/' + arch_dict[cpu]),
+                '-I', join_paths(meson.current_source_dir(), 'scheds/include/bpf-compat'),
+                '-I', join_paths(meson.current_source_dir(), 'lib/include'),]
+
+
+# Common include paths for user C compilation. The following should be
+# passed in as executable::include_directories.
+subdir('lib')
 
 #
 # For rust sub-projects.
@@ -445,6 +463,8 @@ if enable_stress
   endif
 endif
 
+thread_dep = dependency('threads')
+
 subdir('scheds')
 
 systemd = dependency('systemd', required: get_option('systemd'))

scheds/c/scx_nest.bpf.c

@@ -24,6 +24,7 @@
  * Copyright (c) 2023 Tejun Heo <[email protected]>
  */
 #include <scx/common.bpf.h>
+#include <lib/sdt_task.h>
 
 #include "scx_nest.h"
 
@@ -42,7 +43,6 @@ enum {
 };
 
 #define CLOCK_BOOTTIME 7
-#define NUMA_NO_NODE -1
 
 const volatile u64 p_remove_ns = 2 * NSEC_PER_MSEC;
 const volatile u64 r_max = 5;

scheds/c/scx_sdt.bpf.c

@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 #include <scx/common.bpf.h>
-#include <scx/sdt_task_impl.bpf.h>
+#include <scx/bpf_arena_common.h>
+#include <lib/sdt_task.h>
 
 #include "scx_sdt.h"
 
@@ -11,7 +12,7 @@ UEI_DEFINE(uei);
 #define SHARED_DSQ 0
 
 #define DEFINE_SDT_STAT(metric)				\
-static SDT_TASK_FN_ATTRS void				\
+static inline void				\
 stat_inc_##metric(struct scx_stats __arena *stats)	\
 {							\
 	cast_kern(stats);				\
@@ -25,7 +26,7 @@ DEFINE_SDT_STAT(exit);
 DEFINE_SDT_STAT(select_idle_cpu);
 DEFINE_SDT_STAT(select_busy_cpu);
 
-static SDT_TASK_FN_ATTRS void
+static inline void
 scx_stat_global_update(struct scx_stats __arena *stats)
 {
 	cast_kern(stats);

scheds/c/scx_sdt.c

@@ -11,7 +11,6 @@
 #include <libgen.h>
 #include <bpf/bpf.h>
 #include <scx/common.h>
-#include <scx/sdt_task.h>
 #include "scx_sdt.bpf.skel.h"
 
 const char help_fmt[] =

scheds/include/lib

@@ -0,0 +1 @@
+../../lib/include/lib

scheds/include/scx/common.bpf.h

@@ -386,22 +386,22 @@ void bpf_iter_bits_destroy(struct bpf_iter_bits *it) __ksym;
   }
 #define def_for_each_cpu(cpu, name) for_each_##name##_cpu(cpu)
 
-/// Provides iterator for possible and online cpus. 
+/// Provides iterator for possible and online cpus.
 ///
 /// # Example
 ///
 /// ```
 /// static inline void example_use() {
 ///     int *cpu;
-/// 
+///
 ///     for_each_possible_cpu(cpu){
 ///         bpf_printk("CPU %d is possible", *cpu);
 ///     }
-/// 
+///
 ///     for_each_online_cpu(cpu){
 ///         bpf_printk("CPU %d is online", *cpu);
 ///     }
-/// } 
+/// }
 /// ```
 def_iter_struct(possible);
 def_iter_new(possible);
@@ -418,7 +418,7 @@ def_iter_destroy(online);
 /*
  * Access a cpumask in read-only mode (typically to check bits).
  */
-const struct cpumask *cast_mask(struct bpf_cpumask *mask) {
+const inline struct cpumask *cast_mask(struct bpf_cpumask *mask) {
   return (const struct cpumask *)mask;
 }
 

scheds/include/scx/common.h

@@ -72,5 +72,6 @@ typedef int64_t s64;
 #include "user_exit_info.h"
 #include "compat.h"
 #include "enums.h"
+#include <lib/sdt_task.h>
 
 #endif	/* __SCHED_EXT_COMMON_H */

scheds/meson.build

@@ -1,9 +1,3 @@
-# BPF compilation uses the gen_bpf_o generator. The following should be
-# passed in as extra_args.
-bpf_includes = ['-I', join_paths(meson.current_source_dir(), 'include'),
-                '-I', join_paths(meson.current_source_dir(), 'include/arch/' + arch_dict[cpu]),
-                '-I', join_paths(meson.current_source_dir(), 'include/bpf-compat'),]
-
 # Common include paths for user C compilation. The following should be
 # passed in as executable::include_directories.
 user_c_includes = include_directories('include')