Merge pull request #3889 from kinvolk/mauricio/btfgen-integration

libbpf-tools: Add experimental BTFGen integration

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "src/cc/libbpf"]
 	path = src/cc/libbpf
 	url = https://github.com/libbpf/libbpf.git
+[submodule "libbpf-tools/bpftool"]
+	path = libbpf-tools/bpftool
+	url = https://github.com/libbpf/bpftool

libbpf-tools/.gitignore

@@ -1,4 +1,5 @@
 /.output
+/btfhub-archive
 /bashreadline
 /bindsnoop
 /biolatency

libbpf-tools/Makefile

@@ -1,8 +1,9 @@
 # SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
-OUTPUT := .output
+OUTPUT := $(abspath .output)
 CLANG ?= clang
 LLVM_STRIP ?= llvm-strip
-BPFTOOL ?= bin/bpftool
+BPFTOOL_SRC := $(abspath ./bpftool/src)
+BPFTOOL ?= $(OUTPUT)/bpftool/bpftool
 LIBBPF_SRC := $(abspath ../src/cc/libbpf/src)
 LIBBPF_OBJ := $(abspath $(OUTPUT)/libbpf.a)
 INCLUDES := -I$(OUTPUT) -I../src/cc/libbpf/include/uapi
@@ -11,6 +12,7 @@ BPFCFLAGS := -g -O2 -Wall
 INSTALL ?= install
 prefix ?= /usr/local
 ARCH := $(shell uname -m | sed 's/x86_64/x86/' | sed 's/aarch64/arm64/' | sed 's/ppc64le/powerpc/' | sed 's/mips.*/mips/')
+BTFHUB_ARCHIVE ?= $(abspath btfhub-archive)
 
 ifeq ($(wildcard $(ARCH)/),)
 $(error Architecture $(ARCH) is not supported yet. Please open an issue)
@@ -60,6 +62,9 @@ APPS = \
 	vfsstat \
 	#
 
+# export variables that are used in Makefile.btfgen as well.
+export OUTPUT BPFTOOL ARCH BTFHUB_ARCHIVE APPS
+
 FSDIST_ALIASES = btrfsdist ext4dist nfsdist xfsdist
 FSSLOWER_ALIASES = btrfsslower ext4slower nfsslower xfsslower
 APP_ALIASES = $(FSDIST_ALIASES) $(FSSLOWER_ALIASES)
@@ -70,6 +75,8 @@ COMMON_OBJ = \
 	$(OUTPUT)/errno_helpers.o \
 	$(OUTPUT)/map_helpers.o \
 	$(OUTPUT)/uprobe_helpers.o \
+	$(OUTPUT)/btf_helpers.o \
+	$(if $(ENABLE_MIN_CORE_BTFS),$(OUTPUT)/min_core_btf_tar.o) \
 	#
 
 .PHONY: all
@@ -93,6 +100,11 @@ $(OUTPUT) $(OUTPUT)/libbpf:
 	$(call msg,MKDIR,$@)
 	$(Q)mkdir -p $@
 
+.PHONY: bpftool
+bpftool:
+	$(Q)mkdir -p $(OUTPUT)/bpftool
+	$(Q)$(MAKE) OUTPUT=$(OUTPUT)/bpftool/ -C $(BPFTOOL_SRC)
+
 $(APPS): %: $(OUTPUT)/%.o $(LIBBPF_OBJ) $(COMMON_OBJ) | $(OUTPUT)
 	$(call msg,BINARY,$@)
 	$(Q)$(CC) $(CFLAGS) $^ $(LDFLAGS) -lelf -lz -o $@
@@ -103,7 +115,7 @@ $(OUTPUT)/%.o: %.c $(wildcard %.h) $(LIBBPF_OBJ) | $(OUTPUT)
 	$(call msg,CC,$@)
 	$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
 
-$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT)
+$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(OUTPUT) bpftool
 	$(call msg,GEN-SKEL,$@)
 	$(Q)$(BPFTOOL) gen skeleton $< > $@
 
@@ -113,6 +125,16 @@ $(OUTPUT)/%.bpf.o: %.bpf.c $(LIBBPF_OBJ) $(wildcard %.h) $(ARCH)/vmlinux.h | $(O
 		     -I$(ARCH)/ $(INCLUDES) -c $(filter %.c,$^) -o $@ &&      \
 	$(LLVM_STRIP) -g $@
 
+btfhub-archive: force
+	$(call msg,GIT,$@)
+	$(Q)[ -d "$(BTFHUB_ARCHIVE)" ] || git clone -q https://github.com/aquasecurity/btfhub-archive/ $(BTFHUB_ARCHIVE)
+	$(Q)cd $(BTFHUB_ARCHIVE) && git pull
+
+ifdef ENABLE_MIN_CORE_BTFS
+$(OUTPUT)/min_core_btf_tar.o: $(patsubst %,$(OUTPUT)/%.bpf.o,$(APPS)) btfhub-archive | bpftool
+	$(Q)$(MAKE) -f Makefile.btfgen
+endif
+
 # Build libbpf.a
 $(LIBBPF_OBJ): $(wildcard $(LIBBPF_SRC)/*.[ch]) | $(OUTPUT)/libbpf
 	$(call msg,LIB,$@)
@@ -135,6 +157,9 @@ install: $(APPS) $(APP_ALIASES)
 	$(Q)$(INSTALL) $(APPS) $(DESTDIR)$(prefix)/bin
 	$(Q)cp -a $(APP_ALIASES) $(DESTDIR)$(prefix)/bin
 
+.PHONY: force
+force:
+
 # delete failed targets
 .DELETE_ON_ERROR:
 # keep intermediate (.skel.h, .bpf.o, etc) targets

libbpf-tools/Makefile.btfgen

@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+SOURCE_BTF_FILES = $(shell find $(BTFHUB_ARCHIVE)/ -iregex ".*$(subst x86,x86_64,$(ARCH)).*" -type f -name '*.btf.tar.xz')
+MIN_CORE_BTF_FILES = $(patsubst $(BTFHUB_ARCHIVE)/%.btf.tar.xz, $(OUTPUT)/min_core_btfs/%.btf, $(SOURCE_BTF_FILES))
+BPF_O_FILES = $(patsubst %,$(OUTPUT)/%.bpf.o,$(APPS))
+
+.PHONY: all
+all: $(OUTPUT)/min_core_btf_tar.o
+
+ifeq ($(V),1)
+Q =
+msg =
+else
+Q = @
+msg = @printf '  %-8s %s%s\n' "$(1)" "$(notdir $(2))" "$(if $(3), $(3))";
+MAKEFLAGS += --no-print-directory
+endif
+
+$(BTFHUB_ARCHIVE)/%.btf: $(BTFHUB_ARCHIVE)/%.btf.tar.xz
+	$(call msg,UNTAR,$@)
+	$(Q)tar xvfJ $< -C "$(@D)" > /dev/null
+	$(Q)touch $@
+
+$(MIN_CORE_BTF_FILES): $(BPF_O_FILES)
+
+# Create reduced version of BTF files to be embedded within the tools executables
+$(OUTPUT)/min_core_btfs/%.btf: $(BTFHUB_ARCHIVE)/%.btf
+	$(call msg,BTFGEN,$@)
+	$(Q)mkdir -p "$(@D)"
+	$(Q)$(BPFTOOL) gen min_core_btf $< $@ $(OUTPUT)/*.bpf.o
+
+# Compress reduced BTF files and create an object file with its content
+$(OUTPUT)/min_core_btf_tar.o: $(MIN_CORE_BTF_FILES)
+	$(call msg,TAR,$@)
+	$(Q)tar c --gz -f $(OUTPUT)/min_core_btfs.tar.gz -C $(OUTPUT)/min_core_btfs/ .
+	$(Q)cd $(OUTPUT) && ld -r -b binary min_core_btfs.tar.gz -o $@
+
+# delete failed targets
+.DELETE_ON_ERROR:
+# keep intermediate (.skel.h, .bpf.o, etc) targets
+.SECONDARY:

libbpf-tools/README.md

@@ -16,12 +16,12 @@ directory. `make clean` will clean up all the build artifacts, including
 generated binaries.
 
 Given that the libbpf package might not be available across wide variety of
-distributions, all libbpf-based tools are linked statically against a version 
+distributions, all libbpf-based tools are linked statically against a version
 of libbpf that BCC links against (from submodule under src/cc/libbpf). This
 results in binaries with minimal amount of dependencies (libc, libelf, and
-libz are linked dynamically, though, given their widespread availability). 
-If your build fails because the libbpf submodule is outdated, try running `git 
-submodule update --init --recursive`. 
+libz are linked dynamically, though, given their widespread availability).
+If your build fails because the libbpf submodule is outdated, try running `git
+submodule update --init --recursive`.
 
 Tools are expected to follow a simple naming convention:
   - <tool>.c contains userspace C code of a tool.
@@ -92,3 +92,30 @@ CONFIG_DEBUG_INFO=y
 kernel build (it comes from dwarves package). Without it, BTF won't be
 generated, and on older kernels you'd get only warning, but still would
 build kernel successfully
+
+Running in kernels without CONFIG_DEBUG_INFO_BTF=y
+--------------------------------------------------
+
+It's possible to run some tools in kernels that don't expose
+`/sys/kernel/btf/vmlinux`. For those cases,
+[BTFGen](https://lore.kernel.org/bpf/[email protected])
+and [BTFHub](https://github.com/aquasecurity/btfhub) can be used to
+generate small BTF files for the most popular Linux distributions that
+are shipped with the tools in order to provide the needed information to
+perform the CO-RE relocations when loading the eBPF programs.
+
+If you haven't cloned the
+[btfhub-archive](https://github.com/aquasecurity/btfhub) repository, you
+can run make and it'll clone it for you into the `$HOME/.local/share`
+directory:
+
+```bash
+make ENABLE_MIN_CORE_BTFS=1 -j$(nproc)
+```
+
+If you have a local copy of such repository, you can pass it's location
+to avoid cloning it again:
+
+```bash
+make ENABLE_MIN_CORE_BTFS=1 BTF_HUB_ARCHIVE=<path_to_btfhub-archive> -j$(nproc)
+```

libbpf-tools/bashreadline.c

@@ -12,6 +12,7 @@
 #include <bpf/bpf.h>
 #include "bashreadline.h"
 #include "bashreadline.skel.h"
+#include "btf_helpers.h"
 #include "trace_helpers.h"
 #include "uprobe_helpers.h"
 
@@ -143,6 +144,7 @@ static void sig_int(int signo)
 
 int main(int argc, char **argv)
 {
+	LIBBPF_OPTS(bpf_object_open_opts, open_opts);
 	static const struct argp argp = {
 		.options = opts,
 		.parser = parse_arg,
@@ -168,9 +170,21 @@ int main(int argc, char **argv)
 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
 	libbpf_set_print(libbpf_print_fn);
 
-	obj = bashreadline_bpf__open_and_load();
+	err = ensure_core_btf(&open_opts);
+	if (err) {
+		warn("failed to fetch necessary BTF for CO-RE: %s\n", strerror(-err));
+		goto cleanup;
+	}
+
+	obj = bashreadline_bpf__open_opts(&open_opts);
 	if (!obj) {
-		warn("failed to open and load BPF object\n");
+		warn("failed to open BPF object\n");
+		goto cleanup;
+	}
+
+	err = bashreadline_bpf__load(obj);
+	if (err) {
+		warn("failed to load BPF object: %d\n", err);
 		goto cleanup;
 	}
 
@@ -217,6 +231,7 @@ int main(int argc, char **argv)
 		free(readline_so_path);
 	perf_buffer__free(pb);
 	bashreadline_bpf__destroy(obj);
+	cleanup_core_btf(&open_opts);
 
 	return err != 0;
 }

libbpf-tools/bindsnoop.c

@@ -13,12 +13,14 @@
 #include <string.h>
 #include <sys/socket.h>
 #include <time.h>
+#include <unistd.h>
 
 #include <bpf/libbpf.h>
 #include <bpf/bpf.h>
 #include "bindsnoop.h"
 #include "bindsnoop.skel.h"
 #include "trace_helpers.h"
+#include "btf_helpers.h"
 
 #define PERF_BUFFER_PAGES	16
 #define PERF_POLL_TIMEOUT_MS	100
@@ -169,6 +171,7 @@ static void handle_lost_events(void *ctx, int cpu, __u64 lost_cnt)
 
 int main(int argc, char **argv)
 {
+	LIBBPF_OPTS(bpf_object_open_opts, open_opts);
 	static const struct argp argp = {
 		.options = opts,
 		.parser = parse_arg,
@@ -187,7 +190,13 @@ int main(int argc, char **argv)
 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
 	libbpf_set_print(libbpf_print_fn);
 
-	obj = bindsnoop_bpf__open();
+	err = ensure_core_btf(&open_opts);
+	if (err) {
+		fprintf(stderr, "failed to fetch necessary BTF for CO-RE: %s\n", strerror(-err));
+		return 1;
+	}
+
+	obj = bindsnoop_bpf__open_opts(&open_opts);
 	if (!obj) {
 		warn("failed to open BPF object\n");
 		return 1;
@@ -251,6 +260,7 @@ int main(int argc, char **argv)
 cleanup:
 	perf_buffer__free(pb);
 	bindsnoop_bpf__destroy(obj);
+	cleanup_core_btf(&open_opts);
 
 	return err != 0;
 }

libbpf-tools/biopattern.c

@@ -12,6 +12,7 @@
 #include <bpf/bpf.h>
 #include "biopattern.h"
 #include "biopattern.skel.h"
+#include "btf_helpers.h"
 #include "trace_helpers.h"
 
 static struct env {
@@ -158,6 +159,7 @@ static int print_map(struct bpf_map *counters, struct partitions *partitions)
 
 int main(int argc, char **argv)
 {
+	LIBBPF_OPTS(bpf_object_open_opts, open_opts);
 	struct partitions *partitions = NULL;
 	const struct partition *partition;
 	static const struct argp argp = {
@@ -175,7 +177,13 @@ int main(int argc, char **argv)
 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
 	libbpf_set_print(libbpf_print_fn);
 
-	obj = biopattern_bpf__open();
+	err = ensure_core_btf(&open_opts);
+	if (err) {
+		fprintf(stderr, "failed to fetch necessary BTF for CO-RE: %s\n", strerror(-err));
+		return 1;
+	}
+
+	obj = biopattern_bpf__open_opts(&open_opts);
 	if (!obj) {
 		fprintf(stderr, "failed to open BPF object\n");
 		return 1;
@@ -234,6 +242,7 @@ int main(int argc, char **argv)
 cleanup:
 	biopattern_bpf__destroy(obj);
 	partitions__free(partitions);
+	cleanup_core_btf(&open_opts);
 
 	return err != 0;
 }