From 28168861ff3ffe12d21d5bb79e89fa82446b934e Mon Sep 17 00:00:00 2001 From: Wouter Dullaert Date: Mon, 24 Jun 2024 16:56:34 +0100 Subject: [PATCH 1/3] bpf2go: export binding generator Moves the code to generate go bindings to its own dedicated package and export the necessary functions to use it in bpf2go. This change will make it possible to reuse this code in other tools, without having to compile the C programs. The behaviour of bpf2go is unchanged. [ Lorenz: move code and backed out changes around module and identifier. These can come later, with tests. ] Co-authored-by: Lorenz Bauer Signed-off-by: Wouter Dullaert --- cmd/bpf2go/{ => gen}/output.go | 144 ++++++++++------------------ cmd/bpf2go/{ => gen}/output.tpl | 0 cmd/bpf2go/{ => gen}/output_test.go | 50 ++++------ cmd/bpf2go/gen/types.go | 44 +++++++++ cmd/bpf2go/gen/types_test.go | 26 +++++ cmd/bpf2go/internal/module.go | 9 ++ cmd/bpf2go/main.go | 57 ++++++++--- cmd/bpf2go/main_test.go | 3 +- cmd/bpf2go/tools.go | 18 ---- 9 files changed, 192 insertions(+), 159 deletions(-) rename cmd/bpf2go/{ => gen}/output.go (59%) rename cmd/bpf2go/{ => gen}/output.tpl (100%) rename cmd/bpf2go/{ => gen}/output_test.go (55%) create mode 100644 cmd/bpf2go/gen/types.go create mode 100644 cmd/bpf2go/gen/types_test.go create mode 100644 cmd/bpf2go/internal/module.go diff --git a/cmd/bpf2go/output.go b/cmd/bpf2go/gen/output.go similarity index 59% rename from cmd/bpf2go/output.go rename to cmd/bpf2go/gen/output.go index 51f197070..a054fd2f1 100644 --- a/cmd/bpf2go/output.go +++ b/cmd/bpf2go/gen/output.go @@ -1,4 +1,4 @@ -package main +package gen import ( "bytes" @@ -10,9 +10,11 @@ import ( "sort" "strings" "text/template" + "unicode" + "unicode/utf8" - "github.com/cilium/ebpf" "github.com/cilium/ebpf/btf" + b2gInt "github.com/cilium/ebpf/cmd/bpf2go/internal" "github.com/cilium/ebpf/internal" ) @@ -71,38 +73,58 @@ func (n templateName) CloseHelper() string { return "_" + toUpperFirst(string(n)) + "Close" } -type outputArgs struct { +type GenerateArgs struct { // Package of the resulting file. - pkg string + Package string // The prefix of all names declared at the top-level. - stem string - // Build constraints included in the resulting file. - constraints constraint.Expr + Stem string + // Build Constraints included in the resulting file. + Constraints constraint.Expr // Maps to be emitted. - maps []string + Maps []string // Programs to be emitted. - programs []string + Programs []string // Types to be emitted. - types []btf.Type - // Filename of the ELF object to embed. - obj string - out io.Writer + Types []btf.Type + // Filename of the object to embed. + ObjectFile string + // Output to write template to. + Output io.Writer } -func output(args outputArgs) error { +// Generate bindings for a BPF ELF file. +func Generate(args GenerateArgs) error { + if !token.IsIdentifier(args.Stem) { + return fmt.Errorf("%q is not a valid identifier", args.Stem) + } + + if strings.ContainsAny(args.ObjectFile, "\n") { + // Prevent injecting newlines into the template. + return fmt.Errorf("file %q contains an invalid character", args.ObjectFile) + } + + for _, typ := range args.Types { + if _, ok := btf.As[*btf.Datasec](typ); ok { + // Avoid emitting .rodata, .bss, etc. for now. We might want to + // name these types differently, etc. + return fmt.Errorf("can't output btf.Datasec: %s", typ) + } + } + maps := make(map[string]string) - for _, name := range args.maps { + for _, name := range args.Maps { maps[name] = internal.Identifier(name) } programs := make(map[string]string) - for _, name := range args.programs { + for _, name := range args.Programs { programs[name] = internal.Identifier(name) } typeNames := make(map[btf.Type]string) - for _, typ := range args.types { - typeNames[typ] = args.stem + internal.Identifier(typ.TypeName()) + for _, typ := range args.Types { + // NB: This also deduplicates types. + typeNames[typ] = args.Stem + internal.Identifier(typ.TypeName()) } // Ensure we don't have conflicting names and generate a sorted list of @@ -112,8 +134,6 @@ func output(args outputArgs) error { return err } - module := currentModule() - gf := &btf.GoFormatter{ Names: typeNames, Identifier: internal.Identifier, @@ -132,15 +152,15 @@ func output(args outputArgs) error { File string }{ gf, - module, - args.pkg, - args.constraints, - templateName(args.stem), + b2gInt.CurrentModule, + args.Package, + args.Constraints, + templateName(args.Stem), maps, programs, types, typeNames, - args.obj, + args.ObjectFile, } var buf bytes.Buffer @@ -148,74 +168,7 @@ func output(args outputArgs) error { return fmt.Errorf("can't generate types: %s", err) } - return internal.WriteFormatted(buf.Bytes(), args.out) -} - -func collectFromSpec(spec *ebpf.CollectionSpec, cTypes []string, skipGlobalTypes bool) (maps, programs []string, types []btf.Type, _ error) { - for name := range spec.Maps { - // Skip .rodata, .data, .bss, etc. sections - if !strings.HasPrefix(name, ".") { - maps = append(maps, name) - } - } - - for name := range spec.Programs { - programs = append(programs, name) - } - - types, err := collectCTypes(spec.Types, cTypes) - if err != nil { - return nil, nil, nil, fmt.Errorf("collect C types: %w", err) - } - - // Collect map key and value types, unless we've been asked not to. - if skipGlobalTypes { - return maps, programs, types, nil - } - - for _, typ := range collectMapTypes(spec.Maps) { - switch btf.UnderlyingType(typ).(type) { - case *btf.Datasec: - // Avoid emitting .rodata, .bss, etc. for now. We might want to - // name these types differently, etc. - continue - - case *btf.Int: - // Don't emit primitive types by default. - continue - } - - types = append(types, typ) - } - - return maps, programs, types, nil -} - -func collectCTypes(types *btf.Spec, names []string) ([]btf.Type, error) { - var result []btf.Type - for _, cType := range names { - typ, err := types.AnyTypeByName(cType) - if err != nil { - return nil, err - } - result = append(result, typ) - } - return result, nil -} - -// collectMapTypes returns a list of all types used as map keys or values. -func collectMapTypes(maps map[string]*ebpf.MapSpec) []btf.Type { - var result []btf.Type - for _, m := range maps { - if m.Key != nil && m.Key.TypeName() != "" { - result = append(result, m.Key) - } - - if m.Value != nil && m.Value.TypeName() != "" { - result = append(result, m.Value) - } - } - return result + return internal.WriteFormatted(buf.Bytes(), args.Output) } // sortTypes returns a list of types sorted by their (generated) Go type name. @@ -242,3 +195,8 @@ func sortTypes(typeNames map[btf.Type]string) ([]btf.Type, error) { return types, nil } + +func toUpperFirst(str string) string { + first, n := utf8.DecodeRuneInString(str) + return string(unicode.ToUpper(first)) + str[n:] +} diff --git a/cmd/bpf2go/output.tpl b/cmd/bpf2go/gen/output.tpl similarity index 100% rename from cmd/bpf2go/output.tpl rename to cmd/bpf2go/gen/output.tpl diff --git a/cmd/bpf2go/output_test.go b/cmd/bpf2go/gen/output_test.go similarity index 55% rename from cmd/bpf2go/output_test.go rename to cmd/bpf2go/gen/output_test.go index 90db264ce..a0816c529 100644 --- a/cmd/bpf2go/output_test.go +++ b/cmd/bpf2go/gen/output_test.go @@ -1,14 +1,15 @@ -package main +package gen import ( + "bytes" + "fmt" "testing" "github.com/go-quicktest/qt" "github.com/google/go-cmp/cmp" - "github.com/cilium/ebpf" "github.com/cilium/ebpf/btf" - "github.com/cilium/ebpf/internal/testutils" + "github.com/cilium/ebpf/cmd/bpf2go/internal" ) func TestOrderTypes(t *testing.T) { @@ -63,35 +64,20 @@ func TestOrderTypes(t *testing.T) { } } +func TestPackageImport(t *testing.T) { + var buf bytes.Buffer + err := Generate(GenerateArgs{ + Package: "foo", + Stem: "bar", + ObjectFile: "frob.o", + Output: &buf, + }) + qt.Assert(t, qt.IsNil(err)) + // NB: It'd be great to test that this is the case for callers outside of + // this module, but that is kind of tricky. + qt.Assert(t, qt.StringContains(buf.String(), fmt.Sprintf(`"%s"`, internal.CurrentModule))) +} + var typesEqualComparer = cmp.Comparer(func(a, b btf.Type) bool { return a == b }) - -func TestCollectFromSpec(t *testing.T) { - spec, err := ebpf.LoadCollectionSpec(testutils.NativeFile(t, "testdata/minimal-%s.elf")) - if err != nil { - t.Fatal(err) - } - - map1 := spec.Maps["map1"] - - maps, programs, types, err := collectFromSpec(spec, nil, false) - if err != nil { - t.Fatal(err) - } - qt.Assert(t, qt.ContentEquals(maps, []string{"map1"})) - qt.Assert(t, qt.ContentEquals(programs, []string{"filter"})) - qt.Assert(t, qt.CmpEquals(types, []btf.Type{map1.Key, map1.Value}, typesEqualComparer)) - - _, _, types, err = collectFromSpec(spec, nil, true) - if err != nil { - t.Fatal(err) - } - qt.Assert(t, qt.CmpEquals[[]btf.Type](types, nil, typesEqualComparer)) - - _, _, types, err = collectFromSpec(spec, []string{"barfoo"}, true) - if err != nil { - t.Fatal(err) - } - qt.Assert(t, qt.CmpEquals(types, []btf.Type{map1.Value}, typesEqualComparer)) -} diff --git a/cmd/bpf2go/gen/types.go b/cmd/bpf2go/gen/types.go new file mode 100644 index 000000000..37dad0c76 --- /dev/null +++ b/cmd/bpf2go/gen/types.go @@ -0,0 +1,44 @@ +package gen + +import ( + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/btf" +) + +// CollectGlobalTypes finds all types which are used in the global scope. +// +// This currently includes the types of map keys and values. +func CollectGlobalTypes(spec *ebpf.CollectionSpec) []btf.Type { + var types []btf.Type + for _, typ := range collectMapTypes(spec.Maps) { + switch btf.UnderlyingType(typ).(type) { + case *btf.Datasec: + // Avoid emitting .rodata, .bss, etc. for now. We might want to + // name these types differently, etc. + continue + + case *btf.Int: + // Don't emit primitive types by default. + continue + } + + types = append(types, typ) + } + + return types +} + +// collectMapTypes returns a list of all types used as map keys or values. +func collectMapTypes(maps map[string]*ebpf.MapSpec) []btf.Type { + var result []btf.Type + for _, m := range maps { + if m.Key != nil && m.Key.TypeName() != "" { + result = append(result, m.Key) + } + + if m.Value != nil && m.Value.TypeName() != "" { + result = append(result, m.Value) + } + } + return result +} diff --git a/cmd/bpf2go/gen/types_test.go b/cmd/bpf2go/gen/types_test.go new file mode 100644 index 000000000..bb5866301 --- /dev/null +++ b/cmd/bpf2go/gen/types_test.go @@ -0,0 +1,26 @@ +package gen + +import ( + "testing" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/btf" + "github.com/cilium/ebpf/internal/testutils" + + "github.com/go-quicktest/qt" +) + +func TestCollectGlobalTypes(t *testing.T) { + spec, err := ebpf.LoadCollectionSpec(testutils.NativeFile(t, "../testdata/minimal-%s.elf")) + if err != nil { + t.Fatal(err) + } + + map1 := spec.Maps["map1"] + + types := CollectGlobalTypes(spec) + if err != nil { + t.Fatal(err) + } + qt.Assert(t, qt.CmpEquals(types, []btf.Type{map1.Key, map1.Value}, typesEqualComparer)) +} diff --git a/cmd/bpf2go/internal/module.go b/cmd/bpf2go/internal/module.go new file mode 100644 index 000000000..43f7f6d4f --- /dev/null +++ b/cmd/bpf2go/internal/module.go @@ -0,0 +1,9 @@ +package internal + +// We used to have some clever code here which relied on debug.ReadBuildInfo(). +// This is broken due to https://github.com/golang/go/issues/33976, and some build +// systems like bazel also do not generate the necessary data. Let's keep it +// simple instead. + +// The module containing the code in this repository. +const CurrentModule = "github.com/cilium/ebpf" diff --git a/cmd/bpf2go/main.go b/cmd/bpf2go/main.go index 6064d44f3..374e48eec 100644 --- a/cmd/bpf2go/main.go +++ b/cmd/bpf2go/main.go @@ -6,7 +6,6 @@ import ( "flag" "fmt" "go/build/constraint" - "go/token" "io" "os" "os/exec" @@ -18,6 +17,8 @@ import ( "strings" "github.com/cilium/ebpf" + "github.com/cilium/ebpf/btf" + "github.com/cilium/ebpf/cmd/bpf2go/gen" ) const helpText = `Usage: %[1]s [options] [-- ] @@ -190,9 +191,6 @@ func newB2G(stdout io.Writer, args []string) (*bpf2go, error) { } b2g.identStem = args[0] - if !token.IsIdentifier(b2g.identStem) { - return nil, fmt.Errorf("%q is not a valid identifier", b2g.identStem) - } sourceFile, err := filepath.Abs(args[1]) if err != nil { @@ -389,9 +387,26 @@ func (b2g *bpf2go) convert(tgt target, goarches []goarch) (err error) { return fmt.Errorf("can't load BPF from ELF: %s", err) } - maps, programs, types, err := collectFromSpec(spec, b2g.cTypes, b2g.skipGlobalTypes) + var maps []string + for name := range spec.Maps { + // Skip .rodata, .data, .bss, etc. sections + if !strings.HasPrefix(name, ".") { + maps = append(maps, name) + } + } + + var programs []string + for name := range spec.Programs { + programs = append(programs, name) + } + + types, err := collectCTypes(spec.Types, b2g.cTypes) if err != nil { - return err + return fmt.Errorf("collect C types: %w", err) + } + + if !b2g.skipGlobalTypes { + types = append(types, gen.CollectGlobalTypes(spec)...) } // Write out generated go @@ -402,15 +417,15 @@ func (b2g *bpf2go) convert(tgt target, goarches []goarch) (err error) { } defer removeOnError(goFile) - err = output(outputArgs{ - pkg: b2g.pkg, - stem: b2g.identStem, - constraints: constraints, - maps: maps, - programs: programs, - types: types, - obj: filepath.Base(objFileName), - out: goFile, + err = gen.Generate(gen.GenerateArgs{ + Package: b2g.pkg, + Stem: b2g.identStem, + Constraints: constraints, + Maps: maps, + Programs: programs, + Types: types, + ObjectFile: filepath.Base(objFileName), + Output: goFile, }) if err != nil { return fmt.Errorf("can't write %s: %s", goFileName, err) @@ -540,6 +555,18 @@ func collectTargets(targets []string) (map[target][]goarch, error) { return result, nil } +func collectCTypes(types *btf.Spec, names []string) ([]btf.Type, error) { + var result []btf.Type + for _, cType := range names { + typ, err := types.AnyTypeByName(cType) + if err != nil { + return nil, err + } + result = append(result, typ) + } + return result, nil +} + const gopackageEnv = "GOPACKAGE" func main() { diff --git a/cmd/bpf2go/main_test.go b/cmd/bpf2go/main_test.go index da0eca972..c949cf4a6 100644 --- a/cmd/bpf2go/main_test.go +++ b/cmd/bpf2go/main_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + "github.com/cilium/ebpf/cmd/bpf2go/internal" "github.com/go-quicktest/qt" "github.com/google/go-cmp/cmp" ) @@ -43,7 +44,7 @@ func TestRun(t *testing.T) { } } - module := currentModule() + module := internal.CurrentModule execInModule("go", "mod", "init", "bpf2go-test") diff --git a/cmd/bpf2go/tools.go b/cmd/bpf2go/tools.go index d2e020b48..e754d7aca 100644 --- a/cmd/bpf2go/tools.go +++ b/cmd/bpf2go/tools.go @@ -3,10 +3,7 @@ package main import ( "errors" "fmt" - "runtime/debug" "strings" - "unicode" - "unicode/utf8" ) func splitCFlagsFromArgs(in []string) (args, cflags []string) { @@ -77,18 +74,3 @@ func splitArguments(in string) ([]string, error) { return result, nil } - -func toUpperFirst(str string) string { - first, n := utf8.DecodeRuneInString(str) - return string(unicode.ToUpper(first)) + str[n:] -} - -func currentModule() string { - bi, ok := debug.ReadBuildInfo() - if !ok { - // Fall back to constant since bazel doesn't support BuildInfo. - return "github.com/cilium/ebpf" - } - - return bi.Main.Path -} From cded449dcefd0403ee3f070692420358262931dc Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Fri, 21 Jun 2024 16:11:18 +0100 Subject: [PATCH 2/3] bpf2go: export compilation Move the code necessary to compile a C to an ELF. The behaviour of bpf2go is unchanged. The code to fix up make-style depfiles remains in bpf2go since it has little tests, and is probably used only seldomly. Signed-off-by: Lorenz Bauer --- cmd/bpf2go/compile.go | 210 --------------------------------- cmd/bpf2go/compile_test.go | 160 ------------------------- cmd/bpf2go/gen/compile.go | 86 ++++++++++++++ cmd/bpf2go/gen/compile_test.go | 115 ++++++++++++++++++ cmd/bpf2go/gen/doc.go | 2 + cmd/bpf2go/main.go | 61 ++++++---- cmd/bpf2go/main_test.go | 35 +++--- cmd/bpf2go/makedep.go | 106 +++++++++++++++++ cmd/bpf2go/makedep_test.go | 59 +++++++++ internal/testutils/programs.go | 25 ++++ 10 files changed, 446 insertions(+), 413 deletions(-) delete mode 100644 cmd/bpf2go/compile.go delete mode 100644 cmd/bpf2go/compile_test.go create mode 100644 cmd/bpf2go/gen/compile.go create mode 100644 cmd/bpf2go/gen/compile_test.go create mode 100644 cmd/bpf2go/gen/doc.go create mode 100644 cmd/bpf2go/makedep.go create mode 100644 cmd/bpf2go/makedep_test.go create mode 100644 internal/testutils/programs.go diff --git a/cmd/bpf2go/compile.go b/cmd/bpf2go/compile.go deleted file mode 100644 index 2aa08f92a..000000000 --- a/cmd/bpf2go/compile.go +++ /dev/null @@ -1,210 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" -) - -type compileArgs struct { - // Which compiler to use - cc string - cFlags []string - // Absolute working directory - dir string - // Absolute input file name - source string - // Absolute output file name - dest string - // Target to compile for, defaults to "bpf". - target string - // Depfile will be written here if depName is not empty - dep io.Writer -} - -func compile(args compileArgs) error { - // Default cflags that can be overridden by args.cFlags - overrideFlags := []string{ - // Code needs to be optimized, otherwise the verifier will often fail - // to understand it. - "-O2", - // Clang defaults to mcpu=probe which checks the kernel that we are - // compiling on. This isn't appropriate for ahead of time - // compiled code so force the most compatible version. - "-mcpu=v1", - } - - cmd := exec.Command(args.cc, append(overrideFlags, args.cFlags...)...) - cmd.Stderr = os.Stderr - - inputDir := filepath.Dir(args.source) - relInputDir, err := filepath.Rel(args.dir, inputDir) - if err != nil { - return err - } - - target := args.target - if target == "" { - target = "bpf" - } - - // C flags that can't be overridden. - cmd.Args = append(cmd.Args, - "-target", target, - "-c", args.source, - "-o", args.dest, - // Don't include clang version - "-fno-ident", - // Don't output inputDir into debug info - "-fdebug-prefix-map="+inputDir+"="+relInputDir, - "-fdebug-compilation-dir", ".", - // We always want BTF to be generated, so enforce debug symbols - "-g", - fmt.Sprintf("-D__BPF_TARGET_MISSING=%q", "GCC error \"The eBPF is using target specific macros, please provide -target that is not bpf, bpfel or bpfeb\""), - ) - cmd.Dir = args.dir - - var depFile *os.File - if args.dep != nil { - depFile, err = os.CreateTemp("", "bpf2go") - if err != nil { - return err - } - defer depFile.Close() - defer os.Remove(depFile.Name()) - - cmd.Args = append(cmd.Args, - // Output dependency information. - "-MD", - // Create phony targets so that deleting a dependency doesn't - // break the build. - "-MP", - // Write it to temporary file - "-MF"+depFile.Name(), - ) - } - - if err := cmd.Run(); err != nil { - return fmt.Errorf("can't execute %s: %s", args.cc, err) - } - - if depFile != nil { - if _, err := io.Copy(args.dep, depFile); err != nil { - return fmt.Errorf("error writing depfile: %w", err) - } - } - - return nil -} - -func adjustDependencies(baseDir string, deps []dependency) ([]byte, error) { - var buf bytes.Buffer - for _, dep := range deps { - relativeFile, err := filepath.Rel(baseDir, dep.file) - if err != nil { - return nil, err - } - - if len(dep.prerequisites) == 0 { - _, err := fmt.Fprintf(&buf, "%s:\n\n", relativeFile) - if err != nil { - return nil, err - } - continue - } - - var prereqs []string - for _, prereq := range dep.prerequisites { - relativePrereq, err := filepath.Rel(baseDir, prereq) - if err != nil { - return nil, err - } - - prereqs = append(prereqs, relativePrereq) - } - - _, err = fmt.Fprintf(&buf, "%s: \\\n %s\n\n", relativeFile, strings.Join(prereqs, " \\\n ")) - if err != nil { - return nil, err - } - } - return buf.Bytes(), nil -} - -type dependency struct { - file string - prerequisites []string -} - -func parseDependencies(baseDir string, in io.Reader) ([]dependency, error) { - abs := func(path string) string { - if filepath.IsAbs(path) { - return path - } - return filepath.Join(baseDir, path) - } - - scanner := bufio.NewScanner(in) - var line strings.Builder - var deps []dependency - for scanner.Scan() { - buf := scanner.Bytes() - if line.Len()+len(buf) > 1024*1024 { - return nil, errors.New("line too long") - } - - if bytes.HasSuffix(buf, []byte{'\\'}) { - line.Write(buf[:len(buf)-1]) - continue - } - - line.Write(buf) - if line.Len() == 0 { - // Skip empty lines - continue - } - - parts := strings.SplitN(line.String(), ":", 2) - if len(parts) < 2 { - return nil, fmt.Errorf("invalid line without ':'") - } - - // NB: This doesn't handle filenames with spaces in them. - // It seems like make doesn't do that either, so oh well. - var prereqs []string - for _, prereq := range strings.Fields(parts[1]) { - prereqs = append(prereqs, abs(prereq)) - } - - deps = append(deps, dependency{ - abs(string(parts[0])), - prereqs, - }) - line.Reset() - } - if err := scanner.Err(); err != nil { - return nil, err - } - - // There is always at least a dependency for the main file. - if len(deps) == 0 { - return nil, fmt.Errorf("empty dependency file") - } - return deps, nil -} - -// strip DWARF debug info from file by executing exe. -func strip(exe, file string) error { - cmd := exec.Command(exe, "-g", file) - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("%s: %s", exe, err) - } - return nil -} diff --git a/cmd/bpf2go/compile_test.go b/cmd/bpf2go/compile_test.go deleted file mode 100644 index ff87d80a9..000000000 --- a/cmd/bpf2go/compile_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package main - -import ( - "bytes" - "os" - "path/filepath" - "reflect" - "strings" - "testing" -) - -const minimalSocketFilter = `__attribute__((section("socket"), used)) int main() { return 0; }` - -func TestCompile(t *testing.T) { - dir := t.TempDir() - mustWriteFile(t, dir, "test.c", minimalSocketFilter) - - var dep bytes.Buffer - err := compile(compileArgs{ - cc: clangBin(t), - dir: dir, - source: filepath.Join(dir, "test.c"), - dest: filepath.Join(dir, "test.o"), - dep: &dep, - }) - if err != nil { - t.Fatal("Can't compile:", err) - } - - stat, err := os.Stat(filepath.Join(dir, "test.o")) - if err != nil { - t.Fatal("Can't stat output:", err) - } - - if stat.Size() == 0 { - t.Error("Compilation creates an empty file") - } - - if dep.Len() == 0 { - t.Error("Compilation doesn't generate depinfo") - } - - if _, err := parseDependencies(dir, &dep); err != nil { - t.Error("Can't parse dependencies:", err) - } -} - -func TestReproducibleCompile(t *testing.T) { - clangBin := clangBin(t) - dir := t.TempDir() - mustWriteFile(t, dir, "test.c", minimalSocketFilter) - - err := compile(compileArgs{ - cc: clangBin, - dir: dir, - source: filepath.Join(dir, "test.c"), - dest: filepath.Join(dir, "a.o"), - }) - if err != nil { - t.Fatal("Can't compile:", err) - } - - err = compile(compileArgs{ - cc: clangBin, - dir: dir, - source: filepath.Join(dir, "test.c"), - dest: filepath.Join(dir, "b.o"), - }) - if err != nil { - t.Fatal("Can't compile:", err) - } - - aBytes, err := os.ReadFile(filepath.Join(dir, "a.o")) - if err != nil { - t.Fatal(err) - } - - bBytes, err := os.ReadFile(filepath.Join(dir, "b.o")) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(aBytes, bBytes) { - t.Error("Compiling the same file twice doesn't give the same result") - } -} - -func TestTriggerMissingTarget(t *testing.T) { - dir := t.TempDir() - mustWriteFile(t, dir, "test.c", `_Pragma(__BPF_TARGET_MISSING);`) - - err := compile(compileArgs{ - cc: clangBin(t), - dir: dir, - source: filepath.Join(dir, "test.c"), - dest: filepath.Join(dir, "a.o"), - }) - - if err == nil { - t.Fatal("No error when compiling __BPF_TARGET_MISSING") - } -} - -func TestParseDependencies(t *testing.T) { - const input = `main.go: /foo/bar baz - -frob: /gobble \ - gubble - -nothing: -` - - have, err := parseDependencies("/foo", strings.NewReader(input)) - if err != nil { - t.Fatal("Can't parse dependencies:", err) - } - - want := []dependency{ - {"/foo/main.go", []string{"/foo/bar", "/foo/baz"}}, - {"/foo/frob", []string{"/gobble", "/foo/gubble"}}, - {"/foo/nothing", nil}, - } - - if !reflect.DeepEqual(have, want) { - t.Logf("Have: %#v", have) - t.Logf("Want: %#v", want) - t.Error("Result doesn't match") - } - - output, err := adjustDependencies("/foo", want) - if err != nil { - t.Error("Can't adjust dependencies") - } - - const wantOutput = `main.go: \ - bar \ - baz - -frob: \ - ../gobble \ - gubble - -nothing: - -` - - if have := string(output); have != wantOutput { - t.Logf("Have:\n%s", have) - t.Logf("Want:\n%s", wantOutput) - t.Error("Output doesn't match") - } -} - -func mustWriteFile(tb testing.TB, dir, name, contents string) { - tb.Helper() - tmpFile := filepath.Join(dir, name) - if err := os.WriteFile(tmpFile, []byte(contents), 0660); err != nil { - tb.Fatal(err) - } -} diff --git a/cmd/bpf2go/gen/compile.go b/cmd/bpf2go/gen/compile.go new file mode 100644 index 000000000..4c0da5cf8 --- /dev/null +++ b/cmd/bpf2go/gen/compile.go @@ -0,0 +1,86 @@ +package gen + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +type CompileArgs struct { + // Which compiler to use. + CC string + // Command used to strip DWARF from the ELF. + Strip string + // Flags to pass to the compiler. + Flags []string + // Absolute working directory + Workdir string + // Absolute input file name + Source string + // Absolute output file name + Dest string + // Target to compile for, defaults to "bpf". + Target string + DisableStripping bool +} + +// Compile C to a BPF ELF file. +func Compile(args CompileArgs) error { + // Default cflags that can be overridden by args.cFlags + overrideFlags := []string{ + // Code needs to be optimized, otherwise the verifier will often fail + // to understand it. + "-O2", + // Clang defaults to mcpu=probe which checks the kernel that we are + // compiling on. This isn't appropriate for ahead of time + // compiled code so force the most compatible version. + "-mcpu=v1", + } + + cmd := exec.Command(args.CC, append(overrideFlags, args.Flags...)...) + cmd.Stderr = os.Stderr + + inputDir := filepath.Dir(args.Source) + relInputDir, err := filepath.Rel(args.Workdir, inputDir) + if err != nil { + return err + } + + target := args.Target + if target == "" { + target = "bpf" + } + + // C flags that can't be overridden. + cmd.Args = append(cmd.Args, + "-target", target, + "-c", args.Source, + "-o", args.Dest, + // Don't include clang version + "-fno-ident", + // Don't output inputDir into debug info + "-fdebug-prefix-map="+inputDir+"="+relInputDir, + "-fdebug-compilation-dir", ".", + // We always want BTF to be generated, so enforce debug symbols + "-g", + fmt.Sprintf("-D__BPF_TARGET_MISSING=%q", "GCC error \"The eBPF is using target specific macros, please provide -target that is not bpf, bpfel or bpfeb\""), + ) + cmd.Dir = args.Workdir + + if err := cmd.Run(); err != nil { + return err + } + + if args.DisableStripping { + return nil + } + + cmd = exec.Command(args.Strip, "-g", args.Dest) + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("strip %s: %w", args.Dest, err) + } + + return nil +} diff --git a/cmd/bpf2go/gen/compile_test.go b/cmd/bpf2go/gen/compile_test.go new file mode 100644 index 000000000..f5aeb7d76 --- /dev/null +++ b/cmd/bpf2go/gen/compile_test.go @@ -0,0 +1,115 @@ +package gen + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/cilium/ebpf/internal/testutils" +) + +const minimalSocketFilter = `__attribute__((section("socket"), used)) int main() { return 0; }` + +func TestCompile(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + dir := t.TempDir() + mustWriteFile(t, dir, "test.c", minimalSocketFilter) + + err := Compile(CompileArgs{ + CC: testutils.ClangBin(t), + DisableStripping: true, + Workdir: dir, + Source: filepath.Join(dir, "test.c"), + Dest: filepath.Join(dir, "test.o"), + }) + if err != nil { + t.Fatal("Can't compile:", err) + } + + stat, err := os.Stat(filepath.Join(dir, "test.o")) + if err != nil { + t.Fatal("Can't stat output:", err) + } + + if stat.Size() == 0 { + t.Error("Compilation creates an empty file") + } +} + +func TestReproducibleCompile(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + clangBin := testutils.ClangBin(t) + dir := t.TempDir() + mustWriteFile(t, dir, "test.c", minimalSocketFilter) + + err := Compile(CompileArgs{ + CC: clangBin, + DisableStripping: true, + Workdir: dir, + Source: filepath.Join(dir, "test.c"), + Dest: filepath.Join(dir, "a.o"), + }) + if err != nil { + t.Fatal("Can't compile:", err) + } + + err = Compile(CompileArgs{ + CC: clangBin, + DisableStripping: true, + Workdir: dir, + Source: filepath.Join(dir, "test.c"), + Dest: filepath.Join(dir, "b.o"), + }) + if err != nil { + t.Fatal("Can't compile:", err) + } + + aBytes, err := os.ReadFile(filepath.Join(dir, "a.o")) + if err != nil { + t.Fatal(err) + } + + bBytes, err := os.ReadFile(filepath.Join(dir, "b.o")) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(aBytes, bBytes) { + t.Error("Compiling the same file twice doesn't give the same result") + } +} + +func TestTriggerMissingTarget(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + dir := t.TempDir() + mustWriteFile(t, dir, "test.c", `_Pragma(__BPF_TARGET_MISSING);`) + + err := Compile(CompileArgs{ + CC: testutils.ClangBin(t), + Workdir: dir, + Source: filepath.Join(dir, "test.c"), + Dest: filepath.Join(dir, "a.o"), + }) + + if err == nil { + t.Fatal("No error when compiling __BPF_TARGET_MISSING") + } +} + +func mustWriteFile(tb testing.TB, dir, name, contents string) { + tb.Helper() + tmpFile := filepath.Join(dir, name) + if err := os.WriteFile(tmpFile, []byte(contents), 0660); err != nil { + tb.Fatal(err) + } +} diff --git a/cmd/bpf2go/gen/doc.go b/cmd/bpf2go/gen/doc.go new file mode 100644 index 000000000..1f3080f6f --- /dev/null +++ b/cmd/bpf2go/gen/doc.go @@ -0,0 +1,2 @@ +// Package gen contains utilities to generate Go bindings for eBPF ELF files. +package gen diff --git a/cmd/bpf2go/main.go b/cmd/bpf2go/main.go index 374e48eec..eb09e8803 100644 --- a/cmd/bpf2go/main.go +++ b/cmd/bpf2go/main.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "errors" "flag" "fmt" @@ -359,26 +358,43 @@ func (b2g *bpf2go) convert(tgt target, goarches []goarch) (err error) { return fmt.Errorf("remove obsolete output: %w", err) } - var dep bytes.Buffer - err = compile(compileArgs{ - cc: b2g.cc, - cFlags: cFlags, - target: tgt.clang, - dir: cwd, - source: b2g.sourceFile, - dest: objFileName, - dep: &dep, + var depInput *os.File + if b2g.makeBase != "" { + depInput, err = os.CreateTemp("", "bpf2go") + if err != nil { + return err + } + defer depInput.Close() + defer os.Remove(depInput.Name()) + + cFlags = append(cFlags, + // Output dependency information. + "-MD", + // Create phony targets so that deleting a dependency doesn't + // break the build. + "-MP", + // Write it to temporary file + "-MF"+depInput.Name(), + ) + } + + err = gen.Compile(gen.CompileArgs{ + CC: b2g.cc, + Strip: b2g.strip, + DisableStripping: b2g.disableStripping, + Flags: cFlags, + Target: tgt.clang, + Workdir: cwd, + Source: b2g.sourceFile, + Dest: objFileName, }) if err != nil { - return err + return fmt.Errorf("compile: %w", err) } fmt.Fprintln(b2g.stdout, "Compiled", objFileName) if !b2g.disableStripping { - if err := strip(b2g.strip, objFileName); err != nil { - return err - } fmt.Fprintln(b2g.stdout, "Stripped", objFileName) } @@ -437,21 +453,22 @@ func (b2g *bpf2go) convert(tgt target, goarches []goarch) (err error) { return } - deps, err := parseDependencies(cwd, &dep) + deps, err := parseDependencies(cwd, depInput) if err != nil { return fmt.Errorf("can't read dependency information: %s", err) } - // There is always at least a dependency for the main file. - deps[0].file = goFileName - depFile, err := adjustDependencies(b2g.makeBase, deps) + depFileName := goFileName + ".d" + depOutput, err := os.Create(depFileName) if err != nil { - return fmt.Errorf("can't adjust dependency information: %s", err) + return fmt.Errorf("write make dependencies: %w", err) } + defer depOutput.Close() - depFileName := goFileName + ".d" - if err := os.WriteFile(depFileName, depFile, 0666); err != nil { - return fmt.Errorf("can't write dependency file: %s", err) + // There is always at least a dependency for the main file. + deps[0].file = goFileName + if err := adjustDependencies(depOutput, b2g.makeBase, deps); err != nil { + return fmt.Errorf("can't adjust dependency information: %s", err) } fmt.Fprintln(b2g.stdout, "Wrote", depFileName) diff --git a/cmd/bpf2go/main_test.go b/cmd/bpf2go/main_test.go index c949cf4a6..f9605d3d2 100644 --- a/cmd/bpf2go/main_test.go +++ b/cmd/bpf2go/main_test.go @@ -14,12 +14,15 @@ import ( "testing" "github.com/cilium/ebpf/cmd/bpf2go/internal" + "github.com/cilium/ebpf/internal/testutils" "github.com/go-quicktest/qt" "github.com/google/go-cmp/cmp" ) +const minimalSocketFilter = `__attribute__((section("socket"), used)) int main() { return 0; }` + func TestRun(t *testing.T) { - clangBin := clangBin(t) + clangBin := testutils.ClangBin(t) dir := t.TempDir() mustWriteFile(t, dir, "test.c", minimalSocketFilter) @@ -129,7 +132,7 @@ func TestDisableStripping(t *testing.T) { err := run(io.Discard, []string{ "-go-package", "foo", "-output-dir", dir, - "-cc", clangBin(t), + "-cc", testutils.ClangBin(t), "-strip", "binary-that-certainly-doesnt-exist", "-no-strip", "bar", @@ -254,7 +257,7 @@ func TestConvertGOARCH(t *testing.T) { pkg: "test", stdout: io.Discard, identStem: "test", - cc: clangBin(t), + cc: testutils.ClangBin(t), disableStripping: true, sourceFile: tmp + "/test.c", outputDir: tmp, @@ -515,24 +518,6 @@ func TestClangTargets(t *testing.T) { } } -func clangBin(t *testing.T) string { - t.Helper() - - if testing.Short() { - t.Skip("Not compiling with -short") - } - - // Use a floating clang version for local development, but allow CI to run - // against oldest supported clang. - clang := "clang" - if minVersion := os.Getenv("CI_MIN_CLANG_VERSION"); minVersion != "" { - clang = fmt.Sprintf("clang-%s", minVersion) - } - - t.Log("Testing against", clang) - return clang -} - func goBin(t *testing.T) string { t.Helper() @@ -544,3 +529,11 @@ func goBin(t *testing.T) string { return exe } + +func mustWriteFile(tb testing.TB, dir, name, contents string) { + tb.Helper() + tmpFile := filepath.Join(dir, name) + if err := os.WriteFile(tmpFile, []byte(contents), 0660); err != nil { + tb.Fatal(err) + } +} diff --git a/cmd/bpf2go/makedep.go b/cmd/bpf2go/makedep.go new file mode 100644 index 000000000..9f4973c3a --- /dev/null +++ b/cmd/bpf2go/makedep.go @@ -0,0 +1,106 @@ +package main + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "path/filepath" + "strings" +) + +func adjustDependencies(w io.Writer, baseDir string, deps []dependency) error { + for _, dep := range deps { + relativeFile, err := filepath.Rel(baseDir, dep.file) + if err != nil { + return err + } + + if len(dep.prerequisites) == 0 { + _, err := fmt.Fprintf(w, "%s:\n\n", relativeFile) + if err != nil { + return err + } + continue + } + + var prereqs []string + for _, prereq := range dep.prerequisites { + relativePrereq, err := filepath.Rel(baseDir, prereq) + if err != nil { + return err + } + + prereqs = append(prereqs, relativePrereq) + } + + _, err = fmt.Fprintf(w, "%s: \\\n %s\n\n", relativeFile, strings.Join(prereqs, " \\\n ")) + if err != nil { + return err + } + } + return nil +} + +type dependency struct { + file string + prerequisites []string +} + +func parseDependencies(baseDir string, in io.Reader) ([]dependency, error) { + abs := func(path string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(baseDir, path) + } + + scanner := bufio.NewScanner(in) + var line strings.Builder + var deps []dependency + for scanner.Scan() { + buf := scanner.Bytes() + if line.Len()+len(buf) > 1024*1024 { + return nil, errors.New("line too long") + } + + if bytes.HasSuffix(buf, []byte{'\\'}) { + line.Write(buf[:len(buf)-1]) + continue + } + + line.Write(buf) + if line.Len() == 0 { + // Skip empty lines + continue + } + + parts := strings.SplitN(line.String(), ":", 2) + if len(parts) < 2 { + return nil, fmt.Errorf("invalid line without ':'") + } + + // NB: This doesn't handle filenames with spaces in them. + // It seems like make doesn't do that either, so oh well. + var prereqs []string + for _, prereq := range strings.Fields(parts[1]) { + prereqs = append(prereqs, abs(prereq)) + } + + deps = append(deps, dependency{ + abs(string(parts[0])), + prereqs, + }) + line.Reset() + } + if err := scanner.Err(); err != nil { + return nil, err + } + + // There is always at least a dependency for the main file. + if len(deps) == 0 { + return nil, fmt.Errorf("empty dependency file") + } + return deps, nil +} diff --git a/cmd/bpf2go/makedep_test.go b/cmd/bpf2go/makedep_test.go new file mode 100644 index 000000000..6dff2675e --- /dev/null +++ b/cmd/bpf2go/makedep_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "bytes" + "reflect" + "strings" + "testing" +) + +func TestParseDependencies(t *testing.T) { + const input = `main.go: /foo/bar baz + +frob: /gobble \ + gubble + +nothing: +` + + have, err := parseDependencies("/foo", strings.NewReader(input)) + if err != nil { + t.Fatal("Can't parse dependencies:", err) + } + + want := []dependency{ + {"/foo/main.go", []string{"/foo/bar", "/foo/baz"}}, + {"/foo/frob", []string{"/gobble", "/foo/gubble"}}, + {"/foo/nothing", nil}, + } + + if !reflect.DeepEqual(have, want) { + t.Logf("Have: %#v", have) + t.Logf("Want: %#v", want) + t.Error("Result doesn't match") + } + + var output bytes.Buffer + err = adjustDependencies(&output, "/foo", want) + if err != nil { + t.Error("Can't adjust dependencies") + } + + const wantOutput = `main.go: \ + bar \ + baz + +frob: \ + ../gobble \ + gubble + +nothing: + +` + + if have := output.String(); have != wantOutput { + t.Logf("Have:\n%s", have) + t.Logf("Want:\n%s", wantOutput) + t.Error("Output doesn't match") + } +} diff --git a/internal/testutils/programs.go b/internal/testutils/programs.go new file mode 100644 index 000000000..dd99a245b --- /dev/null +++ b/internal/testutils/programs.go @@ -0,0 +1,25 @@ +package testutils + +import ( + "fmt" + "os" + "testing" +) + +func ClangBin(tb testing.TB) string { + tb.Helper() + + if testing.Short() { + tb.Skip("Not compiling with -short") + } + + // Use a floating clang version for local development, but allow CI to run + // against oldest supported clang. + clang := "clang" + if minVersion := os.Getenv("CI_MIN_CLANG_VERSION"); minVersion != "" { + clang = fmt.Sprintf("clang-%s", minVersion) + } + + tb.Log("Testing against", clang) + return clang +} From 69dbf3204861132516735fbdb014afaad055b05a Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Fri, 21 Jun 2024 18:11:40 +0100 Subject: [PATCH 3/3] bpf2go: export targets Move target and force users to use one of the predefined targets. Also export logic to generate build contraints from goarches. Signed-off-by: Lorenz Bauer --- cmd/bpf2go/flags.go | 12 --- cmd/bpf2go/gen/compile.go | 15 ++-- cmd/bpf2go/gen/target.go | 155 ++++++++++++++++++++++++++++++++++ cmd/bpf2go/gen/target_test.go | 155 ++++++++++++++++++++++++++++++++++ cmd/bpf2go/main.go | 143 ++++++------------------------- cmd/bpf2go/main_test.go | 155 +--------------------------------- 6 files changed, 346 insertions(+), 289 deletions(-) create mode 100644 cmd/bpf2go/gen/target.go create mode 100644 cmd/bpf2go/gen/target_test.go diff --git a/cmd/bpf2go/flags.go b/cmd/bpf2go/flags.go index ca8852a29..806d2cd2c 100644 --- a/cmd/bpf2go/flags.go +++ b/cmd/bpf2go/flags.go @@ -43,15 +43,3 @@ func andConstraints(x, y constraint.Expr) constraint.Expr { return &constraint.AndExpr{X: x, Y: y} } - -func orConstraints(x, y constraint.Expr) constraint.Expr { - if x == nil { - return y - } - - if y == nil { - return x - } - - return &constraint.OrExpr{X: x, Y: y} -} diff --git a/cmd/bpf2go/gen/compile.go b/cmd/bpf2go/gen/compile.go index 4c0da5cf8..09d57da8d 100644 --- a/cmd/bpf2go/gen/compile.go +++ b/cmd/bpf2go/gen/compile.go @@ -20,8 +20,8 @@ type CompileArgs struct { Source string // Absolute output file name Dest string - // Target to compile for, defaults to "bpf". - Target string + // Target to compile for, defaults to compiling generic BPF in host endianness. + Target Target DisableStripping bool } @@ -48,13 +48,18 @@ func Compile(args CompileArgs) error { } target := args.Target - if target == "" { - target = "bpf" + if target == (Target{}) { + target.clang = "bpf" } // C flags that can't be overridden. + if linux := target.linux; linux != "" { + cmd.Args = append(cmd.Args, "-D__TARGET_ARCH_"+linux) + } + cmd.Args = append(cmd.Args, - "-target", target, + "-Wunused-command-line-argument", + "-target", target.clang, "-c", args.Source, "-o", args.Dest, // Don't include clang version diff --git a/cmd/bpf2go/gen/target.go b/cmd/bpf2go/gen/target.go new file mode 100644 index 000000000..f5484cccf --- /dev/null +++ b/cmd/bpf2go/gen/target.go @@ -0,0 +1,155 @@ +package gen + +import ( + "errors" + "fmt" + "go/build/constraint" + "maps" + "runtime" + "slices" +) + +var ErrInvalidTarget = errors.New("unsupported target") + +var targetsByGoArch = map[GoArch]Target{ + "386": {"bpfel", "x86"}, + "amd64": {"bpfel", "x86"}, + "arm": {"bpfel", "arm"}, + "arm64": {"bpfel", "arm64"}, + "loong64": {"bpfel", "loongarch"}, + "mips": {"bpfeb", "mips"}, + "mipsle": {"bpfel", ""}, + "mips64": {"bpfeb", ""}, + "mips64le": {"bpfel", ""}, + "ppc64": {"bpfeb", "powerpc"}, + "ppc64le": {"bpfel", "powerpc"}, + "riscv64": {"bpfel", "riscv"}, + "s390x": {"bpfeb", "s390"}, +} + +type Target struct { + // Clang arch string, used to define the clang -target flag, as per + // "clang -print-targets". + clang string + // Linux arch string, used to define __TARGET_ARCH_xzy macros used by + // https://github.com/libbpf/libbpf/blob/master/src/bpf_tracing.h + linux string +} + +// TargetsByGoArch returns all supported targets. +func TargetsByGoArch() map[GoArch]Target { + return maps.Clone(targetsByGoArch) +} + +// IsGeneric returns true if the target will compile to generic BPF. +func (tgt *Target) IsGeneric() bool { + return tgt.linux == "" +} + +// Suffix returns a a string suitable for appending to a file name to +// identify the target. +func (tgt *Target) Suffix() string { + // The output filename must not match any of the following patterns: + // + // *_GOOS + // *_GOARCH + // *_GOOS_GOARCH + // + // Otherwise it is interpreted as a build constraint by the Go toolchain. + stem := tgt.clang + if tgt.linux != "" { + stem = fmt.Sprintf("%s_%s", tgt.linux, tgt.clang) + } + return stem +} + +// ObsoleteSuffix returns an obsolete suffix for a subset of targets. +// +// It's used to work around an old bug and should not be used in new code. +func (tgt *Target) ObsoleteSuffix() string { + if tgt.linux == "" { + return "" + } + + return fmt.Sprintf("%s_%s", tgt.clang, tgt.linux) +} + +// GoArch is a Go arch string. +// +// See https://go.dev/doc/install/source#environment for valid GOARCHes when +// GOOS=linux. +type GoArch string + +type GoArches []GoArch + +// Constraints is satisfied when GOARCH is any of the arches. +func (arches GoArches) Constraint() constraint.Expr { + var archConstraint constraint.Expr + for _, goarch := range arches { + tag := &constraint.TagExpr{Tag: string(goarch)} + archConstraint = orConstraints(archConstraint, tag) + } + return archConstraint +} + +// FindTarget turns a list of identifiers into targets and their respective +// GoArches. +// +// The following are valid identifiers: +// +// - bpf: compile generic BPF for host endianness +// - bpfel: compile generic BPF for little endian +// - bpfeb: compile generic BPF for big endian +// - native: compile BPF for host target +// - $GOARCH: compile BPF for $GOARCH target +// +// Generic BPF can run on any target goarch with the correct endianness, +// but doesn't have access to some arch specific tracing functionality. +func FindTarget(id string) (Target, GoArches, error) { + switch id { + case "bpf", "bpfel", "bpfeb": + var goarches []GoArch + for arch, archTarget := range targetsByGoArch { + if archTarget.clang == id { + // Include tags for all goarches that have the same endianness. + goarches = append(goarches, arch) + } + } + slices.Sort(goarches) + return Target{id, ""}, goarches, nil + + case "native": + id = runtime.GOARCH + fallthrough + + default: + archTarget, ok := targetsByGoArch[GoArch(id)] + if !ok || archTarget.linux == "" { + return Target{}, nil, fmt.Errorf("%q: %w", id, ErrInvalidTarget) + } + + var goarches []GoArch + for goarch, lt := range targetsByGoArch { + if lt == archTarget { + // Include tags for all goarches that have the same + // target. + goarches = append(goarches, goarch) + } + } + + slices.Sort(goarches) + return archTarget, goarches, nil + } +} + +func orConstraints(x, y constraint.Expr) constraint.Expr { + if x == nil { + return y + } + + if y == nil { + return x + } + + return &constraint.OrExpr{X: x, Y: y} +} diff --git a/cmd/bpf2go/gen/target_test.go b/cmd/bpf2go/gen/target_test.go new file mode 100644 index 000000000..021b9dda7 --- /dev/null +++ b/cmd/bpf2go/gen/target_test.go @@ -0,0 +1,155 @@ +package gen + +import ( + "errors" + "os/exec" + "slices" + "testing" + + "github.com/go-quicktest/qt" +) + +func TestCollectTargets(t *testing.T) { + clangArches := make(map[string][]GoArch) + linuxArchesLE := make(map[string][]GoArch) + linuxArchesBE := make(map[string][]GoArch) + for arch, archTarget := range targetsByGoArch { + clangArches[archTarget.clang] = append(clangArches[archTarget.clang], arch) + if archTarget.clang == "bpfel" { + linuxArchesLE[archTarget.linux] = append(linuxArchesLE[archTarget.linux], arch) + continue + } + linuxArchesBE[archTarget.linux] = append(linuxArchesBE[archTarget.linux], arch) + } + for i := range clangArches { + slices.Sort(clangArches[i]) + } + for i := range linuxArchesLE { + slices.Sort(linuxArchesLE[i]) + } + for i := range linuxArchesBE { + slices.Sort(linuxArchesBE[i]) + } + + nativeTarget, nativeArches, err := FindTarget("native") + qt.Assert(t, qt.IsNil(err)) + + tests := []struct { + short string + target Target + arches GoArches + }{ + { + "bpf", + Target{"bpf", ""}, + nil, + }, + { + "bpfel", + Target{"bpfel", ""}, + clangArches["bpfel"], + }, + { + "bpfeb", + Target{"bpfeb", ""}, + clangArches["bpfeb"], + }, + { + "amd64", + Target{"bpfel", "x86"}, + linuxArchesLE["x86"], + }, + { + "386", + Target{"bpfel", "x86"}, + linuxArchesLE["x86"], + }, + { + "ppc64", + Target{"bpfeb", "powerpc"}, + linuxArchesBE["powerpc"], + }, + { + "native", + nativeTarget, + nativeArches, + }, + } + + for _, test := range tests { + t.Run(test.short, func(t *testing.T) { + target, arches, err := FindTarget(test.short) + qt.Assert(t, qt.IsNil(err)) + qt.Assert(t, qt.Equals(target, test.target)) + qt.Assert(t, qt.DeepEquals(arches, test.arches)) + }) + } +} + +func TestCollectTargetsErrors(t *testing.T) { + tests := []struct { + name string + target string + }{ + {"unknown", "frood"}, + {"no linux target", "mipsle"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, _, err := FindTarget(test.target) + if err == nil { + t.Fatal("Function did not return an error") + } + t.Log("Error message:", err) + }) + } +} + +func TestGoarches(t *testing.T) { + exe := goBin(t) + + for GoArch := range targetsByGoArch { + t.Run(string(GoArch), func(t *testing.T) { + goEnv := exec.Command(exe, "env") + goEnv.Env = []string{"GOROOT=/", "GOOS=linux", "GOARCH=" + string(GoArch)} + output, err := goEnv.CombinedOutput() + qt.Assert(t, qt.IsNil(err), qt.Commentf("go output is:\n%s", string(output))) + }) + } +} + +func TestClangTargets(t *testing.T) { + exe := goBin(t) + + clangTargets := map[string]struct{}{} + for _, tgt := range targetsByGoArch { + clangTargets[tgt.clang] = struct{}{} + } + + for target := range clangTargets { + for _, env := range []string{"GOOS", "GOARCH"} { + env += "=" + target + t.Run(env, func(t *testing.T) { + goEnv := exec.Command(exe, "env") + goEnv.Env = []string{"GOROOT=/", env} + output, err := goEnv.CombinedOutput() + t.Log("go output is:", string(output)) + qt.Assert(t, qt.IsNotNil(err), qt.Commentf("No clang target should be a valid build constraint")) + }) + } + + } +} + +func goBin(t *testing.T) string { + t.Helper() + + exe, err := exec.LookPath("go") + if errors.Is(err, exec.ErrNotFound) { + t.Skip("go binary is not in PATH") + } + qt.Assert(t, qt.IsNil(err)) + + return exe +} diff --git a/cmd/bpf2go/main.go b/cmd/bpf2go/main.go index eb09e8803..fb077e139 100644 --- a/cmd/bpf2go/main.go +++ b/cmd/bpf2go/main.go @@ -4,13 +4,11 @@ import ( "errors" "flag" "fmt" - "go/build/constraint" "io" "os" "os/exec" "path/filepath" "regexp" - "runtime" "slices" "sort" "strings" @@ -44,29 +42,6 @@ Options: ` -// Targets understood by bpf2go. -// -// Targets without a Linux string can't be used directly and are only included -// for the generic bpf, bpfel, bpfeb targets. -// -// See https://go.dev/doc/install/source#environment for valid GOARCHes when -// GOOS=linux. -var targetByGoArch = map[goarch]target{ - "386": {"bpfel", "x86"}, - "amd64": {"bpfel", "x86"}, - "arm": {"bpfel", "arm"}, - "arm64": {"bpfel", "arm64"}, - "loong64": {"bpfel", "loongarch"}, - "mips": {"bpfeb", "mips"}, - "mipsle": {"bpfel", ""}, - "mips64": {"bpfeb", ""}, - "mips64le": {"bpfel", ""}, - "ppc64": {"bpfeb", "powerpc"}, - "ppc64le": {"bpfel", "powerpc"}, - "riscv64": {"bpfel", "riscv"}, - "s390x": {"bpfeb", "s390"}, -} - func run(stdout io.Writer, args []string) (err error) { b2g, err := newB2G(stdout, args) switch { @@ -92,7 +67,7 @@ type bpf2go struct { // Valid go identifier. identStem string // Targets to build for. - targetArches map[target][]goarch + targetArches map[gen.Target]gen.GoArches // C compiler. cc string // Command used to strip DWARF. @@ -208,14 +183,18 @@ func newB2G(stdout io.Writer, args []string) (*bpf2go, error) { return nil, fmt.Errorf("-output-stem %q must not contain path separation characters", b2g.outputStem) } - targetArches, err := collectTargets(strings.Split(*flagTarget, ",")) - if errors.Is(err, errInvalidTarget) { - printTargets(b2g.stdout) - fmt.Fprintln(b2g.stdout) - return nil, err - } - if err != nil { - return nil, err + targetArches := make(map[gen.Target]gen.GoArches) + for _, tgt := range strings.Split(*flagTarget, ",") { + target, goarches, err := gen.FindTarget(tgt) + if err != nil { + if errors.Is(err, gen.ErrInvalidTarget) { + printTargets(b2g.stdout) + fmt.Fprintln(b2g.stdout) + } + return nil, err + } + + targetArches[target] = goarches } if len(targetArches) == 0 { @@ -303,7 +282,7 @@ func (b2g *bpf2go) convertAll() (err error) { return nil } -func (b2g *bpf2go) convert(tgt target, goarches []goarch) (err error) { +func (b2g *bpf2go) convert(tgt gen.Target, goarches gen.GoArches) (err error) { removeOnError := func(f *os.File) { if err != nil { os.Remove(f.Name()) @@ -316,17 +295,7 @@ func (b2g *bpf2go) convert(tgt target, goarches []goarch) (err error) { outputStem = strings.ToLower(b2g.identStem) } - // The output filename must not match any of the following patterns: - // - // *_GOOS - // *_GOARCH - // *_GOOS_GOARCH - // - // Otherwise it is interpreted as a build constraint by the Go toolchain. - stem := fmt.Sprintf("%s_%s", outputStem, tgt.clang) - if tgt.linux != "" { - stem = fmt.Sprintf("%s_%s_%s", outputStem, tgt.linux, tgt.clang) - } + stem := fmt.Sprintf("%s_%s", outputStem, tgt.Suffix()) absOutPath, err := filepath.Abs(b2g.outputDir) if err != nil { @@ -340,25 +309,15 @@ func (b2g *bpf2go) convert(tgt target, goarches []goarch) (err error) { return err } - var archConstraint constraint.Expr - for _, goarch := range goarches { - tag := &constraint.TagExpr{Tag: string(goarch)} - archConstraint = orConstraints(archConstraint, tag) - } - + archConstraint := goarches.Constraint() constraints := andConstraints(archConstraint, b2g.tags.Expr) - cFlags := make([]string, len(b2g.cFlags)) - copy(cFlags, b2g.cFlags) - if tgt.linux != "" { - cFlags = append(cFlags, "-D__TARGET_ARCH_"+tgt.linux) - } - if err := b2g.removeOldOutputFiles(outputStem, tgt); err != nil { return fmt.Errorf("remove obsolete output: %w", err) } var depInput *os.File + cFlags := slices.Clone(b2g.cFlags) if b2g.makeBase != "" { depInput, err = os.CreateTemp("", "bpf2go") if err != nil { @@ -383,7 +342,7 @@ func (b2g *bpf2go) convert(tgt target, goarches []goarch) (err error) { Strip: b2g.strip, DisableStripping: b2g.disableStripping, Flags: cFlags, - Target: tgt.clang, + Target: tgt, Workdir: cwd, Source: b2g.sourceFile, Dest: objFileName, @@ -479,12 +438,13 @@ func (b2g *bpf2go) convert(tgt target, goarches []goarch) (err error) { // // In the old scheme some linux targets were interpreted as build constraints // by the go toolchain. -func (b2g *bpf2go) removeOldOutputFiles(outputStem string, tgt target) error { - if tgt.linux == "" { +func (b2g *bpf2go) removeOldOutputFiles(outputStem string, tgt gen.Target) error { + suffix := tgt.ObsoleteSuffix() + if suffix == "" { return nil } - stem := fmt.Sprintf("%s_%s_%s", outputStem, tgt.clang, tgt.linux) + stem := fmt.Sprintf("%s_%s", outputStem, suffix) for _, ext := range []string{".o", ".go"} { filename := filepath.Join(b2g.outputDir, stem+ext) @@ -500,21 +460,10 @@ func (b2g *bpf2go) removeOldOutputFiles(outputStem string, tgt target) error { return nil } -type target struct { - // Clang arch string, used to define the clang -target flag, as per - // "clang -print-targets". - clang string - // Linux arch string, used to define __TARGET_ARCH_xzy macros used by - // https://github.com/libbpf/libbpf/blob/master/src/bpf_tracing.h - linux string -} - -type goarch string - func printTargets(w io.Writer) { var arches []string - for goarch, archTarget := range targetByGoArch { - if archTarget.linux == "" { + for goarch, archTarget := range gen.TargetsByGoArch() { + if archTarget.IsGeneric() { continue } arches = append(arches, string(goarch)) @@ -528,50 +477,6 @@ func printTargets(w io.Writer) { } } -var errInvalidTarget = errors.New("unsupported target") - -func collectTargets(targets []string) (map[target][]goarch, error) { - result := make(map[target][]goarch) - for _, tgt := range targets { - switch tgt { - case "bpf", "bpfel", "bpfeb": - var goarches []goarch - for arch, archTarget := range targetByGoArch { - if archTarget.clang == tgt { - // Include tags for all goarches that have the same endianness. - goarches = append(goarches, arch) - } - } - slices.Sort(goarches) - result[target{tgt, ""}] = goarches - - case "native": - tgt = runtime.GOARCH - fallthrough - - default: - archTarget, ok := targetByGoArch[goarch(tgt)] - if !ok || archTarget.linux == "" { - return nil, fmt.Errorf("%q: %w", tgt, errInvalidTarget) - } - - var goarches []goarch - for goarch, lt := range targetByGoArch { - if lt == archTarget { - // Include tags for all goarches that have the same - // target. - goarches = append(goarches, goarch) - } - } - - slices.Sort(goarches) - result[archTarget] = goarches - } - } - - return result, nil -} - func collectCTypes(types *btf.Spec, names []string) ([]btf.Type, error) { var result []btf.Type for _, cType := range names { diff --git a/cmd/bpf2go/main_test.go b/cmd/bpf2go/main_test.go index f9605d3d2..218bb8742 100644 --- a/cmd/bpf2go/main_test.go +++ b/cmd/bpf2go/main_test.go @@ -2,21 +2,18 @@ package main import ( "bytes" - "errors" "fmt" "io" "os" "os/exec" "path/filepath" - "runtime" - "slices" "strings" "testing" + "github.com/cilium/ebpf/cmd/bpf2go/gen" "github.com/cilium/ebpf/cmd/bpf2go/internal" "github.com/cilium/ebpf/internal/testutils" "github.com/go-quicktest/qt" - "github.com/google/go-cmp/cmp" ) const minimalSocketFilter = `__attribute__((section("socket"), used)) int main() { return 0; }` @@ -144,106 +141,6 @@ func TestDisableStripping(t *testing.T) { } } -func TestCollectTargets(t *testing.T) { - clangArches := make(map[string][]goarch) - linuxArchesLE := make(map[string][]goarch) - linuxArchesBE := make(map[string][]goarch) - for arch, archTarget := range targetByGoArch { - clangArches[archTarget.clang] = append(clangArches[archTarget.clang], arch) - if archTarget.clang == "bpfel" { - linuxArchesLE[archTarget.linux] = append(linuxArchesLE[archTarget.linux], arch) - continue - } - linuxArchesBE[archTarget.linux] = append(linuxArchesBE[archTarget.linux], arch) - } - for i := range clangArches { - slices.Sort(clangArches[i]) - } - for i := range linuxArchesLE { - slices.Sort(linuxArchesLE[i]) - } - for i := range linuxArchesBE { - slices.Sort(linuxArchesBE[i]) - } - - nativeTarget := make(map[target][]goarch) - for arch, archTarget := range targetByGoArch { - if arch == goarch(runtime.GOARCH) { - if archTarget.clang == "bpfel" { - nativeTarget[archTarget] = linuxArchesLE[archTarget.linux] - } else { - nativeTarget[archTarget] = linuxArchesBE[archTarget.linux] - } - break - } - } - - tests := []struct { - targets []string - want map[target][]goarch - }{ - { - []string{"bpf", "bpfel", "bpfeb"}, - map[target][]goarch{ - {"bpf", ""}: nil, - {"bpfel", ""}: clangArches["bpfel"], - {"bpfeb", ""}: clangArches["bpfeb"], - }, - }, - { - []string{"amd64", "386"}, - map[target][]goarch{ - {"bpfel", "x86"}: linuxArchesLE["x86"], - }, - }, - { - []string{"amd64", "ppc64"}, - map[target][]goarch{ - {"bpfeb", "powerpc"}: linuxArchesBE["powerpc"], - {"bpfel", "x86"}: linuxArchesLE["x86"], - }, - }, - { - []string{"native"}, - nativeTarget, - }, - } - - for _, test := range tests { - name := strings.Join(test.targets, ",") - t.Run(name, func(t *testing.T) { - have, err := collectTargets(test.targets) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(test.want, have); diff != "" { - t.Errorf("Result mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestCollectTargetsErrors(t *testing.T) { - tests := []struct { - name string - target string - }{ - {"unknown", "frood"}, - {"no linux target", "mipsle"}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := collectTargets([]string{test.target}) - if err == nil { - t.Fatal("Function did not return an error") - } - t.Log("Error message:", err) - }) - } -} - func TestConvertGOARCH(t *testing.T) { tmp := t.TempDir() mustWriteFile(t, tmp, "test.c", @@ -263,7 +160,7 @@ func TestConvertGOARCH(t *testing.T) { outputDir: tmp, } - if err := b2g.convert(targetByGoArch["amd64"], nil); err != nil { + if err := b2g.convert(gen.TargetsByGoArch()["amd64"], nil); err != nil { t.Fatal("Can't target GOARCH:", err) } } @@ -482,54 +379,6 @@ func TestParseArgs(t *testing.T) { }) } -func TestGoarches(t *testing.T) { - exe := goBin(t) - - for goarch := range targetByGoArch { - t.Run(string(goarch), func(t *testing.T) { - goEnv := exec.Command(exe, "env") - goEnv.Env = []string{"GOROOT=/", "GOOS=linux", "GOARCH=" + string(goarch)} - output, err := goEnv.CombinedOutput() - qt.Assert(t, qt.IsNil(err), qt.Commentf("go output is:\n%s", string(output))) - }) - } -} - -func TestClangTargets(t *testing.T) { - exe := goBin(t) - - clangTargets := map[string]struct{}{} - for _, tgt := range targetByGoArch { - clangTargets[tgt.clang] = struct{}{} - } - - for target := range clangTargets { - for _, env := range []string{"GOOS", "GOARCH"} { - env += "=" + target - t.Run(env, func(t *testing.T) { - goEnv := exec.Command(exe, "env") - goEnv.Env = []string{"GOROOT=/", env} - output, err := goEnv.CombinedOutput() - t.Log("go output is:", string(output)) - qt.Assert(t, qt.IsNotNil(err), qt.Commentf("No clang target should be a valid build constraint")) - }) - } - - } -} - -func goBin(t *testing.T) string { - t.Helper() - - exe, err := exec.LookPath("go") - if errors.Is(err, exec.ErrNotFound) { - t.Skip("go binary is not in PATH") - } - qt.Assert(t, qt.IsNil(err)) - - return exe -} - func mustWriteFile(tb testing.TB, dir, name, contents string) { tb.Helper() tmpFile := filepath.Join(dir, name)