From 4373862f6f33abe5008abc3c6bae5cdca9bf2d58 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana <47679741+eliottness@users.noreply.github.com> Date: Sat, 25 Nov 2023 10:08:58 +0100 Subject: [PATCH] refactor embed storage and packaging for bazel (#50) --- _tools/libddwaf-updater/update.go | 79 ++++++++++-------- internal/lib/darwin-amd64/vendor.go | 9 -- internal/lib/darwin-arm64/vendor.go | 9 -- internal/lib/lib_darwin_amd64.go | 4 +- internal/lib/lib_darwin_arm64.go | 4 +- internal/lib/lib_linux_amd64.go | 4 +- internal/lib/lib_linux_arm64.go | 4 +- ...dwaf.dylib => libddwaf-darwin-amd64.dylib} | Bin ...dwaf.dylib => libddwaf-darwin-arm64.dylib} | Bin .../libddwaf.so => libddwaf-linux-amd64.so} | Bin .../libddwaf.so => libddwaf-linux-arm64.so} | Bin internal/lib/linux-amd64/vendor.go | 9 -- internal/lib/linux-arm64/vendor.go | 9 -- 13 files changed, 56 insertions(+), 75 deletions(-) delete mode 100644 internal/lib/darwin-amd64/vendor.go delete mode 100644 internal/lib/darwin-arm64/vendor.go rename internal/lib/{darwin-amd64/libddwaf.dylib => libddwaf-darwin-amd64.dylib} (100%) rename internal/lib/{darwin-arm64/libddwaf.dylib => libddwaf-darwin-arm64.dylib} (100%) rename internal/lib/{linux-amd64/libddwaf.so => libddwaf-linux-amd64.so} (100%) rename internal/lib/{linux-arm64/libddwaf.so => libddwaf-linux-arm64.so} (100%) delete mode 100644 internal/lib/linux-amd64/vendor.go delete mode 100644 internal/lib/linux-arm64/vendor.go diff --git a/_tools/libddwaf-updater/update.go b/_tools/libddwaf-updater/update.go index 4ba3fe4c..43d39fa5 100644 --- a/_tools/libddwaf-updater/update.go +++ b/_tools/libddwaf-updater/update.go @@ -30,6 +30,10 @@ var ( currentVersion string ) +const ( + goVersionUnsupported = "go1.22" +) + func main() { force := os.Args[1] == "--force" @@ -63,21 +67,7 @@ func main() { wg := sync.WaitGroup{} wg.Add(len(targets)) for _, tgt := range targets { - embedDir := path.Join(libDir, fmt.Sprintf("%s-%s", tgt.os, tgt.arch)) - created := false - if _, err = os.Stat(embedDir); errors.Is(err, os.ErrNotExist) { - if err = os.MkdirAll(embedDir, 0755); err != nil { - panic(err) - } - created = true - } - if created || force { - if err := os.WriteFile(path.Join(embedDir, "vendor.go"), []byte(vendorMarkerFile), 0644); err != nil { - panic(err) - } - createEmbedSource(tgt) - } - go handleTarget(&wg, version, tgt, embedDir, assets) + go handleTarget(&wg, version, tgt, assets, force) } wg.Wait() @@ -98,6 +88,9 @@ func main() { fmt.Println("All done! Don't forget to check in changes to include/ and internal/lib/, check the libddwaf upgrade guide to update bindings!") } +// createEmbedSource creates the embed source file for the given target. +// The go:embed directive MUST be in the same package as the target of the directive. +// See bazelbuild/bazel-gazelle#1316 for more details. func createEmbedSource(tgt target) { gosource := strings.Join( []string{ @@ -106,25 +99,28 @@ func createEmbedSource(tgt target) { "// This product includes software developed at Datadog (https://www.datadoghq.com/).", "// Copyright 2016-present Datadog, Inc.", "", - fmt.Sprintf("//go:build %s && %s && !go1.22", tgt.os, tgt.arch), + tgt.buildConstraintDirective(), + "", "package lib", "", - `import _ "embed" // Needed for go:embed`, + "// THIS FILE IS AUTOGENERATED. DO NOT EDIT.", "", - fmt.Sprintf("//go:embed %s-%s/%s.%s", tgt.os, tgt.arch, tgt.base, tgt.ext), + "import _ \"embed\" // Needed for go:embed", + "", + tgt.embedSourceDirective(), "var libddwaf []byte", "", - fmt.Sprintf(`const embedNamePattern = "%s-*.%s"`, tgt.base, tgt.ext), + tgt.tempFilePatternStatement(), "", // Trailing new line... }, "\n", ) - if err := os.WriteFile(path.Join(libDir, fmt.Sprintf("lib_%s_%s.go", tgt.os, tgt.arch)), []byte(gosource), 0644); err != nil { + if err := os.WriteFile(path.Join(libDir, tgt.embedSourceFilename()), []byte(gosource), 0644); err != nil { panic(err) } } -func handleTarget(wg *sync.WaitGroup, version string, tgt target, embedDir string, assets map[string]*github.ReleaseAsset) { +func handleTarget(wg *sync.WaitGroup, version string, tgt target, assets map[string]*github.ReleaseAsset, force bool) { defer wg.Done() tarName := fmt.Sprintf("libddwaf-%s-%s.tar.gz", version, tgt.assetLabel) @@ -187,15 +183,14 @@ func handleTarget(wg *sync.WaitGroup, version string, tgt target, embedDir strin var destPath string switch name := header.FileInfo().Name(); name { case "libddwaf.so", "libddwaf.dylib", "ddwaf.dll": - destPath = path.Join(embedDir, name) + destPath = path.Join(libDir, tgt.binaryLibName()) foundLib = true case "ddwaf.h": - if tgt.primary { - destPath = path.Join(rootDir, "include", name) - foundHdr = true - } else { + if !tgt.primary { continue } + destPath = path.Join(rootDir, "include", name) + foundHdr = true default: continue } @@ -217,11 +212,16 @@ func handleTarget(wg *sync.WaitGroup, version string, tgt target, embedDir strin } if !foundLib { - panic(fmt.Errorf("could not find libddwaf.so/libddwaf.dylib in %s", tarUrl)) + panic(fmt.Errorf("could not find libddwaf.%s in %s", tgt.ext, tarUrl)) } if tgt.primary && !foundHdr { panic(fmt.Errorf("could not find ddwaf.h in %s", tarUrl)) } + + // If the embed source file doesn't exist, or if --force is set, create it. + if _, err = os.Stat(path.Join(libDir, tgt.embedSourceFilename())); errors.Is(err, os.ErrNotExist) || force { + createEmbedSource(tgt) + } } type target struct { @@ -273,16 +273,25 @@ var targets = []target{ // {os: "linux", arch: "i386", ext: "so", assetLabel: "i386-linux-musl"}, } -const vendorMarkerFile = `// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. +func (t target) binaryLibName() string { + return fmt.Sprintf("%s-%s-%s.%s", t.base, t.os, t.arch, t.ext) +} -package vendor +func (t target) embedSourceFilename() string { + return fmt.Sprintf("lib_%s_%s.go", t.os, t.arch) +} -// This file is there to make sure "go mod vendor" does not drop those directories, as it otherwise removes -// any directiory that contains no ".go" files. -` +func (t target) buildConstraintDirective() string { + return fmt.Sprintf("//go:build %s && %s && !%s && !datadog.no_waf", t.os, t.arch, goVersionUnsupported) +} + +func (t target) tempFilePatternStatement() string { + return fmt.Sprintf("const embedNamePattern = \"%s-*.%s\"", t.base, t.ext) +} + +func (t target) embedSourceDirective() string { + return fmt.Sprintf("//go:embed %s", t.binaryLibName()) +} func init() { _, filename, _, _ := runtime.Caller(0) diff --git a/internal/lib/darwin-amd64/vendor.go b/internal/lib/darwin-amd64/vendor.go deleted file mode 100644 index 788b2d6b..00000000 --- a/internal/lib/darwin-amd64/vendor.go +++ /dev/null @@ -1,9 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package vendor - -// This file is there to make sure "go mod vendor" does not drop those directories, as it otherwise removes -// any directiory that contains no ".go" files. diff --git a/internal/lib/darwin-arm64/vendor.go b/internal/lib/darwin-arm64/vendor.go deleted file mode 100644 index 788b2d6b..00000000 --- a/internal/lib/darwin-arm64/vendor.go +++ /dev/null @@ -1,9 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package vendor - -// This file is there to make sure "go mod vendor" does not drop those directories, as it otherwise removes -// any directiory that contains no ".go" files. diff --git a/internal/lib/lib_darwin_amd64.go b/internal/lib/lib_darwin_amd64.go index 143d3b5e..89aa4e06 100644 --- a/internal/lib/lib_darwin_amd64.go +++ b/internal/lib/lib_darwin_amd64.go @@ -7,9 +7,11 @@ package lib +// THIS FILE IS AUTOGENERATED. DO NOT EDIT. + import _ "embed" // Needed for go:embed -//go:embed darwin-amd64/libddwaf.dylib +//go:embed libddwaf-darwin-amd64.dylib var libddwaf []byte const embedNamePattern = "libddwaf-*.dylib" diff --git a/internal/lib/lib_darwin_arm64.go b/internal/lib/lib_darwin_arm64.go index c92f4df8..c084c7b0 100644 --- a/internal/lib/lib_darwin_arm64.go +++ b/internal/lib/lib_darwin_arm64.go @@ -7,9 +7,11 @@ package lib +// THIS FILE IS AUTOGENERATED. DO NOT EDIT. + import _ "embed" // Needed for go:embed -//go:embed darwin-arm64/libddwaf.dylib +//go:embed libddwaf-darwin-arm64.dylib var libddwaf []byte const embedNamePattern = "libddwaf-*.dylib" diff --git a/internal/lib/lib_linux_amd64.go b/internal/lib/lib_linux_amd64.go index 694e4af7..63a30865 100644 --- a/internal/lib/lib_linux_amd64.go +++ b/internal/lib/lib_linux_amd64.go @@ -7,9 +7,11 @@ package lib +// THIS FILE IS AUTOGENERATED. DO NOT EDIT. + import _ "embed" // Needed for go:embed -//go:embed linux-amd64/libddwaf.so +//go:embed libddwaf-linux-amd64.so var libddwaf []byte const embedNamePattern = "libddwaf-*.so" diff --git a/internal/lib/lib_linux_arm64.go b/internal/lib/lib_linux_arm64.go index 25e6b547..583ebdb9 100644 --- a/internal/lib/lib_linux_arm64.go +++ b/internal/lib/lib_linux_arm64.go @@ -7,9 +7,11 @@ package lib +// THIS FILE IS AUTOGENERATED. DO NOT EDIT. + import _ "embed" // Needed for go:embed -//go:embed linux-arm64/libddwaf.so +//go:embed libddwaf-linux-arm64.so var libddwaf []byte const embedNamePattern = "libddwaf-*.so" diff --git a/internal/lib/darwin-amd64/libddwaf.dylib b/internal/lib/libddwaf-darwin-amd64.dylib similarity index 100% rename from internal/lib/darwin-amd64/libddwaf.dylib rename to internal/lib/libddwaf-darwin-amd64.dylib diff --git a/internal/lib/darwin-arm64/libddwaf.dylib b/internal/lib/libddwaf-darwin-arm64.dylib similarity index 100% rename from internal/lib/darwin-arm64/libddwaf.dylib rename to internal/lib/libddwaf-darwin-arm64.dylib diff --git a/internal/lib/linux-amd64/libddwaf.so b/internal/lib/libddwaf-linux-amd64.so similarity index 100% rename from internal/lib/linux-amd64/libddwaf.so rename to internal/lib/libddwaf-linux-amd64.so diff --git a/internal/lib/linux-arm64/libddwaf.so b/internal/lib/libddwaf-linux-arm64.so similarity index 100% rename from internal/lib/linux-arm64/libddwaf.so rename to internal/lib/libddwaf-linux-arm64.so diff --git a/internal/lib/linux-amd64/vendor.go b/internal/lib/linux-amd64/vendor.go deleted file mode 100644 index 788b2d6b..00000000 --- a/internal/lib/linux-amd64/vendor.go +++ /dev/null @@ -1,9 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package vendor - -// This file is there to make sure "go mod vendor" does not drop those directories, as it otherwise removes -// any directiory that contains no ".go" files. diff --git a/internal/lib/linux-arm64/vendor.go b/internal/lib/linux-arm64/vendor.go deleted file mode 100644 index 788b2d6b..00000000 --- a/internal/lib/linux-arm64/vendor.go +++ /dev/null @@ -1,9 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package vendor - -// This file is there to make sure "go mod vendor" does not drop those directories, as it otherwise removes -// any directiory that contains no ".go" files.