From 0647e324e9013d90f3f6f101c62f37cb1f32f4ee Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:11:24 +0100 Subject: [PATCH] Include Npcap in Packetbeat when bundled with Agentbeat (#41909) (#42081) (cherry picked from commit 3a2c24c530139904fc6ec837abe4bf291f196ce7) Co-authored-by: Michal Pristas --- x-pack/agentbeat/magefile.go | 5 +- x-pack/packetbeat/magefile.go | 76 ++-------------------- x-pack/packetbeat/scripts/mage/pcap.go | 88 ++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 71 deletions(-) create mode 100644 x-pack/packetbeat/scripts/mage/pcap.go diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index b65a3e59af36..888ebb8bb582 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -23,6 +23,7 @@ import ( metricbeat "github.com/elastic/beats/v7/metricbeat/scripts/mage" packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage" osquerybeat "github.com/elastic/beats/v7/x-pack/osquerybeat/scripts/mage" + xpacketbeat "github.com/elastic/beats/v7/x-pack/packetbeat/scripts/mage" //mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" @@ -85,7 +86,9 @@ func GolangCrossBuild() error { // CrossBuild cross-builds the beat for all target platforms. func CrossBuild() error { - return devtools.CrossBuild() + return devtools.CrossBuild( + devtools.ImageSelector(xpacketbeat.ImageSelector), + ) } // BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon). diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go index 543498164683..5e938cbbda72 100644 --- a/x-pack/packetbeat/magefile.go +++ b/x-pack/packetbeat/magefile.go @@ -8,21 +8,16 @@ package main import ( "context" - "errors" "fmt" - "io/fs" "os" - "path/filepath" - "runtime" - "strings" "time" "github.com/magefile/mage/mg" - "github.com/magefile/mage/sh" devtools "github.com/elastic/beats/v7/dev-tools/mage" "github.com/elastic/beats/v7/dev-tools/mage/target/build" packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage" + xpacketbeat "github.com/elastic/beats/v7/x-pack/packetbeat/scripts/mage" //mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" @@ -32,14 +27,6 @@ import ( "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) -// NpcapVersion specifies the version of the OEM Npcap installer to bundle with -// the packetbeat executable. It is used to specify which npcap builder crossbuild -// image to use and the installer to obtain from the cloud store for testing. -const ( - NpcapVersion = "1.80" - installer = "npcap-" + NpcapVersion + "-oem.exe" -) - func init() { common.RegisterCheckDeps(Update) @@ -73,12 +60,10 @@ func Build() error { // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { - if devtools.Platform.GOOS == "windows" && (devtools.Platform.GOARCH == "amd64" || devtools.Platform.GOARCH == "386") { - err := sh.Copy("./npcap/installer/"+installer, "/installer/"+installer) - if err != nil { - return fmt.Errorf("failed to copy Npcap installer into source tree: %w", err) - } + if err := xpacketbeat.CopyNPCAPInstaller(); err != nil { + return err } + return packetbeat.GolangCrossBuild() } @@ -94,20 +79,7 @@ func CrossBuild() error { // by concurrent builds. See https://github.com/elastic/beats/issues/24304. devtools.Serially(), - devtools.ImageSelector(func(platform string) (string, error) { - image, err := devtools.CrossBuildImage(platform) - if err != nil { - return "", err - } - if os.Getenv("CI") != "true" && os.Getenv("NPCAP_LOCAL") != "true" { - return image, nil - } - if platform == "windows/amd64" { - image = strings.ReplaceAll(image, "beats-dev", "observability-ci") // Temporarily work around naming of npcap image. - image = strings.ReplaceAll(image, "main", "npcap-"+NpcapVersion+"-debian9") - } - return image, nil - }), + devtools.ImageSelector(xpacketbeat.ImageSelector), ) } @@ -169,46 +141,10 @@ func SystemTest(ctx context.Context) error { if os.Getenv("CI") == "true" { mg.SerialDeps(devtools.BuildSystemTestBinary) } else { - mg.SerialDeps(getNpcapInstaller, devtools.BuildSystemTestBinary) + mg.SerialDeps(xpacketbeat.GetNpcapInstaller, devtools.BuildSystemTestBinary) } args := devtools.DefaultGoTestIntegrationArgs() args.Packages = []string{"./tests/system/..."} return devtools.GoTest(ctx, args) } - -func getBucketName() string { - return "ingest-buildkite-ci" -} - -// getNpcapInstaller gets the installer from the Google Cloud Storage service. -// -// On Windows platforms, if getNpcapInstaller is invoked with the environment variables -// CI or NPCAP_LOCAL set to "true" and the OEM Npcap installer is not available it is -// obtained from the cloud storage. This behaviour requires access to the private store. -// If NPCAP_LOCAL is set to "true" and the file is in the npcap/installer directory, no -// fetch will be made. -func getNpcapInstaller() error { - // TODO: Consider whether to expose this as a target. - if runtime.GOOS != "windows" { - return nil - } - if os.Getenv("CI") != "true" && os.Getenv("NPCAP_LOCAL") != "true" { - return errors.New("only available if running in the CI or with NPCAP_LOCAL=true") - } - dstPath := filepath.Join("./npcap/installer", installer) - if os.Getenv("NPCAP_LOCAL") == "true" { - fi, err := os.Stat(dstPath) - if err == nil && !fi.IsDir() { - fmt.Println("using local Npcap installer with NPCAP_LOCAL=true") - return nil - } - if !errors.Is(err, fs.ErrNotExist) { - return err - } - } - ciBucketName := getBucketName() - - fmt.Printf("getting %s from private cache\n", installer) - return sh.RunV("gsutil", "cp", "gs://"+ciBucketName+"/private/"+installer, dstPath) -} diff --git a/x-pack/packetbeat/scripts/mage/pcap.go b/x-pack/packetbeat/scripts/mage/pcap.go new file mode 100644 index 000000000000..703daed2b272 --- /dev/null +++ b/x-pack/packetbeat/scripts/mage/pcap.go @@ -0,0 +1,88 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package mage + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/magefile/mage/sh" + + devtools "github.com/elastic/beats/v7/dev-tools/mage" +) + +// NpcapVersion specifies the version of the OEM Npcap installer to bundle with +// the packetbeat executable. It is used to specify which npcap builder crossbuild +// image to use and the installer to obtain from the cloud store for testing. +const ( + NpcapVersion = "1.80" + installer = "npcap-" + NpcapVersion + "-oem.exe" +) + +func ImageSelector(platform string) (string, error) { + image, err := devtools.CrossBuildImage(platform) + if err != nil { + return "", err + } + if os.Getenv("CI") != "true" && os.Getenv("NPCAP_LOCAL") != "true" { + return image, nil + } + if platform == "windows/amd64" { + image = strings.ReplaceAll(image, "beats-dev", "observability-ci") // Temporarily work around naming of npcap image. + image = strings.ReplaceAll(image, "main", "npcap-"+NpcapVersion+"-debian9") + } + return image, nil +} + +func CopyNPCAPInstaller() error { + if devtools.Platform.GOOS == "windows" && (devtools.Platform.GOARCH == "amd64" || devtools.Platform.GOARCH == "386") { + err := sh.Copy("./npcap/installer/"+installer, "/installer/"+installer) + if err != nil { + return fmt.Errorf("failed to copy Npcap installer into source tree: %w", err) + } + } + return nil +} + +// GetNpcapInstaller gets the installer from the Google Cloud Storage service. +// +// On Windows platforms, if getNpcapInstaller is invoked with the environment variables +// CI or NPCAP_LOCAL set to "true" and the OEM Npcap installer is not available it is +// obtained from the cloud storage. This behaviour requires access to the private store. +// If NPCAP_LOCAL is set to "true" and the file is in the npcap/installer directory, no +// fetch will be made. +func GetNpcapInstaller() error { + // TODO: Consider whether to expose this as a target. + if runtime.GOOS != "windows" { + return nil + } + if os.Getenv("CI") != "true" && os.Getenv("NPCAP_LOCAL") != "true" { + return errors.New("only available if running in the CI or with NPCAP_LOCAL=true") + } + dstPath := filepath.Join("./npcap/installer", installer) + if os.Getenv("NPCAP_LOCAL") == "true" { + fi, err := os.Stat(dstPath) + if err == nil && !fi.IsDir() { + fmt.Println("using local Npcap installer with NPCAP_LOCAL=true") //nolint:forbidigo // fmt.Println is ok here + return nil + } + if !errors.Is(err, fs.ErrNotExist) { + return err + } + } + ciBucketName := getBucketName() + + fmt.Printf("getting %s from private cache\n", installer) //nolint:forbidigo // fmt.Println is ok here + return sh.RunV("gsutil", "cp", "gs://"+ciBucketName+"/private/"+installer, dstPath) +} + +func getBucketName() string { + return "ingest-buildkite-ci" +}