Skip to content

Commit

Permalink
Cherry-pick #21979 to 7.x: [Packetbeat] Create x-pack magefile (#22545)
Browse files Browse the repository at this point in the history 
* [Packetbeat] Create x-pack magefile (#21979)

* Create packetbeat x-pack magefile

* add changelog entry

* Fix up packetbeat magefile

* Add makefile for x-pack directory

* Uncomment packaging

* Add desired platforms to Jenkinsfile

* Add back in fields yml and kibana dashboards on package

* Add generated configs in x-pack/packetbeat

* Suggested changes

(cherry picked from commit 3e73c99)

* Fix up changelog
  • Loading branch information
Andrew Stucki authored Nov 11, 2020
1 parent 909c16b commit bcbef06
Show file tree
Hide file tree
Showing 13 changed files with 2,861 additions and 321 deletions.
2 changes: 1 addition & 1 deletion .ci/packaging.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ pipeline {
'x-pack/heartbeat',
// 'x-pack/journalbeat',
'x-pack/metricbeat',
// 'x-pack/packetbeat',
'x-pack/packetbeat',
'x-pack/winlogbeat'
)
}
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,6 +588,7 @@ same journal. {pull}18467[18467]

`host` metadata fields when processing network data from network tap or mirror
port. {pull}19209[19209]
- Change build process for x-pack distribution {pull}21979[21979]


*Functionbeat*
Expand Down
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ XPACK_SUFFIX=x-pack/
# PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support
# in the x-pack directory (rather than having the OSS build produce both sets
# of artifacts). This will be removed once we complete the transition.
PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/heartbeat x-pack/metricbeat x-pack/winlogbeat
PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/heartbeat x-pack/metricbeat x-pack/winlogbeat x-pack/packetbeat
# PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in
# Mage. For compatibility with CI testing these projects support a subset of the
# makefile targets. After all Beats converge to primarily using Mage we can
Expand Down
323 changes: 6 additions & 317 deletions packetbeat/magefile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,9 @@ package main

import (
"fmt"
"log"
"strings"
"time"

"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/pkg/errors"

devtools "github.com/elastic/beats/v7/dev-tools/mage"
packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage"
Expand All @@ -44,7 +40,7 @@ import (

func init() {
common.RegisterCheckDeps(Update)
unittest.RegisterPythonTestDeps(fieldsYML, Dashboards)
unittest.RegisterPythonTestDeps(packetbeat.FieldsYML, Dashboards)

devtools.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch."
}
Expand All @@ -57,21 +53,7 @@ func Build() error {
// GolangCrossBuild build the Beat binary inside of the golang-builder.
// Do not use directly, use crossBuild instead.
func GolangCrossBuild() error {
if dep, found := crossBuildDeps[devtools.Platform.Name]; found {
mg.Deps(dep)
}

params := devtools.DefaultGolangCrossBuildArgs()
if flags, found := libpcapLDFLAGS[devtools.Platform.Name]; found {
params.Env = map[string]string{
"CGO_LDFLAGS": flags,
}
}
if flags, found := libpcapCFLAGS[devtools.Platform.Name]; found {
params.Env["CGO_CFLAGS"] = flags
}

return devtools.GolangCrossBuild(params)
return packetbeat.GolangCrossBuild()
}

// BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon).
Expand Down Expand Up @@ -118,12 +100,12 @@ func Package() {
start := time.Now()
defer func() { fmt.Println("package ran for", time.Since(start)) }()

devtools.UseElasticBeatPackaging()
devtools.UseElasticBeatOSSPackaging()
devtools.PackageKibanaDashboardsFromBuildDir()
customizePackaging()
packetbeat.CustomizePackaging()

mg.Deps(Update)
mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon)
mg.Deps(CrossBuild, CrossBuildGoDaemon)
mg.SerialDeps(devtools.Package, TestPackages)
}

Expand Down Expand Up @@ -151,27 +133,7 @@ func includeList() error {

// Fields generates fields.yml and fields.go files for the Beat.
func Fields() {
mg.Deps(libbeatAndPacketbeatCommonFieldsGo, protosFieldsGo)
mg.Deps(fieldsYML)
}

// libbeatAndPacketbeatCommonFieldsGo generates a fields.go containing both
// libbeat and packetbeat's common fields.
func libbeatAndPacketbeatCommonFieldsGo() error {
if err := devtools.GenerateFieldsYAML(); err != nil {
return err
}
return devtools.GenerateAllInOneFieldsGo()
}

// protosFieldsGo generates a fields.go for each protocol.
func protosFieldsGo() error {
return devtools.GenerateModuleFieldsGo("protos")
}

// fieldsYML generates the fields.yml file containing all fields.
func fieldsYML() error {
return devtools.GenerateFieldsYAML("protos")
packetbeat.Fields()
}

func fieldDocs() error {
Expand All @@ -182,276 +144,3 @@ func fieldDocs() error {
func Dashboards() error {
return devtools.KibanaDashboards("protos")
}

// -----------------------------------------------------------------------------
// Customizations specific to Packetbeat.
// - Config file contains an OS specific device name (affects darwin, windows).
// - Must compile libpcap or winpcap during cross-compilation.
// - On Linux libpcap is statically linked. Darwin and Windows are dynamic.

const (
libpcapURL = "https://s3.amazonaws.com/beats-files/deps/libpcap-1.8.1.tar.gz"
libpcapSHA256 = "673dbc69fdc3f5a86fb5759ab19899039a8e5e6c631749e48dcd9c6f0c83541e"
)

const (
linuxPcapLDFLAGS = "-L/libpcap/libpcap-1.8.1 -lpcap"
linuxPcapCFLAGS = "-I /libpcap/libpcap-1.8.1"
)

var libpcapLDFLAGS = map[string]string{
"linux/386": linuxPcapLDFLAGS,
"linux/amd64": linuxPcapLDFLAGS,
"linux/arm64": linuxPcapLDFLAGS,
"linux/armv5": linuxPcapLDFLAGS,
"linux/armv6": linuxPcapLDFLAGS,
"linux/armv7": linuxPcapLDFLAGS,
"linux/mips": linuxPcapLDFLAGS,
"linux/mipsle": linuxPcapLDFLAGS,
"linux/mips64": linuxPcapLDFLAGS,
"linux/mips64le": linuxPcapLDFLAGS,
"linux/ppc64le": linuxPcapLDFLAGS,
"linux/s390x": linuxPcapLDFLAGS,
"darwin/amd64": "-lpcap",
"windows/amd64": "-L /libpcap/win/WpdPack/Lib/x64 -lwpcap",
"windows/386": "-L /libpcap/win/WpdPack/Lib -lwpcap",
}

var libpcapCFLAGS = map[string]string{
"linux/386": linuxPcapCFLAGS,
"linux/amd64": linuxPcapCFLAGS,
"linux/arm64": linuxPcapCFLAGS,
"linux/armv5": linuxPcapCFLAGS,
"linux/armv6": linuxPcapCFLAGS,
"linux/armv7": linuxPcapCFLAGS,
"linux/mips": linuxPcapCFLAGS,
"linux/mipsle": linuxPcapCFLAGS,
"linux/mips64": linuxPcapCFLAGS,
"linux/mips64le": linuxPcapCFLAGS,
"linux/ppc64le": linuxPcapCFLAGS,
"linux/s390x": linuxPcapCFLAGS,
"windows/amd64": "-I /libpcap/win/WpdPack/Include",
"windows/386": "-I /libpcap/win/WpdPack/Include",
}

var crossBuildDeps = map[string]func() error{
"linux/386": buildLibpcapLinux386,
"linux/amd64": buildLibpcapLinuxAMD64,
"linux/arm64": buildLibpcapLinuxARM64,
"linux/armv5": buildLibpcapLinuxARMv5,
"linux/armv6": buildLibpcapLinuxARMv6,
"linux/armv7": buildLibpcapLinuxARMv7,
"linux/mips": buildLibpcapLinuxMIPS,
"linux/mipsle": buildLibpcapLinuxMIPSLE,
"linux/mips64": buildLibpcapLinuxMIPS64,
"linux/mips64le": buildLibpcapLinuxMIPS64LE,
"linux/ppc64le": buildLibpcapLinuxPPC64LE,
"linux/s390x": buildLibpcapLinuxS390x,
"windows/amd64": installLibpcapWindowsAMD64,
"windows/386": installLibpcapWindows386,
}

// buildLibpcapFromSource builds libpcap from source because the library needs
// to be compiled with -fPIC.
// See https://github.com/elastic/beats/v7/pull/4217.
func buildLibpcapFromSource(params map[string]string) error {
tarFile, err := devtools.DownloadFile(libpcapURL, "/libpcap")
if err != nil {
return errors.Wrap(err, "failed to download libpcap source")
}

if err = devtools.VerifySHA256(tarFile, libpcapSHA256); err != nil {
return err
}

if err = devtools.Extract(tarFile, "/libpcap"); err != nil {
return errors.Wrap(err, "failed to extract libpcap")
}

var configureArgs []string
for k, v := range params {
if strings.HasPrefix(k, "-") {
delete(params, k)
configureArgs = append(configureArgs, k+"="+v)
}
}

// Use sh -c here because sh.Run does not expose a way to change the CWD.
// This command only runs in Linux so this is fine.
return sh.RunWith(params, "sh", "-c",
"cd /libpcap/libpcap-1.8.1 && "+
"./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no "+strings.Join(configureArgs, " ")+"&& "+
"make")
}

func buildLibpcapLinux386() error {
return buildLibpcapFromSource(map[string]string{
"CFLAGS": "-m32",
"LDFLAGS": "-m32",
})
}

func buildLibpcapLinuxAMD64() error {
return buildLibpcapFromSource(map[string]string{})
}

func buildLibpcapLinuxARM64() error {
return buildLibpcapFromSource(map[string]string{
"--host": "aarch64-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxARMv5() error {
return buildLibpcapFromSource(map[string]string{
"--host": "arm-linux-gnueabi",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxARMv6() error {
return buildLibpcapFromSource(map[string]string{
"--host": "arm-linux-gnueabi",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxARMv7() error {
return buildLibpcapFromSource(map[string]string{
"--host": "arm-linux-gnueabihf",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxMIPS() error {
return buildLibpcapFromSource(map[string]string{
"--host": "mips-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxMIPSLE() error {
return buildLibpcapFromSource(map[string]string{
"--host": "mipsle-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxMIPS64() error {
return buildLibpcapFromSource(map[string]string{
"--host": "mips64-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxMIPS64LE() error {
return buildLibpcapFromSource(map[string]string{
"--host": "mips64le-unknown-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxPPC64LE() error {
return buildLibpcapFromSource(map[string]string{
"--host": "powerpc64le-linux-gnu",
"--with-pcap": "linux",
})
}

func buildLibpcapLinuxS390x() error {
return buildLibpcapFromSource(map[string]string{
"--host": "s390x-ibm-linux-gnu",
"--with-pcap": "linux",
})
}

func installLibpcapWindowsAMD64() error {
mg.SerialDeps(installWinpcap, generateWin64StaticWinpcap)
return nil
}

func installLibpcapWindows386() error {
return installWinpcap()
}

func installWinpcap() error {
log.Println("Install Winpcap")
const wpdpackURL = "https://www.winpcap.org/install/bin/WpdPack_4_1_2.zip"

winpcapZip, err := devtools.DownloadFile(wpdpackURL, "/")
if err != nil {
return err
}

if err = devtools.Extract(winpcapZip, "/libpcap/win"); err != nil {
return err
}

return nil
}

func generateWin64StaticWinpcap() error {
log.Println(">> Generating 64-bit winpcap static lib")

// Notes: We are using absolute path to make sure the files
// are available for x-pack build.
// Ref: https://github.com/elastic/beats/v7/issues/1259
defer devtools.DockerChown(devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib"))
return devtools.RunCmds(
// Requires mingw-w64-tools.
[]string{"gendef", devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.dll")},
[]string{"mv", "wpcap.def", devtools.MustExpand("{{ elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")},
[]string{"x86_64-w64-mingw32-dlltool", "--as-flags=--64",
"-m", "i386:x86-64", "-k",
"--output-lib", "/libpcap/win/WpdPack/Lib/x64/libwpcap.a",
"--input-def", devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")},
)
}

// customizePackaging modifies the device in the configuration files based on
// the target OS.
func customizePackaging() {
var (
configYml = devtools.PackageFile{
Mode: 0600,
Source: "{{.PackageDir}}/{{.BeatName}}.yml",
Config: true,
Dep: func(spec devtools.PackageSpec) error {
c := packetbeat.ConfigFileParams()
c.ExtraVars["GOOS"] = spec.OS
c.ExtraVars["GOARCH"] = spec.MustExpand("{{.GOARCH}}")
return devtools.Config(devtools.ShortConfigType, c, spec.MustExpand("{{.PackageDir}}"))
},
}
referenceConfigYml = devtools.PackageFile{
Mode: 0644,
Source: "{{.PackageDir}}/{{.BeatName}}.reference.yml",
Dep: func(spec devtools.PackageSpec) error {
c := packetbeat.ConfigFileParams()
c.ExtraVars["GOOS"] = spec.OS
c.ExtraVars["GOARCH"] = spec.MustExpand("{{.GOARCH}}")
return devtools.Config(devtools.ReferenceConfigType, c, spec.MustExpand("{{.PackageDir}}"))
},
}
)

for _, args := range devtools.Packages {
for _, pkgType := range args.Types {
switch pkgType {
case devtools.TarGz, devtools.Zip:
args.Spec.ReplaceFile("{{.BeatName}}.yml", configYml)
args.Spec.ReplaceFile("{{.BeatName}}.reference.yml", referenceConfigYml)
case devtools.Deb, devtools.RPM, devtools.DMG:
args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", configYml)
args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfigYml)
case devtools.Docker:
args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin=eip")
default:
panic(errors.Errorf("unhandled package type: %v", pkgType))
}

// Match the first package type then continue.
break
}
}
}
Loading

0 comments on commit bcbef06

Please sign in to comment.