Packaging of journalbeat (#8702)

Journalbeat is going to be built using the new Debian 8 container, because systemd version in Debian 7 is too old (v44 instead of the required v187).

Minor changes:
* add missing X-Pack folder to journalbeat
* do not crosscompile journalbeat due to missing dependencies locally

.travis.yml

@@ -162,12 +162,11 @@ addons:
   apt:
     update: true
     packages:
-      - libc6-dev-i386
-      - libpcap-dev
-      - libsystemd-journal-dev
-      - libxml2-utils
       - python-virtualenv
+      - libpcap-dev
       - xsltproc
+      - libxml2-utils
+      - libsystemd-journal-dev
 
 before_install:
   - python --version

Makefile

@@ -1,6 +1,7 @@
 BUILD_DIR=$(CURDIR)/build
 COVERAGE_DIR=$(BUILD_DIR)/coverage
-BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat x-pack/functionbeat
+BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat x-pack/functionbeat
+>>>>>>> Packaging of journalbeat (#8702)
 PROJECTS=libbeat $(BEATS)
 PROJECTS_ENV=libbeat filebeat metricbeat
 PYTHON_ENV?=$(BUILD_DIR)/python-env

dev-tools/mage/crossbuild.go

@@ -48,6 +48,9 @@ func init() {
 // CrossBuildOption defines a option to the CrossBuild target.
 type CrossBuildOption func(params *crossBuildParams)
 
+// ImageSelectorFunc returns the name of the builder image.
+type ImageSelectorFunc func(platform string) (string, error)
+
 // ForPlatforms filters the platforms based on the given expression.
 func ForPlatforms(expr string) func(params *crossBuildParams) {
 	return func(params *crossBuildParams) {
@@ -78,16 +81,24 @@ func Serially() func(params *crossBuildParams) {
 	}
 }
 
+// ImageSelector returns the name of the selected builder image.
+func ImageSelector(f ImageSelectorFunc) func(params *crossBuildParams) {
+	return func(params *crossBuildParams) {
+		params.ImageSelector = f
+	}
+}
+
 type crossBuildParams struct {
-	Platforms BuildPlatformList
-	Target    string
-	Serial    bool
-	InDir     string
+	Platforms     BuildPlatformList
+	Target        string
+	Serial        bool
+	InDir         string
+	ImageSelector ImageSelectorFunc
 }
 
 // CrossBuild executes a given build target once for each target platform.
 func CrossBuild(options ...CrossBuildOption) error {
-	params := crossBuildParams{Platforms: Platforms, Target: defaultCrossBuildTarget}
+	params := crossBuildParams{Platforms: Platforms, Target: defaultCrossBuildTarget, ImageSelector: crossBuildImage}
 	for _, opt := range options {
 		opt(&params)
 	}
@@ -111,10 +122,10 @@ func CrossBuild(options ...CrossBuildOption) error {
 		if !buildPlatform.Flags.CanCrossBuild() {
 			return fmt.Errorf("unsupported cross build platform %v", buildPlatform.Name)
 		}
-		builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir}
+		builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir, params.ImageSelector}
 		if params.Serial {
 			if err := builder.Build(); err != nil {
-				return errors.Wrapf(err, "failed cross-building target=%v for platform=%v",
+				return errors.Wrapf(err, "failed cross-building target=%v for platform=%v %v", params.ImageSelector,
 					params.Target, buildPlatform.Name)
 			}
 		} else {
@@ -174,15 +185,16 @@ func crossBuildImage(platform string) (string, error) {
 		return "", err
 	}
 
-	return beatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil
+	return BeatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil
 }
 
 // GolangCrossBuilder executes the specified mage target inside of the
 // associated golang-crossbuild container image for the platform.
 type GolangCrossBuilder struct {
-	Platform string
-	Target   string
-	InDir    string
+	Platform      string
+	Target        string
+	InDir         string
+	ImageSelector ImageSelectorFunc
 }
 
 // Build executes the build inside of Docker.
@@ -208,7 +220,7 @@ func (b GolangCrossBuilder) Build() error {
 	}
 
 	dockerRun := sh.RunCmd("docker", "run")
-	image, err := crossBuildImage(b.Platform)
+	image, err := b.ImageSelector(b.Platform)
 	if err != nil {
 		return errors.Wrap(err, "failed to determine golang-crossbuild image tag")
 	}

dev-tools/mage/godaemon.go

@@ -23,6 +23,13 @@ import (
 	"os"
 )
 
+var (
+	defaultCrossBuildGoDaemon = []CrossBuildOption{
+		ForPlatforms("linux"),
+		WithTarget("buildGoDaemon"),
+	}
+)
+
 // BuildGoDaemon builds the go-deamon binary.
 func BuildGoDaemon() error {
 	if GOOS != "linux" {
@@ -67,6 +74,7 @@ func BuildGoDaemon() error {
 
 // CrossBuildGoDaemon cross-build the go-daemon binary using the
 // golang-crossbuild environment.
-func CrossBuildGoDaemon() error {
-	return CrossBuild(ForPlatforms("linux"), WithTarget("buildGoDaemon"))
+func CrossBuildGoDaemon(options ...CrossBuildOption) error {
+	opts := append(defaultCrossBuildGoDaemon, options...)
+	return CrossBuild(opts...)
 }

dev-tools/mage/settings.go

@@ -39,8 +39,9 @@ const (
 	fpmVersion = "1.10.0"
 
 	// Docker images. See https://github.com/elastic/golang-crossbuild.
-	beatsFPMImage        = "docker.elastic.co/beats-dev/fpm"
-	beatsCrossBuildImage = "docker.elastic.co/beats-dev/golang-crossbuild"
+	beatsFPMImage = "docker.elastic.co/beats-dev/fpm"
+	// BeatsCrossBuildImage is the image used for crossbuilding Beats.
+	BeatsCrossBuildImage = "docker.elastic.co/beats-dev/golang-crossbuild"
 
 	elasticBeatsImportPath = "github.com/elastic/beats"
 )

journalbeat/Makefile

@@ -3,8 +3,6 @@ BEAT_TITLE=Journalbeat
 SYSTEM_TESTS=false
 TEST_ENVIRONMENT=false
 ES_BEATS?=..
-GOX_FLAGS=-cgo
-GOX_OS=linux
 
 # Path to the libbeat Makefile
 -include $(ES_BEATS)/libbeat/scripts/Makefile

journalbeat/journalbeat.yml

@@ -142,6 +142,14 @@ output.elasticsearch:
   # Client Certificate Key
   #ssl.key: "/etc/pki/client/cert.key"
 
+#================================ Procesors =====================================
+
+# Configure processors to enhance or manipulate events generated by the beat.
+
+processors:
+  - add_host_metadata: ~
+  - add_cloud_metadata: ~
+
 #================================ Logging =====================================
 
 # Sets log level. The default log level is info.

journalbeat/magefile.go

@@ -22,20 +22,26 @@ package main
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
+	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/dev-tools/mage"
 )
 
 func init() {
 	mage.BeatDescription = "Journalbeat ships systemd journal entries to Elasticsearch or Logstash."
 
-	// TODO filter platforms
+	mage.Platforms = mage.Platforms.Filter("linux !linux/ppc64 !linux/mips64")
 }
 
+const (
+	libsystemdDevPkgName = "libsystemd-dev"
+)
+
 // Build builds the Beat binary.
 func Build() error {
 	return mage.Build(mage.DefaultBuildArgs())
@@ -44,6 +50,9 @@ func Build() error {
 // GolangCrossBuild build the Beat binary inside of the golang-builder.
 // Do not use directly, use crossBuild instead.
 func GolangCrossBuild() error {
+	if d, ok := deps[mage.Platform.Name]; ok {
+		mg.Deps(d)
+	}
 	return mage.GolangCrossBuild(mage.DefaultGolangCrossBuildArgs())
 }
 
@@ -54,17 +63,17 @@ func BuildGoDaemon() error {
 
 // CrossBuild cross-builds the beat for all target platforms.
 func CrossBuild() error {
-	return mage.CrossBuild()
+	return mage.CrossBuild(mage.ImageSelector(selectImage))
 }
 
 // CrossBuildXPack cross-builds the beat with XPack for all target platforms.
 func CrossBuildXPack() error {
-	return mage.CrossBuildXPack()
+	return mage.CrossBuildXPack(mage.ImageSelector(selectImage))
 }
 
 // CrossBuildGoDaemon cross-builds the go-daemon binary using Docker.
 func CrossBuildGoDaemon() error {
-	return mage.CrossBuildGoDaemon()
+	return mage.CrossBuildGoDaemon(mage.ImageSelector(selectImage))
 }
 
 // Clean cleans all generated files and build artifacts.
@@ -80,6 +89,7 @@ func Package() {
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
 	mage.UseElasticBeatPackaging()
+
 	mg.Deps(Update)
 	mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon)
 	mg.SerialDeps(mage.Package, TestPackages)
@@ -113,3 +123,106 @@ func GoTestUnit(ctx context.Context) error {
 func GoTestIntegration(ctx context.Context) error {
 	return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs())
 }
+
+// -----------------------------------------------------------------------------
+// Customizations specific to Journalbeat.
+// - Install required headers on builders for different architectures.
+
+var (
+	deps = map[string]func() error{
+		"linux/386":      installLinux386,
+		"linux/amd64":    installLinuxAMD64,
+		"linux/arm64":    installLinuxARM64,
+		"linux/armv5":    installLinuxARMLE,
+		"linux/armv6":    installLinuxARMLE,
+		"linux/armv7":    installLinuxARMHF,
+		"linux/mips":     installLinuxMIPS,
+		"linux/mipsle":   installLinuxMIPSLE,
+		"linux/mips64le": installLinuxMIPS64LE,
+		"linux/ppc64le":  installLinuxPPC64LE,
+		"linux/s390x":    installLinuxS390X,
+
+		// No deb packages
+		//"linux/ppc64": installLinuxPpc64,
+		//"linux/mips64": installLinuxMips64,
+	}
+)
+
+func installLinuxAMD64() error {
+	return installDependencies(libsystemdDevPkgName, "")
+}
+
+func installLinuxARM64() error {
+	return installDependencies(libsystemdDevPkgName+":arm64", "arm64")
+}
+
+func installLinuxARMHF() error {
+	return installDependencies(libsystemdDevPkgName+":armhf", "armhf")
+}
+
+func installLinuxARMLE() error {
+	return installDependencies(libsystemdDevPkgName+":armel", "armel")
+}
+
+func installLinux386() error {
+	return installDependencies(libsystemdDevPkgName+":i386", "i386")
+}
+
+func installLinuxMIPS() error {
+	return installDependencies(libsystemdDevPkgName+":mips", "mips")
+}
+
+func installLinuxMIPS64LE() error {
+	return installDependencies(libsystemdDevPkgName+":mips64el", "mips64el")
+}
+
+func installLinuxMIPSLE() error {
+	return installDependencies(libsystemdDevPkgName+":mipsel", "mipsel")
+}
+
+func installLinuxPPC64LE() error {
+	return installDependencies(libsystemdDevPkgName+":ppc64el", "ppc64el")
+}
+
+func installLinuxS390X() error {
+	return installDependencies(libsystemdDevPkgName+":s390x", "s390x")
+}
+
+func installDependencies(pkg, arch string) error {
+	if arch != "" {
+		err := sh.Run("dpkg", "--add-architecture", arch)
+		if err != nil {
+			return errors.Wrap(err, "error while adding architecture")
+		}
+	}
+
+	if err := sh.Run("apt-get", "update"); err != nil {
+		return err
+	}
+
+	return sh.Run("apt-get", "install", "-y", "--no-install-recommends", pkg)
+}
+
+func selectImage(platform string) (string, error) {
+	tagSuffix := "main"
+
+	switch {
+	case strings.HasPrefix(platform, "linux/arm"):
+		tagSuffix = "arm"
+	case strings.HasPrefix(platform, "linux/mips"):
+		tagSuffix = "mips"
+	case strings.HasPrefix(platform, "linux/ppc"):
+		tagSuffix = "ppc"
+	case platform == "linux/s390x":
+		tagSuffix = "s390x"
+	case strings.HasPrefix(platform, "linux"):
+		tagSuffix = "main-debian8"
+	}
+
+	goVersion, err := mage.GoVersion()
+	if err != nil {
+		return "", err
+	}
+
+	return mage.BeatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil
+}

libbeat/scripts/Makefile

@@ -107,9 +107,11 @@ ${BEAT_NAME}.test: $(GOFILES_ALL)
 .PHONY: crosscompile
 crosscompile: ## @build Cross-compile beat for the OS'es specified in GOX_OS variable. The binaries are placed in the build/bin directory.
 crosscompile: $(GOFILES)
+ifneq ($(shell [[ $(BEAT_NAME) == journalbeat ]] && echo true ),true)
 	go get github.com/mitchellh/gox
 	mkdir -p ${BUILD_DIR}/bin
 	gox -output="${BUILD_DIR}/bin/{{.Dir}}-{{.OS}}-{{.Arch}}" -os="$(strip $(GOX_OS))" -osarch="$(strip $(GOX_OSARCH))" ${GOX_FLAGS}
+endif
 
 .PHONY: check
 check: check-headers python-env prepare-tests ## @build Checks project and source code if everything is according to standard

magefile.go

@@ -31,6 +31,7 @@ var (
 		"auditbeat",
 		"filebeat",
 		"heartbeat",
+		"journalbeat",
 		"metricbeat",
 		"packetbeat",
 		"winlogbeat",

x-pack/journalbeat/cmd/root.go

@@ -0,0 +1,17 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	"github.com/elastic/beats/journalbeat/cmd"
+	xpackcmd "github.com/elastic/beats/x-pack/libbeat/cmd"
+)
+
+// RootCmd to handle beats cli
+var RootCmd = cmd.RootCmd
+
+func init() {
+	xpackcmd.AddXPack(RootCmd, cmd.Name)
+}

x-pack/journalbeat/main.go

@@ -0,0 +1,19 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+import (
+	"os"
+
+	"github.com/elastic/beats/x-pack/journalbeat/cmd"
+
+	_ "github.com/elastic/beats/journalbeat/include"
+)
+
+func main() {
+	if err := cmd.RootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}