Update beats

.gitignore

@@ -17,5 +17,6 @@
 /_meta/kibana.generated/*
 /fields.yml
 /apm-server.template-es.json
+/vendor/github.com/elastic/beats/libbeat/fields.yml
 
 html_docs

Makefile

@@ -11,7 +11,6 @@ ES_BEATS?=./_beats
 BEATS_VERSION?=master
 NOW=$(shell date -u '+%Y-%m-%dT%H:%M:%S')
 GOBUILD_FLAGS=-i -ldflags "-s -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.buildTime=$(NOW) -X $(BEAT_PATH)/vendor/github.com/elastic/beats/libbeat/version.commit=$(COMMIT_ID)"
-FIELDS_FILE_PATH=model
 MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
 
 # Path to the libbeat Makefile
@@ -21,11 +20,12 @@ MAGE_IMPORT_PATH=${BEAT_PATH}/vendor/github.com/magefile/mage
 update-beats:
 	rm -rf vendor/github.com/elastic/beats
 	@govendor fetch github.com/elastic/beats/...@$(BEATS_VERSION)
-	@govendor fetch github.com/elastic/beats/libbeat/generator/fields@$(BEATS_VERSION) github.com/elastic/beats/libbeat/kibana@$(BEATS_VERSION) github.com/elastic/beats/libbeat/outputs/transport/transptest@$(BEATS_VERSION)
+	@govendor fetch github.com/elastic/beats/libbeat/generator/fields@$(BEATS_VERSION) github.com/elastic/beats/libbeat/kibana@$(BEATS_VERSION) github.com/elastic/beats/libbeat/outputs/transport/transptest@$(BEATS_VERSION) github.com/elastic/beats/libbeat/scripts/cmd/global_fields@$(BEATS_VERSION)
 	@BEATS_VERSION=$(BEATS_VERSION) script/update_beats.sh
 	@$(MAKE) update
 	@echo --- Use this commit message: Update beats framework to `cat vendor/vendor.json | python -c 'import sys, json; print([p["revision"] for p in json.load(sys.stdin)["package"] if p["path"] == "github.com/elastic/beats/libbeat/beat"][0][:7])'`
 
+
 .PHONY: is-beats-updated
 is-beats-updated: python-env
 	@$(PYTHON_ENV)/bin/python ./script/is_beats_updated.py ${BEATS_VERSION}

NOTICE.txt

@@ -292,7 +292,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: a314e34ea823c39f0838be59de7f299831a8a018
+Revision: 9b268ba852ae93e1080c745a1031c72243f64df0
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------
@@ -358,7 +358,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: github.com/elastic/go-ucfg
-Revision: 9c66f5c432b1d25bdb449a1e588d58b5d0cd7268
+Revision: 581f7b1fe9d84f4c18ef0694d6e0eb944a925dae
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/go-ucfg/LICENSE:
 --------------------------------------------------------------------
@@ -1630,11 +1630,11 @@ DEALINGS IN THE SOFTWARE.
 
 --------------------------------------------------------------------
 Dependency: github.com/Shopify/sarama
-Revision: 32b4ad5c9537ed14e471779b76713ff65420db39
+Revision: d1575e4abe04acbbe8ac766320585cdf271dd189
 License type (autodetected): MIT
 ./vendor/github.com/Shopify/sarama/LICENSE:
 --------------------------------------------------------------------
-Copyright (c) 2013 Evan Huus
+Copyright (c) 2013 Shopify
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

_beats/dev-tools/jenkins_ci.ps1

@@ -21,6 +21,14 @@ $env:PATH = "$env:GOPATH\bin;C:\tools\mingw64\bin;$env:PATH"
 # each run starts from a clean slate.
 $env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile"
 
+exec { go install github.com/elastic/beats/vendor/github.com/magefile/mage }
+exec { go get -u github.com/jstemmer/go-junit-report }
+
+echo "Building libbeat fields.yml"
+cd libbeat
+exec { mage fields }
+cd ..
+
 if (Test-Path "$env:beat") {
     cd "$env:beat"
 } else {
@@ -35,20 +43,11 @@ New-Item -ItemType directory -Path build\coverage | Out-Null
 New-Item -ItemType directory -Path build\system-tests | Out-Null
 New-Item -ItemType directory -Path build\system-tests\run | Out-Null
 
-exec { go get -u github.com/jstemmer/go-junit-report }
+echo "Building fields.yml"
+exec { mage fields }
 
 echo "Building $env:beat"
-exec { go build } "Build FAILURE"
-
-# always build the libbeat fields
-cp ..\libbeat\_meta\fields.common.yml ..\libbeat\_meta\fields.generated.yml
-cat ..\libbeat\processors\*\_meta\fields.yml | Out-File -append -encoding UTF8 -filepath ..\libbeat\_meta\fields.generated.yml
-cp ..\libbeat\_meta\fields.generated.yml ..\libbeat\fields.yml
-
-if ($env:beat -eq "metricbeat") {
-    cp .\_meta\fields.common.yml .\_meta\fields.generated.yml
-    python .\scripts\fields_collector.py | out-file -append -encoding UTF8 -filepath .\_meta\fields.generated.yml
-}
+exec { mage build } "Build FAILURE"
 
 echo "Unit testing $env:beat"
 go test -v $(go list ./... | select-string -Pattern "vendor" -NotMatch) 2>&1 | Out-File -encoding UTF8 build/TEST-go-unit.out

_beats/dev-tools/mage/fields.go

@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"path/filepath"
+
+	"github.com/magefile/mage/sh"
+)
+
+// GenerateFieldsYAML generates a fields.yml file for a Beat. This will include
+// the common fields specified by libbeat, the common fields for the Beat,
+// and any additional fields.yml files you specify.
+//
+// fieldsFiles specifies additional directories to search recursively for files
+// named fields.yml. The contents of each fields.yml will be included in the
+// generated file.
+func GenerateFieldsYAML(fieldsFiles ...string) error {
+	const globalFieldsCmdPath = "libbeat/scripts/cmd/global_fields/main.go"
+
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		return err
+	}
+
+	globalFieldsCmd := sh.RunCmd("go", "run",
+		filepath.Join(beatsDir, globalFieldsCmdPath),
+		"-es_beats_path", beatsDir,
+		"-beat_path", CWD(),
+	)
+
+	return globalFieldsCmd(fieldsFiles...)
+}

_beats/dev-tools/packaging/templates/common/magefile.go.tmpl

@@ -70,3 +70,8 @@ func TestPackages() error {
 func Update() error {
 	return sh.Run("make", "update")
 }
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("protos")
+}

_beats/libbeat/Makefile

@@ -1,7 +1,6 @@
 BEAT_NAME=libbeat
 TEST_ENVIRONMENT?=true
 SYSTEM_TESTS=true
-FIELDS_FILE_PATH=processors
 
 include scripts/Makefile
 

_beats/libbeat/magefile.go

@@ -0,0 +1,39 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build mage
+
+package main
+
+import (
+	"github.com/elastic/beats/dev-tools/mage"
+)
+
+// Build builds the Beat binary.
+func Build() error {
+	return mage.Build(mage.DefaultBuildArgs())
+}
+
+// Clean cleans all generated files and build artifacts.
+func Clean() error {
+	return mage.Clean()
+}
+
+// Fields generates a fields.yml for the Beat.
+func Fields() error {
+	return mage.GenerateFieldsYAML("processors")
+}

_beats/libbeat/scripts/Makefile

@@ -20,7 +20,9 @@ ELASTIC_LICENSE_FILE?=../licenses/ELASTIC-LICENSE.txt
 SECCOMP_BINARY?=${BEAT_NAME}
 SECCOMP_BLACKLIST?=${ES_BEATS}/libbeat/common/seccomp/seccomp-profiler-blacklist.txt
 SECCOMP_ALLOWLIST?=${ES_BEATS}/libbeat/common/seccomp/seccomp-profiler-allow.txt
+MAGE_PRESENT := $(shell command -v mage 2> /dev/null)
 MAGE_IMPORT_PATH?=github.com/elastic/beats/vendor/github.com/magefile/mage
+export MAGE_IMPORT_PATH
 
 space:=$() #
 comma:=,
@@ -304,9 +306,8 @@ coverage-report:
 
 
 .PHONY: fields
-fields:
-	@go run ${ES_BEATS}/libbeat/scripts/cmd/global_fields/main.go --es_beats_path $(ES_BEATS) --beat_path $(PWD) $(FIELDS_FILE_PATH)
-
+fields: mage
+	mage -v fields
 
 .PHONY: libbeat_fields
 libbeat_fields:
@@ -442,7 +443,9 @@ seccomp-package:
 
 .PHONY: mage
 mage:
-	@go install ${MAGE_IMPORT_PATH}
+ifndef MAGE_PRESENT
+	go install ${MAGE_IMPORT_PATH}
+endif
 
 .PHONY: release
 release: mage

_beats/libbeat/scripts/cmd/global_fields/main.go

@@ -31,41 +31,46 @@ func main() {
 	beatPath := flag.String("beat_path", ".", "Path to your Beat")
 	flag.Parse()
 
-	beatFieldsPath := flag.Args()
+	beatFieldsPaths := flag.Args()
 	name := filepath.Base(*beatPath)
 
-	err := os.MkdirAll(filepath.Join(*beatPath, "_meta"), 0744)
+	if *beatPath == "" {
+		fmt.Fprintf(os.Stderr, "beat_path cannot be empty")
+		os.Exit(1)
+	}
+
+	err := os.MkdirAll(filepath.Join(*beatPath, "_meta"), 0755)
 	if err != nil {
-		fmt.Printf("Cannot create _meta dir for %s: %v\n", name, err)
+		fmt.Fprintf(os.Stderr, "Cannot create _meta dir for %s: %+v\n", name, err)
 		os.Exit(1)
 	}
 
-	if len(beatFieldsPath) == 0 {
+	if len(beatFieldsPaths) == 0 {
 		fmt.Println("No field files to collect")
 		err = fields.AppendFromLibbeat(*esBeatsPath, *beatPath)
 		if err != nil {
-			fmt.Printf("Cannot generate global fields.yml for %s: %v\n", name, err)
+			fmt.Fprintf(os.Stderr, "Cannot generate global fields.yml for %s: %+v\n", name, err)
 			os.Exit(2)
 		}
 		return
 	}
 
-	if *beatPath == "" {
-		fmt.Println("beat_path cannot be empty")
-		os.Exit(1)
-	}
+	var fieldsFiles []*fields.YmlFile
+	for _, fieldsFilePath := range beatFieldsPaths {
+		pathToModules := filepath.Join(*beatPath, fieldsFilePath)
 
-	pathToModules := filepath.Join(*beatPath, beatFieldsPath[0])
-	fieldFiles, err := fields.CollectModuleFiles(pathToModules)
-	if err != nil {
-		fmt.Printf("Cannot collect fields.yml files: %v\n", err)
-		os.Exit(2)
+		fieldsFile, err := fields.CollectModuleFiles(pathToModules)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Cannot collect fields.yml files: %+v\n", err)
+			os.Exit(2)
+		}
 
+		fieldsFiles = append(fieldsFiles, fieldsFile...)
 	}
 
-	err = fields.Generate(*esBeatsPath, *beatPath, fieldFiles)
+	err = fields.Generate(*esBeatsPath, *beatPath, fieldsFiles)
 	if err != nil {
-		fmt.Printf("Cannot generate global fields.yml file for %s: %v\n", name, err)
+		fmt.Fprintf(os.Stderr, "Cannot generate global fields.yml file for %s: %+v\n", name, err)
 		os.Exit(3)
 	}
 

_beats/libbeat/tests/system/requirements.txt

@@ -28,4 +28,4 @@ texttable==0.9.1
 urllib3==1.22
 websocket-client==0.47.0
 parameterized==0.6.1
-jsondiff
+jsondiff==1.1.2

_beats/testing/environments/args.yml

@@ -7,4 +7,4 @@ services:
       args:
         DOWNLOAD_URL: https://snapshots.elastic.co/downloads
         ELASTIC_VERSION: 7.0.0-alpha1-SNAPSHOT
-        CACHE_BUST: 20180501
+        CACHE_BUST: 20180723

_beats/testing/environments/docker/kafka/Dockerfile

@@ -4,7 +4,7 @@ ENV KAFKA_HOME /kafka
 # The advertised host is kafka. This means it will not work if container is started locally and connected from localhost to it
 ENV KAFKA_ADVERTISED_HOST kafka
 ENV KAFKA_LOGS_DIR="/kafka-logs"
-ENV KAFKA_VERSION 1.0.0
+ENV KAFKA_VERSION 1.1.1
 ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
 ENV TERM=linux