elastic · christiangalsterer · Apr 21, 2017 · Apr 21, 2017 · Apr 21, 2017 · Apr 23, 2017
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -30,6 +30,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Add Icinga module. {pull}3904[3904]
 
 *Heartbeat*
+- Event format and field naming changes in Heartbeat and sample Dashboard. {pull}4091[4091]
 
 *Metricbeat*
 - Linux cgroup metrics are now enabled by default for the system process
@@ -54,6 +55,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Fix potential elasticsearch output URL parsing error if protocol scheme is missing. {pull}3671[3671]
 - Improve error message when downloading the dashboards fails. {pull}3805[3805]
 - Fix panic when testing regex-AST to match against date patterns. {issue}3889[3889]
+- Fix panic due to race condition in kafka output. {pull}4098[4098]
 
 *Filebeat*
 - Always use absolute path for event and registry. {pull}3328[3328]
@@ -108,6 +110,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Add kubernetes processor {pull}3888[3888]
 - Add support for include_labels and include_annotations in kubernetes processor {pull}4043[4043]
 - Support new `index_patterns` field when loading templates for Elasticsearch >= 6.0 {pull}4056[4056]
+- Adding goimports support to make check and fmt {pull}4114[4114]
 
 *Filebeat*
 
@@ -120,6 +123,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Add base for supporting prospector level processors {pull}3853[3853]
 - Add auditd module for reading audit logs on Linux. {pull}3750[3750] {pull}3941[3941]
 - Add filebeat.config.path as replacement for config_dir. {pull}4051[4051]
+- Add a `recursive_glob.enabled` setting to expand "**" in patterns. {{pull}}3980[3980]
 
 *Heartbeat*
 
@@ -143,6 +147,9 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Add new MetricSet interfaces for developers (`Closer`, `ReportingFetcher`, and `PushMetricSet`). {pull}3908[3908]
 - Add kubelet module {pull}3916[3916]
 - Add dropwizard module {pull}4022[4022]
+- Adding query APIs for metricsets and modules from metricbeat registry {pull}4102[4102]
+- Fixing nil pointer on prometheus collector when http response is nil {pull}4119[4119]
+- Add http module with json metricset. {pull}4092[4092]
 
 *Packetbeat*
 - Add `fields` and `fields_under_root` to packetbeat protocols configurations. {pull}3518[3518]
@@ -289,6 +296,7 @@ https://github.com/elastic/beats/compare/v5.0.2...v5.1.1[View commits]
 - Add add_cloud_metadata processor for collecting cloud provider metadata. {pull}2728[2728]
 - Added decode_json_fields processor for decoding fields containing JSON strings. {pull}2605[2605]
 - Add Tencent Cloud provider for add_cloud_metadata processor. {pull}4023[4023]
+- Add Alibaba Cloud provider for add_cloud_metadata processor. {pull}4111[4111]
 
 *Metricbeat*
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -51,11 +51,11 @@ Beats](https://github.com/elastic/beats/blob/master/libbeat/docs/communitybeats.
 
 The Beats are Go programs, so install the latest version of
 [golang](http://golang.org/) if you don't have it already. The current Go version
-used for development is Golang 1.7.4.
+used for development is Golang 1.8.1.
 
 The location where you clone is important. Please clone under the source
 directory of your `GOPATH`. If you don't have `GOPATH` already set, you can
-simply set it to your home directory (`export GOPATH=$HOME`).
+simply set it to the `go` directory in your home (`export GOPATH=$HOME/go`).
 
     $ mkdir -p ${GOPATH}/src/github.com/elastic
     $ cd ${GOPATH}/src/github.com/elastic
@@ -116,4 +116,3 @@ To manage the `vendor/` folder we use
 the govendor documentation on how to add or update vendored dependencies.
 
 In most cases `govendor fetch your/dependency@version +out` will get the job done.
-
diff --git a/dev-tools/packer/docker/xgo-image/base/Dockerfile b/dev-tools/packer/docker/xgo-image/base/Dockerfile
@@ -29,14 +29,13 @@ RUN \
     binutils-multiarch rsync                                                    \
     --no-install-recommends
 
-# Configure the container for OSX cross compilation
 # Configure the container for OSX cross compilation
 ENV OSX_SDK     MacOSX10.11.sdk
 ENV OSX_NDK_X86 /usr/local/osx-ndk-x86
 
 RUN \
-  OSX_SDK_PATH=https://s3.dockerproject.org/darwin/v2/$OSX_SDK.tar.xz && \
-  $FETCH $OSX_SDK_PATH dd228a335194e3392f1904ce49aff1b1da26ca62       && \
+  OSX_SDK_PATH=https://github.com/phracker/MacOSX-SDKs/releases/download/MacOSX10.11.sdk/MacOSX10.11.sdk.tar.xz && \
+  $FETCH $OSX_SDK_PATH f3430e3d923644e66c0c13f7a48754e7b6aa2e3f       && \
   \
   git clone https://github.com/tpoechtrager/osxcross.git && \
   mv `basename $OSX_SDK_PATH` /osxcross/tarballs/        && \

diff --git a/docs/devguide/index.asciidoc b/docs/devguide/index.asciidoc
@@ -21,6 +21,8 @@ include::../../libbeat/docs/version.asciidoc[]
 
 include::./contributing.asciidoc[]
 
+include::../../libbeat/docs/communitybeats.asciidoc[]
+
 include::./newbeat.asciidoc[]
 
 include::./event-conventions.asciidoc[]

diff --git a/filebeat/_meta/common.full.p2.yml b/filebeat/_meta/common.full.p2.yml
@@ -16,6 +16,9 @@ filebeat.prospectors:
 #------------------------------ Log prospector --------------------------------
 - input_type: log
 
+  # Change to true to enable this prospector configuration.
+  enabled: false
+
   # Paths that should be crawled and fetched. Glob based paths.
   # To fetch all ".log" files from a specific level of subdirectories
   # /var/log/*/*.log can be used.
@@ -82,6 +85,11 @@ filebeat.prospectors:
   # This is especially useful for multiline log messages which can get large.
   #max_bytes: 10485760
 
+  ### Recursive glob configuration
+
+  # Expand "**" patterns into regular glob patterns.
+  #recursive_glob.enabled: true
+
   ### JSON configuration
 
   # Decode JSON options. Enable this if your logs are structured in JSON.

diff --git a/filebeat/_meta/common.p2.yml b/filebeat/_meta/common.p2.yml
@@ -11,6 +11,9 @@ filebeat.prospectors:
 
 - input_type: log
 
+  # Change to true to enable this prospector configuration.
+  enabled: false
+
   # Paths that should be crawled and fetched. Glob based paths.
   paths:
     - /var/log/*.log

diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go
@@ -11,15 +11,16 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/outputs/elasticsearch"
 
+	"github.com/elastic/beats/filebeat/channel"
 	cfg "github.com/elastic/beats/filebeat/config"
 	"github.com/elastic/beats/filebeat/crawler"
 	"github.com/elastic/beats/filebeat/fileset"
 	"github.com/elastic/beats/filebeat/publisher"
 	"github.com/elastic/beats/filebeat/registrar"
 	"github.com/elastic/beats/filebeat/spooler"
 
-	//Add filebeat level processors
-	_ "github.com/elastic/beats/filebeat/processor/annotate/kubernetes"
+	// Add filebeat level processors
+	_ "github.com/elastic/beats/filebeat/processors/kubernetes"
 )
 
 var (
@@ -57,8 +58,16 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 	// Add prospectors created by the modules
 	config.Prospectors = append(config.Prospectors, moduleProspectors...)
 
-	if !config.ConfigProspector.Enabled() && len(config.Prospectors) == 0 {
-		return nil, errors.New("No prospectors defined. What files do you want me to watch?")
+	haveEnabledProspectors := false
+	for _, prospector := range config.Prospectors {
+		if prospector.Enabled() {
+			haveEnabledProspectors = true
+			break
+		}
+	}
+
+	if !config.ConfigProspector.Enabled() && !haveEnabledProspectors {
+		return nil, errors.New("No modules or prospectors enabled and configuration reloading disabled. What files do you want me to watch?")
 	}
 
 	if *once && config.ConfigProspector.Enabled() {
@@ -140,7 +149,7 @@ func (fb *Filebeat) Run(b *beat.Beat) error {
 		return err
 	}
 
-	crawler, err := crawler.New(newSpoolerOutlet(fb.done, spooler, wgEvents), config.Prospectors, fb.done, *once)
+	crawler, err := crawler.New(channel.NewOutlet(fb.done, spooler.Channel, wgEvents), config.Prospectors, fb.done, *once)
 	if err != nil {
 		logp.Err("Could not init crawler: %v", err)
 		return err

diff --git a/filebeat/channel/outlet.go b/filebeat/channel/outlet.go
@@ -5,6 +5,7 @@ import (
 	"sync/atomic"
 
 	"github.com/elastic/beats/filebeat/input"
+	"github.com/elastic/beats/filebeat/prospector"
 )
 
 // Outlet struct is used to be passed to an object which needs an outlet
@@ -19,13 +20,13 @@ type Outlet struct {
 	wg      *sync.WaitGroup // Use for counting active events
 	done    <-chan struct{}
 	signal  <-chan struct{}
-	channel chan *input.Event
+	channel chan *input.Data
 	isOpen  int32 // atomic indicator
 }
 
 func NewOutlet(
 	done <-chan struct{},
-	c chan *input.Event,
+	c chan *input.Data,
 	wg *sync.WaitGroup,
 ) *Outlet {
 	return &Outlet{
@@ -42,7 +43,7 @@ func (o *Outlet) SetSignal(signal <-chan struct{}) {
 	o.signal = signal
 }
 
-func (o *Outlet) OnEvent(event *input.Event) bool {
+func (o *Outlet) OnEvent(event *input.Data) bool {
 	open := atomic.LoadInt32(&o.isOpen) == 1
 	if !open {
 		return false
@@ -67,7 +68,7 @@ func (o *Outlet) OnEvent(event *input.Event) bool {
 // OnEventSignal can be stopped by the signal that is set with SetSignal
 // This does not close the outlet. Only OnEvent does close the outlet.
 // If OnEventSignal is used, it must be ensured that only one producer is used.
-func (o *Outlet) OnEventSignal(event *input.Event) bool {
+func (o *Outlet) OnEventSignal(event *input.Data) bool {
 	open := atomic.LoadInt32(&o.isOpen) == 1
 	if !open {
 		return false
@@ -88,3 +89,7 @@ func (o *Outlet) OnEventSignal(event *input.Event) bool {
 		return true
 	}
 }
+
+func (o *Outlet) Copy() prospector.Outlet {
+	return NewOutlet(o.done, o.channel, o.wg)
+}
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -17,6 +17,7 @@ grouped in the following categories:
 * <<exported-fields-beat>>
 * <<exported-fields-cloud>>
 * <<exported-fields-icinga>>
+* <<exported-fields-kubernetes>>
 * <<exported-fields-log>>
 * <<exported-fields-mysql>>
 * <<exported-fields-nginx>>
@@ -682,6 +683,53 @@ type: text
 The logged message.
 
 
+[[exported-fields-kubernetes]]
+== Kubernetes info Fields
+
+Kubernetes metadata added by the kubernetes processor
+
+
+
+[float]
+=== kubernetes.pod.name
+
+type: keyword
+
+Kubernetes pod name
+
+
+[float]
+=== kubernetes.namespace
+
+type: keyword
+
+Kubernetes namespace
+
+
+[float]
+=== kubernetes.labels
+
+type: object
+
+Kubernetes labels map
+
+
+[float]
+=== kubernetes.annotations
+
+type: object
+
+Kubernetes annotations map
+
+
+[float]
+=== kubernetes.container.name
+
+type: keyword
+
+Kubernetes container name
+
+
 [[exported-fields-log]]
 == Log File Content Fields
 

diff --git a/filebeat/docs/multiline.asciidoc b/filebeat/docs/multiline.asciidoc
@@ -1,8 +1,15 @@
 [[multiline-examples]]
 == Managing Multiline Messages
 
-You can specify `multiline` settings in the +{beatname_lc}.yml+ file to control how Filebeat deals with messages that
-span multiple lines. At a minimum, you need to configure:
+The files harvested by {beatname_uc} may contain messages that span multiple lines of text. In order to correctly handle
+these multiline events, you need to configure `multiline` settings in the +{beatname_lc}.yml+ file to specify which
+lines are part of a single event. 
+
+IMPORTANT: If you are sending multiline events to Logstash, use the options described here to handle multiline events
+before sending the event data to Logstash. Trying to implement multiline event handling in Logstash (for example, by
+using the Logstash multiline codec) may result in the mixing of streams and corrupted data. 
+
+At a minimum, you need to configure these `multiline` options:
 
 * the `pattern` option, which specifies a regular expression. Depending on how you configure other multiline options, 
 lines that match the specified regular expression are considered either continuations of a previous line or the start of a new multiline event. You can set the `negate` option to negate the pattern.

diff --git a/filebeat/docs/reference/configuration/filebeat-options.asciidoc b/filebeat/docs/reference/configuration/filebeat-options.asciidoc
@@ -41,12 +41,22 @@ supported by https://golang.org/pkg/path/filepath/#Glob[Golang Glob] are also
 supported here. For example, to fetch all files from a predefined level of
 subdirectories, the following pattern can be used: `/var/log/*/*.log`. This
 fetches all `.log` files from the subfolders of `/var/log`. It does not
-fetch log files from the `/var/log` folder itself. Currently it is not possible
-to recursively fetch all files in all subdirectories of a directory.
+fetch log files from the `/var/log` folder itself.
+It is possible to recursively fetch all files in all subdirectories of a directory
+using the optional <<recursive_glob>> settings.
 
 Filebeat starts a harvester for each file that it finds under the specified
 paths. You can specify one path per line. Each line begins with a dash (-).
 
+[[recursive_glob]]
+===== recursive_glob
+
+*`enabled`*:: Enable expanding `**` into recursive glob patterns. With this feature enabled,
+the rightmost `**` in each path is expanded into a fixed
+number of glob patterns. For example: `/foo/**` expands to `/foo`, `/foo/*`,
+`/foo/*/*`, and so on. The feature is disabled by default, and if enabled it expands a single `**`
+into a 8-level deep `*` pattern.
+
 ===== encoding
 
 The file encoding to use for reading files that contain international characters.
@@ -346,6 +356,10 @@ occur.
 [[multiline]]
 ===== multiline
 
+IMPORTANT: If you are sending multiline events to Logstash, use the options described here to handle multiline events
+before sending the event data to Logstash. Trying to implement multiline event handling in Logstash (for example, by
+using the Logstash multiline codec) may result in the mixing of streams and corrupted data.
+
 Options that control how Filebeat deals with log messages that span multiple lines. Multiline messages are common in files that contain Java stack traces.
 
 The following example shows how to configure Filebeat to handle a multiline message where the first line of the message begins with a bracket (`[`).
@@ -587,4 +601,3 @@ include::../../../../libbeat/docs/dashboardsconfig.asciidoc[]
 include::../../../../libbeat/docs/loggingconfig.asciidoc[]
 
 include::../../../../libbeat/docs/processors-config.asciidoc[]
-