Update beats framework to d37c114

.gitignore

@@ -14,7 +14,7 @@
 /apm-server.dev.yml
 /_meta/fields.generated.yml
 /_meta/kibana/*/index-pattern
-
+/_meta/kibana.generated/*
 /fields.yml
 /apm-server.template-es.json
 

NOTICE.txt

@@ -211,7 +211,7 @@ SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: 245b3e1b2ff12449007d6aa01dceee050d40ffa5
+Revision: c941a3d38792137bfc53a3a121f40cebf20b5512
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------

_beats/libbeat/scripts/Makefile

@@ -149,8 +149,7 @@ clean:: ## @build  Cleans up all files generated by the build steps
 	@rm -f docker-compose.yml.lock
 	@rm -f ${BEAT_NAME} ${BEAT_NAME}.test ${BEAT_NAME}.exe ${BEAT_NAME}.test.exe
 	@rm -f _meta/fields.generated.yml fields.yml
-	@rm -f $(PWD)/_meta/kibana/5/index-pattern/${BEAT_NAME}.json
-	@rm -f $(PWD)/_meta/kibana/6/index-pattern/${BEAT_NAME}.json
+	@rm -fr $(PWD)/_meta/kibana.generated
 	@rm -f ${BEAT_NAME}.template*.json
 
 .PHONY: ci
@@ -343,9 +342,15 @@ endif
 	@mkdir -p docs
 	@${PYTHON_ENV}/bin/python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_TITLE} ${ES_BEATS}
 
-	@# Generate Kibana index pattern
-	@mkdir -p $(PWD)/_meta/kibana/5/index-pattern
-	@mkdir -p $(PWD)/_meta/kibana/6/index-pattern
+	@mkdir -p $(PWD)/_meta/kibana.generated
+	@# Generate Kibana index pattern and copy dashboard files
+	if [ -d $(PWD)/_meta/kibana ]; then \
+		cp -pr $(PWD)/_meta/kibana/* $(PWD)/_meta/kibana.generated ; \
+	fi
+	@# Convert all dashboards to string
+	@python ${ES_BEATS}/libbeat/scripts/unpack_dashboards.py --glob="./_meta/kibana.generated/6/dashboard/*.json"
+	@mkdir -p $(PWD)/_meta/kibana.generated/5/index-pattern
+	@mkdir -p $(PWD)/_meta/kibana.generated/6/index-pattern
 	@go run ${ES_BEATS}/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go -index '${BEAT_INDEX_PREFIX}-*' -beat-name ${BEAT_NAME} -beat-dir $(PWD) -version ${BEAT_VERSION}
 
 .PHONY: docs
@@ -366,7 +371,7 @@ ${ES_BEATS}/dev-tools/cmd/dashboards/export_dashboards:
 
 .PHONY: import-dashboards
 import-dashboards: update ${BEAT_NAME}
-	${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME} setup -E setup.dashboards.directory=${PWD}/_meta/kibana -E setup.kibana.host=${KIBANA_URL} --dashboards
+	${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME} setup -E setup.dashboards.directory=${PWD}/_meta/kibana.generated -E setup.kibana.host=${KIBANA_URL} --dashboards
 
 ### CONTAINER ENVIRONMENT ####
 
@@ -417,9 +422,9 @@ install-home:
 		rsync -av _meta/module.generated/ ${HOME_PREFIX}/module/; \
 		chmod -R go-w ${HOME_PREFIX}/module/; \
 	fi
-	if [ -d _meta/kibana ]; then \
+	if [ -d _meta/kibana.generated ]; then \
 		install -d -m 755 ${HOME_PREFIX}/kibana; \
-		rsync -av _meta/kibana/ ${HOME_PREFIX}/kibana/; \
+		rsync -av _meta/kibana.generated/ ${HOME_PREFIX}/kibana/; \
 	fi
 
 # Prepares for packaging. Builds binaries and creates homedir data
@@ -541,7 +546,7 @@ package-all: package-elastic package-oss
 
 package-dashboards: package-setup
 	mkdir -p ${BUILD_DIR}
-	cp -r _meta/kibana ${BUILD_DIR}/dashboards
+	cp -r _meta/kibana.generated ${BUILD_DIR}/dashboards
 	# build the dashboards package
 	BEAT_NAME=${BEAT_NAME} BUILD_DIR=${BUILD_DIR} SNAPSHOT=$(SNAPSHOT) $(MAKE) -C ${ES_BEATS}/dev-tools/packer package-dashboards ${shell pwd}/build/upload/build_id.txt
 

_beats/libbeat/tests/system/beat/beat.py

@@ -579,3 +579,17 @@ def get_kibana_url(self):
             host=os.getenv("KIBANA_HOST", "localhost"),
             port=os.getenv("KIBANA_PORT", "5601"),
         )
+
+    def assert_fields_are_documented(self, evt):
+        """
+        Assert that all keys present in evt are documented in fields.yml.
+        This reads from the global fields.yml, means `make collect` has to be run before the check.
+        """
+        expected_fields, dict_fields = self.load_fields()
+        flat = self.flatten_object(evt, dict_fields)
+
+        for key in flat.keys():
+            documented = key in expected_fields
+            metaKey = key.startswith('@metadata.')
+            if not(documented or metaKey):
+                raise Exception("Key '{}' found in event is not documented!".format(key))

docs/copied-from-beats/command-reference.asciidoc

@@ -397,10 +397,10 @@ ifeval::["{beatname_lc}"=="filebeat"]
 
 *`--once`*::
 When the `--once` flag is used, {beatname_uc} starts all configured harvesters
-and inputs, and runs each input until the harvesters are closed. If you set the
-`--once` flag, you should also set `close_eof` so the harvester is closed when
-the end of the file is reached. By default harvesters are closed after
-`close_inactive` is reached.
+and prospectors, and runs each prospector until the harvesters are closed. If
+you set the `--once` flag, you should also set `close_eof` so the harvester is
+closed when the end of the file is reached. By default harvesters are closed
+after `close_inactive` is reached.
 
 endif::[]
 
@@ -481,9 +481,6 @@ ifeval::["{beatname_lc}"=="filebeat"]
 Specifies a comma-separated list of modules. Use this flag to avoid errors when
 there are no modules defined in the +{beatname_lc}.yml+ file.
 
-*`--pipelines`*::
-Sets up ingest pipelines for configured filesets.
-
 endif::[]
 
 *`--template`*::

docs/copied-from-beats/dashboards.asciidoc

@@ -18,7 +18,7 @@ command (as described here) or
 +{beatname_lc}.yml+ config file.
 
 This requires a Kibana endpoint configuration. If you didn't already configure
-a Kibana endpoint, see <<{beatname_lc}-configuration,configure {beatname_uc}>>. 
+a Kibana endpoint, see <<{beatname_lc}-configuration,configured {beatname_uc}>> 
 
 Make sure Kibana is running before you perform this step. If you are accessing a
 secured Kibana instance, make sure you've configured credentials as described in
@@ -67,12 +67,13 @@ endif::[]
 endif::allplatforms[]
 
 Open a PowerShell prompt as an Administrator (right-click the PowerShell icon
-and select *Run As Administrator*).
+and select *Run As Administrator*). If you are running Windows XP, you may need
+to download and install PowerShell.
 
 From the PowerShell prompt, change to the directory where you installed {beatname_uc},
 and run:
 
-["source","sh",subs="attributes"]
+["source","sh",subs="attributes,callouts"]
 ----------------------------------------------------------------------
-PS > .{backslash}{beatname_lc}.exe setup --dashboards
+PS > {beatname_lc} setup --dashboards
 ----------------------------------------------------------------------

docs/copied-from-beats/keystore.asciidoc

@@ -10,11 +10,7 @@
 //////////////////////////////////////////////////////////////////////////
 
 [[keystore]]
-=== Secrets keystore for secure settings
-
-++++
-<titleabbrev>Secrets keystore</titleabbrev>
-++++
+=== Secrets keystore
 
 When you configure {beatname_uc}, you might need to specify sensitive settings,
 such as passwords. Rather than relying on file system permissions to protect

docs/copied-from-beats/outputconfig.asciidoc

@@ -81,9 +81,7 @@ output.elasticsearch:
 
 ==== Compatibility
 
-This output works with all compatible versions of Elasticsearch. See the
-https://www.elastic.co/support/matrix#matrix_compatibility[Elastic Support
-Matrix].
+This output works with all compatible versions of Elasticsearch. See "Supported Beats Versions" in the https://www.elastic.co/support/matrix#show_compatibility[Elastic Support Matrix].
 
 ==== Configuration options
 
@@ -258,7 +256,7 @@ Example elasticsearch output with `pipelines`:
 
 ["source","yaml"]
 ------------------------------------------------------------------------------
-filebeat.inputs:
+filebeat.prospectors:
 - type: log
   paths: ["/var/log/app/normal/*.log"]
   fields:
@@ -403,9 +401,7 @@ will be similar to events directly indexed by Beats into Elasticsearch.
 
 ==== Compatibility
 
-This output works with all compatible versions of Logstash. See the
-https://www.elastic.co/support/matrix#matrix_compatibility[Elastic Support
-Matrix].
+This output works with all compatible versions of Logstash. See "Supported Beats Versions" in the https://www.elastic.co/support/matrix#show_compatibility[Elastic Support Matrix].
 
 ==== Configuration options
 

docs/copied-from-beats/security/basic-auth.asciidoc

@@ -43,7 +43,7 @@ following request creates a ++{beat_default_index_prefix}_internal++ user that h
 ---------------------------------------------------------------
 POST /_xpack/security/user/{beat_default_index_prefix}_internal
 {
-  "password" : "test-password",
+  "password" : "x-pack-test-password",
   "roles" : [ "{beat_default_index_prefix}_writer"],
   "full_name" : "Internal {beatname_uc} User"
 }
@@ -76,7 +76,7 @@ output.elasticsearch:
     hosts: ["localhost:9200"]
     index: "{beat_default_index_prefix}"
     username: "{beat_default_index_prefix}_internal"
-    password: "test-password"
+    password: "x-pack-test-password"
 --------------------------------------------------
 
 .. To use PKI authentication, configure the `certificate` and

docs/copied-from-beats/security/user-access.asciidoc

@@ -39,7 +39,7 @@ example, the following request grants ++{beat_default_index_prefix}_user++ the
 ---------------------------------------------------------------
 POST /_xpack/security/user/{beat_default_index_prefix}_user
 {
-  "password" : "test-password",
+  "password" : "x-pack-test-password",
   "roles" : [ "{beat_default_index_prefix}_reader"],
   "full_name" : "{beatname_uc} User"
 }

docs/copied-from-beats/shared-configuring.asciidoc

@@ -1,24 +1,24 @@
 //Added conditional coding to support Beats that don't offer all of these install options
 
-To configure {beatname_uc}, you edit the configuration file. 
-
 ifeval::["{beatname_lc}"!="auditbeat"]
-For rpm and deb, you'll find the configuration file at +/etc/{beatname_lc}/{beatname_lc}.yml+. 
-For mac and win, look in the archive that you just extracted.
-Under Docker, it's located at +/usr/share/{beatname_lc}/{beatname_lc}.yml+.
+
+To configure {beatname_uc}, you edit the configuration file. For rpm and deb,
+you'll find the configuration file at +/etc/{beatname_lc}/{beatname_lc}.yml+. Under
+Docker, it's located at +/usr/share/{beatname_lc}/{beatname_lc}.yml+. For mac and win,
+look in the archive that you just extracted.
+
 endif::[]
 
 ifeval::["{beatname_lc}"=="auditbeat"]
-For rpm and deb, you'll find the configuration file at +/etc/{beatname_lc}/{beatname_lc}.yml+. 
-For mac and win, look in the archive that you just extracted.
-endif::[]
 
+To configure {beatname_uc}, you edit the configuration file. For rpm and deb,
+you'll find the configuration file at +/etc/{beatname_lc}/{beatname_lc}.yml+.
+For mac and win, look in the archive that you just extracted. There’s also a
+full example configuration file called +{beatname_lc}.reference.yml+ that shows
+all non-deprecated options.
 
-ifeval::["{beatname_lc}"!="apm-server"]
-There’s also a full example configuration file called +{beatname_lc}.reference.yml+ 
-that shows all non-deprecated options.
 endif::[]
 
-TIP: See the
+See the
 {libbeat}/config-file-format.html[Config File Format] section of the
 _Beats Platform Reference_ for more about the structure of the config file.

docs/copied-from-beats/shared-ssl-config.asciidoc

@@ -168,17 +168,3 @@ are `never`, `once`, and `freely`. The default value is never.
 * `never` - Disables renegotiation.
 * `once` - Allows a remote server to request renegotiation once per connection.
 * `freely` - Allows a remote server to repeatedly request renegotiation.
-
-ifeval::["{beatname_lc}" == "filebeat"]
-[float]
-==== `client_authentication`
-
-This configures what types of client authentication are supported. The valid options
-are `none`, `optional`, and `required`. The default value is required.
-
-NOTE: This option is only valid with the TCP or the Syslog input.
-
-* `none` - Disables client authentification.
-* `optional` - When a client certificate is given, the server will verify it.
-* `required` - Will require clients to provide a valid certificate.
-endif::[]

docs/copied-from-beats/shared-template-load.asciidoc

@@ -176,14 +176,15 @@ endif::[]
 endif::allplatforms[]
 
 Open a PowerShell prompt as an Administrator (right-click the PowerShell icon
-and select *Run As Administrator*).
+and select *Run As Administrator*). If you are running Windows XP, you may need
+to download and install PowerShell.
 
 From the PowerShell prompt, change to the directory where you installed {beatname_uc},
 and run:
 
-["source","sh",subs="attributes"]
+["source","sh",subs="attributes,callouts"]
 ----------------------------------------------------------------------
-PS > .{backslash}{beatname_lc}.exe setup --template{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]'
+PS > .{backslash}{beatname_lc} setup --template{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]'
 ----------------------------------------------------------------------
 
 
@@ -238,12 +239,12 @@ ifdef::allplatforms[]
 ./{beatname_lc} export template > {beatname_lc}.template.json
 ----
 +
-*win:*
+*win*:
 +
 endif::allplatforms[]
 ["source","sh",subs="attributes"]
 ----
-PS > .{backslash}{beatname_lc}.exe export template --es.version {stack-version} | Out-File -Encoding UTF8 {beatname_lc}.template.json
+PS> .{backslash}{beatname_lc}.exe export template --es.version {stack-version} | Out-File -Encoding UTF8 {beatname_lc}.template.json
 ----
 
 . Install the template:
@@ -255,7 +256,7 @@ PS > .{backslash}{beatname_lc}.exe export template --es.version {stack-version}
 curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_template/{beatname_lc}-{stack-version} -d@{beatname_lc}.template.json
 ----
 +
-*win:*
+*win*:
 +
 ["source","sh",subs="attributes"]
 ----

docs/copied-from-beats/template-config.asciidoc

@@ -87,16 +87,4 @@ ifeval::["{beatname_lc}"!="apm-server"]
 
 NOTE: With append_fields only new fields can be added an no existing one overwritten or changed. This is especially useful if data is collected through the http/json metricset where the data structure is not known in advance. Changing the config of append_fields means the template has to be overwritten and only applies to new indices. If there are 2 Beats with different append_fields configs the last one writing the template will win. Any changes will also have an affect on the Kibana Index pattern.
 
-*`setup.template.json.enabled`*:: Set to true to load a json based template file. Specify the path to your Elasticsearch
-index template file and set the name of the template. experimental[]
-
-["source","yaml",subs="attributes"]
-----------------------------------------------------------------------
-setup.template.json.enabled: true
-setup.template.json.path: "template.json"
-setup.template.json.name: "template-name
-----------------------------------------------------------------------
-
-NOTE: If the JSON template is used, the fields.yml is skipped for the template generation.
-
 endif::[]