Merge remote-tracking branch 'upstream/master' into zeek-kibana-dashb…

…oard-fix

CHANGELOG.asciidoc

@@ -273,7 +273,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...v7.0.0-beta1[Check the
 - Added support for ingesting structured Elasticsearch server logs {pull}10428[10428]
 - Populate more ECS fields in the Suricata module. {pull}10006[10006]
 - Add module zeek. {issue}9931[9931] {pull}10034[10034]
-- Add support for CRI-O based logs autodiscover {pull}10687[10687]
 
 *Heartbeat*
 

CHANGELOG.next.asciidoc

@@ -46,6 +46,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Set `ecs: true` in user_agent processors when loading pipelines with Filebeat 7.0.x into Elasticsearch 6.7.x. {issue}10655[10655] {pull}10875[10875]
+
 *Heartbeat*
 
 - Remove monitor generator script that was rarely used. {pull}9648[9648]
@@ -90,6 +92,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update rabbitmq.* fields to map to ECS. {pull}10563[10563]
 - Update haproxy.* fields to map to ECS. {pull}10558[10558] {pull}10568[10568]
 - Collect all EC2 meta data from all instances in all states. {pull}10628[10628]
+- Migrate docker module to ECS. {pull}10927[10927]
 
 *Packetbeat*
 
@@ -169,6 +172,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed data types for various hosts fields in `mongodb/replstatus` metricset {pull}10307[10307]
 - Added function to close sql database connection. {pull}10355[10355]
 - Fix issue with `elasticsearch/node_stats` metricset (x-pack) not indexing `source_node` field. {pull}10639[10639]
+- Migrate docker autodiscover to ECS. {issue}10757[10757] {pull}10862[10862]
 
 *Packetbeat*
 
@@ -206,6 +210,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Calls to Elasticsearch X-Pack APIs made by Beats won't cause deprecation logs in Elasticsearch logs. {9656}9656[9656]
 - Add `network` condition to processors for matching IP addresses against CIDRs. {pull}10743[10743]
 - Add if/then/else support to processors. {pull}10744[10744]
+- Add `community_id` processor for computing network flow hashes. {pull}10745[10745]
+
 
 *Auditbeat*
 
@@ -244,6 +250,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Populate more ECS fields in the Suricata module. {pull}10006[10006]
 - Add ISO8601 timestamp support in syslog metricset. {issue}8716[8716] {pull}10736[10736]
 - Add more info to message logged when a duplicated symlink file is found {pull}10845[10845]
+- Add option to configure docker input with paths {pull}10687[10687]
+- Add Netflow module to enrich flow events with geoip data. {pull}10877[10877]
+- Set `event.category: network_traffic` for Suricata. {pull}10882[10882]
+- Add configuration knob for auto-discover hints to control whether log harvesting is enabled for the pod/container. {issue}10811[10811] {pull}10911[10911]
+- Change Suricata module pipeline to handle `destination.domain` being set if a reverse DNS processor is used. {issue}10510[10510]
 
 *Heartbeat*
 
@@ -298,6 +309,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add overview dashboard to Zookeeper Metricbeat module {pull}10379[10379]
 - Add Consul Metricbeat module with Agent Metricset {pull}8631[8631]
 - Add filters and pie chart for AWS EC2 dashboard. {pull}10596[10596]
+- Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053]
 
 *Packetbeat*
 

filebeat/autodiscover/builder/hints/logs_test.go

@@ -47,6 +47,18 @@ func TestGenerateHints(t *testing.T) {
 			len:    0,
 			result: common.MapStr{},
 		},
+		{
+			msg: "Hints with logs.disable should return nothing",
+			event: bus.Event{
+				"hints": common.MapStr{
+					"logs": common.MapStr{
+						"disable": "true",
+					},
+				},
+			},
+			len:    0,
+			result: common.MapStr{},
+		},
 		{
 			msg: "Empty event hints should return default config",
 			event: bus.Event{

filebeat/docs/autodiscover-hints.asciidoc

@@ -94,6 +94,25 @@ filebeat.autodiscover:
       hints.enabled: true
 -------------------------------------------------------------------------------------
 
+Autodiscover provides a way to control whether log harvesting is by default disabled for the pods/containers when auto-discovery is in use. To enable it, just set `default.disable` to true:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+filebeat.autodiscover:
+  providers:
+    - type: kubernetes
+      hints.enabled: true
+      default.disable: true
+-------------------------------------------------------------------------------------
+
+Then, for the pods/containers that log harvesting should be enabled, you can annotate with hint:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+annotations:
+  co.elastic.logs/disable: false
+-------------------------------------------------------------------------------------
+
 You can annotate Kubernetes Pods with useful info to spin up {beatname_uc} inputs or modules:
 
 ["source","yaml",subs="attributes"]
@@ -137,6 +156,26 @@ filebeat.autodiscover:
       hints.enabled: true
 -------------------------------------------------------------------------------------
 
+Autodiscover provides a way to control whether log harvesting is by default disabled for the 
+containers when auto-discovery is in use. To enable it, just set `default.disable` to true:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+      default.disable: true
+-------------------------------------------------------------------------------------
+
+Then, for the containers that log harvesting should be enabled, you can label Docker containers with:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+annotations:
+  co.elastic.logs/disable: false
+-------------------------------------------------------------------------------------
+
 You can label Docker containers with useful info to spin up {beatname_uc} inputs, for example:
 
 ["source","yaml",subs="attributes"]

filebeat/docs/fields.asciidoc

@@ -33,6 +33,7 @@ grouped in the following categories:
 * <<exported-fields-mysql>>
 * <<exported-fields-nats>>
 * <<exported-fields-netflow>>
+* <<exported-fields-netflow-module>>
 * <<exported-fields-nginx>>
 * <<exported-fields-osquery>>
 * <<exported-fields-postgresql>>
@@ -10621,6 +10622,12 @@ type: short
 
 --
 
+[[exported-fields-netflow-module]]
+== NetFlow fields
+
+Module for receiving NetFlow and IPFIX flow records over UDP. The module does not add fields beyond what the netflow input provides.
+
+
 [[exported-fields-nginx]]
 == Nginx fields
 

filebeat/docs/inputs/input-docker.asciidoc

@@ -42,6 +42,24 @@ The list of Docker container IDs to read logs from. Specify
 The base path where Docker logs are located. The default
 is `/var/lib/docker/containers`.
 
+===== `containers.paths`
+
+The list of paths to read logs from. This can be used as an alternative to
+`containers.ids` for other runtimes that use the same logging format as docker
+but place their logs in different paths. For example when using CRI-O runtime in
+Kubernetes the following configuration can be used:
+
+["source","yaml",subs="attributes"]
+----
+{beatname_lc}.inputs:
+- type: docker
+  containers.paths:
+    - /var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log
+----
+
+When `containers.paths` is used, `containers.path` is ignored.
+
+
 ===== `containers.stream`
 
 Reads from the specified streams only: `all`, `stdout` or `stderr`. The default

filebeat/docs/modules-getting-started.asciidoc

@@ -100,7 +100,7 @@ load the ingest pipelines manually. To do this, run the `setup` command with
 the `--pipelines` option specified. If you used the
 <<modules-command,`modules`>> command to enable modules in the `modules.d`
 directory, also specify the `--modules` flag. For example, the following command
-loads the ingest pipelines used by all metricsets enabled in the system, nginx,
+loads the ingest pipelines used by all filesets enabled in the system, nginx,
 and mysql modules:
 
 // override modulename attribute so it works with the --modules option
@@ -134,5 +134,9 @@ and mysql modules:
 PS > .{backslash}{beatname_lc}.exe setup --pipelines --modules {modulename}
 ----
 
+TIP: If you're loading ingest pipelines manually because you want to send events
+to {ls}, also see
+{logstash-ref}/filebeat-modules.html[Working with {beatname_uc} modules].
+
 :has_module_steps!:
 :modulename!:

filebeat/docs/modules/netflow.asciidoc

@@ -0,0 +1,80 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[filebeat-module-netflow]]
+[role="xpack"]
+
+:modulename: netflow
+:has-dashboards: false
+
+== NetFlow module
+
+This is a module for receiving NetFlow and IPFIX flow records over UDP. This
+input supports NetFlow versions 1, 5, 6, 7, 8 and 9, as well as IPFIX. For
+NetFlow versions older than 9, fields are mapped automatically to NetFlow v9.
+
+This module wraps the <<filebeat-input-netflow,netflow input>> to enrich the
+flow records with geolocation information about the IP endpoints by using
+Elasticsearch Ingest Node.
+
+[float]
+=== Compatibility
+
+This module requires the {elasticsearch-plugins}/ingest-geoip.html[ingest-geoip]
+Elasticsearch plugins.
+
+include::../include/running-modules.asciidoc[]
+
+include::../include/configuring-intro.asciidoc[]
+
+:fileset_ex: log
+
+include::../include/config-option-intro.asciidoc[]
+
+[float]
+==== `log` fileset settings
+
+The fileset is by default configured to listen for UDP traffic on
+`localhost:2055`. For most uses cases you will want to set the `netflow_host`
+variable to allow the input bind to all interfaces so that it can receive
+traffic from network devices.
+
+["source","yaml",subs="attributes"]
+-----
+- module: netflow
+  log:
+    enabled: true
+    var:
+      netflow_host: 0.0.0.0
+      netflow_port: 2055
+-----
+
+`var.netflow_host`:: Address to find to. Defaults to `localhost`.
+
+`var.netflow_port`:: Port to listen on. Defaults to `2055`.
+
+`var.max_message_size`:: The maximum size of the message received over UDP.
+The default is `10KiB`.
+
+`var.expiration_timeout`:: The time before an idle session or unused template is
+expired. Only applicable to v9 and IPFIX protocols. A value of zero disables
+expiration.
+
+`var.queue_size`:: The maximum number of packets that can be queued for
+processing. Use this setting to avoid packet-loss when dealing with occasional
+bursts of traffic.
+
+:has-dashboards!:
+
+:fileset_ex!:
+
+:modulename!:
+
+
+[float]
+=== Fields
+
+For a description of each field in the module, see the
+<<exported-fields-netflow,exported fields>> section.
+

filebeat/docs/modules_list.asciidoc

@@ -16,6 +16,7 @@ This file is generated! See scripts/docs_collector.py
   * <<filebeat-module-mongodb>>
   * <<filebeat-module-mysql>>
   * <<filebeat-module-nats>>
+  * <<filebeat-module-netflow>>
   * <<filebeat-module-nginx>>
   * <<filebeat-module-osquery>>
   * <<filebeat-module-postgresql>>
@@ -43,6 +44,7 @@ include::modules/logstash.asciidoc[]
 include::modules/mongodb.asciidoc[]
 include::modules/mysql.asciidoc[]
 include::modules/nats.asciidoc[]
+include::modules/netflow.asciidoc[]
 include::modules/nginx.asciidoc[]
 include::modules/osquery.asciidoc[]
 include::modules/postgresql.asciidoc[]

filebeat/fileset/pipelines.go

@@ -121,6 +121,12 @@ func loadPipeline(esClient PipelineLoader, pipelineID string, content map[string
 			return nil
 		}
 	}
+
+	err := setECSProcessors(esClient.GetVersion(), pipelineID, content)
+	if err != nil {
+		return fmt.Errorf("failed to adapt pipeline for ECS compatibility: %v", err)
+	}
+
 	body, err := esClient.LoadJSON(path, content)
 	if err != nil {
 		return interpretError(err, body)
@@ -129,6 +135,40 @@ func loadPipeline(esClient PipelineLoader, pipelineID string, content map[string
 	return nil
 }
 
+// setECSProcessors sets required ECS options in processors when filebeat version is >= 7.0.0
+// and ES is 6.7.X to ease migration to ECS.
+func setECSProcessors(esVersion common.Version, pipelineID string, content map[string]interface{}) error {
+	ecsVersion := common.MustNewVersion("7.0.0")
+	if !esVersion.LessThan(ecsVersion) {
+		return nil
+	}
+
+	p, ok := content["processors"]
+	if !ok {
+		return nil
+	}
+	processors, ok := p.([]interface{})
+	if !ok {
+		return fmt.Errorf("'processors' in pipeline '%s' expected to be a list, found %T", pipelineID, p)
+	}
+
+	minUserAgentVersion := common.MustNewVersion("6.7.0")
+	for _, p := range processors {
+		processor, ok := p.(map[string]interface{})
+		if !ok {
+			continue
+		}
+		if options, ok := processor["user_agent"].(map[string]interface{}); ok {
+			if esVersion.LessThan(minUserAgentVersion) {
+				return fmt.Errorf("user_agent processor requires option 'ecs: true', but Elasticsearch %v does not support this option (Elasticsearch %v or newer is required)", esVersion, minUserAgentVersion)
+			}
+			logp.Debug("modules", "Setting 'ecs: true' option in user_agent processor for field '%v' in pipeline '%s'", options["field"], pipelineID)
+			options["ecs"] = true
+		}
+	}
+	return nil
+}
+
 func deletePipeline(esClient PipelineLoader, pipelineID string) error {
 	path := makeIngestPipelinePath(pipelineID)
 	_, _, err := esClient.Request("DELETE", path, "", nil, nil)