merge conflicts

CHANGELOG.asciidoc

@@ -3,10 +3,16 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.0.0-alpha2]]
+=== Beats version 8.0.0-alpha2
+
+Changes will be described in a later alpha / beta.
+
 [[release-notes-8.0.0-alpha1]]
 === Beats version 8.0.0-alpha1
 
 Changes will be described in a later alpha / beta.
+
 [[release-notes-7.14.1]]
 === Beats version 7.14.1
 https://github.com/elastic/beats/compare/v7.14.0...v7.14.1[View commits]

CHANGELOG.next.asciidoc

@@ -86,6 +86,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Crowdstrike ingest pipeline that was creating flattened `process` fields. {issue}27622[27622] {pull}27623[27623]
 - Rename `log.path` to `log.file.path` in filestream to be consistent with `log` input and ECS. {pull}27761[27761]
 - Removes old module name aliases (gsuite) and removing old cyberark module in favor of the new cyberarkpas{pull}27915[27915]
+- Only filesets that are explicitly configured will be enabled. {issue}17256[17256] {pull}27526[27526]
+- All filesets are disabled in the default configuration. {issue}17256[17256] {pull}27762[27762]
 
 *Heartbeat*
 - Remove long deprecated `watch_poll` functionality. {pull}27166[27166]
@@ -111,6 +113,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for kube-state-metrics v2.0.0 {pull}27552[27552]
 - Add User-Agent header to HTTP requests. {issue}18160[18160] {pull}27509[27509]
 - Errors should be thrown as errors. Metricsets inside Metricbeat will now throw errors as the `error` log level. {pull}27804[27804]
+- Remove deprecated fields in Docker module. {issue}11835[11835] {pull}27933[27933]
 
 *Packetbeat*
 
@@ -210,7 +213,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Do not try to load ILM policy if `check_exists` is `false`. {pull}27508[27508] {issue}26322[26322]
 - Fix bug with cgroups hierarchy override path in cgroups {pull}27620[27620]
 - Beat `setup kibana` command may use the elasticsearch API key defined in `output.elasticsearch.api_key`. {issue}24015[24015] {pull}27540[27540]
+- Fix `decode_xml` handling of array merging when using `to_lower: true`. {pull}27922[27922]
 - Seperate namespaces for V1 and V2 controller paths {pull}27676[27676]
+- Beats dashboards use custom index when `setup.dashboards.index` is set. {issue}21232[21232] {pull}27901[27901]
 
 *Auditbeat*
 
@@ -754,7 +759,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `join` and `sprintf` functions to `httpjson` input. {pull}27735[27735]
 - Improve memory usage of line reader of `log` and `filestream` input. {pull}27782[27782]
 - Add `ignore_empty_value` flag to `httpjson` `split` processor. {pull}27880[27880]
-
+- Update Cisco ASA/FTD ingest pipeline grok/dissect patterns for multiple message IDs. {issue}26869[26869] {pull}26879[26879]
+- Add write access to `url.value` from `request.transforms` in `httpjson` input. {pull}27937[27937]
 
 *Heartbeat*
 

dev-tools/mage/check.go

@@ -36,6 +36,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
+	"github.com/elastic/beats/v7/libbeat/dashboards"
 	"github.com/elastic/beats/v7/libbeat/processors/dissect"
 )
 
@@ -260,6 +261,14 @@ func checkDashboardForErrors(file string, d []byte) bool {
 		fmt.Println("  ", err)
 	}
 
+	replaced := dashboards.ReplaceIndexInDashboardObject("my-test-index-*", d)
+	if bytes.Contains(replaced, []byte(BeatName+"-*")) {
+		hasErrors = true
+		fmt.Printf(">> Cannot modify all index pattern references in dashboard - %s\n", file)
+		fmt.Println("Please edit the dashboard override function named ReplaceIndexInDashboardObject in libbeat.")
+		fmt.Println(string(replaced))
+	}
+
 	return hasErrors
 }
 

dev-tools/mage/modules.go

@@ -18,10 +18,15 @@
 package mage
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/joeshaw/multierror"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
 )
 
 var modulesDConfigTemplate = `
@@ -71,3 +76,68 @@ func GenerateDirModulesD() error {
 	}
 	return nil
 }
+
+type datasetDefinition struct {
+	Enabled *bool
+}
+
+type moduleDefinition struct {
+	Name     string                       `yaml:"module"`
+	Filesets map[string]datasetDefinition `yaml:",inline"`
+}
+
+// ValidateDirModulesD validates a modules.d directory containing the
+// <module>.yml.disabled files. It checks that the files are valid
+// yaml and conform to module definitions.
+func ValidateDirModulesD() error {
+	_, err := loadModulesD()
+	return err
+}
+
+// ValidateDirModulesDDatasetsDisabled ensures that all the datasets
+// are disabled by default.
+func ValidateDirModulesDDatasetsDisabled() error {
+	cfgs, err := loadModulesD()
+	if err != nil {
+		return err
+	}
+	var errs multierror.Errors
+	for path, cfg := range cfgs {
+		// A config.yml is a list of module configurations.
+		for modIdx, mod := range cfg {
+			// A module config is a map of datasets.
+			for dsName, ds := range mod.Filesets {
+				if ds.Enabled == nil || *ds.Enabled {
+					var entry string
+					if len(cfg) > 1 {
+						entry = fmt.Sprintf(" (entry #%d)", modIdx+1)
+					}
+					err = fmt.Errorf("in file '%s': %s module%s dataset %s must be explicitly disabled (needs `enabled: false`)",
+						path, mod.Name, entry, dsName)
+					errs = append(errs, err)
+				}
+			}
+		}
+	}
+	return errs.Err()
+}
+
+func loadModulesD() (modules map[string][]moduleDefinition, err error) {
+	files, err := filepath.Glob("modules.d/*.disabled")
+	if err != nil {
+		return nil, err
+	}
+	modules = make(map[string][]moduleDefinition, len(files))
+	for _, file := range files {
+		contents, err := ioutil.ReadFile(file)
+		if err != nil {
+			return nil, errors.Wrapf(err, "reading %s", file)
+		}
+		var cfg []moduleDefinition
+		if err = yaml.Unmarshal(contents, &cfg); err != nil {
+			return nil, errors.Wrapf(err, "parsing %s as YAML", file)
+		}
+		modules[file] = cfg
+	}
+	return modules, nil
+}

filebeat/autodiscover/builder/hints/logs.go

@@ -201,7 +201,7 @@ func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*f
 	var configured bool
 	filesets := make(map[string]*filesetConfig)
 
-	moduleFilesets, err := l.registry.ModuleFilesets(module)
+	moduleFilesets, err := l.registry.ModuleAvailableFilesets(module)
 	if err != nil {
 		logp.Err("Error retrieving module filesets: %+v", err)
 		return nil

filebeat/beater/filebeat.go

@@ -113,6 +113,19 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *common.Config) (b
 	}
 	if !moduleRegistry.Empty() {
 		logp.Info("Enabled modules/filesets: %s", moduleRegistry.InfoString())
+		for _, mod := range moduleRegistry.ModuleNames() {
+			if mod == "" {
+				continue
+			}
+			filesets, err := moduleRegistry.ModuleConfiguredFilesets(mod)
+			if err != nil {
+				logp.Err("Failed listing filesets for module %s", mod)
+				continue
+			}
+			if len(filesets) == 0 {
+				logp.Warn("Module %s is enabled but has no enabled filesets", mod)
+			}
+		}
 	}
 
 	moduleInputs, err := moduleRegistry.GetInputConfigs()

filebeat/docs/fields.asciidoc

@@ -21807,6 +21807,26 @@ type: keyword
 The WebVPN group name the user belongs to
 
 
+type: keyword
+
+--
+
+*`cisco.asa.termination_initiator`*::
++
+--
+Interface name of the side that initiated the teardown
+
+
+type: keyword
+
+--
+
+*`cisco.asa.tunnel_type`*::
++
+--
+SA type (remote access or L2L)
+
+
 type: keyword
 
 --
@@ -22045,6 +22065,16 @@ type: keyword
 The WebVPN group name the user belongs to
 
 
+type: keyword
+
+--
+
+*`cisco.ftd.termination_initiator`*::
++
+--
+Interface name of the side that initiated the teardown
+
+
 type: keyword
 
 --

filebeat/docs/filebeat-modules-options.asciidoc

@@ -75,8 +75,12 @@ The following example shows a configuration that runs the `nginx`,`mysql`, and
 ----
 {beatname_lc}.modules:
 - module: nginx
+  access:
+  error:
 - module: mysql
+  slowlog:
 - module: system
+  auth:
 ----
 
 [[advanced-settings]]

filebeat/docs/getting-started.asciidoc

@@ -86,8 +86,8 @@ configs:
 include::{libbeat-dir}/tab-widgets/enable-modules-widget.asciidoc[]
 --
 
-. In the module configs under `modules.d`, change the module settings to match
-your environment.
+. In the module configs under `modules.d`, enable the desired datasets and
+change the module settings to match your environment.
 +
 For example, log locations are set based on the OS. If your logs aren't in
 default locations, set the `paths` variable:
@@ -97,6 +97,7 @@ default locations, set the `paths` variable:
 ----
 - module: nginx
   access:
+    enabled: true
     var.paths: ["/var/log/nginx/access.log*"] <1>
 ----
 --

filebeat/docs/modules/sophos.asciidoc

@@ -16,17 +16,17 @@ logs in syslog format or from a file for the following devices:
 - `xg` fileset: supports Sophos XG SFOS logs.
 - `utm` fileset: supports Sophos UTM logs.
 
-To configure a remote syslog destination, please reference the https://community.sophos.com/kb/en-us/123184[SophosXG/SFOS Documentation].
+To configure a remote syslog destination, please reference the https://docs.sophos.com/nsg/sophos-firewall/18.5/Help/en-us/webhelp/onlinehelp/nsg/tasks/SyslogServerAdd.html[SophosXG/SFOS Documentation].
 
-The syslog format choosen should be `Default`.
+The syslog format choosen in Sophos configuration should be `Central Reporting Format`.
 
 include::../include/gs-link.asciidoc[]
 
 [float]
 === Compatibility
 
-This module has been tested against SFOS version 17.5.x and 18.0.x.
-Versions above this are expected to work but have not been tested.
+This module has been tested against SFOS version 17.5.x, 18.0.x, and 18.5.x.
+Versions above this and between 18.0 - 18.5 are expected to work but have not been tested.
 
 include::../include/configuring-intro.asciidoc[]