Filebeat: Make all filesets disabled in default configuration (elasti…

…c#27762)

What does this PR do?

Changes the default configuration for Filebeat's filesets to make them
disabled by default (enabled: false).

Adds a check to the config and update targets of mage to check that
default configurations have an explicit disable:

> error: in file 'modules.d/checkpoint.yml.disabled': checkpoint module
> dataset firewall must be explicitly disabled (needs enabled: false)

Why is it important?

The previous default of having all filesets enabled, paired with
the configuration loader enabling all non-explicitly-disabled
filesets (changed in elastic#27526) has been causing trouble for our users
for quite some time.

CHANGELOG.next.asciidoc

@@ -86,6 +86,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Crowdstrike ingest pipeline that was creating flattened `process` fields. {issue}27622[27622] {pull}27623[27623]
 - Rename `log.path` to `log.file.path` in filestream to be consistent with `log` input and ECS. {pull}27761[27761]
 - Only filesets that are explicitly configured will be enabled. {issue}17256[17256] {pull}27526[27526]
+- All filesets are disabled in the default configuration. {issue}17256[17256] {pull}27762[27762]
 
 *Heartbeat*
 - Remove long deprecated `watch_poll` functionality. {pull}27166[27166]

dev-tools/mage/modules.go

@@ -18,10 +18,15 @@
 package mage
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/joeshaw/multierror"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
 )
 
 var modulesDConfigTemplate = `
@@ -71,3 +76,68 @@ func GenerateDirModulesD() error {
 	}
 	return nil
 }
+
+type datasetDefinition struct {
+	Enabled *bool
+}
+
+type moduleDefinition struct {
+	Name     string                       `yaml:"module"`
+	Filesets map[string]datasetDefinition `yaml:",inline"`
+}
+
+// ValidateDirModulesD validates a modules.d directory containing the
+// <module>.yml.disabled files. It checks that the files are valid
+// yaml and conform to module definitions.
+func ValidateDirModulesD() error {
+	_, err := loadModulesD()
+	return err
+}
+
+// ValidateDirModulesDDatasetsDisabled ensures that all the datasets
+// are disabled by default.
+func ValidateDirModulesDDatasetsDisabled() error {
+	cfgs, err := loadModulesD()
+	if err != nil {
+		return err
+	}
+	var errs multierror.Errors
+	for path, cfg := range cfgs {
+		// A config.yml is a list of module configurations.
+		for modIdx, mod := range cfg {
+			// A module config is a map of datasets.
+			for dsName, ds := range mod.Filesets {
+				if ds.Enabled == nil || *ds.Enabled {
+					var entry string
+					if len(cfg) > 1 {
+						entry = fmt.Sprintf(" (entry #%d)", modIdx+1)
+					}
+					err = fmt.Errorf("in file '%s': %s module%s dataset %s must be explicitly disabled (needs `enabled: false`)",
+						path, mod.Name, entry, dsName)
+					errs = append(errs, err)
+				}
+			}
+		}
+	}
+	return errs.Err()
+}
+
+func loadModulesD() (modules map[string][]moduleDefinition, err error) {
+	files, err := filepath.Glob("modules.d/*.disabled")
+	if err != nil {
+		return nil, err
+	}
+	modules = make(map[string][]moduleDefinition, len(files))
+	for _, file := range files {
+		contents, err := ioutil.ReadFile(file)
+		if err != nil {
+			return nil, errors.Wrapf(err, "reading %s", file)
+		}
+		var cfg []moduleDefinition
+		if err = yaml.Unmarshal(contents, &cfg); err != nil {
+			return nil, errors.Wrapf(err, "parsing %s as YAML", file)
+		}
+		modules[file] = cfg
+	}
+	return modules, nil
+}

filebeat/docs/getting-started.asciidoc

@@ -86,8 +86,8 @@ configs:
 include::{libbeat-dir}/tab-widgets/enable-modules-widget.asciidoc[]
 --
 
-. In the module configs under `modules.d`, change the module settings to match
-your environment.
+. In the module configs under `modules.d`, enable the desired datasets and
+change the module settings to match your environment.
 +
 For example, log locations are set based on the OS. If your logs aren't in
 default locations, set the `paths` variable:
@@ -97,6 +97,7 @@ default locations, set the `paths` variable:
 ----
 - module: nginx
   access:
+    enabled: true
     var.paths: ["/var/log/nginx/access.log*"] <1>
 ----
 --

filebeat/filebeat.reference.yml

@@ -80,32 +80,32 @@ filebeat.modules:
 - module: elasticsearch
   # Server log
   server:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   gc:
-    enabled: true
+    enabled: false
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   audit:
-    enabled: true
+    enabled: false
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   slowlog:
-    enabled: true
+    enabled: false
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   deprecation:
-    enabled: true
+    enabled: false
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
@@ -114,7 +114,7 @@ filebeat.modules:
 - module: haproxy
   # All logs
   log:
-    enabled: true
+    enabled: false
 
     # Set which input to use between syslog (default) or file.
     #var.input:
@@ -191,7 +191,7 @@ filebeat.modules:
 - module: kafka
   # All logs
   log:
-    enabled: true
+    enabled: false
 
     # Set custom paths for Kafka. If left empty,
     # Filebeat will look under /opt.
@@ -205,15 +205,15 @@ filebeat.modules:
 - module: kibana
   # Server logs
   log:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Audit logs
   audit:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
@@ -281,7 +281,7 @@ filebeat.modules:
 - module: nats
   # All logs
   log:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
@@ -322,9 +322,9 @@ filebeat.modules:
   #  #var.paths:
 
 #------------------------------- Osquery Module -------------------------------
-- module: osquery
-  result:
-    enabled: true
+#- module: osquery
+  #result:
+    #enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
@@ -339,7 +339,7 @@ filebeat.modules:
 - module: pensando
 # Firewall logs
   dfw:
-    enabled: true
+    enabled: false
     var.syslog_host: 0.0.0.0
     var.syslog_port: 9001
 
@@ -384,7 +384,7 @@ filebeat.modules:
 #----------------------------- Google Santa Module -----------------------------
 - module: santa
   log:
-    enabled: true
+    enabled: false
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the the default path.
     #var.paths:

filebeat/magefile.go

@@ -105,6 +105,7 @@ func Update() {
 // modules.d directory.
 func Config() {
 	mg.Deps(devtools.GenerateDirModulesD, configYML)
+	mg.SerialDeps(devtools.ValidateDirModulesD, devtools.ValidateDirModulesDDatasetsDisabled)
 }
 
 func configYML() error {

filebeat/module/apache/_meta/config.yml

@@ -1,15 +1,15 @@
 - module: apache
   # Access logs
   access:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Error logs
   error:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.

filebeat/module/auditd/_meta/config.yml

@@ -1,6 +1,6 @@
 - module: auditd
   log:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.

filebeat/module/elasticsearch/_meta/config.yml

@@ -1,32 +1,32 @@
 - module: elasticsearch
   # Server log
   server:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   gc:
-    enabled: true
+    enabled: false
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   audit:
-    enabled: true
+    enabled: false
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   slowlog:
-    enabled: true
+    enabled: false
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   deprecation:
-    enabled: true
+    enabled: false
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:

filebeat/module/haproxy/_meta/config.yml

@@ -1,7 +1,7 @@
 - module: haproxy
   # All logs
   log:
-    enabled: true
+    enabled: false
 
     # Set which input to use between syslog (default) or file.
     #var.input:

filebeat/module/icinga/_meta/config.yml

@@ -1,23 +1,23 @@
 - module: icinga
   # Main logs
   main:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Debug logs
   debug:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Startup logs
   startup:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.

filebeat/module/iis/_meta/config.yml

@@ -1,17 +1,17 @@
 - module: iis
   # Access logs
   access:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Error logs
   error:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
-
+

filebeat/module/kafka/_meta/config.yml

@@ -1,7 +1,7 @@
 - module: kafka
   # All logs
   log:
-    enabled: true
+    enabled: false
 
     # Set custom paths for Kafka. If left empty,
     # Filebeat will look under /opt.

filebeat/module/kibana/_meta/config.yml

@@ -1,15 +1,15 @@
 - module: kibana
   # Server logs
   log:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
   # Audit logs
   audit:
-    enabled: true
+    enabled: false
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.