Address {issue}10811[10811] - Add configuration knob for auto-discover default.disable

CHANGELOG.next.asciidoc

@@ -251,6 +251,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add more info to message logged when a duplicated symlink file is found {pull}10845[10845]
 - Add Netflow module to enrich flow events with geoip data. {pull}10877[10877]
 - Set `event.category: network_traffic` for Suricata. {pull}10882[10882]
+- Add configuration knob for auto-discover hints to control whether log harvesting is enabled for the pod/container. {issue}10811[10811] {pull}10911[10911]
 
 *Heartbeat*
 

filebeat/autodiscover/builder/hints/logs_test.go

@@ -47,6 +47,18 @@ func TestGenerateHints(t *testing.T) {
 			len:    0,
 			result: common.MapStr{},
 		},
+		{
+			msg: "Hints with logs.disable should return nothing",
+			event: bus.Event{
+				"hints": common.MapStr{
+					"logs": common.MapStr{
+						"disable": "true",
+					},
+				},
+			},
+			len:    0,
+			result: common.MapStr{},
+		},
 		{
 			msg: "Empty event hints should return default config",
 			event: bus.Event{

filebeat/docs/autodiscover-hints.asciidoc

@@ -94,6 +94,25 @@ filebeat.autodiscover:
       hints.enabled: true
 -------------------------------------------------------------------------------------
 
+Autodiscover provides a way to control whether log harvesting is by default disabled for the pods/containers when auto-discovery is in use. To enable it, just set `default.disable` to true:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+filebeat.autodiscover:
+  providers:
+    - type: kubernetes
+      hints.enabled: true
+      default.disable: true
+-------------------------------------------------------------------------------------
+
+Then, for the pods/containers that log harvesting should be enabled, you can annotate with hint:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+annotations:
+  co.elastic.logs/disable: false
+-------------------------------------------------------------------------------------
+
 You can annotate Kubernetes Pods with useful info to spin up {beatname_uc} inputs or modules:
 
 ["source","yaml",subs="attributes"]
@@ -137,6 +156,26 @@ filebeat.autodiscover:
       hints.enabled: true
 -------------------------------------------------------------------------------------
 
+Autodiscover provides a way to control whether log harvesting is by default disabled for the 
+containers when auto-discovery is in use. To enable it, just set `default.disable` to true:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+      default.disable: true
+-------------------------------------------------------------------------------------
+
+Then, for the containers that log harvesting should be enabled, you can label Docker containers with:
+
+["source","yaml",subs="attributes"]
+-------------------------------------------------------------------------------------
+annotations:
+  co.elastic.logs/disable: false
+-------------------------------------------------------------------------------------
+
 You can label Docker containers with useful info to spin up {beatname_uc} inputs, for example:
 
 ["source","yaml",subs="attributes"]

libbeat/autodiscover/builder/helper.go

@@ -155,7 +155,7 @@ func IsNoOp(hints common.MapStr, key string) bool {
 }
 
 // GenerateHints parses annotations based on a prefix and sets up hints that can be picked up by individual Beats.
-func GenerateHints(annotations common.MapStr, container, prefix string) common.MapStr {
+func GenerateHints(annotations common.MapStr, container, prefix string, defaultDisable bool) common.MapStr {
 	hints := common.MapStr{}
 	if rawEntries, err := annotations.GetValue(prefix); err == nil {
 		if entries, ok := rawEntries.(common.MapStr); ok {
@@ -195,5 +195,10 @@ func GenerateHints(annotations common.MapStr, container, prefix string) common.M
 		}
 	}
 
+	// Update hints: if .disabled annotation does not exist, set according to disabledByDefault flag
+	if _, err := hints.GetValue("logs.disable"); err != nil && defaultDisable {
+		hints.Put("logs.disable", "true")
+	}
+
 	return hints
 }

libbeat/autodiscover/builder/helper_test.go

@@ -27,13 +27,15 @@ import (
 
 func TestGenerateHints(t *testing.T) {
 	tests := []struct {
-		annotations map[string]string
-		result      common.MapStr
+		annotations    map[string]string
+		defaultDisable bool
+		result         common.MapStr
 	}{
 		// Empty annotations should return empty hints
 		{
-			annotations: map[string]string{},
-			result:      common.MapStr{},
+			annotations:    map[string]string{},
+			defaultDisable: false,
+			result:         common.MapStr{},
 		},
 
 		// Scenarios being tested:
@@ -50,6 +52,7 @@ func TestGenerateHints(t *testing.T) {
 				"co.elastic.metrics.foobar1/period": "15s",
 				"not.to.include":                    "true",
 			},
+			defaultDisable: false,
 			result: common.MapStr{
 				"logs": common.MapStr{
 					"multiline": common.MapStr{
@@ -62,13 +65,74 @@ func TestGenerateHints(t *testing.T) {
 				},
 			},
 		},
+		// Scenarios being tested:
+		// logs.disable must be generated when defaultDisable is set and annotations does not
+		// have co.elastic.logs/disable set to false.
+		// logs/multiline.pattern must be a nested common.MapStr under hints.logs
+		// metrics/module must be found in hints.metrics
+		// not.to.include must not be part of hints
+		// period is annotated at both container and pod level. Container level value must be in hints
+		{
+			annotations: map[string]string{
+				"co.elastic.logs/multiline.pattern": "^test",
+				"co.elastic.metrics/module":         "prometheus",
+				"co.elastic.metrics/period":         "10s",
+				"co.elastic.metrics.foobar/period":  "15s",
+				"co.elastic.metrics.foobar1/period": "15s",
+				"not.to.include":                    "true",
+			},
+			defaultDisable: true,
+			result: common.MapStr{
+				"logs": common.MapStr{
+					"multiline": common.MapStr{
+						"pattern": "^test",
+					},
+					"disable": "true",
+				},
+				"metrics": common.MapStr{
+					"module": "prometheus",
+					"period": "15s",
+				},
+			},
+		},
+		// Scenarios being tested:
+		// logs.disable must not be generated when defaultDisable is set, but annotations
+		// have co.elastic.logs/disable set to false.
+		// logs/multiline.pattern must be a nested common.MapStr under hints.logs
+		// metrics/module must be found in hints.metrics
+		// not.to.include must not be part of hints
+		// period is annotated at both container and pod level. Container level value must be in hints
+		{
+			annotations: map[string]string{
+				"co.elastic.logs/disable":           "false",
+				"co.elastic.logs/multiline.pattern": "^test",
+				"co.elastic.metrics/module":         "prometheus",
+				"co.elastic.metrics/period":         "10s",
+				"co.elastic.metrics.foobar/period":  "15s",
+				"co.elastic.metrics.foobar1/period": "15s",
+				"not.to.include":                    "true",
+			},
+			defaultDisable: true,
+			result: common.MapStr{
+				"logs": common.MapStr{
+					"multiline": common.MapStr{
+						"pattern": "^test",
+					},
+					"disable": "false",
+				},
+				"metrics": common.MapStr{
+					"module": "prometheus",
+					"period": "15s",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
 		annMap := common.MapStr{}
 		for k, v := range test.annotations {
 			annMap.Put(k, v)
 		}
-		assert.Equal(t, GenerateHints(annMap, "foobar", "co.elastic"), test.result)
+		assert.Equal(t, GenerateHints(annMap, "foobar", "co.elastic", test.defaultDisable), test.result)
 	}
 }

libbeat/autodiscover/providers/docker/config.go

@@ -25,14 +25,15 @@ import (
 
 // Config for docker autodiscover provider
 type Config struct {
-	Host         string                  `config:"host"`
-	TLS          *docker.TLSConfig       `config:"ssl"`
-	Prefix       string                  `config:"prefix"`
-	HintsEnabled bool                    `config:"hints.enabled"`
-	Builders     []*common.Config        `config:"builders"`
-	Appenders    []*common.Config        `config:"appenders"`
-	Templates    template.MapperSettings `config:"templates"`
-	Dedot        bool                    `config:"labels.dedot"`
+	Host           string                  `config:"host"`
+	TLS            *docker.TLSConfig       `config:"ssl"`
+	Prefix         string                  `config:"prefix"`
+	HintsEnabled   bool                    `config:"hints.enabled"`
+	DefaultDisable bool                    `config:"default.disable"`
+	Builders       []*common.Config        `config:"builders"`
+	Appenders      []*common.Config        `config:"appenders"`
+	Templates      template.MapperSettings `config:"templates"`
+	Dedot          bool                    `config:"labels.dedot"`
 }
 
 func defaultConfig() *Config {

libbeat/autodiscover/providers/docker/docker.go

@@ -267,7 +267,7 @@ func (d *Provider) generateHints(event bus.Event) bus.Event {
 		e["port"] = port
 	}
 	if labels, err := dockerMeta.GetValue("labels"); err == nil {
-		hints := builder.GenerateHints(labels.(common.MapStr), "", d.config.Prefix)
+		hints := builder.GenerateHints(labels.(common.MapStr), "", d.config.Prefix, d.config.DefaultDisable)
 		e["hints"] = hints
 	}
 	return e

libbeat/autodiscover/providers/kubernetes/config.go

@@ -33,11 +33,12 @@ type Config struct {
 	SyncPeriod     time.Duration `config:"sync_period"`
 	CleanupTimeout time.Duration `config:"cleanup_timeout"`
 
-	Prefix       string                  `config:"prefix"`
-	HintsEnabled bool                    `config:"hints.enabled"`
-	Builders     []*common.Config        `config:"builders"`
-	Appenders    []*common.Config        `config:"appenders"`
-	Templates    template.MapperSettings `config:"templates"`
+	Prefix         string                  `config:"prefix"`
+	HintsEnabled   bool                    `config:"hints.enabled"`
+	DefaultDisable bool                    `config:"default.disable"`
+	Builders       []*common.Config        `config:"builders"`
+	Appenders      []*common.Config        `config:"appenders"`
+	Templates      template.MapperSettings `config:"templates"`
 }
 
 func defaultConfig() *Config {

libbeat/autodiscover/providers/kubernetes/kubernetes.go

@@ -275,12 +275,13 @@ func (p *Provider) generateHints(event bus.Event) bus.Event {
 	}
 
 	cname := builder.GetContainerName(container)
-	hints := builder.GenerateHints(annotations, cname, p.config.Prefix)
+	hints := builder.GenerateHints(annotations, cname, p.config.Prefix, p.config.DefaultDisable)
+	logp.Debug("kubernetes", "Generated hints %+v", hints)
 	if len(hints) != 0 {
 		e["hints"] = hints
 	}
 
-	logp.Debug("kubernetes", "Generated builder event %v", event)
+	logp.Debug("kubernetes", "Generated builder event %+v", e)
 
 	return e
 }

testing/environments/snapshot.yml

@@ -3,7 +3,7 @@
 version: '2.3'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.0.0-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.1.0-SNAPSHOT
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
       retries: 300
@@ -16,7 +16,7 @@ services:
     - "xpack.security.enabled=false"
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:7.0.0-SNAPSHOT
+    image: docker.elastic.co/logstash/logstash:7.1.0-SNAPSHOT
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"]
       retries: 600
@@ -26,7 +26,7 @@ services:
       - ./docker/logstash/pki:/etc/pki:ro
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:7.0.0-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:7.1.0-SNAPSHOT
     healthcheck:
       test: ["CMD-SHELL", 'python -c ''import urllib, json; response = urllib.urlopen("http://localhost:5601/api/status"); data = json.loads(response.read()); exit(1) if data["status"]["overall"]["state"] != "green" else exit(0);''']
       retries: 600