Use Merge with AppendValues option when merging default config with h…

…ints generated config (#36857) * Use Merge with AppendValues option when merging default config with hints generated config * Update filebeat/autodiscover/builder/hints/logs.go Co-authored-by: Andrew Gizas <[email protected]> * Update changelog --------- Co-authored-by: Andrew Gizas <[email protected]>
elastic · Oct 24, 2023 · 670b682 · 670b682
1 parent 546a413
commit 670b682
Show file tree

Hide file tree

Showing 3 changed files with 76 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -112,6 +112,7 @@ is collected by it.
 - Revert error introduced in {pull}35734[35734] when symlinks can't be resolved in filestream. {pull}36557[36557]
 - Fix ignoring external input configuration in `take_over: true` mode {issue}36378[36378] {pull}36395[36395]
 - Add validation to http_endpoint config for empty URL {pull}36816[36816] {issue}36772[36772]
+- Fix merging of array fields(processors, paths, parsers) in configurations generated from hints and default config. {issue}36838[36838] {pull}36857[36857]
 
 *Heartbeat*
 

diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go
@@ -139,7 +139,8 @@ func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*conf
 			kubernetes.ShouldPut(tempCfg, json, jsonOpts, l.log)
 		}
 		// Merge config template with the configs from the annotations
-		if err := config.Merge(tempCfg); err != nil {
+		// AppendValues option is used to append arrays from annotations to existing arrays while merging
+		if err := config.MergeWithOpts(tempCfg, ucfg.AppendValues); err != nil {
 			logp.Debug("hints.builder", "config merge failed with error: %v", err)
 			continue
 		}

diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go
@@ -43,6 +43,24 @@ func TestGenerateHints(t *testing.T) {
 		},
 	})
 
+	customProcessorCfg := conf.MustNewConfigFrom(map[string]interface{}{
+		"default_config": map[string]interface{}{
+			"type": "container",
+			"paths": []string{
+				"/var/lib/docker/containers/${data.container.id}/*-json.log",
+			},
+			"close_timeout": "true",
+			"processors": []interface{}{
+				map[string]interface{}{
+					"add_tags": map[string]interface{}{
+						"tags":   []string{"web"},
+						"target": "environment",
+					},
+				},
+			},
+		},
+	})
+
 	defaultCfg := conf.NewConfig()
 
 	defaultDisabled := conf.MustNewConfigFrom(map[string]interface{}{
@@ -389,6 +407,61 @@ func TestGenerateHints(t *testing.T) {
 				},
 			},
 		},
+		{
+			msg:    "Processors in hints must be appended in the processors of the default config",
+			config: customProcessorCfg,
+			event: bus.Event{
+				"host": "1.2.3.4",
+				"kubernetes": mapstr.M{
+					"container": mapstr.M{
+						"name": "foobar",
+						"id":   "abc",
+					},
+				},
+				"container": mapstr.M{
+					"name": "foobar",
+					"id":   "abc",
+				},
+				"hints": mapstr.M{
+					"logs": mapstr.M{
+						"processors": mapstr.M{
+							"1": mapstr.M{
+								"dissect": mapstr.M{
+									"tokenizer": "%{key1} %{key2}",
+								},
+							},
+							"drop_event": mapstr.M{},
+						},
+					},
+				},
+			},
+			len: 1,
+			result: []mapstr.M{
+				{
+					"type": "container",
+					"paths": []interface{}{
+						"/var/lib/docker/containers/abc/*-json.log",
+					},
+					"close_timeout": "true",
+					"processors": []interface{}{
+						map[string]interface{}{
+							"add_tags": map[string]interface{}{
+								"tags":   []interface{}{"web"},
+								"target": "environment",
+							},
+						},
+						map[string]interface{}{
+							"dissect": map[string]interface{}{
+								"tokenizer": "%{key1} %{key2}",
+							},
+						},
+						map[string]interface{}{
+							"drop_event": nil,
+						},
+					},
+				},
+			},
+		},
 		{
 			msg:    "Hint with module should attach input to its filesets",
 			config: customCfg,