allow/merge fields.yml overrides (elastic#9188)

CHANGELOG-developer.asciidoc

@@ -69,3 +69,4 @@ The list below covers the major changes between 6.3.0 and master only.
 - Add docker image building to `mage.Package`. {pull}8898[8898]
 - Simplified exporting of dashboards. {pull}7730[7730]
 - Update Beats to use go 1.11.2 {pull}8746[8746]
+- Allow/Merge fields.yml overrides {pull}9188[9188]

dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go

@@ -79,7 +79,7 @@ func main() {
 
 		file, err := indexPattern.Generate()
 		if err != nil {
-			log.Fatal(err)
+			log.Fatalf("ERROR: %s", err)
 		}
 
 		// Log output file location.

libbeat/common/field.go

@@ -69,7 +69,8 @@ type Field struct {
 	UrlTemplate          []VersionizedString `config:"url_template"`
 	OpenLinkInCurrentTab *bool               `config:"open_link_in_current_tab"`
 
-	Path string
+	Overwrite bool `config:"overwrite"`
+	Path      string
 }
 
 type VersionizedString struct {

libbeat/generator/fields/fields.go

@@ -32,18 +32,13 @@ type YmlFile struct {
 }
 
 func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]*YmlFile, error) {
-	commonFields := []string{
-		// Fields for custom beats
-		filepath.Join(beatPath, "_meta/fields.yml"),
-		filepath.Join(beatPath, "_meta/fields.common.yml"),
-	}
-
+	var commonFields []string
 	var libbeatFieldFiles []*YmlFile
 	var err error
 	if !isLibbeat(beatPath) {
 		commonFields = append(commonFields,
-			filepath.Join(esBeatsPath, "libbeat/_meta/fields.common.yml"),
 			filepath.Join(esBeatsPath, "libbeat/_meta/fields.ecs.yml"),
+			filepath.Join(esBeatsPath, "libbeat/_meta/fields.common.yml"),
 		)
 
 		libbeatModulesPath := filepath.Join(esBeatsPath, "libbeat/processors")
@@ -53,6 +48,12 @@ func collectCommonFiles(esBeatsPath, beatPath string, fieldFiles []*YmlFile) ([]
 		}
 	}
 
+	// Fields for custom beats last, to enable overriding more generically defined fields
+	commonFields = append(commonFields,
+		filepath.Join(beatPath, "_meta/fields.common.yml"),
+		filepath.Join(beatPath, "_meta/fields.yml"),
+	)
+
 	var files []*YmlFile
 	for _, cf := range commonFields {
 		_, err := os.Stat(cf)

libbeat/kibana/fields_transformer.go

@@ -29,7 +29,7 @@ type fieldsTransformer struct {
 	transformedFields         []common.MapStr
 	transformedFieldFormatMap common.MapStr
 	version                   *common.Version
-	keys                      common.MapStr
+	keys                      map[string]int
 }
 
 func newFieldsTransformer(version *common.Version, fields common.Fields) (*fieldsTransformer, error) {
@@ -41,7 +41,7 @@ func newFieldsTransformer(version *common.Version, fields common.Fields) (*field
 		version:                   version,
 		transformedFields:         []common.MapStr{},
 		transformedFieldFormatMap: common.MapStr{},
-		keys:                      common.MapStr{},
+		keys:                      map[string]int{},
 	}, nil
 }
 
@@ -79,17 +79,11 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
 			f.Path = path + "." + f.Name
 		}
 
-		if t.keys[f.Path] != nil {
-			msg := fmt.Sprintf("ERROR: Field <%s> is duplicated. Please update and try again.\n", f.Path)
-			panic(errors.New(msg))
-		}
-
 		if f.Type == "group" {
 			if f.Enabled == nil || *f.Enabled {
 				t.transformFields(f.Fields, f.Path)
 			}
 		} else {
-			t.keys[f.Path] = true
 			t.add(f)
 
 			if f.MultiFields != nil {
@@ -104,13 +98,35 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
 	}
 }
 
+func (t *fieldsTransformer) update(target *common.MapStr, override common.Field) error {
+	field, _ := transformField(t.version, override)
+	if override.Type == "" || (*target)["type"] == field["type"] {
+		target.Update(field)
+		if !override.Overwrite {
+			// compatible duplication
+			return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true'", override.Path)
+		}
+		return nil
+	}
+	// incompatible duplication
+	return fmt.Errorf("field <%s> is duplicated", override.Path)
+}
+
 func (t *fieldsTransformer) add(f common.Field) {
+	if idx := t.keys[f.Path]; idx > 0 {
+		target := &t.transformedFields[idx-1] // 1-indexed
+		if err := t.update(target, f); err != nil {
+			panic(err)
+		}
+		return
+	}
+
 	field, fieldFormat := transformField(t.version, f)
 	t.transformedFields = append(t.transformedFields, field)
+	t.keys[f.Path] = len(t.transformedFields) // 1-index
 	if fieldFormat != nil {
 		t.transformedFieldFormatMap[field["name"].(string)] = fieldFormat
 	}
-
 }
 
 func transformField(version *common.Version, f common.Field) (common.MapStr, common.MapStr) {

libbeat/kibana/fields_transformer_test.go

@@ -22,6 +22,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/libbeat/common"
 )
@@ -97,14 +98,75 @@ func TestMissingVersion(t *testing.T) {
 }
 
 func TestDuplicateField(t *testing.T) {
+	testCases := []struct {
+		commonFields []common.Field
+	}{
+		// type change
+		{commonFields: []common.Field{
+			{Name: "context", Path: "something"},
+			{Name: "context", Path: "something", Type: "date"},
+		}},
+		// missing overwrite
+		{commonFields: []common.Field{
+			{Name: "context", Path: "something"},
+			{Name: "context", Path: "something"},
+		}},
+		// missing overwrite in source
+		{commonFields: []common.Field{
+			{Name: "context", Path: "something", Overwrite: true},
+			{Name: "context", Path: "something"},
+		}},
+	}
+	for _, testCase := range testCases {
+		trans, err := newFieldsTransformer(version, testCase.commonFields)
+		require.NoError(t, err)
+		_, err = trans.transform()
+		assert.Error(t, err)
+	}
+}
+
+func TestValidDuplicateField(t *testing.T) {
 	commonFields := common.Fields{
-		common.Field{Name: "context", Path: "something"},
-		common.Field{Name: "context", Path: "something", Type: "keyword"},
+		common.Field{Name: "context", Path: "something", Type: "keyword", Description: "original description"},
+		common.Field{Name: "context", Path: "something", Overwrite: true, Description: "updated description",
+			Aggregatable: &falsy,
+			Analyzed:     &truthy,
+			Count:        2,
+			DocValues:    &falsy,
+			Index:        &falsy,
+			Searchable:   &falsy,
+		},
+		common.Field{
+			Name: "context",
+			Type: "group",
+			Fields: common.Fields{
+				common.Field{Name: "another", Type: "date"},
+			},
+		},
+		common.Field{
+			Name: "context",
+			Type: "group",
+			Fields: common.Fields{
+				common.Field{Name: "another", Overwrite: true},
+			},
+		},
 	}
 	trans, err := newFieldsTransformer(version, commonFields)
-	assert.NoError(t, err)
-	_, err = trans.transform()
-	assert.Error(t, err)
+	require.NoError(t, err)
+	transformed, err := trans.transform()
+	require.NoError(t, err)
+	out := transformed["fields"].([]common.MapStr)[0]
+	assert.Equal(t, out, common.MapStr{
+		"aggregatable": false,
+		"analyzed":     true,
+		"count":        2,
+		"doc_values":   false,
+		"indexed":      false,
+		"name":         "context",
+		"scripted":     false,
+		"searchable":   false,
+		"type":         "string",
+	})
 }
 
 func TestInvalidVersion(t *testing.T) {

libbeat/scripts/generate_fields_docs.py

@@ -1,6 +1,8 @@
-import yaml
-import os
 import argparse
+from collections import OrderedDict
+import os
+
+import yaml
 
 
 def document_fields(output, section, sections, path):
@@ -102,6 +104,21 @@ def fields_to_asciidoc(input, output, beat):
         print("fields.yml file is empty. fields.asciidoc cannot be generated.")
         return
 
+    # deduplicate fields, last one wins
+    for section in docs:
+        if not section.get("fields"):
+            continue
+        fields = OrderedDict()
+        for field in section["fields"]:
+            name = field["name"]
+            if name in fields:
+                assert field["type"] == fields[name]["type"], 'field "{}" redefined with different type "{}"'.format(
+                    name, field["type"])
+                fields[name].update(field)
+            else:
+                fields[name] = field
+        section["fields"] = list(fields.values())
+
     # Create sections from available fields
     sections = {}
     for v in docs: