Add nested field yml support in elasticsearch template rendering (ela…

…stic#23183) * Add nested field yml support in elasticsearch template rendering * Handle empty properties * Update changelog (cherry picked from commit ba7dbb6)
andrewstucki · Dec 16, 2020 · d579eba · d579eba
1 parent 4d1c891
commit d579eba
Show file tree

Hide file tree

Showing 3 changed files with 121 additions and 24 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -161,6 +161,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
   as gauges (rather than counters). {pull}22877[22877]
 - Use PROGRAMDATA environment variable instead of C:\ProgramData for windows install service {pull}22874[22874]
 - Fix reporting of cgroup metrics when running under Docker {pull}22879[22879]
+- Fix `nested` subfield handling in generated Elasticsearch templates. {issue}23178[23178] {pull}23183[23183]
 
 
 *Auditbeat*

diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go
@@ -91,33 +91,18 @@ func (p *Processor) Process(fields mapping.Fields, state *fieldState, output com
 			indexMapping = p.alias(&field)
 		case "histogram":
 			indexMapping = p.histogram(&field)
-		case "group":
-			indexMapping = common.MapStr{}
-			if field.Dynamic.Value != nil {
-				indexMapping["dynamic"] = field.Dynamic.Value
-			}
-
-			// Combine properties with previous field definitions (if any)
-			properties := common.MapStr{}
-			key := mapping.GenerateKey(field.Name) + ".properties"
-			currentProperties, err := output.GetValue(key)
-			if err == nil {
-				var ok bool
-				properties, ok = currentProperties.(common.MapStr)
-				if !ok {
-					// This should never happen
-					return errors.New(key + " is expected to be a MapStr")
-				}
-			}
-
-			groupState := &fieldState{Path: field.Name, DefaultField: *field.DefaultField}
-			if state.Path != "" {
-				groupState.Path = state.Path + "." + field.Name
+		case "nested":
+			mapping, err := p.nested(&field, output)
+			if err != nil {
+				return err
 			}
-			if err := p.Process(field.Fields, groupState, properties); err != nil {
+			indexMapping = mapping
+		case "group":
+			mapping, err := p.group(&field, output)
+			if err != nil {
 				return err
 			}
-			indexMapping["properties"] = properties
+			indexMapping = mapping
 		default:
 			indexMapping = p.other(&field)
 		}
@@ -187,6 +172,47 @@ func (p *Processor) scaledFloat(f *mapping.Field, params ...common.MapStr) commo
 	return property
 }
 
+func (p *Processor) nested(f *mapping.Field, output common.MapStr) (common.MapStr, error) {
+	mapping, err := p.group(f, output)
+	if err != nil {
+		return nil, err
+	}
+	mapping["type"] = "nested"
+	return mapping, nil
+}
+
+func (p *Processor) group(f *mapping.Field, output common.MapStr) (common.MapStr, error) {
+	indexMapping := common.MapStr{}
+	if f.Dynamic.Value != nil {
+		indexMapping["dynamic"] = f.Dynamic.Value
+	}
+
+	// Combine properties with previous field definitions (if any)
+	properties := common.MapStr{}
+	key := mapping.GenerateKey(f.Name) + ".properties"
+	currentProperties, err := output.GetValue(key)
+	if err == nil {
+		var ok bool
+		properties, ok = currentProperties.(common.MapStr)
+		if !ok {
+			// This should never happen
+			return nil, errors.New(key + " is expected to be a MapStr")
+		}
+	}
+
+	groupState := &fieldState{Path: f.Name, DefaultField: *f.DefaultField}
+	if f.Path != "" {
+		groupState.Path = f.Path + "." + f.Name
+	}
+	if err := p.Process(f.Fields, groupState, properties); err != nil {
+		return nil, err
+	}
+	if len(properties) != 0 {
+		indexMapping["properties"] = properties
+	}
+	return indexMapping, nil
+}
+
 func (p *Processor) halfFloat(f *mapping.Field) common.MapStr {
 	property := getDefaultProperties(f)
 	property["type"] = "half_float"

diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go
@@ -802,3 +802,73 @@ func TestProcessWildcardPreSupport(t *testing.T) {
 
 	assert.Equal(t, expectedOutput, output)
 }
+
+func TestProcessNestedSupport(t *testing.T) {
+	fields := mapping.Fields{
+		mapping.Field{
+			Name: "test",
+			Type: "nested",
+			Fields: mapping.Fields{
+				mapping.Field{
+					Name: "one",
+					Type: "keyword",
+				},
+			},
+		},
+	}
+
+	output := common.MapStr{}
+	version, err := common.NewVersion("7.8.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := Processor{EsVersion: *version, ElasticLicensed: true}
+	err = p.Process(fields, nil, output)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedOutput := common.MapStr{
+		"test": common.MapStr{
+			"type": "nested",
+			"properties": common.MapStr{
+				"one": common.MapStr{
+					"ignore_above": 1024,
+					"type":         "keyword",
+				},
+			},
+		},
+	}
+
+	assert.Equal(t, expectedOutput, output)
+}
+
+func TestProcessNestedSupportNoSubfields(t *testing.T) {
+	fields := mapping.Fields{
+		mapping.Field{
+			Name: "test",
+			Type: "nested",
+		},
+	}
+
+	output := common.MapStr{}
+	version, err := common.NewVersion("7.8.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := Processor{EsVersion: *version, ElasticLicensed: true}
+	err = p.Process(fields, nil, output)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedOutput := common.MapStr{
+		"test": common.MapStr{
+			"type": "nested",
+		},
+	}
+
+	assert.Equal(t, expectedOutput, output)
+}