libbeat: support explicit dynamic templates

CHANGELOG.next.asciidoc

@@ -607,6 +607,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add kubernetes.pod.ip field in kubernetes metadata. {pull}25037[25037]
 - Discover changes in Kubernetes namespace metadata as soon as they happen. {pull}25117[25117]
 - Add `decode_xml_wineventlog` processor. {issue}23910[23910] {pull}25115[25115]
+- Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422]
 
 *Auditbeat*
 

libbeat/mapping/field.go

@@ -56,6 +56,14 @@ type Field struct {
 	MigrationAlias bool        `config:"migration"`
 	Dimension      *bool       `config:"dimension"`
 
+	// DynamicTemplate controls whether this field represents an explicitly
+	// named dynamic template.
+	//
+	// Such dynamic templates are only suitable for use in dynamic_template
+	// parameter in bulk requests or in ingest pipelines, as they will have
+	// no path or type match criteria.
+	DynamicTemplate bool `config:"dynamic_template"`
+
 	ObjectType            string          `config:"object_type"`
 	ObjectTypeMappingType string          `config:"object_type_mapping_type"`
 	ScalingFactor         int             `config:"scaling_factor"`

libbeat/template/processor.go

@@ -120,7 +120,15 @@ func (p *Processor) Process(fields mapping.Fields, state *fieldState, output com
 		}
 
 		if len(indexMapping) > 0 {
-			output.Put(mapping.GenerateKey(field.Name), indexMapping)
+			if field.DynamicTemplate {
+				// Explicit dynamic templates were introduced in
+				// Elasticsearch 7.13, ignore if unsupported
+				if !p.EsVersion.LessThan(common.MustNewVersion("7.13.0")) {
+					p.addDynamicTemplate(field.Name, "", "", indexMapping)
+				}
+			} else {
+				output.Put(mapping.GenerateKey(field.Name), indexMapping)
+			}
 		}
 	}
 	return nil
@@ -375,8 +383,11 @@ func (p *Processor) object(f *mapping.Field) common.MapStr {
 	if len(f.ObjectTypeParams) != 0 {
 		otParams = f.ObjectTypeParams
 	} else {
-		otParams = []mapping.ObjectTypeCfg{mapping.ObjectTypeCfg{
-			ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}}
+		otParams = []mapping.ObjectTypeCfg{{
+			ObjectType:            f.ObjectType,
+			ObjectTypeMappingType: f.ObjectTypeMappingType,
+			ScalingFactor:         f.ScalingFactor,
+		}}
 	}
 
 	for _, otp := range otParams {
@@ -425,7 +436,7 @@ func (p *Processor) object(f *mapping.Field) common.MapStr {
 		if len(otParams) > 1 {
 			path = fmt.Sprintf("%s_%s", path, matchingType)
 		}
-		p.addDynamicTemplate(path, pathMatch, dynProperties, matchingType)
+		p.addDynamicTemplate(path, pathMatch, matchingType, dynProperties)
 	}
 
 	properties := getDefaultProperties(f)
@@ -442,14 +453,14 @@ func (p *Processor) object(f *mapping.Field) common.MapStr {
 }
 
 type dynamicTemplateKey struct {
-	path      string
+	name      string
 	pathMatch string
 	matchType string
 }
 
-func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) {
+func (p *Processor) addDynamicTemplate(name, pathMatch, matchType string, properties common.MapStr) bool {
 	key := dynamicTemplateKey{
-		path:      path,
+		name:      name,
 		pathMatch: pathMatch,
 		matchType: matchType,
 	}
@@ -458,21 +469,25 @@ func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties
 	} else {
 		if _, ok := p.dynamicTemplatesMap[key]; ok {
 			// Dynamic template already added.
-			return
+			return false
 		}
 	}
+	dynamicTemplateProperties := common.MapStr{
+		"mapping": properties,
+	}
+	if matchType != "" {
+		dynamicTemplateProperties["match_mapping_type"] = matchType
+	}
+	if pathMatch != "" {
+		dynamicTemplateProperties["path_match"] = pathMatch
+	}
 	dynamicTemplate := common.MapStr{
-		// Set the path of the field as name
-		path: common.MapStr{
-			"mapping":            properties,
-			"match_mapping_type": matchType,
-			"path_match":         pathMatch,
-		},
+		name: dynamicTemplateProperties,
 	}
 	p.dynamicTemplatesMap[key] = dynamicTemplate
 	p.dynamicTemplates = append(p.dynamicTemplates, dynamicTemplate)
+	return true
 }
-
 func getDefaultProperties(f *mapping.Field) common.MapStr {
 	// Currently no defaults exist
 	properties := common.MapStr{}

libbeat/template/processor_test.go

@@ -21,6 +21,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/mapping"
@@ -465,6 +466,20 @@ func TestDynamicTemplates(t *testing.T) {
 				},
 			},
 		},
+		{
+			field: mapping.Field{
+				Name:            "dynamic_histogram",
+				Type:            "histogram",
+				DynamicTemplate: true,
+			},
+			expected: []common.MapStr{
+				{
+					"dynamic_histogram": common.MapStr{
+						"mapping": common.MapStr{"type": "histogram"},
+					},
+				},
+			},
+		},
 	}
 
 	for _, numericType := range []string{"byte", "double", "float", "long", "short", "boolean"} {
@@ -492,9 +507,13 @@ func TestDynamicTemplates(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		p := &Processor{}
-		p.object(&test.field)
-		p.object(&test.field) // should not be added twice
+		output := make(common.MapStr)
+		p := &Processor{EsVersion: *common.MustNewVersion("8.0.0")}
+		err := p.Process(mapping.Fields{
+			test.field,
+			test.field, // should not be added twice
+		}, &fieldState{Path: test.field.Path}, output)
+		require.NoError(t, err)
 		assert.Equal(t, test.expected, p.dynamicTemplates)
 	}
 }