From 1e51bb3c514f0031d0f0e93078403e0cc3a2de82 Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Tue, 2 Jun 2020 11:34:09 +0200 Subject: [PATCH] Avoid duplicate names in dynamic_templates (#18849) When multiple dynamic templates are detected for the same field, add the matching type to the template name for uniqueness. closes #17203 --- CHANGELOG.next.asciidoc | 1 + libbeat/template/processor.go | 45 ++++++++++++++++++++---------- libbeat/template/processor_test.go | 12 ++++---- 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 876f0fd9530..ce4dd81fb75 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Remove `AddDockerMetadata` and `AddKubernetesMetadata` processors from the `script` processor. They can still be used as normal processors in the configuration. {issue}16349[16349] {pull}16514[16514] - Introduce APM libbeat instrumentation, active when running the beat with ELASTIC_APM_ACTIVE=true. {pull}17938[17938] - Make error message about locked data path actionable. {pull}18667[18667] +- Ensure dynamic template names are unique for the same field. {pull}18849[18849] *Auditbeat* diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 36fd8c549a8..0f372b49838 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -19,6 +19,7 @@ package template import ( "errors" + "fmt" "strings" "github.com/elastic/beats/v7/libbeat/common" @@ -319,11 +320,12 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { for _, otp := range otParams { dynProperties := getDefaultProperties(f) + var matchingType string switch otp.ObjectType { case "scaled_float": dynProperties = p.scaledFloat(f, common.MapStr{scalingFactorKey: otp.ScalingFactor}) - addDynamicTemplate(f, dynProperties, matchType("*", otp.ObjectTypeMappingType)) + matchingType = matchType("*", otp.ObjectTypeMappingType) case "text": dynProperties["type"] = "text" @@ -331,17 +333,38 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { dynProperties["type"] = "string" dynProperties["index"] = "analyzed" } - addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType)) + matchingType = matchType("string", otp.ObjectTypeMappingType) case "keyword": dynProperties["type"] = otp.ObjectType - addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType)) + matchingType = matchType("string", otp.ObjectTypeMappingType) case "byte", "double", "float", "long", "short", "boolean": dynProperties["type"] = otp.ObjectType - addDynamicTemplate(f, dynProperties, matchType(otp.ObjectType, otp.ObjectTypeMappingType)) + matchingType = matchType(otp.ObjectType, otp.ObjectTypeMappingType) case "histogram": dynProperties["type"] = otp.ObjectType - addDynamicTemplate(f, dynProperties, matchType("*", otp.ObjectTypeMappingType)) + matchingType = matchType("*", otp.ObjectTypeMappingType) + default: + continue + } + + path := f.Path + if len(path) > 0 { + path += "." } + path += f.Name + pathMatch := path + // ensure the `path_match` string ends with a `*` + if !strings.ContainsRune(path, '*') { + pathMatch += ".*" + } + // When multiple object type parameters are detected for a field, + // add a unique part to the name of the dynamic template. + // Duplicated dynamic template names can lead to errors when template + // inheritance is applied, and will not be supported in future versions + if len(otParams) > 1 { + path = fmt.Sprintf("%s_%s", path, matchingType) + } + addDynamicTemplate(path, pathMatch, dynProperties, matchingType) } properties := getDefaultProperties(f) @@ -357,18 +380,10 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { return properties } -func addDynamicTemplate(f *mapping.Field, properties common.MapStr, matchType string) { - path := "" - if len(f.Path) > 0 { - path = f.Path + "." - } - pathMatch := path + f.Name - if !strings.ContainsRune(pathMatch, '*') { - pathMatch += ".*" - } +func addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) { template := common.MapStr{ // Set the path of the field as name - path + f.Name: common.MapStr{ + path: common.MapStr{ "mapping": properties, "match_mapping_type": matchType, "path_match": pathMatch, diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index 8d4e6c2235a..2ffc03dad24 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -443,22 +443,22 @@ func TestDynamicTemplates(t *testing.T) { Name: "context", }, expected: []common.MapStr{ - common.MapStr{ - "context": common.MapStr{ + { + "context_float": common.MapStr{ "mapping": common.MapStr{"type": "float"}, "match_mapping_type": "float", "path_match": "context.*", }, }, - common.MapStr{ - "context": common.MapStr{ + { + "context_boolean": common.MapStr{ "mapping": common.MapStr{"type": "boolean"}, "match_mapping_type": "boolean", "path_match": "context.*", }, }, - common.MapStr{ - "context": common.MapStr{ + { + "context_*": common.MapStr{ "mapping": common.MapStr{"type": "scaled_float", "scaling_factor": 10000}, "match_mapping_type": "*", "path_match": "context.*",