diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 15bc2d2790eb..6a6add512fa9 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -31,6 +31,11 @@ type Processor struct { EsVersion common.Version Migration bool ElasticLicensed bool + + // dynamicTemplatesMap records which dynamic templates have been added, to prevent duplicates. + dynamicTemplatesMap map[dynamicTemplateKey]common.MapStr + // dynamicTemplates records the dynamic templates in the order they were added. + dynamicTemplates []common.MapStr } var ( @@ -420,7 +425,7 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { if len(otParams) > 1 { path = fmt.Sprintf("%s_%s", path, matchingType) } - addDynamicTemplate(path, pathMatch, dynProperties, matchingType) + p.addDynamicTemplate(path, pathMatch, dynProperties, matchingType) } properties := getDefaultProperties(f) @@ -436,8 +441,27 @@ func (p *Processor) object(f *mapping.Field) common.MapStr { return properties } -func addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) { - template := common.MapStr{ +type dynamicTemplateKey struct { + path string + pathMatch string + matchType string +} + +func (p *Processor) addDynamicTemplate(path string, pathMatch string, properties common.MapStr, matchType string) { + key := dynamicTemplateKey{ + path: path, + pathMatch: pathMatch, + matchType: matchType, + } + if p.dynamicTemplatesMap == nil { + p.dynamicTemplatesMap = make(map[dynamicTemplateKey]common.MapStr) + } else { + if _, ok := p.dynamicTemplatesMap[key]; ok { + // Dynamic template already added. + return + } + } + dynamicTemplate := common.MapStr{ // Set the path of the field as name path: common.MapStr{ "mapping": properties, @@ -445,8 +469,8 @@ func addDynamicTemplate(path string, pathMatch string, properties common.MapStr, "path_match": pathMatch, }, } - - dynamicTemplates = append(dynamicTemplates, template) + p.dynamicTemplatesMap[key] = dynamicTemplate + p.dynamicTemplates = append(p.dynamicTemplates, dynamicTemplate) } func getDefaultProperties(f *mapping.Field) common.MapStr { diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index 0765e84d69f2..1fb3cfb08e07 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -317,7 +317,6 @@ func TestProcessor(t *testing.T) { } func TestDynamicTemplates(t *testing.T) { - p := &Processor{} tests := []struct { field mapping.Field expected []common.MapStr @@ -493,9 +492,10 @@ func TestDynamicTemplates(t *testing.T) { } for _, test := range tests { - dynamicTemplates = nil + p := &Processor{} p.object(&test.field) - assert.Equal(t, test.expected, dynamicTemplates) + p.object(&test.field) // should not be added twice + assert.Equal(t, test.expected, p.dynamicTemplates) } } diff --git a/libbeat/template/template.go b/libbeat/template/template.go index f6366ddc6761..8ed8886e9196 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -37,9 +37,6 @@ var ( defaultNumberOfRoutingShards = 30 defaultMaxDocvalueFieldsSearch = 200 - // Array to store dynamicTemplate parts in - dynamicTemplates []common.MapStr - defaultFields []string ) @@ -147,7 +144,6 @@ func (t *Template) load(fields mapping.Fields) (common.MapStr, error) { t.Lock() defer t.Unlock() - dynamicTemplates = nil defaultFields = nil var err error @@ -164,7 +160,8 @@ func (t *Template) load(fields mapping.Fields) (common.MapStr, error) { if err := processor.Process(fields, nil, properties); err != nil { return nil, err } - output := t.Generate(properties, dynamicTemplates) + + output := t.Generate(properties, processor.dynamicTemplates) return output, nil } @@ -255,17 +252,16 @@ func (t *Template) GetPattern() string { func (t *Template) Generate(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { switch t.templateType { case IndexTemplateLegacy: - return t.generateLegacy(properties) + return t.generateLegacy(properties, dynamicTemplates) case IndexTemplateComponent: - return t.generateComponent(properties) + return t.generateComponent(properties, dynamicTemplates) case IndexTemplateIndex: - return t.generateIndex(properties) - default: + return t.generateIndex(properties, dynamicTemplates) } return nil } -func (t *Template) generateLegacy(properties common.MapStr) common.MapStr { +func (t *Template) generateLegacy(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) return common.MapStr{ keyPattern: patterns, @@ -284,7 +280,7 @@ func (t *Template) generateLegacy(properties common.MapStr) common.MapStr { } } -func (t *Template) generateComponent(properties common.MapStr) common.MapStr { +func (t *Template) generateComponent(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { return common.MapStr{ "template": common.MapStr{ "mappings": buildMappings( @@ -302,8 +298,8 @@ func (t *Template) generateComponent(properties common.MapStr) common.MapStr { } } -func (t *Template) generateIndex(properties common.MapStr) common.MapStr { - tmpl := t.generateComponent(properties) +func (t *Template) generateIndex(properties common.MapStr, dynamicTemplates []common.MapStr) common.MapStr { + tmpl := t.generateComponent(properties, dynamicTemplates) tmpl["priority"] = t.priority keyPattern, patterns := buildPatternSettings(t.esVersion, t.GetPattern()) tmpl[keyPattern] = patterns