From 3072a79b333bd369d6dd2b8fc216052c09de8b8a Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Mon, 15 May 2017 13:31:57 +0200 Subject: [PATCH] Add option to adjust _source options + improvements This adds/fixes: * Ability to disable _source, or set other _source related options * Moves template index settings under `settings.index` * Fixes the overwrite logic (was using the wrong template name on the check) * Fixes error handling * Integration tests for overwritting Part of #3654 and #4112. --- CHANGELOG.asciidoc | 2 +- libbeat/beat/beat.go | 8 +- libbeat/docs/template-config.asciidoc | 22 +++- libbeat/template/config.go | 17 +-- libbeat/template/load.go | 22 ++-- libbeat/template/load_integration_test.go | 121 +++++++++++++++++++--- libbeat/template/template.go | 28 ++--- 7 files changed, 169 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6c9a3c4513af..94c739c3b516 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -46,7 +46,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d *Affecting all Beats* -- Added the possibility to set Elasticsearch mapping template settings from the Beat configuration file. {pull}4284[4284] +- Added the possibility to set Elasticsearch mapping template settings from the Beat configuration file. {pull}4284[4284] {pull}4317[4317] *Filebeat* diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 51b6838df459..2be4e7b2f3ed 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -487,12 +487,14 @@ func (b *Beat) registerTemplateLoading() error { loader, err := template.NewLoader(b.Config.Template, esClient, b.Info) if err != nil { - return fmt.Errorf("Error loading elasticsearch template: %v", err) + return fmt.Errorf("Error creating Elasticsearch template: %v", err) } - loader.Load() + err = loader.Load() + if err != nil { + return fmt.Errorf("Error loading Elasticsearch template: %v", err) + } - logp.Info("ES template successfully loaded.") return nil } diff --git a/libbeat/docs/template-config.asciidoc b/libbeat/docs/template-config.asciidoc index 3c758c56e15a..a27b2fd9ebb6 100644 --- a/libbeat/docs/template-config.asciidoc +++ b/libbeat/docs/template-config.asciidoc @@ -20,11 +20,11 @@ section for details. *`overwrite`*:: A boolean that specifies whether to overwrite the existing template. The default is false. -*`settings`*:: A dictionary of settings to place into the `settings` dictionary of the Elasticsearch -template. For more details about the available Elasticsearch mapping options, please see the -Elasticsearch {elasticsearch}/mapping.html[mapping reference]. +*`settings.index`*:: A dictionary of settings to place into the `settings.index` dictionary of the +Elasticsearch template. For more details about the available Elasticsearch mapping options, please +see the Elasticsearch {elasticsearch}/mapping.html[mapping reference]. -For example: +Example: ["source","yaml",subs="attributes,callouts"] ---------------------------------------------------------------------- @@ -35,3 +35,17 @@ setup.template.settings: index.number_of_shards: 1 index.number_of_replicas: 1 ---------------------------------------------------------------------- + +*`settings._source`*:: A dictionary of settings for the `_source` field. For the available settings, +please see the Elasticsearch {elasticsearch}/mapping-source-field.html[reference]. + +Example: + +["source","yaml",subs="attributes,callouts"] +---------------------------------------------------------------------- +setup.template.name: "{beatname_lc}" +setup.template.fields: "fields.yml" +setup.template.overwrite: false +setup.template.settings: + _source.enabled: false +---------------------------------------------------------------------- diff --git a/libbeat/template/config.go b/libbeat/template/config.go index fe6185972e27..ea5d55285eac 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -1,12 +1,17 @@ package template type TemplateConfig struct { - Enabled bool `config:"enabled"` - Name string `config:"name"` - Fields string `config:"fields"` - Overwrite bool `config:"overwrite"` - OutputToFile string `config:"output_to_file"` - Settings map[string]interface{} `config:"settings"` + Enabled bool `config:"enabled"` + Name string `config:"name"` + Fields string `config:"fields"` + Overwrite bool `config:"overwrite"` + OutputToFile string `config:"output_to_file"` + Settings templateSettings `config:"settings"` +} + +type templateSettings struct { + Index map[string]interface{} `config:"index"` + Source map[string]interface{} `config:"_source"` } var ( diff --git a/libbeat/template/load.go b/libbeat/template/load.go index b1b49b9c0b53..d319e006dd1b 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -43,25 +43,25 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo common.BeatInfo) (* // template is written to index func (l *Loader) Load() error { + if l.config.Name == "" { + l.config.Name = l.beatInfo.Beat + } + + tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings) + if err != nil { + return fmt.Errorf("error creating template instance: %v", err) + } + // Check if template already exist or should be overwritten - exists := l.CheckTemplate(l.config.Name) + exists := l.CheckTemplate(tmpl.GetName()) if !exists || l.config.Overwrite { - logp.Info("Loading template for elasticsearch version: %s", l.client.GetVersion()) + logp.Info("Loading template for Elasticsearch version: %s", l.client.GetVersion()) if l.config.Overwrite { logp.Info("Existing template will be overwritten, as overwrite is enabled.") } - if l.config.Name == "" { - l.config.Name = l.beatInfo.Beat - } - - tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings) - if err != nil { - return fmt.Errorf("error creating template instance: %v", err) - } - fieldsPath := paths.Resolve(paths.Config, l.config.Fields) output, err := tmpl.Load(fieldsPath) diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go index 579f1d912a8b..7dc65783b3b9 100644 --- a/libbeat/template/load_integration_test.go +++ b/libbeat/template/load_integration_test.go @@ -4,7 +4,6 @@ package template import ( "encoding/json" - "fmt" "path/filepath" "testing" "time" @@ -46,7 +45,7 @@ func TestLoadTemplate(t *testing.T) { fieldsPath := absPath + "/fields.yml" index := "testbeat" - tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{}) + tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, templateSettings{}) assert.NoError(t, err) content, err := tmpl.Load(fieldsPath) assert.NoError(t, err) @@ -95,6 +94,25 @@ func TestLoadInvalidTemplate(t *testing.T) { assert.False(t, loader.CheckTemplate(templateName)) } +func getTemplate(t *testing.T, client ESClient, templateName string) common.MapStr { + + status, body, err := client.Request("GET", "/_template/"+templateName, "", nil, nil) + assert.NoError(t, err) + assert.Equal(t, status, 200) + + var response common.MapStr + err = json.Unmarshal(body, &response) + assert.NoError(t, err) + + return common.MapStr(response[templateName].(map[string]interface{})) +} + +func newConfigFrom(t *testing.T, from interface{}) *common.Config { + cfg, err := common.NewConfigFrom(from) + assert.NoError(t, err) + return cfg +} + // Tests loading the templates for each beat func TestLoadBeatsTemplate(t *testing.T) { @@ -117,7 +135,7 @@ func TestLoadBeatsTemplate(t *testing.T) { fieldsPath := absPath + "/fields.yml" index := beat - tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{}) + tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, templateSettings{}) assert.NoError(t, err) content, err := tmpl.Load(fieldsPath) assert.NoError(t, err) @@ -155,10 +173,13 @@ func TestTemplateSettings(t *testing.T) { fieldsPath := absPath + "/fields.yml" - settings := common.MapStr{ - "index": common.MapStr{ + settings := templateSettings{ + Index: common.MapStr{ "number_of_shards": 1, }, + Source: common.MapStr{ + "enabled": false, + }, } tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), "testbeat", settings) assert.NoError(t, err) @@ -174,18 +195,14 @@ func TestTemplateSettings(t *testing.T) { assert.Nil(t, err) // Check that it contains the mapping - status, body, err := loader.client.Request("GET", "/_template/"+tmpl.GetName(), "", nil, nil) - assert.NoError(t, err) - assert.Equal(t, status, 200) - - var response common.MapStr - err = json.Unmarshal(body, &response) + templateJSON := getTemplate(t, client, tmpl.GetName()) + val, err := templateJSON.GetValue("settings.index.number_of_shards") assert.NoError(t, err) + assert.Equal(t, val.(string), "1") - templateJSON := common.MapStr(response[tmpl.GetName()].(map[string]interface{})) - val, err := templateJSON.GetValue(fmt.Sprintf("settings.index.number_of_shards")) + val, err = templateJSON.GetValue("mappings._default_._source.enabled") assert.NoError(t, err) - assert.Equal(t, val.(string), "1") + assert.Equal(t, val.(bool), false) // Delete template again to clean up client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil) @@ -193,3 +210,79 @@ func TestTemplateSettings(t *testing.T) { // Make sure it was removed assert.False(t, loader.CheckTemplate(tmpl.GetName())) } + +func TestOverwrite(t *testing.T) { + + // Setup ES + client := elasticsearch.GetTestingElasticsearch() + err := client.Connect(5 * time.Second) + assert.Nil(t, err) + + beatInfo := common.BeatInfo{ + Beat: "testbeat", + Version: version.GetDefaultVersion(), + } + templateName := "testbeat-" + version.GetDefaultVersion() + + absPath, err := filepath.Abs("../") + assert.NotNil(t, absPath) + assert.Nil(t, err) + + // make sure no template is already there + client.Request("DELETE", "/_template/"+templateName, "", nil, nil) + + // Load template + config := newConfigFrom(t, TemplateConfig{ + Enabled: true, + Fields: absPath + "/fields.yml", + }) + loader, err := NewLoader(config, client, beatInfo) + assert.NoError(t, err) + err = loader.Load() + assert.NoError(t, err) + + // Load template again, this time with custom settings + config = newConfigFrom(t, TemplateConfig{ + Enabled: true, + Fields: absPath + "/fields.yml", + Settings: templateSettings{ + Source: map[string]interface{}{ + "enabled": false, + }, + }, + }) + loader, err = NewLoader(config, client, beatInfo) + assert.NoError(t, err) + err = loader.Load() + assert.NoError(t, err) + + // Overwrite was not enabled, so the first version should still be there + templateJSON := getTemplate(t, client, templateName) + _, err = templateJSON.GetValue("mappings._default_._source.enabled") + assert.Error(t, err) + + // Load template again, this time with custom settings AND overwrite: true + config = newConfigFrom(t, TemplateConfig{ + Enabled: true, + Overwrite: true, + Fields: absPath + "/fields.yml", + Settings: templateSettings{ + Source: map[string]interface{}{ + "enabled": false, + }, + }, + }) + loader, err = NewLoader(config, client, beatInfo) + assert.NoError(t, err) + err = loader.Load() + assert.NoError(t, err) + + // Overwrite was enabled, so the custom setting should be there + templateJSON = getTemplate(t, client, templateName) + val, err := templateJSON.GetValue("mappings._default_._source.enabled") + assert.NoError(t, err) + assert.Equal(t, val.(bool), false) + + // Delete template again to clean up + client.Request("DELETE", "/_template/"+templateName, "", nil, nil) +} diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 602baf40f8f2..f848bedb3afe 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -20,11 +20,11 @@ type Template struct { index string beatVersion Version esVersion Version - settings common.MapStr + settings templateSettings } // New creates a new template instance -func New(beatVersion string, esVersion string, index string, settings common.MapStr) (*Template, error) { +func New(beatVersion string, esVersion string, index string, settings templateSettings) (*Template, error) { bV, err := NewVersion(beatVersion) if err != nil { @@ -92,17 +92,15 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. dynamicTemplates = append(dynamicTemplates, dynamicTemplateBase) - settings := common.MapStr{ - "index": common.MapStr{ - "refresh_interval": "5s", - "mapping": common.MapStr{ - "total_fields": common.MapStr{ - "limit": defaultTotalFieldsLimit, - }, + indexSettings := common.MapStr{ + "refresh_interval": "5s", + "mapping": common.MapStr{ + "total_fields": common.MapStr{ + "limit": defaultTotalFieldsLimit, }, }, } - settings.DeepUpdate(t.settings) + indexSettings.DeepUpdate(t.settings.Index) // Load basic structure basicStructure := common.MapStr{ @@ -116,8 +114,14 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. "properties": properties, }, }, - "order": 1, - "settings": settings, + "order": 1, + "settings": common.MapStr{ + "index": indexSettings, + }, + } + + if len(t.settings.Source) > 0 { + basicStructure.Put("mappings._default_._source", t.settings.Source) } // ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009