Add option to adjust _source options + improvements

CHANGELOG.asciidoc

@@ -46,7 +46,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d
 
 *Affecting all Beats*
 
-- Added the possibility to set Elasticsearch mapping template settings from the Beat configuration file. {pull}4284[4284]
+- Added the possibility to set Elasticsearch mapping template settings from the Beat configuration file. {pull}4284[4284] {pull}4317[4317]
 
 *Filebeat*
 

libbeat/beat/beat.go

@@ -487,12 +487,14 @@ func (b *Beat) registerTemplateLoading() error {
 
 			loader, err := template.NewLoader(b.Config.Template, esClient, b.Info)
 			if err != nil {
-				return fmt.Errorf("Error loading elasticsearch template: %v", err)
+				return fmt.Errorf("Error creating Elasticsearch template: %v", err)
 			}
 
-			loader.Load()
+			err = loader.Load()
+			if err != nil {
+				return fmt.Errorf("Error loading Elasticsearch template: %v", err)
+			}
 
-			logp.Info("ES template successfully loaded.")
 			return nil
 		}
 

libbeat/docs/template-config.asciidoc

@@ -20,11 +20,11 @@ section for details.
 *`overwrite`*:: A boolean that specifies whether to overwrite the existing template. The default
 is false.
 
-*`settings`*:: A dictionary of settings to place into the `settings` dictionary of the Elasticsearch
-template. For more details about the available Elasticsearch mapping options, please see the
-Elasticsearch {elasticsearch}/mapping.html[mapping reference].
+*`settings.index`*:: A dictionary of settings to place into the `settings.index` dictionary of the
+Elasticsearch template. For more details about the available Elasticsearch mapping options, please
+see the Elasticsearch {elasticsearch}/mapping.html[mapping reference].
 
-For example:
+Example:
 
 ["source","yaml",subs="attributes,callouts"]
 ----------------------------------------------------------------------
@@ -35,3 +35,17 @@ setup.template.settings:
   index.number_of_shards: 1
   index.number_of_replicas: 1
 ----------------------------------------------------------------------
+
+*`settings._source`*:: A dictionary of settings for the `_source` field. For the available settings,
+please see the Elasticsearch {elasticsearch}/mapping-source-field.html[reference].
+
+Example:
+
+["source","yaml",subs="attributes,callouts"]
+----------------------------------------------------------------------
+setup.template.name: "{beatname_lc}"
+setup.template.fields: "fields.yml"
+setup.template.overwrite: false
+setup.template.settings:
+  _source.enabled: false
+----------------------------------------------------------------------

libbeat/template/config.go

@@ -1,12 +1,17 @@
 package template
 
 type TemplateConfig struct {
-	Enabled      bool                   `config:"enabled"`
-	Name         string                 `config:"name"`
-	Fields       string                 `config:"fields"`
-	Overwrite    bool                   `config:"overwrite"`
-	OutputToFile string                 `config:"output_to_file"`
-	Settings     map[string]interface{} `config:"settings"`
+	Enabled      bool             `config:"enabled"`
+	Name         string           `config:"name"`
+	Fields       string           `config:"fields"`
+	Overwrite    bool             `config:"overwrite"`
+	OutputToFile string           `config:"output_to_file"`
+	Settings     templateSettings `config:"settings"`
+}
+
+type templateSettings struct {
+	Index  map[string]interface{} `config:"index"`
+	Source map[string]interface{} `config:"_source"`
 }
 
 var (

libbeat/template/load.go

@@ -43,25 +43,25 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo common.BeatInfo) (*
 // template is written to index
 func (l *Loader) Load() error {
 
+	if l.config.Name == "" {
+		l.config.Name = l.beatInfo.Beat
+	}
+
+	tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings)
+	if err != nil {
+		return fmt.Errorf("error creating template instance: %v", err)
+	}
+
 	// Check if template already exist or should be overwritten
-	exists := l.CheckTemplate(l.config.Name)
+	exists := l.CheckTemplate(tmpl.GetName())
 	if !exists || l.config.Overwrite {
 
-		logp.Info("Loading template for elasticsearch version: %s", l.client.GetVersion())
+		logp.Info("Loading template for Elasticsearch version: %s", l.client.GetVersion())
 
 		if l.config.Overwrite {
 			logp.Info("Existing template will be overwritten, as overwrite is enabled.")
 		}
 
-		if l.config.Name == "" {
-			l.config.Name = l.beatInfo.Beat
-		}
-
-		tmpl, err := New(l.beatInfo.Version, l.client.GetVersion(), l.config.Name, l.config.Settings)
-		if err != nil {
-			return fmt.Errorf("error creating template instance: %v", err)
-		}
-
 		fieldsPath := paths.Resolve(paths.Config, l.config.Fields)
 
 		output, err := tmpl.Load(fieldsPath)

libbeat/template/load_integration_test.go

@@ -4,7 +4,6 @@ package template
 
 import (
 	"encoding/json"
-	"fmt"
 	"path/filepath"
 	"testing"
 	"time"
@@ -46,7 +45,7 @@ func TestLoadTemplate(t *testing.T) {
 	fieldsPath := absPath + "/fields.yml"
 	index := "testbeat"
 
-	tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{})
+	tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, templateSettings{})
 	assert.NoError(t, err)
 	content, err := tmpl.Load(fieldsPath)
 	assert.NoError(t, err)
@@ -95,6 +94,25 @@ func TestLoadInvalidTemplate(t *testing.T) {
 	assert.False(t, loader.CheckTemplate(templateName))
 }
 
+func getTemplate(t *testing.T, client ESClient, templateName string) common.MapStr {
+
+	status, body, err := client.Request("GET", "/_template/"+templateName, "", nil, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, status, 200)
+
+	var response common.MapStr
+	err = json.Unmarshal(body, &response)
+	assert.NoError(t, err)
+
+	return common.MapStr(response[templateName].(map[string]interface{}))
+}
+
+func newConfigFrom(t *testing.T, from interface{}) *common.Config {
+	cfg, err := common.NewConfigFrom(from)
+	assert.NoError(t, err)
+	return cfg
+}
+
 // Tests loading the templates for each beat
 func TestLoadBeatsTemplate(t *testing.T) {
 
@@ -117,7 +135,7 @@ func TestLoadBeatsTemplate(t *testing.T) {
 		fieldsPath := absPath + "/fields.yml"
 		index := beat
 
-		tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, common.MapStr{})
+		tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), index, templateSettings{})
 		assert.NoError(t, err)
 		content, err := tmpl.Load(fieldsPath)
 		assert.NoError(t, err)
@@ -155,10 +173,13 @@ func TestTemplateSettings(t *testing.T) {
 
 	fieldsPath := absPath + "/fields.yml"
 
-	settings := common.MapStr{
-		"index": common.MapStr{
+	settings := templateSettings{
+		Index: common.MapStr{
 			"number_of_shards": 1,
 		},
+		Source: common.MapStr{
+			"enabled": false,
+		},
 	}
 	tmpl, err := New(version.GetDefaultVersion(), client.GetVersion(), "testbeat", settings)
 	assert.NoError(t, err)
@@ -174,22 +195,94 @@ func TestTemplateSettings(t *testing.T) {
 	assert.Nil(t, err)
 
 	// Check that it contains the mapping
-	status, body, err := loader.client.Request("GET", "/_template/"+tmpl.GetName(), "", nil, nil)
-	assert.NoError(t, err)
-	assert.Equal(t, status, 200)
-
-	var response common.MapStr
-	err = json.Unmarshal(body, &response)
+	templateJSON := getTemplate(t, client, tmpl.GetName())
+	val, err := templateJSON.GetValue("settings.index.number_of_shards")
 	assert.NoError(t, err)
+	assert.Equal(t, val.(string), "1")
 
-	templateJSON := common.MapStr(response[tmpl.GetName()].(map[string]interface{}))
-	val, err := templateJSON.GetValue(fmt.Sprintf("settings.index.number_of_shards"))
+	val, err = templateJSON.GetValue("mappings._default_._source.enabled")
 	assert.NoError(t, err)
-	assert.Equal(t, val.(string), "1")
+	assert.Equal(t, val.(bool), false)
 
 	// Delete template again to clean up
 	client.Request("DELETE", "/_template/"+tmpl.GetName(), "", nil, nil)
 
 	// Make sure it was removed
 	assert.False(t, loader.CheckTemplate(tmpl.GetName()))
 }
+
+func TestOverwrite(t *testing.T) {
+
+	// Setup ES
+	client := elasticsearch.GetTestingElasticsearch()
+	err := client.Connect(5 * time.Second)
+	assert.Nil(t, err)
+
+	beatInfo := common.BeatInfo{
+		Beat:    "testbeat",
+		Version: version.GetDefaultVersion(),
+	}
+	templateName := "testbeat-" + version.GetDefaultVersion()
+
+	absPath, err := filepath.Abs("../")
+	assert.NotNil(t, absPath)
+	assert.Nil(t, err)
+
+	// make sure no template is already there
+	client.Request("DELETE", "/_template/"+templateName, "", nil, nil)
+
+	// Load template
+	config := newConfigFrom(t, TemplateConfig{
+		Enabled: true,
+		Fields:  absPath + "/fields.yml",
+	})
+	loader, err := NewLoader(config, client, beatInfo)
+	assert.NoError(t, err)
+	err = loader.Load()
+	assert.NoError(t, err)
+
+	// Load template again, this time with custom settings
+	config = newConfigFrom(t, TemplateConfig{
+		Enabled: true,
+		Fields:  absPath + "/fields.yml",
+		Settings: templateSettings{
+			Source: map[string]interface{}{
+				"enabled": false,
+			},
+		},
+	})
+	loader, err = NewLoader(config, client, beatInfo)
+	assert.NoError(t, err)
+	err = loader.Load()
+	assert.NoError(t, err)
+
+	// Overwrite was not enabled, so the first version should still be there
+	templateJSON := getTemplate(t, client, templateName)
+	_, err = templateJSON.GetValue("mappings._default_._source.enabled")
+	assert.Error(t, err)
+
+	// Load template again, this time with custom settings AND overwrite: true
+	config = newConfigFrom(t, TemplateConfig{
+		Enabled:   true,
+		Overwrite: true,
+		Fields:    absPath + "/fields.yml",
+		Settings: templateSettings{
+			Source: map[string]interface{}{
+				"enabled": false,
+			},
+		},
+	})
+	loader, err = NewLoader(config, client, beatInfo)
+	assert.NoError(t, err)
+	err = loader.Load()
+	assert.NoError(t, err)
+
+	// Overwrite was enabled, so the custom setting should be there
+	templateJSON = getTemplate(t, client, templateName)
+	val, err := templateJSON.GetValue("mappings._default_._source.enabled")
+	assert.NoError(t, err)
+	assert.Equal(t, val.(bool), false)
+
+	// Delete template again to clean up
+	client.Request("DELETE", "/_template/"+templateName, "", nil, nil)
+}

libbeat/template/template.go

@@ -20,11 +20,11 @@ type Template struct {
 	index       string
 	beatVersion Version
 	esVersion   Version
-	settings    common.MapStr
+	settings    templateSettings
 }
 
 // New creates a new template instance
-func New(beatVersion string, esVersion string, index string, settings common.MapStr) (*Template, error) {
+func New(beatVersion string, esVersion string, index string, settings templateSettings) (*Template, error) {
 
 	bV, err := NewVersion(beatVersion)
 	if err != nil {
@@ -92,17 +92,15 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 
 	dynamicTemplates = append(dynamicTemplates, dynamicTemplateBase)
 
-	settings := common.MapStr{
-		"index": common.MapStr{
-			"refresh_interval": "5s",
-			"mapping": common.MapStr{
-				"total_fields": common.MapStr{
-					"limit": defaultTotalFieldsLimit,
-				},
+	indexSettings := common.MapStr{
+		"refresh_interval": "5s",
+		"mapping": common.MapStr{
+			"total_fields": common.MapStr{
+				"limit": defaultTotalFieldsLimit,
 			},
 		},
 	}
-	settings.DeepUpdate(t.settings)
+	indexSettings.DeepUpdate(t.settings.Index)
 
 	// Load basic structure
 	basicStructure := common.MapStr{
@@ -116,8 +114,14 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 				"properties":        properties,
 			},
 		},
-		"order":    1,
-		"settings": settings,
+		"order": 1,
+		"settings": common.MapStr{
+			"index": indexSettings,
+		},
+	}
+
+	if len(t.settings.Source) > 0 {
+		basicStructure.Put("mappings._default_._source", t.settings.Source)
 	}
 
 	// ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009