From dabc10ce4b4200d8f1a7d3030b192f3baa103869 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Tue, 16 May 2017 11:48:24 +0200 Subject: [PATCH] Add the option to generate the template into a file Part of #3654. --- libbeat/beat/beat.go | 26 +++++++- libbeat/template/config.go | 9 ++- libbeat/template/load.go | 37 +++++++++++ libbeat/template/load_test.go | 112 ++++++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 libbeat/template/load_test.go diff --git a/libbeat/beat/beat.go b/libbeat/beat/beat.go index 2be4e7b2f3e..e9d440aed12 100644 --- a/libbeat/beat/beat.go +++ b/libbeat/beat/beat.go @@ -469,8 +469,30 @@ func (b *Beat) registerTemplateLoading() error { } } - esConfig := b.Config.Output["elasticsearch"] + // Check if outputting to file is enabled, and output to file if it is + if b.Config.Template != nil && b.Config.Template.Enabled() { + var cfg template.TemplateConfig + err := b.Config.Template.Unpack(&cfg) + if err != nil { + return fmt.Errorf("unpacking template config fails: %v", err) + } + if len(cfg.OutputToFile.Path) > 0 { + // output to file is enabled + loader, err := template.NewLoader(b.Config.Template, nil, b.Info) + if err != nil { + return fmt.Errorf("Error creating Elasticsearch template loader: %v", err) + } + err = loader.Generate() + if err != nil { + return fmt.Errorf("Error generating template: %v", err) + } + // XXX: Should we kill the Beat here or just continue? + return fmt.Errorf("Stopping after successfully writing the template to the file.") + } + } + + esConfig := b.Config.Output["elasticsearch"] // Loads template by default if esOutput is enabled if (b.Config.Template == nil && esConfig.Enabled()) || (b.Config.Template != nil && b.Config.Template.Enabled()) { if esConfig == nil || !esConfig.Enabled() { @@ -487,7 +509,7 @@ func (b *Beat) registerTemplateLoading() error { loader, err := template.NewLoader(b.Config.Template, esClient, b.Info) if err != nil { - return fmt.Errorf("Error creating Elasticsearch template: %v", err) + return fmt.Errorf("Error creating Elasticsearch template loader: %v", err) } err = loader.Load() diff --git a/libbeat/template/config.go b/libbeat/template/config.go index ea5d55285ea..f5aeb3a5d5e 100644 --- a/libbeat/template/config.go +++ b/libbeat/template/config.go @@ -5,8 +5,15 @@ type TemplateConfig struct { Name string `config:"name"` Fields string `config:"fields"` Overwrite bool `config:"overwrite"` - OutputToFile string `config:"output_to_file"` Settings templateSettings `config:"settings"` + OutputToFile OutputToFile `config:"output_to_file"` +} + +// OutputToFile contains the configuration options for generating +// and writing the template into a file. +type OutputToFile struct { + Path string `config:"path"` + Version string `config:"version"` } type templateSettings struct { diff --git a/libbeat/template/load.go b/libbeat/template/load.go index d319e006dd1..c3a8bb268bf 100644 --- a/libbeat/template/load.go +++ b/libbeat/template/load.go @@ -1,7 +1,9 @@ package template import ( + "encoding/json" "fmt" + "io/ioutil" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" @@ -80,6 +82,41 @@ func (l *Loader) Load() error { return nil } +func (l *Loader) Generate() error { + if l.config.OutputToFile.Version == "" { + l.config.OutputToFile.Version = l.beatInfo.Version + } + + if l.config.Name == "" { + l.config.Name = l.beatInfo.Beat + } + + tmpl, err := New(l.beatInfo.Version, l.config.OutputToFile.Version, l.config.Name, l.config.Settings) + if err != nil { + return fmt.Errorf("error creating template instance: %v", err) + } + + fieldsPath := paths.Resolve(paths.Config, l.config.Fields) + + output, err := tmpl.Load(fieldsPath) + if err != nil { + return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err) + } + + jsonBytes, err := json.MarshalIndent(output, "", " ") + if err != nil { + return fmt.Errorf("error marshaling template: %v", err) + } + + err = ioutil.WriteFile(l.config.OutputToFile.Path, jsonBytes, 0644) + if err != nil { + return fmt.Errorf("error writing to file %s: %v", l.config.OutputToFile.Path, err) + } + + logp.Info("Template for Elasticsearch %s written to: %s", l.config.OutputToFile.Version, l.config.OutputToFile.Path) + return nil +} + // LoadTemplate loads a template into Elasticsearch overwriting the existing // template if it exists. If you wish to not overwrite an existing template // then use CheckTemplate prior to calling this method. diff --git a/libbeat/template/load_test.go b/libbeat/template/load_test.go new file mode 100644 index 00000000000..1fcf83b6f9d --- /dev/null +++ b/libbeat/template/load_test.go @@ -0,0 +1,112 @@ +// +build !integration + +package template + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/version" + "github.com/stretchr/testify/assert" +) + +func TestGenerateTemplate(t *testing.T) { + + // Load template + absPath, err := filepath.Abs("../") + assert.NotNil(t, absPath) + assert.Nil(t, err) + + beatInfo := common.BeatInfo{ + Beat: "testbeat", + Version: version.GetDefaultVersion(), + } + + dir, err := ioutil.TempDir("", "test-template") + assert.NoError(t, err) + defer os.RemoveAll(dir) + path := filepath.Join(dir, "template.json") + + config := newConfigFrom(t, TemplateConfig{ + Enabled: true, + Fields: absPath + "/fields.yml", + OutputToFile: OutputToFile{ + Path: path, + }, + }) + + loader, err := NewLoader(config, nil, beatInfo) + assert.NoError(t, err) + + err = loader.Generate() + assert.NoError(t, err) + + // Read it back to check it + fp, err := os.Open(path) + assert.NoError(t, err) + jsonParser := json.NewDecoder(fp) + var parsed common.MapStr + err = jsonParser.Decode(&parsed) + assert.NoError(t, err) + + val, err := parsed.GetValue("mappings._default_._meta.version") + assert.NoError(t, err) + assert.Equal(t, val.(string), version.GetDefaultVersion()) + +} + +func TestGenerateTemplateWithVersion(t *testing.T) { + + // Load template + absPath, err := filepath.Abs("../") + assert.NotNil(t, absPath) + assert.Nil(t, err) + + beatInfo := common.BeatInfo{ + Beat: "testbeat", + Version: version.GetDefaultVersion(), + } + + dir, err := ioutil.TempDir("", "test-template") + assert.NoError(t, err) + defer os.RemoveAll(dir) + path := filepath.Join(dir, "template.json") + + config := newConfigFrom(t, TemplateConfig{ + Enabled: true, + Fields: absPath + "/fields.yml", + OutputToFile: OutputToFile{ + Path: path, + Version: "2.4.0", + }, + }) + + loader, err := NewLoader(config, nil, beatInfo) + assert.NoError(t, err) + + err = loader.Generate() + assert.NoError(t, err) + + // Read it back to check it + fp, err := os.Open(path) + assert.NoError(t, err) + jsonParser := json.NewDecoder(fp) + var parsed common.MapStr + err = jsonParser.Decode(&parsed) + assert.NoError(t, err) + + // check a setting specific to that version + val, err := parsed.GetValue("mappings._default_._all.norms.enabled") + assert.NoError(t, err) + assert.Equal(t, val.(bool), false) +} + +func newConfigFrom(t *testing.T, from interface{}) *common.Config { + cfg, err := common.NewConfigFrom(from) + assert.NoError(t, err) + return cfg +}