Add migration.enabled config option (#9805)

The migration.enabled config option is off by default. Enabling it will add all the alias fields which are used for making Beats 7.x data compatible with 6.x data for queries and dashboards. A user that upgrades can either set `migration.enabled: true` in the config file or run `-E migration.enabled=true` on the command line to also have the migration aliases in the Elasticsearch index template. Further changes: * Add system test to check the generated template * Fix autodiscovery tests. Were broken as probably the removal of the aliases by default did not work here as it relied on apache2.
elastic · Jan 21, 2019 · 4ee3a64 · 4ee3a64
1 parent b292ba2
commit 4ee3a64
Show file tree

Hide file tree

Showing 36 changed files with 256 additions and 26 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -129,6 +129,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `name` config option to `add_host_metadata` processor. {pull}9943[9943]
 - Add `add_labels` and `add_tags` processors. {pull}9973[9973]
 - Add missing file encoding to readers. {pull}10080[10080]
+- Introduce `migration.enabled` configuration. {pull}9805[9805]
 - Add alias field support in Kibana index pattern. {pull}10075[10075]
 
 *Auditbeat*

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -1246,3 +1246,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml
@@ -177,3 +177,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -1941,3 +1941,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
@@ -205,3 +205,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -1390,3 +1390,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml
@@ -162,3 +162,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml
@@ -1180,3 +1180,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/journalbeat/journalbeat.yml b/journalbeat/journalbeat.yml
@@ -168,3 +168,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml
@@ -1134,3 +1134,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/libbeat/_meta/config.yml b/libbeat/_meta/config.yml
@@ -122,3 +122,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false
diff --git a/libbeat/cmd/export/template.go b/libbeat/cmd/export/template.go
@@ -66,7 +66,7 @@ func GenTemplateConfigCmd(settings instance.Settings, name, idxPrefix, beatVersi
 				fmt.Fprintf(os.Stderr, "Invalid Elasticsearch version: %s\n", err)
 			}
 
-			tmpl, err := template.New(b.Info.Version, index, *esVersion, cfg)
+			tmpl, err := template.New(b.Info.Version, index, *esVersion, cfg, b.Config.Migration.Enabled())
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Error generating template: %+v", err)
 				os.Exit(1)

diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go
@@ -63,7 +63,7 @@ import (
 	svc "github.com/elastic/beats/libbeat/service"
 	"github.com/elastic/beats/libbeat/template"
 	"github.com/elastic/beats/libbeat/version"
-	"github.com/elastic/go-sysinfo"
+	sysinfo "github.com/elastic/go-sysinfo"
 	"github.com/elastic/go-sysinfo/types"
 	ucfg "github.com/elastic/go-ucfg"
 )
@@ -105,6 +105,7 @@ type beatConfig struct {
 	Dashboards *common.Config `config:"setup.dashboards"`
 	Template   *common.Config `config:"setup.template"`
 	Kibana     *common.Config `config:"setup.kibana"`
+	Migration  *common.Config `config:"migration"`
 
 	// ILM Config options
 	ILM *common.Config `config:"output.elasticsearch.ilm"`
@@ -877,7 +878,7 @@ func (b *Beat) templateLoadingCallback() (func(esClient *elasticsearch.Client) e
 			b.Config.Template = common.NewConfig()
 		}
 
-		loader, err := template.NewLoader(b.Config.Template, esClient, b.Info, b.Fields)
+		loader, err := template.NewLoader(b.Config.Template, esClient, b.Info, b.Fields, b.Config.Migration.Enabled())
 		if err != nil {
 			return fmt.Errorf("Error creating Elasticsearch template loader: %v", err)
 		}

diff --git a/libbeat/common/field.go b/libbeat/common/field.go
@@ -52,6 +52,7 @@ type Field struct {
 	CopyTo         string      `config:"copy_to"`
 	IgnoreAbove    int         `config:"ignore_above"`
 	AliasPath      string      `config:"path"`
+	MigrationAlias bool        `config:"migration"`
 
 	ObjectType            string          `config:"object_type"`
 	ObjectTypeMappingType string          `config:"object_type_mapping_type"`
@@ -64,6 +65,7 @@ type Field struct {
 	Searchable   *bool  `config:"searchable"`
 	Aggregatable *bool  `config:"aggregatable"`
 	Script       string `config:"script"`
+
 	// Kibana params
 	Pattern              string              `config:"pattern"`
 	InputFormat          string              `config:"input_format"`

diff --git a/libbeat/template/load.go b/libbeat/template/load.go
@@ -38,14 +38,15 @@ type ESClient interface {
 }
 
 type Loader struct {
-	config   TemplateConfig
-	client   ESClient
-	beatInfo beat.Info
-	fields   []byte
+	config    TemplateConfig
+	client    ESClient
+	beatInfo  beat.Info
+	fields    []byte
+	migration bool
 }
 
 // NewLoader creates a new template loader
-func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields []byte) (*Loader, error) {
+func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields []byte, migration bool) (*Loader, error) {
 	config := DefaultConfig
 
 	err := cfg.Unpack(&config)
@@ -54,10 +55,11 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields [
 	}
 
 	return &Loader{
-		config:   config,
-		client:   client,
-		beatInfo: beatInfo,
-		fields:   fields,
+		config:    config,
+		client:    client,
+		beatInfo:  beatInfo,
+		fields:    fields,
+		migration: migration,
 	}, nil
 }
 
@@ -66,7 +68,7 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo beat.Info, fields [
 // template is written to index
 func (l *Loader) Load() error {
 
-	tmpl, err := New(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.client.GetVersion(), l.config)
+	tmpl, err := New(l.beatInfo.Version, l.beatInfo.IndexPrefix, l.client.GetVersion(), l.config, l.migration)
 	if err != nil {
 		return fmt.Errorf("error creating template instance: %v", err)
 	}

diff --git a/libbeat/template/load_integration_test.go b/libbeat/template/load_integration_test.go
@@ -70,7 +70,7 @@ func TestLoadTemplate(t *testing.T) {
 	fieldsPath := absPath + "/fields.yml"
 	index := "testbeat"
 
-	tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{})
+	tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}, false)
 	assert.NoError(t, err)
 	content, err := tmpl.LoadFile(fieldsPath)
 	assert.NoError(t, err)
@@ -140,7 +140,7 @@ func TestLoadBeatsTemplate(t *testing.T) {
 		fieldsPath := absPath + "/fields.yml"
 		index := beat
 
-		tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{})
+		tmpl, err := New(version.GetDefaultVersion(), index, client.GetVersion(), TemplateConfig{}, false)
 		assert.NoError(t, err)
 		content, err := tmpl.LoadFile(fieldsPath)
 		assert.NoError(t, err)
@@ -189,7 +189,7 @@ func TestTemplateSettings(t *testing.T) {
 	config := TemplateConfig{
 		Settings: settings,
 	}
-	tmpl, err := New(version.GetDefaultVersion(), "testbeat", client.GetVersion(), config)
+	tmpl, err := New(version.GetDefaultVersion(), "testbeat", client.GetVersion(), config, false)
 	assert.NoError(t, err)
 	content, err := tmpl.LoadFile(fieldsPath)
 	assert.NoError(t, err)
@@ -240,7 +240,7 @@ func TestOverwrite(t *testing.T) {
 		Enabled: true,
 		Fields:  absPath + "/fields.yml",
 	})
-	loader, err := NewLoader(config, client, beatInfo, nil)
+	loader, err := NewLoader(config, client, beatInfo, nil, false)
 	assert.NoError(t, err)
 	err = loader.Load()
 	assert.NoError(t, err)
@@ -255,7 +255,7 @@ func TestOverwrite(t *testing.T) {
 			},
 		},
 	})
-	loader, err = NewLoader(config, client, beatInfo, nil)
+	loader, err = NewLoader(config, client, beatInfo, nil, false)
 	assert.NoError(t, err)
 	err = loader.Load()
 	assert.NoError(t, err)
@@ -275,7 +275,7 @@ func TestOverwrite(t *testing.T) {
 			},
 		},
 	})
-	loader, err = NewLoader(config, client, beatInfo, nil)
+	loader, err = NewLoader(config, client, beatInfo, nil, false)
 	assert.NoError(t, err)
 	err = loader.Load()
 	assert.NoError(t, err)
@@ -339,7 +339,7 @@ func TestTemplateWithData(t *testing.T) {
 	// Setup ES
 	client := estest.GetTestingElasticsearch(t)
 
-	tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplateConfig{})
+	tmpl, err := New(version.GetDefaultVersion(), "testindex", client.GetVersion(), TemplateConfig{}, false)
 	assert.NoError(t, err)
 	content, err := tmpl.LoadFile(fieldsPath)
 	assert.NoError(t, err)

diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go
@@ -27,6 +27,7 @@ import (
 // Processor struct to process fields to template
 type Processor struct {
 	EsVersion common.Version
+	Migration bool
 }
 
 var (
@@ -264,9 +265,14 @@ func (p *Processor) alias(f *common.Field) common.MapStr {
 		return nil
 	}
 
+	// In case migration is disabled and it's a migration alias, field is not created
+	if !p.Migration && f.MigrationAlias {
+		return nil
+	}
 	properties := getDefaultProperties(f)
 	properties["type"] = "alias"
 	properties["path"] = f.AliasPath
+
 	return properties
 }
 

diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go
@@ -28,7 +28,8 @@ import (
 func TestProcessor(t *testing.T) {
 	falseVar := false
 	trueVar := true
-	p := &Processor{}
+	p := &Processor{EsVersion: *common.MustNewVersion("7.0.0")}
+	migrationP := &Processor{EsVersion: *common.MustNewVersion("7.0.0"), Migration: true}
 	pEsVersion2 := &Processor{EsVersion: *common.MustNewVersion("2.0.0")}
 	pEsVersion64 := &Processor{EsVersion: *common.MustNewVersion("6.4.0")}
 	pEsVersion63 := &Processor{EsVersion: *common.MustNewVersion("6.3.6")}
@@ -282,6 +283,22 @@ func TestProcessor(t *testing.T) {
 				"type": "text", "doc_values": true,
 			},
 		},
+		{
+			output:   p.alias(&common.Field{Type: "alias", AliasPath: "a.c", MigrationAlias: false}),
+			expected: common.MapStr{"path": "a.c", "type": "alias"},
+		},
+		{
+			output:   p.alias(&common.Field{Type: "alias", AliasPath: "a.d", MigrationAlias: true}),
+			expected: nil,
+		},
+		{
+			output:   migrationP.alias(&common.Field{Type: "alias", AliasPath: "a.e", MigrationAlias: false}),
+			expected: common.MapStr{"path": "a.e", "type": "alias"},
+		},
+		{
+			output:   migrationP.alias(&common.Field{Type: "alias", AliasPath: "a.f", MigrationAlias: true}),
+			expected: common.MapStr{"path": "a.f", "type": "alias"},
+		},
 	}
 
 	for _, test := range tests {

diff --git a/libbeat/template/template.go b/libbeat/template/template.go
@@ -48,10 +48,11 @@ type Template struct {
 	beatVersion common.Version
 	esVersion   common.Version
 	config      TemplateConfig
+	migration   bool
 }
 
 // New creates a new template instance
-func New(beatVersion string, beatName string, esVersion common.Version, config TemplateConfig) (*Template, error) {
+func New(beatVersion string, beatName string, esVersion common.Version, config TemplateConfig, migration bool) (*Template, error) {
 	bV, err := common.NewVersion(beatVersion)
 	if err != nil {
 		return nil, err
@@ -116,6 +117,7 @@ func New(beatVersion string, beatName string, esVersion common.Version, config T
 		beatVersion: *bV,
 		esVersion:   esVersion,
 		config:      config,
+		migration:   migration,
 	}, nil
 }
 
@@ -139,7 +141,7 @@ func (t *Template) load(fields common.Fields) (common.MapStr, error) {
 
 	// Start processing at the root
 	properties := common.MapStr{}
-	processor := Processor{EsVersion: t.esVersion}
+	processor := Processor{EsVersion: t.esVersion, Migration: t.migration}
 	if err := processor.Process(fields, "", properties); err != nil {
 		return nil, err
 	}

diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go
@@ -35,7 +35,7 @@ func TestNumberOfRoutingShards(t *testing.T) {
 
 	// Test it exists in 6.1
 	ver := common.MustNewVersion("6.1.0")
-	template, err := New(beatVersion, beatName, *ver, config)
+	template, err := New(beatVersion, beatName, *ver, config, false)
 	assert.NoError(t, err)
 
 	data := template.Generate(nil, nil)
@@ -46,7 +46,7 @@ func TestNumberOfRoutingShards(t *testing.T) {
 
 	// Test it does not exist in 6.0
 	ver = common.MustNewVersion("6.0.0")
-	template, err = New(beatVersion, beatName, *ver, config)
+	template, err = New(beatVersion, beatName, *ver, config, false)
 	assert.NoError(t, err)
 
 	data = template.Generate(nil, nil)
@@ -67,7 +67,7 @@ func TestNumberOfRoutingShardsOverwrite(t *testing.T) {
 
 	// Test it exists in 6.1
 	ver := common.MustNewVersion("6.1.0")
-	template, err := New(beatVersion, beatName, *ver, config)
+	template, err := New(beatVersion, beatName, *ver, config, false)
 	assert.NoError(t, err)
 
 	data := template.Generate(nil, nil)

diff --git a/libbeat/template/testdata/fields.yml b/libbeat/template/testdata/fields.yml
@@ -24,3 +24,13 @@
     - name: alias
       type: alias
       path: keyword
+
+    - name: migration_alias_true
+      type: alias
+      path: keyword
+      migration: true
+
+    - name: migration_alias_false
+      type: alias
+      path: keyword
+      migration_alias: false