Merge remote-tracking branch 'origin/master' into gosigar-0.10.0

CHANGELOG.next.asciidoc

@@ -130,6 +130,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `name` config option to `add_host_metadata` processor. {pull}9943[9943]
 - Add `add_labels` and `add_tags` processors. {pull}9973[9973]
 - Add missing file encoding to readers. {pull}10080[10080]
+- Introduce `migration.enabled` configuration. {pull}9805[9805]
+- Add alias field support in Kibana index pattern. {pull}10075[10075]
 
 *Auditbeat*
 
@@ -166,8 +168,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add more meaningful metrics to 'performance' Metricset on 'MSSQL' module {pull}10011[10011]
 - Rename some fields in `performance` Metricset on MSSQL module to match the updated documentation from Microsoft {pull}10074[10074]
 - Add AWS EC2 module. {pull}9257[9257] {issue}9300[9300]
+- Release traefik Metricbeat module as GA. {pull}10166[10166]
 - Release Elastic stack modules (Elasticsearch, Logstash, and Kibana) as GA. {pull}10094[10094]
 - List filesystems on Windows that have an access path but not an assigned letter {issue}8916[8916] {pull}10196[10196]
+- Release Ceph module as GA. {pull}10202[10202]
+- Release aerospike module as GA. {pull}10203[10203]
 
 *Packetbeat*
 

auditbeat/auditbeat.reference.yml

@@ -1246,3 +1246,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

auditbeat/auditbeat.yml

@@ -177,3 +177,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

filebeat/filebeat.reference.yml

@@ -1941,3 +1941,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

filebeat/filebeat.yml

@@ -205,3 +205,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

heartbeat/heartbeat.reference.yml

@@ -1390,3 +1390,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

heartbeat/heartbeat.yml

@@ -162,3 +162,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

journalbeat/journalbeat.reference.yml

@@ -1180,3 +1180,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

journalbeat/journalbeat.yml

@@ -168,3 +168,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

libbeat/_meta/config.reference.yml

@@ -1134,3 +1134,8 @@ logging.files:
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.
 #seccomp.enabled: true
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

libbeat/_meta/config.yml

@@ -122,3 +122,8 @@ processors:
 # have the Elasticsearch output configured, you can simply uncomment the
 # following line.
 #xpack.monitoring.elasticsearch:
+
+#================================= Migration ==================================
+
+# This allows to enable migration aliases
+#migration.enabled: false

libbeat/cmd/export/template.go

@@ -66,7 +66,7 @@ func GenTemplateConfigCmd(settings instance.Settings, name, idxPrefix, beatVersi
 				fmt.Fprintf(os.Stderr, "Invalid Elasticsearch version: %s\n", err)
 			}
 
-			tmpl, err := template.New(b.Info.Version, index, *esVersion, cfg)
+			tmpl, err := template.New(b.Info.Version, index, *esVersion, cfg, b.Config.Migration.Enabled())
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Error generating template: %+v", err)
 				os.Exit(1)

libbeat/cmd/instance/beat.go

@@ -63,7 +63,7 @@ import (
 	svc "github.com/elastic/beats/libbeat/service"
 	"github.com/elastic/beats/libbeat/template"
 	"github.com/elastic/beats/libbeat/version"
-	"github.com/elastic/go-sysinfo"
+	sysinfo "github.com/elastic/go-sysinfo"
 	"github.com/elastic/go-sysinfo/types"
 	ucfg "github.com/elastic/go-ucfg"
 )
@@ -105,6 +105,7 @@ type beatConfig struct {
 	Dashboards *common.Config `config:"setup.dashboards"`
 	Template   *common.Config `config:"setup.template"`
 	Kibana     *common.Config `config:"setup.kibana"`
+	Migration  *common.Config `config:"migration"`
 
 	// ILM Config options
 	ILM *common.Config `config:"output.elasticsearch.ilm"`
@@ -877,7 +878,7 @@ func (b *Beat) templateLoadingCallback() (func(esClient *elasticsearch.Client) e
 			b.Config.Template = common.NewConfig()
 		}
 
-		loader, err := template.NewLoader(b.Config.Template, esClient, b.Info, b.Fields)
+		loader, err := template.NewLoader(b.Config.Template, esClient, b.Info, b.Fields, b.Config.Migration.Enabled())
 		if err != nil {
 			return fmt.Errorf("Error creating Elasticsearch template loader: %v", err)
 		}

libbeat/common/field.go

@@ -52,6 +52,7 @@ type Field struct {
 	CopyTo         string      `config:"copy_to"`
 	IgnoreAbove    int         `config:"ignore_above"`
 	AliasPath      string      `config:"path"`
+	MigrationAlias bool        `config:"migration"`
 
 	ObjectType            string          `config:"object_type"`
 	ObjectTypeMappingType string          `config:"object_type_mapping_type"`
@@ -64,6 +65,7 @@ type Field struct {
 	Searchable   *bool  `config:"searchable"`
 	Aggregatable *bool  `config:"aggregatable"`
 	Script       string `config:"script"`
+
 	// Kibana params
 	Pattern              string              `config:"pattern"`
 	InputFormat          string              `config:"input_format"`
@@ -142,6 +144,13 @@ func (f Fields) HasKey(key string) bool {
 	return f.hasKey(keys)
 }
 
+// GetField returns the field in case it exists
+func (f Fields) GetField(key string) *Field {
+	keys := strings.Split(key, ".")
+	return f.getField(keys)
+
+}
+
 // HasNode checks if inside fields the given node exists
 // In contrast to HasKey it not only compares the leaf nodes but
 // every single key it traverses.
@@ -216,6 +225,32 @@ func (f Fields) hasKey(keys []string) bool {
 	return false
 }
 
+func (f Fields) getField(keys []string) *Field {
+	// Nothing to compare anymore
+	if len(keys) == 0 {
+		return nil
+	}
+
+	key := keys[0]
+	keys = keys[1:]
+
+	for _, field := range f {
+		if field.Name == key {
+
+			if len(field.Fields) > 0 {
+				return field.Fields.getField(keys)
+			}
+			// Last entry in the tree but still more keys
+			if len(keys) > 0 {
+				return nil
+			}
+
+			return &field
+		}
+	}
+	return nil
+}
+
 // GetKeys returns a flat list of keys this Fields contains
 func (f Fields) GetKeys() []string {
 	return f.getKeys("")

libbeat/common/field_test.go

@@ -192,6 +192,58 @@ func TestGetKeys(t *testing.T) {
 	}
 }
 
+func TestGetField(t *testing.T) {
+	tests := []struct {
+		fields Fields
+		key    string
+		field  *Field
+	}{
+		{
+			fields: Fields{
+				Field{
+					Name: "test", Fields: Fields{
+						Field{
+							Name: "find",
+						},
+					},
+				},
+			},
+			key:   "test.find",
+			field: &Field{Name: "find"},
+		},
+		{
+			fields: Fields{
+				Field{
+					Name: "test", Fields: Fields{
+						Field{
+							Name: "find",
+						},
+					},
+				},
+			},
+			key:   "test",
+			field: nil,
+		},
+		{
+			fields: Fields{
+				Field{
+					Name: "test", Fields: Fields{
+						Field{
+							Name: "find",
+						},
+					},
+				},
+			},
+			key:   "test.find.more",
+			field: nil,
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.field, test.fields.GetField(test.key))
+	}
+}
+
 func TestFieldValidate(t *testing.T) {
 	tests := []struct {
 		cfg   MapStr

libbeat/docs/communitybeats.asciidoc

@@ -79,6 +79,7 @@ https://github.com/eBay/nvidiagpubeat[nvidiagpubeat]:: Uses nvidia-smi to grab m
 https://github.com/aristanetworks/openconfigbeat[openconfigbeat]:: Streams data from http://openconfig.net[OpenConfig]-enabled network devices
 https://github.com/joehillen/packagebeat[packagebeat]:: Collects information about system packages from package
 managers.
+https://github.com/WuerthIT/perfstatbeat[perfstatbeat]:: Collect performance metrics on the AIX operating system.
 https://github.com/kozlice/phpfpmbeat[phpfpmbeat]:: Reads status from PHP-FPM.
 https://github.com/joshuar/pingbeat[pingbeat]:: Sends ICMP pings to a list
 of targets and stores the round trip time (RTT) in Elasticsearch.

libbeat/kibana/fields_transformer.go

@@ -24,6 +24,8 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 )
 
+var v640 = common.MustNewVersion("6.4.0")
+
 type fieldsTransformer struct {
 	fields                    common.Fields
 	transformedFields         []common.MapStr
@@ -84,6 +86,19 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
 				t.transformFields(f.Fields, f.Path)
 			}
 		} else {
+			if f.Type == "alias" {
+				if t.version.LessThan(v640) {
+					continue
+				}
+				if ff := t.fields.GetField(f.AliasPath); ff != nil {
+					// copy the field, keep
+					path := f.Path
+					name := f.Name
+					f = *ff
+					f.Path = path
+					f.Name = name
+				}
+			}
 			t.add(f)
 
 			if f.MultiFields != nil {
@@ -104,7 +119,7 @@ func (t *fieldsTransformer) update(target *common.MapStr, override common.Field)
 		target.Update(field)
 		if !override.Overwrite {
 			// compatible duplication
-			return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true'", override.Path)
+			return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true', %+v, %+v", override.Path, override, field)
 		}
 		return nil
 	}

libbeat/kibana/index_pattern_generator_test.go

@@ -122,7 +122,7 @@ func TestGenerate(t *testing.T) {
 	tmpDir := tmpPath(t)
 	defer os.RemoveAll(tmpDir)
 
-	v6, _ := common.NewVersion("6.0.0")
+	v6, _ := common.NewVersion("6.4.0")
 	versions := []*common.Version{v6}
 	for _, version := range versions {
 		generator, err := NewGenerator("beat-*", "b eat ?!", fieldsYml, tmpDir, "7.0.0-alpha1", *version)
@@ -149,7 +149,7 @@ func TestGenerateExtensive(t *testing.T) {
 	tmpDir := tmpPath(t)
 	defer os.RemoveAll(tmpDir)
 
-	version6, _ := common.NewVersion("6.0.0")
+	version6, _ := common.NewVersion("6.4.0")
 	versions := []*common.Version{version6}
 	for _, version := range versions {
 		generator, err := NewGenerator("metricbeat-*", "metric be at ?!", "testdata/extensive/fields.yml", tmpDir, "7.0.0-alpha1", *version)

libbeat/kibana/testdata/beat-6.json

@@ -2,8 +2,8 @@
   "objects": [
     {
       "attributes": {
-        "fieldFormatMap": "{\"long\":{\"id\":\"url\",\"params\":{\"inputFormat\":\"string\",\"labelTemplate\":\"long template\",\"outputFormat\":\"float\",\"outputPrecision\":5,\"urlTemplate\":\"_a=(query:(language:lucene,query:'context.app.name:\\\"{{value}}\\\"'))\"}}}",
-        "fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"long\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field.keyword\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"blob\",\"scripted\":false,\"searchable\":false,\"type\":\"binary\"}]",
+        "fieldFormatMap": "{\"long\":{\"id\":\"url\",\"params\":{\"inputFormat\":\"string\",\"labelTemplate\":\"long template\",\"outputFormat\":\"float\",\"outputPrecision\":5,\"urlTemplate\":\"_a=(query:(language:lucene,query:'context.app.name:\\\"{{value}}\\\"'))\"}},\"alias\":{\"id\":\"url\",\"params\":{\"inputFormat\":\"string\",\"labelTemplate\":\"long template\",\"outputFormat\":\"float\",\"outputPrecision\":5,\"urlTemplate\":\"_a=(query:(language:lucene,query:'context.app.name:\\\"{{value}}\\\"'))\"}}}",
+        "fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"long\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field.keyword\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"blob\",\"scripted\":false,\"searchable\":false,\"type\":\"binary\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"alias\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"}]",
         "timeFieldName": "@timestamp",
         "title": "beat-*",
         "sourceFilters": "[{\"value\":\"user.email\"}, {\"value\": \"user.name\"},{\"value\": \"url.*\"}]"

libbeat/kibana/testdata/fields.yml

@@ -25,7 +25,11 @@
         - min_version: 6.0.0
           value: "_a=(query:(language:lucene,query:'context.app.name:\"{{value}}\"'))"
 
-- key: with source filter 
+    - name: alias
+      type: alias
+      path: long
+
+- key: with source filter
   title: Test
   kibana:
     source_filters: