diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2357e94c205..9f126d136a4 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4419,6 +4419,16 @@ type: keyword The state of this shard. +-- + +*`elasticsearch.shard.relocating_node.name`*:: ++ +-- +type: keyword + +The node the shard was relocated from. + + -- [[exported-fields-envoyproxy]] diff --git a/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go b/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go index 0ac7223e158..d6bb4e4dfed 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go +++ b/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go @@ -32,16 +32,6 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) -func passthruField(fieldPath string, sourceData, targetData common.MapStr) error { - fieldValue, err := sourceData.GetValue(fieldPath) - if err != nil { - return elastic.MakeErrorForMissingField(fieldPath, elastic.Elasticsearch) - } - - targetData.Put(fieldPath, fieldValue) - return nil -} - func clusterNeedsTLSEnabled(license, stackStats common.MapStr) (bool, error) { // TLS does not need to be enabled if license type is something other than trial value, err := license.GetValue("license.type") @@ -184,7 +174,7 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { return err } - if err = passthruField("status", clusterStats, clusterState); err != nil { + if err = elasticsearch.PassThruField("status", clusterStats, clusterState); err != nil { return err } diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 72d62c1d0dd..c384e59b0e2 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -26,6 +26,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/helper/elastic" ) // Global clusterIdCache. Assumption is that the same node id never can belong to a different cluster id @@ -224,6 +225,18 @@ func GetStackUsage(http *helper.HTTP, resetURI string) (common.MapStr, error) { return stackUsage, err } +// PassThruField copies the field at the given path from the given source data object into +// the same path in the given target data object +func PassThruField(fieldPath string, sourceData, targetData common.MapStr) error { + fieldValue, err := sourceData.GetValue(fieldPath) + if err != nil { + return elastic.MakeErrorForMissingField(fieldPath, elastic.Elasticsearch) + } + + targetData.Put(fieldPath, fieldValue) + return nil +} + // Global cache for license information. Assumption is that license information changes infrequently var licenseCache = &_licenseCache{} diff --git a/metricbeat/module/elasticsearch/fields.go b/metricbeat/module/elasticsearch/fields.go index 6ad6d79844d..97b59175742 100644 --- a/metricbeat/module/elasticsearch/fields.go +++ b/metricbeat/module/elasticsearch/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsW81u4zYQvvspBjm1wEYP4EMvi26bApsuutkCRVF4aXEsM+GPlqScuE9fkJIc/VCWIimxF7Vvkezv+2Y4Q86QzDU84H4JyImxLDZIdLxdAFhmOS7h6ufq86sFAEUTa5ZapuQSfloAANS+A0LRjOMCQCNHYnAJa7RkAWDQWiYTs4S/r4zhV/+4Z1ul7SpWcsOSJWwIN+6XG4acmqUHvwZJBLYFuo/dp7iERKssLZ4E1NXhqpAxz4xFHbm/Di9L1AfcPypNK8+D2Pmn7oEC17NEi05aRl+DlNEjlMYSizMTe0xPe4zVtCirw9ZD+L6wrQnUjLDy0xzwqiiHkZnaq24n9OhqassM/JBoRPkO9si5enwHGumPUVCIVBTDOpqeGaDi1oF5DcwPTdT4RsgjlYFSmbQtzFwMVzJpvepRA3CnLOEgM7FGDWqTGwtMHmKiQ4kg7u2sUm4PInLwa+QsYWuOg0VRUouuOSU56B4dpQomKYvni5ibHO6MY6YweMAAmS3RtOmZY74ZoOazwzzinS7/9HnoqI8G6Gr7Kbf+iJueJaWaCaJZK4peQ1bOte+XV4rz/nQJEQkUSu+j9d4GlE4JsY8eGDKDFDZKVyhbixeTFJ+mLFpNgDGLVaMygWlL1Y1TVFQlITbrRnDw/NIX/lTFJnqrHKAqzgRK6+PMbjH3fncieHEUOVpsOvFV5OVML5ZprNIYGfYvdiRDj9SN0oLYJXT9eLApToIz5KDZGeBRj4jHxNv6ZkGQCzvQ9gs7Os+8lWuf9YvK7LTee1+XUgPuDs9YkcZY7VDvTz11sWZedXqzx1P5QlyaVWs0qnwOfr6pskHqkMK0xSoXZF4rxZHIlzHf6QyBNZbPMLexJJnR5j9Kaz1uINCq3JboBG3UMcqj+O88pK+Gu0c5p90q05xQcmL2wiq4yulAgVCq0Zij7POuylUJ9bU5POgq0zHO6vjPHvK44wva2Rxf5ex3fME+r+OrEuqOr8+oJhONFD/JhNpVu18qtFnkXSq0S4XWq/9lFRpceqtL5l4y9zvM3MN2NI/u1XrKyi/46/RRoyqeL5J9yxAEh3u17i71LLEzllm/qXUOGWajxJKVj2ITpVrFaAzSleu8NF2FontsH/mpBM833XHXDuSQJiZ3hDO6osTirHrutliJz9xgA4/MbgGZ3aIGAoIZw2TiBGEeJaDcc/+33RILsco4BaksrBFSog3SQAPRCmxX9E4J68bvT7+3eduu4qtkO9SGqWYrPpWvQA1T3u/E4PW+L4X+/Ag3cqOGndD0Wd1n+QBBpaigA6oKilVgiySNmGT2ZOvBr0hScApqS4CzoX+xrRohyNNpbRDkabwJUsnTD8WtktczDEdpyylH5GDK8FF5buf9chQJruIHwsO9wagdxJtNCQ4OG6lTUjgtuBBMv6DhUFZz3M6Y+ZDbz9KsctLdRD6rRgufmLF+uS9bmfNtss6oNfnempKy3+vZOYAZe9VRvagn7ZnF7nfCOTVKleKzZa2bPIt+zeGOSlzFu9MifCNkkJd+57QtrT1u3fqgdtmpe+WCviGFASHZNKrjKwAfyVNXCFYFp0gezkTxJyQPQyWvzsfRXrYY5m1XT5yJ7C95aXN0jtqrLChkcs795YAvWXcOii9Zd25ZZzK9YzvVvjI7Q+J9LrAvuXcOii+5d+rc66yAkziKFecYW6Vnq4J/eQ8H0HDWDaiBS13HdgEnlsPPDJDEYyeGrl4Rhgz6AKGHCWh0ZPWQ1KqgV/Z6XhD9v/0ezMWNCVzWgSlJ+IFxBLM3FgWEofuS0B/8n2TH4eAVjac54i4FkB1hnKz526po/hdaipIymawsMQ+LJvUL9jq/hgC/QqykJUwaIFC8APeiilRN0nGbowa1XSlNW/8iNfYQ8sZDQhuycutMaWbDCTXmEDYAV7/lF2SacMEvgg9KAz4RkXJnUGavBUlT1pB+yFcmcMXk6luGGUateWv0aS8Tfi/Nw7Zi1F/ynRKUHqAInolR9mo3mu2WGWDG7y0OuN2cb/DO5f7aYbtX0qAOR+S8VyHu/K4qsdgU8V8AAAD//yuE2oA=" + return "eJzsW9tu4zYQffdXDPLUAht9gB/6sui2KbDpopstUBSFlxbHNhNetCTlxP36gpTk6EJdIimxF7XfItnnnBnOkDMkcw0PeFgCcmIsiw0SHe8WAJZZjku4+rn8/GoBQNHEmiWWKbmEnxYAAJXvgFA05bgA0MiRGFzCGi1ZABi0lsmtWcLfV8bwq3/cs53SdhUruWHbJWwIN+6XG4acmqUHvwZJBDYFuo89JLiErVZpkj8JqKvClSFjnhqLOnJ/HV8WqA94eFSalp4HsbNP1QM5rmeJFq20jL4GKaMdlMYSizMTe0xP28VqGpTlYeshfJ/bVgeqR1jxqQ94WZTDSE3lVbsTenTVtaUGfthqRPkODsi5enwHGumPUVCIVBTDOuqeGaDi1oF5DcwPTVT7RsgjpYFSqbQNzEwMV3LbeNWjBuBOWcJBpmKNGtQmMxaYPMZEixJB3NtZpdweRWTg18jZlq05DhZFSSW65pTkoHt0FCqYpCyeL2JuMrgzjpnc4AEDZHZE07pnunwzQM1nh9nhnTb/9Hmo00cDdDX9lFnf4aZnSYlmgmjWiKLXkJVxHfrlFeK8P11CRAKF0odofbABpVNC7KMHhtQghY3SJcrG4sUkxacpi1YdYMxiVatMYNpSdeMU5VVJiM26ERw8v/SFP1Wxid4qB6iKU4HS+jizO8y8354IXhxFjhbrTnwVeRnTi2UaqzRGhv2LLcnQI3WjtCB2CW0/HmyKk+AMOWp2BnjUDvG49ba+WRBkwo60/cI655m3cu2zflGandYH7+tCasDd4Rkr0hirPerDqacuVs+rVm/2eCpbiAuzKo1Gmc/BzzdV1kgdUpg2X+WCzGulOBL5MuY7nSKw2vIZ5jaWbGe0+Y/CWo8bCLQytyV6izZqGeVR/Hce0lfD7aOc0e6UqU8oGTF7YRVc5nSgQCjVaEwn+7yrcllCdW0OD7pKdYyzOv6zh+x2fE47m+PLnP2Oz9nndXxZQtXx1RnVpKKW4ieZUNtq90uFNou8S4V2qdB69b+sQoNLb3XJ3EvmfoeZe9yO5tG9Wk9Z+QV/nT5qVMXzRbJvKYLgcK/W7aWeJXbGMus3tc4gw2yUWLLyUWyiRKsYjUG6cp2XpqtQdI/tIz8V4NmmO+6bgRzSxOSecEZXlFicVc/dDkvxmRls4JHZHSCzO9RAQDBjmNw6QZhFCSj33P9td8RCrFJOQSoLa4SEaIM00EA0AtsVvVPCuvb70+9t3jar+DLZHrVhqt6KT+XLUcOU93sxeL3vS6E/P8KN3KhhJzR9VvdZPkBQISrogLKCfBXYIUkiJpk92XrwK5IEnILKEuBs6F9sy0YI8nRaGwR5Gm+CVPL0Q3Gr5PUMw1HYcsoROZoyfFSe23m/HEWCq/iB8HBvMGoH8WZTgIPDRuqU5E4LLgTTL2g4lNUctzNmPuT2szQrnXTXkc+q0cInZqxf7otW5nybrDNqTb63pqTo93p2DmDGXnVUL+pJe2ax+71wTo0SpfhsWesmz7xfc7ijElfx9rQI3wgZ5KXfOW1Ka45buz6oXHZqX7mgb0hhQEjWjWr5CsBH8tQWgmXBCZKHM1H8CcnDUMmr83G0ly2GedvVE2ci+0tW2nTOUQeVBoVMzrm/HPAl685B8SXrzi3rTKr3bK+aV2ZnSLzPOfYl985B8SX3Tp17rRXwNo5ixTnGVunZquBf3sMRNJx1A2rgQlfXLuDEcviZAbbx2ImhrVeEIYM+QOhxAhodWT0klSrolb2eFUT/b78Hc3FjApd1YEoSfmAcwRyMRQFh6L4k9Af/J9lxOHpF42mOuAsBZE8YJ2v+tirq/4WWoKRMbleWmIdFnfoFe51fQ4BfIVbSEiYNEMhfgHtRRion6bjNUYParpSmjX+RGnsIeeMhoQlZunWmNLPhhBpzCBuAq97yCzJNuOAXwQelAZ+ISLgzKLXXgiQJq0k/5isTuGJy9S3FFKPGvDX6tJcJv5fmYRsx6i/5TglKD5AHz8Qoe7UbzXbHDDDj9xYH3G7ONnjncn/lsN0r6b5YPecNiDu/mUosDuHWyFVMrJtY/NnMzJeNd/mFV7/B64PmkZiCFClstBLR4r8AAAD//+uzAwg=" } diff --git a/metricbeat/module/elasticsearch/shard/_meta/data.json b/metricbeat/module/elasticsearch/shard/_meta/data.json index 23f198b6569..daa0aa8464a 100644 --- a/metricbeat/module/elasticsearch/shard/_meta/data.json +++ b/metricbeat/module/elasticsearch/shard/_meta/data.json @@ -8,16 +8,17 @@ "cluster": { "name": "elasticsearch", "state": { - "id": "dJaHX6fxSqSVMOsL4QZwwQ" + "id": "MBE4XrQOSf6ScXRTuCO1Pw" } }, "index": { - "name": "filebeat-7.0.0-alpha1-2018.05.09" + "name": "heartbeat-7.0.0-alpha1-2018.08.27" }, "node": { - "name": "523zXyT6TRWiqXcQItnkyQ" + "name": "Z4hBonPxQVW9qPKEHpwWCg" }, "shard": { + "number": 0, "primary": true, "state": "STARTED" } diff --git a/metricbeat/module/elasticsearch/shard/_meta/fields.yml b/metricbeat/module/elasticsearch/shard/_meta/fields.yml index 97337cfce0d..bb62b9bf2ae 100644 --- a/metricbeat/module/elasticsearch/shard/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/shard/_meta/fields.yml @@ -12,8 +12,11 @@ type: long description: > The number of this shard. - - name: state type: keyword description: > The state of this shard. + - name: relocating_node.name + type: keyword + description: > + The node the shard was relocated from. diff --git a/metricbeat/module/elasticsearch/shard/_meta/test/routing_table.700.json b/metricbeat/module/elasticsearch/shard/_meta/test/routing_table.700.json new file mode 100644 index 00000000000..1874c435302 --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/_meta/test/routing_table.700.json @@ -0,0 +1,117 @@ +{ + "cluster_name": "elasticsearch", + "compressed_size_in_bytes": 11865, + "cluster_uuid": "d0xWWzE8S_2x3Z-lMagVxw", + "version": 26, + "state_uuid": "K6wBBv1ZQk-1HgpnPx5awQ", + "master_node": "PL_V6KgiSXCO-vI2Rm-B_A", + "routing_table": { + "indices": { + ".monitoring-es-6-2018.09.05": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "PL_V6KgiSXCO-vI2Rm-B_A", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-es-6-2018.09.05", + "allocation_id": { + "id": "mOtgQ2xsTGWbRSlRbkbrcA" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 0, + "index": ".monitoring-es-6-2018.09.05", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "REPLICA_ADDED", + "at": "2018-09-05T16:23:21.269Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ] + } + }, + "testindex": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "PL_V6KgiSXCO-vI2Rm-B_A", + "relocating_node": null, + "shard": 0, + "index": "testindex", + "allocation_id": { + "id": "epzwd1uKQGSLa745KCBYQw" + } + }, + { + "state": "INITIALIZING", + "primary": false, + "node": "DdU2Zw-gSRCmAi0Kp46aCg", + "relocating_node": null, + "shard": 0, + "index": "testindex", + "recovery_source": { + "type": "PEER" + }, + "allocation_id": { + "id": "foAIE9r0S-Ob8vjstFg1qA" + }, + "unassigned_info": { + "reason": "INDEX_CREATED", + "at": "2018-09-05T16:22:02.488Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ] + } + }, + ".kibana": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "PL_V6KgiSXCO-vI2Rm-B_A", + "relocating_node": null, + "shard": 0, + "index": ".kibana", + "allocation_id": { + "id": "R4ydh8SOS4iPcKsbgMwkoA" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 0, + "index": ".kibana", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "REPLICA_ADDED", + "at": "2018-09-05T16:23:21.269Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ] + } + } + } + } + } diff --git a/metricbeat/module/elasticsearch/shard/data.go b/metricbeat/module/elasticsearch/shard/data.go index 00de2076896..e731c82f175 100644 --- a/metricbeat/module/elasticsearch/shard/data.go +++ b/metricbeat/module/elasticsearch/shard/data.go @@ -24,16 +24,15 @@ import ( s "github.com/elastic/beats/libbeat/common/schema" c "github.com/elastic/beats/libbeat/common/schema/mapstriface" "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/module/elasticsearch" ) var ( schema = s.Schema{ - "state": c.Str("state"), - "primary": c.Bool("primary"), - "node": c.Str("node"), - "index": c.Str("index"), - "shard": c.Int("number"), - "relocating_node": c.Str("relocating_node"), + "state": c.Str("state"), + "primary": c.Bool("primary"), + "index": c.Str("index"), + "shard": c.Int("shard"), } ) @@ -61,13 +60,38 @@ func eventsMapping(r mb.ReporterV2, content []byte) { for _, shard := range shards { event := mb.Event{} - fields, _ := schema.Apply(shard) + fields, err := schema.Apply(shard) + if err != nil { + r.Error(err) + continue + } + + // Handle node field: could be string or null + err = elasticsearch.PassThruField("node", shard, fields) + if err != nil { + continue + } + + // Handle relocating_node field: could be string or null + err = elasticsearch.PassThruField("relocating_node", shard, fields) + if err != nil { + continue + } + event.ModuleFields = common.MapStr{} event.ModuleFields.Put("node.name", fields["node"]) delete(fields, "node") + event.ModuleFields.Put("index.name", fields["index"]) delete(fields, "index") + event.MetricSetFields = fields + event.MetricSetFields.Put("number", fields["shard"]) + delete(event.MetricSetFields, "shard") + + delete(event.MetricSetFields, "relocating_node") + event.MetricSetFields.Put("relocating_node.name", fields["relocating_node"]) + event.ModuleFields.Put("cluster.state.id", stateData.StateID) event.ModuleFields.Put("cluster.name", stateData.ClusterName) diff --git a/metricbeat/module/elasticsearch/shard/data_xpack.go b/metricbeat/module/elasticsearch/shard/data_xpack.go index 6ed9c917e9e..8f15e6abae0 100644 --- a/metricbeat/module/elasticsearch/shard/data_xpack.go +++ b/metricbeat/module/elasticsearch/shard/data_xpack.go @@ -22,6 +22,7 @@ import ( "time" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/helper/elastic" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/elasticsearch" ) @@ -30,13 +31,11 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) { stateData := &stateStruct{} err := json.Unmarshal(content, stateData) if err != nil { - r.Error(err) return } nodeInfo, err := elasticsearch.GetNodeInfo(m.HTTP, m.HostData().SanitizedURI+statePath, stateData.MasterNode) if err != nil { - r.Error(err) return } @@ -44,7 +43,6 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) { // Will be fixed in: https://github.com/elastic/elasticsearch/pull/30656 clusterID, err := elasticsearch.GetClusterID(m.HTTP, m.HostData().SanitizedURI+statePath, stateData.MasterNode) if err != nil { - r.Error(err) return } @@ -64,12 +62,20 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) { event := mb.Event{} fields, err := schema.Apply(shard) if err != nil { - r.Error(err) continue } - fields["shard"] = fields["number"] - delete(fields, "number") + // Handle node field: could be string or null + err = elasticsearch.PassThruField("node", shard, fields) + if err != nil { + continue + } + + // Handle relocating_node field: could be string or null + err = elasticsearch.PassThruField("relocating_node", shard, fields) + if err != nil { + continue + } event.RootFields = common.MapStr{} @@ -82,7 +88,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) { "shard": fields, "state_uuid": stateData.StateID, } - event.Index = ".monitoring-es-6-mb" + event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) r.Event(event)