From 462f42f5e9b635d1cbf1e43bce04bf18502a40a7 Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Thu, 4 Nov 2021 15:50:08 +0100 Subject: [PATCH] Write to .monitoring index on elasticsearch if xpack.enabled is true on standalone metricbeat (#28365) --- CHANGELOG.next.asciidoc | 1 + metricbeat/module/beat/config.go | 30 + metricbeat/module/beat/metricset.go | 7 + metricbeat/module/beat/state/_meta/data.json | 86 ++- metricbeat/module/beat/state/data.go | 11 +- metricbeat/module/beat/state/data_test.go | 2 +- metricbeat/module/beat/state/state.go | 2 +- metricbeat/module/beat/stats/_meta/data.json | 556 +++--------------- metricbeat/module/beat/stats/data.go | 11 +- metricbeat/module/beat/stats/data_test.go | 2 +- metricbeat/module/beat/stats/stats.go | 2 +- .../module/elasticsearch/ccr/_meta/data.json | 66 +-- metricbeat/module/elasticsearch/ccr/ccr.go | 2 +- metricbeat/module/elasticsearch/ccr/data.go | 11 +- .../module/elasticsearch/ccr/data_test.go | 2 +- .../cluster_stats/_meta/data.json | 68 ++- .../cluster_stats/cluster_stats.go | 2 +- .../elasticsearch/cluster_stats/data.go | 9 +- .../elasticsearch/enrich/_meta/data.json | 10 +- .../module/elasticsearch/enrich/data.go | 14 +- .../module/elasticsearch/enrich/data_test.go | 2 +- .../module/elasticsearch/enrich/enrich.go | 2 +- .../elasticsearch/index/_meta/data.json | 54 +- metricbeat/module/elasticsearch/index/data.go | 9 +- .../module/elasticsearch/index/data_test.go | 2 +- .../module/elasticsearch/index/index.go | 2 +- .../index_recovery/_meta/data.json | 15 +- .../elasticsearch/index_recovery/data.go | 9 +- .../index_recovery/index_recovery.go | 2 +- .../index_summary/_meta/data.json | 36 +- .../elasticsearch/index_summary/data.go | 12 +- .../elasticsearch/index_summary/data_test.go | 2 +- .../index_summary/index_summary.go | 2 +- metricbeat/module/elasticsearch/metricset.go | 10 +- .../elasticsearch/ml_job/_meta/data.json | 14 +- .../module/elasticsearch/ml_job/data.go | 9 +- .../module/elasticsearch/ml_job/ml_job.go | 2 +- .../module/elasticsearch/node/_meta/data.json | 35 +- metricbeat/module/elasticsearch/node/data.go | 11 +- .../module/elasticsearch/node/data_test.go | 2 +- metricbeat/module/elasticsearch/node/node.go | 2 +- .../elasticsearch/node_stats/_meta/data.json | 95 ++- .../module/elasticsearch/node_stats/data.go | 9 +- .../elasticsearch/node_stats/node_stats.go | 2 +- .../elasticsearch/pending_tasks/data.go | 9 +- .../elasticsearch/pending_tasks/data_test.go | 18 +- .../pending_tasks/pending_tasks.go | 2 +- .../elasticsearch/shard/_meta/data.json | 14 +- metricbeat/module/elasticsearch/shard/data.go | 9 +- .../module/elasticsearch/shard/data_test.go | 2 +- .../module/elasticsearch/shard/shard.go | 2 +- metricbeat/module/elasticsearch/testing.go | 12 +- .../module/kibana/settings/_meta/data.json | 14 +- .../module/kibana/stats/_meta/data.json | 54 +- metricbeat/module/kibana/stats/data.go | 9 +- metricbeat/module/kibana/stats/data_test.go | 2 +- metricbeat/module/kibana/stats/stats.go | 2 +- metricbeat/module/kibana/status/data.go | 9 +- metricbeat/module/kibana/status/data_test.go | 2 +- metricbeat/module/kibana/status/status.go | 2 +- metricbeat/module/logstash/logstash.go | 11 + .../logstash/logstash_integration_test.go | 1 - .../module/logstash/node/_meta/data.json | 136 ++--- metricbeat/module/logstash/node/data.go | 9 +- metricbeat/module/logstash/node/node.go | 2 +- .../logstash/node_stats/_meta/data.json | 79 +-- metricbeat/module/logstash/node_stats/data.go | 11 +- .../module/logstash/node_stats/data_test.go | 3 +- .../module/logstash/node_stats/node_stats.go | 2 +- 69 files changed, 705 insertions(+), 945 deletions(-) create mode 100644 metricbeat/module/beat/config.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 025ace44edd..f9d04da9a4b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -227,6 +227,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - `beat` module respects `basepath` config option. {pull}28162[28162] - Fix list_docker.go {pull}28374[28374] - Divide RDS metric cpu.total.pct by 100. {pull}28456[28456] +- Use xpack.enabled on SM modules to write into .monitoring indices when using Metricbeat standalone {pull}28365[28365] *Packetbeat* diff --git a/metricbeat/module/beat/config.go b/metricbeat/module/beat/config.go new file mode 100644 index 00000000000..83ea1879729 --- /dev/null +++ b/metricbeat/module/beat/config.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package beat + +// Config defines the structure for the Beat module configuration options +type Config struct { + XPackEnabled bool `config:"xpack.enabled"` +} + +// DefaultConfig returns the default configuration for the Beat module +func DefaultConfig() Config { + return Config{ + XPackEnabled: false, + } +} diff --git a/metricbeat/module/beat/metricset.go b/metricbeat/module/beat/metricset.go index b018b84343a..fe0d2d9acf8 100644 --- a/metricbeat/module/beat/metricset.go +++ b/metricbeat/module/beat/metricset.go @@ -26,11 +26,17 @@ import ( type MetricSet struct { mb.BaseMetricSet *helper.HTTP + XPackEnabled bool } // NewMetricSet creates a metricset that can be used to build other metricsets // within the Beat module. func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { + config := DefaultConfig() + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + http, err := helper.NewHTTP(base) if err != nil { return nil, err @@ -39,6 +45,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { ms := &MetricSet{ base, http, + config.XPackEnabled, } return ms, nil diff --git a/metricbeat/module/beat/state/_meta/data.json b/metricbeat/module/beat/state/_meta/data.json index 9ab38170a68..6e85c8a3018 100644 --- a/metricbeat/module/beat/state/_meta/data.json +++ b/metricbeat/module/beat/state/_meta/data.json @@ -1,77 +1,61 @@ { - "@timestamp": "2021-03-30T18:16:03.880Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "ed46455e-9adf-4493-83ba-a145ad0371a9", - "id": "7a0a00bb-faa3-4b2b-9d92-4a0a17bcb8f3", - "name": "anonymous", - "type": "metricbeat" - }, - "ecs": { - "version": "1.8.0" - }, - "metricset": { - "name": "state", - "period": 10000 - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "beat": { "state": { "beat": { - "name": "anonymous", - "host": "anonymous", + "host": "2963d991095f", + "name": "2963d991095f", "type": "metricbeat", - "uuid": "1be2b779-2479-44c4-9027-2e6fcc0cee93", - "version": "8.0.0" - }, - "module": { - "count": 10 + "uuid": "c4c9bc08-e990-4529-8bf5-715c00fa6615", + "version": "7.12.0" }, - "output": { - "name": "elasticsearch" - }, - "queue": { - "name": "mem" + "cluster": { + "uuid": "foobar" }, "host": { "containerized": "containerized", "os": { - "platform": "antergos", - "version": "", - "kernel": "5.11.7-arch1-1", - "name": "Antergos Linux" + "kernel": "5.14.13-200.fc34.x86_64", + "name": "CentOS Linux", + "platform": "centos", + "version": "7 (Core)" } }, "management": { "enabled": false }, + "module": { + "count": 10 + }, + "output": { + "name": "elasticsearch" + }, + "queue": { + "name": "mem" + }, "service": { - "id": "1be2b779-2479-44c4-9027-2e6fcc0cee93", + "id": "c4c9bc08-e990-4529-8bf5-715c00fa6615", "name": "metricbeat", - "version": "8.0.0" - }, - "cluster": { - "uuid": "-6NL06F2Td6bMGdkyaJ0PA" + "version": "7.12.0" } } }, + "event": { + "dataset": "beat.state", + "duration": 115000, + "module": "beat" + }, "host": { - "name": "anonymous", "architecture": "x86_64", - "hostname": "anonymous", - "id": "54f70115bae545cbac2b150f254472a0" + "hostname": "2963d991095f", + "id": "44ccc339d95bd51386fcfc5d8f041927" + }, + "metricset": { + "name": "state", + "period": 10000 }, "service": { - "address": "http://localhost:5066/state", + "address": "172.19.0.2:5066", "type": "beat" - }, - "event": { - "module": "beat", - "duration": 4263542, - "dataset": "beat.state" } -} +} \ No newline at end of file diff --git a/metricbeat/module/beat/state/data.go b/metricbeat/module/beat/state/data.go index a9985edf1c5..2a356407ef7 100644 --- a/metricbeat/module/beat/state/data.go +++ b/metricbeat/module/beat/state/data.go @@ -20,6 +20,8 @@ package state import ( "encoding/json" + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/elastic/beats/v7/libbeat/common" "github.com/pkg/errors" @@ -65,7 +67,7 @@ var ( } ) -func eventMapping(r mb.ReporterV2, info beat.Info, content []byte) error { +func eventMapping(r mb.ReporterV2, info beat.Info, content []byte, isXpack bool) error { event := mb.Event{ RootFields: common.MapStr{}, } @@ -135,6 +137,13 @@ func eventMapping(r mb.ReporterV2, info beat.Info, content []byte) error { event.MetricSetFields["host"] = hostMap } + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Beats) + event.Index = index + } + r.Event(event) return nil diff --git a/metricbeat/module/beat/state/data_test.go b/metricbeat/module/beat/state/data_test.go index 74c581e3e89..090cd0238bc 100644 --- a/metricbeat/module/beat/state/data_test.go +++ b/metricbeat/module/beat/state/data_test.go @@ -47,7 +47,7 @@ func TestEventMapping(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, info, input) + err = eventMapping(reporter, info, input, true) require.NoError(t, err, f) require.True(t, len(reporter.GetEvents()) >= 1, f) diff --git a/metricbeat/module/beat/state/state.go b/metricbeat/module/beat/state/state.go index 54f9ff7c3c5..2da5916ab61 100644 --- a/metricbeat/module/beat/state/state.go +++ b/metricbeat/module/beat/state/state.go @@ -66,5 +66,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventMapping(r, *info, content) + return eventMapping(r, *info, content, m.XPackEnabled) } diff --git a/metricbeat/module/beat/stats/_meta/data.json b/metricbeat/module/beat/stats/_meta/data.json index a34d9d1f4b0..ecda8583492 100644 --- a/metricbeat/module/beat/stats/_meta/data.json +++ b/metricbeat/module/beat/stats/_meta/data.json @@ -1,500 +1,136 @@ { - "@timestamp": "2021-04-01T15:45:36.780Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "beat": { - "type": "apm-server", + "id": "c4c9bc08-e990-4529-8bf5-715c00fa6615", "stats": { - "runtime": { - "goroutines": 38 - }, - "handles": { - "limit": { - "hard": 524288, - "soft": 1024 - }, - "open": 13 - }, - "info": { - "ephemeral_id": "dda08800-0daa-45b6-82dc-f8793d7b0a26", - "uptime": { - "ms": 211012 - } - }, "beat": { - "type": "apm-server", - "uuid": "7a60d23c-9fca-4182-90c5-f71296212641", - "version": "7.12.0", - "name": "anonymous", - "host": "anonymous" - }, - "cgroup": { - "cpu": { - "cfs": { - "quota": { - "us": 0 - }, - "period": { - "us": 100000 - } - }, - "id": "user.slice", - "stats": { - "periods": 0, - "throttled": { - "ns": 0, - "periods": 0 - } - } - }, - "cpuacct": { - "total": { - "ns": 7.052757783198e+12 - }, - "id": "user.slice" - }, - "memory": { - "mem": { - "limit": { - "bytes": 9.223372036854772e+18 - }, - "usage": { - "bytes": 1.1569999872e+10 - } - }, - "id": "user@1000.service" - } - }, - "memstats": { - "gc_next": 10417888, - "memory": { - "alloc": 7708704, - "total": 20617352 - }, - "rss": 63864832 - }, - "apm_server": { - "acm": { - "request": { - "count": 0 - }, - "response": { - "errors": { - "notfound": 0, - "decode": 0, - "validate": 0, - "toolarge": 0, - "count": 0, - "forbidden": 0, - "queue": 0, - "ratelimit": 0, - "internal": 0, - "unavailable": 0, - "unauthorized": 0, - "closed": 0, - "method": 0, - "invalidquery": 0 - }, - "valid": { - "ok": 0, - "accepted": 0, - "count": 0, - "notmodified": 0 - }, - "count": 0 - }, - "unset": 0 - }, - "decoder": { - "reader": { - "count": 0 - }, - "uncompressed": { - "content-length": 0, - "count": 0 - }, - "deflate": { - "content-length": 0, - "count": 0 - }, - "gzip": { - "content-length": 0, - "count": 0 - }, - "missing-content-length": { - "count": 0 - } - }, - "root": { - "request": { - "count": 0 - }, - "response": { - "errors": { - "validate": 0, - "queue": 0, - "toolarge": 0, - "closed": 0, - "forbidden": 0, - "decode": 0, - "count": 0, - "invalidquery": 0, - "method": 0, - "unauthorized": 0, - "internal": 0, - "ratelimit": 0, - "unavailable": 0, - "notfound": 0 - }, - "valid": { - "count": 0, - "notmodified": 0, - "ok": 0, - "accepted": 0 - }, - "count": 0 - }, - "unset": 0 - }, - "otlp": { - "grpc": { - "traces": { - "request": { - "count": 0 - }, - "response": { - "count": 0, - "errors": { - "count": 0 - }, - "valid": { - "count": 0 - } - } - }, - "metrics": { - "consumer": { - "unsupported_dropped": 0 - }, - "request": { - "count": 0 - }, - "response": { - "count": 0, - "errors": { - "count": 0 - }, - "valid": { - "count": 0 - } - } - } - } - }, - "processor": { - "transaction": { - "transformations": 0 - }, - "error": { - "frames": 0, - "stacktraces": 0, - "transformations": 0 - }, - "metric": { - "transformations": 0 - }, - "sourcemap": { - "counter": 0 - }, - "span": { - "stacktraces": 0, - "transformations": 0, - "frames": 0 - }, - "stream": { - "accepted": 0, - "errors": { - "queue": 0, - "server": 0, - "toolarge": 0, - "closed": 0, - "invalid": 0 - } - } - }, - "server": { - "request": { - "count": 0 - }, - "response": { - "count": 0, - "errors": { - "validate": 0, - "closed": 0, - "invalidquery": 0, - "ratelimit": 0, - "count": 0, - "queue": 0, - "notfound": 0, - "forbidden": 0, - "unavailable": 0, - "unauthorized": 0, - "decode": 0, - "internal": 0, - "method": 0, - "toolarge": 0 - }, - "valid": { - "accepted": 0, - "count": 0, - "notmodified": 0, - "ok": 0 - } - }, - "unset": 0 - }, - "sourcemap": { - "validation": { - "count": 0, - "errors": 0 - }, - "decoding": { - "count": 0, - "errors": 0 - }, - "request": { - "count": 0 - }, - "response": { - "count": 0, - "errors": { - "ratelimit": 0, - "unauthorized": 0, - "notfound": 0, - "unavailable": 0, - "toolarge": 0, - "validate": 0, - "closed": 0, - "queue": 0, - "count": 0, - "invalidquery": 0, - "method": 0, - "internal": 0, - "decode": 0, - "forbidden": 0 - }, - "valid": { - "accepted": 0, - "count": 0, - "notmodified": 0, - "ok": 0 - } - }, - "unset": 0 - }, - "jaeger": { - "grpc": { - "collect": { - "response": { - "count": 0, - "errors": { - "count": 0 - }, - "valid": { - "count": 0 - } - }, - "event": { - "dropped": { - "count": 0 - }, - "received": { - "count": 0 - } - }, - "request": { - "count": 0 - } - }, - "sampling": { - "request": { - "count": 0 - }, - "response": { - "count": 0, - "errors": { - "count": 0 - }, - "valid": { - "count": 0 - } - }, - "event": { - "dropped": { - "count": 0 - }, - "received": { - "count": 0 - } - } - } - }, - "http": { - "response": { - "errors": { - "count": 0 - }, - "valid": { - "count": 0 - }, - "count": 0 - }, - "event": { - "received": { - "count": 0 - }, - "dropped": { - "count": 0 - } - }, - "request": { - "count": 0 - } - } - }, - "profile": { - "response": { - "valid": { - "notmodified": 0, - "ok": 0, - "accepted": 0, - "count": 0 - }, - "count": 0, - "errors": { - "count": 0, - "decode": 0, - "toolarge": 0, - "closed": 0, - "unauthorized": 0, - "method": 0, - "internal": 0, - "queue": 0, - "unavailable": 0, - "notfound": 0, - "validate": 0, - "forbidden": 0, - "ratelimit": 0, - "invalidquery": 0 - } - }, - "unset": 0, - "request": { - "count": 0 - } - }, - "sampling": { - "transactions_dropped": 0 - } + "host": "2963d991095f", + "name": "2963d991095f", + "type": "metricbeat", + "uuid": "c4c9bc08-e990-4529-8bf5-715c00fa6615", + "version": "7.12.0" }, "cpu": { - "user": { - "ticks": 80, - "time": { - "ms": 85 - } - }, "system": { - "ticks": 30, + "ticks": 60, "time": { - "ms": 36 + "ms": 65 } }, "total": { - "value": 110, + "ticks": 170, + "time": { + "ms": 178 + }, + "value": 170 + }, + "user": { "ticks": 110, "time": { - "ms": 121 + "ms": 113 } } }, - "uptime": { - "ms": 211012 + "handles": { + "limit": { + "hard": 1024, + "soft": 1024 + }, + "open": 17 + }, + "info": { + "ephemeral_id": "16f5a19a-8ed6-4bfa-ae0a-85530262b63e", + "uptime": { + "ms": 3260 + } }, "libbeat": { - "pipeline": { - "queue": { - "acked": 1 - }, + "config": { + "running": 3, + "starts": 3, + "stops": 0 + }, + "output": { "events": { - "published": 1, - "retry": 1, - "total": 1, + "acked": 0, "active": 0, + "batches": 0, "dropped": 0, + "duplicates": 0, "failed": 0, - "filtered": 0 + "toomany": 0, + "total": 0 }, - "clients": 1 - }, - "config": { - "starts": 0, - "stops": 0, - "running": 0 - }, - "output": { "read": { - "bytes": 8533, - "errors": 1 - }, - "write": { - "bytes": 4309, + "bytes": 0, "errors": 0 }, "type": "elasticsearch", + "write": { + "bytes": 0, + "errors": 0 + } + }, + "pipeline": { + "clients": 10, "events": { - "duplicates": 0, + "active": 11, + "dropped": 0, "failed": 0, - "toomany": 0, - "total": 1, - "acked": 1, - "active": 0, - "batches": 1, - "dropped": 0 + "filtered": 0, + "published": 11, + "retry": 0, + "total": 11 + }, + "queue": { + "acked": 0 + } + } + }, + "memstats": { + "gc_next": 16263952, + "memory": { + "alloc": 12278440, + "total": 31749920 + }, + "rss": 83816448 + }, + "runtime": { + "goroutines": 63 + }, + "system": { + "cpu": { + "cores": 12 + }, + "load": { + "1": 2.14, + "15": 1.5, + "5": 1.44, + "norm": { + "1": 0.1783, + "15": 0.125, + "5": 0.12 } } + }, + "uptime": { + "ms": 3260 } }, - "id": "7a60d23c-9fca-4182-90c5-f71296212641" - }, - "ecs": { - "version": "1.8.0" + "type": "metricbeat" }, - "host": { - "name": "anonymous" + "event": { + "dataset": "beat.stats", + "duration": 115000, + "module": "beat" }, - "agent": { - "name": "anonymous", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "0d4eecfc-9661-4115-9f5e-72b7b5c720cd", - "id": "7a0a00bb-faa3-4b2b-9d92-4a0a17bcb8f3" + "metricset": { + "name": "stats", + "period": 10000 }, "service": { + "address": "172.19.0.2:5066", "name": "beat", - "address": "http://localhost:5066/stats", "type": "beat" - }, - "event": { - "dataset": "beat.stats", - "module": "beat", - "duration": 3375001 - }, - "metricset": { - "period": 10000, - "name": "stats" } -} +} \ No newline at end of file diff --git a/metricbeat/module/beat/stats/data.go b/metricbeat/module/beat/stats/data.go index 728eeee1f66..5e95f27a22e 100644 --- a/metricbeat/module/beat/stats/data.go +++ b/metricbeat/module/beat/stats/data.go @@ -20,6 +20,8 @@ package stats import ( "encoding/json" + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" @@ -108,7 +110,7 @@ var ( } ) -func eventMapping(r mb.ReporterV2, info beat.Info, content []byte) error { +func eventMapping(r mb.ReporterV2, info beat.Info, content []byte, isXpack bool) error { event := mb.Event{ RootFields: common.MapStr{}, ModuleFields: common.MapStr{}, @@ -135,6 +137,13 @@ func eventMapping(r mb.ReporterV2, info beat.Info, content []byte) error { "version": info.Version, }) + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Beats) + event.Index = index + } + r.Event(event) return nil } diff --git a/metricbeat/module/beat/stats/data_test.go b/metricbeat/module/beat/stats/data_test.go index 24b97dc4d9f..5a58a7c7ee1 100644 --- a/metricbeat/module/beat/stats/data_test.go +++ b/metricbeat/module/beat/stats/data_test.go @@ -47,7 +47,7 @@ func TestEventMapping(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, info, input) + err = eventMapping(reporter, info, input, true) require.NoError(t, err, f) require.True(t, len(reporter.GetEvents()) >= 1, f) diff --git a/metricbeat/module/beat/stats/stats.go b/metricbeat/module/beat/stats/stats.go index 8feaa60c934..7058a54614b 100644 --- a/metricbeat/module/beat/stats/stats.go +++ b/metricbeat/module/beat/stats/stats.go @@ -66,5 +66,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventMapping(r, *info, content) + return eventMapping(r, *info, content, m.XPackEnabled) } diff --git a/metricbeat/module/elasticsearch/ccr/_meta/data.json b/metricbeat/module/elasticsearch/ccr/_meta/data.json index cf390ddbc35..c600cea5046 100644 --- a/metricbeat/module/elasticsearch/ccr/_meta/data.json +++ b/metricbeat/module/elasticsearch/ccr/_meta/data.json @@ -13,52 +13,36 @@ }, "success": { "follow_indices": { - "count": 1 + "count": 0 } } }, - "bytes_read": 32768, + "bytes_read": 0, "follower": { - "global_checkpoint": 768, - "index": "follower_index", - "max_seq_no": 896, + "aliases_version": 1, + "global_checkpoint": -1, + "index": "rats", + "max_seq_no": -1, "operations": { "read": { - "count": 896 + "count": 0 } }, - "operations_written": 832, - "settings_version": 2, + "operations_written": 0, + "settings_version": 1, "shard": { "number": 0 }, "time_since_last_read": { - "ms": 8 + "ms": 3095 } }, "leader": { - "global_checkpoint": 1024, - "index": "leader_index", - "max_seq_no": 1536 + "global_checkpoint": -1, + "index": "pied_piper", + "max_seq_no": -1 }, - "read_exceptions": [ - { - "exception": { - "reason": "my_reason", - "type": "my_warn" - }, - "from_seq_no": 1234, - "retries": 5 - }, - { - "exception": { - "reason": "my_reason", - "type": "my_warn" - }, - "from_seq_no": 1234, - "retries": 5 - } - ], + "read_exceptions": [], "requests": { "failed": { "read": { @@ -70,43 +54,43 @@ }, "outstanding": { "read": { - "count": 8 + "count": 1 }, "write": { - "count": 2 + "count": 0 } }, "successful": { "read": { - "count": 32 + "count": 0 }, "write": { - "count": 16 + "count": 0 } } }, "total_time": { "read": { - "ms": 32768, + "ms": 0, "remote_exec": { - "ms": 16384 + "ms": 0 } }, "write": { - "ms": 16384 + "ms": 0 } }, "write_buffer": { "operation": { - "count": 64 + "count": 0 }, "size": { - "bytes": 1536 + "bytes": 0 } } }, "cluster": { - "id": "8l_zoGznQRmtoX9iSC-goA", + "id": "WocBBA0QRma0sGpdQ7vLfQ", "name": "docker-cluster" } }, @@ -120,7 +104,7 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:37735", + "address": "172.19.0.2:9200", "name": "elasticsearch", "type": "elasticsearch" } diff --git a/metricbeat/module/elasticsearch/ccr/ccr.go b/metricbeat/module/elasticsearch/ccr/ccr.go index bf6cf9c7893..0cc564ea865 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr.go +++ b/metricbeat/module/elasticsearch/ccr/ccr.go @@ -89,7 +89,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventsMapping(r, *info, content) + return eventsMapping(r, *info, content, m.XPackEnabled) } func (m *MetricSet) checkCCRAvailability(currentElasticsearchVersion *common.Version) (message string, err error) { diff --git a/metricbeat/module/elasticsearch/ccr/data.go b/metricbeat/module/elasticsearch/ccr/data.go index 36c6e93ef37..71119599430 100644 --- a/metricbeat/module/elasticsearch/ccr/data.go +++ b/metricbeat/module/elasticsearch/ccr/data.go @@ -20,6 +20,8 @@ package ccr import ( "encoding/json" + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/elastic/beats/v7/libbeat/common" "github.com/joeshaw/multierror" @@ -133,7 +135,7 @@ type response struct { } `json:"follow_stats"` } -func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXpack bool) error { var data response err := json.Unmarshal(content, &data) if err != nil { @@ -156,6 +158,13 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err autoFollow, _ := autoFollowSchema.Apply(data.AutoFollowStats) event.MetricSetFields["auto_follow"] = autoFollow + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) } } diff --git a/metricbeat/module/elasticsearch/ccr/data_test.go b/metricbeat/module/elasticsearch/ccr/data_test.go index 9557e772ee4..3f40a18c025 100644 --- a/metricbeat/module/elasticsearch/ccr/data_test.go +++ b/metricbeat/module/elasticsearch/ccr/data_test.go @@ -46,7 +46,7 @@ func TestEmpty(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - eventsMapping(reporter, info, input) + eventsMapping(reporter, info, input, true) require.Equal(t, 0, len(reporter.GetErrors())) require.Equal(t, 0, len(reporter.GetEvents())) } diff --git a/metricbeat/module/elasticsearch/cluster_stats/_meta/data.json b/metricbeat/module/elasticsearch/cluster_stats/_meta/data.json index 77111c45f6d..a8a5f10362d 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/_meta/data.json +++ b/metricbeat/module/elasticsearch/cluster_stats/_meta/data.json @@ -1,14 +1,14 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "cluster_settings": {}, "elasticsearch": { "cluster": { - "id": "8l_zoGznQRmtoX9iSC-goA", + "id": "WocBBA0QRma0sGpdQ7vLfQ", "name": "docker-cluster", "stats": { + "expiry_date_in_millis": 1638548539004, "indices": { "docs": { - "total": 223 + "total": 81 }, "fielddata": { "memory": { @@ -16,81 +16,95 @@ } }, "shards": { - "count": 8, - "primaries": 8 + "count": 17, + "primaries": 17 }, "store": { "size": { - "bytes": 11701629 + "bytes": 44156720 } }, - "total": 8 + "total": 17 }, "license": { - "status": "", - "type": "platinum", - "expiry_date_in_millis": 0 + "status": "active", + "type": "trial", + "expiry_date_in_millis": 1638548539004 }, "nodes": { "count": 1, "fs": { "available": { - "bytes": 182713794560 + "bytes": 126640869376 }, "total": { - "bytes": 958613114880 + "bytes": 330994548736 } }, "jvm": { "max_uptime": { - "ms": 17857098 + "ms": 795905 }, "memory": { "heap": { "max": { - "bytes": 1073741824 + "bytes": 268435456 }, "used": { - "bytes": 615251232 + "bytes": 166911488 } } } }, "master": 1, "versions": [ - "8.0.0" + "7.14.0" ] }, "stack": { "xpack": { "ccr": { - "available": false, + "available": true, "enabled": true } } }, "state": { - "master_node": "0sZBDd6VQ4ObLacVSh65jw", + "master_node": "f5i3v9hMT_q__q6B9WOo5A", "nodes": { - "0sZBDd6VQ4ObLacVSh65jw": { + "f5i3v9hMT_q__q6B9WOo5A": { "attributes": { - "ml.machine_memory": "33300463616", - "ml.max_open_jobs": "20", + "ml.machine_memory": "33295835136", + "ml.max_jvm_size": "268435456", + "ml.max_open_jobs": "512", "transform.node": "true", "xpack.installed": "true" }, - "ephemeral_id": "nqDXltxJTly70OWy95QfBw", - "name": "7d86b192e7ce", + "ephemeral_id": "aQVRUnEIQ5GA3Qe4uBDqLA", + "name": "27fb1c2fd783", + "roles": [ + "data", + "data_cold", + "data_content", + "data_frozen", + "data_hot", + "data_warm", + "ingest", + "master", + "ml", + "remote_cluster_client", + "transform" + ], "transport_address": "127.0.0.1:9300" } }, - "nodes_hash": -575310727, - "state_uuid": "N0SOO0GZQICpIp19KZ27dg" + "nodes_hash": -19752840, + "state_uuid": "XNIdeSZxQwyItvGfR5fHSw" }, "status": "yellow" } }, - "version": 65 + "version": 106 }, "event": { "dataset": "elasticsearch.cluster.stats", @@ -102,7 +116,7 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:39279", + "address": "172.19.0.2:9200", "type": "elasticsearch" } } \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go index a8757d978fc..78bb16368ee 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go +++ b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go @@ -67,5 +67,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventMapping(r, m.HTTP, *info, content) + return eventMapping(r, m.HTTP, *info, content, m.XPackEnabled) } diff --git a/metricbeat/module/elasticsearch/cluster_stats/data.go b/metricbeat/module/elasticsearch/cluster_stats/data.go index d331beba9da..8de27d71561 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/data.go +++ b/metricbeat/module/elasticsearch/cluster_stats/data.go @@ -212,7 +212,7 @@ func getClusterMetadataSettings(httpClient *helper.HTTP) (common.MapStr, error) return clusterSettings, nil } -func eventMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch.Info, content []byte) error { +func eventMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch.Info, content []byte, isXpack bool) error { var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { @@ -327,6 +327,13 @@ func eventMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch.I event.MetricSetFields = metricSetFields + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) return nil diff --git a/metricbeat/module/elasticsearch/enrich/_meta/data.json b/metricbeat/module/elasticsearch/enrich/_meta/data.json index 49aa1fadfd8..970e7e74116 100644 --- a/metricbeat/module/elasticsearch/enrich/_meta/data.json +++ b/metricbeat/module/elasticsearch/enrich/_meta/data.json @@ -2,23 +2,23 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "elasticsearch": { "cluster": { - "id": "8l_zoGznQRmtoX9iSC-goA", + "id": "WocBBA0QRma0sGpdQ7vLfQ", "name": "docker-cluster" }, "enrich": { "executed_searches": { - "total": 0 + "total": 1 }, "queue": { "size": 0 }, "remote_requests": { "current": 0, - "total": 0 + "total": 1 } }, "node": { - "id": "1sFM8cmSROZYhPxVsiWew" + "id": "f5i3v9hMT_q__q6B9WOo5A" } }, "event": { @@ -31,7 +31,7 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:51380", + "address": "172.19.0.2:9200", "type": "elasticsearch" } } \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/enrich/data.go b/metricbeat/module/elasticsearch/enrich/data.go index 447cbff7d3d..dc105e29f00 100644 --- a/metricbeat/module/elasticsearch/enrich/data.go +++ b/metricbeat/module/elasticsearch/enrich/data.go @@ -20,6 +20,8 @@ package enrich import ( "encoding/json" + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/joeshaw/multierror" "github.com/pkg/errors" @@ -67,7 +69,7 @@ type response struct { CoordinatorStats []map[string]interface{} `json:"coordinator_stats"` } -func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXpack bool) error { var data response err := json.Unmarshal(content, &data) if err != nil { @@ -99,6 +101,9 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err event.MetricSetFields = fields + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + r.Event(event) } @@ -139,6 +144,13 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err event.MetricSetFields.Put("executing_policy.name", policyName) event.MetricSetFields.Put("executing_policy.task", fields) + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) } diff --git a/metricbeat/module/elasticsearch/enrich/data_test.go b/metricbeat/module/elasticsearch/enrich/data_test.go index b2930c6b79a..f3eb8c1be4d 100644 --- a/metricbeat/module/elasticsearch/enrich/data_test.go +++ b/metricbeat/module/elasticsearch/enrich/data_test.go @@ -46,7 +46,7 @@ func TestEmpty(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - eventsMapping(reporter, info, input) + eventsMapping(reporter, info, input, true) require.Equal(t, 0, len(reporter.GetErrors())) require.Equal(t, 0, len(reporter.GetEvents())) } diff --git a/metricbeat/module/elasticsearch/enrich/enrich.go b/metricbeat/module/elasticsearch/enrich/enrich.go index a9eef5480d6..765b4e6393c 100644 --- a/metricbeat/module/elasticsearch/enrich/enrich.go +++ b/metricbeat/module/elasticsearch/enrich/enrich.go @@ -86,7 +86,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventsMapping(r, *info, content) + return eventsMapping(r, *info, content, m.XPackEnabled) } func (m *MetricSet) checkEnrichAvailability(currentElasticsearchVersion *common.Version) (message string, err error) { diff --git a/metricbeat/module/elasticsearch/index/_meta/data.json b/metricbeat/module/elasticsearch/index/_meta/data.json index ab6059386dd..0b81a8e5405 100644 --- a/metricbeat/module/elasticsearch/index/_meta/data.json +++ b/metricbeat/module/elasticsearch/index/_meta/data.json @@ -2,32 +2,32 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "elasticsearch": { "cluster": { - "id": "8l_zoGznQRmtoX9iSC-goA", + "id": "WocBBA0QRma0sGpdQ7vLfQ", "name": "docker-cluster" }, "index": { "hidden": false, - "name": ".kibana-event-log-8.0.0-000001", + "name": ".kibana_task_manager_7.14.0_001", "primaries": { "docs": { - "count": 2 + "count": 14 }, "indexing": { - "index_time_in_millis": 109, - "index_total": 1, + "index_time_in_millis": 493, + "index_total": 719, "throttle_time_in_millis": 0 }, "merges": { - "total_size_in_bytes": 0 + "total_size_in_bytes": 5885647 }, "refresh": { - "total_time_in_millis": 366 + "total_time_in_millis": 2315 }, "segments": { - "count": 2 + "count": 6 }, "store": { - "size_in_bytes": 11301 + "size_in_bytes": 148860 } }, "shards": { @@ -36,44 +36,44 @@ "status": "green", "total": { "docs": { - "count": 2 + "count": 14 }, "fielddata": { "memory_size_in_bytes": 0 }, "indexing": { - "index_time_in_millis": 109, - "index_total": 1, + "index_time_in_millis": 493, + "index_total": 719, "throttle_time_in_millis": 0 }, "merges": { - "total_size_in_bytes": 0 + "total_size_in_bytes": 5885647 }, "refresh": { - "total_time_in_millis": 366 + "total_time_in_millis": 2315 }, "search": { - "query_time_in_millis": 0, - "query_total": 1 + "query_time_in_millis": 807, + "query_total": 1339 }, "segments": { - "count": 2, - "doc_values_memory_in_bytes": 152, - "fixed_bit_set_memory_in_bytes": 96, + "count": 6, + "doc_values_memory_in_bytes": 16616, + "fixed_bit_set_memory_in_bytes": 376, "index_writer_memory_in_bytes": 0, - "memory_in_bytes": 4392, - "norms_memory_in_bytes": 0, + "memory_in_bytes": 36280, + "norms_memory_in_bytes": 1536, "points_memory_in_bytes": 0, - "stored_fields_memory_in_bytes": 976, + "stored_fields_memory_in_bytes": 2960, "term_vectors_memory_in_bytes": 0, - "terms_memory_in_bytes": 3264, + "terms_memory_in_bytes": 15168, "version_map_memory_in_bytes": 0 }, "store": { - "size_in_bytes": 11301 + "size_in_bytes": 148860 } }, - "uuid": "3765e_aCRh28_UoF-iWnuQ" + "uuid": "1E4r6Df2RhOa5pgdPvOZEA" } }, "event": { @@ -86,7 +86,7 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:35043", + "address": "172.19.0.2:9200", "type": "elasticsearch" } -} +} \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/index/data.go b/metricbeat/module/elasticsearch/index/data.go index 11b5c55439d..041b04c6aea 100644 --- a/metricbeat/module/elasticsearch/index/data.go +++ b/metricbeat/module/elasticsearch/index/data.go @@ -136,7 +136,7 @@ type bulkStats struct { AvgSizeInBytes int `json:"avg_size_in_bytes"` } -func eventsMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch.Info, content []byte) error { +func eventsMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch.Info, content []byte, isXpack bool) error { clusterStateMetrics := []string{"routing_table"} clusterState, err := elasticsearch.GetClusterState(httpClient, httpClient.GetURI(), clusterStateMetrics) if err != nil { @@ -191,6 +191,13 @@ func eventsMapping(r mb.ReporterV2, httpClient *helper.HTTP, info elasticsearch. event.MetricSetFields.Put("name", name) delete(event.MetricSetFields, "index") + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) } diff --git a/metricbeat/module/elasticsearch/index/data_test.go b/metricbeat/module/elasticsearch/index/data_test.go index fe339f395f5..8d8a73816e6 100644 --- a/metricbeat/module/elasticsearch/index/data_test.go +++ b/metricbeat/module/elasticsearch/index/data_test.go @@ -77,7 +77,7 @@ func TestEmpty(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - eventsMapping(reporter, httpClient, info, input) + eventsMapping(reporter, httpClient, info, input, true) require.Equal(t, 0, len(reporter.GetEvents())) } diff --git a/metricbeat/module/elasticsearch/index/index.go b/metricbeat/module/elasticsearch/index/index.go index 7fda1ef815a..325931ebb29 100644 --- a/metricbeat/module/elasticsearch/index/index.go +++ b/metricbeat/module/elasticsearch/index/index.go @@ -85,7 +85,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventsMapping(r, m.HTTP, *info, content) + return eventsMapping(r, m.HTTP, *info, content, m.XPackEnabled) } func (m *MetricSet) updateServicePath(esVersion common.Version) error { diff --git a/metricbeat/module/elasticsearch/index_recovery/_meta/data.json b/metricbeat/module/elasticsearch/index_recovery/_meta/data.json index 5d4fd419394..6444aee8160 100644 --- a/metricbeat/module/elasticsearch/index_recovery/_meta/data.json +++ b/metricbeat/module/elasticsearch/index_recovery/_meta/data.json @@ -2,11 +2,11 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "elasticsearch": { "cluster": { - "id": "8l_zoGznQRmtoX9iSC-goA", + "id": "WocBBA0QRma0sGpdQ7vLfQ", "name": "docker-cluster" }, "index": { - "name": ".kibana-event-log-8.0.0-000001", + "name": "users", "recovery": { "id": 0, "index": { @@ -22,19 +22,20 @@ "total_in_bytes": 0 } }, + "name": "users", "primary": true, "source": {}, "stage": "DONE", "start_time": { - "ms": 1605819056123 + "ms": 1635956540240 }, "stop_time": { - "ms": 1605819058696 + "ms": 1635956540265 }, "target": { "host": "127.0.0.1", - "id": "Fkj12lAFQOex0DwK0HMwHw", - "name": "082618b4bb36", + "id": "f5i3v9hMT_q__q6B9WOo5A", + "name": "27fb1c2fd783", "transport_address": "127.0.0.1:9300" }, "translog": { @@ -56,7 +57,7 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:43035", + "address": "172.19.0.2:9200", "name": "elasticsearch", "type": "elasticsearch" } diff --git a/metricbeat/module/elasticsearch/index_recovery/data.go b/metricbeat/module/elasticsearch/index_recovery/data.go index 53e3540850b..2d37bbae6b3 100644 --- a/metricbeat/module/elasticsearch/index_recovery/data.go +++ b/metricbeat/module/elasticsearch/index_recovery/data.go @@ -83,7 +83,7 @@ var ( } ) -func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXpack bool) error { var data map[string]map[string][]map[string]interface{} @@ -117,6 +117,13 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err } event.MetricSetFields.Put("name", indexName) + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) } } diff --git a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go index c37f8fd30ab..eb2eaf93bf6 100644 --- a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go +++ b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go @@ -81,5 +81,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventsMapping(r, *info, content) + return eventsMapping(r, *info, content, m.XPackEnabled) } diff --git a/metricbeat/module/elasticsearch/index_summary/_meta/data.json b/metricbeat/module/elasticsearch/index_summary/_meta/data.json index ab7cc8abdd5..08fbbef7db4 100644 --- a/metricbeat/module/elasticsearch/index_summary/_meta/data.json +++ b/metricbeat/module/elasticsearch/index_summary/_meta/data.json @@ -2,68 +2,68 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "elasticsearch": { "cluster": { - "id": "8l_zoGznQRmtoX9iSC-goA", + "id": "WocBBA0QRma0sGpdQ7vLfQ", "name": "docker-cluster" }, "index": { "summary": { "primaries": { "docs": { - "count": 1257, - "deleted": 11 + "count": 74, + "deleted": 1424 }, "indexing": { "index": { - "count": 1885 + "count": 1495 } }, "search": { "query": { - "count": 81, + "count": 15307, "time": { - "ms": 39 + "ms": 2912 } } }, "segments": { - "count": 20, + "count": 21, "memory": { - "bytes": 190357 + "bytes": 70532 } }, "store": { "size": { - "bytes": 1686190 + "bytes": 44131181 } } }, "total": { "docs": { - "count": 1257, - "deleted": 11 + "count": 74, + "deleted": 1424 }, "indexing": { "index": { - "count": 1885 + "count": 1495 } }, "search": { "query": { - "count": 81, + "count": 15307, "time": { - "ms": 39 + "ms": 2912 } } }, "segments": { - "count": 20, + "count": 21, "memory": { - "bytes": 190357 + "bytes": 70532 } }, "store": { "size": { - "bytes": 1686190 + "bytes": 44131181 } } } @@ -80,7 +80,7 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:32943", + "address": "172.19.0.2:9200", "name": "elasticsearch", "type": "elasticsearch" } diff --git a/metricbeat/module/elasticsearch/index_summary/data.go b/metricbeat/module/elasticsearch/index_summary/data.go index 0177eba1831..e4dc8ce5dff 100644 --- a/metricbeat/module/elasticsearch/index_summary/data.go +++ b/metricbeat/module/elasticsearch/index_summary/data.go @@ -20,6 +20,8 @@ package index_summary import ( "encoding/json" + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" @@ -86,7 +88,7 @@ var bulkStatsDict = c.Dict("bulk", s.Schema{ }, }, c.DictOptional) -func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { +func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXpack bool) error { var all struct { Data map[string]interface{} `json:"_all"` } @@ -111,6 +113,14 @@ func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) erro event.MetricSetFields = fields + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) + return nil } diff --git a/metricbeat/module/elasticsearch/index_summary/data_test.go b/metricbeat/module/elasticsearch/index_summary/data_test.go index 3383b0a6d76..26d1218ea64 100644 --- a/metricbeat/module/elasticsearch/index_summary/data_test.go +++ b/metricbeat/module/elasticsearch/index_summary/data_test.go @@ -91,7 +91,7 @@ func TestEmpty(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - eventMapping(reporter, info, input) + eventMapping(reporter, info, input, true) require.Empty(t, reporter.GetErrors()) require.Equal(t, 1, len(reporter.GetEvents())) } diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary.go b/metricbeat/module/elasticsearch/index_summary/index_summary.go index c163bef237a..b09d94f551d 100644 --- a/metricbeat/module/elasticsearch/index_summary/index_summary.go +++ b/metricbeat/module/elasticsearch/index_summary/index_summary.go @@ -80,5 +80,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return errors.Wrap(err, "failed to get info from Elasticsearch") } - return eventMapping(r, *info, content) + return eventMapping(r, *info, content, m.XPackEnabled) } diff --git a/metricbeat/module/elasticsearch/metricset.go b/metricbeat/module/elasticsearch/metricset.go index a1762793570..7a7b3d863f5 100644 --- a/metricbeat/module/elasticsearch/metricset.go +++ b/metricbeat/module/elasticsearch/metricset.go @@ -78,7 +78,8 @@ type MetricSet struct { mb.BaseMetricSet servicePath string *helper.HTTP - Scope Scope + Scope Scope + XPackEnabled bool } // NewMetricSet creates an metric set that can be used to build other metric @@ -90,9 +91,11 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error) } config := struct { - Scope Scope `config:"scope"` + Scope Scope `config:"scope"` + XPackEnabled bool `config:"xpack.enabled"` }{ - Scope: ScopeNode, + Scope: ScopeNode, + XPackEnabled: false, } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err @@ -103,6 +106,7 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error) servicePath, http, config.Scope, + config.XPackEnabled, } ms.SetServiceURI(servicePath) diff --git a/metricbeat/module/elasticsearch/ml_job/_meta/data.json b/metricbeat/module/elasticsearch/ml_job/_meta/data.json index 934ea661d64..0e79d5bad91 100644 --- a/metricbeat/module/elasticsearch/ml_job/_meta/data.json +++ b/metricbeat/module/elasticsearch/ml_job/_meta/data.json @@ -2,27 +2,27 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "elasticsearch": { "cluster": { - "id": "8l_zoGznQRmtoX9iSC-goA", + "id": "WocBBA0QRma0sGpdQ7vLfQ", "name": "docker-cluster" }, "ml": { "job": { "data_counts": { "invalid_date_count": 0, - "processed_record_count": 1216 + "processed_record_count": 0 }, "forecasts_stats": { - "total": 1 + "total": 0 }, - "id": "low_request_rate", + "id": "total-requests", "model_size": { "memory_status": "ok" }, - "state": "opened" + "state": "closed" } }, "node": { - "id": "a14cf47ef7f2" + "id": "27fb1c2fd783" } }, "event": { @@ -35,7 +35,7 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:38585", + "address": "172.19.0.2:9200", "name": "elasticsearch", "type": "elasticsearch" } diff --git a/metricbeat/module/elasticsearch/ml_job/data.go b/metricbeat/module/elasticsearch/ml_job/data.go index c5cd87c6862..28c296e1e5f 100644 --- a/metricbeat/module/elasticsearch/ml_job/data.go +++ b/metricbeat/module/elasticsearch/ml_job/data.go @@ -53,7 +53,7 @@ type jobsStruct struct { Jobs []map[string]interface{} `json:"jobs"` } -func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXpack bool) error { jobsData := &jobsStruct{} err := json.Unmarshal(content, jobsData) @@ -85,6 +85,13 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err event.MetricSetFields, _ = schema.Apply(job) + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) } diff --git a/metricbeat/module/elasticsearch/ml_job/ml_job.go b/metricbeat/module/elasticsearch/ml_job/ml_job.go index fd9d06526c5..0a4f0817806 100644 --- a/metricbeat/module/elasticsearch/ml_job/ml_job.go +++ b/metricbeat/module/elasticsearch/ml_job/ml_job.go @@ -70,5 +70,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventsMapping(r, *info, content) + return eventsMapping(r, *info, content, m.XPackEnabled) } diff --git a/metricbeat/module/elasticsearch/node/_meta/data.json b/metricbeat/module/elasticsearch/node/_meta/data.json index fa04d1b7561..0dae2d153f2 100644 --- a/metricbeat/module/elasticsearch/node/_meta/data.json +++ b/metricbeat/module/elasticsearch/node/_meta/data.json @@ -1,24 +1,20 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "elasticsearch": { "cluster": { - "id": "wafoCXEDTrGxpYViNueSaA", - "name": "es1" + "id": "WocBBA0QRma0sGpdQ7vLfQ", + "name": "docker-cluster" }, "node": { - "id": "v5gHTHqKSRa4bZ9vbyDy7g", + "id": "f5i3v9hMT_q__q6B9WOo5A", "jvm": { "memory": { "heap": { "init": { - "bytes": 1073741824 + "bytes": 268435456 }, "max": { - "bytes": 1037959168 + "bytes": 268435456 } }, "nonheap": { @@ -30,22 +26,27 @@ } } }, - "version": "11.0.1" + "version": "16.0.1" }, - "name": "es1_1", + "name": "27fb1c2fd783", "process": { "mlockall": false }, - "version": "7.0.0" + "version": "7.14.0" } }, + "event": { + "dataset": "elasticsearch.node", + "duration": 115000, + "module": "elasticsearch" + }, "metricset": { - "host": "127.0.0.1:9200", - "module": "elasticsearch", "name": "node", - "rtt": 115 + "period": 10000 }, "service": { - "name": "elasticsearch" + "address": "172.19.0.2:9200", + "name": "elasticsearch", + "type": "elasticsearch" } -} +} \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/node/data.go b/metricbeat/module/elasticsearch/node/data.go index 56acc36752a..11bb39a01d2 100644 --- a/metricbeat/module/elasticsearch/node/data.go +++ b/metricbeat/module/elasticsearch/node/data.go @@ -20,6 +20,8 @@ package node import ( "encoding/json" + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/joeshaw/multierror" "github.com/pkg/errors" @@ -61,7 +63,7 @@ var ( } ) -func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXpack bool) error { nodesStruct := struct { ClusterName string `json:"cluster_name"` Nodes map[string]map[string]interface{} `json:"nodes"` @@ -91,6 +93,13 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err event.MetricSetFields["id"] = id + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) } diff --git a/metricbeat/module/elasticsearch/node/data_test.go b/metricbeat/module/elasticsearch/node/data_test.go index acc1a409a1d..f8a34e99024 100644 --- a/metricbeat/module/elasticsearch/node/data_test.go +++ b/metricbeat/module/elasticsearch/node/data_test.go @@ -47,6 +47,6 @@ func TestInvalid(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, info, content) + err = eventsMapping(reporter, info, content, true) require.Error(t, err) } diff --git a/metricbeat/module/elasticsearch/node/node.go b/metricbeat/module/elasticsearch/node/node.go index 29587a1dd3e..8543eb351b3 100644 --- a/metricbeat/module/elasticsearch/node/node.go +++ b/metricbeat/module/elasticsearch/node/node.go @@ -74,5 +74,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return errors.Wrap(err, "failed to get info from Elasticsearch") } - return eventsMapping(r, *info, content) + return eventsMapping(r, *info, content, m.XPackEnabled) } diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/data.json b/metricbeat/module/elasticsearch/node_stats/_meta/data.json index 4ae46d3fd98..c6246987ce4 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/data.json +++ b/metricbeat/module/elasticsearch/node_stats/_meta/data.json @@ -2,33 +2,37 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "elasticsearch": { "cluster": { - "id": "w3oo88LcQ1i-7K4f-wrEgQ", + "id": "WocBBA0QRma0sGpdQ7vLfQ", "name": "docker-cluster" }, "node": { - "id": "EjV2AqqvQNq5ZF5cVlaPDQ", + "id": "f5i3v9hMT_q__q6B9WOo5A", "master": true, "mlockall": false, - "name": "foo", + "name": "27fb1c2fd783", "stats": { "fs": { "io_stats": {}, "summary": { "available": { - "bytes": 45897547776 + "bytes": 126640869376 }, "free": { - "bytes": 49114263552 + "bytes": 127357620224 }, "total": { - "bytes": 62725623808 + "bytes": 330994548736 } + }, + "total": { + "available_in_bytes": 126640869376, + "total_in_bytes": 330994548736 } }, "indices": { "docs": { - "count": 9207, - "deleted": 43 + "count": 81, + "deleted": 1424 }, "fielddata": { "memory": { @@ -37,10 +41,10 @@ }, "indexing": { "index_time": { - "ms": 21 + "ms": 2232 }, "index_total": { - "count": 4 + "count": 1502 }, "throttle_time": { "ms": 0 @@ -53,27 +57,27 @@ }, "request_cache": { "memory": { - "bytes": 3736 + "bytes": 30174 } }, "search": { "query_time": { - "ms": 83 + "ms": 2914 }, "query_total": { - "count": 18 + "count": 15325 } }, "segments": { - "count": 63, + "count": 24, "doc_values": { "memory": { - "bytes": 117620 + "bytes": 18672 } }, "fixed_bit_set": { "memory": { - "bytes": 3912 + "bytes": 704 } }, "index_writer": { @@ -82,11 +86,11 @@ } }, "memory": { - "bytes": 330956 + "bytes": 78080 }, "norms": { "memory": { - "bytes": 2688 + "bytes": 3392 } }, "points": { @@ -96,7 +100,7 @@ }, "stored_fields": { "memory": { - "bytes": 31000 + "bytes": 11920 } }, "term_vectors": { @@ -106,7 +110,7 @@ }, "terms": { "memory": { - "bytes": 179648 + "bytes": 44096 } }, "version_map": { @@ -117,7 +121,7 @@ }, "store": { "size": { - "bytes": 18049725 + "bytes": 44156720 } } }, @@ -132,8 +136,8 @@ }, "young": { "collection": { - "count": 10, - "ms": 290 + "count": 200, + "ms": 566 } } } @@ -141,56 +145,25 @@ "mem": { "heap": { "max": { - "bytes": 1073741824 + "bytes": 268435456 }, "used": { - "bytes": 177654272, - "pct": 16 + "bytes": 166911488, + "pct": 62 } } } }, "os": { - "cgroup": { - "cpu": { - "cfs": { - "quota": { - "us": -1 - } - }, - "stat": { - "elapsed_periods": { - "count": 0 - }, - "times_throttled": { - "count": 0 - } - } - }, - "cpuacct": { - "usage": { - "ns": 57724017512 - } - }, - "memory": { - "control_group": "/", - "limit": { - "bytes": "9223372036854771712" - }, - "usage": { - "bytes": "1508503552" - } - } - }, "cpu": { "load_avg": { - "1m": 2.06 + "1m": 2.51 } } }, "process": { "cpu": { - "pct": 32 + "pct": 5 } }, "thread_pool": { @@ -232,8 +205,8 @@ "period": 10000 }, "service": { - "address": "localhost:9200", + "address": "172.19.0.2:9200", "name": "elasticsearch", "type": "elasticsearch" } -} +} \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/node_stats/data.go b/metricbeat/module/elasticsearch/node_stats/data.go index fe00cec0283..fb8c1092560 100644 --- a/metricbeat/module/elasticsearch/node_stats/data.go +++ b/metricbeat/module/elasticsearch/node_stats/data.go @@ -280,7 +280,7 @@ type nodesStruct struct { Nodes map[string]map[string]interface{} `json:"nodes"` } -func eventsMapping(r mb.ReporterV2, m elasticsearch.MetricSetAPI, info elasticsearch.Info, content []byte) error { +func eventsMapping(r mb.ReporterV2, m elasticsearch.MetricSetAPI, info elasticsearch.Info, content []byte, isXpack bool) error { nodeData := &nodesStruct{} err := json.Unmarshal(content, nodeData) if err != nil { @@ -339,6 +339,13 @@ func eventsMapping(r mb.ReporterV2, m elasticsearch.MetricSetAPI, info elasticse event.ModuleFields.Put("node.name", nameStr) event.MetricSetFields.Delete("name") + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) } return errs.Err() diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats.go b/metricbeat/module/elasticsearch/node_stats/node_stats.go index 3d8192f6a71..18c60a1c6c1 100644 --- a/metricbeat/module/elasticsearch/node_stats/node_stats.go +++ b/metricbeat/module/elasticsearch/node_stats/node_stats.go @@ -70,7 +70,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventsMapping(r, m.MetricSet, *info, content) + return eventsMapping(r, m.MetricSet, *info, content, m.XPackEnabled) } func (m *MetricSet) updateServiceURI() error { diff --git a/metricbeat/module/elasticsearch/pending_tasks/data.go b/metricbeat/module/elasticsearch/pending_tasks/data.go index 0f414750bd6..e3ab8d6909b 100644 --- a/metricbeat/module/elasticsearch/pending_tasks/data.go +++ b/metricbeat/module/elasticsearch/pending_tasks/data.go @@ -40,7 +40,7 @@ var ( } ) -func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte, isXpack bool) error { tasksStruct := struct { Tasks []map[string]interface{} `json:"tasks"` }{} @@ -71,6 +71,13 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err continue } + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) } diff --git a/metricbeat/module/elasticsearch/pending_tasks/data_test.go b/metricbeat/module/elasticsearch/pending_tasks/data_test.go index a1bf1a220d4..3a14d7c6258 100644 --- a/metricbeat/module/elasticsearch/pending_tasks/data_test.go +++ b/metricbeat/module/elasticsearch/pending_tasks/data_test.go @@ -47,7 +47,7 @@ func TestEmptyQueueShouldGiveNoError(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, info, content) + err = eventsMapping(reporter, info, content, true) require.NoError(t, err) } @@ -57,7 +57,7 @@ func TestNotEmptyQueueShouldGiveNoError(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, info, content) + err = eventsMapping(reporter, info, content, true) require.NoError(t, err) require.True(t, len(reporter.GetEvents()) >= 1) require.Zero(t, len(reporter.GetErrors())) @@ -69,7 +69,7 @@ func TestEmptyQueueShouldGiveZeroEvent(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, info, content) + err = eventsMapping(reporter, info, content, true) require.Zero(t, len(reporter.GetEvents())) require.Zero(t, len(reporter.GetErrors())) } @@ -80,7 +80,7 @@ func TestNotEmptyQueueShouldGiveSeveralEvents(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, info, content) + err = eventsMapping(reporter, info, content, true) require.Equal(t, 3, len(reporter.GetEvents())) require.Zero(t, len(reporter.GetErrors())) } @@ -91,7 +91,7 @@ func TestInvalidJsonForRequiredFieldShouldThrowError(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, info, content) + err = eventsMapping(reporter, info, content, true) require.Error(t, err) } @@ -101,7 +101,7 @@ func TestInvalidJsonForBadFormatShouldThrowError(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, info, content) + err = eventsMapping(reporter, info, content, true) require.Error(t, err) } @@ -198,16 +198,16 @@ func TestEventsMappedMatchToContentReceived(t *testing.T) { }}, } - for _, testCase := range testCases { + for iter, testCase := range testCases { content, err := ioutil.ReadFile(testCase.given) require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, info, content) + err = eventsMapping(reporter, info, content, false) events := reporter.GetEvents() if !reflect.DeepEqual(testCase.expected, events) { - t.Errorf("Expected %v, actual: %v", testCase.expected, events) + t.Errorf("Iteration %d, Given '%s'. Expected %v, actual: %v", iter, testCase.given, testCase.expected, events) } } } diff --git a/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go b/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go index 68add3146cc..f8719e042a4 100644 --- a/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go +++ b/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go @@ -74,5 +74,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventsMapping(r, *info, content) + return eventsMapping(r, *info, content, m.XPackEnabled) } diff --git a/metricbeat/module/elasticsearch/shard/_meta/data.json b/metricbeat/module/elasticsearch/shard/_meta/data.json index c64df616017..b6278e225f5 100644 --- a/metricbeat/module/elasticsearch/shard/_meta/data.json +++ b/metricbeat/module/elasticsearch/shard/_meta/data.json @@ -2,17 +2,17 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "elasticsearch": { "cluster": { - "id": "tMjf3CQ_TyCXNfcoR9eTWw", + "id": "WocBBA0QRma0sGpdQ7vLfQ", "name": "docker-cluster", "state": { - "id": "n-UoXaqYRoOe9qAC76IG6A" + "id": "XNIdeSZxQwyItvGfR5fHSw" } }, "index": { - "name": ".apm-agent-configuration" + "name": "pied_piper" }, "node": { - "id": "hx-oJ1-aT_-5pRG22JMI1Q" + "id": "f5i3v9hMT_q__q6B9WOo5A" }, "shard": { "number": 0, @@ -22,8 +22,8 @@ "name": null }, "source_node": { - "name": "1fb2aa83efac", - "uuid": "hx-oJ1-aT_-5pRG22JMI1Q" + "name": "27fb1c2fd783", + "uuid": "f5i3v9hMT_q__q6B9WOo5A" }, "state": "STARTED" } @@ -38,7 +38,7 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:44037", + "address": "172.19.0.2:9200", "type": "elasticsearch" } } \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/shard/data.go b/metricbeat/module/elasticsearch/shard/data.go index 56f32373f79..2dd2f93c023 100644 --- a/metricbeat/module/elasticsearch/shard/data.go +++ b/metricbeat/module/elasticsearch/shard/data.go @@ -57,7 +57,7 @@ type stateStruct struct { } `json:"routing_table"` } -func eventsMapping(r mb.ReporterV2, content []byte) error { +func eventsMapping(r mb.ReporterV2, content []byte, isXpack bool) error { stateData := &stateStruct{} err := json.Unmarshal(content, stateData) if err != nil { @@ -131,6 +131,13 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.MetricSetFields.Put("relocating_node.name", relocatingNode) event.MetricSetFields.Put("relocating_node.id", relocatingNode) + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) + event.Index = index + } + r.Event(event) } } diff --git a/metricbeat/module/elasticsearch/shard/data_test.go b/metricbeat/module/elasticsearch/shard/data_test.go index 5ba34901a52..9abb100484b 100644 --- a/metricbeat/module/elasticsearch/shard/data_test.go +++ b/metricbeat/module/elasticsearch/shard/data_test.go @@ -40,7 +40,7 @@ func TestStats(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - eventsMapping(reporter, input) + eventsMapping(reporter, input, true) require.True(t, len(reporter.GetEvents()) >= 1) require.Equal(t, 0, len(reporter.GetErrors())) diff --git a/metricbeat/module/elasticsearch/shard/shard.go b/metricbeat/module/elasticsearch/shard/shard.go index 1483b053451..63650d55596 100644 --- a/metricbeat/module/elasticsearch/shard/shard.go +++ b/metricbeat/module/elasticsearch/shard/shard.go @@ -64,5 +64,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventsMapping(r, content) + return eventsMapping(r, content, m.XPackEnabled) } diff --git a/metricbeat/module/elasticsearch/testing.go b/metricbeat/module/elasticsearch/testing.go index f76e95ed0e7..2b3f605b1b5 100644 --- a/metricbeat/module/elasticsearch/testing.go +++ b/metricbeat/module/elasticsearch/testing.go @@ -55,7 +55,7 @@ func TestMapper(t *testing.T, glob string, mapper func(mb.ReporterV2, []byte) er } // TestMapperWithInfo tests mapping methods with Info fields -func TestMapperWithInfo(t *testing.T, glob string, mapper func(mb.ReporterV2, Info, []byte) error) { +func TestMapperWithInfo(t *testing.T, glob string, mapper func(mb.ReporterV2, Info, []byte, bool) error) { files, err := filepath.Glob(glob) require.NoError(t, err) // Makes sure glob matches at least 1 file @@ -72,7 +72,7 @@ func TestMapperWithInfo(t *testing.T, glob string, mapper func(mb.ReporterV2, In require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = mapper(reporter, info, input) + err = mapper(reporter, info, input, true) require.NoError(t, err) require.True(t, len(reporter.GetEvents()) >= 1) require.Equal(t, 0, len(reporter.GetErrors())) @@ -81,7 +81,7 @@ func TestMapperWithInfo(t *testing.T, glob string, mapper func(mb.ReporterV2, In } // TestMapperWithMetricSetAndInfo tests mapping methods with Info fields -func TestMapperWithMetricSetAndInfo(t *testing.T, glob string, ms MetricSetAPI, mapper func(mb.ReporterV2, MetricSetAPI, Info, []byte) error) { +func TestMapperWithMetricSetAndInfo(t *testing.T, glob string, ms MetricSetAPI, mapper func(mb.ReporterV2, MetricSetAPI, Info, []byte, bool) error) { files, err := filepath.Glob(glob) require.NoError(t, err) // Makes sure glob matches at least 1 file @@ -98,7 +98,7 @@ func TestMapperWithMetricSetAndInfo(t *testing.T, glob string, ms MetricSetAPI, require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = mapper(reporter, ms, info, input) + err = mapper(reporter, ms, info, input, true) require.NoError(t, err) require.True(t, len(reporter.GetEvents()) >= 1) require.Equal(t, 0, len(reporter.GetErrors())) @@ -108,7 +108,7 @@ func TestMapperWithMetricSetAndInfo(t *testing.T, glob string, ms MetricSetAPI, // TestMapperWithMetricSetAndInfo tests mapping methods with Info fields func TestMapperWithHttpHelper(t *testing.T, glob string, httpClient *helper.HTTP, - mapper func(mb.ReporterV2, *helper.HTTP, Info, []byte) error) { + mapper func(mb.ReporterV2, *helper.HTTP, Info, []byte, bool) error) { files, err := filepath.Glob(glob) require.NoError(t, err) // Makes sure glob matches at least 1 file @@ -130,7 +130,7 @@ func TestMapperWithHttpHelper(t *testing.T, glob string, httpClient *helper.HTTP require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = mapper(reporter, httpClient, info, input) + err = mapper(reporter, httpClient, info, input, true) require.NoError(t, err) require.True(t, len(reporter.GetEvents()) >= 1) require.Equal(t, 0, len(reporter.GetErrors())) diff --git a/metricbeat/module/kibana/settings/_meta/data.json b/metricbeat/module/kibana/settings/_meta/data.json index 85c1c53c543..89211564306 100644 --- a/metricbeat/module/kibana/settings/_meta/data.json +++ b/metricbeat/module/kibana/settings/_meta/data.json @@ -8,20 +8,20 @@ "kibana": { "elasticsearch": { "cluster": { - "id": "9Am6m3bHTJ6YrNg1pmac2Q" + "id": "WocBBA0QRma0sGpdQ7vLfQ" } }, "settings": { - "host": "0", + "host": "0.0.0.0", "index": ".kibana", "locale": "en", - "name": "kibana", + "name": "b04775fa6831", "port": 5601, "snapshot": false, "status": "green", - "transport_address": "0:5601", - "uuid": "2076c0f0-fd6e-4ce0-bf04-59541bd3ba0c", - "version": "7.10.0" + "transport_address": "0.0.0.0:5601", + "uuid": "f87393a7-e8ae-48fd-af1e-91f229fd93fd", + "version": "7.14.0" } }, "metricset": { @@ -29,7 +29,7 @@ "period": 10000 }, "service": { - "address": "172.20.0.3:5601", + "address": "172.19.0.3:5601", "type": "kibana" } } \ No newline at end of file diff --git a/metricbeat/module/kibana/stats/_meta/data.json b/metricbeat/module/kibana/stats/_meta/data.json index 761cc5bb7ac..dde2be30ac3 100644 --- a/metricbeat/module/kibana/stats/_meta/data.json +++ b/metricbeat/module/kibana/stats/_meta/data.json @@ -8,71 +8,71 @@ "kibana": { "elasticsearch": { "cluster": { - "id": "9Am6m3bHTJ6YrNg1pmac2Q" + "id": "WocBBA0QRma0sGpdQ7vLfQ" } }, "stats": { - "concurrent_connections": 0, + "concurrent_connections": 4, "host": { - "name": "0" + "name": "0.0.0.0" }, - "index": "kibana", - "name": "kibana", + "index": "b04775fa6831", + "name": "b04775fa6831", "os": { "distro": "CentOS", - "distroRelease": "CentOS-8.2.2004", + "distroRelease": "CentOS-8.4.2105\n", "load": { - "15m": 2.291015625, - "1m": 2.34912109375, - "5m": 2.49072265625 + "15m": 1.12, + "1m": 0.97, + "5m": 1.01 }, "memory": { - "free_in_bytes": 12719185920, - "total_in_bytes": 33291984896, - "used_in_bytes": 20572798976 + "free_in_bytes": 323694592, + "total_in_bytes": 33295835136, + "used_in_bytes": 32972140544 }, "platform": "linux", - "platformRelease": "linux-5.10.14-arch1-1" + "platformRelease": "linux-5.14.13-200.fc34.x86_64" }, "process": { "event_loop_delay": { - "ms": 0.7828890010714531 + "ms": 0.14306800067424774 }, "memory": { "heap": { "size_limit": { - "bytes": 1526909922 + "bytes": 4345298944 }, "total": { - "bytes": 238911488 + "bytes": 264982528 }, "used": { - "bytes": 169990928 + "bytes": 226086648 } }, "resident_set_size": { - "bytes": 354504704 + "bytes": 346976256 } }, "uptime": { - "ms": 986355 + "ms": 79081 } }, "request": { "disconnects": 0, - "total": 4 + "total": 7 }, "response_time": { "avg": { - "ms": 279 + "ms": 74 }, "max": { - "ms": 300 + "ms": 389 } }, "snapshot": false, "status": "green", - "transport_address": "0:5601" + "transport_address": "0.0.0.0:5601" } }, "metricset": { @@ -80,12 +80,12 @@ "period": 10000 }, "process": { - "pid": 7 + "pid": 1222 }, "service": { - "address": "172.20.0.3:5601", - "id": "2076c0f0-fd6e-4ce0-bf04-59541bd3ba0c", + "address": "172.19.0.3:5601", + "id": "f87393a7-e8ae-48fd-af1e-91f229fd93fd", "type": "kibana", - "version": "7.10.0" + "version": "7.14.0" } } \ No newline at end of file diff --git a/metricbeat/module/kibana/stats/data.go b/metricbeat/module/kibana/stats/data.go index 6417e6fd984..79e6454f805 100644 --- a/metricbeat/module/kibana/stats/data.go +++ b/metricbeat/module/kibana/stats/data.go @@ -103,7 +103,7 @@ var ( }) ) -func eventMapping(r mb.ReporterV2, content []byte) error { +func eventMapping(r mb.ReporterV2, content []byte, isXpack bool) error { var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { @@ -168,6 +168,13 @@ func eventMapping(r mb.ReporterV2, content []byte) error { event.MetricSetFields = dataFields + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Kibana) + event.Index = index + } + r.Event(event) return nil diff --git a/metricbeat/module/kibana/stats/data_test.go b/metricbeat/module/kibana/stats/data_test.go index aeda8279787..71bc6130605 100644 --- a/metricbeat/module/kibana/stats/data_test.go +++ b/metricbeat/module/kibana/stats/data_test.go @@ -40,7 +40,7 @@ func TestEventMapping(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, input) + err = eventMapping(reporter, input, true) require.NoError(t, err, f) require.True(t, len(reporter.GetEvents()) >= 1, f) require.Equal(t, 0, len(reporter.GetErrors()), f) diff --git a/metricbeat/module/kibana/stats/stats.go b/metricbeat/module/kibana/stats/stats.go index 889ced3c7f5..5551e57823f 100644 --- a/metricbeat/module/kibana/stats/stats.go +++ b/metricbeat/module/kibana/stats/stats.go @@ -118,5 +118,5 @@ func (m *MetricSet) fetchStats(r mb.ReporterV2) error { return err } - return eventMapping(r, content) + return eventMapping(r, content, m.XPackEnabled) } diff --git a/metricbeat/module/kibana/status/data.go b/metricbeat/module/kibana/status/data.go index 41dd5e493c0..ee842b8fcd5 100644 --- a/metricbeat/module/kibana/status/data.go +++ b/metricbeat/module/kibana/status/data.go @@ -52,7 +52,7 @@ var ( } ) -func eventMapping(r mb.ReporterV2, content []byte) error { +func eventMapping(r mb.ReporterV2, content []byte, isXpack bool) error { var event mb.Event event.RootFields = common.MapStr{} event.RootFields.Put("service.name", kibana.ModuleName) @@ -83,6 +83,13 @@ func eventMapping(r mb.ReporterV2, content []byte) error { event.MetricSetFields = dataFields + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Kibana) + event.Index = index + } + r.Event(event) return nil } diff --git a/metricbeat/module/kibana/status/data_test.go b/metricbeat/module/kibana/status/data_test.go index 89cace6d04c..800b14a12c0 100644 --- a/metricbeat/module/kibana/status/data_test.go +++ b/metricbeat/module/kibana/status/data_test.go @@ -35,7 +35,7 @@ func TestEventMapping(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, content) + err = eventMapping(reporter, content, true) require.NoError(t, err, f) require.True(t, len(reporter.GetEvents()) >= 1, f) require.Equal(t, 0, len(reporter.GetErrors()), f) diff --git a/metricbeat/module/kibana/status/status.go b/metricbeat/module/kibana/status/status.go index 55e95918507..32b38723324 100644 --- a/metricbeat/module/kibana/status/status.go +++ b/metricbeat/module/kibana/status/status.go @@ -74,5 +74,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - return eventMapping(r, content) + return eventMapping(r, content, m.XPackEnabled) } diff --git a/metricbeat/module/logstash/logstash.go b/metricbeat/module/logstash/logstash.go index 500b46107ca..7670d1c3ef6 100644 --- a/metricbeat/module/logstash/logstash.go +++ b/metricbeat/module/logstash/logstash.go @@ -54,6 +54,7 @@ var PipelineGraphAPIsAvailableVersion = common.MustNewVersion("7.3.0") type MetricSet struct { mb.BaseMetricSet *helper.HTTP + XPackEnabled bool } type Graph struct { @@ -87,9 +88,19 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { return nil, err } + config := struct { + XPackEnabled bool `config:"xpack.enabled"` + }{ + XPackEnabled: false, + } + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + return &MetricSet{ base, http, + config.XPackEnabled, }, nil } diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index a5b45f7002f..a884644075a 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -41,7 +41,6 @@ var metricSets = []string{ } func TestFetch(t *testing.T) { - t.Skip("flaky test: https://github.com/elastic/beats/issues/25043") service := compose.EnsureUpWithTimeout(t, 300, "logstash") for _, metricSet := range metricSets { diff --git a/metricbeat/module/logstash/node/_meta/data.json b/metricbeat/module/logstash/node/_meta/data.json index ae1ab625ce3..a847bea944a 100644 --- a/metricbeat/module/logstash/node/_meta/data.json +++ b/metricbeat/module/logstash/node/_meta/data.json @@ -1,99 +1,43 @@ { - "@timestamp": "2020-10-05T10:50:11.757Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0", - "_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a" - }, - "host": { - "name": "mcastro" - }, - "agent": { - "ephemeral_id": "c4b22628-7b30-4a5d-8e28-7b6de81c9974", - "id": "803dfdba-e638-4590-a2de-80cb1cebe78d", - "name": "mcastro", - "type": "metricbeat", - "version": "8.0.0" - }, + "@timestamp": "2017-10-12T08:05:34.853Z", "event": { - "duration": 9740086, "dataset": "logstash.node", + "duration": 115000, "module": "logstash" }, - "metricset": { - "name": "node", - "period": 10000 - }, - "service": { - "address": "localhost:9600", - "type": "logstash" - }, "logstash": { + "cluster": { + "id": "VUwnkX_lTzCFP9VUoXT1IQ" + }, "node": { - "host": "2cb47f6e0eab", - "version": "8.0.0", + "host": "7dc1b688baf4", + "id": "9a1f83e1-52b9-4625-a98a-6aa336f41719", "jvm": { - "version": "11.0.5" + "version": "11.0.10" }, - "id": "4cc683ce-3ddc-46e3-bea3-aefbf37bc082", "state": { "pipeline": { - "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd", - "ephemeral_id": "afb1a50a-95f0-484a-b7d7-e683ddddc75a", + "id": "main", + "hash": "5022893e09c573cc517eef12da3df428d41a45370e168b19031f5b6b0da3db22", + "ephemeral_id": "0a6062d1-85e8-4e9b-b943-d80573912692", "representation": { "graph": { - "edges": [ - { - "from": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", - "id": "b3db599ec6ae0b9493158bd7024dcd922c8a3e76295c37fef0da440086bf3f8c", - "to": "__QUEUE__", - "type": "plain" - }, - { - "type": "plain", - "from": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", - "id": "9db20a77b3e1eb91229a50bd33388425d59725f9093e076a37e6565f8d5a20ad", - "to": "__QUEUE__" - }, - { - "id": "9b2bc571e978746fb9b55b83521a6603c3c940144cde0e3f4296298cea6585cf", - "to": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", - "type": "plain", - "from": "__QUEUE__" - } - ], "vertices": [ { "config_name": "beats", "explicit_id": false, - "id": "1bf3a9cc73ceb7c3a9cbe885df249b23f3496c52a342a6d513153cc865d78182", + "id": "293e8492626c637c237bf394279270356589b77d285ea009a555e19977779693", "meta": { "source": { - "line": 2, - "protocol": "file", "column": 3, - "id": "/usr/share/logstash/pipeline/default.conf" + "id": "/usr/share/logstash/pipeline/logstash.conf", + "line": 2, + "protocol": "file" } }, "plugin_type": "input", "type": "plugin" }, - { - "plugin_type": "input", - "type": "plugin", - "config_name": "beats", - "explicit_id": false, - "id": "71b91bc85b66ab25c5fb16e63db4dd7111c183f96d1f18e19078051ed5fc74f7", - "meta": { - "source": { - "protocol": "file", - "column": 3, - "id": "/usr/share/logstash/pipeline/default.conf", - "line": 7 - } - } - }, { "explicit_id": false, "id": "__QUEUE__", @@ -103,32 +47,58 @@ { "config_name": "elasticsearch", "explicit_id": false, - "id": "a339cb309b29181703c6adf321da3d639f5b60713de5a1e5519ebfea069556d8", + "id": "942f820402532e034dc5b27a680547732f49dc4b5e48df0475d280eb31382e0d", "meta": { "source": { - "id": "/usr/share/logstash/pipeline/default.conf", - "line": 17, - "protocol": "file", - "column": 3 + "column": 3, + "id": "/usr/share/logstash/pipeline/logstash.conf", + "line": 8, + "protocol": "file" } }, "plugin_type": "output", "type": "plugin" } + ], + "edges": [ + { + "from": "293e8492626c637c237bf394279270356589b77d285ea009a555e19977779693", + "id": "af355ece7f2645e789bd864dd4745756782431bb85e6ea2e5b3ad6d546bca889", + "to": "__QUEUE__", + "type": "plain" + }, + { + "from": "__QUEUE__", + "id": "7eb9677f7cb798312f3dc1ac5ab2469a9f2166f6144da511fe339a31f9fd9b38", + "to": "942f820402532e034dc5b27a680547732f49dc4b5e48df0475d280eb31382e0d", + "type": "plain" + } ] }, "type": "lir", "version": "0.0.0", - "hash": "3000c3abf87d4dfa4a57aaf6af0a1f5bee2e0fc1c48a8e8636e2a33d7d2e91dd" + "hash": "5022893e09c573cc517eef12da3df428d41a45370e168b19031f5b6b0da3db22" }, "batch_size": 125, - "workers": 12, - "id": "main" + "workers": 12 } - } + }, + "version": "7.12.0" } }, - "ecs": { - "version": "1.5.0" + "metricset": { + "name": "node", + "period": 10000 + }, + "process": { + "pid": 1 + }, + "service": { + "address": "172.20.0.3:9600", + "hostname": "7dc1b688baf4", + "id": "9a1f83e1-52b9-4625-a98a-6aa336f41719", + "name": "logstash", + "type": "logstash", + "version": "7.12.0" } -} +} \ No newline at end of file diff --git a/metricbeat/module/logstash/node/data.go b/metricbeat/module/logstash/node/data.go index 6117683623c..bb9f51c9847 100644 --- a/metricbeat/module/logstash/node/data.go +++ b/metricbeat/module/logstash/node/data.go @@ -81,7 +81,7 @@ func commonFieldsMapping(event *mb.Event, fields common.MapStr) error { return nil } -func eventMapping(r mb.ReporterV2, content []byte, pipelines []logstash.PipelineState, overrideClusterUUID string) error { +func eventMapping(r mb.ReporterV2, content []byte, pipelines []logstash.PipelineState, overrideClusterUUID string, isXpack bool) error { var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { @@ -126,6 +126,13 @@ func eventMapping(r mb.ReporterV2, content []byte, pipelines []logstash.Pipeline event.ID = pipeline.EphemeralID + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Logstash) + event.Index = index + } + r.Event(event) } } diff --git a/metricbeat/module/logstash/node/node.go b/metricbeat/module/logstash/node/node.go index a70706558e8..e209a6c8918 100644 --- a/metricbeat/module/logstash/node/node.go +++ b/metricbeat/module/logstash/node/node.go @@ -79,7 +79,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - if err = eventMapping(r, content, pipelinesContent, overrideClusterUUID); err != nil { + if err = eventMapping(r, content, pipelinesContent, overrideClusterUUID, m.XPackEnabled); err != nil { return err } diff --git a/metricbeat/module/logstash/node_stats/_meta/data.json b/metricbeat/module/logstash/node_stats/_meta/data.json index 4b24bae61b2..5343737c3b4 100644 --- a/metricbeat/module/logstash/node_stats/_meta/data.json +++ b/metricbeat/module/logstash/node_stats/_meta/data.json @@ -6,6 +6,7 @@ "module": "logstash" }, "logstash": { + "cluster.id": "VUwnkX_lTzCFP9VUoXT1IQ", "node": { "stats": { "events": { @@ -19,20 +20,20 @@ "collectors": { "old": { "collection_count": 3, - "collection_time_in_millis": 449 + "collection_time_in_millis": 738 }, "young": { - "collection_count": 7, - "collection_time_in_millis": 231 + "collection_count": 9, + "collection_time_in_millis": 422 } } }, "mem": { "heap_max_in_bytes": 1037959168, - "heap_used_in_bytes": 371211952, - "heap_used_percent": 35 + "heap_used_in_bytes": 309034360, + "heap_used_percent": 29 }, - "uptime_in_millis": 33546 + "uptime_in_millis": 551958 }, "reloads": { "failures": 0, @@ -42,41 +43,34 @@ "events_count": 0 }, "process": { - "open_file_descriptors": 103, - "max_file_descriptors": 524288, + "open_file_descriptors": 171, + "max_file_descriptors": 1024, "cpu": { - "percent": 1 + "percent": 0 } }, "os": { "cpu": { "percent": 0, "load_average": { - "15m": 1.7, - "1m": 2.52, - "5m": 1.8 + "15m": 1.69, + "1m": 1.51, + "5m": 1.69 } }, "cgroup": { - "cpuacct": { - "control_group": "/user.slice", - "usage_nanos": 24041075965919 - }, + "cpuacct": null, "cpu": { - "stat": { - "number_of_elapsed_periods": 0, - "number_of_times_throttled": 0, - "time_throttled_nanos": 0 - }, - "control_group": "/user.slice" + "stat": null, + "control_group": "" } } }, "pipelines": [ { "id": "main", - "hash": "6984380a58d40b7ebe6ba7ba8fc3134b95ffadfcef9ff328f0864ae002943792", - "ephemeral_id": "eb977feb-0052-4651-a995-39e545213723", + "hash": "5022893e09c573cc517eef12da3df428d41a45370e168b19031f5b6b0da3db22", + "ephemeral_id": "0a6062d1-85e8-4e9b-b943-d80573912692", "events": { "duration_in_millis": 0, "filtered": 0, @@ -94,24 +88,39 @@ "queue_size_in_bytes": 0, "type": "memory" }, - "vertices": null + "vertices": [ + { + "events_out": 0, + "id": "293e8492626c637c237bf394279270356589b77d285ea009a555e19977779693", + "pipeline_ephemeral_id": "0a6062d1-85e8-4e9b-b943-d80573912692", + "queue_push_duration_in_millis": 0 + }, + { + "cluster_uuid": "VUwnkX_lTzCFP9VUoXT1IQ", + "duration_in_millis": 6011, + "events_in": 0, + "events_out": 0, + "id": "942f820402532e034dc5b27a680547732f49dc4b5e48df0475d280eb31382e0d", + "pipeline_ephemeral_id": "0a6062d1-85e8-4e9b-b943-d80573912692" + } + ] } ], "logstash": { - "uuid": "2fc9524f-a36b-4611-82e4-5246d3ac4714", - "ephemeral_id": "10311adf-a5ba-4920-ada2-8dadc54618fe", - "name": "anonymous", - "host": "anonymous", - "version": "7.8.1", + "uuid": "9a1f83e1-52b9-4625-a98a-6aa336f41719", + "ephemeral_id": "cead1f28-1e2d-4ef3-9f0b-93c10447e6e7", + "name": "7dc1b688baf4", + "host": "7dc1b688baf4", + "version": "7.12.0", "snapshot": false, "status": "green", - "http_address": "127.0.0.1:9600", + "http_address": "0.0.0.0:9600", "pipeline": { "batch_size": 125, "workers": 12 } }, - "timestamp": "2020-12-09T16:16:17.796Z" + "timestamp": "2021-11-03T16:22:10.651Z" } } }, @@ -120,11 +129,11 @@ "period": 10000 }, "service": { - "address": "127.0.0.1:33437", - "hostname": "anonymous", + "address": "172.20.0.3:9600", + "hostname": "7dc1b688baf4", "id": "", "name": "logstash", "type": "logstash", - "version": "7.8.1" + "version": "7.12.0" } } \ No newline at end of file diff --git a/metricbeat/module/logstash/node_stats/data.go b/metricbeat/module/logstash/node_stats/data.go index fe9732498f5..85eaaa8b493 100644 --- a/metricbeat/module/logstash/node_stats/data.go +++ b/metricbeat/module/logstash/node_stats/data.go @@ -21,6 +21,8 @@ import ( "encoding/json" "time" + "github.com/elastic/beats/v7/metricbeat/helper/elastic" + "github.com/elastic/beats/v7/metricbeat/module/logstash" "github.com/pkg/errors" @@ -142,7 +144,7 @@ type PipelineStats struct { Vertices []map[string]interface{} `json:"vertices"` } -func eventMapping(r mb.ReporterV2, content []byte) error { +func eventMapping(r mb.ReporterV2, content []byte, isXpack bool) error { var nodeStats NodeStats err := json.Unmarshal(content, &nodeStats) if err != nil { @@ -205,6 +207,13 @@ func eventMapping(r mb.ReporterV2, content []byte) error { event.ModuleFields["cluster.id"] = clusterUUID } + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.Logstash) + event.Index = index + } + r.Event(event) } diff --git a/metricbeat/module/logstash/node_stats/data_test.go b/metricbeat/module/logstash/node_stats/data_test.go index 58591349342..ab133c4d900 100644 --- a/metricbeat/module/logstash/node_stats/data_test.go +++ b/metricbeat/module/logstash/node_stats/data_test.go @@ -35,7 +35,6 @@ import ( ) func TestEventMapping(t *testing.T) { - files, err := filepath.Glob("./_meta/test/node_stats.*.json") require.NoError(t, err) @@ -44,7 +43,7 @@ func TestEventMapping(t *testing.T) { require.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventMapping(reporter, input) + err = eventMapping(reporter, input, true) require.NoError(t, err, f) require.True(t, len(reporter.GetEvents()) >= 1, f) diff --git a/metricbeat/module/logstash/node_stats/node_stats.go b/metricbeat/module/logstash/node_stats/node_stats.go index 681183cfe6d..f7500348400 100644 --- a/metricbeat/module/logstash/node_stats/node_stats.go +++ b/metricbeat/module/logstash/node_stats/node_stats.go @@ -77,7 +77,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - if err = eventMapping(r, content); err != nil { + if err = eventMapping(r, content, m.XPackEnabled); err != nil { return err }