diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a2a9ba049eb..f8095f363dc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -175,6 +175,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add k8s metadata in state_cronjob metricset. {pull}29572[29572] - Add `elasticsearch.cluster.id` field to Beat and Kibana modules. {pull}29577[29577] - Add `elasticsearch.cluster.id` field to Logstash module. {pull}29625[29625] +- Add `xpack.enabled` support for Enterprise Search module. {pull}29871[29871] *Packetbeat* @@ -211,9 +212,3 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d ==== Known Issue *Journalbeat* - - - - - - diff --git a/metricbeat/docs/modules/enterprisesearch.asciidoc b/metricbeat/docs/modules/enterprisesearch.asciidoc index 1f34a3158f1..a7c0948012a 100644 --- a/metricbeat/docs/modules/enterprisesearch.asciidoc +++ b/metricbeat/docs/modules/enterprisesearch.asciidoc @@ -20,6 +20,17 @@ The module has been tested with Enterprise Search versions 7.16.0 and higher. Ve === Usage The Enterprise Search module requires a set of credentials (a username and a password) for an Elasticserch user for a user that has a `monitor` https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html#privileges-list-cluster[cluster privilege]. +[float] +=== Usage for {stack} Monitoring + +The Enterprise Search module can be used to collect metrics shown in our {stack-monitor-app} +UI in {kib}. To enable this usage, set `xpack.enabled: true` in configuration. + +NOTE: When this module is used for {stack} Monitoring, it sends metrics to the +monitoring index instead of the default index typically used by {metricbeat}. +For more details about the monitoring index, see +{ref}/config-monitoring-indices.html[Configuring indices for monitoring]. + [float] === Example configuration @@ -35,8 +46,8 @@ metricbeat.modules: enabled: true period: 10s hosts: ["http://localhost:3002"] - username: elastic - password: changeme + #username: "user" + #password: "secret" ---- This module supports TLS connections when using `ssl` config field, as described in <>. diff --git a/metricbeat/helper/elastic/elastic.go b/metricbeat/helper/elastic/elastic.go index 837ff615064..c51179747e1 100644 --- a/metricbeat/helper/elastic/elastic.go +++ b/metricbeat/helper/elastic/elastic.go @@ -44,6 +44,9 @@ const ( // Beats product Beats + + // Enterprise Search product + EnterpriseSearch ) func (p Product) xPackMonitoringIndexString() string { @@ -52,6 +55,7 @@ func (p Product) xPackMonitoringIndexString() string { "kibana", "logstash", "beats", + "ent-search", } if int(p) < 0 || int(p) > len(indexProductNames) { @@ -67,6 +71,7 @@ func (p Product) String() string { "kibana", "logstash", "beats", + "enterprisesearch", } if int(p) < 0 || int(p) > len(productNames) { diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 2c3a35d801b..61875105aae 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -510,8 +510,8 @@ metricbeat.modules: enabled: true period: 10s hosts: ["http://localhost:3002"] - username: elastic - password: changeme + #username: "user" + #password: "secret" #------------------------------ Envoyproxy Module ------------------------------ - module: envoyproxy diff --git a/x-pack/metricbeat/module/enterprisesearch/_meta/config-xpack.yml b/x-pack/metricbeat/module/enterprisesearch/_meta/config-xpack.yml new file mode 100644 index 00000000000..d80e6d349b6 --- /dev/null +++ b/x-pack/metricbeat/module/enterprisesearch/_meta/config-xpack.yml @@ -0,0 +1,8 @@ +- module: enterprisesearch + xpack.enabled: true + metricsets: ["health", "stats"] + enabled: true + period: 10s + hosts: ["http://localhost:3002"] + #username: "user" + #password: "secret" diff --git a/x-pack/metricbeat/module/enterprisesearch/_meta/config.yml b/x-pack/metricbeat/module/enterprisesearch/_meta/config.yml index 7c2efca0a61..e90fa79f9ff 100644 --- a/x-pack/metricbeat/module/enterprisesearch/_meta/config.yml +++ b/x-pack/metricbeat/module/enterprisesearch/_meta/config.yml @@ -3,5 +3,5 @@ enabled: true period: 10s hosts: ["http://localhost:3002"] - username: elastic - password: changeme + #username: "user" + #password: "secret" diff --git a/x-pack/metricbeat/module/enterprisesearch/_meta/docs.asciidoc b/x-pack/metricbeat/module/enterprisesearch/_meta/docs.asciidoc index 3251d02c09a..e417e57b682 100644 --- a/x-pack/metricbeat/module/enterprisesearch/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/enterprisesearch/_meta/docs.asciidoc @@ -7,3 +7,14 @@ The module has been tested with Enterprise Search versions 7.16.0 and higher. Ve [float] === Usage The Enterprise Search module requires a set of credentials (a username and a password) for an Elasticserch user for a user that has a `monitor` https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html#privileges-list-cluster[cluster privilege]. + +[float] +=== Usage for {stack} Monitoring + +The Enterprise Search module can be used to collect metrics shown in our {stack-monitor-app} +UI in {kib}. To enable this usage, set `xpack.enabled: true` in configuration. + +NOTE: When this module is used for {stack} Monitoring, it sends metrics to the +monitoring index instead of the default index typically used by {metricbeat}. +For more details about the monitoring index, see +{ref}/config-monitoring-indices.html[Configuring indices for monitoring]. diff --git a/x-pack/metricbeat/module/enterprisesearch/docker-compose.yml b/x-pack/metricbeat/module/enterprisesearch/docker-compose.yml index 3e9dbfdf9bf..09d7addb71c 100644 --- a/x-pack/metricbeat/module/enterprisesearch/docker-compose.yml +++ b/x-pack/metricbeat/module/enterprisesearch/docker-compose.yml @@ -24,7 +24,7 @@ services: - 3002:3002 elasticsearch: - image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.15.0}-1 + image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.0.0-SNAPSHOT}-1 build: args: ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.15.0} diff --git a/x-pack/metricbeat/module/enterprisesearch/health/data.go b/x-pack/metricbeat/module/enterprisesearch/health/data.go index 517b0d1fbbc..32fa3849788 100644 --- a/x-pack/metricbeat/module/enterprisesearch/health/data.go +++ b/x-pack/metricbeat/module/enterprisesearch/health/data.go @@ -13,6 +13,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/helper/elastic" "github.com/elastic/beats/v7/metricbeat/mb" ) @@ -73,7 +74,7 @@ var ( } ) -func eventMapping(report mb.ReporterV2, input []byte) error { +func eventMapping(report mb.ReporterV2, input []byte, isXpack bool) error { var data map[string]interface{} err := json.Unmarshal(input, &data) if err != nil { @@ -115,6 +116,13 @@ func eventMapping(report mb.ReporterV2, input []byte) error { // Set the process info we have collected data["process"] = process + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.EnterpriseSearch) + event.Index = index + } + event.MetricSetFields, err = schema.Apply(data) if err != nil { errs = append(errs, errors.Wrap(err, "failure to apply health schema")) diff --git a/x-pack/metricbeat/module/enterprisesearch/health/health.go b/x-pack/metricbeat/module/enterprisesearch/health/health.go index e649748c2b7..d263314bf95 100644 --- a/x-pack/metricbeat/module/enterprisesearch/health/health.go +++ b/x-pack/metricbeat/module/enterprisesearch/health/health.go @@ -38,7 +38,8 @@ func init() { type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP + http *helper.HTTP + XPackEnabled bool } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { @@ -48,9 +49,19 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err != nil { return nil, err } + config := struct { + XPackEnabled bool `config:"xpack.enabled"` + }{ + XPackEnabled: false, + } + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + return &MetricSet{ base, http, + config.XPackEnabled, }, nil } @@ -63,7 +74,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return errors.Wrap(err, "error in fetch") } - err = eventMapping(report, content) + err = eventMapping(report, content, m.XPackEnabled) if err != nil { return errors.Wrap(err, "error converting event") } diff --git a/x-pack/metricbeat/module/enterprisesearch/stats/data.go b/x-pack/metricbeat/module/enterprisesearch/stats/data.go index 049145dda19..d11767b316e 100644 --- a/x-pack/metricbeat/module/enterprisesearch/stats/data.go +++ b/x-pack/metricbeat/module/enterprisesearch/stats/data.go @@ -13,6 +13,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/helper/elastic" "github.com/elastic/beats/v7/metricbeat/mb" ) @@ -155,7 +156,7 @@ var ( } ) -func eventMapping(report mb.ReporterV2, input []byte) error { +func eventMapping(report mb.ReporterV2, input []byte, isXpack bool) error { var data map[string]interface{} err := json.Unmarshal(input, &data) if err != nil { @@ -185,6 +186,13 @@ func eventMapping(report mb.ReporterV2, input []byte) error { errs = append(errs, errors.New("queues is not a map")) } + // xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*` + // When using Agent, the index name is overwritten anyways. + if isXpack { + index := elastic.MakeXPackMonitoringIndexName(elastic.EnterpriseSearch) + event.Index = index + } + event.MetricSetFields, err = schema.Apply(data) if err != nil { errs = append(errs, errors.Wrap(err, "failure to apply stats schema")) diff --git a/x-pack/metricbeat/module/enterprisesearch/stats/stats.go b/x-pack/metricbeat/module/enterprisesearch/stats/stats.go index 8a1f2036235..c59b3481af4 100644 --- a/x-pack/metricbeat/module/enterprisesearch/stats/stats.go +++ b/x-pack/metricbeat/module/enterprisesearch/stats/stats.go @@ -38,7 +38,8 @@ func init() { type MetricSet struct { mb.BaseMetricSet - http *helper.HTTP + http *helper.HTTP + XPackEnabled bool } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { @@ -48,9 +49,19 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err != nil { return nil, err } + + config := struct { + XPackEnabled bool `config:"xpack.enabled"` + }{ + XPackEnabled: false, + } + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } return &MetricSet{ base, http, + config.XPackEnabled, }, nil } @@ -63,7 +74,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { return errors.Wrap(err, "error in fetch") } - err = eventMapping(report, content) + err = eventMapping(report, content, m.XPackEnabled) if err != nil { return errors.Wrap(err, "error converting event") } diff --git a/x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled b/x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled new file mode 100644 index 00000000000..e42dde843c2 --- /dev/null +++ b/x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled @@ -0,0 +1,11 @@ +# Module: enterprisesearch +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-enterprisesearch.html + +- module: enterprisesearch + xpack.enabled: true + metricsets: ["health", "stats"] + enabled: true + period: 10s + hosts: ["http://localhost:3002"] + #username: "user" + #password: "secret" diff --git a/x-pack/metricbeat/modules.d/enterprisesearch.yml.disabled b/x-pack/metricbeat/modules.d/enterprisesearch.yml.disabled index 78c9ca01448..241791cc203 100644 --- a/x-pack/metricbeat/modules.d/enterprisesearch.yml.disabled +++ b/x-pack/metricbeat/modules.d/enterprisesearch.yml.disabled @@ -6,5 +6,5 @@ enabled: true period: 10s hosts: ["http://localhost:3002"] - username: elastic - password: changeme + #username: "user" + #password: "secret"