Metricbeat enterprise search module: add xpack.enabled support

CHANGELOG.next.asciidoc

@@ -175,6 +175,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add k8s metadata in state_cronjob metricset. {pull}29572[29572]
 - Add `elasticsearch.cluster.id` field to Beat and Kibana modules. {pull}29577[29577]
 - Add `elasticsearch.cluster.id` field to Logstash module. {pull}29625[29625]
+- Add `xpack.enabled` support for Enterprise Search module. {pull}29871[29871]
 
 *Packetbeat*
 
@@ -211,9 +212,3 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 ==== Known Issue
 
 *Journalbeat*
-
-
-
-
-
-

metricbeat/docs/modules/enterprisesearch.asciidoc

@@ -20,6 +20,17 @@ The module has been tested with Enterprise Search versions 7.16.0 and higher. Ve
 === Usage
 The Enterprise Search module requires a set of credentials (a username and a password) for an Elasticserch user for a user that has a `monitor` https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html#privileges-list-cluster[cluster privilege].
 
+[float]
+=== Usage for {stack} Monitoring
+
+The Enterprise Search module can be used to collect metrics shown in our {stack-monitor-app}
+UI in {kib}. To enable this usage, set `xpack.enabled: true` in configuration.
+
+NOTE: When this module is used for {stack} Monitoring, it sends metrics to the
+monitoring index instead of the default index typically used by {metricbeat}.
+For more details about the monitoring index, see
+{ref}/config-monitoring-indices.html[Configuring indices for monitoring].
+
 
 [float]
 === Example configuration
@@ -35,8 +46,8 @@ metricbeat.modules:
   enabled: true
   period: 10s
   hosts: ["http://localhost:3002"]
-  username: elastic
-  password: changeme
+  #username: "user"
+  #password: "secret"
 ----
 
 This module supports TLS connections when using `ssl` config field, as described in <<configuration-ssl>>.

metricbeat/helper/elastic/elastic.go

@@ -44,6 +44,9 @@ const (
 
 	// Beats product
 	Beats
+
+	// Enterprise Search product
+	EnterpriseSearch
 )
 
 func (p Product) xPackMonitoringIndexString() string {
@@ -52,6 +55,7 @@ func (p Product) xPackMonitoringIndexString() string {
 		"kibana",
 		"logstash",
 		"beats",
+		"ent-search",
 	}
 
 	if int(p) < 0 || int(p) > len(indexProductNames) {
@@ -67,6 +71,7 @@ func (p Product) String() string {
 		"kibana",
 		"logstash",
 		"beats",
+		"enterprisesearch",
 	}
 
 	if int(p) < 0 || int(p) > len(productNames) {

x-pack/metricbeat/metricbeat.reference.yml

@@ -510,8 +510,8 @@ metricbeat.modules:
   enabled: true
   period: 10s
   hosts: ["http://localhost:3002"]
-  username: elastic
-  password: changeme
+  #username: "user"
+  #password: "secret"
 
 #------------------------------ Envoyproxy Module ------------------------------
 - module: envoyproxy

x-pack/metricbeat/module/enterprisesearch/_meta/config-xpack.yml

@@ -0,0 +1,8 @@
+- module: enterprisesearch
+  xpack.enabled: true
+  metricsets: ["health", "stats"]
+  enabled: true
+  period: 10s
+  hosts: ["http://localhost:3002"]
+  #username: "user"
+  #password: "secret"

x-pack/metricbeat/module/enterprisesearch/_meta/config.yml

@@ -3,5 +3,5 @@
   enabled: true
   period: 10s
   hosts: ["http://localhost:3002"]
-  username: elastic
-  password: changeme
+  #username: "user"
+  #password: "secret"

x-pack/metricbeat/module/enterprisesearch/_meta/docs.asciidoc

@@ -7,3 +7,14 @@ The module has been tested with Enterprise Search versions 7.16.0 and higher. Ve
 [float]
 === Usage
 The Enterprise Search module requires a set of credentials (a username and a password) for an Elasticserch user for a user that has a `monitor` https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html#privileges-list-cluster[cluster privilege].
+
+[float]
+=== Usage for {stack} Monitoring
+
+The Enterprise Search module can be used to collect metrics shown in our {stack-monitor-app}
+UI in {kib}. To enable this usage, set `xpack.enabled: true` in configuration.
+
+NOTE: When this module is used for {stack} Monitoring, it sends metrics to the
+monitoring index instead of the default index typically used by {metricbeat}.
+For more details about the monitoring index, see
+{ref}/config-monitoring-indices.html[Configuring indices for monitoring].

x-pack/metricbeat/module/enterprisesearch/docker-compose.yml

@@ -24,7 +24,7 @@ services:
       - 3002:3002
 
   elasticsearch:
-    image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.15.0}-1
+    image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.0.0-SNAPSHOT}-1
     build:
       args:
         ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.15.0}

x-pack/metricbeat/module/enterprisesearch/health/data.go

@@ -13,6 +13,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	s "github.com/elastic/beats/v7/libbeat/common/schema"
 	c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface"
+	"github.com/elastic/beats/v7/metricbeat/helper/elastic"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
@@ -73,7 +74,7 @@ var (
 	}
 )
 
-func eventMapping(report mb.ReporterV2, input []byte) error {
+func eventMapping(report mb.ReporterV2, input []byte, isXpack bool) error {
 	var data map[string]interface{}
 	err := json.Unmarshal(input, &data)
 	if err != nil {
@@ -115,6 +116,13 @@ func eventMapping(report mb.ReporterV2, input []byte) error {
 	// Set the process info we have collected
 	data["process"] = process
 
+	// xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*`
+	// When using Agent, the index name is overwritten anyways.
+	if isXpack {
+		index := elastic.MakeXPackMonitoringIndexName(elastic.EnterpriseSearch)
+		event.Index = index
+	}
+
 	event.MetricSetFields, err = schema.Apply(data)
 	if err != nil {
 		errs = append(errs, errors.Wrap(err, "failure to apply health schema"))

x-pack/metricbeat/module/enterprisesearch/health/health.go

@@ -38,7 +38,8 @@ func init() {
 
 type MetricSet struct {
 	mb.BaseMetricSet
-	http *helper.HTTP
+	http         *helper.HTTP
+	XPackEnabled bool
 }
 
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
@@ -48,9 +49,19 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if err != nil {
 		return nil, err
 	}
+	config := struct {
+		XPackEnabled bool `config:"xpack.enabled"`
+	}{
+		XPackEnabled: false,
+	}
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
 		base,
 		http,
+		config.XPackEnabled,
 	}, nil
 }
 
@@ -63,7 +74,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 		return errors.Wrap(err, "error in fetch")
 	}
 
-	err = eventMapping(report, content)
+	err = eventMapping(report, content, m.XPackEnabled)
 	if err != nil {
 		return errors.Wrap(err, "error converting event")
 	}

x-pack/metricbeat/module/enterprisesearch/stats/data.go

@@ -13,6 +13,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 	s "github.com/elastic/beats/v7/libbeat/common/schema"
 	c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface"
+	"github.com/elastic/beats/v7/metricbeat/helper/elastic"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
@@ -155,7 +156,7 @@ var (
 	}
 )
 
-func eventMapping(report mb.ReporterV2, input []byte) error {
+func eventMapping(report mb.ReporterV2, input []byte, isXpack bool) error {
 	var data map[string]interface{}
 	err := json.Unmarshal(input, &data)
 	if err != nil {
@@ -185,6 +186,13 @@ func eventMapping(report mb.ReporterV2, input []byte) error {
 		errs = append(errs, errors.New("queues is not a map"))
 	}
 
+	// xpack.enabled in config using standalone metricbeat writes to `.monitoring` instead of `metricbeat-*`
+	// When using Agent, the index name is overwritten anyways.
+	if isXpack {
+		index := elastic.MakeXPackMonitoringIndexName(elastic.EnterpriseSearch)
+		event.Index = index
+	}
+
 	event.MetricSetFields, err = schema.Apply(data)
 	if err != nil {
 		errs = append(errs, errors.Wrap(err, "failure to apply stats schema"))

x-pack/metricbeat/module/enterprisesearch/stats/stats.go

@@ -38,7 +38,8 @@ func init() {
 
 type MetricSet struct {
 	mb.BaseMetricSet
-	http *helper.HTTP
+	http         *helper.HTTP
+	XPackEnabled bool
 }
 
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
@@ -48,9 +49,19 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	config := struct {
+		XPackEnabled bool `config:"xpack.enabled"`
+	}{
+		XPackEnabled: false,
+	}
+	if err := base.Module().UnpackConfig(&config); err != nil {
+		return nil, err
+	}
 	return &MetricSet{
 		base,
 		http,
+		config.XPackEnabled,
 	}, nil
 }
 
@@ -63,7 +74,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error {
 		return errors.Wrap(err, "error in fetch")
 	}
 
-	err = eventMapping(report, content)
+	err = eventMapping(report, content, m.XPackEnabled)
 	if err != nil {
 		return errors.Wrap(err, "error converting event")
 	}

x-pack/metricbeat/modules.d/enterprisesearch-xpack.yml.disabled

@@ -0,0 +1,11 @@
+# Module: enterprisesearch
+# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-enterprisesearch.html
+
+- module: enterprisesearch
+  xpack.enabled: true
+  metricsets: ["health", "stats"]
+  enabled: true
+  period: 10s
+  hosts: ["http://localhost:3002"]
+  #username: "user"
+  #password: "secret"

x-pack/metricbeat/modules.d/enterprisesearch.yml.disabled

@@ -6,5 +6,5 @@
   enabled: true
   period: 10s
   hosts: ["http://localhost:3002"]
-  username: elastic
-  password: changeme
+  #username: "user"
+  #password: "secret"