[elasticsearch] support api_key (elastic#36274)

* support api_key in es module

* changelog entry

* add unit test

* fix errorf call

* update documentation

* mage update

* x-pack mage update

CHANGELOG.next.asciidoc

@@ -187,6 +187,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Resolve statsd module's prematurely halting of metrics parsing upon encountering an invalid packet. {pull}35075[35075]
 - Fix the gap in fetching forecast API metrics at the end of each month for Azure billing module  {pull}36142[36142]
 - Add option in SQL module to execute queries for all dbs. {pull}35688[35688]
+- Add support for api_key authentication in elasticsearch module  {pull}36274[36274]
 
 *Osquerybeat*
 

metricbeat/docs/modules/elasticsearch.asciidoc

@@ -71,6 +71,7 @@ metricbeat.modules:
   hosts: ["http://localhost:9200"]
   #username: "elastic"
   #password: "changeme"
+  #api_key: "foo:bar"
   #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
 
   #index_recovery.active_only: true

metricbeat/metricbeat.reference.yml

@@ -282,6 +282,7 @@ metricbeat.modules:
   hosts: ["http://localhost:9200"]
   #username: "elastic"
   #password: "changeme"
+  #api_key: "foo:bar"
   #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
 
   #index_recovery.active_only: true

metricbeat/module/elasticsearch/_meta/config-xpack.yml

@@ -4,4 +4,5 @@
   hosts: ["http://localhost:9200"]
   #username: "user"
   #password: "secret"
+  #api_key: "foo:bar"
 

metricbeat/module/elasticsearch/_meta/config.reference.yml

@@ -12,6 +12,7 @@
   hosts: ["http://localhost:9200"]
   #username: "elastic"
   #password: "changeme"
+  #api_key: "foo:bar"
   #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
 
   #index_recovery.active_only: true

metricbeat/module/elasticsearch/_meta/config.yml

@@ -6,3 +6,4 @@
   hosts: ["http://localhost:9200"]
   #username: "user"
   #password: "secret"
+  #api_key: "foo:bar"

metricbeat/module/elasticsearch/metricset.go

@@ -18,6 +18,7 @@
 package elasticsearch
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -90,19 +91,29 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error)
 		return nil, err
 	}
 
-	http.SetHeaderDefault(productorigin.Header, productorigin.Beats)
-
 	config := struct {
-		Scope        Scope `config:"scope"`
-		XPackEnabled bool  `config:"xpack.enabled"`
+		Scope        Scope  `config:"scope"`
+		XPackEnabled bool   `config:"xpack.enabled"`
+		ApiKey       string `config:"api_key"`
 	}{
 		Scope:        ScopeNode,
 		XPackEnabled: false,
+		ApiKey:       "",
 	}
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
 
+	http.SetHeaderDefault(productorigin.Header, productorigin.Beats)
+
+	if config.ApiKey != "" {
+		hostData := base.HostData()
+		if hostData.User != "" || hostData.Password != "" {
+			return nil, fmt.Errorf("cannot set both api_key and username/password")
+		}
+		http.SetHeader("Authorization", "ApiKey "+base64.StdEncoding.EncodeToString([]byte(config.ApiKey)))
+	}
+
 	ms := &MetricSet{
 		base,
 		servicePath,

metricbeat/module/elasticsearch/node/node_test.go

@@ -20,6 +20,7 @@
 package node
 
 import (
+	"encoding/base64"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -78,4 +79,48 @@ func TestFetch(t *testing.T) {
 			t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint())
 		})
 	}
+
+	t.Run("with api key", func(t *testing.T) {
+		apiKey := "foo:bar"
+		expectedHeader := "ApiKey " + base64.StdEncoding.EncodeToString([]byte(apiKey))
+
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch r.RequestURI {
+			case "/_nodes/_local":
+				apiKey := r.Header.Get("Authorization")
+				if apiKey != expectedHeader {
+					t.Errorf("expected api key to be %s but got %s", expectedHeader, apiKey)
+				}
+				w.WriteHeader(200)
+				w.Header().Set("Content-Type", "application/json;")
+				w.Write([]byte([]byte("{}")))
+
+			case "/":
+				apiKey := r.Header.Get("Authorization")
+				if apiKey != expectedHeader {
+					t.Errorf("expected api key to be %s but got %s", expectedHeader, apiKey)
+				}
+
+				w.WriteHeader(200)
+				w.Header().Set("Content-Type", "application/json")
+				w.Write([]byte("{}"))
+
+			default:
+				t.FailNow()
+			}
+
+		}))
+		defer server.Close()
+
+		config := map[string]interface{}{
+			"module":     "elasticsearch",
+			"metricsets": []string{"node"},
+			"hosts":      []string{server.URL},
+			"api_key":    "foo:bar",
+		}
+		reporter := &mbtest.CapturingReporterV2{}
+
+		metricSet := mbtest.NewReportingMetricSetV2Error(t, config)
+		metricSet.Fetch(reporter)
+	})
 }

metricbeat/modules.d/elasticsearch-xpack.yml.disabled

@@ -7,4 +7,5 @@
   hosts: ["http://localhost:9200"]
   #username: "user"
   #password: "secret"
+  #api_key: "foo:bar"
 

metricbeat/modules.d/elasticsearch.yml.disabled

@@ -9,3 +9,4 @@
   hosts: ["http://localhost:9200"]
   #username: "user"
   #password: "secret"
+  #api_key: "foo:bar"

x-pack/metricbeat/metricbeat.reference.yml

@@ -525,6 +525,7 @@ metricbeat.modules:
   hosts: ["http://localhost:9200"]
   #username: "elastic"
   #password: "changeme"
+  #api_key: "foo:bar"
   #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
 
   #index_recovery.active_only: true