Report index hidden setting in index stats

metricbeat/module/elasticsearch/elasticsearch.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -359,6 +360,45 @@ func GetXPack(http *helper.HTTP, resetURI string) (XPack, error) {
 	return xpack, err
 }
 
+type IndexSettings struct {
+	Hidden bool
+}
+
+// GetIndicesSettings returns a map of index names to their settings.
+// Note that as of now it is optimized to fetch only the "hidden" index setting to keep the memory
+// footprint of this function call as low as possible.
+func GetIndicesSettings(http *helper.HTTP, resetURI string) (map[string]IndexSettings, error) {
+	content, err := fetchPath(http, resetURI, "*/_settings", "filter_path=*.settings.index.hidden&expand_wildcards=all")
+
+	if err != nil {
+		return nil, errors.Wrap(err, "could not fetch indices settings")
+	}
+
+	var resp map[string]struct {
+		Settings struct {
+			Index struct {
+				Hidden string `json:"hidden"`
+			} `json:"index"`
+		} `json:"settings"`
+	}
+
+	err = json.Unmarshal(content, &resp)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not parse indices settings response")
+	}
+
+	ret := make(map[string]IndexSettings, len(resp))
+	for index, settings := range resp {
+		isHidden, err := strconv.ParseBool(settings.Settings.Index.Hidden)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not parse hidden setting as boolean")
+		}
+		ret[index] = IndexSettings{Hidden: isHidden}
+	}
+
+	return ret, nil
+}
+
 // IsMLockAllEnabled returns if the given Elasticsearch node has mlockall enabled
 func IsMLockAllEnabled(http *helper.HTTP, resetURI, nodeID string) (bool, error) {
 	content, err := fetchPath(http, resetURI, "_nodes/"+nodeID, "filter_path=nodes.*.process.mlockall")

metricbeat/module/elasticsearch/elasticsearch_integration_test.go

@@ -226,8 +226,10 @@ func TestGetAllIndices(t *testing.T) {
 			switch idx.Index {
 			case indexVisible:
 				idxVisibleExists = true
+				require.False(t, idx.Hidden)
 			case indexHidden:
 				idxHiddenExists = true
+				require.True(t, idx.Hidden)
 			}
 		}
 

metricbeat/module/elasticsearch/index/data_xpack.go

@@ -49,6 +49,7 @@ type Index struct {
 	Index   string     `json:"index"`
 	Created int64      `json:"created"`
 	Status  string     `json:"status"`
+	Hidden  bool       `json:"hidden"`
 	Shards  shardStats `json:"shards"`
 }
 
@@ -141,11 +142,21 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info,
 		return errors.Wrap(err, "failure parsing Indices Stats Elasticsearch API response")
 	}
 
+	indicesSettings, err := elasticsearch.GetIndicesSettings(m.HTTP, m.HTTP.GetURI())
+	if err != nil {
+		return errors.Wrap(err, "failure retrieving indices settings from Elasticsearch")
+	}
+
 	var errs multierror.Errors
 	for name, idx := range indicesStats.Indices {
 		event := mb.Event{}
 		idx.Index = name
 
+		settings, exists := indicesSettings[name]
+		if exists {
+			idx.Hidden = settings.Hidden
+		}
+
 		err = addClusterStateFields(&idx, clusterState)
 		if err != nil {
 			errs = append(errs, errors.Wrap(err, "failure adding cluster state fields"))