Cherry-pick to 7.9: Only request wildcard expansion for hidden indice…

…s if supported (elastic#20938) (elastic#20962)

* Only request wildcard expansion for hidden indices if supported (elastic#20938)

* Refactoring: inverting logic to make room for another case

* Expand hidden indices wildcards if monitored ES supports that option

* Adding CHANGELOG entry

* Fixing formatting

* Avoid unnecessary setting

* Removing unnecessary suffix existence checks

* Fixing feature version

* Add test cases to unit test

* Updating test
# Conflicts:
#	metricbeat/module/elasticsearch/index/index_test.go

* Fixing CHANGELOG

CHANGELOG.next.asciidoc

@@ -97,6 +97,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix storage metricset to allow config without region/zone. {issue}17623[17623] {pull}17624[17624]
 - Fix overflow on Prometheus rates when new buckets are added on the go. {pull}17753[17753]
 - Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
+- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
 
 *Packetbeat*
 

metricbeat/module/elasticsearch/elasticsearch.go

@@ -60,17 +60,23 @@ func NewModule(base mb.BaseModule) (mb.Module, error) {
 	return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName))
 }
 
-// CCRStatsAPIAvailableVersion is the version of Elasticsearch since when the CCR stats API is available.
-var CCRStatsAPIAvailableVersion = common.MustNewVersion("6.5.0")
+var (
+	// CCRStatsAPIAvailableVersion is the version of Elasticsearch since when the CCR stats API is available.
+	CCRStatsAPIAvailableVersion = common.MustNewVersion("6.5.0")
+
+	// EnrichStatsAPIAvailableVersion is the version of Elasticsearch since when the Enrich stats API is available.
+	EnrichStatsAPIAvailableVersion = common.MustNewVersion("7.5.0")
 
-// EnrichStatsAPIAvailableVersion is the version of Elasticsearch since when the Enrich stats API is available.
-var EnrichStatsAPIAvailableVersion = common.MustNewVersion("7.5.0")
+	// BulkStatsAvailableVersion is the version since when bulk indexing stats are available
+	BulkStatsAvailableVersion = common.MustNewVersion("8.0.0")
 
-// BulkStatsAvailableVersion is the version since when bulk indexing stats are available
-var BulkStatsAvailableVersion = common.MustNewVersion("8.0.0")
+	//ExpandWildcardsHiddenAvailableVersion is the version since when the "expand_wildcards" query parameter to
+	// the Indices Stats API can accept "hidden" as a value.
+	ExpandWildcardsHiddenAvailableVersion = common.MustNewVersion("7.7.0")
 
-// Global clusterIdCache. Assumption is that the same node id never can belong to a different cluster id.
-var clusterIDCache = map[string]string{}
+	// Global clusterIdCache. Assumption is that the same node id never can belong to a different cluster id.
+	clusterIDCache = map[string]string{}
+)
 
 // ModuleName is the name of this module.
 const ModuleName = "elasticsearch"

metricbeat/module/elasticsearch/index/index.go

@@ -38,8 +38,12 @@ func init() {
 }
 
 const (
-	statsMetrics = "docs,fielddata,indexing,merge,search,segments,store,refresh,query_cache,request_cache"
-	statsPath    = "/_stats/" + statsMetrics + "?filter_path=indices&expand_wildcards=open,hidden"
+	statsMetrics    = "docs,fielddata,indexing,merge,search,segments,store,refresh,query_cache,request_cache"
+	expandWildcards = "expand_wildcards=open"
+	statsPath       = "/_stats/" + statsMetrics + "?filter_path=indices&" + expandWildcards
+
+	bulkSuffix   = ",bulk"
+	hiddenSuffix = ",hidden"
 )
 
 // MetricSet type defines all fields of the MetricSet
@@ -118,21 +122,18 @@ func (m *MetricSet) updateServicePath(esVersion common.Version) error {
 
 func getServicePath(esVersion common.Version) (string, error) {
 	currPath := statsPath
-	if esVersion.LessThan(elasticsearch.BulkStatsAvailableVersion) {
-		// Can't request bulk stats so don't change service URI
-		return currPath, nil
-	}
-
 	u, err := url.Parse(currPath)
 	if err != nil {
 		return "", err
 	}
 
-	if strings.HasSuffix(u.Path, ",bulk") {
-		// Bulk stats already being requested so don't change service URI
-		return currPath, nil
+	if !esVersion.LessThan(elasticsearch.BulkStatsAvailableVersion) {
+		u.Path += bulkSuffix
+	}
+
+	if !esVersion.LessThan(elasticsearch.ExpandWildcardsHiddenAvailableVersion) {
+		u.RawQuery = strings.Replace(u.RawQuery, expandWildcards, expandWildcards+hiddenSuffix, 1)
 	}
 
-	u.Path += ",bulk"
 	return u.String(), nil
 }

metricbeat/module/elasticsearch/index/index_test.go

@@ -27,18 +27,29 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestGetServiceURI(t *testing.T) {
+func TestGetServiceURIExpectedPath(t *testing.T) {
+	path770 := strings.Replace(statsPath, expandWildcards, expandWildcards+hiddenSuffix, 1)
+	path800 := strings.Replace(path770, statsMetrics, statsMetrics+bulkSuffix, 1)
+
 	tests := map[string]struct {
 		esVersion    *common.Version
 		expectedPath string
 	}{
 		"bulk_stats_unavailable": {
-			esVersion:    common.MustNewVersion("7.7.0"),
+			esVersion:    common.MustNewVersion("7.6.0"),
 			expectedPath: statsPath,
 		},
 		"bulk_stats_available": {
 			esVersion:    common.MustNewVersion("8.0.0"),
-			expectedPath: strings.Replace(statsPath, statsMetrics, statsMetrics+",bulk", 1),
+			expectedPath: path800,
+		},
+		"expand_wildcards_hidden_unavailable": {
+			esVersion:    common.MustNewVersion("7.6.0"),
+			expectedPath: statsPath,
+		},
+		"expand_wildcards_hidden_available": {
+			esVersion:    common.MustNewVersion("7.7.0"),
+			expectedPath: path770,
 		},
 	}
 
@@ -52,6 +63,9 @@ func TestGetServiceURI(t *testing.T) {
 }
 
 func TestGetServiceURIMultipleCalls(t *testing.T) {
+	path := strings.Replace(statsPath, expandWildcards, expandWildcards+hiddenSuffix, 1)
+	path = strings.Replace(path, statsMetrics, statsMetrics+bulkSuffix, 1)
+
 	err := quick.Check(func(r uint) bool {
 		numCalls := 2 + (r % 10) // between 2 and 11
 
@@ -64,7 +78,7 @@ func TestGetServiceURIMultipleCalls(t *testing.T) {
 			}
 		}
 
-		return err == nil && uri == strings.Replace(statsPath, statsMetrics, statsMetrics+",bulk", 1)
+		return err == nil && uri == path
 	}, nil)
 	require.NoError(t, err)
 }