Allow setting index field limit during stack monitoring e2e test. (#8284

) * Allow setting index field limit during stack monitoring e2e test. * Use beats setup to handle initial index settings. --------- Signed-off-by: Michael Montgomery <[email protected]>
elastic · Dec 3, 2024 · 7f816b6 · 7f816b6
1 parent 1e644a9
commit 7f816b6
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 15 deletions.
diff --git a/config/recipes/beats/stack_monitoring.yaml b/config/recipes/beats/stack_monitoring.yaml
@@ -61,6 +61,11 @@ spec:
     processors:
     - add_cloud_metadata: {}
     logging.json: true
+    # See https://github.com/elastic/beats/pull/41640
+    # Since 8.15 the default index template for metricbeat exceeds the default
+    # index mapping total fields limit.
+    setup.template.settings:
+      index.mapping.total_fields.limit: 12500
   deployment:
     podTemplate:
       spec:

diff --git a/pkg/controller/common/stackmon/config.go b/pkg/controller/common/stackmon/config.go
@@ -168,13 +168,14 @@ func mergeConfig(rawConfig string, config map[string]interface{}) ([]byte, error
 // inputConfigData holds data to configure the Metricbeat Elasticsearch and Kibana modules used
 // to collect metrics for Stack Monitoring
 type inputConfigData struct {
-	URL      string
-	Username string
-	Password string
-	IsSSL    bool
-	HasCA    bool
-	CAPath   string
-	Version  semver.Version
+	URL              string
+	Username         string
+	Password         string
+	IsSSL            bool
+	HasCA            bool
+	CAPath           string
+	Version          semver.Version
+	TotalFieldsLimit int
 }
 
 // buildMetricbeatBaseConfig builds the base configuration for Metricbeat with the Elasticsearch or Kibana modules used
@@ -209,6 +210,13 @@ func buildMetricbeatBaseConfig(
 		Version:  version, // Version of the monitored resource
 	}
 
+	// See https://github.com/elastic/cloud-on-k8s/pull/8284
+	// The default index template for metricbeat exceeds the default
+	// index mapping total fields limit.
+	if version.GTE(semver.MustParse("8.15.0")) {
+		configData.TotalFieldsLimit = 12500
+	}
+
 	var caVolume volume.VolumeLike
 	if configData.HasCA {
 		caVolume = volume.NewSecretVolumeWithMountPath(

diff --git a/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml b/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml
@@ -35,4 +35,9 @@ processors:
   - add_cloud_metadata: {}
   - add_host_metadata: {}
 
+{{- if gt .TotalFieldsLimit 0 }}
+setup.template.settings:
+  index.mapping.total_fields.limit: {{ .TotalFieldsLimit }}
+{{- end }}
+
 # Elasticsearch output configuration is generated
diff --git a/test/e2e/beat/recipes_test.go b/test/e2e/beat/recipes_test.go
@@ -99,14 +99,6 @@ func TestMetricbeatHostsRecipe(t *testing.T) {
 }
 
 func TestMetricbeatStackMonitoringRecipe(t *testing.T) {
-	v := version.MustParse(test.Ctx().ElasticStackVersion)
-
-	// https://github.com/elastic/cloud-on-k8s/issues/8250
-	// Update when the referenced issue is resolved.
-	if v.GE(version.MinFor(8, 16, 0)) {
-		t.SkipNow()
-	}
-
 	name := "fb-autodiscover"
 	pod, loggedString := loggingTestPod(name)
 	customize := func(builder beat.Builder) beat.Builder {