diff --git a/config/recipes/beats/stack_monitoring.yaml b/config/recipes/beats/stack_monitoring.yaml index 670a437340..0ac668736b 100644 --- a/config/recipes/beats/stack_monitoring.yaml +++ b/config/recipes/beats/stack_monitoring.yaml @@ -61,6 +61,11 @@ spec: processors: - add_cloud_metadata: {} logging.json: true + # See https://github.com/elastic/beats/pull/41640 + # Since 8.15 the default index template for metricbeat exceeds the default + # index mapping total fields limit. + setup.template.settings: + index.mapping.total_fields.limit: 12500 deployment: podTemplate: spec: diff --git a/pkg/controller/common/stackmon/config.go b/pkg/controller/common/stackmon/config.go index 50b84d538b..72cfed3f45 100644 --- a/pkg/controller/common/stackmon/config.go +++ b/pkg/controller/common/stackmon/config.go @@ -168,13 +168,14 @@ func mergeConfig(rawConfig string, config map[string]interface{}) ([]byte, error // inputConfigData holds data to configure the Metricbeat Elasticsearch and Kibana modules used // to collect metrics for Stack Monitoring type inputConfigData struct { - URL string - Username string - Password string - IsSSL bool - HasCA bool - CAPath string - Version semver.Version + URL string + Username string + Password string + IsSSL bool + HasCA bool + CAPath string + Version semver.Version + TotalFieldsLimit int } // buildMetricbeatBaseConfig builds the base configuration for Metricbeat with the Elasticsearch or Kibana modules used @@ -209,6 +210,13 @@ func buildMetricbeatBaseConfig( Version: version, // Version of the monitored resource } + // See https://github.com/elastic/cloud-on-k8s/pull/8284 + // The default index template for metricbeat exceeds the default + // index mapping total fields limit. + if version.GTE(semver.MustParse("8.15.0")) { + configData.TotalFieldsLimit = 12500 + } + var caVolume volume.VolumeLike if configData.HasCA { caVolume = volume.NewSecretVolumeWithMountPath( diff --git a/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml b/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml index e7a781a742..39be4e138c 100644 --- a/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml +++ b/pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml @@ -35,4 +35,9 @@ processors: - add_cloud_metadata: {} - add_host_metadata: {} +{{- if gt .TotalFieldsLimit 0 }} +setup.template.settings: + index.mapping.total_fields.limit: {{ .TotalFieldsLimit }} +{{- end }} + # Elasticsearch output configuration is generated diff --git a/test/e2e/beat/recipes_test.go b/test/e2e/beat/recipes_test.go index 4c67638b4e..b91b2e5a62 100644 --- a/test/e2e/beat/recipes_test.go +++ b/test/e2e/beat/recipes_test.go @@ -99,14 +99,6 @@ func TestMetricbeatHostsRecipe(t *testing.T) { } func TestMetricbeatStackMonitoringRecipe(t *testing.T) { - v := version.MustParse(test.Ctx().ElasticStackVersion) - - // https://github.com/elastic/cloud-on-k8s/issues/8250 - // Update when the referenced issue is resolved. - if v.GE(version.MinFor(8, 16, 0)) { - t.SkipNow() - } - name := "fb-autodiscover" pod, loggedString := loggingTestPod(name) customize := func(builder beat.Builder) beat.Builder {