Add fields validation for histogram subfields (#17759)

Add validation to histogram subfields in a way that they don't need to
be added to the mappings, but they can still be checked. This is helpful
to avoid having to add exceptions to all the modules based on Prometheus
when Elasticsearch types are used.

CHANGELOG-developer.next.asciidoc

@@ -79,3 +79,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - New mage target for Functionbeat: generate pkg folder to make manager easier. {pull}15580[15880]
 - Add support for MODULE environment variable in `mage goIntegTest` in metricbeat to run integration tests for a single module. {pull}17147[17147]
 - Add support for a `TEST_TAGS` environment variable to add tags for tests selection following go build tags semantics, this environment variable is used by mage test targets to add build tags. Python tests can also be tagged with a decorator (`@beat.tag('sometag')`). {pull}16937[16937] {pull}17075[17075]
+- Add fields validation for histogram subfields. {pull}17759[17759]

libbeat/mapping/field.go

@@ -354,6 +354,10 @@ func (f Fields) getKeys(namespace string) []string {
 		} else {
 			keys = append(keys, field.Fields.getKeys(fieldName)...)
 		}
+		if field.ObjectType == "histogram" {
+			keys = append(keys, fieldName+".values")
+			keys = append(keys, fieldName+".counts")
+		}
 	}
 
 	return keys

libbeat/tests/system/beat/beat.py

@@ -573,6 +573,10 @@ def extract_fields(doc_list, name):
                     if field.get("type") in ["object", "geo_point"]:
                         dictfields.append(newName)
 
+                if field.get("type") == "object" and field.get("object_type") == "histogram":
+                    fields.append(newName + ".values")
+                    fields.append(newName + ".counts")
+
                 if field.get("type") == "alias":
                     aliases.append(newName)
 

x-pack/metricbeat/module/prometheus/collector/_meta/testdata/config.yml

@@ -1,13 +1,7 @@
 type: http
 url: "/metrics"
 suffix: plain
-omit_documented_fields_check:
-   # these are not mapped by this module but the oss one
-  - prometheus.labels.*
-  # histogram values & counts are not mapped (it's part of the type data)
-  - '*.histogram.values'
-  - '*.histogram.counts'
 remove_fields_from_comparison: ["prometheus.labels.instance"]
 module:
   use_types: true
-  rate_counters: true
+  rate_counters: true

x-pack/metricbeat/module/prometheus/collector/collector_test.go

@@ -12,6 +12,9 @@ import (
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 
 	_ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus"
+
+	// Import common fields for validation
+	_ "github.com/elastic/beats/v7/metricbeat/module/prometheus"
 )
 
 func TestData(t *testing.T) {