Update Prometheus package to leverage types in remote write (#243)

dev/import-beats-resources/prometheus/docs/README.md

@@ -197,6 +197,82 @@ The fields reported are:
 
 {{fields "remote_write"}}
 
+#### Histograms and types [x-pack]
+
+`use_types` parameter (default: false) enables a different layout for metrics storage, leveraging Elasticsearch
+types, including https://www.elastic.co/guide/en/elasticsearch/reference/current/histogram.html[histograms].
+
+`rate_counters` parameter (default: false) enables calculating a rate out of Prometheus counters. When enabled, Metricbeat stores
+the counter increment since the last collection. This metric should make some aggregations easier and with better
+performance. This parameter can only be enabled in combination with `use_types`.
+
+When `use_types` and `rate_counters` are enabled, metrics are stored like this:
+
+```$json
+{
+    "prometheus": {
+        "labels": {
+            "instance": "172.27.0.2:9090",
+            "job": "prometheus"
+        },
+        "prometheus_target_interval_length_seconds_count": {
+            "counter": 1,
+            "rate": 0
+        },
+        "prometheus_target_interval_length_seconds_sum": {
+            "counter": 15.000401344,
+            "rate": 0
+        }
+        "prometheus_tsdb_compaction_chunk_range_seconds_bucket": {
+            "histogram": {
+                "values": [50, 300, 1000, 4000, 16000],
+                "counts": [10, 2, 34, 7]
+            }
+        }
+    },
+}
+```
+
+#### Types' patterns
+
+Unlike `collector` metricset, `remote_write` receives metrics in raw format from the prometheus server.
+In this, the module has to internally use a heuristic in order to identify efficiently the type of each raw metric.
+For these purpose some name patterns are used in order to identify the type of each metric.
+The default patterns are the following:
+
+. `_total` suffix: the metric is of Counter type
+. `_sum` suffix: the metric is of Counter type
+. `_count` suffix: the metric is of Counter type
+. `_bucket` suffix and `le` in labels: the metric is of Histogram type
+
+Everything else is handled as a Gauge. In addition there is no special handling for Summaries so it is expected that
+Summary's quantiles are handled as Gauges and Summary's sum and count as Counters.
+
+Users have the flexibility to add their own patterns using the following configuration:
+
+```$yml
+types_patterns:
+    counter_patterns: ["_my_counter_suffix"]
+    histogram_patterns: ["_my_histogram_suffix"]
+```
+
+The configuration above will consider metrics with names that match `_my_counter_suffix` as Counters
+and those that match `_my_histogram_suffix` (and have `le` in their labels) as Histograms.
+
+
+To match only specific metrics, anchor the start and the end of the regexp of each metric:
+
+- the caret `^` matches the beginning of a text or line,
+- the dollar sign `$` matches the end of a text.
+
+```$yml
+types_patterns:
+    histogram_patterns: ["^my_histogram_metric$"]
+```
+
+Note that when using `types_patterns`, the provided patterns have higher priority than the default patterns.
+For instance if `_histogram_total` is a defined histogram pattern, then a metric like `network_bytes_histogram_total`
+will be handled as a histogram, even if it has the suffix `_total` which is a default pattern for counters.
 
 ### Query Metrics
 

packages/prometheus/dataset/remote_write/agent/stream/stream.yml.hbs

@@ -4,3 +4,13 @@ port: {{port}}
 ssl.enabled: {{ssl.enabled}}
 ssl.certificate: {{ssl.certificate}}
 ssl.key: {{ssl.key}}
+rate_counters: {{rate_counters}}
+use_types: {{use_types}}
+types_patterns.exclude:
+{{#each types_patterns.counter_patterns}}
+  - {{this}}
+{{/each}}
+types_patterns.include:
+{{#each types_patterns.histogram_patterns}}
+  - {{this}}
+{{/each}}

packages/prometheus/dataset/remote_write/fields/fields.yml

@@ -12,3 +12,30 @@
     object_type_mapping_type: "*"
     description: |-
       Prometheus metric
+- name: prometheus.*.value
+  type: object
+  object_type: double
+  object_type_mapping_type: "*"
+  description: >
+    Prometheus gauge metric
+
+- name: prometheus.*.counter
+  type: object
+  object_type: double
+  object_type_mapping_type: "*"
+  description: >
+    Prometheus counter metric
+
+- name: prometheus.*.rate
+  type: object
+  object_type: double
+  object_type_mapping_type: "*"
+  description: >
+    Prometheus rated counter metric
+
+- name: prometheus.*.histogram
+  type: object
+  object_type: histogram
+  object_type_mapping_type: "*"
+  description: >-
+    Prometheus histogram metric

packages/prometheus/dataset/remote_write/manifest.yml

@@ -39,5 +39,33 @@ streams:
     required: false
     show_user: true
     default: /etc/pki/server/cert.key
+  - name: rate_counters
+    type: bool
+    title: Rate Counters
+    multi: false
+    required: true
+    show_user: true
+    default: true
+  - name: use_types
+    type: bool
+    title: Use Types
+    multi: false
+    required: true
+    show_user: true
+    default: true
+  - name: types_patterns.counter_patterns
+    type: text
+    title: Counter Type Patterns
+    multi: true
+    required: false
+    show_user: true
+    default: []
+  - name: types_patterns.histogram_patterns
+    type: text
+    title: Histogram Type Patterns
+    multi: true
+    required: false
+    show_user: true
+    default: []
   title: Prometheus remote write metrics
   description: Collect Prometheus remote write metrics

packages/prometheus/docs/README.md

@@ -333,10 +333,90 @@ The fields reported are:
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
+| prometheus.*.counter | Prometheus counter metric | object |
+| prometheus.*.histogram | Prometheus histogram metric | object |
+| prometheus.*.rate | Prometheus rated counter metric | object |
+| prometheus.*.value | Prometheus gauge metric | object |
 | prometheus.labels.* | Prometheus metric labels | object |
 | prometheus.metrics.* | Prometheus metric | object |
 
 
+#### Histograms and types [x-pack]
+
+`use_types` parameter (default: false) enables a different layout for metrics storage, leveraging Elasticsearch
+types, including https://www.elastic.co/guide/en/elasticsearch/reference/current/histogram.html[histograms].
+
+`rate_counters` parameter (default: false) enables calculating a rate out of Prometheus counters. When enabled, Metricbeat stores
+the counter increment since the last collection. This metric should make some aggregations easier and with better
+performance. This parameter can only be enabled in combination with `use_types`.
+
+When `use_types` and `rate_counters` are enabled, metrics are stored like this:
+
+```$json
+{
+    "prometheus": {
+        "labels": {
+            "instance": "172.27.0.2:9090",
+            "job": "prometheus"
+        },
+        "prometheus_target_interval_length_seconds_count": {
+            "counter": 1,
+            "rate": 0
+        },
+        "prometheus_target_interval_length_seconds_sum": {
+            "counter": 15.000401344,
+            "rate": 0
+        }
+        "prometheus_tsdb_compaction_chunk_range_seconds_bucket": {
+            "histogram": {
+                "values": [50, 300, 1000, 4000, 16000],
+                "counts": [10, 2, 34, 7]
+            }
+        }
+    },
+}
+```
+
+#### Types' patterns
+
+Unlike `collector` metricset, `remote_write` receives metrics in raw format from the prometheus server.
+In this, the module has to internally use a heuristic in order to identify efficiently the type of each raw metric.
+For these purpose some name patterns are used in order to identify the type of each metric.
+The default patterns are the following:
+
+. `_total` suffix: the metric is of Counter type
+. `_sum` suffix: the metric is of Counter type
+. `_count` suffix: the metric is of Counter type
+. `_bucket` suffix and `le` in labels: the metric is of Histogram type
+
+Everything else is handled as a Gauge. In addition there is no special handling for Summaries so it is expected that
+Summary's quantiles are handled as Gauges and Summary's sum and count as Counters.
+
+Users have the flexibility to add their own patterns using the following configuration:
+
+```$yml
+types_patterns:
+    counter_patterns: ["_my_counter_suffix"]
+    histogram_patterns: ["_my_histogram_suffix"]
+```
+
+The configuration above will consider metrics with names that match `_my_counter_suffix` as Counters
+and those that match `_my_histogram_suffix` (and have `le` in their labels) as Histograms.
+
+
+To match only specific metrics, anchor the start and the end of the regexp of each metric:
+
+- the caret `^` matches the beginning of a text or line,
+- the dollar sign `$` matches the end of a text.
+
+```$yml
+types_patterns:
+    histogram_patterns: ["^my_histogram_metric$"]
+```
+
+Note that when using `types_patterns`, the provided patterns have higher priority than the default patterns.
+For instance if `_histogram_total` is a defined histogram pattern, then a metric like `network_bytes_histogram_total`
+will be handled as a histogram, even if it has the suffix `_total` which is a default pattern for counters.
 
 ### Query Metrics
 

packages/prometheus/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: prometheus
 title: Prometheus
-version: 0.1.4
+version: 0.2.0
 license: basic
 description: Prometheus Integration
 type: integration
@@ -10,7 +10,8 @@ categories:
 - datastore
 release: experimental
 conditions:
-  kibana.version: '^7.9.0'
+  kibana.version: ">=7.10.0"
+  agent.version: ">=7.10.0"
 screenshots:
 - src: /img/metricbeat-prometheus-overview.png
   title: Metricbeat Prometheus Overview