{,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add `metrics_count` to Prometheus module if `metrics_count` is `true` #40411

shmsr · 2024-08-01T07:20:01Z

Proposed commit message

Add metrics_count metric (counter) to each document (if enabled) for collector and remote_write datastreams. Please see the sample docs for more info.

Checklist

My code follows the style guidelines of this project
I have commented my code, particularly in hard-to-understand areas
I have made corresponding changes to the documentation
I have made corresponding change to the default configuration files
I have added tests that prove my fix is effective or that my feature works
I have added an entry in CHANGELOG.next.asciidoc or CHANGELOG-developer.next.asciidoc.

Sample output

Collector

{
  "_index": ".ds-metricbeat-8.16.0-2024.08.01-000001",
  "_id": "i14sKZEBcmwqWRmK-rWQ",
  "_version": 1,
  "_score": 0,
  "_source": {
    "@timestamp": "2024-08-06T19:31:39.257Z",
    "prometheus": {
      "labels": {
        "instance": "localhost:8081",
        "job": "prometheus"
      },
      "system_memory_usage_bytes": {
        "value": 7351702269.7480135
      },
      "up": {
        "value": 1
      }
    },
    "metrics_count": 2,
    "metricset": {
      "name": "collector",
      "period": 10000
    },
    "service": {
      "type": "prometheus",
      "address": "http://localhost:8081/metrics"
    },
    "event": {
      "dataset": "prometheus.collector",
      "module": "prometheus",
      "duration": 2130125
    },
    "agent": {
      "name": "Subhams-MacBook-Pro.local",
      "type": "metricbeat",
      "version": "8.16.0",
      "ephemeral_id": "6d63b998-1566-4299-aa59-ced8c7a0b7e3",
      "id": "00e1e439-0173-4fb3-8d90-74f4b06ea202"
    },
    "ecs": {
      "version": "8.0.0"
    },
    "host": {
      "mac": [
        "36-1C-20-59-8C-80",
        "36-1C-20-59-8C-84",
        "36-1C-20-59-8C-88",
        "6A-03-C5-9F-E3-B5",
        "6A-03-C5-9F-E3-B6",
        "6A-03-C5-9F-E3-B7",
        "6A-03-C5-9F-E3-D5",
        "6A-03-C5-9F-E3-D6",
        "6A-03-C5-9F-E3-D7",
        "AA-B7-74-48-81-60",
        "BC-D0-74-0D-6F-A5",
        "BE-D0-74-0D-6F-A5",
        "BE-D0-74-D0-53-64",
        "CA-59-C1-CF-D2-0F"
      ],
      "hostname": "Subhams-MacBook-Pro.local",
      "name": "subhams-macbook-pro.local",
      "architecture": "arm64",
      "os": {
        "type": "macos",
        "platform": "darwin",
        "version": "13.6",
        "family": "darwin",
        "name": "macOS",
        "kernel": "22.6.0",
        "build": "22G120"
      },
      "id": "C2677768-AA3E-5023-BD42-EB78BC3453A2",
      "ip": [
        "fe80::6803:c5ff:fe9f:e3d7",
        "fe80::6803:c5ff:fe9f:e3d5",
        "fe80::6803:c5ff:fe9f:e3d6",
        "fe80::bcd0:74ff:fe0d:6fa5",
        "fe80::8e1:bdcb:6244:6bc2",
        "2405:201:d02a:1114:1042:40ca:c385:f622",
        "2405:201:d02a:1114:da8:4a8:4c93:310f",
        "192.168.29.202",
        "fe80::c859:c1ff:fecf:d20f",
        "fe80::c859:c1ff:fecf:d20f",
        "fe80::c646:a13d:9b18:7919",
        "fe80::db4f:7fe7:4b9:cc51",
        "fe80::ce81:b1c:bd2c:69e",
        "fe80::8c9c:d4c3:99a0:8bf8",
        "fe80::781e:7cd2:79db:db56",
        "fe80::2376:e283:a58f:4f88",
        "fe80::7ff1:4c06:56e6:d8e2",
        "192.168.65.1",
        "fe80::bcd0:74ff:fed0:5364",
        "fda2:9011:41d4:6fd3:14bc:39e2:c7ad:76c4"
      ]
    }
  },
  "fields": {
    "beats_state.state.host.hostname": [
      "Subhams-MacBook-Pro.local"
    ],
    "host.os.name.text": [
      "macOS"
    ],
    "host.hostname": [
      "Subhams-MacBook-Pro.local"
    ],
    "host.mac": [
      "36-1C-20-59-8C-80",
      "36-1C-20-59-8C-84",
      "36-1C-20-59-8C-88",
      "6A-03-C5-9F-E3-B5",
      "6A-03-C5-9F-E3-B6",
      "6A-03-C5-9F-E3-B7",
      "6A-03-C5-9F-E3-D5",
      "6A-03-C5-9F-E3-D6",
      "6A-03-C5-9F-E3-D7",
      "AA-B7-74-48-81-60",
      "BC-D0-74-0D-6F-A5",
      "BE-D0-74-0D-6F-A5",
      "BE-D0-74-D0-53-64",
      "CA-59-C1-CF-D2-0F"
    ],
    "host.os.build": [
      "22G120"
    ],
    "service.type": [
      "prometheus"
    ],
    "host.ip": [
      "fe80::6803:c5ff:fe9f:e3d7",
      "fe80::6803:c5ff:fe9f:e3d5",
      "fe80::6803:c5ff:fe9f:e3d6",
      "fe80::bcd0:74ff:fe0d:6fa5",
      "fe80::8e1:bdcb:6244:6bc2",
      "2405:201:d02a:1114:1042:40ca:c385:f622",
      "2405:201:d02a:1114:da8:4a8:4c93:310f",
      "192.168.29.202",
      "fe80::c859:c1ff:fecf:d20f",
      "fe80::c859:c1ff:fecf:d20f",
      "fe80::c646:a13d:9b18:7919",
      "fe80::db4f:7fe7:4b9:cc51",
      "fe80::ce81:b1c:bd2c:69e",
      "fe80::8c9c:d4c3:99a0:8bf8",
      "fe80::781e:7cd2:79db:db56",
      "fe80::2376:e283:a58f:4f88",
      "fe80::7ff1:4c06:56e6:d8e2",
      "192.168.65.1",
      "fe80::bcd0:74ff:fed0:5364",
      "fda2:9011:41d4:6fd3:14bc:39e2:c7ad:76c4"
    ],
    "agent.type": [
      "metricbeat"
    ],
    "logstash_stats.timestamp": [
      "2024-08-06T19:31:39.257Z"
    ],
    "event.module": [
      "prometheus"
    ],
    "host.os.version": [
      "13.6"
    ],
    "host.os.kernel": [
      "22.6.0"
    ],
    "host.os.name": [
      "macOS"
    ],
    "metrics_count": [
      2
    ],
    "agent.name": [
      "Subhams-MacBook-Pro.local"
    ],
    "host.name": [
      "subhams-macbook-pro.local"
    ],
    "beats_state.timestamp": [
      "2024-08-06T19:31:39.257Z"
    ],
    "host.id": [
      "C2677768-AA3E-5023-BD42-EB78BC3453A2"
    ],
    "beats_state.state.host.name": [
      "subhams-macbook-pro.local"
    ],
    "timestamp": [
      "2024-08-06T19:31:39.257Z"
    ],
    "prometheus.system_memory_usage_bytes.value": [
      7351702269.7480135
    ],
    "prometheus.up.value": [
      1
    ],
    "host.os.type": [
      "macos"
    ],
    "kibana_stats.timestamp": [
      "2024-08-06T19:31:39.257Z"
    ],
    "metricset.period": [
      10000
    ],
    "agent.hostname": [
      "Subhams-MacBook-Pro.local"
    ],
    "event.duration": [
      2130125
    ],
    "host.architecture": [
      "arm64"
    ],
    "metricset.name": [
      "collector"
    ],
    "prometheus.labels.instance": [
      "localhost:8081"
    ],
    "@timestamp": [
      "2024-08-06T19:31:39.257Z"
    ],
    "agent.id": [
      "00e1e439-0173-4fb3-8d90-74f4b06ea202"
    ],
    "ecs.version": [
      "8.0.0"
    ],
    "host.os.platform": [
      "darwin"
    ],
    "service.address": [
      "http://localhost:8081/metrics"
    ],
    "agent.ephemeral_id": [
      "6d63b998-1566-4299-aa59-ced8c7a0b7e3"
    ],
    "prometheus.labels.job": [
      "prometheus"
    ],
    "agent.version": [
      "8.16.0"
    ],
    "beats_state.state.host.architecture": [
      "arm64"
    ],
    "host.os.family": [
      "darwin"
    ],
    "event.dataset": [
      "prometheus.collector"
    ]
  }
}

Remote Write

{
  "@timestamp": [
    "2024-08-06T20:11:35.881Z"
  ],
  "agent.ephemeral_id": [
    "4b5d7b27-4dec-4d60-89c5-5214e9805929"
  ],
  "agent.hostname": [
    "Subhams-MacBook-Pro.local"
  ],
  "agent.id": [
    "00e1e439-0173-4fb3-8d90-74f4b06ea202"
  ],
  "agent.name": [
    "Subhams-MacBook-Pro.local"
  ],
  "agent.type": [
    "metricbeat"
  ],
  "agent.version": [
    "8.16.0"
  ],
  "beats_state.state.host.architecture": [
    "arm64"
  ],
  "beats_state.state.host.hostname": [
    "Subhams-MacBook-Pro.local"
  ],
  "beats_state.state.host.name": [
    "subhams-macbook-pro.local"
  ],
  "beats_state.timestamp": [
    "2024-08-06T20:11:35.881Z"
  ],
  "ecs.version": [
    "8.0.0"
  ],
  "event.dataset": [
    "prometheus.remote_write"
  ],
  "event.module": [
    "prometheus"
  ],
  "host.architecture": [
    "arm64"
  ],
  "host.hostname": [
    "Subhams-MacBook-Pro.local"
  ],
  "host.id": [
    "C2677768-AA3E-5023-BD42-EB78BC3453A2"
  ],
  "host.ip": [
    "fe80::6803:c5ff:fe9f:e3d7",
    "fe80::6803:c5ff:fe9f:e3d5",
    "fe80::6803:c5ff:fe9f:e3d6",
    "fe80::bcd0:74ff:fe0d:6fa5",
    "fe80::8e1:bdcb:6244:6bc2",
    "2405:201:d02a:1114:1042:40ca:c385:f622",
    "2405:201:d02a:1114:da8:4a8:4c93:310f",
    "192.168.29.202",
    "fe80::c859:c1ff:fecf:d20f",
    "fe80::c859:c1ff:fecf:d20f",
    "fe80::c646:a13d:9b18:7919",
    "fe80::db4f:7fe7:4b9:cc51",
    "fe80::ce81:b1c:bd2c:69e",
    "fe80::8c9c:d4c3:99a0:8bf8",
    "fe80::781e:7cd2:79db:db56",
    "fe80::2376:e283:a58f:4f88",
    "fe80::7ff1:4c06:56e6:d8e2",
    "192.168.65.1",
    "fe80::bcd0:74ff:fed0:5364",
    "fda2:9011:41d4:6fd3:14bc:39e2:c7ad:76c4"
  ],
  "host.mac": [
    "36-1C-20-59-8C-80",
    "36-1C-20-59-8C-84",
    "36-1C-20-59-8C-88",
    "6A-03-C5-9F-E3-B5",
    "6A-03-C5-9F-E3-B6",
    "6A-03-C5-9F-E3-B7",
    "6A-03-C5-9F-E3-D5",
    "6A-03-C5-9F-E3-D6",
    "6A-03-C5-9F-E3-D7",
    "AA-B7-74-48-81-60",
    "BC-D0-74-0D-6F-A5",
    "BE-D0-74-0D-6F-A5",
    "BE-D0-74-D0-53-64",
    "CA-59-C1-CF-D2-0F"
  ],
  "host.name": [
    "subhams-macbook-pro.local"
  ],
  "host.os.build": [
    "22G120"
  ],
  "host.os.family": [
    "darwin"
  ],
  "host.os.kernel": [
    "22.6.0"
  ],
  "host.os.name": [
    "macOS"
  ],
  "host.os.name.text": [
    "macOS"
  ],
  "host.os.platform": [
    "darwin"
  ],
  "host.os.type": [
    "macos"
  ],
  "host.os.version": [
    "13.6"
  ],
  "kibana_stats.timestamp": [
    "2024-08-06T20:11:35.881Z"
  ],
  "logstash_stats.timestamp": [
    "2024-08-06T20:11:35.881Z"
  ],
  "metrics_count": [
    6
  ],
  "metricset.name": [
    "remote_write"
  ],
  "prometheus.labels.instance": [
    "localhost:8081"
  ],
  "prometheus.labels.job": [
    "prometheus"
  ],
  "prometheus.scrape_duration_seconds.value": [
    0.0047865
  ],
  "prometheus.scrape_samples_post_metric_relabeling.value": [
    17
  ],
  "prometheus.scrape_samples_scraped.value": [
    17
  ],
  "prometheus.scrape_series_added.value": [
    0
  ],
  "prometheus.system_memory_usage_bytes.value": [
    2583889525.1153283
  ],
  "prometheus.up.value": [
    1
  ],
  "service.type": [
    "prometheus"
  ],
  "timestamp": [
    "2024-08-06T20:11:35.881Z"
  ],
  "_id": "-l5RKZEBcmwqWRmKn-Xc",
  "_index": ".ds-metricbeat-8.16.0-2024.08.01-000001",
  "_score": null
}

mergify · 2024-08-01T07:20:39Z

This pull request does not have a backport label.
If this is a bug or security fix, could you label this PR @shmsr? 🙏.
For such, you'll need to label your PR with:

The upcoming major version of the Elastic Stack
The upcoming minor version of the Elastic Stack (if you're not pushing a breaking change)

To fixup this pull request, you need to add the backport labels for the needed
branches, such as:

backport-v8./d.0 is the label to automatically backport to the 8./d branch. /d is the digit

botelastic · 2024-08-06T07:09:31Z

This pull request doesn't have a Team:<team> label.

shmsr · 2024-08-13T07:35:04Z

/test

elasticmachine · 2024-08-20T12:41:44Z

Pinging @elastic/elastic-agent-data-plane (Team:Elastic-Agent-Data-Plane)

shmsr · 2024-09-10T09:57:40Z

@gizas Did you get a chance to look at this PR again?

mergify · 2024-09-10T10:07:42Z

This pull request is now in conflicts. Could you fix it? 🙏
To fixup this pull request, you can check out it locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/

git fetch upstream
git checkout -b prometheus_count_metrics upstream/prometheus_count_metrics
git merge upstream/main
git push upstream prometheus_count_metrics

x-pack/metricbeat/module/prometheus/remote_write/data.go

mergify · 2024-09-11T03:51:12Z

This pull request is now in conflicts. Could you fix it? 🙏
To fixup this pull request, you can check out it locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/

git fetch upstream
git checkout -b prometheus_count_metrics upstream/prometheus_count_metrics
git merge upstream/main
git push upstream prometheus_count_metrics

metricbeat/module/prometheus/remote_write/remote_write_test.go

x-pack/metricbeat/metricbeat.reference.yml

Co-authored-by: Aman <[email protected]>

metricbeat/module/prometheus/collector/collector.go

devamanv

Left one last comment. See if that makes sense. Rest all looks good.

mergify · 2024-09-11T11:46:45Z

backport-8.x has been added to help with the transition to the new branch 8.x.

ishleenk17

Looks good!

gizas · 2024-09-12T07:28:43Z

Thanks for the tests added. LGTM and also help to understand how the metrics_count is calculated

…`metrics_count` to Prometheus module if `metrics_count` is `true` (#40411) Co-authored-by: Ishleen Kaur <[email protected]> Co-authored-by: Aman <[email protected]> (cherry picked from commit 3c9563c)

…tor,remote_write}: Add `metrics_count` to Prometheus module if `metrics_count` is `true` (#40784) * {,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add `metrics_count` to Prometheus module if `metrics_count` is `true` (#40411) Co-authored-by: Ishleen Kaur <[email protected]> Co-authored-by: Aman <[email protected]> (cherry picked from commit 3c9563c) * Update CHANGELOG.next.asciidoc --------- Co-authored-by: subham sarkar <[email protected]>

shmsr self-assigned this Aug 1, 2024

botelastic bot added the needs_team Indicates that the issue/PR needs a Team:* label label Aug 1, 2024

shmsr added the Team:Service-Integrations Label for the Service Integrations team label Aug 6, 2024

botelastic bot removed the needs_team Indicates that the issue/PR needs a Team:* label label Aug 6, 2024

shmsr added needs_team Indicates that the issue/PR needs a Team:* label Team:Obs-InfraObs Label for the Observability Infrastructure Monitoring team labels Aug 6, 2024

botelastic bot removed the needs_team Indicates that the issue/PR needs a Team:* label label Aug 6, 2024

shmsr added needs_team Indicates that the issue/PR needs a Team:* label and removed Team:Service-Integrations Label for the Service Integrations team labels Aug 6, 2024

botelastic bot removed the needs_team Indicates that the issue/PR needs a Team:* label label Aug 6, 2024

Add metrics_count to Prometheus module

b7741c6

shmsr force-pushed the prometheus_count_metrics branch from 144568b to b7741c6 Compare August 7, 2024 09:36

Merge branch 'main' into prometheus_count_metrics

c311957

shmsr changed the title ~~Add metrics_count to Prometheus module~~ Add metrics_count to Prometheus module Aug 15, 2024

shmsr marked this pull request as ready for review August 20, 2024 12:23

shmsr requested review from a team as code owners August 20, 2024 12:23

shmsr requested review from gizas, tetianakravchenko, rdner and mauri870 August 20, 2024 12:23

Merge branch 'main' into prometheus_count_metrics

c6d6e3c

pierrehilbert added the Team:Elastic-Agent-Data-Plane Label for the Agent Data Plane team label Aug 20, 2024

pierrehilbert requested review from belimawr and removed request for rdner August 20, 2024 12:41

Fix CI

f2e735e

mauri870 approved these changes Sep 6, 2024

View reviewed changes

mergify bot added the backport-v8.x label Sep 10, 2024

shmsr added 2 commits September 10, 2024 16:06

Use FO pattern for remote_write

5ea8d13

Merge branch 'main' into prometheus_count_metrics

875f8b6

devamanv reviewed Sep 10, 2024

View reviewed changes

x-pack/metricbeat/module/prometheus/remote_write/data.go Outdated Show resolved Hide resolved

x-pack/metricbeat/module/prometheus/remote_write/data.go Outdated Show resolved Hide resolved

Address review comments

b1408a7

shmsr added 2 commits September 11, 2024 11:23

Merge branch 'main' into prometheus_count_metrics

85d0738

Address review comments

2936ea2

devamanv reviewed Sep 11, 2024

View reviewed changes

metricbeat/module/prometheus/remote_write/remote_write_test.go Outdated Show resolved Hide resolved

metricbeat/module/prometheus/remote_write/remote_write_test.go Outdated Show resolved Hide resolved

metricbeat/module/prometheus/remote_write/remote_write_test.go Show resolved Hide resolved

shmsr added 2 commits September 11, 2024 14:11

Address review comments

274f104

Rename vars

142b79d

devamanv reviewed Sep 11, 2024

View reviewed changes

x-pack/metricbeat/metricbeat.reference.yml Show resolved Hide resolved

Update metricbeat/module/prometheus/remote_write/remote_write_test.go

4c43a9e

Co-authored-by: Aman <[email protected]>

devamanv reviewed Sep 11, 2024

View reviewed changes

metricbeat/module/prometheus/collector/collector.go Show resolved Hide resolved

devamanv approved these changes Sep 11, 2024

View reviewed changes

mergify bot added the backport-8.x Automated backport to the 8.x branch with mergify label Sep 11, 2024

v1v removed the backport-v8.x label Sep 11, 2024

ishleenk17 approved these changes Sep 12, 2024

View reviewed changes

gizas approved these changes Sep 12, 2024

View reviewed changes

shmsr merged commit 3c9563c into elastic:main Sep 12, 2024
29 checks passed

mergify bot mentioned this pull request Sep 12, 2024

[8.x](backport #40411) {,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add metrics_count to Prometheus module if metrics_count is true #40784

Merged

6 tasks

shmsr mentioned this pull request Oct 17, 2024

packages/prometheus: Add metrics_count to collector and remote_write if metrics_count: true elastic/integrations#11447

Merged

5 tasks

leehinman mentioned this pull request Nov 28, 2024

add metricbeat benchmark module #41801

Merged

6 tasks

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

{,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add `metrics_count` to Prometheus module if `metrics_count` is `true` #40411

{,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add `metrics_count` to Prometheus module if `metrics_count` is `true` #40411

shmsr commented Aug 1, 2024 •

edited

Loading

mergify bot commented Aug 1, 2024

botelastic bot commented Aug 6, 2024

shmsr commented Aug 13, 2024

elasticmachine commented Aug 20, 2024

shmsr commented Sep 10, 2024

mergify bot commented Sep 10, 2024

mergify bot commented Sep 11, 2024

devamanv left a comment

mergify bot commented Sep 11, 2024

ishleenk17 left a comment

gizas commented Sep 12, 2024

{,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add metrics_count to Prometheus module if metrics_count is true #40411

{,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add metrics_count to Prometheus module if metrics_count is true #40411

Conversation

shmsr commented Aug 1, 2024 • edited Loading

Proposed commit message

Checklist

Sample output

mergify bot commented Aug 1, 2024

botelastic bot commented Aug 6, 2024

shmsr commented Aug 13, 2024

elasticmachine commented Aug 20, 2024

shmsr commented Sep 10, 2024

mergify bot commented Sep 10, 2024

mergify bot commented Sep 11, 2024

devamanv left a comment

Choose a reason for hiding this comment

mergify bot commented Sep 11, 2024

ishleenk17 left a comment

Choose a reason for hiding this comment

gizas commented Sep 12, 2024

{,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add `metrics_count` to Prometheus module if `metrics_count` is `true` #40411

{,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add `metrics_count` to Prometheus module if `metrics_count` is `true` #40411

shmsr commented Aug 1, 2024 •

edited

Loading