From c16b299be3f08e0de29e2163965202a5514d2c43 Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Fri, 21 Feb 2020 17:26:04 +0100 Subject: [PATCH] Cherry-pick #15559 to 7.x: [Metricbeat] Add Load Balancing metricset to Google Cloud Platform module (#16422) * [Metricbeat] Add Load Balancing metricset to Google Cloud Platform module (#15559) (cherry picked from commit 978d676d8bf024fef8987a2157bdbf2924ec08f1) * Update changelog entry --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/googlecloud.asciidoc | 4 + .../googlecloud/loadbalancing.asciidoc | 23 ++ metricbeat/docs/modules_list.asciidoc | 3 +- .../module/googlecloud/constants.go | 9 +- .../metricbeat/module/googlecloud/fields.go | 2 +- .../googlecloud/loadbalancing/_meta/data.json | 174 ++++++++++++ .../loadbalancing/_meta/docs.asciidoc | 27 ++ .../loadbalancing/_meta/fields.yml | 249 ++++++++++++++++++ .../googlecloud/loadbalancing/manifest.yml | 33 +++ .../metricbeat/module/googlecloud/module.yml | 1 + .../stackdriver/metadata_services.go | 2 + .../stackdriver/metrics_requester.go | 24 +- .../stackdriver/metrics_requester_test.go | 24 ++ .../timeseries_metadata_collector.go | 5 +- 15 files changed, 573 insertions(+), 8 deletions(-) create mode 100644 metricbeat/docs/modules/googlecloud/loadbalancing.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json create mode 100644 x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/googlecloud/loadbalancing/manifest.yml create mode 100644 x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1723bd0249e..a59fa692d6a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -245,6 +245,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for NATS 2.1. {pull}16317[16317] - Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333] - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254] +- Add Load Balancing metricset to GCP {pull}15559[15559] - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280] *Packetbeat* diff --git a/metricbeat/docs/modules/googlecloud.asciidoc b/metricbeat/docs/modules/googlecloud.asciidoc index a1f596aef1e..b510835ad43 100644 --- a/metricbeat/docs/modules/googlecloud.asciidoc +++ b/metricbeat/docs/modules/googlecloud.asciidoc @@ -96,5 +96,9 @@ The following metricsets are available: * <> +* <> + include::googlecloud/compute.asciidoc[] +include::googlecloud/loadbalancing.asciidoc[] + diff --git a/metricbeat/docs/modules/googlecloud/loadbalancing.asciidoc b/metricbeat/docs/modules/googlecloud/loadbalancing.asciidoc new file mode 100644 index 00000000000..22ea8bf98b4 --- /dev/null +++ b/metricbeat/docs/modules/googlecloud/loadbalancing.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-googlecloud-loadbalancing]] +=== Google Cloud Platform loadbalancing metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 12b6d346700..b55c9625c2e 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -98,7 +98,8 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> |<> |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> beta[] +.2+| .2+| |<> beta[] +|<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | diff --git a/x-pack/metricbeat/module/googlecloud/constants.go b/x-pack/metricbeat/module/googlecloud/constants.go index 9bc61eb1267..b5ded8bf9bc 100644 --- a/x-pack/metricbeat/module/googlecloud/constants.go +++ b/x-pack/metricbeat/module/googlecloud/constants.go @@ -18,10 +18,11 @@ const ( // Metricsets / GCP services names const ( - ServiceCompute = "compute" - ServicePubsub = "pubsub" - ServiceFirestore = "firestore" - ServiceStorage = "storage" + ServiceCompute = "compute" + ServicePubsub = "pubsub" + ServiceLoadBalancing = "loadbalancing" + ServiceFirestore = "firestore" + ServiceStorage = "storage" ) //Paths within the GCP monitoring.TimeSeries response, if converted to JSON, where you can find each ECS field required for the output event diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 8d22aa84558..4a45e5b4b7a 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJy8l7Fy2zwMx3c/BbbcffclD6Chi4c0Q9sMTVcfRUIya4rQgaB96tP3KEWqZFuK04u62ST4xw9/ApZ8DwdsMiiJSofaUTQbALHiMIO7x3YVtmkZnp2Sgri62wAwOlQBM8hR1AbAYNBsa7HkM/i0AQB43D5DRSY63AAUFp0JWbtxD15VOEn54FSOLrTbANLUmAHlP1HL69L4/FgjBuSH/4blq2f/RFcoyihR7zrBVofbD4QmCFa3x2uq6ig4Cj+3dipTMsV6tDoxfnJd2065L2F05tzMMY/1QZTXONmcSz4nNhYsLONJOXcRsCS6JDwWN0x1jWaXN4Jhpyl6uRrf53Lky5mAiZFPXlNlfQmtcJ8G8gZkj0slXaLVSh9QVoF7lb4Zb+i5Oq5yG4wB+Yhmp4kxLNZqKObuvMmuVvs1VjkyUAGt6pAEyLfV7ilI2k2fZ5p3ShnFOvtLJfUPQvyeTGel07ceRTlHWgka2D6/gOyVgA2gIzN6cQ1Yn367+iJuAw+qxJ3Yag7qvdwvSRAK4kT7aq/1EFCTN5f3N3S2DYeV+kd9/Chvk1K6lW6UUw4omKq5Ms5wqF4FJiXvWJ6+AdXIbT9en5me58RWcG1/UhJBD0JvG9QBrelQm+ENi4bxqGdGYxFgkvwzndq4diZ/fIG9CpAjeuDovfXl/7eMh0c5Ea81IRrtcYUH3sWUdHm6SUluzFd1he7jn3kDX//M+zvCgF7W9i7lADoiv4/qH3i2THY/++bZ/kEIovTBsD0ir/pS+jsAAP//l2hICA==" + return "eJzsW0tz2zgSvvtX9M3JVuIc9ubDViWqrSS1ycY1cXJVQWBLwhhEc4CmFc2vnwL4ECmJJGiLSmZKN5uP7q+b6O9rPPQaHnB7CyuilUapKU+uAFixxlu4fh+uwsxfhjsteEk2vb4CsKhROLyFBbK4AkjQSasyVmRu4T9XAADvZ3eQUpJrvAJYKtSJuw03XoMRKbZc3mixQO3CbQDeZngLtPgdJZeXmu83beQO7c2/6stH3909nSKLRLAY9YZV0sW/4LaOMY1/XlKa5YyNx/dT2zazspRnjautxLc+16ywXIXQeGc/mU08yjgWRmLrZpfzLmNNg0tlcSO0Pnigz2if4abxxFKWYTJfbBndXFJu+OjzlS9NZtXxQCuRH42kVJkVBMOVG1hsgdfYF9IhtEzIB+RJwJWmo+HVYy7LJ/kaFh3aR0zmkiy63lgTyhd6f5Adjfb/ebpAC7SEYLV2AmRCtGty7O/6vzsGbxtlzkqrP4W3fiKI9z7pVkj/XwVFaE1SMCYwu/sGvBYMyoHMrUXDegvKeO6qgogD7sQK56zSLlBjcX/zBmFJ1qMt06sMOJRkksPvV49s5R4mGj/i9KU885b8VylK2fuApaW0K4w9OJRNAsY7L7B8/AKUoQ3j8XjNVHg2VjFOnR/vhNEA03CCCkBTZih4GEhRXR5ZR2n0Amg5/0Cb8Fyoye+fYS0cLBAN2NwYZVavYsrDIG/ITlUhEtXjBIJ3UCWFn6JSfDa6ozqC7vSaV+OrNO9pCB0anjp33gfQI9pxqM6Qs35kFRpNIlkILYxULT+n6ks/kUjgXeVgZHu6Zs72S6+7rJ4BogtIE8zC59Ukcy0YjVRHW54WhLee1diqRd5sFYrXtyCFlrkOPUMY1Js1Ft2BxT9ydAwbUX7DstHLLP3Yeqb2/5RgIDesdOP2YaFUTxa+HYdx68FYdBkZhzeTcNczBvZuNib6e7ZBC4vcl0JRZP0daqwpyrqVeyhtMJC6prsvvX7ifMX4a/r874+MDBpWQr8L0fa5j4cQC6MJZWVpw+v5UkgmO/hWVC986MRJEfF0BA8fmjZ5Ol8qEzq4qFQO+NnnoJIjBsQtvhXycxpTT70ayiZcxUeuoJQP9/d3b74G2YBCN7zAUIXMHbJJF/ZTo65xljPGxbYG5W8fAx4DtiDJM2W6cFamukb/gixIIdf40md6XCBLS4Z9JCyzueVj4CNE67f7e0/HLrdeWsgCCrkGScZgMQ9eIG986yy1CsGYpBCki7hcxOUiLoPGfwlxUYbRGnHKtdsxvfM/p3seHrz9QzaG5iJGTgTVjbASQ3fjzfVTXhwLDNd/PPHF804c6zyZ/sYQYDwFPp0ER9LgeCIcRYXPJMMBX7sVrpguexD4EzvtopVy8S3fIfApIB9ps8fDi2qqT5nYVmPdNYcpM94HnYmFPrOM1arUkrJ97VoRw9vZ/5pjB8gUolXFH9JyUaxnmbso1rOBxIOBi2JF+BqnV1OsCo3WqlilOuFi0FhgP2/hZ7Q+xavT31CbLstFHaYuy0WjIcTCgMty0VE/9a7xv286VovOvD1bAcKVRed6ybondXFEHdjr46d3u0WhipLhxZIs3M/uYKlp40DxtStYZ3eqkAyILNNKhoMt4NiiSIGM3r7cZ729oPrPCjwtrNYhgZ7ASk3wYXWhVOZcuS8xMbXATpP7KqrzJP9oZF3QLHOPxg8ofGsnJ5wO8anb7eS4sL3jUYR0HgpyH9EOLwQ/ca4/KMNRbw9L8BgzffJ7mtn0kPCedqo5UnKnmXWOE9sRUjtOaKNl9hkiOyCxLLO5c3oeOuifK7KN8tXkwonvmi6eNSdqnvbeEVA4Pr1Bi8BoU2XCFKSiqjdfv37q2loe0Qw8DeW+IH3/3JDK3IVT+v3QTrMd71IiXmMS2PyFMpC6lztWb07Frl3gd8dCPrwqdu1TZXLGliBqsUVbRpcJV05D6y2zIsDL3OwyN7vMzQaN/yJb+eeiwF3n+v1zHAUa3JxNQKTF0epBGZqTAZwVPwFqNP+Us2NhEp+pNmhL+WodqLsD6evOU9nhR52B5BOrHrFZCqf/IeFfAQAA///d9YAm" } diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json new file mode 100644 index 00000000000..b997e5c61af --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/data.json @@ -0,0 +1,174 @@ +{ + "@timestamp": "2020-01-14T21:14:00.000Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "host": { + "name": "mcastro", + "architecture": "x86_64", + "os": { + "version": "", + "family": "", + "name": "Antergos Linux", + "kernel": "5.4.3-arch1-1", + "platform": "antergos" + }, + "id": "54f70115bae545cbac2b150f254472a0", + "containerized": false, + "hostname": "mcastro" + }, + "agent": { + "type": "metricbeat", + "ephemeral_id": "ddb5f424-578e-4c51-ac74-08447827a4b1", + "hostname": "mcastro", + "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228", + "version": "8.0.0" + }, + "service": { + "type": "googlecloud" + }, + "cloud": { + "account": { + "id": "elastic-metricbeat" + }, + "provider": "googlecloud" + }, + "event": { + "dataset": "googlecloud.loadbalancing", + "module": "googlecloud", + "duration": 908872754 + }, + "metricset": { + "name": "loadbalancing", + "period": 900000 + }, + "googlecloud": { + "labels": { + "metrics": { + "response_code": "502", + "cache_result": "DISABLED", + "proxy_continent": "Europe", + "client_country": "France", + "response_code_class": "500", + "protocol": "HTTP/1.1" + }, + "resource": { + "matched_url_path_rule": "UNMATCHED", + "backend_target_name": "my-backed", + "backend_type": "INVALID_BACKEND", + "backend_scope_type": "INVALID_BACKEND", + "url_map_name": "test-lb", + "forwarding_rule_name": "test-lb-forwarding-rule", + "region": "global", + "backend_scope": "INVALID_BACKEND", + "target_proxy_name": "test-lb-target-proxy", + "backend_target_type": "BACKEND_SERVICE", + "backend_name": "INVALID_BACKEND" + } + }, + "loadbalancing": { + "https": { + "frontend_tcp_rtt": { + "count": 3, + "mean": 27, + "bucket_options": { + "Options": { + "ExponentialBuckets": { + "growth_factor": 1.4, + "scale": 1, + "num_finite_buckets": 66 + } + } + }, + "bucket_counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3 + ] + }, + "request_bytes_count": 1281, + "request_count": 3, + "response_bytes_count": 1464, + "backend_latencies": { + "mean": 112.094, + "bucket_options": { + "Options": { + "ExponentialBuckets": { + "growth_factor": 1.4, + "scale": 1, + "num_finite_buckets": 66 + } + } + }, + "bucket_counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3 + ], + "count": 3 + }, + "total_latencies": { + "bucket_counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 3 + ], + "count": 3, + "mean": 195, + "bucket_options": { + "Options": { + "ExponentialBuckets": { + "growth_factor": 1.4, + "scale": 1, + "num_finite_buckets": 66 + } + } + } + }, + "backend_request_count": 3, + "backend_response_bytes_count": 1464, + "backend_request_bytes_count": 1281 + } + } + }, + "ecs": { + "version": "1.4.0" + } +} diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/docs.asciidoc b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/docs.asciidoc new file mode 100644 index 00000000000..256e744ba6b --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/docs.asciidoc @@ -0,0 +1,27 @@ +Load Balancing metricset to fetch metrics from https://cloud.google.com/load-balancing/[Load Balancing] in Google Cloud Platform. + +The `loadbalancing` metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-loadbalancing[Stackdriver API]. The field names have been left untouched for people already familiar with them. + +[float] +=== Fields + +- `loadbalancing.https.backend_latencies`: A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. +- `loadbalancing.https.backend_request_bytes_count`: The number of bytes sent as requests from HTTP/S load balancer to backends. +- `loadbalancing.https.backend_request_count`: The number of requests served by backends of HTTP/S load balancer. +- `loadbalancing.https.backend_response_bytes_count`: The number of bytes sent as responses from backends (or cache) to HTTP/S load balancer. +- `loadbalancing.https.frontend_tcp_rtt`: A distribution of the RTT measured for each connection between client and proxy. +- `loadbalancing.https.request_bytes_count`: The number of bytes sent as requests from clients to HTTP/S load balancer. +- `loadbalancing.https.request_count`: The number of requests served by HTTP/S load balancer. +- `loadbalancing.https.response_bytes_count`: The number of bytes sent as responses from HTTP/S load balancer to clients. +- `loadbalancing.https.total_latencies`: A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte. +- `loadbalancing.l3.internal.egress_bytes_count`: The number of bytes sent from ILB backend to client (for TCP flows it's counting bytes on application stream only). +- `loadbalancing.l3.internal.egress_packets_count`: The number of packets sent from ILB backend to client of the flow. +- `loadbalancing.l3.internal.ingress_bytes_count`: The number of bytes sent from client to ILB backend (for TCP flows it's counting bytes on application stream only). +- `loadbalancing.l3.internal.ingress_packets_count`: The number of packets sent from client to ILB backend. +- `loadbalancing.l3.internal.rtt_latencies`: A distribution of RTT measured over TCP connections for ILB flows. +- `loadbalancing.tcp_ssl_proxy.closed_connections`: Number of connections that were terminated over TCP/SSL proxy. +- `loadbalancing.tcp_ssl_proxy.egress_bytes_count`: Number of bytes sent from VM to client using proxy. +- `loadbalancing.tcp_ssl_proxy.frontend_tcp_rtt`: A distribution of the smoothed RTT (in ms) measured by the proxy's TCP stack, each minute application layer bytes pass from proxy to client. +- `loadbalancing.tcp_ssl_proxy.ingress_bytes_count`: Number of bytes sent from client to VM using proxy. +- `loadbalancing.tcp_ssl_proxy.new_connections`: Number of connections that were created over TCP/SSL proxy. +- `loadbalancing.tcp_ssl_proxy.open_connections`: Current number of outstanding connections through the TCP/SSL proxy. diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/fields.yml new file mode 100644 index 00000000000..57761c8ea1b --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/_meta/fields.yml @@ -0,0 +1,249 @@ +- name: loadbalancing + release: beta + type: group + description: Google Cloud Load Balancing metrics + fields: + - name: https + type: group + description: Google Cloud Load Balancing metrics + fields: + - name: backend_latencies + description: A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. + type: group + fields: + - name: count + type: long + - name: mean + type: long + - name: bucket_counts + type: long + - name: bucket_options + type: group + fields: + - name: Options + type: group + fields: + - name: ExponentialBuckets + type: group + fields: + - name: growth_factor + type: double + - name: scale + type: long + - name: num_finite_buckets + type: long + - name: backend_request_bytes_count + type: long + description: The number of bytes sent as requests from HTTP/S load balancer to backends. + - name: backend_request_count + type: long + description: The number of requests served by backends of HTTP/S load balancer. + - name: backend_response_bytes_count + type: long + description: The number of bytes sent as responses from backends (or cache) to HTTP/S load balancer. + - name: frontend_tcp_rtt + description: A distribution of the RTT measured for each connection between client and proxy. + type: group + fields: + - name: count + type: long + - name: mean + type: long + - name: bucket_counts + type: long + - name: bucket_options + type: group + fields: + - name: Options + type: group + fields: + - name: ExponentialBuckets + type: group + fields: + - name: growth_factor + type: double + - name: scale + type: long + - name: num_finite_buckets + type: long + - name: internal + type: group + fields: + - name: backend_latencies + description: A distribution of the latency calculated from when the request was sent by the proxy to the backend until the proxy received from the backend the last byte of response. + type: group + fields: + - name: count + type: long + - name: mean + type: long + - name: bucket_counts + type: long + - name: bucket_options + type: group + fields: + - name: Options + type: group + fields: + - name: ExponentialBuckets + type: group + fields: + - name: growth_factor + type: double + - name: scale + type: long + - name: num_finite_buckets + type: long + - name: request_bytes_count + type: long + description: The number of bytes sent as requests from clients to HTTP/S load balancer. + - name: request_count + type: long + description: The number of requests served by HTTP/S load balancer. + - name: response_bytes_count + type: long + description: The number of bytes sent as responses from HTTP/S load balancer to clients. + - name: total_latencies + description: A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte. + type: group + fields: + - name: count + type: long + - name: mean + type: long + - name: bucket_counts + type: long + - name: bucket_options + type: group + fields: + - name: Options + type: group + fields: + - name: ExponentialBuckets + type: group + fields: + - name: growth_factor + type: double + - name: scale + type: long + - name: num_finite_buckets + type: long + - name: request_bytes_count + type: long + description: The number of bytes sent as requests from clients to HTTP/S load balancer. + - name: request_count + type: long + description: The number of requests served by HTTP/S load balancer. + - name: response_bytes_count + type: long + description: The number of bytes sent as responses from HTTP/S load balancer to clients. + - name: total_latencies + description: A distribution of the latency calculated from when the request was received by the proxy until the proxy got ACK from client on last response byte. + type: group + fields: + - name: count + type: long + - name: mean + type: long + - name: bucket_counts + type: long + - name: bucket_options + type: group + fields: + - name: Options + type: group + fields: + - name: ExponentialBuckets + type: group + fields: + - name: growth_factor + type: double + - name: scale + type: long + - name: num_finite_buckets + type: long + - name: l3.internal + type: group + description: Google Cloud Load Balancing metrics + fields: + - name: egress_bytes_count + type: long + description: The number of bytes sent from ILB backend to client (for TCP flows it's counting bytes on application stream only). + - name: egress_packets_count + type: long + description: The number of packets sent from ILB backend to client of the flow. + - name: ingress_bytes_count + type: long + description: The number of bytes sent from client to ILB backend (for TCP flows it's counting bytes on application stream only). + - name: ingress_packets_count + type: long + description: The number of packets sent from client to ILB backend. + - name: rtt_latencies + description: A distribution of RTT measured over TCP connections for ILB flows. + type: group + fields: + - name: count + type: long + - name: mean + type: long + - name: bucket_counts + type: long + - name: bucket_options + type: group + fields: + - name: Options + type: group + fields: + - name: ExponentialBuckets + type: group + fields: + - name: growth_factor + type: double + - name: scale + type: long + - name: num_finite_buckets + type: long + - name: tcp_ssl_proxy + type: group + description: Google Cloud Load Balancing metrics + fields: + - name: closed_connections + type: long + description: Number of connections that were terminated over TCP/SSL proxy. + - name: egress_bytes_count + type: long + description: Number of bytes sent from VM to client using proxy. + - name: frontend_tcp_rtt + description: A distribution of the smoothed RTT (in ms) measured by the proxy's TCP stack, each minute application layer bytes pass from proxy to client. + type: group + fields: + - name: count + type: long + - name: mean + type: long + - name: bucket_counts + type: long + - name: bucket_options + type: group + fields: + - name: Options + type: group + fields: + - name: ExponentialBuckets + type: group + fields: + - name: growth_factor + type: double + - name: scale + type: long + - name: num_finite_buckets + type: long + - name: ingress_bytes_count + type: long + description: Number of bytes sent from client to VM using proxy. + - name: new_connections + type: long + description: Number of connections that were created over TCP/SSL proxy. + - name: open_connections + type: long + description: Current number of outstanding connections through the TCP/SSL proxy. diff --git a/x-pack/metricbeat/module/googlecloud/loadbalancing/manifest.yml b/x-pack/metricbeat/module/googlecloud/loadbalancing/manifest.yml new file mode 100644 index 00000000000..479f92c94d4 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/loadbalancing/manifest.yml @@ -0,0 +1,33 @@ +default: false +input: + module: googlecloud + metricset: stackdriver + defaults: + stackdriver: + service: loadbalancing + metrics: + - "loadbalancing.googleapis.com/https/backend_latencies" + - "loadbalancing.googleapis.com/https/backend_latencies" + - "loadbalancing.googleapis.com/https/backend_request_bytes_count" + - "loadbalancing.googleapis.com/https/backend_request_count" + - "loadbalancing.googleapis.com/https/backend_response_bytes_count" + - "loadbalancing.googleapis.com/https/frontend_tcp_rtt" + - "loadbalancing.googleapis.com/https/request_bytes_count" + - "loadbalancing.googleapis.com/https/request_bytes_count" + - "loadbalancing.googleapis.com/https/request_count" + - "loadbalancing.googleapis.com/https/request_count" + - "loadbalancing.googleapis.com/https/response_bytes_count" + - "loadbalancing.googleapis.com/https/response_bytes_count" + - "loadbalancing.googleapis.com/https/total_latencies" + - "loadbalancing.googleapis.com/https/total_latencies" + - "loadbalancing.googleapis.com/l3/internal/egress_bytes_count" + - "loadbalancing.googleapis.com/l3/internal/egress_packets_count" + - "loadbalancing.googleapis.com/l3/internal/ingress_bytes_count" + - "loadbalancing.googleapis.com/l3/internal/ingress_packets_count" + - "loadbalancing.googleapis.com/l3/internal/rtt_latencies" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/closed_connections" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/egress_bytes_count" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/frontend_tcp_rtt" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/ingress_bytes_count" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/new_connections" + - "loadbalancing.googleapis.com/tcp_ssl_proxy/open_connections" diff --git a/x-pack/metricbeat/module/googlecloud/module.yml b/x-pack/metricbeat/module/googlecloud/module.yml index c7de504341e..6b6704ca08d 100644 --- a/x-pack/metricbeat/module/googlecloud/module.yml +++ b/x-pack/metricbeat/module/googlecloud/module.yml @@ -1,3 +1,4 @@ name: googlecloud metricsets: - compute +- loadbalancing diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go index d5956bb565c..01302e9b184 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go @@ -17,6 +17,8 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error) switch c.ServiceName { case googlecloud.ServiceCompute: return compute.NewMetadataService(c.ProjectID, c.Zone, c.opt...) + case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing: + return nil, nil default: return nil, errors.Errorf("service '%s' not supported", c.ServiceName) } diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go index bb59d6fffc3..8f46113ae03 100644 --- a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester.go @@ -7,6 +7,7 @@ package stackdriver import ( "context" "fmt" + "regexp" "sync" "time" @@ -51,11 +52,13 @@ type stackdriverMetricsRequester struct { func (r *stackdriverMetricsRequester) Metric(ctx context.Context, m string) (out []*monitoringpb.TimeSeries) { out = make([]*monitoringpb.TimeSeries, 0) + filter := r.getFilterForMetric(m) + req := &monitoringpb.ListTimeSeriesRequest{ Name: "projects/" + r.config.ProjectID, Interval: r.interval, View: monitoringpb.ListTimeSeriesRequest_FULL, - Filter: fmt.Sprintf(`metric.type="%s" AND resource.labels.zone = "%s"`, m, r.config.Zone), + Filter: filter, } it := r.client.ListTimeSeries(ctx, req) @@ -104,6 +107,25 @@ func (r *stackdriverMetricsRequester) Metrics(ctx context.Context, ms []string) return results, nil } +var serviceRegexp = regexp.MustCompile(`^(?P[a-z]+)\.googleapis.com.*`) + +// getFilterForMetric returns the filter associated with the corresponding filter. Some services like Pub/Sub fails +// if they have a region specified. +func (r *stackdriverMetricsRequester) getFilterForMetric(m string) (f string) { + f = fmt.Sprintf(`metric.type="%s"`, m) + + service := serviceRegexp.ReplaceAllString(m, "${service}") + + switch service { + case googlecloud.ServicePubsub, googlecloud.ServiceLoadBalancing: + return + default: + f = fmt.Sprintf(`%s AND resource.labels.zone = "%s"`, f, r.config.Zone) + } + + return +} + // Returns a GCP TimeInterval based on the provided config func getTimeInterval(windowTime time.Duration) (*monitoringpb.TimeInterval, error) { var startTime, endTime time.Time diff --git a/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go new file mode 100644 index 00000000000..308e32559b8 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/stackdriver/metrics_requester_test.go @@ -0,0 +1,24 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package stackdriver + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetFilterForMetric(t *testing.T) { + r := stackdriverMetricsRequester{config: config{Zone: "us-central1-a"}} + + s := r.getFilterForMetric("compute.googleapis.com/firewall/dropped_bytes_count") + assert.Equal(t, `metric.type="compute.googleapis.com/firewall/dropped_bytes_count" AND resource.labels.zone = "us-central1-a"`, s) + + s = r.getFilterForMetric("pubsub.googleapis.com/subscription/ack_message_count") + assert.Equal(t, `metric.type="pubsub.googleapis.com/subscription/ack_message_count"`, s) + + s = r.getFilterForMetric("loadbalancing.googleapis.com/https/backend_latencies") + assert.Equal(t, `metric.type="loadbalancing.googleapis.com/https/backend_latencies"`, s) +} diff --git a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go index bc1d39fba2a..90e217afd6d 100644 --- a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go @@ -52,7 +52,6 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i ecs := common.MapStr{ ECSCloud: common.MapStr{ - ECSCloudAvailabilityZone: availabilityZone, ECSCloudAccount: common.MapStr{ ECSCloudAccountID: accountID, }, @@ -60,6 +59,10 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i }, } + if availabilityZone != "" { + ecs[ECSCloud+"."+ECSCloudAvailabilityZone] = availabilityZone + } + //Remove keys from resource that refers to ECS fields if s.timeSeries == nil {