From a2e66a71d04efb934a366393b6e5848714c1aa47 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 31 Jul 2020 15:25:20 -0600 Subject: [PATCH 1/3] Add host inventory metrics to googlecloud compute metricset --- metricbeat/docs/fields.asciidoc | 242 +++++++++++------- .../module/googlecloud/_meta/fields.yml | 24 ++ .../googlecloud/compute/_meta/data.json | 33 ++- .../googlecloud/compute/_meta/data_cpu.json | 49 ++-- .../googlecloud/compute/_meta/data_disk.json | 26 +- .../compute/_meta/data_firewall.json | 45 ++-- .../compute/_meta/data_memory.json | 31 ++- .../compute/_meta/data_network.json | 31 ++- .../compute/compute_integration_test.go | 1 - .../metricbeat/module/googlecloud/fields.go | 2 +- .../module/googlecloud/metrics/metricset.go | 42 ++- 11 files changed, 331 insertions(+), 195 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 05d336127f5..c0effc2baab 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -18408,6 +18408,70 @@ type: object -- + +*`host.cpu.pct`*:: ++ +-- +Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 100%. + +type: scaled_float + +-- + +*`host.network.in.bytes`*:: ++ +-- +The number of bytes received on all network interfaces by the host in a given period of time. + +type: scaled_float + +-- + +*`host.network.out.bytes`*:: ++ +-- +The number of bytes sent out on all network interfaces by the host in a given period of time. + +type: scaled_float + +-- + +*`host.network.in.packets`*:: ++ +-- +The number of packets received on all network interfaces by the host in a given period of time. + +type: scaled_float + +-- + +*`host.network.out.packets`*:: ++ +-- +The number of packets sent out on all network interfaces by the host in a given period of time. + +type: scaled_float + +-- + +*`host.disk.read.bytes`*:: ++ +-- +The total number of bytes read successfully in a given period of time. + +type: scaled_float + +-- + +*`host.disk.write.bytes`*:: ++ +-- +The total number of bytes write successfully in a given period of time. + +type: scaled_float + +-- + [float] === compute @@ -18416,7 +18480,7 @@ Google Cloud Compute metrics -*`googlecloud.compute.instance.firewall.dropped_bytes_count.value`*:: +*`host.compute.instance.firewall.dropped_bytes_count.value`*:: + -- Incoming bytes dropped by the firewall @@ -18425,7 +18489,7 @@ type: long -- -*`googlecloud.compute.instance.firewall.dropped_packets_count.value`*:: +*`host.compute.instance.firewall.dropped_packets_count.value`*:: + -- Incoming packets dropped by the firewall @@ -18435,7 +18499,7 @@ type: long -- -*`googlecloud.compute.instance.cpu.reserved_cores.value`*:: +*`host.compute.instance.cpu.reserved_cores.value`*:: + -- Number of cores reserved on the host of the instance @@ -18444,7 +18508,7 @@ type: double -- -*`googlecloud.compute.instance.cpu.utilization.value`*:: +*`host.compute.instance.cpu.utilization.value`*:: + -- The fraction of the allocated CPU that is currently in use on the instance @@ -18453,7 +18517,7 @@ type: double -- -*`googlecloud.compute.instance.cpu.usage_time.value`*:: +*`host.compute.instance.cpu.usage_time.value`*:: + -- Usage for all cores in seconds @@ -18463,7 +18527,7 @@ type: double -- -*`googlecloud.compute.instance.disk.read_bytes_count.value`*:: +*`host.compute.instance.disk.read_bytes_count.value`*:: + -- Count of bytes read from disk @@ -18472,7 +18536,7 @@ type: long -- -*`googlecloud.compute.instance.disk.read_ops_count.value`*:: +*`host.compute.instance.disk.read_ops_count.value`*:: + -- Count of disk read IO operations @@ -18481,7 +18545,7 @@ type: long -- -*`googlecloud.compute.instance.disk.write_bytes_count.value`*:: +*`host.compute.instance.disk.write_bytes_count.value`*:: + -- Count of bytes written to disk @@ -18490,7 +18554,7 @@ type: long -- -*`googlecloud.compute.instance.disk.write_ops_count.value`*:: +*`host.compute.instance.disk.write_ops_count.value`*:: + -- Count of disk write IO operations @@ -18499,7 +18563,7 @@ type: long -- -*`googlecloud.compute.instance.uptime.value`*:: +*`host.compute.instance.uptime.value`*:: + -- How long the VM has been running, in seconds @@ -18509,7 +18573,7 @@ type: long -- -*`googlecloud.compute.instance.network.received_bytes_count.value`*:: +*`host.compute.instance.network.received_bytes_count.value`*:: + -- Count of bytes received from the network @@ -18518,7 +18582,7 @@ type: long -- -*`googlecloud.compute.instance.network.received_packets_count.value`*:: +*`host.compute.instance.network.received_packets_count.value`*:: + -- Count of packets received from the network @@ -18527,7 +18591,7 @@ type: long -- -*`googlecloud.compute.instance.network.sent_bytes_count.value`*:: +*`host.compute.instance.network.sent_bytes_count.value`*:: + -- Count of bytes sent over the network @@ -18536,7 +18600,7 @@ type: long -- -*`googlecloud.compute.instance.network.sent_packets_count.value`*:: +*`host.compute.instance.network.sent_packets_count.value`*:: + -- Count of packets sent over the network @@ -18546,7 +18610,7 @@ type: long -- -*`googlecloud.compute.instance.memory.balloon.ram_size.value`*:: +*`host.compute.instance.memory.balloon.ram_size.value`*:: + -- The total amount of memory in the VM. This metric is only available for VMs that belong to the e2 family. @@ -18555,7 +18619,7 @@ type: long -- -*`googlecloud.compute.instance.memory.balloon.ram_used.value`*:: +*`host.compute.instance.memory.balloon.ram_used.value`*:: + -- Memory currently used in the VM. This metric is only available for VMs that belong to the e2 family. @@ -18564,7 +18628,7 @@ type: long -- -*`googlecloud.compute.instance.memory.balloon.swap_in_bytes_count.value`*:: +*`host.compute.instance.memory.balloon.swap_in_bytes_count.value`*:: + -- The amount of memory read into the guest from its own swap space. This metric is only available for VMs that belong to the e2 family. @@ -18573,7 +18637,7 @@ type: long -- -*`googlecloud.compute.instance.memory.balloon.swap_out_bytes_count.value`*:: +*`host.compute.instance.memory.balloon.swap_out_bytes_count.value`*:: + -- The amount of memory written from the guest to its own swap space. This metric is only available for VMs that belong to the e2 family. @@ -18594,7 +18658,7 @@ Google Cloud Load Balancing metrics Google Cloud Load Balancing metrics -*`googlecloud.loadbalancing.https.backend_request_bytes_count.value`*:: +*`host.loadbalancing.https.backend_request_bytes_count.value`*:: + -- The number of bytes sent as requests from HTTP/S load balancer to backends. @@ -18603,7 +18667,7 @@ type: long -- -*`googlecloud.loadbalancing.https.backend_request_count.value`*:: +*`host.loadbalancing.https.backend_request_count.value`*:: + -- The number of requests served by backends of HTTP/S load balancer. @@ -18612,7 +18676,7 @@ type: long -- -*`googlecloud.loadbalancing.https.request_bytes_count.value`*:: +*`host.loadbalancing.https.request_bytes_count.value`*:: + -- The number of bytes sent as requests from clients to HTTP/S load balancer. @@ -18621,7 +18685,7 @@ type: long -- -*`googlecloud.loadbalancing.https.request_count.value`*:: +*`host.loadbalancing.https.request_count.value`*:: + -- The number of requests served by HTTP/S load balancer. @@ -18630,7 +18694,7 @@ type: long -- -*`googlecloud.loadbalancing.https.response_bytes_count.value`*:: +*`host.loadbalancing.https.response_bytes_count.value`*:: + -- The number of bytes sent as responses from HTTP/S load balancer to clients. @@ -18645,7 +18709,7 @@ type: long Google Cloud Load Balancing metrics -*`googlecloud.loadbalancing.l3.internal.egress_bytes_count.value`*:: +*`host.loadbalancing.l3.internal.egress_bytes_count.value`*:: + -- The number of bytes sent from ILB backend to client (for TCP flows it's counting bytes on application stream only). @@ -18654,7 +18718,7 @@ type: long -- -*`googlecloud.loadbalancing.l3.internal.egress_packets_count.value`*:: +*`host.loadbalancing.l3.internal.egress_packets_count.value`*:: + -- The number of packets sent from ILB backend to client of the flow. @@ -18663,7 +18727,7 @@ type: long -- -*`googlecloud.loadbalancing.l3.internal.ingress_bytes_count.value`*:: +*`host.loadbalancing.l3.internal.ingress_bytes_count.value`*:: + -- The number of bytes sent from client to ILB backend (for TCP flows it's counting bytes on application stream only). @@ -18672,7 +18736,7 @@ type: long -- -*`googlecloud.loadbalancing.l3.internal.ingress_packets_count.value`*:: +*`host.loadbalancing.l3.internal.ingress_packets_count.value`*:: + -- The number of packets sent from client to ILB backend. @@ -18687,7 +18751,7 @@ type: long Google Cloud Load Balancing metrics -*`googlecloud.loadbalancing.tcp_ssl_proxy.closed_connections.value`*:: +*`host.loadbalancing.tcp_ssl_proxy.closed_connections.value`*:: + -- Number of connections that were terminated over TCP/SSL proxy. @@ -18696,7 +18760,7 @@ type: long -- -*`googlecloud.loadbalancing.tcp_ssl_proxy.egress_bytes_count.value`*:: +*`host.loadbalancing.tcp_ssl_proxy.egress_bytes_count.value`*:: + -- Number of bytes sent from VM to client using proxy. @@ -18705,7 +18769,7 @@ type: long -- -*`googlecloud.loadbalancing.tcp_ssl_proxy.ingress_bytes_count.value`*:: +*`host.loadbalancing.tcp_ssl_proxy.ingress_bytes_count.value`*:: + -- Number of bytes sent from client to VM using proxy. @@ -18714,7 +18778,7 @@ type: long -- -*`googlecloud.loadbalancing.tcp_ssl_proxy.new_connections.value`*:: +*`host.loadbalancing.tcp_ssl_proxy.new_connections.value`*:: + -- Number of connections that were created over TCP/SSL proxy. @@ -18723,7 +18787,7 @@ type: long -- -*`googlecloud.loadbalancing.tcp_ssl_proxy.open_connections.value`*:: +*`host.loadbalancing.tcp_ssl_proxy.open_connections.value`*:: + -- Current number of outstanding connections through the TCP/SSL proxy. @@ -18744,7 +18808,7 @@ Google Cloud PubSub metrics Suscription related metrics -*`googlecloud.pubsub.subscription.ack_message_count.value`*:: +*`host.pubsub.subscription.ack_message_count.value`*:: + -- Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. @@ -18753,7 +18817,7 @@ type: long -- -*`googlecloud.pubsub.subscription.backlog_bytes.value`*:: +*`host.pubsub.subscription.backlog_bytes.value`*:: + -- Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. @@ -18762,7 +18826,7 @@ type: long -- -*`googlecloud.pubsub.subscription.num_outstanding_messages.value`*:: +*`host.pubsub.subscription.num_outstanding_messages.value`*:: + -- Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. @@ -18771,7 +18835,7 @@ type: long -- -*`googlecloud.pubsub.subscription.num_undelivered_messages.value`*:: +*`host.pubsub.subscription.num_undelivered_messages.value`*:: + -- Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. @@ -18780,7 +18844,7 @@ type: long -- -*`googlecloud.pubsub.subscription.oldest_unacked_message_age.value`*:: +*`host.pubsub.subscription.oldest_unacked_message_age.value`*:: + -- Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. @@ -18789,7 +18853,7 @@ type: long -- -*`googlecloud.pubsub.subscription.pull_ack_message_operation_count.value`*:: +*`host.pubsub.subscription.pull_ack_message_operation_count.value`*:: + -- Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. @@ -18798,7 +18862,7 @@ type: long -- -*`googlecloud.pubsub.subscription.pull_ack_request_count.value`*:: +*`host.pubsub.subscription.pull_ack_request_count.value`*:: + -- Cumulative count of acknowledge requests, grouped by result. @@ -18807,7 +18871,7 @@ type: long -- -*`googlecloud.pubsub.subscription.pull_message_operation_count.value`*:: +*`host.pubsub.subscription.pull_message_operation_count.value`*:: + -- Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. @@ -18816,7 +18880,7 @@ type: long -- -*`googlecloud.pubsub.subscription.pull_request_count.value`*:: +*`host.pubsub.subscription.pull_request_count.value`*:: + -- Cumulative count of pull requests, grouped by result. @@ -18825,7 +18889,7 @@ type: long -- -*`googlecloud.pubsub.subscription.push_request_count.value`*:: +*`host.pubsub.subscription.push_request_count.value`*:: + -- Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. @@ -18834,7 +18898,7 @@ type: long -- -*`googlecloud.pubsub.subscription.push_request_latencies.value`*:: +*`host.pubsub.subscription.push_request_latencies.value`*:: + -- Distribution of push request latencies (in microseconds), grouped by result. @@ -18843,7 +18907,7 @@ type: long -- -*`googlecloud.pubsub.subscription.sent_message_count.value`*:: +*`host.pubsub.subscription.sent_message_count.value`*:: + -- Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. @@ -18852,7 +18916,7 @@ type: long -- -*`googlecloud.pubsub.subscription.streaming_pull_ack_message_operation_count.value`*:: +*`host.pubsub.subscription.streaming_pull_ack_message_operation_count.value`*:: + -- Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. @@ -18861,7 +18925,7 @@ type: long -- -*`googlecloud.pubsub.subscription.streaming_pull_ack_request_count.value`*:: +*`host.pubsub.subscription.streaming_pull_ack_request_count.value`*:: + -- Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. @@ -18870,7 +18934,7 @@ type: long -- -*`googlecloud.pubsub.subscription.streaming_pull_message_operation_count.value`*:: +*`host.pubsub.subscription.streaming_pull_message_operation_count.value`*:: + -- Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count @@ -18879,7 +18943,7 @@ type: long -- -*`googlecloud.pubsub.subscription.streaming_pull_response_count.value`*:: +*`host.pubsub.subscription.streaming_pull_response_count.value`*:: + -- Cumulative count of streaming pull responses, grouped by result. @@ -18888,7 +18952,7 @@ type: long -- -*`googlecloud.pubsub.subscription.dead_letter_message_count.value`*:: +*`host.pubsub.subscription.dead_letter_message_count.value`*:: + -- Cumulative count of messages published to dead letter topic, grouped by result. @@ -18897,7 +18961,7 @@ type: long -- -*`googlecloud.pubsub.subscription.mod_ack_deadline_message_count.value`*:: +*`host.pubsub.subscription.mod_ack_deadline_message_count.value`*:: + -- Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. @@ -18906,7 +18970,7 @@ type: long -- -*`googlecloud.pubsub.subscription.mod_ack_deadline_message_operation_count.value`*:: +*`host.pubsub.subscription.mod_ack_deadline_message_operation_count.value`*:: + -- Cumulative count of ModifyAckDeadline message operations, grouped by result. @@ -18915,7 +18979,7 @@ type: long -- -*`googlecloud.pubsub.subscription.mod_ack_deadline_request_count.value`*:: +*`host.pubsub.subscription.mod_ack_deadline_request_count.value`*:: + -- Cumulative count of ModifyAckDeadline requests, grouped by result. @@ -18924,7 +18988,7 @@ type: long -- -*`googlecloud.pubsub.subscription.oldest_retained_acked_message_age.value`*:: +*`host.pubsub.subscription.oldest_retained_acked_message_age.value`*:: + -- Age (in seconds) of the oldest acknowledged message retained in a subscription. @@ -18933,7 +18997,7 @@ type: long -- -*`googlecloud.pubsub.subscription.oldest_retained_acked_message_age_by_region.value`*:: +*`host.pubsub.subscription.oldest_retained_acked_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. @@ -18942,7 +19006,7 @@ type: long -- -*`googlecloud.pubsub.subscription.oldest_unacked_message_age_by_region.value`*:: +*`host.pubsub.subscription.oldest_unacked_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. @@ -18951,7 +19015,7 @@ type: long -- -*`googlecloud.pubsub.subscription.retained_acked_bytes.value`*:: +*`host.pubsub.subscription.retained_acked_bytes.value`*:: + -- Total byte size of the acknowledged messages retained in a subscription. @@ -18960,7 +19024,7 @@ type: long -- -*`googlecloud.pubsub.subscription.retained_acked_bytes_by_region.value`*:: +*`host.pubsub.subscription.retained_acked_bytes_by_region.value`*:: + -- Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. @@ -18969,7 +19033,7 @@ type: long -- -*`googlecloud.pubsub.subscription.seek_request_count.value`*:: +*`host.pubsub.subscription.seek_request_count.value`*:: + -- Cumulative count of seek attempts, grouped by result. @@ -18978,7 +19042,7 @@ type: long -- -*`googlecloud.pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation_count.value`*:: +*`host.pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation_count.value`*:: + -- Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. @@ -18987,7 +19051,7 @@ type: long -- -*`googlecloud.pubsub.subscription.streaming_pull_mod_ack_deadline_request_count.value`*:: +*`host.pubsub.subscription.streaming_pull_mod_ack_deadline_request_count.value`*:: + -- Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. @@ -18996,7 +19060,7 @@ type: long -- -*`googlecloud.pubsub.subscription.byte_cost.value`*:: +*`host.pubsub.subscription.byte_cost.value`*:: + -- Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. @@ -19005,7 +19069,7 @@ type: long -- -*`googlecloud.pubsub.subscription.config_updates_count.value`*:: +*`host.pubsub.subscription.config_updates_count.value`*:: + -- Cumulative count of configuration changes for each subscription, grouped by operation type and result. @@ -19014,7 +19078,7 @@ type: long -- -*`googlecloud.pubsub.subscription.unacked_bytes_by_region.value`*:: +*`host.pubsub.subscription.unacked_bytes_by_region.value`*:: + -- Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. @@ -19029,7 +19093,7 @@ type: long Topic related metrics -*`googlecloud.pubsub.topic.streaming_pull_response_count.value`*:: +*`host.pubsub.topic.streaming_pull_response_count.value`*:: + -- Cumulative count of streaming pull responses, grouped by result. @@ -19038,7 +19102,7 @@ type: long -- -*`googlecloud.pubsub.topic.send_message_operation_count.value`*:: +*`host.pubsub.topic.send_message_operation_count.value`*:: + -- Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. @@ -19047,7 +19111,7 @@ type: long -- -*`googlecloud.pubsub.topic.send_request_count.value`*:: +*`host.pubsub.topic.send_request_count.value`*:: + -- Cumulative count of publish requests, grouped by result. @@ -19056,7 +19120,7 @@ type: long -- -*`googlecloud.pubsub.topic.oldest_retained_acked_message_age_by_region.value`*:: +*`host.pubsub.topic.oldest_retained_acked_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. @@ -19065,7 +19129,7 @@ type: long -- -*`googlecloud.pubsub.topic.oldest_unacked_message_age_by_region.value`*:: +*`host.pubsub.topic.oldest_unacked_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. @@ -19074,7 +19138,7 @@ type: long -- -*`googlecloud.pubsub.topic.retained_acked_bytes_by_region.value`*:: +*`host.pubsub.topic.retained_acked_bytes_by_region.value`*:: + -- Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. @@ -19083,7 +19147,7 @@ type: long -- -*`googlecloud.pubsub.topic.byte_cost.value`*:: +*`host.pubsub.topic.byte_cost.value`*:: + -- Cost of operations, measured in bytes. This is used to measure utilization for quotas. @@ -19092,7 +19156,7 @@ type: long -- -*`googlecloud.pubsub.topic.config_updates_count.value`*:: +*`host.pubsub.topic.config_updates_count.value`*:: + -- Cumulative count of configuration changes, grouped by operation type and result. @@ -19101,7 +19165,7 @@ type: long -- -*`googlecloud.pubsub.topic.message_sizes.value`*:: +*`host.pubsub.topic.message_sizes.value`*:: + -- Distribution of publish message sizes (in bytes) @@ -19110,7 +19174,7 @@ type: long -- -*`googlecloud.pubsub.topic.unacked_bytes_by_region.value`*:: +*`host.pubsub.topic.unacked_bytes_by_region.value`*:: + -- Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. @@ -19125,7 +19189,7 @@ type: long Snapshot related metrics -*`googlecloud.pubsub.snapshot.oldest_message_age.value`*:: +*`host.pubsub.snapshot.oldest_message_age.value`*:: + -- Age (in seconds) of the oldest message retained in a snapshot. @@ -19134,7 +19198,7 @@ type: long -- -*`googlecloud.pubsub.snapshot.oldest_message_age_by_region.value`*:: +*`host.pubsub.snapshot.oldest_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. @@ -19143,7 +19207,7 @@ type: long -- -*`googlecloud.pubsub.snapshot.backlog_bytes.value`*:: +*`host.pubsub.snapshot.backlog_bytes.value`*:: + -- Total byte size of the messages retained in a snapshot. @@ -19152,7 +19216,7 @@ type: long -- -*`googlecloud.pubsub.snapshot.backlog_bytes_by_region.value`*:: +*`host.pubsub.snapshot.backlog_bytes_by_region.value`*:: + -- Total byte size of the messages retained in a snapshot, broken down by Cloud region. @@ -19161,7 +19225,7 @@ type: long -- -*`googlecloud.pubsub.snapshot.num_messages.value`*:: +*`host.pubsub.snapshot.num_messages.value`*:: + -- Number of messages retained in a snapshot. @@ -19170,7 +19234,7 @@ type: long -- -*`googlecloud.pubsub.snapshot.num_messages_by_region.value`*:: +*`host.pubsub.snapshot.num_messages_by_region.value`*:: + -- Number of messages retained in a snapshot, broken down by Cloud region. @@ -19179,7 +19243,7 @@ type: long -- -*`googlecloud.pubsub.snapshot.config_updates_count.value`*:: +*`host.pubsub.snapshot.config_updates_count.value`*:: + -- Cumulative count of configuration changes, grouped by operation type and result. @@ -19195,7 +19259,7 @@ Google Cloud Storage metrics -*`googlecloud.storage.api.request_count.value`*:: +*`host.storage.api.request_count.value`*:: + -- Delta count of API calls, grouped by the API method name and response code. @@ -19205,7 +19269,7 @@ type: long -- -*`googlecloud.storage.authz.acl_based_object_access_count.value`*:: +*`host.storage.authz.acl_based_object_access_count.value`*:: + -- Delta count of requests that result in an object being granted access solely due to object ACLs. @@ -19214,7 +19278,7 @@ type: long -- -*`googlecloud.storage.authz.acl_operations_count.value`*:: +*`host.storage.authz.acl_operations_count.value`*:: + -- Usage of ACL operations broken down by type. @@ -19223,7 +19287,7 @@ type: long -- -*`googlecloud.storage.authz.object_specific_acl_mutation_count.value`*:: +*`host.storage.authz.object_specific_acl_mutation_count.value`*:: + -- Delta count of changes made to object specific ACLs. @@ -19233,7 +19297,7 @@ type: long -- -*`googlecloud.storage.network.received_bytes_count.value`*:: +*`host.storage.network.received_bytes_count.value`*:: + -- Delta count of bytes received over the network, grouped by the API method name and response code. @@ -19242,7 +19306,7 @@ type: long -- -*`googlecloud.storage.network.sent_bytes_count.value`*:: +*`host.storage.network.sent_bytes_count.value`*:: + -- Delta count of bytes sent over the network, grouped by the API method name and response code. @@ -19252,7 +19316,7 @@ type: long -- -*`googlecloud.storage.storage.object_count.value`*:: +*`host.storage.storage.object_count.value`*:: + -- Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. @@ -19261,7 +19325,7 @@ type: long -- -*`googlecloud.storage.storage.total_byte_seconds.value`*:: +*`host.storage.storage.total_byte_seconds.value`*:: + -- Delta count of bytes received over the network, grouped by the API method name and response code. @@ -19270,7 +19334,7 @@ type: long -- -*`googlecloud.storage.storage.total_bytes.value`*:: +*`host.storage.storage.total_bytes.value`*:: + -- Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. diff --git a/x-pack/metricbeat/module/googlecloud/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/_meta/fields.yml index 84e340de576..6619b23d84b 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/fields.yml @@ -30,3 +30,27 @@ object_type_mapping_type: "*" description: > Metrics that returned from Google Cloud API query. + - name: host + type: group + fields: + - name: cpu.pct + type: scaled_float + description: Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 100%. + - name: network.in.bytes + type: scaled_float + description: The number of bytes received on all network interfaces by the host in a given period of time. + - name: network.out.bytes + type: scaled_float + description: The number of bytes sent out on all network interfaces by the host in a given period of time. + - name: network.in.packets + type: scaled_float + description: The number of packets received on all network interfaces by the host in a given period of time. + - name: network.out.packets + type: scaled_float + description: The number of packets sent out on all network interfaces by the host in a given period of time. + - name: disk.read.bytes + type: scaled_float + description: The total number of bytes read successfully in a given period of time. + - name: disk.write.bytes + type: scaled_float + description: The total number of bytes write successfully in a given period of time. diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json index 13c4958d13f..5ceebb5e516 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json @@ -5,15 +5,15 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "17557085450298074", + "name": "gke-chrismark-leaderelec-default-pool-896893cb-t17x" }, "machine": { - "type": "e2-standard-2" + "type": "e2-medium" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "us-central1-c", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -23,31 +23,31 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 171 + "value": 432 }, "dropped_packets_count": { - "value": 4 + "value": 6 } }, "instance": { "cpu": { "reserved_cores": { - "value": 2 + "value": 1 }, "usage_time": { - "value": 0.1803964574089818 + "value": 9.515682164234931 }, "utilization": { - "value": 0.001503303811741515 + "value": 0.15859470273724885 } }, "memory": { "balloon": { "ram_size": { - "value": 7896264704 + "value": 4140863488 }, "ram_used": { - "value": 416518144 + "value": 2765357056 }, "swap_in_bytes_count": { "value": 0 @@ -58,16 +58,23 @@ } }, "uptime": { - "value": 60.000000000000455 + "value": 60.000000000007276 } } }, "labels": { "user": { - "created-by": "ks" + "goog-gke-node": "" } } }, + "host": { + "cpu": { + "pct": 15.859470273724884 + }, + "id": "17557085450298074", + "name": "gke-chrismark-leaderelec-default-pool-896893cb-t17x" + }, "metricset": { "name": "compute", "period": 10000 diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json index 13c4958d13f..93fb745c1a9 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json @@ -5,15 +5,15 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "8825965883407652424", + "name": "heartbeat-los-angeles" }, "machine": { - "type": "e2-standard-2" + "type": "n1-standard-8" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "us-west2-a", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -23,50 +23,37 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 171 + "value": 586 }, "dropped_packets_count": { - "value": 4 + "value": 6 } }, "instance": { "cpu": { "reserved_cores": { - "value": 2 + "value": 0.2 }, "usage_time": { - "value": 0.1803964574089818 + "value": 1.0218381873273756 }, "utilization": { - "value": 0.001503303811741515 - } - }, - "memory": { - "balloon": { - "ram_size": { - "value": 7896264704 - }, - "ram_used": { - "value": 416518144 - }, - "swap_in_bytes_count": { - "value": 0 - }, - "swap_out_bytes_count": { - "value": 0 - } + "value": 0.0851531822772813 } }, "uptime": { - "value": 60.000000000000455 + "value": 60 } } }, - "labels": { - "user": { - "created-by": "ks" - } - } + "labels": {} + }, + "host": { + "cpu": { + "pct": 8.51531822772813 + }, + "id": "8825965883407652424", + "name": "heartbeat-los-angeles" }, "metricset": { "name": "compute", diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json index 38b0e2a5b4f..197c29a8cee 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json @@ -5,11 +5,11 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "2572129596446793084", + "name": "johns-test-node" }, "machine": { - "type": "e2-standard-2" + "type": "e2-medium" }, "provider": "googlecloud" }, @@ -30,25 +30,37 @@ "value": 0 }, "write_bytes_count": { - "value": 158506 + "value": 163242 }, "write_ops_count": { - "value": 25 + "value": 27 } } } }, "labels": { "metrics": { - "device_name": "instance-test-ks", + "device_name": "johns-test-node", "device_type": "permanent", "storage_type": "pd-standard" }, "user": { - "created-by": "ks" + "goog-gke-node": "" } } }, + "host": { + "disk": { + "read": { + "bytes": 0 + }, + "write": { + "bytes": 163242 + } + }, + "id": "2572129596446793084", + "name": "johns-test-node" + }, "metricset": { "name": "compute", "period": 10000 diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json index f147f276f32..6e1a0a73323 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json @@ -5,11 +5,11 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "2572129596446793084", + "name": "johns-test-node" }, "machine": { - "type": "e2-standard-2" + "type": "n1-standard-8" }, "provider": "googlecloud" }, @@ -23,38 +23,22 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 277 + "value": 1613 }, "dropped_packets_count": { - "value": 5 + "value": 8 } }, "instance": { "cpu": { "reserved_cores": { - "value": 2 + "value": 8 }, "usage_time": { - "value": 0.18404532833979204 + "value": 0.10854628651577514 }, "utilization": { - "value": 0.001533711069498267 - } - }, - "memory": { - "balloon": { - "ram_size": { - "value": 7896264704 - }, - "ram_used": { - "value": 416260096 - }, - "swap_in_bytes_count": { - "value": 0 - }, - "swap_out_bytes_count": { - "value": 0 - } + "value": 0.00022613809690786487 } }, "uptime": { @@ -62,11 +46,14 @@ } } }, - "labels": { - "user": { - "created-by": "ks" - } - } + "labels": {} + }, + "host": { + "cpu": { + "pct": 0.022613809690786486 + }, + "id": "2572129596446793084", + "name": "johns-test-node" }, "metricset": { "name": "compute", diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json index f147f276f32..1fea94f0f9d 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json @@ -5,15 +5,15 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "7920542032463815386", + "name": "gke-chrismark-leaderelec-default-pool-896893cb-nm8x" }, "machine": { - "type": "e2-standard-2" + "type": "e2-medium" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "us-central1-c", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -23,31 +23,31 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 277 + "value": 211 }, "dropped_packets_count": { - "value": 5 + "value": 3 } }, "instance": { "cpu": { "reserved_cores": { - "value": 2 + "value": 1 }, "usage_time": { - "value": 0.18404532833979204 + "value": 8.026239710991831 }, "utilization": { - "value": 0.001533711069498267 + "value": 0.13377066184986386 } }, "memory": { "balloon": { "ram_size": { - "value": 7896264704 + "value": 4140863488 }, "ram_used": { - "value": 416260096 + "value": 2498785280 }, "swap_in_bytes_count": { "value": 0 @@ -64,10 +64,17 @@ }, "labels": { "user": { - "created-by": "ks" + "goog-gke-node": "" } } }, + "host": { + "cpu": { + "pct": 13.377066184986386 + }, + "id": "7920542032463815386", + "name": "gke-chrismark-leaderelec-default-pool-896893cb-nm8x" + }, "metricset": { "name": "compute", "period": 10000 diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json index 1b86e390e16..7acf27dcb7c 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json @@ -5,15 +5,15 @@ "id": "elastic-observability" }, "instance": { - "id": "7692260917184259934", - "name": "instance-test-ks" + "id": "8825965883407652424", + "name": "heartbeat-los-angeles" }, "machine": { - "type": "e2-standard-2" + "type": "n1-standard-8" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "us-west2-a", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -24,16 +24,16 @@ "instance": { "network": { "received_bytes_count": { - "value": 13073 + "value": 2817591 }, "received_packets_count": { - "value": 95 + "value": 2528 }, "sent_bytes_count": { - "value": 0 + "value": 121736 }, "sent_packets_count": { - "value": 0 + "value": 754 } } } @@ -41,9 +41,20 @@ "labels": { "metrics": { "loadbalanced": "false" + } + } + }, + "host": { + "id": "8825965883407652424", + "name": "heartbeat-los-angeles", + "network": { + "in": { + "bytes": 121736, + "packets": 754 }, - "user": { - "created-by": "ks" + "out": { + "bytes": 2817591, + "packets": 2528 } } }, diff --git a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go index a91c26948d0..30c7061760e 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go @@ -2,7 +2,6 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -// +build integration // +build googlecloud package compute diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index ad15833fa63..015b1e85bc6 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzcXN1u2zgWvu9THPSm7SJ1gd27YjFAxsXOFGh2AyTtrUBRxzbXFKnyJ1736RckJVmyJVmWJSWZzs3EtsjvO//nUNJH2OL+M6ylXHOkXNrkDYBhhuNnePeH/xSW7mO458SspErfvQFQyJFo/AwxGvIGIEFNFcsMk+Iz/PYGAOCP5T2kMrEc3wCsGPJEf/ZffARBUjze0v0z+8x9rqTN8k+q11Wv5SRGrsuPi0tl/F+kpvJxA67iX87N7w+pFMxIxcQaUjSKUX26xTGWKh6rUS3+VvuqFZP7Fz6Mwi+2uN9JlTQunKIhCTFkqsUd1UnW1nttMB1t6WLZtyXm8N/b8zZQWzeRNvYW2fBtlJIsY2Kd//RtbfEOS7rLLcZsiAGFxiqBCayUTKHmQLf3X+GnRbVfnNCiMs2sqeI69rA6yaqPnKCrbboMKxe67mnPTGhDBMVG7R1v3rZYdcEVU7gjnJ/8oGvRroWriydKZhkmUbw3qCMqrTCLJ8LtMfz6jlyKdcsPauL8KqhMXWTwyxebQbwHs8EuYqcAM0K3aCaEmG/QG2Rpf5mdRDMKNaonTCIqFeoejE/8s5Xzv20aowK5Ar92uRVI4TlvpDbuW/f/LeZcx2oN4+wXcauPCvTRKUAR6v4qABHOJSUGE1jefw+Bg2mgVikUhu+BCZdSCir94GuyxsiwFEdF/90tCyupHOZc1EyARipFolsNKmF6O5FFkakcfenWcxoKju52ClG8hcwRKJlNCMlBCIi+/gdkhsrb6an8q6h2ihmcR1ZuK4MCjDwvrABremn5fc6Iq3SerNNxOmHUIPwpd/533m9/3MGGaIgRBSgrBBPrmz7OI9DspJrKfyiyp8mS5YkPhd2CHzmZtHNrwDhVvixRFvlyGE6NwswjR7cTyCdUl2GbTX798B36jlSq/SJ2WVCKacycpJFmv/rkwr6kXR430hAOJC34BybOp4O/L+Bxw3RebLuULgXfA3kijJOYhzz64y5vFWIMkUL6i/HvsCIp45XuoI2Y1ZiMSOwukDhUH279OTnpHckiJiZyJae3E435VMpEjnNtUZvg+8xokDvhMYHOCMWZ+Es7VSxpFECRscuAF2Rg5OQSKGc4kiQx4URQVuMxVuf7TZIEfi82uLAB3hiTHSfo9qh0BYg2IFUwsYuyIokU/nRK6mUm/UsWZx6ibKYq+Ya4vOh31MFM/nx8vP/04BUHQXMu4MsCnz618jYG02Av0eadYLwvobmvm+C3Q35uYVPOUBjt5DsM+GwyvhSezqTQ/RqTsQQbtjxjxrnEF40Bgf9jwYRBJcjxFGXmsFAAwrVCrXtIsUOG/STohfb12++FMx1kBe9d6H9c3sOKy50GZt5p8FgO8zIpgGQZZ9S3YKCNQpL6DPLh2EiOqPWpXoeRq5WtHfTyaY0j14aViXn1kCMzsgZ5Gj0U3OZURCO/Zpc0NIu05lGm5P/2LyVXUy61n3cKgX7k1z717B/cqrPOct1Qbu1QIRhUKRN+nuibsMfl/aeHh2/gJdMeinuHkGFYj033x13Ftaz28+pugP2daxyEB+P7cdcPocDdzLqmCi9WtMxQjAxzGbrFii9La7QhInFSq0NX0q43PpK24P3YWvT7w+lTT5zuFKwQWWZjbeMJ2pJ7Gz/Y+MJ2RNu4XG9YpHuw5R+OijehYRGO0G2UovbHDWO55NKmlhPDnjCkrdCo+j2021DIHcdkHerN28PfZTl6EyQQfpAgZ0+o9h5Ad1PC5TpElzFKUD8ccquBZr+wqB6sqBEoab0ni+2CLAoY5RcfgAkgNZV3xB+bRhXHKxQzbiAqIeeCRV8j1SG+05BZvQEUSSaZMDcQWwNCGtijqamwm4wV5SbTkJlUHZInrvXyexwIRGQ9xtnC7Rrh/eH44ENhYGHTFmJtvC6ilVnOo6rfl0cqk0aACp+SzuEwp+byCrXlZgH/kgoIJLhighUHrk2XaqzE40+HgFwTyKdUJp52giThTGAr//OSG7srPyevxsCYS6kb7bw6dlu+auXOoVgvpIEa1ZuZIOoNEGMwzZohwnfB2RY9F33jA5e/xs+RFLA045iiMKE9TSRqnz1iYujG319XRuQFPEhAQjeFSMI8mkphCBOu08XaBQvfdFY3U84cQk+MSknlbMYltDV7QlG7FijxfTMSBanlhmUcwbAUO4adNZm7SktQNkoS+8K0USy2he17SoUMyo18jkgZVbJIFBdZjD84nLW+8/1XvD9yWSMLd3WJO5/NDSnzwqTDFUfPlMYeCgD3zo3/AkmtQaJzhJhy23o8hB0zGxBSfHSxZ18TMEsuC5dHzOa1kyN+cxnHP6lM8LdBJtJXjuXMf1bzyIf+FxmAYxxxNAbVvEEwszFnehNaK4cCAgowMmP0Ig6t2puFyG4jNUKxN+yIBpslft4Q7+FOJmy1v6XbL8UPrunge4eyaTifkunns/35zBFWe6rkHPa89VXoijD0PObugBv73wLQgDa+nUsU7yOF665bhZ+L1Q3ESm7RldA7cSircrQDJhdzcm2ZYYxF8kihEw/+mudMQ+yxCfeoahmJwUC1aMSZKkjEbWeT2rswfN7EU+8oTgP4wPRzjuQLKvJPOYdji4v4OnuPqNSjcwlPnlS1kCLRVgWfCWEn3FPHdLjb0sjiJ/DTSkNqT6O0HzZLsWLrKNRX4x2QNuklbGUDIaAbIlwoWEkVJjH1KFDRQSkEvz8QkZxVS5GDZgtzbUcC18S38iYFV74PO7J7dJdeeVj3ahsxjSKZfRTtW7BXO5WZ4gbLLjlN0ya8sNI6b7//ojX1NexeclF6Da9Rq4LrSoFKEeBTrS8NOk4enrseuDbzFx7i1D3NgUk9wvt9vO94RXx4JSXJpdZdZghBMr2Rx2PbvncQ5VdfWZHkQXHWwVDL1CQndDaAP1Pg7kY9NLrNc9NT27DgrMhr+ObwtDNIB4pZ2HTie7EuFWwV0ahy7Y1toChfW1I7tF5SkfUU73R5CCtfeA8pyVjvwN/96EmfJqPv8wBfkBty0MDt/VegxN8hUpG681P3TYpmIxOPohC9by2BygSbcx6xZvPrSt7Ve195FBONSZS/N4hQino8ozySRjn9yl8t5AzN+5TI31sEMbo+e62IcBk5oAEtOfI9JBZdMZn/8nb5raNydMQORepofMLbQ5xel98qRfBxIOg+5ctFrTOkbMVo5LCm1ow7EDiSfDHgSklSFWKBoUGa3S9vGG5yF72wYTDfo9c1HD/Kf603wpC3JlxJpvGdBGOFldPgfp2Wcxsf7WlIX+tUns3wy2vIUEFs6RZNTRA5G6Cc6KId9SDAP/ad96tSUPQrJGR/4wXlBFj+TmEWnkwhJp8JkzTjLjz5ByefCC8eBZHW+EsT0vHMin/VgreTKK+PX7XVH+iMV3UXZSzhvNRw/tqGF6bk/wcAAP//nHNPVA==" + return "eJzcXN2O27gVvs9THCxQZFNMnG17FxQLzM6iuwEy7QAzm1uBIo9t1hSp8Gdc5+kLkpIs2ZIs25Ims8nV2Bb5fef/HEp6DxvcfYSVUiuBVCjH3gBYbgV+hLe/hU/hzn8MD4LYpdLZ2zcAGgUSgx8hRUveADA0VPPcciU/ws9vAAB+u3uATDEn8A3AkqNg5mP44j1IkuHhlv6f3eX+c61cXnxSv65+rSApClN9XF6q0v8itbWPW3CV/wpuYX/IlORWaS5XkKHVnJrjLQ6x1PE4g3rx18ZXnZj8v/hhEn+xwd1Wada6cIaWMGLJVIt7qpOsbXbGYjba0uWyP1SY4/8fTttAY12mXBossuXbJCN5zuWq+OkPjcV7LOm+sBi7JhY0WqclMlhqlUHDgW4fPsFXh3q3aPjBWhl7tgPQ3C3yBs14raFEIEuWQpFOP3hATVFauHv4wxsuW8DTmht4JsIhcANS6YwI/g0ZpDuwawTpshQ1qGW4hiqNBohkwC1oIldoItufwCr4208//WVxhFei3Sq9WXC5SHcWj113CPCnBpSwDmikyJ+RgZJAhCg3Ai4t6iWhaEoSXs7AJRBY8WeUkKPmivmVLM+wG7JydmTMxktfOTsdZi4XOaEbtGOALlaaSdTj455O3IybzUIjYVcaiFWWiBbTJgyMoxSNWTohdmdD22pucRJsYeWLwVGV5c7Ww/BhQdHEWo+IRyAbMfYurlymtoHpm0tjiaTYmqwON+9arL7gkmvcEiGOftC3aN/C9cWZVnmOLAmaSKhy0i5C8G69qtxRKLnq+EFDnJ8kVZkvhKKii81KD+khdgywcMDpIJYePhRkLXtOohmNBvUzsiSkyAGMj8qRTs7/rhwwpt9yKx/UqtDlHW6NXebcxOosF/wb8auPCtTHjKUm1P9VAiJCKEosslA/hDqJG6BOa5Q2hg5nsKQyDL4hK0xCeBkT/R9+WVgqHTJFFDWXYJAqyUynQfloO5FFkakc/c6vd5BtQhnXQeYAlMonhOQhRESf/gMqRx3s9Fj+dVQhJc0jK7+VRenL3ZPCirCml1bMyP3iqpwn73WcXhgNCL+rbfhd8Nsv97AmBlJECdpJyeXqZojzFPXYRP4Ta9a5fKiokIMfhc6pk1sLxqnyZYXyqJI/C6evo+eRY6zYn1Gfh202+Q3Dtx+zZErvFqnPgkpOY+YkSwz/NiQXDiW9r/1JVvKPTLxPR38vJgex2PYpXUmxA/JMuCCpiHn0y30xGUkxRgoVLsa/w5JkXOwWJ4mFGcV4xO4jiX314defk5PZkjzhciJX8no70lhIpVwWOFcOjY2+z60BtZUBE5icUJyJv3JTxZJWAZQZuwp4UQZWTS6BamStCEuJIJLyBo+xOt/PijD4pdzgzAZ4bW1+mKC7o9IVILqA1MGkPspKlmj86pU0yEyGlyydAzni82LYsRhn/v709PDhMSgOouZ8wFclPnNs5V0MpsFeoS06wXRXQfNft8HvhvzSwqaCo7TGy/cy4LPJ+Fx4JlfSDGtMxhJs3PKEGRcSX7QGBPGPRRiPSnI4RZk5LJSAcKXRmAFS7JHhMAkGoX36/EvpTHtZwY8+9D/dPcBSqK0Bbt8aCFj28zIlgeS54DS0YGCsRpKFDPLu0EgOqA2pXi8j1yhbe+gV0xpPrgsrl/PqoUBmVQPyNHoouc2piFZ+7S5paZ4YI5Jcq//tvpdcTYUyYd4pJYaRX/fUc3hwq886q3VjubVFjWBRZ1yGeWJowp7uHj48Pn6GIJnuUDw4hFyG9dB0v9zXXMuZMK/uBzjcucZBuDe+L/fDEErczqxrqvFsRasc5cgw72K3WPNl5ayxRDIvtSZ0rdxqHSJpB973nUV/uBfn2BOnOwUrRZa71Lh0grbkwaWPLj2zHTEurda7LNI9uuoPTyWY0GURjtBNkqEJxw1jueSdy5wglj9jTFuxUQ17GL+hVFuBbBXrzdv931U5ehMlEH/AUPBn1LsAoL8pEWoVo8sYJWgYDvnVwPBvWFYPTjYIVLR+JIvNgixKGNUX7+LBcV3lPfHHZUnN8UrFjBuIKsiFYDHUSE2Ibw3kzqwBJcsVl/YGUmdBKgs7tA0V9pNxstpkGjKTqkMJ5luvsMeeQEJWY5wt3K4QftwfH7wrDSxu2kGsi9dZtHInRFL3++pIZdIIUONT0dkf5jRcXqNxwi7gX0oDAYZLLnl54Np2qcFaPP6wD8gNgXzIFAu0GRImuMRO/qclN3ZXfkperYGxkFI/2nl17Ld81cqdQ7FBSBdq1KxngmjWQKzFLG+HCH9IwTcYuJibELjCNWGOpIFnucAMpY3tKVNoQvZIiaXrcDtxFZEX8KgACV2XIonzaKqkJVz6ThcbFyxC01nfTHtziD0xaq20txlV3axVvxYoCX0zEg2ZE5bnAsM9XD3DzobMfaUlKR8lif3KjdU8daXtB0qlDKqNQo7IONWqTBRnWUw4OJy1vgv9V7o7cFmrSnf1ibuYzV1S5sVJhy+OXiiNPZYAHrwb/wmSWotE5wgx1bbNeAhbbtcglXzvY8+uIWDOzguXB8zmtZMDfnMZxz+pYvjzRSYyVI7VzH9W8yiG/mcZgGecCLQW9bxBMHep4GYdWyuPAiIKsCrn9CwOndqbhch2rQxCuTdsiQGXszBvSHdwrxhf7m7p5tfyB9d08IND2TScj8kM89nhfOYIqwNVcgp70fpq9EUYBh5zd8Ct/W8J6II2vptLku4Sjau+W4VfitUNpFpt0JfQW7kvqwq0F0wu5uTaMcMYi+SBQice/LXPmS6xxzbco6plJAYXqsUgzlRBIm56m9TBheHLJp5mR3EcwC9MP6dIfkdF/jHneGxxFl9v7wlVZnQu8cmTuhYyJMbp6DMx7MR76riJd1taVf4EvjplSeNplO7DZiWXfJXE+mq8A9I2vcStXCQEdF085ap0nMQ0o0BNB5UQwv7hIdlTailz0GxhrutI4Jr4Vt2k4Mv3y47snvylVx7WvdpGzKBks4+iQwv2aqcyU9xg2SenadqE76y0LtrvP2lNfQ2777kovYbXqFXBdaVArQgIqTaUBj0nDy9dD1yb+UsP8eqe5sCkGeHDPsF3giLevZKS5FzrrjKEJLlZq8Ox7dA7iIqrr6xIiqA462CoY2pSEDoZwF8ocPejvjS6zXPTU9ew4KTIG/jm8LQTSC8Us3TZxPdinSvYOqJR5ToY24WifG1Jbd96KU1WU7zT5TGufOY9pCTngwN//6MnQ5qMoc8D/IrCkr0Gbh8+ASXhDpGa1L2f+m8ytGvFAopS9KG1BKoYtuc84uz625W86/e+iiQlBllSvCaNhPf9jGaUB9Kopl/Fm9S8oQWfksVr2iBF32evNJE+I0c0YJRAsQPm0BeTxS9v7z73VI6e2L5IHY1PfHuI1+vd51oRfBgI+k/5ClGbHClfcpp4rJmz4w4EDiRfDrgywupCLDG0SLP/5Q2Xm9xZL2y4mO/hu+MOHuW/1hvhkrcmXEmm9Z0EY4WV4+B+nZYLGx/taciDd5TF5Q3kqCF1dIO2IYiCDVBBjDl8+2HVrypJMazAyO4mCMoLsPqdxjw+mUJsMRMmWS58eAoPTj4TUT4KopwNlzLS88xKeNVCsJOkqI9ftdXv6YxXdZdlLBGi0nDx2obvTMn/DwAA//8MI+Hs" } diff --git a/x-pack/metricbeat/module/googlecloud/metrics/metricset.go b/x-pack/metricbeat/module/googlecloud/metrics/metricset.go index 894e5734d0f..b3273e10fa7 100644 --- a/x-pack/metricbeat/module/googlecloud/metrics/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/metrics/metricset.go @@ -169,8 +169,7 @@ func (m *MetricSet) eventMapping(ctx context.Context, tss []timeSeriesWithAligne events := make([]mb.Event, 0) for _, groupedEvents := range tsGrouped { event := mb.Event{ - Timestamp: groupedEvents[0].Timestamp, - RootFields: groupedEvents[0].ECS, + Timestamp: groupedEvents[0].Timestamp, ModuleFields: common.MapStr{ "labels": groupedEvents[0].Labels, }, @@ -181,6 +180,12 @@ func (m *MetricSet) eventMapping(ctx context.Context, tss []timeSeriesWithAligne event.MetricSetFields.Put(singleEvent.Key, singleEvent.Value) } + if serviceName == "compute" { + event.RootFields = addHostFields(groupedEvents) + } else { + event.RootFields = groupedEvents[0].ECS + } + events = append(events, event) } @@ -266,3 +271,36 @@ func (m *MetricSet) getMetadata(out *metric.MetricDescriptor, metricsWithMeta ma metricsWithMeta[out.Type] = meta return metricsWithMeta } + +func addHostFields(groupedEvents []KeyValuePoint) common.MapStr { + hostRootFields := groupedEvents[0].ECS + // add host.id and host.name + if hostID, err := groupedEvents[0].ECS.GetValue("cloud.instance.id"); err == nil { + hostRootFields.Put("host.id", hostID) + } + + if hostName, err := groupedEvents[0].ECS.GetValue("cloud.instance.name"); err == nil { + hostRootFields.Put("host.name", hostName) + } + + hostFieldTable := map[string]string{ + "instance.cpu.utilization.value": "host.cpu.pct", + "instance.network.sent_bytes_count.value": "host.network.in.bytes", + "instance.network.received_bytes_count.value": "host.network.out.bytes", + "instance.network.sent_packets_count.value": "host.network.in.packets", + "instance.network.received_packets_count.value": "host.network.out.packets", + "instance.disk.read_bytes_count.value": "host.disk.read.bytes", + "instance.disk.write_bytes_count.value": "host.disk.write.bytes", + } + + for _, singleEvent := range groupedEvents { + if hostMetricName, ok := hostFieldTable[singleEvent.Key]; ok { + if hostMetricName == "host.cpu.pct" { + hostRootFields.Put(hostMetricName, singleEvent.Value.(float64)*100) + } else { + hostRootFields.Put(hostMetricName, singleEvent.Value) + } + } + } + return hostRootFields +} From cf7f96306f23bdaa6283c4c69d4366c8bd497b9d Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 19 Aug 2020 16:28:39 -0600 Subject: [PATCH 2/3] Change host.cpu.pct to be in between 0 to 1 and add cloud.region --- metricbeat/docs/fields.asciidoc | 242 +++++++----------- .../module/googlecloud/_meta/fields.yml | 24 -- .../googlecloud/compute/_meta/data.json | 49 ++-- .../googlecloud/compute/_meta/data_cpu.json | 23 +- .../googlecloud/compute/_meta/data_disk.json | 24 +- .../compute/_meta/data_firewall.json | 25 +- .../compute/_meta/data_memory.json | 85 ------ .../compute/_meta/data_network.json | 29 ++- .../compute/compute_integration_test.go | 1 + .../metricbeat/module/googlecloud/fields.go | 2 +- .../module/googlecloud/metrics/metricset.go | 6 +- .../timeseries_metadata_collector.go | 16 ++ .../timeseries_metadata_collector_test.go | 17 ++ 13 files changed, 191 insertions(+), 352 deletions(-) delete mode 100644 x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json create mode 100644 x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 81201e8b413..eda84195962 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -18472,70 +18472,6 @@ type: object -- - -*`host.cpu.pct`*:: -+ --- -Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 100%. - -type: scaled_float - --- - -*`host.network.in.bytes`*:: -+ --- -The number of bytes received on all network interfaces by the host in a given period of time. - -type: scaled_float - --- - -*`host.network.out.bytes`*:: -+ --- -The number of bytes sent out on all network interfaces by the host in a given period of time. - -type: scaled_float - --- - -*`host.network.in.packets`*:: -+ --- -The number of packets received on all network interfaces by the host in a given period of time. - -type: scaled_float - --- - -*`host.network.out.packets`*:: -+ --- -The number of packets sent out on all network interfaces by the host in a given period of time. - -type: scaled_float - --- - -*`host.disk.read.bytes`*:: -+ --- -The total number of bytes read successfully in a given period of time. - -type: scaled_float - --- - -*`host.disk.write.bytes`*:: -+ --- -The total number of bytes write successfully in a given period of time. - -type: scaled_float - --- - [float] === compute @@ -18544,7 +18480,7 @@ Google Cloud Compute metrics -*`host.compute.instance.firewall.dropped_bytes_count.value`*:: +*`googlecloud.compute.instance.firewall.dropped_bytes_count.value`*:: + -- Incoming bytes dropped by the firewall @@ -18553,7 +18489,7 @@ type: long -- -*`host.compute.instance.firewall.dropped_packets_count.value`*:: +*`googlecloud.compute.instance.firewall.dropped_packets_count.value`*:: + -- Incoming packets dropped by the firewall @@ -18563,7 +18499,7 @@ type: long -- -*`host.compute.instance.cpu.reserved_cores.value`*:: +*`googlecloud.compute.instance.cpu.reserved_cores.value`*:: + -- Number of cores reserved on the host of the instance @@ -18572,7 +18508,7 @@ type: double -- -*`host.compute.instance.cpu.utilization.value`*:: +*`googlecloud.compute.instance.cpu.utilization.value`*:: + -- The fraction of the allocated CPU that is currently in use on the instance @@ -18581,7 +18517,7 @@ type: double -- -*`host.compute.instance.cpu.usage_time.value`*:: +*`googlecloud.compute.instance.cpu.usage_time.value`*:: + -- Usage for all cores in seconds @@ -18591,7 +18527,7 @@ type: double -- -*`host.compute.instance.disk.read_bytes_count.value`*:: +*`googlecloud.compute.instance.disk.read_bytes_count.value`*:: + -- Count of bytes read from disk @@ -18600,7 +18536,7 @@ type: long -- -*`host.compute.instance.disk.read_ops_count.value`*:: +*`googlecloud.compute.instance.disk.read_ops_count.value`*:: + -- Count of disk read IO operations @@ -18609,7 +18545,7 @@ type: long -- -*`host.compute.instance.disk.write_bytes_count.value`*:: +*`googlecloud.compute.instance.disk.write_bytes_count.value`*:: + -- Count of bytes written to disk @@ -18618,7 +18554,7 @@ type: long -- -*`host.compute.instance.disk.write_ops_count.value`*:: +*`googlecloud.compute.instance.disk.write_ops_count.value`*:: + -- Count of disk write IO operations @@ -18627,7 +18563,7 @@ type: long -- -*`host.compute.instance.uptime.value`*:: +*`googlecloud.compute.instance.uptime.value`*:: + -- How long the VM has been running, in seconds @@ -18637,7 +18573,7 @@ type: long -- -*`host.compute.instance.network.received_bytes_count.value`*:: +*`googlecloud.compute.instance.network.received_bytes_count.value`*:: + -- Count of bytes received from the network @@ -18646,7 +18582,7 @@ type: long -- -*`host.compute.instance.network.received_packets_count.value`*:: +*`googlecloud.compute.instance.network.received_packets_count.value`*:: + -- Count of packets received from the network @@ -18655,7 +18591,7 @@ type: long -- -*`host.compute.instance.network.sent_bytes_count.value`*:: +*`googlecloud.compute.instance.network.sent_bytes_count.value`*:: + -- Count of bytes sent over the network @@ -18664,7 +18600,7 @@ type: long -- -*`host.compute.instance.network.sent_packets_count.value`*:: +*`googlecloud.compute.instance.network.sent_packets_count.value`*:: + -- Count of packets sent over the network @@ -18674,7 +18610,7 @@ type: long -- -*`host.compute.instance.memory.balloon.ram_size.value`*:: +*`googlecloud.compute.instance.memory.balloon.ram_size.value`*:: + -- The total amount of memory in the VM. This metric is only available for VMs that belong to the e2 family. @@ -18683,7 +18619,7 @@ type: long -- -*`host.compute.instance.memory.balloon.ram_used.value`*:: +*`googlecloud.compute.instance.memory.balloon.ram_used.value`*:: + -- Memory currently used in the VM. This metric is only available for VMs that belong to the e2 family. @@ -18692,7 +18628,7 @@ type: long -- -*`host.compute.instance.memory.balloon.swap_in_bytes_count.value`*:: +*`googlecloud.compute.instance.memory.balloon.swap_in_bytes_count.value`*:: + -- The amount of memory read into the guest from its own swap space. This metric is only available for VMs that belong to the e2 family. @@ -18701,7 +18637,7 @@ type: long -- -*`host.compute.instance.memory.balloon.swap_out_bytes_count.value`*:: +*`googlecloud.compute.instance.memory.balloon.swap_out_bytes_count.value`*:: + -- The amount of memory written from the guest to its own swap space. This metric is only available for VMs that belong to the e2 family. @@ -18722,7 +18658,7 @@ Google Cloud Load Balancing metrics Google Cloud Load Balancing metrics -*`host.loadbalancing.https.backend_request_bytes_count.value`*:: +*`googlecloud.loadbalancing.https.backend_request_bytes_count.value`*:: + -- The number of bytes sent as requests from HTTP/S load balancer to backends. @@ -18731,7 +18667,7 @@ type: long -- -*`host.loadbalancing.https.backend_request_count.value`*:: +*`googlecloud.loadbalancing.https.backend_request_count.value`*:: + -- The number of requests served by backends of HTTP/S load balancer. @@ -18740,7 +18676,7 @@ type: long -- -*`host.loadbalancing.https.request_bytes_count.value`*:: +*`googlecloud.loadbalancing.https.request_bytes_count.value`*:: + -- The number of bytes sent as requests from clients to HTTP/S load balancer. @@ -18749,7 +18685,7 @@ type: long -- -*`host.loadbalancing.https.request_count.value`*:: +*`googlecloud.loadbalancing.https.request_count.value`*:: + -- The number of requests served by HTTP/S load balancer. @@ -18758,7 +18694,7 @@ type: long -- -*`host.loadbalancing.https.response_bytes_count.value`*:: +*`googlecloud.loadbalancing.https.response_bytes_count.value`*:: + -- The number of bytes sent as responses from HTTP/S load balancer to clients. @@ -18773,7 +18709,7 @@ type: long Google Cloud Load Balancing metrics -*`host.loadbalancing.l3.internal.egress_bytes_count.value`*:: +*`googlecloud.loadbalancing.l3.internal.egress_bytes_count.value`*:: + -- The number of bytes sent from ILB backend to client (for TCP flows it's counting bytes on application stream only). @@ -18782,7 +18718,7 @@ type: long -- -*`host.loadbalancing.l3.internal.egress_packets_count.value`*:: +*`googlecloud.loadbalancing.l3.internal.egress_packets_count.value`*:: + -- The number of packets sent from ILB backend to client of the flow. @@ -18791,7 +18727,7 @@ type: long -- -*`host.loadbalancing.l3.internal.ingress_bytes_count.value`*:: +*`googlecloud.loadbalancing.l3.internal.ingress_bytes_count.value`*:: + -- The number of bytes sent from client to ILB backend (for TCP flows it's counting bytes on application stream only). @@ -18800,7 +18736,7 @@ type: long -- -*`host.loadbalancing.l3.internal.ingress_packets_count.value`*:: +*`googlecloud.loadbalancing.l3.internal.ingress_packets_count.value`*:: + -- The number of packets sent from client to ILB backend. @@ -18815,7 +18751,7 @@ type: long Google Cloud Load Balancing metrics -*`host.loadbalancing.tcp_ssl_proxy.closed_connections.value`*:: +*`googlecloud.loadbalancing.tcp_ssl_proxy.closed_connections.value`*:: + -- Number of connections that were terminated over TCP/SSL proxy. @@ -18824,7 +18760,7 @@ type: long -- -*`host.loadbalancing.tcp_ssl_proxy.egress_bytes_count.value`*:: +*`googlecloud.loadbalancing.tcp_ssl_proxy.egress_bytes_count.value`*:: + -- Number of bytes sent from VM to client using proxy. @@ -18833,7 +18769,7 @@ type: long -- -*`host.loadbalancing.tcp_ssl_proxy.ingress_bytes_count.value`*:: +*`googlecloud.loadbalancing.tcp_ssl_proxy.ingress_bytes_count.value`*:: + -- Number of bytes sent from client to VM using proxy. @@ -18842,7 +18778,7 @@ type: long -- -*`host.loadbalancing.tcp_ssl_proxy.new_connections.value`*:: +*`googlecloud.loadbalancing.tcp_ssl_proxy.new_connections.value`*:: + -- Number of connections that were created over TCP/SSL proxy. @@ -18851,7 +18787,7 @@ type: long -- -*`host.loadbalancing.tcp_ssl_proxy.open_connections.value`*:: +*`googlecloud.loadbalancing.tcp_ssl_proxy.open_connections.value`*:: + -- Current number of outstanding connections through the TCP/SSL proxy. @@ -18872,7 +18808,7 @@ Google Cloud PubSub metrics Suscription related metrics -*`host.pubsub.subscription.ack_message_count.value`*:: +*`googlecloud.pubsub.subscription.ack_message_count.value`*:: + -- Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type. @@ -18881,7 +18817,7 @@ type: long -- -*`host.pubsub.subscription.backlog_bytes.value`*:: +*`googlecloud.pubsub.subscription.backlog_bytes.value`*:: + -- Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription. @@ -18890,7 +18826,7 @@ type: long -- -*`host.pubsub.subscription.num_outstanding_messages.value`*:: +*`googlecloud.pubsub.subscription.num_outstanding_messages.value`*:: + -- Number of messages delivered to a subscription's push endpoint, but not yet acknowledged. @@ -18899,7 +18835,7 @@ type: long -- -*`host.pubsub.subscription.num_undelivered_messages.value`*:: +*`googlecloud.pubsub.subscription.num_undelivered_messages.value`*:: + -- Number of unacknowledged messages (a.k.a. backlog messages) in a subscription. @@ -18908,7 +18844,7 @@ type: long -- -*`host.pubsub.subscription.oldest_unacked_message_age.value`*:: +*`googlecloud.pubsub.subscription.oldest_unacked_message_age.value`*:: + -- Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription. @@ -18917,7 +18853,7 @@ type: long -- -*`host.pubsub.subscription.pull_ack_message_operation_count.value`*:: +*`googlecloud.pubsub.subscription.pull_ack_message_operation_count.value`*:: + -- Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. @@ -18926,7 +18862,7 @@ type: long -- -*`host.pubsub.subscription.pull_ack_request_count.value`*:: +*`googlecloud.pubsub.subscription.pull_ack_request_count.value`*:: + -- Cumulative count of acknowledge requests, grouped by result. @@ -18935,7 +18871,7 @@ type: long -- -*`host.pubsub.subscription.pull_message_operation_count.value`*:: +*`googlecloud.pubsub.subscription.pull_message_operation_count.value`*:: + -- Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. @@ -18944,7 +18880,7 @@ type: long -- -*`host.pubsub.subscription.pull_request_count.value`*:: +*`googlecloud.pubsub.subscription.pull_request_count.value`*:: + -- Cumulative count of pull requests, grouped by result. @@ -18953,7 +18889,7 @@ type: long -- -*`host.pubsub.subscription.push_request_count.value`*:: +*`googlecloud.pubsub.subscription.push_request_count.value`*:: + -- Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times. @@ -18962,7 +18898,7 @@ type: long -- -*`host.pubsub.subscription.push_request_latencies.value`*:: +*`googlecloud.pubsub.subscription.push_request_latencies.value`*:: + -- Distribution of push request latencies (in microseconds), grouped by result. @@ -18971,7 +18907,7 @@ type: long -- -*`host.pubsub.subscription.sent_message_count.value`*:: +*`googlecloud.pubsub.subscription.sent_message_count.value`*:: + -- Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type. @@ -18980,7 +18916,7 @@ type: long -- -*`host.pubsub.subscription.streaming_pull_ack_message_operation_count.value`*:: +*`googlecloud.pubsub.subscription.streaming_pull_ack_message_operation_count.value`*:: + -- Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. @@ -18989,7 +18925,7 @@ type: long -- -*`host.pubsub.subscription.streaming_pull_ack_request_count.value`*:: +*`googlecloud.pubsub.subscription.streaming_pull_ack_request_count.value`*:: + -- Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result. @@ -18998,7 +18934,7 @@ type: long -- -*`host.pubsub.subscription.streaming_pull_message_operation_count.value`*:: +*`googlecloud.pubsub.subscription.streaming_pull_message_operation_count.value`*:: + -- Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count @@ -19007,7 +18943,7 @@ type: long -- -*`host.pubsub.subscription.streaming_pull_response_count.value`*:: +*`googlecloud.pubsub.subscription.streaming_pull_response_count.value`*:: + -- Cumulative count of streaming pull responses, grouped by result. @@ -19016,7 +18952,7 @@ type: long -- -*`host.pubsub.subscription.dead_letter_message_count.value`*:: +*`googlecloud.pubsub.subscription.dead_letter_message_count.value`*:: + -- Cumulative count of messages published to dead letter topic, grouped by result. @@ -19025,7 +18961,7 @@ type: long -- -*`host.pubsub.subscription.mod_ack_deadline_message_count.value`*:: +*`googlecloud.pubsub.subscription.mod_ack_deadline_message_count.value`*:: + -- Cumulative count of messages whose deadline was updated by ModifyAckDeadline requests, grouped by delivery type. @@ -19034,7 +18970,7 @@ type: long -- -*`host.pubsub.subscription.mod_ack_deadline_message_operation_count.value`*:: +*`googlecloud.pubsub.subscription.mod_ack_deadline_message_operation_count.value`*:: + -- Cumulative count of ModifyAckDeadline message operations, grouped by result. @@ -19043,7 +18979,7 @@ type: long -- -*`host.pubsub.subscription.mod_ack_deadline_request_count.value`*:: +*`googlecloud.pubsub.subscription.mod_ack_deadline_request_count.value`*:: + -- Cumulative count of ModifyAckDeadline requests, grouped by result. @@ -19052,7 +18988,7 @@ type: long -- -*`host.pubsub.subscription.oldest_retained_acked_message_age.value`*:: +*`googlecloud.pubsub.subscription.oldest_retained_acked_message_age.value`*:: + -- Age (in seconds) of the oldest acknowledged message retained in a subscription. @@ -19061,7 +18997,7 @@ type: long -- -*`host.pubsub.subscription.oldest_retained_acked_message_age_by_region.value`*:: +*`googlecloud.pubsub.subscription.oldest_retained_acked_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest acknowledged message retained in a subscription, broken down by Cloud region. @@ -19070,7 +19006,7 @@ type: long -- -*`host.pubsub.subscription.oldest_unacked_message_age_by_region.value`*:: +*`googlecloud.pubsub.subscription.oldest_unacked_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest unacknowledged message in a subscription, broken down by Cloud region. @@ -19079,7 +19015,7 @@ type: long -- -*`host.pubsub.subscription.retained_acked_bytes.value`*:: +*`googlecloud.pubsub.subscription.retained_acked_bytes.value`*:: + -- Total byte size of the acknowledged messages retained in a subscription. @@ -19088,7 +19024,7 @@ type: long -- -*`host.pubsub.subscription.retained_acked_bytes_by_region.value`*:: +*`googlecloud.pubsub.subscription.retained_acked_bytes_by_region.value`*:: + -- Total byte size of the acknowledged messages retained in a subscription, broken down by Cloud region. @@ -19097,7 +19033,7 @@ type: long -- -*`host.pubsub.subscription.seek_request_count.value`*:: +*`googlecloud.pubsub.subscription.seek_request_count.value`*:: + -- Cumulative count of seek attempts, grouped by result. @@ -19106,7 +19042,7 @@ type: long -- -*`host.pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation_count.value`*:: +*`googlecloud.pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation_count.value`*:: + -- Cumulative count of StreamingPull ModifyAckDeadline operations, grouped by result. @@ -19115,7 +19051,7 @@ type: long -- -*`host.pubsub.subscription.streaming_pull_mod_ack_deadline_request_count.value`*:: +*`googlecloud.pubsub.subscription.streaming_pull_mod_ack_deadline_request_count.value`*:: + -- Cumulative count of streaming pull requests with non-empty ModifyAckDeadline fields, grouped by result. @@ -19124,7 +19060,7 @@ type: long -- -*`host.pubsub.subscription.byte_cost.value`*:: +*`googlecloud.pubsub.subscription.byte_cost.value`*:: + -- Cumulative cost of operations, measured in bytes. This is used to measure quota utilization. @@ -19133,7 +19069,7 @@ type: long -- -*`host.pubsub.subscription.config_updates_count.value`*:: +*`googlecloud.pubsub.subscription.config_updates_count.value`*:: + -- Cumulative count of configuration changes for each subscription, grouped by operation type and result. @@ -19142,7 +19078,7 @@ type: long -- -*`host.pubsub.subscription.unacked_bytes_by_region.value`*:: +*`googlecloud.pubsub.subscription.unacked_bytes_by_region.value`*:: + -- Total byte size of the unacknowledged messages in a subscription, broken down by Cloud region. @@ -19157,7 +19093,7 @@ type: long Topic related metrics -*`host.pubsub.topic.streaming_pull_response_count.value`*:: +*`googlecloud.pubsub.topic.streaming_pull_response_count.value`*:: + -- Cumulative count of streaming pull responses, grouped by result. @@ -19166,7 +19102,7 @@ type: long -- -*`host.pubsub.topic.send_message_operation_count.value`*:: +*`googlecloud.pubsub.topic.send_message_operation_count.value`*:: + -- Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count. @@ -19175,7 +19111,7 @@ type: long -- -*`host.pubsub.topic.send_request_count.value`*:: +*`googlecloud.pubsub.topic.send_request_count.value`*:: + -- Cumulative count of publish requests, grouped by result. @@ -19184,7 +19120,7 @@ type: long -- -*`host.pubsub.topic.oldest_retained_acked_message_age_by_region.value`*:: +*`googlecloud.pubsub.topic.oldest_retained_acked_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest acknowledged message retained in a topic, broken down by Cloud region. @@ -19193,7 +19129,7 @@ type: long -- -*`host.pubsub.topic.oldest_unacked_message_age_by_region.value`*:: +*`googlecloud.pubsub.topic.oldest_unacked_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest unacknowledged message in a topic, broken down by Cloud region. @@ -19202,7 +19138,7 @@ type: long -- -*`host.pubsub.topic.retained_acked_bytes_by_region.value`*:: +*`googlecloud.pubsub.topic.retained_acked_bytes_by_region.value`*:: + -- Total byte size of the acknowledged messages retained in a topic, broken down by Cloud region. @@ -19211,7 +19147,7 @@ type: long -- -*`host.pubsub.topic.byte_cost.value`*:: +*`googlecloud.pubsub.topic.byte_cost.value`*:: + -- Cost of operations, measured in bytes. This is used to measure utilization for quotas. @@ -19220,7 +19156,7 @@ type: long -- -*`host.pubsub.topic.config_updates_count.value`*:: +*`googlecloud.pubsub.topic.config_updates_count.value`*:: + -- Cumulative count of configuration changes, grouped by operation type and result. @@ -19229,7 +19165,7 @@ type: long -- -*`host.pubsub.topic.message_sizes.value`*:: +*`googlecloud.pubsub.topic.message_sizes.value`*:: + -- Distribution of publish message sizes (in bytes) @@ -19238,7 +19174,7 @@ type: long -- -*`host.pubsub.topic.unacked_bytes_by_region.value`*:: +*`googlecloud.pubsub.topic.unacked_bytes_by_region.value`*:: + -- Total byte size of the unacknowledged messages in a topic, broken down by Cloud region. @@ -19253,7 +19189,7 @@ type: long Snapshot related metrics -*`host.pubsub.snapshot.oldest_message_age.value`*:: +*`googlecloud.pubsub.snapshot.oldest_message_age.value`*:: + -- Age (in seconds) of the oldest message retained in a snapshot. @@ -19262,7 +19198,7 @@ type: long -- -*`host.pubsub.snapshot.oldest_message_age_by_region.value`*:: +*`googlecloud.pubsub.snapshot.oldest_message_age_by_region.value`*:: + -- Age (in seconds) of the oldest message retained in a snapshot, broken down by Cloud region. @@ -19271,7 +19207,7 @@ type: long -- -*`host.pubsub.snapshot.backlog_bytes.value`*:: +*`googlecloud.pubsub.snapshot.backlog_bytes.value`*:: + -- Total byte size of the messages retained in a snapshot. @@ -19280,7 +19216,7 @@ type: long -- -*`host.pubsub.snapshot.backlog_bytes_by_region.value`*:: +*`googlecloud.pubsub.snapshot.backlog_bytes_by_region.value`*:: + -- Total byte size of the messages retained in a snapshot, broken down by Cloud region. @@ -19289,7 +19225,7 @@ type: long -- -*`host.pubsub.snapshot.num_messages.value`*:: +*`googlecloud.pubsub.snapshot.num_messages.value`*:: + -- Number of messages retained in a snapshot. @@ -19298,7 +19234,7 @@ type: long -- -*`host.pubsub.snapshot.num_messages_by_region.value`*:: +*`googlecloud.pubsub.snapshot.num_messages_by_region.value`*:: + -- Number of messages retained in a snapshot, broken down by Cloud region. @@ -19307,7 +19243,7 @@ type: long -- -*`host.pubsub.snapshot.config_updates_count.value`*:: +*`googlecloud.pubsub.snapshot.config_updates_count.value`*:: + -- Cumulative count of configuration changes, grouped by operation type and result. @@ -19323,7 +19259,7 @@ Google Cloud Storage metrics -*`host.storage.api.request_count.value`*:: +*`googlecloud.storage.api.request_count.value`*:: + -- Delta count of API calls, grouped by the API method name and response code. @@ -19333,7 +19269,7 @@ type: long -- -*`host.storage.authz.acl_based_object_access_count.value`*:: +*`googlecloud.storage.authz.acl_based_object_access_count.value`*:: + -- Delta count of requests that result in an object being granted access solely due to object ACLs. @@ -19342,7 +19278,7 @@ type: long -- -*`host.storage.authz.acl_operations_count.value`*:: +*`googlecloud.storage.authz.acl_operations_count.value`*:: + -- Usage of ACL operations broken down by type. @@ -19351,7 +19287,7 @@ type: long -- -*`host.storage.authz.object_specific_acl_mutation_count.value`*:: +*`googlecloud.storage.authz.object_specific_acl_mutation_count.value`*:: + -- Delta count of changes made to object specific ACLs. @@ -19361,7 +19297,7 @@ type: long -- -*`host.storage.network.received_bytes_count.value`*:: +*`googlecloud.storage.network.received_bytes_count.value`*:: + -- Delta count of bytes received over the network, grouped by the API method name and response code. @@ -19370,7 +19306,7 @@ type: long -- -*`host.storage.network.sent_bytes_count.value`*:: +*`googlecloud.storage.network.sent_bytes_count.value`*:: + -- Delta count of bytes sent over the network, grouped by the API method name and response code. @@ -19380,7 +19316,7 @@ type: long -- -*`host.storage.storage.object_count.value`*:: +*`googlecloud.storage.storage.object_count.value`*:: + -- Total number of objects per bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. @@ -19389,7 +19325,7 @@ type: long -- -*`host.storage.storage.total_byte_seconds.value`*:: +*`googlecloud.storage.storage.total_byte_seconds.value`*:: + -- Delta count of bytes received over the network, grouped by the API method name and response code. @@ -19398,7 +19334,7 @@ type: long -- -*`host.storage.storage.total_bytes.value`*:: +*`googlecloud.storage.storage.total_bytes.value`*:: + -- Total size of all objects in the bucket, grouped by storage class. This value is measured once per day, and the value is repeated at each sampling interval throughout the day. diff --git a/x-pack/metricbeat/module/googlecloud/_meta/fields.yml b/x-pack/metricbeat/module/googlecloud/_meta/fields.yml index 6619b23d84b..84e340de576 100644 --- a/x-pack/metricbeat/module/googlecloud/_meta/fields.yml +++ b/x-pack/metricbeat/module/googlecloud/_meta/fields.yml @@ -30,27 +30,3 @@ object_type_mapping_type: "*" description: > Metrics that returned from Google Cloud API query. - - name: host - type: group - fields: - - name: cpu.pct - type: scaled_float - description: Percent CPU used. This value is normalized by the number of CPU cores and it ranges from 0 to 100%. - - name: network.in.bytes - type: scaled_float - description: The number of bytes received on all network interfaces by the host in a given period of time. - - name: network.out.bytes - type: scaled_float - description: The number of bytes sent out on all network interfaces by the host in a given period of time. - - name: network.in.packets - type: scaled_float - description: The number of packets received on all network interfaces by the host in a given period of time. - - name: network.out.packets - type: scaled_float - description: The number of packets sent out on all network interfaces by the host in a given period of time. - - name: disk.read.bytes - type: scaled_float - description: The total number of bytes read successfully in a given period of time. - - name: disk.write.bytes - type: scaled_float - description: The total number of bytes write successfully in a given period of time. diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json index 5ceebb5e516..36c46744b50 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "17557085450298074", - "name": "gke-chrismark-leaderelec-default-pool-896893cb-t17x" + "id": "2438671519308240892", + "name": "heartbeat-mumbai" }, "machine": { - "type": "e2-medium" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-c", + "cloud.availability_zone": "asia-south1-c", + "cloud.region": "asia-south1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -23,57 +24,37 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 432 + "value": 435 }, "dropped_packets_count": { - "value": 6 + "value": 7 } }, "instance": { "cpu": { "reserved_cores": { - "value": 1 + "value": 0.2 }, "usage_time": { - "value": 9.515682164234931 + "value": 0.8653667340986431 }, "utilization": { - "value": 0.15859470273724885 - } - }, - "memory": { - "balloon": { - "ram_size": { - "value": 4140863488 - }, - "ram_used": { - "value": 2765357056 - }, - "swap_in_bytes_count": { - "value": 0 - }, - "swap_out_bytes_count": { - "value": 0 - } + "value": 0.07211389450822024 } }, "uptime": { - "value": 60.000000000007276 + "value": 60 } } }, - "labels": { - "user": { - "goog-gke-node": "" - } - } + "labels": {} }, "host": { "cpu": { - "pct": 15.859470273724884 + "pct": 0.07211389450822024 }, - "id": "17557085450298074", - "name": "gke-chrismark-leaderelec-default-pool-896893cb-t17x" + "id": "2438671519308240892", + "name": "heartbeat-mumbai" }, "metricset": { "name": "compute", diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json index 93fb745c1a9..b43490d1915 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_cpu.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "8825965883407652424", - "name": "heartbeat-los-angeles" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "machine": { - "type": "n1-standard-8" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-west2-a", + "cloud.availability_zone": "asia-northeast1-b", + "cloud.region": "asia-northeast1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -23,10 +24,10 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 586 + "value": 422 }, "dropped_packets_count": { - "value": 6 + "value": 7 } }, "instance": { @@ -35,10 +36,10 @@ "value": 0.2 }, "usage_time": { - "value": 1.0218381873273756 + "value": 0.8921228461549617 }, "utilization": { - "value": 0.0851531822772813 + "value": 0.07434357051291347 } }, "uptime": { @@ -50,10 +51,10 @@ }, "host": { "cpu": { - "pct": 8.51531822772813 + "pct": 0.07434357051291347 }, - "id": "8825965883407652424", - "name": "heartbeat-los-angeles" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "metricset": { "name": "compute", diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json index 197c29a8cee..7a8f923569e 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_disk.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "2572129596446793084", - "name": "johns-test-node" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "machine": { - "type": "e2-medium" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "asia-northeast1-b", + "cloud.region": "asia-northeast1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -30,22 +31,19 @@ "value": 0 }, "write_bytes_count": { - "value": 163242 + "value": 0 }, "write_ops_count": { - "value": 27 + "value": 0 } } } }, "labels": { "metrics": { - "device_name": "johns-test-node", + "device_name": "disk-1", "device_type": "permanent", "storage_type": "pd-standard" - }, - "user": { - "goog-gke-node": "" } } }, @@ -55,11 +53,11 @@ "bytes": 0 }, "write": { - "bytes": 163242 + "bytes": 0 } }, - "id": "2572129596446793084", - "name": "johns-test-node" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "metricset": { "name": "compute", diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json index 6e1a0a73323..b43490d1915 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_firewall.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "2572129596446793084", - "name": "johns-test-node" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "machine": { - "type": "n1-standard-8" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-central1-a", + "cloud.availability_zone": "asia-northeast1-b", + "cloud.region": "asia-northeast1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -23,22 +24,22 @@ "compute": { "firewall": { "dropped_bytes_count": { - "value": 1613 + "value": 422 }, "dropped_packets_count": { - "value": 8 + "value": 7 } }, "instance": { "cpu": { "reserved_cores": { - "value": 8 + "value": 0.2 }, "usage_time": { - "value": 0.10854628651577514 + "value": 0.8921228461549617 }, "utilization": { - "value": 0.00022613809690786487 + "value": 0.07434357051291347 } }, "uptime": { @@ -50,10 +51,10 @@ }, "host": { "cpu": { - "pct": 0.022613809690786486 + "pct": 0.07434357051291347 }, - "id": "2572129596446793084", - "name": "johns-test-node" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "metricset": { "name": "compute", diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json deleted file mode 100644 index 1fea94f0f9d..00000000000 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_memory.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "@timestamp": "2017-10-12T08:05:34.853Z", - "cloud": { - "account": { - "id": "elastic-observability" - }, - "instance": { - "id": "7920542032463815386", - "name": "gke-chrismark-leaderelec-default-pool-896893cb-nm8x" - }, - "machine": { - "type": "e2-medium" - }, - "provider": "googlecloud" - }, - "cloud.availability_zone": "us-central1-c", - "event": { - "dataset": "googlecloud.compute", - "duration": 115000, - "module": "googlecloud" - }, - "googlecloud": { - "compute": { - "firewall": { - "dropped_bytes_count": { - "value": 211 - }, - "dropped_packets_count": { - "value": 3 - } - }, - "instance": { - "cpu": { - "reserved_cores": { - "value": 1 - }, - "usage_time": { - "value": 8.026239710991831 - }, - "utilization": { - "value": 0.13377066184986386 - } - }, - "memory": { - "balloon": { - "ram_size": { - "value": 4140863488 - }, - "ram_used": { - "value": 2498785280 - }, - "swap_in_bytes_count": { - "value": 0 - }, - "swap_out_bytes_count": { - "value": 0 - } - } - }, - "uptime": { - "value": 60 - } - } - }, - "labels": { - "user": { - "goog-gke-node": "" - } - } - }, - "host": { - "cpu": { - "pct": 13.377066184986386 - }, - "id": "7920542032463815386", - "name": "gke-chrismark-leaderelec-default-pool-896893cb-nm8x" - }, - "metricset": { - "name": "compute", - "period": 10000 - }, - "service": { - "type": "googlecloud" - } -} \ No newline at end of file diff --git a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json index 7acf27dcb7c..22257a87521 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json +++ b/x-pack/metricbeat/module/googlecloud/compute/_meta/data_network.json @@ -5,15 +5,16 @@ "id": "elastic-observability" }, "instance": { - "id": "8825965883407652424", - "name": "heartbeat-los-angeles" + "id": "7925831082803123728", + "name": "heartbeat-tokyo" }, "machine": { - "type": "n1-standard-8" + "type": "f1-micro" }, "provider": "googlecloud" }, - "cloud.availability_zone": "us-west2-a", + "cloud.availability_zone": "asia-northeast1-b", + "cloud.region": "asia-northeast1", "event": { "dataset": "googlecloud.compute", "duration": 115000, @@ -24,16 +25,16 @@ "instance": { "network": { "received_bytes_count": { - "value": 2817591 + "value": 2866053 }, "received_packets_count": { - "value": 2528 + "value": 2449 }, "sent_bytes_count": { - "value": 121736 + "value": 119338 }, "sent_packets_count": { - "value": 754 + "value": 695 } } } @@ -45,16 +46,16 @@ } }, "host": { - "id": "8825965883407652424", - "name": "heartbeat-los-angeles", + "id": "7925831082803123728", + "name": "heartbeat-tokyo", "network": { "in": { - "bytes": 121736, - "packets": 754 + "bytes": 119338, + "packets": 695 }, "out": { - "bytes": 2817591, - "packets": 2528 + "bytes": 2866053, + "packets": 2449 } } }, diff --git a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go index 30c7061760e..a91c26948d0 100644 --- a/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go +++ b/x-pack/metricbeat/module/googlecloud/compute/compute_integration_test.go @@ -2,6 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. +// +build integration // +build googlecloud package compute diff --git a/x-pack/metricbeat/module/googlecloud/fields.go b/x-pack/metricbeat/module/googlecloud/fields.go index 015b1e85bc6..ad15833fa63 100644 --- a/x-pack/metricbeat/module/googlecloud/fields.go +++ b/x-pack/metricbeat/module/googlecloud/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGooglecloud returns asset data. // This is the base64 encoded gzipped contents of module/googlecloud. func AssetGooglecloud() string { - return "eJzcXN2O27gVvs9THCxQZFNMnG17FxQLzM6iuwEy7QAzm1uBIo9t1hSp8Gdc5+kLkpIs2ZIs25Ims8nV2Bb5fef/HEp6DxvcfYSVUiuBVCjH3gBYbgV+hLe/hU/hzn8MD4LYpdLZ2zcAGgUSgx8hRUveADA0VPPcciU/ws9vAAB+u3uATDEn8A3AkqNg5mP44j1IkuHhlv6f3eX+c61cXnxSv65+rSApClN9XF6q0v8itbWPW3CV/wpuYX/IlORWaS5XkKHVnJrjLQ6x1PE4g3rx18ZXnZj8v/hhEn+xwd1Wada6cIaWMGLJVIt7qpOsbXbGYjba0uWyP1SY4/8fTttAY12mXBossuXbJCN5zuWq+OkPjcV7LOm+sBi7JhY0WqclMlhqlUHDgW4fPsFXh3q3aPjBWhl7tgPQ3C3yBs14raFEIEuWQpFOP3hATVFauHv4wxsuW8DTmht4JsIhcANS6YwI/g0ZpDuwawTpshQ1qGW4hiqNBohkwC1oIldoItufwCr4208//WVxhFei3Sq9WXC5SHcWj113CPCnBpSwDmikyJ+RgZJAhCg3Ai4t6iWhaEoSXs7AJRBY8WeUkKPmivmVLM+wG7JydmTMxktfOTsdZi4XOaEbtGOALlaaSdTj455O3IybzUIjYVcaiFWWiBbTJgyMoxSNWTohdmdD22pucRJsYeWLwVGV5c7Ww/BhQdHEWo+IRyAbMfYurlymtoHpm0tjiaTYmqwON+9arL7gkmvcEiGOftC3aN/C9cWZVnmOLAmaSKhy0i5C8G69qtxRKLnq+EFDnJ8kVZkvhKKii81KD+khdgywcMDpIJYePhRkLXtOohmNBvUzsiSkyAGMj8qRTs7/rhwwpt9yKx/UqtDlHW6NXebcxOosF/wb8auPCtTHjKUm1P9VAiJCKEosslA/hDqJG6BOa5Q2hg5nsKQyDL4hK0xCeBkT/R9+WVgqHTJFFDWXYJAqyUynQfloO5FFkakc/c6vd5BtQhnXQeYAlMonhOQhRESf/gMqRx3s9Fj+dVQhJc0jK7+VRenL3ZPCirCml1bMyP3iqpwn73WcXhgNCL+rbfhd8Nsv97AmBlJECdpJyeXqZojzFPXYRP4Ta9a5fKiokIMfhc6pk1sLxqnyZYXyqJI/C6evo+eRY6zYn1Gfh202+Q3Dtx+zZErvFqnPgkpOY+YkSwz/NiQXDiW9r/1JVvKPTLxPR38vJgex2PYpXUmxA/JMuCCpiHn0y30xGUkxRgoVLsa/w5JkXOwWJ4mFGcV4xO4jiX314defk5PZkjzhciJX8no70lhIpVwWOFcOjY2+z60BtZUBE5icUJyJv3JTxZJWAZQZuwp4UQZWTS6BamStCEuJIJLyBo+xOt/PijD4pdzgzAZ4bW1+mKC7o9IVILqA1MGkPspKlmj86pU0yEyGlyydAzni82LYsRhn/v709PDhMSgOouZ8wFclPnNs5V0MpsFeoS06wXRXQfNft8HvhvzSwqaCo7TGy/cy4LPJ+Fx4JlfSDGtMxhJs3PKEGRcSX7QGBPGPRRiPSnI4RZk5LJSAcKXRmAFS7JHhMAkGoX36/EvpTHtZwY8+9D/dPcBSqK0Bbt8aCFj28zIlgeS54DS0YGCsRpKFDPLu0EgOqA2pXi8j1yhbe+gV0xpPrgsrl/PqoUBmVQPyNHoouc2piFZ+7S5paZ4YI5Jcq//tvpdcTYUyYd4pJYaRX/fUc3hwq886q3VjubVFjWBRZ1yGeWJowp7uHj48Pn6GIJnuUDw4hFyG9dB0v9zXXMuZMK/uBzjcucZBuDe+L/fDEErczqxrqvFsRasc5cgw72K3WPNl5ayxRDIvtSZ0rdxqHSJpB973nUV/uBfn2BOnOwUrRZa71Lh0grbkwaWPLj2zHTEurda7LNI9uuoPTyWY0GURjtBNkqEJxw1jueSdy5wglj9jTFuxUQ17GL+hVFuBbBXrzdv931U5ehMlEH/AUPBn1LsAoL8pEWoVo8sYJWgYDvnVwPBvWFYPTjYIVLR+JIvNgixKGNUX7+LBcV3lPfHHZUnN8UrFjBuIKsiFYDHUSE2Ibw3kzqwBJcsVl/YGUmdBKgs7tA0V9pNxstpkGjKTqkMJ5luvsMeeQEJWY5wt3K4QftwfH7wrDSxu2kGsi9dZtHInRFL3++pIZdIIUONT0dkf5jRcXqNxwi7gX0oDAYZLLnl54Np2qcFaPP6wD8gNgXzIFAu0GRImuMRO/qclN3ZXfkperYGxkFI/2nl17Ld81cqdQ7FBSBdq1KxngmjWQKzFLG+HCH9IwTcYuJibELjCNWGOpIFnucAMpY3tKVNoQvZIiaXrcDtxFZEX8KgACV2XIonzaKqkJVz6ThcbFyxC01nfTHtziD0xaq20txlV3axVvxYoCX0zEg2ZE5bnAsM9XD3DzobMfaUlKR8lif3KjdU8daXtB0qlDKqNQo7IONWqTBRnWUw4OJy1vgv9V7o7cFmrSnf1ibuYzV1S5sVJhy+OXiiNPZYAHrwb/wmSWotE5wgx1bbNeAhbbtcglXzvY8+uIWDOzguXB8zmtZMDfnMZxz+pYvjzRSYyVI7VzH9W8yiG/mcZgGecCLQW9bxBMHep4GYdWyuPAiIKsCrn9CwOndqbhch2rQxCuTdsiQGXszBvSHdwrxhf7m7p5tfyB9d08IND2TScj8kM89nhfOYIqwNVcgp70fpq9EUYBh5zd8Ct/W8J6II2vptLku4Sjau+W4VfitUNpFpt0JfQW7kvqwq0F0wu5uTaMcMYi+SBQice/LXPmS6xxzbco6plJAYXqsUgzlRBIm56m9TBheHLJp5mR3EcwC9MP6dIfkdF/jHneGxxFl9v7wlVZnQu8cmTuhYyJMbp6DMx7MR76riJd1taVf4EvjplSeNplO7DZiWXfJXE+mq8A9I2vcStXCQEdF085ap0nMQ0o0BNB5UQwv7hIdlTailz0GxhrutI4Jr4Vt2k4Mv3y47snvylVx7WvdpGzKBks4+iQwv2aqcyU9xg2SenadqE76y0LtrvP2lNfQ2777kovYbXqFXBdaVArQgIqTaUBj0nDy9dD1yb+UsP8eqe5sCkGeHDPsF3giLevZKS5FzrrjKEJLlZq8Ox7dA7iIqrr6xIiqA462CoY2pSEDoZwF8ocPejvjS6zXPTU9ew4KTIG/jm8LQTSC8Us3TZxPdinSvYOqJR5ToY24WifG1Jbd96KU1WU7zT5TGufOY9pCTngwN//6MnQ5qMoc8D/IrCkr0Gbh8+ASXhDpGa1L2f+m8ytGvFAopS9KG1BKoYtuc84uz625W86/e+iiQlBllSvCaNhPf9jGaUB9Kopl/Fm9S8oQWfksVr2iBF32evNJE+I0c0YJRAsQPm0BeTxS9v7z73VI6e2L5IHY1PfHuI1+vd51oRfBgI+k/5ClGbHClfcpp4rJmz4w4EDiRfDrgywupCLDG0SLP/5Q2Xm9xZL2y4mO/hu+MOHuW/1hvhkrcmXEmm9Z0EY4WV4+B+nZYLGx/taciDd5TF5Q3kqCF1dIO2IYiCDVBBjDl8+2HVrypJMazAyO4mCMoLsPqdxjw+mUJsMRMmWS58eAoPTj4TUT4KopwNlzLS88xKeNVCsJOkqI9ftdXv6YxXdZdlLBGi0nDx2obvTMn/DwAA//8MI+Hs" + return "eJzcXN1u2zgWvu9THPSm7SJ1gd27YjFAxsXOFGh2AyTtrUBRxzbXFKnyJ1736RckJVmyJVmWJSWZzs3EtsjvO//nUNJH2OL+M6ylXHOkXNrkDYBhhuNnePeH/xSW7mO458SspErfvQFQyJFo/AwxGvIGIEFNFcsMk+Iz/PYGAOCP5T2kMrEc3wCsGPJEf/ZffARBUjze0v0z+8x9rqTN8k+q11Wv5SRGrsuPi0tl/F+kpvJxA67iX87N7w+pFMxIxcQaUjSKUX26xTGWKh6rUS3+VvuqFZP7Fz6Mwi+2uN9JlTQunKIhCTFkqsUd1UnW1nttMB1t6WLZtyXm8N/b8zZQWzeRNvYW2fBtlJIsY2Kd//RtbfEOS7rLLcZsiAGFxiqBCayUTKHmQLf3X+GnRbVfnNCiMs2sqeI69rA6yaqPnKCrbboMKxe67mnPTGhDBMVG7R1v3rZYdcEVU7gjnJ/8oGvRroWriydKZhkmUbw3qCMqrTCLJ8LtMfz6jlyKdcsPauL8KqhMXWTwyxebQbwHs8EuYqcAM0K3aCaEmG/QG2Rpf5mdRDMKNaonTCIqFeoejE/8s5Xzv20aowK5Ar92uRVI4TlvpDbuW/f/LeZcx2oN4+wXcauPCvTRKUAR6v4qABHOJSUGE1jefw+Bg2mgVikUhu+BCZdSCir94GuyxsiwFEdF/90tCyupHOZc1EyARipFolsNKmF6O5FFkakcfenWcxoKju52ClG8hcwRKJlNCMlBCIi+/gdkhsrb6an8q6h2ihmcR1ZuK4MCjDwvrABremn5fc6Iq3SerNNxOmHUIPwpd/533m9/3MGGaIgRBSgrBBPrmz7OI9DspJrKfyiyp8mS5YkPhd2CHzmZtHNrwDhVvixRFvlyGE6NwswjR7cTyCdUl2GbTX798B36jlSq/SJ2WVCKacycpJFmv/rkwr6kXR430hAOJC34BybOp4O/L+Bxw3RebLuULgXfA3kijJOYhzz64y5vFWIMkUL6i/HvsCIp45XuoI2Y1ZiMSOwukDhUH279OTnpHckiJiZyJae3E435VMpEjnNtUZvg+8xokDvhMYHOCMWZ+Es7VSxpFECRscuAF2Rg5OQSKGc4kiQx4URQVuMxVuf7TZIEfi82uLAB3hiTHSfo9qh0BYg2IFUwsYuyIokU/nRK6mUm/UsWZx6ibKYq+Ya4vOh31MFM/nx8vP/04BUHQXMu4MsCnz618jYG02Av0eadYLwvobmvm+C3Q35uYVPOUBjt5DsM+GwyvhSezqTQ/RqTsQQbtjxjxrnEF40Bgf9jwYRBJcjxFGXmsFAAwrVCrXtIsUOG/STohfb12++FMx1kBe9d6H9c3sOKy50GZt5p8FgO8zIpgGQZZ9S3YKCNQpL6DPLh2EiOqPWpXoeRq5WtHfTyaY0j14aViXn1kCMzsgZ5Gj0U3OZURCO/Zpc0NIu05lGm5P/2LyVXUy61n3cKgX7k1z717B/cqrPOct1Qbu1QIRhUKRN+nuibsMfl/aeHh2/gJdMeinuHkGFYj033x13Ftaz28+pugP2daxyEB+P7cdcPocDdzLqmCi9WtMxQjAxzGbrFii9La7QhInFSq0NX0q43PpK24P3YWvT7w+lTT5zuFKwQWWZjbeMJ2pJ7Gz/Y+MJ2RNu4XG9YpHuw5R+OijehYRGO0G2UovbHDWO55NKmlhPDnjCkrdCo+j2021DIHcdkHerN28PfZTl6EyQQfpAgZ0+o9h5Ad1PC5TpElzFKUD8ccquBZr+wqB6sqBEoab0ni+2CLAoY5RcfgAkgNZV3xB+bRhXHKxQzbiAqIeeCRV8j1SG+05BZvQEUSSaZMDcQWwNCGtijqamwm4wV5SbTkJlUHZInrvXyexwIRGQ9xtnC7Rrh/eH44ENhYGHTFmJtvC6ilVnOo6rfl0cqk0aACp+SzuEwp+byCrXlZgH/kgoIJLhighUHrk2XaqzE40+HgFwTyKdUJp52giThTGAr//OSG7srPyevxsCYS6kb7bw6dlu+auXOoVgvpIEa1ZuZIOoNEGMwzZohwnfB2RY9F33jA5e/xs+RFLA045iiMKE9TSRqnz1iYujG319XRuQFPEhAQjeFSMI8mkphCBOu08XaBQvfdFY3U84cQk+MSknlbMYltDV7QlG7FijxfTMSBanlhmUcwbAUO4adNZm7SktQNkoS+8K0USy2he17SoUMyo18jkgZVbJIFBdZjD84nLW+8/1XvD9yWSMLd3WJO5/NDSnzwqTDFUfPlMYeCgD3zo3/AkmtQaJzhJhy23o8hB0zGxBSfHSxZ18TMEsuC5dHzOa1kyN+cxnHP6lM8LdBJtJXjuXMf1bzyIf+FxmAYxxxNAbVvEEwszFnehNaK4cCAgowMmP0Ig6t2puFyG4jNUKxN+yIBpslft4Q7+FOJmy1v6XbL8UPrunge4eyaTifkunns/35zBFWe6rkHPa89VXoijD0PObugBv73wLQgDa+nUsU7yOF665bhZ+L1Q3ESm7RldA7cSircrQDJhdzcm2ZYYxF8kihEw/+mudMQ+yxCfeoahmJwUC1aMSZKkjEbWeT2rswfN7EU+8oTgP4wPRzjuQLKvJPOYdji4v4OnuPqNSjcwlPnlS1kCLRVgWfCWEn3FPHdLjb0sjiJ/DTSkNqT6O0HzZLsWLrKNRX4x2QNuklbGUDIaAbIlwoWEkVJjH1KFDRQSkEvz8QkZxVS5GDZgtzbUcC18S38iYFV74PO7J7dJdeeVj3ahsxjSKZfRTtW7BXO5WZ4gbLLjlN0ya8sNI6b7//ojX1NexeclF6Da9Rq4LrSoFKEeBTrS8NOk4enrseuDbzFx7i1D3NgUk9wvt9vO94RXx4JSXJpdZdZghBMr2Rx2PbvncQ5VdfWZHkQXHWwVDL1CQndDaAP1Pg7kY9NLrNc9NT27DgrMhr+ObwtDNIB4pZ2HTie7EuFWwV0ahy7Y1toChfW1I7tF5SkfUU73R5CCtfeA8pyVjvwN/96EmfJqPv8wBfkBty0MDt/VegxN8hUpG681P3TYpmIxOPohC9by2BygSbcx6xZvPrSt7Ve195FBONSZS/N4hQino8ozySRjn9yl8t5AzN+5TI31sEMbo+e62IcBk5oAEtOfI9JBZdMZn/8nb5raNydMQORepofMLbQ5xel98qRfBxIOg+5ctFrTOkbMVo5LCm1ow7EDiSfDHgSklSFWKBoUGa3S9vGG5yF72wYTDfo9c1HD/Kf603wpC3JlxJpvGdBGOFldPgfp2Wcxsf7WlIX+tUns3wy2vIUEFs6RZNTRA5G6Cc6KId9SDAP/ad96tSUPQrJGR/4wXlBFj+TmEWnkwhJp8JkzTjLjz5ByefCC8eBZHW+EsT0vHMin/VgreTKK+PX7XVH+iMV3UXZSzhvNRw/tqGF6bk/wcAAP//nHNPVA==" } diff --git a/x-pack/metricbeat/module/googlecloud/metrics/metricset.go b/x-pack/metricbeat/module/googlecloud/metrics/metricset.go index 486414b28ff..b0823feeace 100644 --- a/x-pack/metricbeat/module/googlecloud/metrics/metricset.go +++ b/x-pack/metricbeat/module/googlecloud/metrics/metricset.go @@ -298,11 +298,7 @@ func addHostFields(groupedEvents []KeyValuePoint) common.MapStr { for _, singleEvent := range groupedEvents { if hostMetricName, ok := hostFieldTable[singleEvent.Key]; ok { - if hostMetricName == "host.cpu.pct" { - hostRootFields.Put(hostMetricName, singleEvent.Value.(float64)*100) - } else { - hostRootFields.Put(hostMetricName, singleEvent.Value) - } + hostRootFields.Put(hostMetricName, singleEvent.Value) } } return hostRootFields diff --git a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go index 90603389426..fe14aff5966 100644 --- a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector.go @@ -6,6 +6,7 @@ package googlecloud import ( "context" + "strings" "time" "github.com/golang/protobuf/ptypes" @@ -62,6 +63,12 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i if availabilityZone != "" { ecs[ECSCloud+"."+ECSCloudAvailabilityZone] = availabilityZone + + // Get region name from availability zone name + region := getRegionName(availabilityZone) + if region != "" { + ecs[ECSCloud+"."+ECSCloudRegion] = region + } } //Remove keys from resource that refers to ECS fields @@ -169,3 +176,12 @@ func (s *StackdriverTimeSeriesMetadataCollector) getTimestamp(p *monitoringpb.Po return time.Time{}, errors.New("error trying to extract the timestamp from the point data") } + +func getRegionName(availabilityZone string) (region string) { + azSplit := strings.Split(availabilityZone, "-") + if len(azSplit) != 3 { + return "" + } + region = azSplit[0] + "-" + azSplit[1] + return +} diff --git a/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector_test.go b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector_test.go new file mode 100644 index 00000000000..38ac903e2e3 --- /dev/null +++ b/x-pack/metricbeat/module/googlecloud/timeseries_metadata_collector_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package googlecloud + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetRegionName(t *testing.T) { + availabilityZone := "us-central1-a" + region := getRegionName(availabilityZone) + assert.Equal(t, "us-central1", region) +} From 1286bd45e2a3e4ab4511c4b33e841d0797c493db Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 19 Aug 2020 16:31:56 -0600 Subject: [PATCH 3/3] add changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0dcf9c23716..b7a9a49bc26 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -646,6 +646,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077] - Add host inventory metrics into aws ec2 metricset. {pull}20171[20171] - Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547] +- Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391] *Packetbeat*