[Metricbeat] Add PubSub metricset to Google Cloud Platform module

CHANGELOG.next.asciidoc

@@ -599,6 +599,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add DynamoDB AWS Metricbeat light module {pull}15097[15097]
 - Release elb module as GA. {pull}15485[15485]
 - Add a `system/network_summary` metricset {pull}15196[15196]
+- Add PubSub metricset to Google Cloud Platform module {pull}15536[15536]
 
 *Packetbeat*
 

metricbeat/docs/modules/googlecloud.asciidoc

@@ -43,7 +43,7 @@ Authentication and authorization in Google Cloud Platform can be achieved in man
 }
 ----
 
-Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role:
+Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role (applies to all metricsets):
 
 * `Compute Viewer` (1 permission in use of 138):
 - `monitoring.metricDescriptors.list`
@@ -82,6 +82,7 @@ metricbeat.modules:
 - module: googlecloud
   metricsets:
     - compute
+    - pubsub
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"
@@ -96,5 +97,9 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-googlecloud-compute,compute>>
 
+* <<metricbeat-metricset-googlecloud-pubsub,pubsub>>
+
 include::googlecloud/compute.asciidoc[]
 
+include::googlecloud/pubsub.asciidoc[]
+

metricbeat/docs/modules/googlecloud/pubsub.asciidoc

@@ -0,0 +1,18 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-googlecloud-pubsub]]
+=== Google Cloud Platform pubsub metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-googlecloud,exported fields>> section.
+

metricbeat/docs/modules_list.asciidoc

@@ -91,7 +91,8 @@ This file is generated! See scripts/mage/docs_collector.go
 .2+| .2+|  |<<metricbeat-metricset-golang-expvar,expvar>>   
 |<<metricbeat-metricset-golang-heap,heap>>   
 |<<metricbeat-module-googlecloud,Google Cloud Platform>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.1+| .1+|  |<<metricbeat-metricset-googlecloud-compute,compute>> beta[]  
+.2+| .2+|  |<<metricbeat-metricset-googlecloud-compute,compute>> beta[]  
+|<<metricbeat-metricset-googlecloud-pubsub,pubsub>> beta[]  
 |<<metricbeat-module-graphite,Graphite>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-graphite-server,server>>   
 |<<metricbeat-module-haproxy,HAProxy>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  

x-pack/metricbeat/metricbeat.reference.yml

@@ -437,6 +437,7 @@ metricbeat.modules:
 - module: googlecloud
   metricsets:
     - compute
+    - pubsub
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"

x-pack/metricbeat/module/googlecloud/_meta/config.yml

@@ -1,6 +1,7 @@
 - module: googlecloud
   metricsets:
     - compute
+    - pubsub
   zone: "us-central1-a"
   project_id: "your project id"
   credentials_file_path: "your JSON credentials file path"

x-pack/metricbeat/module/googlecloud/_meta/docs.asciidoc

@@ -33,7 +33,7 @@ Authentication and authorization in Google Cloud Platform can be achieved in man
 }
 ----
 
-Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role:
+Generally, you have to create a Service Account and assign it the following roles or the permissions described on each role (applies to all metricsets):
 
 * `Compute Viewer` (1 permission in use of 138):
 - `monitoring.metricDescriptors.list`

x-pack/metricbeat/module/googlecloud/module.yml

@@ -1,3 +1,4 @@
 name: googlecloud
 metricsets:
 - compute
+- pubsub

x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_subscription.json

@@ -0,0 +1,53 @@
+{
+    "@timestamp": "2020-01-14T11:46:00.000Z",
+    "@metadata": {
+        "beat": "metricbeat",
+        "type": "_doc",
+        "version": "8.0.0"
+    },
+    "googlecloud": {
+        "labels": {
+            "resource": {
+                "subscription_id": "test-subscription"
+            }
+        },
+        "pubsub": {
+            "subscription": {
+                "num_undelivered_messages": 0,
+                "oldest_unacked_message_age": 0,
+                "backlog_bytes": 0
+            }
+        }
+    },
+    "service": {
+        "type": "googlecloud"
+    },
+    "cloud": {
+        "account": {
+            "id": "elastic-metricbeat"
+        },
+        "provider": "googlecloud"
+    },
+    "host": {
+        "name": "mcastro"
+    },
+    "agent": {
+        "version": "8.0.0",
+        "type": "metricbeat",
+        "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa",
+        "hostname": "mcastro",
+        "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228"
+    },
+    "ecs": {
+        "version": "1.4.0"
+    },
+    "event": {
+        "dataset": "googlecloud.pubsub",
+        "module": "googlecloud",
+        "duration": 933264720
+    },
+    "metricset": {
+        "name": "pubsub",
+        "period": 300000
+    }
+}

x-pack/metricbeat/module/googlecloud/pubsub/_meta/data_topic.json

@@ -0,0 +1,61 @@
+{
+    "@timestamp": "2020-01-14T11:42:00.000Z",
+    "@metadata": {
+        "beat": "metricbeat",
+        "type": "_doc",
+        "version": "8.0.0"
+    },
+    "ecs": {
+        "version": "1.4.0"
+    },
+    "cloud": {
+        "provider": "googlecloud",
+        "account": {
+            "id": "elastic-metricbeat"
+        }
+    },
+    "metricset": {
+        "name": "pubsub",
+        "period": 300000
+    },
+    "event": {
+        "duration": 932060852,
+        "dataset": "googlecloud.pubsub",
+        "module": "googlecloud"
+    },
+    "googlecloud": {
+        "labels": {
+            "resource": {
+                "subscription_id": "test-subscription"
+            }
+        },
+        "pubsub": {
+            "topic": {
+                "message_sizes": {
+                    "bucket_options": {
+                        "Options": {
+                            "ExponentialBuckets": {
+                                "growth_factor": 4,
+                                "scale": 1,
+                                "num_finite_buckets": 16
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    },
+    "service": {
+        "type": "googlecloud"
+    },
+    "host": {
+        "name": "mcastro"
+    },
+    "agent": {
+        "ephemeral_id": "b3b8d9da-0094-4d54-b1dc-4d5e103d38aa",
+        "hostname": "mcastro",
+        "id": "7e36a073-1a32-4a94-b65b-4c7f971fb228",
+        "version": "8.0.0",
+        "type": "metricbeat"
+    }
+}

x-pack/metricbeat/module/googlecloud/pubsub/_meta/docs.asciidoc

@@ -0,0 +1,25 @@
+PubSub Metricset to fetch metrics from https://cloud.google.com/pubsub/[Pub/Sub] topics and subscriptions in Google Cloud Platform.
+
+The `pubsub` Metricset contains all metrics exported from the https://cloud.google.com/monitoring/api/metrics_gcp#gcp-pubsub[Stackdriver API]. The field names have been left untouched for people already familiar with them.
+
+No special permissions are needed apart from the ones detailed in the module section of the docs.
+
+- `pubsub.subscription.ack_message_count`: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type.
+- `pubsub.subscription.backlog_bytes`: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription.
+- `pubsub.subscription.num_outstanding_messages`: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged.
+- `pubsub.subscription.num_undelivered_messages`: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription.
+- `pubsub.subscription.oldest_unacked_message_age`: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription.
+- `pubsub.subscription.pull_ack_message_operation_count`: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count.
+- `pubsub.subscription.pull_ack_request_count`: Cumulative count of acknowledge requests, grouped by result.
+- `pubsub.subscription.pull_message_operation_count`: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count.
+- `pubsub.subscription.pull_request_count`: Cumulative count of pull requests, grouped by result.
+- `pubsub.subscription.push_request_count`: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times.
+- `pubsub.subscription.push_request_latencies`: Distribution of push request latencies (in microseconds), grouped by result.
+- `pubsub.subscription.sent_message_count`: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type.
+- `pubsub.subscription.streaming_pull_ack_message_operation_count`: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count.
+- `pubsub.subscription.streaming_pull_ack_request_count`: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result.
+- `pubsub.subscription.streaming_pull_message_operation_count`: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric <code>subscription/mod_ack_deadline_message_operation_count
+- `pubsub.subscription.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result.
+- `pubsub.topic.streaming_pull_response_count`: Cumulative count of streaming pull responses, grouped by result.
+- `pubsub.topic.send_message_operation_count`: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count.
+- `pubsub.topic.send_request_count`: Cumulative count of publish requests, grouped by result.

x-pack/metricbeat/module/googlecloud/pubsub/_meta/fields.yml

@@ -0,0 +1,70 @@
+- name: pubsub
+  release: beta
+  type: group
+  description: Google Cloud PubSub metrics
+  fields:
+    - name: subscription
+      type: group
+      description: Suscription related metrics
+      fields:
+        - name: ack_message_count
+          type: long
+          description: Cumulative count of messages acknowledged by Acknowledge requests, grouped by delivery type.
+        - name: backlog_bytes
+          type: long
+          description: Total byte size of the unacknowledged messages (a.k.a. backlog messages) in a subscription.
+        - name: num_outstanding_messages
+          type: long
+          description: Number of messages delivered to a subscription's push endpoint, but not yet acknowledged.
+        - name: num_undelivered_messages
+          type: long
+          description: Number of unacknowledged messages (a.k.a. backlog messages) in a subscription.
+        - name: oldest_unacked_message_age
+          type: long
+          description: Age (in seconds) of the oldest unacknowledged message (a.k.a. backlog message) in a subscription.
+        - name: pull_ack_message_operation_count
+          type: long
+          description: Cumulative count of acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count.
+        - name: pull_ack_request_count
+          type: long
+          description: Cumulative count of acknowledge requests, grouped by result.
+        - name: pull_message_operation_count
+          type: long
+          description: Cumulative count of pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count.
+        - name: pull_request_count
+          type: long
+          description: Cumulative count of pull requests, grouped by result.
+        - name: push_request_count
+          type: long
+          description: Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times.
+        - name: push_request_latencies
+          type: long
+          description: Distribution of push request latencies (in microseconds), grouped by result.
+        - name: sent_message_count
+          type: long
+          description: Cumulative count of messages sent by Cloud Pub/Sub to subscriber clients, grouped by delivery type.
+        - name: streaming_pull_ack_message_operation_count
+          type: long
+          description: Cumulative count of StreamingPull acknowledge message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count.
+        - name: streaming_pull_ack_request_count
+          type: long
+          description: Cumulative count of streaming pull requests with non-empty acknowledge ids, grouped by result.
+        - name: streaming_pull_message_operation_count
+          type: long
+          description: Cumulative count of streaming pull message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric <code>subscription/mod_ack_deadline_message_operation_count
+        - name: streaming_pull_response_count
+          type: long
+          description: Cumulative count of streaming pull responses, grouped by result.
+    - name: topic
+      type: group
+      description: Topic related metrics
+      fields:
+        - name: streaming_pull_response_count
+          type: long
+          description: Cumulative count of streaming pull responses, grouped by result.
+        - name: send_message_operation_count
+          type: long
+          description: Cumulative count of publish message operations, grouped by result. For a definition of message operations, see Cloud Pub/Sub metric subscription/mod_ack_deadline_message_operation_count.
+        - name: send_request_count
+          type: long
+          description: Cumulative count of publish requests, grouped by result.

x-pack/metricbeat/module/googlecloud/pubsub/manifest.yml

@@ -0,0 +1,27 @@
+default: true
+input:
+  module: googlecloud
+  metricset: stackdriver
+  defaults:
+    stackdriver:
+      service: pubsub
+      metrics:
+        - "pubsub.googleapis.com/subscription/ack_message_count"
+        - "pubsub.googleapis.com/subscription/backlog_bytes"
+        - "pubsub.googleapis.com/subscription/num_outstanding_messages"
+        - "pubsub.googleapis.com/subscription/num_undelivered_messages"
+        - "pubsub.googleapis.com/subscription/oldest_unacked_message_age"
+        - "pubsub.googleapis.com/subscription/pull_ack_message_operation_count"
+        - "pubsub.googleapis.com/subscription/pull_ack_request_count"
+        - "pubsub.googleapis.com/subscription/pull_message_operation_count"
+        - "pubsub.googleapis.com/subscription/pull_request_count"
+        - "pubsub.googleapis.com/subscription/push_request_count"
+        - "pubsub.googleapis.com/subscription/push_request_latencies"
+        - "pubsub.googleapis.com/subscription/sent_message_count"
+        - "pubsub.googleapis.com/subscription/streaming_pull_ack_message_operation_count"
+        - "pubsub.googleapis.com/subscription/streaming_pull_ack_request_count"
+        - "pubsub.googleapis.com/subscription/streaming_pull_message_operation_count"
+        - "pubsub.googleapis.com/subscription/streaming_pull_response_count"
+        - "pubsub.googleapis.com/topic/message_sizes"
+        - "pubsub.googleapis.com/topic/send_message_operation_count"
+        - "pubsub.googleapis.com/topic/send_request_count"

x-pack/metricbeat/module/googlecloud/stackdriver/metadata_services.go

@@ -17,6 +17,8 @@ func NewMetadataServiceForConfig(c config) (googlecloud.MetadataService, error)
 	switch c.ServiceName {
 	case googlecloud.ServiceCompute:
 		return compute.NewMetadataService(c.ProjectID, c.Zone, c.opt...)
+	case googlecloud.ServicePubsub:
+		return nil, nil
 	default:
 		return nil, errors.Errorf("service '%s' not supported", c.ServiceName)
 	}