GCP Carbon footprint metricset

CHANGELOG.next.asciidoc

@@ -263,6 +263,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Changed cloudwatch module to call ListMetrics API only once per region, instead of per AWS namespace {pull}34055[34055]
 - Add beta ingest_pipeline metricset to Elasticsearch module for ingest pipeline monitoring {pull}34012[34012]
 - Handle duplicated TYPE line for prometheus metrics {issue}18813[18813] {pull}33865[33865]
+- Add GCP Carbon Footprint metricbeat data {pull}34820[34820]
 
 *Packetbeat*
 

go.mod

@@ -184,6 +184,7 @@ require (
 )
 
 require (
+	cloud.google.com/go v0.105.0
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1
 	github.com/Azure/go-autorest/autorest/adal v0.9.14
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.17
@@ -213,7 +214,6 @@ require (
 )
 
 require (
-	cloud.google.com/go v0.105.0 // indirect
 	cloud.google.com/go/compute v1.10.0 // indirect
 	cloud.google.com/go/iam v0.6.0 // indirect
 	code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f // indirect

metricbeat/docs/fields.asciidoc

@@ -35358,6 +35358,102 @@ type: float
 
 --
 
+[float]
+=== carbon
+
+Google Cloud Carbon Footprint metrics
+
+
+*`gcp.carbon.project_id`*::
++
+--
+Project ID the carbon footprint report belongs to.
+
+type: keyword
+
+--
+
+*`gcp.carbon.project_name`*::
++
+--
+Project name the carbon footprint report belongs to.
+
+type: keyword
+
+--
+
+*`gcp.carbon.service_id`*::
++
+--
+Service ID for the carbon footprint usage.
+
+type: keyword
+
+--
+
+*`gcp.carbon.service_description`*::
++
+--
+Service description for the carbon footprint usage.
+
+type: keyword
+
+--
+
+*`gcp.carbon.region`*::
++
+--
+Region for the carbon fooprint usage.
+
+type: keyword
+
+--
+
+*`gcp.carbon.footprint.scope1`*::
++
+--
+Scope 1 carbon footprint.
+
+type: float
+
+--
+
+*`gcp.carbon.footprint.scope2.location`*::
++
+--
+Scope 2 carbon footprint using location-based methodology.
+
+type: float
+
+--
+
+*`gcp.carbon.footprint.scope2.market`*::
++
+--
+Scope 2 carbon footprint using market-based methodology.
+
+type: float
+
+--
+
+*`gcp.carbon.footprint.scope3`*::
++
+--
+Scope 3 carbon footprint.
+
+type: float
+
+--
+
+*`gcp.carbon.footprint.offsets`*::
++
+--
+Total carbon offsets.
+
+type: float
+
+--
+
 [float]
 === compute
 

metricbeat/docs/modules/gcp.asciidoc

@@ -329,6 +329,15 @@ metricbeat.modules:
   dataset_id: "dataset id"
   table_pattern: "table pattern"
   cost_type: "regular"
+
+- module: gcp
+  metricsets:
+    - carbon
+  period: 24h
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  dataset_id: "dataset id"
+  table_pattern: "table pattern"
 ----
 
 [float]
@@ -338,6 +347,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-gcp-billing,billing>>
 
+* <<metricbeat-metricset-gcp-carbon,carbon>>
+
 * <<metricbeat-metricset-gcp-compute,compute>>
 
 * <<metricbeat-metricset-gcp-dataproc,dataproc>>
@@ -356,6 +367,8 @@ The following metricsets are available:
 
 include::gcp/billing.asciidoc[]
 
+include::gcp/carbon.asciidoc[]
+
 include::gcp/compute.asciidoc[]
 
 include::gcp/dataproc.asciidoc[]

metricbeat/docs/modules/gcp/carbon.asciidoc

@@ -0,0 +1,29 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+:edit_url: https://github.com/elastic/beats/edit/main/x-pack/metricbeat/module/gcp/carbon/_meta/docs.asciidoc
+
+
+[[metricbeat-metricset-gcp-carbon]]
+[role="xpack"]
+=== Google Cloud Platform carbon metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/gcp/carbon/_meta/docs.asciidoc[]
+
+
+:edit_url:
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-gcp,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/gcp/carbon/_meta/data.json[]
+----
+:edit_url!:

metricbeat/docs/modules_list.asciidoc

@@ -121,7 +121,8 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-etcd-self,self>>   
 |<<metricbeat-metricset-etcd-store,store>>   
 |<<metricbeat-module-gcp,Google Cloud Platform>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.9+| .9+|  |<<metricbeat-metricset-gcp-billing,billing>> beta[]  
+.10+| .10+|  |<<metricbeat-metricset-gcp-billing,billing>> beta[]  
+|<<metricbeat-metricset-gcp-carbon,carbon>> beta[]  
 |<<metricbeat-metricset-gcp-compute,compute>> beta[]  
 |<<metricbeat-metricset-gcp-dataproc,dataproc>> beta[]  
 |<<metricbeat-metricset-gcp-firestore,firestore>> beta[]  

x-pack/metricbeat/metricbeat.reference.yml

@@ -611,6 +611,15 @@ metricbeat.modules:
   table_pattern: "table pattern"
   cost_type: "regular"
 
+- module: gcp
+  metricsets:
+    - carbon
+  period: 24h
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  dataset_id: "dataset id"
+  table_pattern: "table pattern"
+
 #-------------------------------- Golang Module --------------------------------
 - module: golang
   #metricsets:

x-pack/metricbeat/module/gcp/_meta/config.yml

@@ -60,3 +60,12 @@
   dataset_id: "dataset id"
   table_pattern: "table pattern"
   cost_type: "regular"
+
+- module: gcp
+  metricsets:
+    - carbon
+  period: 24h
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  dataset_id: "dataset id"
+  table_pattern: "table pattern"

x-pack/metricbeat/module/gcp/carbon/_meta/data.json

@@ -0,0 +1,32 @@
+{
+  "@timestamp": "2017-10-12T08:05:34.853Z",
+    "cloud.account.id": "01475F-5B1080-1137E7",
+    "cloud.project.id": "elastic-bi",
+    "cloud.project.name": "elastic-containerlib-prod",
+    "cloud.provider": "gcp",
+    "event": {
+      "dataset": "gcp.carbon",
+      "duration": 115000,
+      "module": "gcp"
+    },
+    "gcp": {
+      "carbon": {
+        "project_id": "containerlib-prod-12763",
+        "project_name": "elastic-containerlib-prod",
+        "service_id": "24E6-581D-38E5",
+        "service_description": "BigQuery",
+        "footprint.scope1":          4.044,
+        "footprint.scope2.location": 1.797,
+        "footprint.scope2.market":   null,
+        "footprint.scope3":          2.337,
+        "footprint.offsets":         null
+      }
+    },
+    "metricset": {
+      "name": "carbon",
+      "period": 10000
+    },
+    "service": {
+      "type": "gcp"
+    }
+}

x-pack/metricbeat/module/gcp/carbon/_meta/docs.asciidoc

@@ -0,0 +1,31 @@
+The `carbon` metricset is designed to collect Carbon Footprint data from GCP
+BigQuery monthly cost detail table. BigQuery is a fully-managed, serverless
+data warehouse.
+
+Cloud Carbon export to BigQuery enables you to export detailed Google Cloud
+carbon footprint data (such as carbon produced by tier and service) automatically
+throughout the month to a BigQuery dataset that you specify. Then you can access
+your Cloud Carbon data from BigQuery for detailed analysis using Metricbeat.
+Please see https://cloud.google.com/carbon-footprint/docs/export[export carbon
+footprint data to BigQuery] for more details on how to export carbon footprint data.
+
+[float]
+=== Metricset-specific configuration notes
+* *dataset_id*: (Required) Dataset ID that points to the top-level container which contains
+the actual carbon footprint tables.
+* *table_pattern*: (Optional) The name of the table where carbon footprint data
+is stored. Default to `carbon_footprint`.
+
+[float]
+=== Configuration example
+[source,yaml]
+----
+- module: gcp
+  metricsets:
+    - carbon
+  period: 24h
+  project_id: "your project id"
+  credentials_file_path: "your JSON credentials file path"
+  dataset_id: "dataset id"
+  table_name: "table name"
+----

x-pack/metricbeat/module/gcp/carbon/_meta/fields.yml

@@ -0,0 +1,35 @@
+- name: carbon
+  description: Google Cloud Carbon Footprint metrics
+  release: beta
+  type: group
+  fields:
+  - name: project_id
+    type: keyword
+    description: Project ID the carbon footprint report belongs to.
+  - name: project_name
+    type: keyword
+    description: Project name the carbon footprint report belongs to.
+  - name: service_id
+    type: keyword
+    description: Service ID for the carbon footprint usage.
+  - name: service_description
+    type: keyword
+    description: Service description for the carbon footprint usage.
+  - name: region
+    type: keyword
+    description: Region for the carbon fooprint usage.
+  - name: footprint.scope1
+    type: float
+    description: Scope 1 carbon footprint.
+  - name: footprint.scope2.location
+    type: float
+    description: Scope 2 carbon footprint using location-based methodology.
+  - name: footprint.scope2.market
+    type: float
+    description: Scope 2 carbon footprint using market-based methodology.
+  - name: footprint.scope3
+    type: float
+    description: Scope 3 carbon footprint.
+  - name: footprint.offsets
+    type: float
+    description: Total carbon offsets.