[Metricbeat] Adding cloudwatch metricset into AWS module (#11798)

* Initial draft for adding cloudwatch metricset * Combine metrics into same events by identifiers(instanceID, queueName,...) * Split metricDataQueries into 100 length in GetMetricDataResults * Add support to read metricname, dimensions from config for cloudwatch * Move CloudwatchMetricsConfigs into cloudwatch metricset from aws module * Change Period from string to time.Duration type
elastic · May 1, 2019 · 55e9628 · 55e9628
1 parent 0a2ae60
commit 55e9628
Show file tree

Hide file tree

Showing 22 changed files with 947 additions and 97 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -174,6 +174,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added new disk states and raid level to the system/raid metricset. {pull}11613[11613]
 - Added `path_name` and `start_name` to service metricset on windows module {issue}8364[8364] {pull}11877[11877]
 - Add check on object name in the counter path if the instance name is missing {issue}6528[6528] {pull}11878[11878]
+- Add AWS cloudwatch metricset. {pull}11798[11798] {issue}11734[11734]
 
 *Packetbeat*
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -830,6 +830,43 @@ Total.
 
 
 
+[float]
+== cloudwatch fields
+
+`cloudwatch` contains the metrics that were scraped from AWS CloudWatch which contains monitoring metrics sent by different namespaces.
+
+
+
+*`aws.cloudwatch.namespace`*::
++
+--
+type: keyword
+
+The namespace specified when query cloudwatch api.
+
+
+--
+
+*`aws.cloudwatch.metrics.*`*::
++
+--
+type: object
+
+Metrics that returned from Cloudwatch api query.
+
+
+--
+
+*`aws.cloudwatch.dimensions.*`*::
++
+--
+type: object
+
+Cloudwatch metric dimensions.
+
+
+--
+
 [float]
 == ec2 fields
 

diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc
@@ -9,7 +9,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using
 https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for AWS services.
 Note: extra AWS charges on GetMetricData API requests will be generated by this module.
 
-The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`.
+The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `cloudwatch`.
 
 [float]
 === Module-specific configuration notes
@@ -50,8 +50,8 @@ metricbeat.modules:
 - module: aws
   period: 300s
   metricsets:
-    - "ec2"
-    - "sqs"
+    - ec2
+    - sqs
   access_key_id: '${AWS_ACCESS_KEY_ID}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY}'
   session_token: '${AWS_SESSION_TOKEN}'
@@ -102,6 +102,11 @@ https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[Ho
 Configure Request Metrics for S3] for instructions on how to enable request metrics for
 each S3 bucket.
 
+[float]
+=== `cloudwatch`
+This metricset gives users the freedom to query metrics from AWS Cloudwatch with
+any given namespaces or specific instance with a given period.
+
 
 [float]
 === Example configuration
@@ -115,28 +120,45 @@ metricbeat.modules:
 - module: aws
   period: 300s
   metricsets:
-    - "ec2"
-    - "sqs"
+    - ec2
+    - sqs
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
 - module: aws
   period: 86400s
   metricsets:
-    - "s3_request"
-    - "s3_daily_storage"
+    - s3_request
+    - s3_daily_storage
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+  cloudwatch_metrics:
+    - namespace: AWS/EC2
+      metricname: CPUUtilization
+      dimensions:
+        - name: InstanceId
+          value: i-0686946e22cf9494a
+    - namespace: AWS/EBS
 ----
 
 [float]
 === Metricsets
 
 The following metricsets are available:
 
+* <<metricbeat-metricset-aws-cloudwatch,cloudwatch>>
+
 * <<metricbeat-metricset-aws-ec2,ec2>>
 
 * <<metricbeat-metricset-aws-s3_daily_storage,s3_daily_storage>>
@@ -145,6 +167,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-aws-sqs,sqs>>
 
+include::aws/cloudwatch.asciidoc[]
+
 include::aws/ec2.asciidoc[]
 
 include::aws/s3_daily_storage.asciidoc[]

diff --git a/metricbeat/docs/modules/aws/cloudwatch.asciidoc b/metricbeat/docs/modules/aws/cloudwatch.asciidoc
@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-aws-cloudwatch]]
+=== aws cloudwatch metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/aws/cloudwatch/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-aws,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/aws/cloudwatch/_meta/data.json[]
+----
diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc
@@ -10,7 +10,8 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-module-apache,Apache>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .1+| .1+|  |<<metricbeat-metricset-apache-status,status>>   
 |<<metricbeat-module-aws,aws>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.4+| .4+|  |<<metricbeat-metricset-aws-ec2,ec2>>   
+.5+| .5+|  |<<metricbeat-metricset-aws-cloudwatch,cloudwatch>> beta[]  
+|<<metricbeat-metricset-aws-ec2,ec2>>   
 |<<metricbeat-metricset-aws-s3_daily_storage,s3_daily_storage>> beta[]  
 |<<metricbeat-metricset-aws-s3_request,s3_request>> beta[]  
 |<<metricbeat-metricset-aws-sqs,sqs>> beta[]  

diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go
diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
@@ -160,21 +160,36 @@ metricbeat.modules:
 - module: aws
   period: 300s
   metricsets:
-    - "ec2"
-    - "sqs"
+    - ec2
+    - sqs
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
 - module: aws
   period: 86400s
   metricsets:
-    - "s3_request"
-    - "s3_daily_storage"
+    - s3_request
+    - s3_daily_storage
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+  cloudwatch_metrics:
+    - namespace: AWS/EC2
+      metricname: CPUUtilization
+      dimensions:
+        - name: InstanceId
+          value: i-0686946e22cf9494a
+    - namespace: AWS/EBS
 
 #--------------------------------- Ceph Module ---------------------------------
 - module: ceph

diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml
@@ -1,18 +1,33 @@
 - module: aws
   period: 300s
   metricsets:
-    - "ec2"
-    - "sqs"
+    - ec2
+    - sqs
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
 - module: aws
   period: 86400s
   metricsets:
-    - "s3_request"
-    - "s3_daily_storage"
+    - s3_request
+    - s3_daily_storage
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
   session_token: '${AWS_SESSION_TOKEN:""}'
   default_region: '${AWS_REGION:us-west-1}'
+- module: aws
+  period: 300s
+  metricsets:
+    - cloudwatch
+  access_key_id: '${AWS_ACCESS_KEY_ID:""}'
+  secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
+  session_token: '${AWS_SESSION_TOKEN:""}'
+  default_region: '${AWS_REGION:us-west-1}'
+  cloudwatch_metrics:
+    - namespace: AWS/EC2
+      metricname: CPUUtilization
+      dimensions:
+        - name: InstanceId
+          value: i-0686946e22cf9494a
+    - namespace: AWS/EBS
diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc
@@ -2,7 +2,7 @@ This module periodically fetches monitoring metrics from AWS Cloudwatch using
 https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[GetMetricData API] for AWS services.
 Note: extra AWS charges on GetMetricData API requests will be generated by this module.
 
-The default metricsets are `ec2`, `sqs`, `s3_request` and `s3_daily_storage`.
+The default metricsets are `ec2`, `sqs`, `s3_request`, `s3_daily_storage` and `cloudwatch`.
 
 [float]
 === Module-specific configuration notes
@@ -43,8 +43,8 @@ metricbeat.modules:
 - module: aws
   period: 300s
   metricsets:
-    - "ec2"
-    - "sqs"
+    - ec2
+    - sqs
   access_key_id: '${AWS_ACCESS_KEY_ID}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY}'
   session_token: '${AWS_SESSION_TOKEN}'
@@ -94,3 +94,8 @@ always adjust this to the granularity they want. Request metrics are not enabled
 https://docs.aws.amazon.com/AmazonS3/latest/user-guide/configure-metrics.html[How to
 Configure Request Metrics for S3] for instructions on how to enable request metrics for
 each S3 bucket.
+
+[float]
+=== `cloudwatch`
+This metricset gives users the freedom to query metrics from AWS Cloudwatch with
+any given namespaces or specific instance with a given period.
diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go
@@ -5,7 +5,7 @@
 package aws
 
 import (
-	"strconv"
+	"time"
 
 	"github.com/elastic/beats/libbeat/common"
 
@@ -20,11 +20,11 @@ import (
 
 // Config defines all required and optional parameters for aws metricsets
 type Config struct {
-	Period          string `config:"period"`
-	AccessKeyID     string `config:"access_key_id"`
-	SecretAccessKey string `config:"secret_access_key"`
-	SessionToken    string `config:"session_token"`
-	DefaultRegion   string `config:"default_region"`
+	Period          time.Duration `config:"period" validate:"nonzero,required"`
+	AccessKeyID     string        `config:"access_key_id"`
+	SecretAccessKey string        `config:"secret_access_key"`
+	SessionToken    string        `config:"session_token"`
+	DefaultRegion   string        `config:"default_region"`
 }
 
 // MetricSet is the base metricset for all aws metricsets
@@ -82,13 +82,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) {
 		return nil, err
 	}
 
-	// Calculate duration based on period
-	if config.Period == "" {
-		err = errors.New("period is not set in AWS module config")
-		return nil, err
-	}
-
-	durationString, periodSec, err := convertPeriodToDuration(config.Period)
+	durationString, periodSec := convertPeriodToDuration(config.Period)
 	if err != nil {
 		return nil, err
 	}
@@ -118,28 +112,12 @@ func getRegions(svc ec2iface.EC2API) (regionsList []string, err error) {
 	return
 }
 
-func convertPeriodToDuration(period string) (string, int, error) {
+func convertPeriodToDuration(period time.Duration) (string, int) {
 	// Set starttime double the default frequency earlier than the endtime in order to make sure
 	// GetMetricDataRequest gets the latest data point for each metric.
-	numberPeriod, err := strconv.Atoi(period[0 : len(period)-1])
-	if err != nil {
-		return "", 0, err
-	}
-
-	unitPeriod := period[len(period)-1:]
-	switch unitPeriod {
-	case "s":
-		duration := "-" + strconv.Itoa(numberPeriod*2) + unitPeriod
-		return duration, numberPeriod, nil
-	case "m":
-		duration := "-" + strconv.Itoa(numberPeriod*2) + unitPeriod
-		periodInSec := numberPeriod * 60
-		return duration, periodInSec, nil
-	default:
-		err = errors.New("invalid period in config. Please reset period in config")
-		duration := "-" + strconv.Itoa(numberPeriod*2) + "s"
-		return duration, numberPeriod, err
-	}
+	duration := "-" + (period * 2).String()
+	numberPeriod := int(period.Seconds())
+	return duration, numberPeriod
 }
 
 // StringInSlice checks if a string is already exists in list
@@ -156,8 +134,8 @@ func StringInSlice(str string, list []string) bool {
 func InitEvent(metricsetName string, regionName string) mb.Event {
 	event := mb.Event{}
 	event.Service = metricsetName
-	event.RootFields = common.MapStr{}
 	event.MetricSetFields = common.MapStr{}
+	event.RootFields = common.MapStr{}
 	event.RootFields.Put("service.name", metricsetName)
 	event.RootFields.Put("cloud.provider", "aws")
 	if regionName != "" {