Add metrics collection from cost explorer into aws/billing metricset (e…

…lastic#20527)

* Add cost metricset for aws module
* Add cost_explorer_config for billing metricset with group_by_dimension_keys and group_by_tag_keys

CHANGELOG.next.asciidoc

@@ -677,6 +677,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add state_daemonset metricset for Kubernetes Metricbeat module {pull}20649[20649]
 - Add host inventory metrics to azure compute_vm metricset. {pull}20641[20641]
 - Add host inventory metrics to googlecloud compute metricset. {pull}20391[20391]
+- Add billing data collection from Cost Explorer into aws billing metricset. {pull}20527[20527] {issue}20103[20103]
 - Migrate `compute_vm` metricset to a light one, map `cloud.instance.id` field. {pull}20889[20889]
 - Request prometheus endpoints to be gzipped by default {pull}20766[20766]
 - Release all kubernetes `state` metricsets as GA {pull}20901[20901]

metricbeat/docs/fields.asciidoc

@@ -1569,8 +1569,7 @@ type: object
 
 
 
-
-*`aws.billing.metrics.EstimatedCharges.max`*::
+*`aws.billing.EstimatedCharges`*::
 +
 --
 Maximum estimated charges for AWS acccount.
@@ -1579,6 +1578,166 @@ type: long
 
 --
 
+*`aws.billing.Currency`*::
++
+--
+Estimated charges currency unit.
+
+type: keyword
+
+--
+
+*`aws.billing.ServiceName`*::
++
+--
+Service name for the maximum estimated charges.
+
+type: keyword
+
+--
+
+
+*`aws.billing.AmortizedCost.amount`*::
++
+--
+Amortized cost amount
+
+type: double
+
+--
+
+*`aws.billing.AmortizedCost.unit`*::
++
+--
+Amortized cost unit
+
+type: keyword
+
+--
+
+
+*`aws.billing.BlendedCost.amount`*::
++
+--
+Blended cost amount
+
+type: double
+
+--
+
+*`aws.billing.BlendedCost.unit`*::
++
+--
+Blended cost unit
+
+type: keyword
+
+--
+
+
+*`aws.billing.NormalizedUsageAmount.amount`*::
++
+--
+Normalized usage amount
+
+type: double
+
+--
+
+*`aws.billing.NormalizedUsageAmount.unit`*::
++
+--
+Normalized usage amount unit
+
+type: keyword
+
+--
+
+
+*`aws.billing.UnblendedCost.amount`*::
++
+--
+Unblended cost amount
+
+type: double
+
+--
+
+*`aws.billing.UnblendedCost.unit`*::
++
+--
+Unblended cost unit
+
+type: keyword
+
+--
+
+
+*`aws.billing.UsageQuantity.amount`*::
++
+--
+Usage quantity amount
+
+type: double
+
+--
+
+*`aws.billing.UsageQuantity.unit`*::
++
+--
+Usage quantity unit
+
+type: keyword
+
+--
+
+*`aws.billing.start_date`*::
++
+--
+Start date for retrieving AWS costs
+
+type: keyword
+
+--
+
+*`aws.billing.end_date`*::
++
+--
+End date for retrieving AWS costs
+
+type: keyword
+
+--
+
+
+*`aws.billing.group_definition.key`*::
++
+--
+The string that represents a key for a specified group
+
+type: keyword
+
+--
+
+*`aws.billing.group_definition.type`*::
++
+--
+The string that represents the type of group
+
+type: keyword
+
+--
+
+*`aws.billing.group_by.*`*::
++
+--
+Cost explorer group by key values
+
+
+type: object
+
+--
+
 [float]
 === cloudwatch
 

x-pack/metricbeat/module/aws/_meta/config.yml

@@ -30,11 +30,16 @@
     - sns
     - sqs
 - module: aws
-  period: 12h
+  period: 24h
   metricsets:
     - billing
-  regions:
-    - us-east-1
+  cost_explorer_config:
+    group_by_dimension_keys:
+      - "AZ"
+      - "INSTANCE_TYPE"
+      - "SERVICE"
+#    group_by_tag_keys:
+#      - "aws:createdBy"
 - module: aws
   period: 24h
   metricsets: