[receiver/awscloudwatchmetricsreceiver] AWS CloudWatch metrics receiver

.github/CODEOWNERS

@@ -158,6 +158,7 @@ receiver/activedirectorydsreceiver/                      @open-telemetry/collect
 receiver/aerospikereceiver/                              @open-telemetry/collector-contrib-approvers @djaglowski @antonblock
 receiver/apachereceiver/                                 @open-telemetry/collector-contrib-approvers @djaglowski
 receiver/awscloudwatchreceiver/                          @open-telemetry/collector-contrib-approvers @djaglowski @schmikei
+receiver/awscloudwatchmetricsreceiver                    @open-telemetry/collector-contrib-approvers @lewis262626 @jpkrohling @kovrus
 receiver/awscontainerinsightreceiver/                    @open-telemetry/collector-contrib-approvers @Aneurysm9 @pxaws
 receiver/awsecscontainermetricsreceiver/                 @open-telemetry/collector-contrib-approvers @Aneurysm9
 receiver/awsfirehosereceiver/                            @open-telemetry/collector-contrib-approvers @Aneurysm9
@@ -238,4 +239,4 @@ receiver/zookeeperreceiver/                              @open-telemetry/collect
 
 testbed/                                                 @open-telemetry/collector-contrib-approvers @open-telemetry/collector-approvers
 testbed/mockdatareceivers/mockawsxrayreceiver/           @open-telemetry/collector-contrib-approvers @willarmiros
-testbed/mockdatasenders/mockdatadogagentexporter/        @open-telemetry/collector-contrib-approvers @boostchicken
+testbed/mockdatasenders/mockdatadogagentexporter/        @open-telemetry/collector-contrib-approvers @boostchicken

cmd/configschema/go.mod

@@ -115,6 +115,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/aerospikereceiver v0.73.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apachereceiver v0.73.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchreceiver v0.73.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchmetricsreceiver v0.73.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.73.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver v0.73.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver v0.73.0 // indirect
@@ -875,6 +876,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apach
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchreceiver => ../../receiver/awscloudwatchreceiver
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchmetricsreceiver => ../../receiver/awscloudwatchmetricsreceiver
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => ../../receiver/awscontainerinsightreceiver
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver => ../../receiver/awsecscontainermetricsreceiver

go.mod

@@ -86,6 +86,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/aerospikereceiver v0.73.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apachereceiver v0.73.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchreceiver v0.73.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchmetricsreceiver v0.73.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.73.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver v0.73.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver v0.73.0
@@ -874,6 +875,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apach
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchreceiver => ./receiver/awscloudwatchreceiver
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchmetricsreceiver => ./receiver/awscloudwatchmetricsreceiver
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => ./receiver/awscontainerinsightreceiver
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver => ./receiver/awsecscontainermetricsreceiver

internal/components/components.go

@@ -109,6 +109,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/activedirectorydsreceiver"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/aerospikereceiver"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/apachereceiver"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchmetricsreceiver"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchreceiver"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver"
@@ -221,6 +222,7 @@ func Components() (otelcol.Factories, error) {
 		activedirectorydsreceiver.NewFactory(),
 		aerospikereceiver.NewFactory(),
 		apachereceiver.NewFactory(),
+		awscloudwatchmetricsreceiver.NewFactory(),
 		awscontainerinsightreceiver.NewFactory(),
 		awsecscontainermetricsreceiver.NewFactory(),
 		awsfirehosereceiver.NewFactory(),

receiver/awscloudwatchmetricsreceiver/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

receiver/awscloudwatchmetricsreceiver/README.md

@@ -0,0 +1,145 @@
+# CloudWatch Metrics Receiver
+
+| Status                   |               |
+| ------------------------ | ------------- |
+| Stability                | [development] |
+| Supported pipeline types | metrics       |
+| Distributions            | [contrib]     |
+
+Receives Cloudwatch metrics from [AWS Cloudwatch](https://aws.amazon.com/cloudwatch/) via the [AWS SDK for Cloudwatch Logs](https://docs.aws.amazon.com/sdk-for-go/api/service/cloudwatchlogs/)
+
+## Getting Started
+
+This receiver uses the [AWS SDK](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html) as mode of authentication, which includes [Profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) and [IMDS](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) authentication for EC2 instances.
+
+## Configuration
+
+### Top Level Parameters
+
+| Parameter       | Notes      | type   | Description                                                                                                                                                                                                                                                                       |
+| --------------- | ---------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `region`        | *required* | string | The AWS recognized region string  |
+| `profile`       | *optional* | string | The AWS profile used to authenticate, if none is specified the default is chosen from the list of profiles  |
+| `IMDSEndpoint`  | *optional* | string | The IMDS endpoint to authenticate to AWS  |                                                                                                                                 
+| `poll_interval`   | `default=1m` | duration   | The duration waiting in between requests | 
+| `metrics`          | *optional* | `Metrics` | Configuration for metrics ingestion of this receiver    |
+
+### Metrics Parameters
+
+
+| Parameter                | Notes        | type                   | Description                                                                                |
+| ------------------------ | ------------ | ---------------------- | ------------------------------------------------------------------------------------------ |
+| `named`                 | *required*   | `See Named Parameters` | Configuration for Named Metrics, by default no metrics are collected |
+
+
+### Named Parameters
+
+| Parameter                | Notes        | type                   | Description                                                                                |
+| ------------------------ | ------------ | ---------------------- | ------------------------------------------------------------------------------------------ |
+| `namespace`                 | *required*   | `string` | AWS Metric namespace, all AWS namespaces are prefixed with `AWS`, eg: `AWS/EC2` for EC2 metrics |
+| `metric_name` | *required* | `string` | AWS metric name |
+| `period` | `default=5m` | duration | Aggregation period |
+| `aws_aggregation` | `default=sum` | string | type of AWS aggregation, eg: sum, min, max, average |
+| `dimensions` | *optional* | `see Dimensions Parameters` | Configuration for metric dimensions |
+
+### Dimension Parameters
+
+| Parameter                | Notes        | type                   | Description                                                                                |
+| ------------------------ | ------------ | ---------------------- | ------------------------------------------------------------------------------------------ |
+| `name`                 | *required*   | `string` | Dimensions name, can't start with a colon |
+| `value` | *required* | `string` | Dimension value |
+
+
+#### Named Example
+
+```yaml
+awscloudwatchmetrics:
+  region: us-east-1
+  poll_interval: 1m
+  metrics:
+    named:
+      - namespace: "AWS/EC2"
+        metric_name: "CPUUtilization"
+        period: "5m"
+        aws_aggregation: "Sum"
+        dimensions:
+          - Name: "InstanceId"
+            Value: "i-1234567890abcdef0"
+      - namespace: "AWS/S3"
+        metric_name: "BucketSizeBytes"
+        period: "5m"
+        aws_aggregation: "p99"
+        dimensions:
+          - Name: "BucketName"
+            Value: "OpenTelemetry"
+          - Name: "StorageType"
+            Value: "StandardStorage"
+```
+
+## Sample Configs
+
+
+```yaml
+receivers:
+  awscloudwatchmetrics:
+    region: eu-west-1
+    poll_interval: 10m
+    metrics:
+      named:
+        - namespace: "AWS/EC2"
+          metric_name: "CPUUtilization"
+          period: "5m"
+          aws_aggregation: "Sum"
+          dimensions:
+            - Name: "InstanceId"
+              Value: "i-035e091c31292427a"
+
+processors:
+
+exporters:
+  logging:
+    verbosity: detailed
+
+service:
+  pipelines:
+    metrics:
+      receivers: [awscloudwatchmetrics]
+      processors: []
+      exporters: [logging]
+```
+
+## AWS Costs
+
+This receiver uses the [GetMetricData](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html) API call, this call is *not* in the AWS free tier. Please refer to [Amazon's pricing](https://aws.amazon.com/cloudwatch/pricing/) for further information about expected costs. For `us-east-1`, the current pricing is $0.01 per 1,000 metrics requested as of February 2023.
+
+
+[alpha]:https://github.com/open-telemetry/opentelemetry-collector#alpha
+[contrib]:https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
+[Issue]:https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/15667
+
+## Troubleshooting / Debugging
+
+## My metrics are intermittent / not receing any metrics
+
+Try a bigger `poll_interval`. CloudWatch returns no data if the period of the metric, by default for AWS supplied metrics, it's 300 seconds (5 minutes). Try out a period of 600 seconds and a poll interval of 600 seconds.
+
+## Help, I'm getting IAM permission denied
+
+Make sure your IAM role/user has the required permissions:
+
+```yaml
+"cloudwatch:GetMetricData",
+"cloudwatch:GetMetricStatistics",
+"cloudwatch:ListMetrics"
+```
+
+The following IAM permissions are required for transit gateways to work:
+
+```
+"ec2:DescribeTags",
+"ec2:DescribeInstances",
+"ec2:DescribeRegions",
+"ec2:DescribeTransitGateway*"
+```
+
+

receiver/awscloudwatchmetricsreceiver/config.go

@@ -0,0 +1,180 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awscloudwatchmetricsreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscloudwatchmetricsreceiver"
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"go.uber.org/multierr"
+)
+
+var (
+	defaultPollInterval = time.Minute
+)
+
+// Config is the overall config structure for the awscloudwatchmetricsreceiver
+type Config struct {
+	Region       string         `mapstructure:"region"`
+	Profile      string         `mapstructure:"profile"`
+	IMDSEndpoint string         `mapstructure:"imds_endpoint"`
+	PollInterval time.Duration  `mapstructure:"poll_interval"`
+	nilToZero    bool           `mapstrucuture:"nil_to_zero"` // Return 0 value if Cloudwatch returns no metrics at all. By default NaN will be reported
+	Metrics      *MetricsConfig `mapstructure:"metrics"`
+}
+
+// MetricsConfig is the configuration for the metrics part of the receiver
+// added this so we could expand to other inputs such as autodiscover
+type MetricsConfig struct {
+	Names []*NamedConfig `mapstructure:"named"`
+}
+
+// NamesConfig is the configuration for the metric namespace and metric names
+// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html
+type NamedConfig struct {
+	Namespace      string                   `mapstructure:"namespace"`
+	MetricName     string                   `mapstructure:"metric_name"`
+	Period         time.Duration            `mapstructure:"period"`
+	AwsAggregation string                   `mapstructure:"aws_aggregation"`
+	Dimensions     []MetricDimensionsConfig `mapstructure:"dimensions"`
+}
+
+// MetricDimensionConfig is the configuration for the metric dimensions
+type MetricDimensionsConfig struct {
+	Name  string `mapstructure:"Name"`
+	Value string `mapstructure:"Value"`
+}
+
+var (
+	errNoMetricsConfigured = errors.New("no metrics configured")
+	errNoRegion            = errors.New("no region was specified")
+	errInvalidPollInterval = errors.New("poll interval is incorrect, it must be a duration greater than one second")
+
+	// https://docs.aws.amazon.com/cli/latest/reference/cloudwatch/get-metric-data.html
+	// GetMetricData supports up to 500 metrics per API call
+	errTooManyMetrics = errors.New("too many metrics defined")
+
+	// https://docs.aws.amazon.com/cli/latest/reference/cloudwatch/get-metric-data.html
+	errEmptyDimensions      = errors.New("dimensions name and value is empty")
+	errTooManyDimensions    = errors.New("you cannot define more than 30 dimensions for a metric")
+	errDimensionColonPrefix = errors.New("dimension name cannot start with a colon")
+
+	errInvalidAwsAggregation = errors.New("invalid AWS aggregation")
+)
+
+func (cfg *Config) Validate() error {
+	if cfg.Region == "" {
+		return errNoRegion
+	}
+
+	if cfg.IMDSEndpoint != "" {
+		_, err := url.ParseRequestURI(cfg.IMDSEndpoint)
+		if err != nil {
+			return fmt.Errorf("unable to parse URI for imds_endpoint: %w", err)
+		}
+	}
+
+	if cfg.PollInterval < time.Second {
+		return errInvalidPollInterval
+	}
+	var errs error
+	errs = multierr.Append(errs, cfg.validateMetricsConfig())
+	return errs
+}
+
+func (cfg *Config) validateMetricsConfig() error {
+	if cfg.Metrics == nil {
+		return errNoMetricsConfigured
+	}
+	return cfg.validateNamedConfig()
+}
+
+func (cfg *Config) validateNamedConfig() error {
+	if cfg.Metrics.Names == nil {
+		return errNoMetricsConfigured
+	}
+	return cfg.validateDimensionsConfig()
+}
+
+func (cfg *Config) validateDimensionsConfig() error {
+	var errs error
+
+	metricsNames := cfg.Metrics.Names
+	if len(metricsNames) > 500 {
+		return errTooManyMetrics
+	}
+	for _, name := range metricsNames {
+		if name.Namespace == "" {
+			return errNoMetricsConfigured
+		}
+		err := validateAwsAggregation(name.AwsAggregation)
+		if err != nil {
+			return err
+		}
+		if name.MetricName == "" {
+			return errNoMetricsConfigured
+		}
+		errs = multierr.Append(errs, validate(name.Dimensions))
+	}
+	return errs
+}
+
+// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Statistics-definitions.html
+func validateAwsAggregation(agg string) error {
+	switch {
+	case agg == "SampleCount":
+		return nil
+	case agg == "Sum":
+		return nil
+	case agg == "Average":
+		return nil
+	case agg == "Minimum":
+		return nil
+	case agg == "Maximum":
+		return nil
+	case strings.HasPrefix(agg, "p"):
+		return nil
+	case strings.HasPrefix(agg, "TM"):
+		return nil
+	case agg == "IQM":
+		return nil
+	case strings.HasPrefix(agg, "PR"):
+		return nil
+	case strings.HasPrefix(agg, "TC"):
+		return nil
+	case strings.HasPrefix(agg, "TS"):
+		return nil
+	default:
+		return errInvalidAwsAggregation
+	}
+}
+
+func validate(nmd []MetricDimensionsConfig) error {
+	for _, dimensionConfig := range nmd {
+		if dimensionConfig.Name == "" || dimensionConfig.Value == "" {
+			return errEmptyDimensions
+		}
+		if strings.HasPrefix(dimensionConfig.Name, ":") {
+			return errDimensionColonPrefix
+		}
+	}
+	if len(nmd) > 30 {
+		return errTooManyDimensions
+	}
+	return nil
+}