From ed181c24145b9964299832b11bdc40708cdd662d Mon Sep 17 00:00:00 2001 From: Lewis Robbins Date: Sat, 18 Feb 2023 22:09:24 +0000 Subject: [PATCH] Start on tests for config --- .../awscloudwatchmetricsreceiver/README.md | 31 ++++-- .../awscloudwatchmetricsreceiver/config.go | 104 ++++++++++++++++-- .../config_test.go | 34 +++++- .../awscloudwatchmetricsreceiver/factory.go | 1 + receiver/awscloudwatchmetricsreceiver/go.mod | 6 +- receiver/awscloudwatchmetricsreceiver/go.sum | 8 ++ 6 files changed, 165 insertions(+), 19 deletions(-) diff --git a/receiver/awscloudwatchmetricsreceiver/README.md b/receiver/awscloudwatchmetricsreceiver/README.md index cb5027f6b3aa..c752a6e43498 100644 --- a/receiver/awscloudwatchmetricsreceiver/README.md +++ b/receiver/awscloudwatchmetricsreceiver/README.md @@ -20,7 +20,7 @@ This receiver uses the [AWS SDK](https://docs.aws.amazon.com/sdk-for-go/v1/devel | --------------- | ---------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `region` | *required* | string | The AWS recognized region string | | `profile` | *optional* | string | The AWS profile used to authenticate, if none is specified the default is chosen from the list of profiles | -| `imds_endpoint` | *optional* | string | A way of specifying a custom URL to be used by the EC2 IMDS client to validate the session. If unset, and the environment variable `AWS_EC2_METADATA_SERVICE_ENDPOINT` has a value the client will use the value of the environment variable as the endpoint for operation calls. | + | `metrics` | *optional* | `Metrics` | Configuration for metrics ingestion of this receiver | ### Metrics Parameters @@ -28,8 +28,7 @@ This receiver uses the [AWS SDK](https://docs.aws.amazon.com/sdk-for-go/v1/devel | Parameter | Notes | type | Description | | ------------------------ | ------------ | ---------------------- | ------------------------------------------------------------------------------------------ | | `poll_interval` | `default=1m` | duration | The duration waiting in between requests. | -| `max_events_per_request` | `default=50` | int | The maximum number of events to process per request to Cloudwatch | -| `groups` | *optional* | `See Group Parameters` | Configuration for Log Groups, by default all Log Groups and Log Streams will be collected. | +| `named` | *optional* | `See Group Parameters` | Configuration for Log Groups, by default no metrics will be collected | ### Group Parameters @@ -46,16 +45,34 @@ This receiver uses the [AWS SDK](https://docs.aws.amazon.com/sdk-for-go/v1/devel ```yaml awscloudwatchmetrics: - region: us-west-1 - poll_interval: 5m + region: us-east-1 + poll_interval: 1m metrics: named: - - namespace: AWS/EC2: - metric_names: [DiskWriteOps,DiskReadBytes] + - namespace: "AWS/EC2" + metric_name: "CPUUtilization" + period: "5m" + aws_aggregation: "Sum" + dimensions: + - Name: "InstanceId" + Value: "i-1234567890abcdef0" + - namespace: "AWS/S3" + metric_name: "BucketSizeBytes" + period: "5m" + aws_aggregation: "p99" + dimensions: + - Name: "BucketName" + Value: "OpenTelemetry" + - Name: "StorageType" + Value: "StandardStorage" ``` ## Sample Configs +## AWS Costs + +This receiver uses the [GetMetricData](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html) API call, this call is *not* in the AWS free tier. Please refer to [Amazon's pricing](https://aws.amazon.com/cloudwatch/pricing/) for futher information about expected costs. For `us-east-1`, the current pricing is $0.01 per 1,000 metrics requested as of February 2023. + [alpha]:https://github.com/open-telemetry/opentelemetry-collector#alpha [contrib]:https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib diff --git a/receiver/awscloudwatchmetricsreceiver/config.go b/receiver/awscloudwatchmetricsreceiver/config.go index bfb65f303dbc..e73c398774da 100644 --- a/receiver/awscloudwatchmetricsreceiver/config.go +++ b/receiver/awscloudwatchmetricsreceiver/config.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "net/url" + "strings" "time" "go.uber.org/multierr" @@ -33,25 +34,46 @@ type Config struct { Profile string `mapstructure:"profile"` IMDSEndpoint string `mapstructure:"imds_endpoint"` PollInterval time.Duration `mapstructure:"poll_interval"` + nilToZero bool `mapstrucuture:"nil_to_zero"` // Return 0 value if Cloudwatch returns no metrics at all. By default NaN will be reported Metrics *MetricsConfig `mapstructure:"metrics"` } // MetricsConfig is the configuration for the metrics part of the receiver +// added this so we could expand to other inputs such as autodiscover type MetricsConfig struct { - Names []NamesConfig `mapstructure:"named"` + Names []*NamedConfig `mapstructure:"named"` } // NamesConfig is the configuration for the metric namespace and metric names // https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html -type NamesConfig struct { - Namespace string `mapstructure:"namespace"` - MetricNames []*string `mapstructure:"metric_names"` +type NamedConfig struct { + Namespace string `mapstructure:"namespace"` + MetricName string `mapstructure:"metric_name"` + Period time.Duration `mapstructure:"period"` + AwsAggregation string `mapstructure:"aws_aggregation"` + Dimensions []MetricDimensionsConfig `mapstructure:"dimensions"` +} + +type MetricDimensionsConfig struct { + Name string `mapstructure:"Name"` + Value string `mapstructure:"Value"` } var ( errNoMetricsConfigured = errors.New("no metrics configured") errNoRegion = errors.New("no region was specified") errInvalidPollInterval = errors.New("poll interval is incorrect, it must be a duration greater than one second") + + // https://docs.aws.amazon.com/cli/latest/reference/cloudwatch/get-metric-data.html + // GetMetricData supports up to 500 metrics per API call + errTooManyMetrics = errors.New("too many metrics defined") + + // https://docs.aws.amazon.com/cli/latest/reference/cloudwatch/get-metric-data.html + errEmptyDimensions = errors.New("dimensions name and value is empty") + errTooManyDimensions = errors.New("you cannot define more than 30 dimensions for a metric") + errDimensionColonPrefix = errors.New("dimension name cannot start with a colon") + + errInvalidAwsAggregation = errors.New("invalid AWS aggregation") ) func (cfg *Config) Validate() error { @@ -78,18 +100,80 @@ func (cfg *Config) validateMetricsConfig() error { if cfg.Metrics == nil { return errNoMetricsConfigured } - return validate(cfg.Metrics.Names) + return cfg.validateNamedConfig() } -func validate(g []NamesConfig) error { - for _, metric := range g { - if metric.Namespace == "" { +func (cfg *Config) validateNamedConfig() error { + if cfg.Metrics.Names == nil { + return errNoMetricsConfigured + } + return cfg.validateDimensionsConfig() +} + +func (cfg *Config) validateDimensionsConfig() error { + var errs error + + metricsNames := cfg.Metrics.Names + if len(metricsNames) > 500 { + return errTooManyMetrics + } + for _, name := range metricsNames { + if name.Namespace == "" { return errNoMetricsConfigured } - - if len(metric.MetricNames) <= 0 { + err := validateAwsAggregation(name.AwsAggregation) + if err != nil { + return err + } + if name.MetricName == "" { return errNoMetricsConfigured } + errs = multierr.Append(errs, validate(name.Dimensions)) + } + return errs +} + +// https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Statistics-definitions.html +func validateAwsAggregation(agg string) error { + switch { + case agg == "SampleCount": + return nil + case agg == "Sum": + return nil + case agg == "Average": + return nil + case agg == "Minimum": + return nil + case agg == "Maximum": + return nil + case strings.HasPrefix(agg, "p"): + return nil + case strings.HasPrefix(agg, "TM"): + return nil + case agg == "IQM": + return nil + case strings.HasPrefix(agg, "PR"): + return nil + case strings.HasPrefix(agg, "TC"): + return nil + case strings.HasPrefix(agg, "TS"): + return nil + default: + return errInvalidAwsAggregation + } +} + +func validate(nmd []MetricDimensionsConfig) error { + for _, dimensionConfig := range nmd { + if dimensionConfig.Name == "" || dimensionConfig.Value == "" { + return errEmptyDimensions + } + if strings.HasPrefix(dimensionConfig.Name, ":") { + return errDimensionColonPrefix + } + } + if len(nmd) > 30 { + return errTooManyDimensions } return nil } diff --git a/receiver/awscloudwatchmetricsreceiver/config_test.go b/receiver/awscloudwatchmetricsreceiver/config_test.go index 89d49b346511..08c4d2a8181a 100644 --- a/receiver/awscloudwatchmetricsreceiver/config_test.go +++ b/receiver/awscloudwatchmetricsreceiver/config_test.go @@ -12,4 +12,36 @@ // See the License for the specific language governing permissions and // limitations under the License. -package awscloudwatchmetricsreceiver_test +package awscloudwatchmetricsreceiver + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidate(t *testing.T) { + cases := []struct { + name string + config Config + expectedErr error + }{ + { + name: "Invalid region", + config: Config{ + Region: "", + }, + expectedErr: errNoRegion, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := tc.config.Validate() + if tc.expectedErr != nil { + require.ErrorContains(t, err, tc.expectedErr.Error()) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/receiver/awscloudwatchmetricsreceiver/factory.go b/receiver/awscloudwatchmetricsreceiver/factory.go index a0ff50d7f2f6..4516b0657214 100644 --- a/receiver/awscloudwatchmetricsreceiver/factory.go +++ b/receiver/awscloudwatchmetricsreceiver/factory.go @@ -43,6 +43,7 @@ func createMetricsRceiver(_ context.Context, params receiver.CreateSettings, bas func createDefaultConfig() component.Config { return &Config{ PollInterval: defaultPollInterval, + nilToZero: false, Metrics: &MetricsConfig{}, } } diff --git a/receiver/awscloudwatchmetricsreceiver/go.mod b/receiver/awscloudwatchmetricsreceiver/go.mod index 537f7d364065..a0786a5b2209 100644 --- a/receiver/awscloudwatchmetricsreceiver/go.mod +++ b/receiver/awscloudwatchmetricsreceiver/go.mod @@ -3,13 +3,16 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsclo go 1.19 require ( + github.com/stretchr/testify v1.8.1 go.opentelemetry.io/collector v0.71.0 go.opentelemetry.io/collector/component v0.71.0 go.opentelemetry.io/collector/consumer v0.71.0 go.uber.org/multierr v1.9.0 + go.uber.org/zap v1.24.0 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -19,6 +22,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/collector/confmap v0.71.0 // indirect go.opentelemetry.io/collector/featuregate v0.71.0 // indirect go.opentelemetry.io/collector/pdata v1.0.0-rc5 // indirect @@ -26,11 +30,11 @@ require ( go.opentelemetry.io/otel/metric v0.36.0 // indirect go.opentelemetry.io/otel/trace v1.13.0 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/zap v1.24.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sys v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect google.golang.org/grpc v1.52.3 // indirect google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/receiver/awscloudwatchmetricsreceiver/go.sum b/receiver/awscloudwatchmetricsreceiver/go.sum index 2411835a46eb..5c1b751649ba 100644 --- a/receiver/awscloudwatchmetricsreceiver/go.sum +++ b/receiver/awscloudwatchmetricsreceiver/go.sum @@ -150,8 +150,10 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -232,12 +234,17 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -410,6 +417,7 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=