diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0e7edfce6f2..2b75b0e9005 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -582,6 +582,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Honor kube event resysncs to handle missed watch events {pull}22668[22668] - Add autodiscover provider and metadata processor for Nomad. {pull}14954[14954] {pull}23324[23324] - Add `processors.rate_limit.n.dropped` monitoring counter metric for the `rate_limit` processor. {pull}23330[23330] +- Deprecate aws_partition config parameter for AWS, use endpoint instead. {pull}23539[23539] - Update the baseline version of Sarama (Kafka support library) to 1.27.2. {pull}23595[23595] - Add kubernetes.volume.fs.used.pct field. {pull}23564[23564] diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index e02a7a81460..085436b05d9 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -31,7 +31,7 @@ Please see <> for more details. This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available -AWS regions. +AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter. * *latency* @@ -40,6 +40,27 @@ process larger than Metricbeat collection period. This case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *endpoint* + +Most AWS services offer a regional endpoint that can be used to make requests. +The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. +Some services, such as IAM, do not support regions. The endpoints for these +services do not include a region. In `aws` module, `endpoint` config is to set +the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, +`sc2s.sgov.gov`. + +If endpoint is specified, `regions` config becomes required. For example: + +[source,yaml] +---- +- module: aws + period: 5m + endpoint: amazonaws.com.cn + regions: cn-north-1 + metricsets: + - ec2 +---- + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index 584d306b1c7..98d8c60d77b 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -113,6 +113,7 @@ func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3C log.Debug("s3 service name = ", s3Servicename) log.Debug("s3 input config max_number_of_messages = ", in.config.MaxNumberOfMessages) + log.Debug("s3 input config endpoint = ", in.config.AwsConfig.Endpoint) return &s3Collector{ cancellation: ctxtool.FromCanceller(ctx.Cancelation), logger: log, diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 97d06a6a255..fafe8e7336b 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/logp" ) @@ -24,7 +25,7 @@ type ConfigAWS struct { SharedCredentialFile string `config:"shared_credential_file"` Endpoint string `config:"endpoint"` RoleArn string `config:"role_arn"` - AWSPartition string `config:"aws_partition"` + AWSPartition string `config:"aws_partition"` // Deprecated. } // GetAWSCredentials function gets aws credentials from the config. @@ -104,3 +105,11 @@ func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName } return awsConfig } + +// Validate checks for deprecated config option +func (c ConfigAWS) Validate() error { + if c.AWSPartition != "" { + cfgwarn.Deprecate("8.0.0", "aws_partition is deprecated. Please use endpoint instead.") + } + return nil +} diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index a9cb4ab8e88..a3a242554b4 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -9,9 +9,14 @@ To configure AWS credentials, either put the credentials into the {beatname_uc} * *session_token*: required when using temporary security credentials. * *credential_profile_name*: profile name in shared credentials file. * *shared_credential_file*: directory of the shared credentials file. -* *endpoint*: URL of the entry point for an AWS web service. * *role_arn*: AWS IAM Role to assume. -* *aws_partition*: AWS region parttion name, value is one of `aws, aws-cn, aws-us-gov`, default is `aws`. +* *endpoint*: URL of the entry point for an AWS web service. +Most AWS services offer a regional endpoint that can be used to make requests. +The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. +Some services, such as IAM, do not support regions. The endpoints for these +services do not include a region. In `aws` module, `endpoint` config is to set +the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, +`sc2s.sgov.gov`. [float] ==== Supported Formats diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index e4e55e82136..df18966b2af 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -23,7 +23,7 @@ Please see <> for more details. This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available -AWS regions. +AWS regions. If `endpoint` is specified, `regions` becomes a required config parameter. * *latency* @@ -32,6 +32,27 @@ process larger than Metricbeat collection period. This case, please specify a `latency` parameter so collection start time and end time will be shifted by the given latency amount. +* *endpoint* + +Most AWS services offer a regional endpoint that can be used to make requests. +The general syntax of a regional endpoint is `protocol://service-code.region-code.endpoint-code`. +Some services, such as IAM, do not support regions. The endpoints for these +services do not include a region. In `aws` module, `endpoint` config is to set +the `endpoint-code` part, such as `amazonaws.com`, `amazonaws.com.cn`, `c2s.ic.gov`, +`sc2s.sgov.gov`. + +If endpoint is specified, `regions` config becomes required. For example: + +[source,yaml] +---- +- module: aws + period: 5m + endpoint: amazonaws.com.cn + regions: cn-north-1 + metricsets: + - ec2 +---- + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 167e6a088a0..f3a7caf6cc8 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -6,6 +6,7 @@ package aws import ( "context" + "fmt" "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" @@ -16,7 +17,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go-v2/service/sts" - "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" @@ -78,20 +78,22 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { awsConfig, err := awscommon.GetAWSCredentials(config.AWSConfig) if err != nil { - return nil, errors.Wrap(err, "failed to get aws credentials, please check AWS credential in config") + return nil, fmt.Errorf("failed to get aws credentials, please check AWS credential in config: %w", err) } _, err = awsConfig.Credentials.Retrieve() if err != nil { - return nil, errors.Wrap(err, "failed to retrieve aws credentials, please check AWS credential in config") + return nil, fmt.Errorf("failed to retrieve aws credentials, please check AWS credential in config: %w", err) } + base.Logger().Debug("aws config endpoint = ", config.AWSConfig.Endpoint) metricSet := MetricSet{ BaseMetricSet: base, Period: config.Period, Latency: config.Latency, AwsConfig: &awsConfig, TagsFilter: config.TagsFilter, + Endpoint: config.AWSConfig.Endpoint, } base.Logger().Debug("Metricset level config for period: ", metricSet.Period) @@ -100,6 +102,9 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // Get IAM account name, set region by aws_partition, default is aws global partition // refer https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html + if config.AWSConfig.AWSPartition != "" && config.AWSConfig.Endpoint != "" { + base.Logger().Warn("aws_partition is deprecated. Please use endpoint and regions instead.") + } switch config.AWSConfig.AWSPartition { case "aws-cn": awsConfig.Region = "cn-north-1" @@ -109,6 +114,11 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { awsConfig.Region = "us-east-1" } + // If regions in config is not empty, then overwrite the awsConfig.Region + if len(config.Regions) > 0 { + awsConfig.Region = config.Regions[0] + } + // Get IAM account id svcSts := sts.New(awscommon.EnrichAWSConfigWithEndpoint( config.AWSConfig.Endpoint, "sts", "", awsConfig)) @@ -151,7 +161,7 @@ func getRegions(svc ec2iface.ClientAPI) (completeRegionsList []string, err error req := svc.DescribeRegionsRequest(input) output, err := req.Send(context.TODO()) if err != nil { - err = errors.Wrap(err, "Failed DescribeRegions") + err = fmt.Errorf("failed DescribeRegions: %w", err) return } for _, region := range output.Regions {