From 6ac480f1d700aa22723cb2270d1447ffbb04f7d4 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 11 Feb 2020 16:38:33 -0700 Subject: [PATCH 1/8] Add endpoint into AWS config --- x-pack/libbeat/common/aws/credentials.go | 18 +++++++++++++++++- .../docs/aws-credentials-config.asciidoc | 9 +++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 3a3bf5b80e61..d0286493e7ac 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -17,6 +17,7 @@ type ConfigAWS struct { SessionToken string `config:"session_token"` ProfileName string `config:"credential_profile_name"` SharedCredentialFile string `config:"shared_credential_file"` + Endpoint string `config:"endpoint"` } // GetAWSCredentials function gets aws credentials from the config. @@ -39,6 +40,11 @@ func GetAWSCredentials(config ConfigAWS) (awssdk.Config, error) { awsConfig.Credentials = awssdk.StaticCredentialsProvider{ Value: awsCredentials, } + + // check if endpoint is given from configuration + if config.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL(config.Endpoint) + } return awsConfig, nil } @@ -57,5 +63,15 @@ func GetAWSCredentials(config ConfigAWS) (awssdk.Config, error) { if config.SharedCredentialFile != "" { options = append(options, external.WithSharedConfigFiles([]string{config.SharedCredentialFile})) } - return external.LoadDefaultAWSConfig(options...) + + awsConfig, err := external.LoadDefaultAWSConfig(options...) + if err != nil { + return awsConfig, err + } + + // check if endpoint is given from configuration + if config.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL(config.Endpoint) + } + return awsConfig, nil } diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index a83da149b06f..33deb5a09414 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -2,6 +2,15 @@ === AWS Credentials Configuration To configure AWS credentials, either put the credentials into the {beatname_uc} configuration, or use a shared credentials file, as shown in the following examples. +[float] +==== Configuration parameters +* *access_key_id*: first part of access key. +* *secret_access_key*: second part of access key. +* *session_token*: required when using temporary security credentials. +* *credential_profile_name*: profile name in shared credentials file. +* *shared_credential_file*: directory of the shared credentials file. +* *endpoint*: URL of the entry point for an AWS web service. + [float] ==== Supported Formats * Use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or `AWS_SESSION_TOKEN` From c8fed1926d693dc6eb6dcc7a244712ec806d6fc8 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 12 Feb 2020 16:30:33 -0700 Subject: [PATCH 2/8] add EndpointResolver for aws config --- x-pack/libbeat/common/aws/credentials.go | 1 + x-pack/metricbeat/module/aws/aws.go | 24 +++++++++++++++++++ .../module/aws/cloudwatch/cloudwatch.go | 13 ++++++++++ x-pack/metricbeat/module/aws/ec2/ec2.go | 15 ++++++++++-- x-pack/metricbeat/module/aws/rds/rds.go | 15 ++++++++++-- .../aws/s3_daily_storage/s3_daily_storage.go | 7 ++++++ .../module/aws/s3_request/s3_request.go | 7 ++++++ x-pack/metricbeat/module/aws/sqs/sqs.go | 11 +++++++++ 8 files changed, 89 insertions(+), 4 deletions(-) diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index d0286493e7ac..c5d2bd59ba28 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -18,6 +18,7 @@ type ConfigAWS struct { ProfileName string `config:"credential_profile_name"` SharedCredentialFile string `config:"shared_credential_file"` Endpoint string `config:"endpoint"` + EndpointRegion string `config:"endpoint_region"` } // GetAWSCredentials function gets aws credentials from the config. diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index a4bba395bc58..960257dbe8e6 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -32,6 +32,7 @@ type Config struct { type MetricSet struct { mb.BaseMetricSet RegionsList []string + Endpoint string Period time.Duration AwsConfig *awssdk.Config AccountName string @@ -86,6 +87,12 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { } // Get IAM account name + // check if endpoint is given from configuration + if config.AWSConfig.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://iam." + config.AWSConfig.Endpoint) + } + + awsConfig.Region = "us-east-1" svcIam := iam.New(awsConfig) req := svcIam.ListAccountAliasesRequest(&iam.ListAccountAliasesInput{}) output, err := req.Send(context.TODO()) @@ -100,6 +107,10 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { } // Get IAM account id + // check if endpoint is given from configuration + if config.AWSConfig.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sts." + config.AWSConfig.Endpoint) + } svcSts := sts.New(awsConfig) reqIdentity := svcSts.GetCallerIdentityRequest(&sts.GetCallerIdentityInput{}) outputIdentity, err := reqIdentity.Send(context.TODO()) @@ -109,6 +120,19 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { metricSet.AccountID = *outputIdentity.Account } + // Construct MetricSet when endpoint is given + // When endpoint is given, use endpoint_region as RegionsList. + if config.AWSConfig.Endpoint != "" { + // When use an endpoint with no Region, AWS routes the request to US East + // (N. Virginia) (us-east-1), which is the default Region for API calls. + metricSet.RegionsList = []string{"us-east-1"} + if config.AWSConfig.EndpointRegion != "" { + metricSet.RegionsList = []string{config.AWSConfig.EndpointRegion} + } + + return &metricSet, nil + } + // Construct MetricSet with a full regions list if config.Regions == nil { // set default region to make initial aws api call diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 40be8a846cdd..35aee4872bd9 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -11,6 +11,7 @@ import ( "strings" "time" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" @@ -136,6 +137,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + } + svcCloudwatch := cloudwatch.New(awsConfig) svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) @@ -154,6 +161,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + } + svcCloudwatch := cloudwatch.New(awsConfig) svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 41770c495ab1..b1a75e1e50d3 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -11,13 +11,13 @@ import ( "strings" "time" - "github.com/elastic/beats/libbeat/common" - + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -91,6 +91,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + m.Endpoint) + } + svcEC2 := ec2.New(awsConfig) instanceIDs, instancesOutputs, err := getInstancesPerRegion(svcEC2) if err != nil { @@ -100,6 +106,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + } + svcCloudwatch := cloudwatch.New(awsConfig) namespace := "AWS/EC2" listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index e2f2d1eb7f53..48bdde998722 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -11,10 +11,10 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" - + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" "github.com/pkg/errors" "github.com/elastic/beats/metricbeat/mb" @@ -82,6 +82,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://rds." + m.Endpoint) + } + svc := rds.New(awsConfig) // Get DBInstance IDs per region dbInstanceIDs, dbDetailsMap, err := getDBInstancesPerRegion(svc) @@ -96,6 +102,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + } + svcCloudwatch := cloudwatch.New(awsConfig) namespace := "AWS/RDS" listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index de9d2574380e..4e983f84ea7a 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -10,6 +10,7 @@ import ( "strings" "time" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/pkg/errors" @@ -74,6 +75,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + } + svcCloudwatch := cloudwatch.New(awsConfig) listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index 5474cbc0117b..a9ce5a82dd29 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -10,6 +10,7 @@ import ( "strings" "time" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/pkg/errors" @@ -74,6 +75,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + } + svcCloudwatch := cloudwatch.New(awsConfig) listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { diff --git a/x-pack/metricbeat/module/aws/sqs/sqs.go b/x-pack/metricbeat/module/aws/sqs/sqs.go index 5e70cdb0b1b1..f9e40d11acaa 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs.go @@ -11,6 +11,7 @@ import ( "strings" "time" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface" @@ -71,7 +72,17 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + } svcCloudwatch := cloudwatch.New(awsConfig) + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sqs." + m.Endpoint) + } svcSQS := sqs.New(awsConfig) // Get queueUrls for each region From 05bcc71b28d4f4c95aa86ef93f16f23446672c78 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 13 Feb 2020 09:54:52 -0700 Subject: [PATCH 3/8] Update endpoints with region name --- x-pack/metricbeat/module/aws/aws.go | 1 + x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go | 9 +++++++-- x-pack/metricbeat/module/aws/ec2/ec2.go | 4 ++-- x-pack/metricbeat/module/aws/rds/rds.go | 4 ++-- .../module/aws/s3_daily_storage/s3_daily_storage.go | 2 +- x-pack/metricbeat/module/aws/s3_request/s3_request.go | 2 +- x-pack/metricbeat/module/aws/sqs/sqs.go | 4 ++-- 7 files changed, 16 insertions(+), 10 deletions(-) diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 960257dbe8e6..5fe7f586a8b0 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -89,6 +89,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // Get IAM account name // check if endpoint is given from configuration if config.AWSConfig.Endpoint != "" { + metricSet.Endpoint = config.AWSConfig.Endpoint awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://iam." + config.AWSConfig.Endpoint) } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 35aee4872bd9..67125e73d545 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -140,10 +140,15 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) } svcCloudwatch := cloudwatch.New(awsConfig) + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://resourcegroupstaggingapi." + awsConfig.Region + "." + m.Endpoint) + } svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) eventsWithIdentifier, eventsNoIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, regionName, startTime, endTime) @@ -164,7 +169,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) } svcCloudwatch := cloudwatch.New(awsConfig) diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index b1a75e1e50d3..c6995c9c952e 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -94,7 +94,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + awsConfig.Region + "." + m.Endpoint) } svcEC2 := ec2.New(awsConfig) @@ -108,7 +108,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) } svcCloudwatch := cloudwatch.New(awsConfig) diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index 48bdde998722..fce4b3c8c30d 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -85,7 +85,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://rds." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://rds." + awsConfig.Region + "." + m.Endpoint) } svc := rds.New(awsConfig) @@ -104,7 +104,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) } svcCloudwatch := cloudwatch.New(awsConfig) diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index 4e983f84ea7a..4ddcc447e8f1 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -78,7 +78,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) } svcCloudwatch := cloudwatch.New(awsConfig) diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index a9ce5a82dd29..4a33926ff381 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -78,7 +78,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) } svcCloudwatch := cloudwatch.New(awsConfig) diff --git a/x-pack/metricbeat/module/aws/sqs/sqs.go b/x-pack/metricbeat/module/aws/sqs/sqs.go index f9e40d11acaa..7ed5be7a5e70 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs.go @@ -75,13 +75,13 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) } svcCloudwatch := cloudwatch.New(awsConfig) // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sqs." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sqs." + awsConfig.Region + "." + m.Endpoint) } svcSQS := sqs.New(awsConfig) From 8f440083b643ef887e0c63d71255392a9a0cebdb Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 13 Feb 2020 12:01:38 -0700 Subject: [PATCH 4/8] replace endpoint_region with regions config option --- x-pack/libbeat/common/aws/credentials.go | 1 - x-pack/metricbeat/module/aws/aws.go | 16 +--------------- .../module/aws/cloudwatch/cloudwatch.go | 12 ++++++++---- x-pack/metricbeat/module/aws/ec2/ec2.go | 9 ++++----- 4 files changed, 13 insertions(+), 25 deletions(-) diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index c5d2bd59ba28..d0286493e7ac 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -18,7 +18,6 @@ type ConfigAWS struct { ProfileName string `config:"credential_profile_name"` SharedCredentialFile string `config:"shared_credential_file"` Endpoint string `config:"endpoint"` - EndpointRegion string `config:"endpoint_region"` } // GetAWSCredentials function gets aws credentials from the config. diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index 5fe7f586a8b0..b5937a0bc77d 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -121,23 +121,9 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { metricSet.AccountID = *outputIdentity.Account } - // Construct MetricSet when endpoint is given - // When endpoint is given, use endpoint_region as RegionsList. - if config.AWSConfig.Endpoint != "" { - // When use an endpoint with no Region, AWS routes the request to US East - // (N. Virginia) (us-east-1), which is the default Region for API calls. - metricSet.RegionsList = []string{"us-east-1"} - if config.AWSConfig.EndpointRegion != "" { - metricSet.RegionsList = []string{config.AWSConfig.EndpointRegion} - } - - return &metricSet, nil - } - // Construct MetricSet with a full regions list if config.Regions == nil { - // set default region to make initial aws api call - awsConfig.Region = "us-west-1" + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + config.AWSConfig.Endpoint) svcEC2 := ec2.New(awsConfig) completeRegionsList, err := getRegions(svcEC2) if err != nil { diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 67125e73d545..540b5f5536d9 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -140,14 +140,14 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + regionName + "." + m.Endpoint) } svcCloudwatch := cloudwatch.New(awsConfig) // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://resourcegroupstaggingapi." + awsConfig.Region + "." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://tagging." + regionName + "." + m.Endpoint) } svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) @@ -169,10 +169,14 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + regionName + "." + m.Endpoint) } - svcCloudwatch := cloudwatch.New(awsConfig) + + // check if endpoint is given from configuration + if m.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://tagging." + regionName + "." + m.Endpoint) + } svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) // Create events based on namespaceDetailTotal from configuration diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index c6995c9c952e..6aa8ae8897da 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -94,10 +94,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + awsConfig.Region + "." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + regionName + "." + m.Endpoint) } - svcEC2 := ec2.New(awsConfig) + instanceIDs, instancesOutputs, err := getInstancesPerRegion(svcEC2) if err != nil { err = errors.Wrap(err, "getInstancesPerRegion failed, skipping region "+regionName) @@ -108,10 +108,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // check if endpoint is given from configuration if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + regionName + "." + m.Endpoint) } - svcCloudwatch := cloudwatch.New(awsConfig) + namespace := "AWS/EC2" listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { @@ -142,7 +142,6 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Create Cloudwatch Events for EC2 events, err := m.createCloudWatchEvents(metricDataOutput, instancesOutputs, regionName) - if err != nil { m.Logger().Error(err.Error()) report.Error(err) From 636a9125aebf2b24394a85293b0ffdbde1d26c8c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 13 Feb 2020 12:54:20 -0700 Subject: [PATCH 5/8] Add endpoint to filebeat s3 input and aws module --- filebeat/docs/modules/aws.asciidoc | 29 ++++++++++++------- x-pack/filebeat/filebeat.reference.yml | 12 ++++++++ x-pack/filebeat/input/s3/input.go | 10 +++++++ x-pack/filebeat/module/aws/_meta/config.yml | 12 ++++++++ .../filebeat/module/aws/_meta/docs.asciidoc | 29 ++++++++++++------- .../aws/cloudtrail/config/cloudtrail.yml | 4 +++ .../module/aws/cloudtrail/manifest.yml | 1 + x-pack/filebeat/module/aws/elb/config/s3.yml | 4 +++ x-pack/filebeat/module/aws/elb/manifest.yml | 1 + .../module/aws/s3access/config/s3.yml | 4 +++ .../filebeat/module/aws/s3access/manifest.yml | 1 + .../module/aws/vpcflow/config/input.yml | 4 +++ .../filebeat/module/aws/vpcflow/manifest.yml | 1 + x-pack/filebeat/modules.d/aws.yml.disabled | 12 ++++++++ 14 files changed, 102 insertions(+), 22 deletions(-) diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index 79daf4a1865a..e8e971b66901 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -33,18 +33,12 @@ Example config: - module: aws s3access: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com elb: enabled: false @@ -104,6 +98,19 @@ Filename of AWS credential file. AWS credential profile name. +*`var.visibility_timeout`*:: + +The duration that the received messages are hidden from ReceiveMessage request. +Default to be 300 seconds. + +*`var.api_timeout`*:: + +Maximum duration before AWS API request will be interrupted. Default to be 120 seconds. + +*`var.endpoint`*:: + +Custom endpoint used to access AWS APIs. + [float] === cloudtrail fileset diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 03739cb0122b..adad505bb02a 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -119,6 +119,9 @@ filebeat.modules: # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + elb: enabled: false @@ -142,6 +145,9 @@ filebeat.modules: # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + vpcflow: enabled: false @@ -165,6 +171,9 @@ filebeat.modules: # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + cloudtrail: enabled: false @@ -188,6 +197,9 @@ filebeat.modules: # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + #-------------------------------- Azure Module -------------------------------- - module: azure # All logs diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index dd254cffebf2..b0c194b8b46b 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -181,7 +181,17 @@ func (p *s3Input) Run() { awsConfig := p.awsConfig.Copy() awsConfig.Region = regionName + + // check if endpoint is given from configuration + if p.config.AwsConfig.Endpoint!= "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sqs." + regionName + "."+ p.config.AwsConfig.Endpoint) + } svcSQS := sqs.New(awsConfig) + + // check if endpoint is given from configuration + if p.config.AwsConfig.Endpoint!= "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://s3." + regionName + "."+ p.config.AwsConfig.Endpoint) + } svcS3 := s3.New(awsConfig) p.workerWg.Add(1) diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index e87956f950ce..ee54cc545586 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -22,6 +22,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + elb: enabled: false @@ -45,6 +48,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + vpcflow: enabled: false @@ -68,6 +74,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + cloudtrail: enabled: false @@ -90,3 +99,6 @@ # Maximum duration before AWS API request will be interrupted # Default to be 120s #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index 552b90c20d54..e222f55d23f5 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -28,18 +28,12 @@ Example config: - module: aws s3access: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com elb: enabled: false @@ -99,6 +93,19 @@ Filename of AWS credential file. AWS credential profile name. +*`var.visibility_timeout`*:: + +The duration that the received messages are hidden from ReceiveMessage request. +Default to be 300 seconds. + +*`var.api_timeout`*:: + +Maximum duration before AWS API request will be interrupted. Default to be 120 seconds. + +*`var.endpoint`*:: + +Custom endpoint used to access AWS APIs. + [float] === cloudtrail fileset diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml b/x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml index 87219497aef3..18087c4769b6 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml @@ -20,6 +20,10 @@ visibility_timeout: {{ .visibility_timeout }} api_timeout: {{ .api_timeout }} {{ end }} +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} + {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index d809dec5fb51..5e55c965175f 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -7,6 +7,7 @@ var: - name: credential_profile_name - name: visibility_timeout - name: api_timeout + - name: endpoint ingest_pipeline: ingest/pipeline.yml input: config/cloudtrail.yml diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/s3.yml index 40212723e968..2af7bebff30c 100644 --- a/x-pack/filebeat/module/aws/elb/config/s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/s3.yml @@ -16,3 +16,7 @@ visibility_timeout: {{ .visibility_timeout }} {{ if .api_timeout }} api_timeout: {{ .api_timeout }} {{ end }} + +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml index 01857d90643d..f5a3befce10d 100644 --- a/x-pack/filebeat/module/aws/elb/manifest.yml +++ b/x-pack/filebeat/module/aws/elb/manifest.yml @@ -7,6 +7,7 @@ var: - name: credential_profile_name - name: visibility_timeout - name: api_timeout + - name: endpoint ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/s3.yml index 40212723e968..2af7bebff30c 100644 --- a/x-pack/filebeat/module/aws/s3access/config/s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/s3.yml @@ -16,3 +16,7 @@ visibility_timeout: {{ .visibility_timeout }} {{ if .api_timeout }} api_timeout: {{ .api_timeout }} {{ end }} + +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml index b4fdbb7ca49f..ce14df8f1711 100644 --- a/x-pack/filebeat/module/aws/s3access/manifest.yml +++ b/x-pack/filebeat/module/aws/s3access/manifest.yml @@ -7,6 +7,7 @@ var: - name: credential_profile_name - name: visibility_timeout - name: api_timeout + - name: endpoint ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index 1838d695eb4a..af77a19e14f9 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -19,6 +19,10 @@ visibility_timeout: {{ .visibility_timeout }} api_timeout: {{ .api_timeout }} {{ end }} +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} + {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml index e438f1e91c30..27d8c8f126ea 100644 --- a/x-pack/filebeat/module/aws/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/aws/vpcflow/manifest.yml @@ -7,6 +7,7 @@ var: - name: credential_profile_name - name: visibility_timeout - name: api_timeout + - name: endpoint ingest_pipeline: ingest/pipeline.yml input: config/input.yml diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index 689f697e0237..47cead91df5b 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -25,6 +25,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + elb: enabled: false @@ -48,6 +51,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + vpcflow: enabled: false @@ -71,6 +77,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + cloudtrail: enabled: false @@ -93,3 +102,6 @@ # Maximum duration before AWS API request will be interrupted # Default to be 120s #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com From 4e6099bd8134ae64a27f032ed0b38cd2afe86c82 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 13 Feb 2020 13:56:11 -0700 Subject: [PATCH 6/8] Add endpoint to autodiscover aws ec2 and elb providers --- .../autodiscover/providers/aws/ec2/provider.go | 12 ++++++++++++ .../autodiscover/providers/aws/elb/provider.go | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 4c9aa3e8b436..70770f489c37 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -10,6 +10,7 @@ import ( "github.com/gofrs/uuid" "github.com/pkg/errors" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/elastic/beats/libbeat/autodiscover" "github.com/elastic/beats/libbeat/autodiscover/template" "github.com/elastic/beats/libbeat/common" @@ -57,7 +58,13 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis if config.Regions == nil { // set default region to make initial aws api call awsCfg.Region = "us-west-1" + + // check if endpoint is given from configuration + if config.AWSConfig.Endpoint != "" { + awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + awsCfg.Region + "." + config.AWSConfig.Endpoint) + } svcEC2 := ec2.New(awsCfg) + completeRegionsList, err := awsauto.GetRegions(svcEC2) if err != nil { return nil, err @@ -72,6 +79,11 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis logp.Error(errors.Wrap(err, "error loading AWS config for aws_ec2 autodiscover provider")) } awsCfg.Region = region + + // check if endpoint is given from configuration + if config.AWSConfig.Endpoint != "" { + awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + region + "." + config.AWSConfig.Endpoint) + } clients = append(clients, ec2.New(awsCfg)) } diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index b965f9ee1e2f..c32a5492170f 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -10,6 +10,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/elasticloadbalancingv2iface" "github.com/gofrs/uuid" + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/elastic/beats/libbeat/autodiscover" "github.com/elastic/beats/libbeat/autodiscover/template" "github.com/elastic/beats/libbeat/common" @@ -58,7 +59,13 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis if config.Regions == nil { // set default region to make initial aws api call awsCfg.Region = "us-west-1" + + // check if endpoint is given from configuration + if config.AWSConfig.Endpoint != "" { + awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + awsCfg.Region + "." + config.AWSConfig.Endpoint) + } svcEC2 := ec2.New(awsCfg) + completeRegionsList, err := awsauto.GetRegions(svcEC2) if err != nil { return nil, err @@ -79,6 +86,11 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis logp.Err("error loading AWS config for aws_elb autodiscover provider: %s", err) } awsCfg.Region = region + + // check if endpoint is given from configuration + if config.AWSConfig.Endpoint != "" { + awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://elasticloadbalancing." + region + "." + config.AWSConfig.Endpoint) + } clients = append(clients, elasticloadbalancingv2.New(awsCfg)) } From 268fb0ddd242171f0f269d0fc245525ebe3cfed1 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 13 Feb 2020 15:16:32 -0700 Subject: [PATCH 7/8] update changelog --- CHANGELOG.asciidoc | 1 + x-pack/filebeat/input/s3/input.go | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 7aed292e2d7a..b16d40b7bc14 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -128,6 +128,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit - Libbeat HTTP's Server can listen to a unix socket using the `unix:///tmp/hello.sock` syntax. {pull}13655[13655] - Libbeat HTTP's Server can listen to a Windows named pipe using the `npipe:///hello` syntax. {pull}13655[13655] - Adding new `Enterprise` license type to the licenser. {issue}14246[14246] +- Add endpoint config in AWS config to support using custom endpoint accessing AWS APIs. {issue}16245[16245] {pull}16263[16263] *Auditbeat* diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index b0c194b8b46b..853ff7313d8d 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -183,14 +183,14 @@ func (p *s3Input) Run() { awsConfig.Region = regionName // check if endpoint is given from configuration - if p.config.AwsConfig.Endpoint!= "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sqs." + regionName + "."+ p.config.AwsConfig.Endpoint) + if p.config.AwsConfig.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sqs." + regionName + "." + p.config.AwsConfig.Endpoint) } svcSQS := sqs.New(awsConfig) // check if endpoint is given from configuration - if p.config.AwsConfig.Endpoint!= "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://s3." + regionName + "."+ p.config.AwsConfig.Endpoint) + if p.config.AwsConfig.Endpoint != "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://s3." + regionName + "." + p.config.AwsConfig.Endpoint) } svcS3 := s3.New(awsConfig) From deceb45d6117fca39f44f3d95a80eea4959c3e6c Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 14 Feb 2020 10:05:31 -0700 Subject: [PATCH 8/8] Add EnrichAWSConfigWithEndpoint function in libbeat/common for both FB and MB --- x-pack/filebeat/input/s3/input.go | 13 +----- .../providers/aws/ec2/provider.go | 17 ++------ .../providers/aws/elb/provider.go | 17 ++------ x-pack/libbeat/common/aws/credentials.go | 21 ++++++---- x-pack/libbeat/common/aws/credentials_test.go | 42 ++++++++++++++++++- x-pack/metricbeat/module/aws/aws.go | 20 +++------ .../module/aws/cloudwatch/cloudwatch.go | 31 ++++---------- x-pack/metricbeat/module/aws/ec2/ec2.go | 16 +++---- x-pack/metricbeat/module/aws/rds/rds.go | 16 +++---- .../aws/s3_daily_storage/s3_daily_storage.go | 9 ++-- .../module/aws/s3_request/s3_request.go | 9 ++-- x-pack/metricbeat/module/aws/sqs/sqs.go | 16 +++---- 12 files changed, 98 insertions(+), 129 deletions(-) diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 853ff7313d8d..c038797a807c 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -182,17 +182,8 @@ func (p *s3Input) Run() { awsConfig := p.awsConfig.Copy() awsConfig.Region = regionName - // check if endpoint is given from configuration - if p.config.AwsConfig.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sqs." + regionName + "." + p.config.AwsConfig.Endpoint) - } - svcSQS := sqs.New(awsConfig) - - // check if endpoint is given from configuration - if p.config.AwsConfig.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://s3." + regionName + "." + p.config.AwsConfig.Endpoint) - } - svcS3 := s3.New(awsConfig) + svcSQS := sqs.New(awscommon.EnrichAWSConfigWithEndpoint(p.config.AwsConfig.Endpoint, "sqs", regionName, awsConfig)) + svcS3 := s3.New(awscommon.EnrichAWSConfigWithEndpoint(p.config.AwsConfig.Endpoint, "s3", regionName, awsConfig)) p.workerWg.Add(1) go p.run(svcSQS, svcS3, visibilityTimeout) diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 70770f489c37..a29bf84a1862 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -10,7 +10,6 @@ import ( "github.com/gofrs/uuid" "github.com/pkg/errors" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/elastic/beats/libbeat/autodiscover" "github.com/elastic/beats/libbeat/autodiscover/template" "github.com/elastic/beats/libbeat/common" @@ -58,12 +57,8 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis if config.Regions == nil { // set default region to make initial aws api call awsCfg.Region = "us-west-1" - - // check if endpoint is given from configuration - if config.AWSConfig.Endpoint != "" { - awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + awsCfg.Region + "." + config.AWSConfig.Endpoint) - } - svcEC2 := ec2.New(awsCfg) + svcEC2 := ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "ec2", awsCfg.Region, awsCfg)) completeRegionsList, err := awsauto.GetRegions(svcEC2) if err != nil { @@ -79,12 +74,8 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis logp.Error(errors.Wrap(err, "error loading AWS config for aws_ec2 autodiscover provider")) } awsCfg.Region = region - - // check if endpoint is given from configuration - if config.AWSConfig.Endpoint != "" { - awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + region + "." + config.AWSConfig.Endpoint) - } - clients = append(clients, ec2.New(awsCfg)) + clients = append(clients, ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "ec2", region, awsCfg))) } return internalBuilder(uuid, bus, config, newAPIFetcher(clients)) diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index c32a5492170f..1f1eec2621e0 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -10,7 +10,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/elasticloadbalancingv2iface" "github.com/gofrs/uuid" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/elastic/beats/libbeat/autodiscover" "github.com/elastic/beats/libbeat/autodiscover/template" "github.com/elastic/beats/libbeat/common" @@ -59,12 +58,8 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis if config.Regions == nil { // set default region to make initial aws api call awsCfg.Region = "us-west-1" - - // check if endpoint is given from configuration - if config.AWSConfig.Endpoint != "" { - awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + awsCfg.Region + "." + config.AWSConfig.Endpoint) - } - svcEC2 := ec2.New(awsCfg) + svcEC2 := ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "ec2", awsCfg.Region, awsCfg)) completeRegionsList, err := awsauto.GetRegions(svcEC2) if err != nil { @@ -86,12 +81,8 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis logp.Err("error loading AWS config for aws_elb autodiscover provider: %s", err) } awsCfg.Region = region - - // check if endpoint is given from configuration - if config.AWSConfig.Endpoint != "" { - awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://elasticloadbalancing." + region + "." + config.AWSConfig.Endpoint) - } - clients = append(clients, elasticloadbalancingv2.New(awsCfg)) + clients = append(clients, elasticloadbalancingv2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "elasticloadbalancing", region, awsCfg))) } return internalBuilder(uuid, bus, config, newAPIFetcher(clients)) diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index d0286493e7ac..a438cc1f032a 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -40,11 +40,6 @@ func GetAWSCredentials(config ConfigAWS) (awssdk.Config, error) { awsConfig.Credentials = awssdk.StaticCredentialsProvider{ Value: awsCredentials, } - - // check if endpoint is given from configuration - if config.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL(config.Endpoint) - } return awsConfig, nil } @@ -68,10 +63,18 @@ func GetAWSCredentials(config ConfigAWS) (awssdk.Config, error) { if err != nil { return awsConfig, err } + return awsConfig, nil +} - // check if endpoint is given from configuration - if config.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL(config.Endpoint) +// EnrichAWSConfigWithEndpoint function enabled endpoint resolver for AWS +// service clients when endpoint is given in config. +func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName string, awsConfig awssdk.Config) awssdk.Config { + if endpoint != "" { + if regionName == "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://" + serviceName + "." + endpoint) + } else { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://" + serviceName + "." + regionName + "." + endpoint) + } } - return awsConfig, nil + return awsConfig } diff --git a/x-pack/libbeat/common/aws/credentials_test.go b/x-pack/libbeat/common/aws/credentials_test.go index 8cab023f6530..2e438033c9cd 100644 --- a/x-pack/libbeat/common/aws/credentials_test.go +++ b/x-pack/libbeat/common/aws/credentials_test.go @@ -5,9 +5,9 @@ package aws import ( - "testing" - + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/stretchr/testify/assert" + "testing" ) func TestGetAWSCredentials(t *testing.T) { @@ -26,3 +26,41 @@ func TestGetAWSCredentials(t *testing.T) { assert.Equal(t, inputConfig.SecretAccessKey, retrievedAWSConfig.SecretAccessKey) assert.Equal(t, inputConfig.SessionToken, retrievedAWSConfig.SessionToken) } + +func TestEnrichAWSConfigWithEndpoint(t *testing.T) { + cases := []struct { + title string + endpoint string + serviceName string + region string + awsConfig awssdk.Config + expectedAWSConfig awssdk.Config + }{ + { + "endpoint and serviceName given", + "amazonaws.com", + "ec2", + "", + awssdk.Config{}, + awssdk.Config{ + EndpointResolver: awssdk.ResolveWithEndpointURL("https://ec2.amazonaws.com"), + }, + }, + { + "endpoint, serviceName and region given", + "amazonaws.com", + "cloudwatch", + "us-west-1", + awssdk.Config{}, + awssdk.Config{ + EndpointResolver: awssdk.ResolveWithEndpointURL("https://cloudwatch.us-west-1.amazonaws.com"), + }, + }, + } + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + enrichedAWSConfig := EnrichAWSConfigWithEndpoint(c.endpoint, c.serviceName, c.region, c.awsConfig) + assert.Equal(t, c.expectedAWSConfig, enrichedAWSConfig) + }) + } +} diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index b5937a0bc77d..e1ef1f7c1009 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -87,14 +87,9 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { } // Get IAM account name - // check if endpoint is given from configuration - if config.AWSConfig.Endpoint != "" { - metricSet.Endpoint = config.AWSConfig.Endpoint - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://iam." + config.AWSConfig.Endpoint) - } - awsConfig.Region = "us-east-1" - svcIam := iam.New(awsConfig) + svcIam := iam.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "iam", "", awsConfig)) req := svcIam.ListAccountAliasesRequest(&iam.ListAccountAliasesInput{}) output, err := req.Send(context.TODO()) if err != nil { @@ -108,11 +103,8 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { } // Get IAM account id - // check if endpoint is given from configuration - if config.AWSConfig.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sts." + config.AWSConfig.Endpoint) - } - svcSts := sts.New(awsConfig) + svcSts := sts.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "sts", "", awsConfig)) reqIdentity := svcSts.GetCallerIdentityRequest(&sts.GetCallerIdentityInput{}) outputIdentity, err := reqIdentity.Send(context.TODO()) if err != nil { @@ -123,8 +115,8 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // Construct MetricSet with a full regions list if config.Regions == nil { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + config.AWSConfig.Endpoint) - svcEC2 := ec2.New(awsConfig) + svcEC2 := ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "ec2", "", awsConfig)) completeRegionsList, err := getRegions(svcEC2) if err != nil { return nil, err diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 540b5f5536d9..43b021aa3656 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -11,7 +11,6 @@ import ( "strings" "time" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface" "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" @@ -20,6 +19,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -138,18 +138,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + regionName + "." + m.Endpoint) - } - - svcCloudwatch := cloudwatch.New(awsConfig) + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://tagging." + regionName + "." + m.Endpoint) - } - svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) + svcResourceAPI := resourcegroupstaggingapi.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "tagging", regionName, awsConfig)) eventsWithIdentifier, eventsNoIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, regionName, startTime, endTime) if err != nil { @@ -167,17 +160,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + regionName + "." + m.Endpoint) - } - svcCloudwatch := cloudwatch.New(awsConfig) + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://tagging." + regionName + "." + m.Endpoint) - } - svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) + svcResourceAPI := resourcegroupstaggingapi.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "tagging", regionName, awsConfig)) // Create events based on namespaceDetailTotal from configuration for namespace, namespaceDetails := range namespaceDetailTotal { diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 6aa8ae8897da..023788e905e8 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -11,7 +11,6 @@ import ( "strings" "time" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" @@ -19,6 +18,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -92,11 +92,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + regionName + "." + m.Endpoint) - } - svcEC2 := ec2.New(awsConfig) + svcEC2 := ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "ec2", regionName, awsConfig)) instanceIDs, instancesOutputs, err := getInstancesPerRegion(svcEC2) if err != nil { @@ -106,11 +103,8 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + regionName + "." + m.Endpoint) - } - svcCloudwatch := cloudwatch.New(awsConfig) + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) namespace := "AWS/EC2" listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index fce4b3c8c30d..34e00571b023 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -11,13 +11,13 @@ import ( "strings" "time" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/rds" "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" "github.com/pkg/errors" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -83,12 +83,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://rds." + awsConfig.Region + "." + m.Endpoint) - } + svc := rds.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "rds", regionName, awsConfig)) - svc := rds.New(awsConfig) // Get DBInstance IDs per region dbInstanceIDs, dbDetailsMap, err := getDBInstancesPerRegion(svc) if err != nil { @@ -102,12 +99,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) - } + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) - svcCloudwatch := cloudwatch.New(awsConfig) namespace := "AWS/RDS" listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index 4ddcc447e8f1..3c87f254c05e 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -10,11 +10,11 @@ import ( "strings" "time" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/pkg/errors" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -76,12 +76,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) - } + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) - svcCloudwatch := cloudwatch.New(awsConfig) listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { err = errors.Wrap(err, "GetListMetricsOutput failed, skipping region "+regionName) diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index 4a33926ff381..34f3e1891092 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -10,11 +10,11 @@ import ( "strings" "time" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/pkg/errors" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -76,12 +76,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) - } + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) - svcCloudwatch := cloudwatch.New(awsConfig) listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { m.Logger().Error(err.Error()) diff --git a/x-pack/metricbeat/module/aws/sqs/sqs.go b/x-pack/metricbeat/module/aws/sqs/sqs.go index 7ed5be7a5e70..9bfe9bd90757 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs.go @@ -11,7 +11,6 @@ import ( "strings" "time" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface" @@ -19,6 +18,7 @@ import ( s "github.com/elastic/beats/libbeat/common/schema" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -73,17 +73,11 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://monitoring." + awsConfig.Region + "." + m.Endpoint) - } - svcCloudwatch := cloudwatch.New(awsConfig) + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) - // check if endpoint is given from configuration - if m.Endpoint != "" { - awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sqs." + awsConfig.Region + "." + m.Endpoint) - } - svcSQS := sqs.New(awsConfig) + svcSQS := sqs.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "sqs", regionName, awsConfig)) // Get queueUrls for each region queueURLs, err := getQueueUrls(svcSQS)