From cc78671ec64ab2753a7df47e61465f88269b5437 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 18 Feb 2020 13:16:41 -0700 Subject: [PATCH] Add endpoint into AWS config (#16263) (#16389) * Add endpoint into AWS config * Add EnrichAWSConfigWithEndpoint function in libbeat/common for both FB and MB (cherry picked from commit 97501ad1460636eb2cf0e497f09f93332ab30729) --- CHANGELOG.asciidoc | 1 + filebeat/docs/modules/aws.asciidoc | 29 ++++++++----- x-pack/filebeat/filebeat.reference.yml | 12 ++++++ x-pack/filebeat/input/s3/input.go | 5 ++- x-pack/filebeat/module/aws/_meta/config.yml | 12 ++++++ .../filebeat/module/aws/_meta/docs.asciidoc | 29 ++++++++----- .../aws/cloudtrail/config/cloudtrail.yml | 4 ++ .../module/aws/cloudtrail/manifest.yml | 1 + x-pack/filebeat/module/aws/elb/config/s3.yml | 4 ++ x-pack/filebeat/module/aws/elb/manifest.yml | 1 + .../module/aws/s3access/config/s3.yml | 4 ++ .../filebeat/module/aws/s3access/manifest.yml | 1 + .../module/aws/vpcflow/config/input.yml | 4 ++ .../filebeat/module/aws/vpcflow/manifest.yml | 1 + x-pack/filebeat/modules.d/aws.yml.disabled | 12 ++++++ .../providers/aws/ec2/provider.go | 7 +++- .../providers/aws/elb/provider.go | 7 +++- x-pack/libbeat/common/aws/credentials.go | 21 +++++++++- x-pack/libbeat/common/aws/credentials_test.go | 42 ++++++++++++++++++- .../docs/aws-credentials-config.asciidoc | 9 ++++ x-pack/metricbeat/module/aws/aws.go | 13 +++--- .../module/aws/cloudwatch/cloudwatch.go | 17 ++++++-- x-pack/metricbeat/module/aws/ec2/ec2.go | 14 ++++--- x-pack/metricbeat/module/aws/rds/rds.go | 13 ++++-- .../aws/s3_daily_storage/s3_daily_storage.go | 6 ++- .../module/aws/s3_request/s3_request.go | 6 ++- x-pack/metricbeat/module/aws/sqs/sqs.go | 9 +++- 27 files changed, 231 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 5c94fb28677..1e82aa3bec7 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -128,6 +128,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit - Libbeat HTTP's Server can listen to a unix socket using the `unix:///tmp/hello.sock` syntax. {pull}13655[13655] - Libbeat HTTP's Server can listen to a Windows named pipe using the `npipe:///hello` syntax. {pull}13655[13655] - Adding new `Enterprise` license type to the licenser. {issue}14246[14246] +- Add endpoint config in AWS config to support using custom endpoint accessing AWS APIs. {issue}16245[16245] {pull}16263[16263] *Auditbeat* diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index 79daf4a1865..e8e971b6690 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -33,18 +33,12 @@ Example config: - module: aws s3access: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com elb: enabled: false @@ -104,6 +98,19 @@ Filename of AWS credential file. AWS credential profile name. +*`var.visibility_timeout`*:: + +The duration that the received messages are hidden from ReceiveMessage request. +Default to be 300 seconds. + +*`var.api_timeout`*:: + +Maximum duration before AWS API request will be interrupted. Default to be 120 seconds. + +*`var.endpoint`*:: + +Custom endpoint used to access AWS APIs. + [float] === cloudtrail fileset diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 0f99430a5ac..529e1ffce9b 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -119,6 +119,9 @@ filebeat.modules: # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + elb: enabled: false @@ -142,6 +145,9 @@ filebeat.modules: # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + vpcflow: enabled: false @@ -165,6 +171,9 @@ filebeat.modules: # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + cloudtrail: enabled: false @@ -188,6 +197,9 @@ filebeat.modules: # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + #-------------------------------- Azure Module -------------------------------- - module: azure # All logs diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index dd254cffebf..c038797a807 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -181,8 +181,9 @@ func (p *s3Input) Run() { awsConfig := p.awsConfig.Copy() awsConfig.Region = regionName - svcSQS := sqs.New(awsConfig) - svcS3 := s3.New(awsConfig) + + svcSQS := sqs.New(awscommon.EnrichAWSConfigWithEndpoint(p.config.AwsConfig.Endpoint, "sqs", regionName, awsConfig)) + svcS3 := s3.New(awscommon.EnrichAWSConfigWithEndpoint(p.config.AwsConfig.Endpoint, "s3", regionName, awsConfig)) p.workerWg.Add(1) go p.run(svcSQS, svcS3, visibilityTimeout) diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index e87956f950c..ee54cc54558 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -22,6 +22,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + elb: enabled: false @@ -45,6 +48,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + vpcflow: enabled: false @@ -68,6 +74,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + cloudtrail: enabled: false @@ -90,3 +99,6 @@ # Maximum duration before AWS API request will be interrupted # Default to be 120s #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index 552b90c20d5..e222f55d23f 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -28,18 +28,12 @@ Example config: - module: aws s3access: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue - - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com elb: enabled: false @@ -99,6 +93,19 @@ Filename of AWS credential file. AWS credential profile name. +*`var.visibility_timeout`*:: + +The duration that the received messages are hidden from ReceiveMessage request. +Default to be 300 seconds. + +*`var.api_timeout`*:: + +Maximum duration before AWS API request will be interrupted. Default to be 120 seconds. + +*`var.endpoint`*:: + +Custom endpoint used to access AWS APIs. + [float] === cloudtrail fileset diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml b/x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml index 87219497aef..18087c4769b 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml @@ -20,6 +20,10 @@ visibility_timeout: {{ .visibility_timeout }} api_timeout: {{ .api_timeout }} {{ end }} +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} + {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml index d809dec5fb5..5e55c965175 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml @@ -7,6 +7,7 @@ var: - name: credential_profile_name - name: visibility_timeout - name: api_timeout + - name: endpoint ingest_pipeline: ingest/pipeline.yml input: config/cloudtrail.yml diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/s3.yml index 40212723e96..2af7bebff30 100644 --- a/x-pack/filebeat/module/aws/elb/config/s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/s3.yml @@ -16,3 +16,7 @@ visibility_timeout: {{ .visibility_timeout }} {{ if .api_timeout }} api_timeout: {{ .api_timeout }} {{ end }} + +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml index 01857d90643..f5a3befce10 100644 --- a/x-pack/filebeat/module/aws/elb/manifest.yml +++ b/x-pack/filebeat/module/aws/elb/manifest.yml @@ -7,6 +7,7 @@ var: - name: credential_profile_name - name: visibility_timeout - name: api_timeout + - name: endpoint ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/s3.yml index 40212723e96..2af7bebff30 100644 --- a/x-pack/filebeat/module/aws/s3access/config/s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/s3.yml @@ -16,3 +16,7 @@ visibility_timeout: {{ .visibility_timeout }} {{ if .api_timeout }} api_timeout: {{ .api_timeout }} {{ end }} + +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml index b4fdbb7ca49..ce14df8f171 100644 --- a/x-pack/filebeat/module/aws/s3access/manifest.yml +++ b/x-pack/filebeat/module/aws/s3access/manifest.yml @@ -7,6 +7,7 @@ var: - name: credential_profile_name - name: visibility_timeout - name: api_timeout + - name: endpoint ingest_pipeline: ingest/pipeline.yml input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index 1838d695eb4..af77a19e14f 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -19,6 +19,10 @@ visibility_timeout: {{ .visibility_timeout }} api_timeout: {{ .api_timeout }} {{ end }} +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} + {{ else if eq .input "file" }} type: log diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml index e438f1e91c3..27d8c8f126e 100644 --- a/x-pack/filebeat/module/aws/vpcflow/manifest.yml +++ b/x-pack/filebeat/module/aws/vpcflow/manifest.yml @@ -7,6 +7,7 @@ var: - name: credential_profile_name - name: visibility_timeout - name: api_timeout + - name: endpoint ingest_pipeline: ingest/pipeline.yml input: config/input.yml diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index 9c0284b0c64..bb6ff14e8e5 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -25,6 +25,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + elb: enabled: false @@ -48,6 +51,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + vpcflow: enabled: false @@ -71,6 +77,9 @@ # Default to be 120s #var.api_timeout: 120s + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + cloudtrail: enabled: false @@ -93,3 +102,6 @@ # Maximum duration before AWS API request will be interrupted # Default to be 120s #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 4c9aa3e8b43..a29bf84a186 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -57,7 +57,9 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis if config.Regions == nil { // set default region to make initial aws api call awsCfg.Region = "us-west-1" - svcEC2 := ec2.New(awsCfg) + svcEC2 := ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "ec2", awsCfg.Region, awsCfg)) + completeRegionsList, err := awsauto.GetRegions(svcEC2) if err != nil { return nil, err @@ -72,7 +74,8 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis logp.Error(errors.Wrap(err, "error loading AWS config for aws_ec2 autodiscover provider")) } awsCfg.Region = region - clients = append(clients, ec2.New(awsCfg)) + clients = append(clients, ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "ec2", region, awsCfg))) } return internalBuilder(uuid, bus, config, newAPIFetcher(clients)) diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index b965f9ee1e2..1f1eec2621e 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -58,7 +58,9 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis if config.Regions == nil { // set default region to make initial aws api call awsCfg.Region = "us-west-1" - svcEC2 := ec2.New(awsCfg) + svcEC2 := ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "ec2", awsCfg.Region, awsCfg)) + completeRegionsList, err := awsauto.GetRegions(svcEC2) if err != nil { return nil, err @@ -79,7 +81,8 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis logp.Err("error loading AWS config for aws_elb autodiscover provider: %s", err) } awsCfg.Region = region - clients = append(clients, elasticloadbalancingv2.New(awsCfg)) + clients = append(clients, elasticloadbalancingv2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "elasticloadbalancing", region, awsCfg))) } return internalBuilder(uuid, bus, config, newAPIFetcher(clients)) diff --git a/x-pack/libbeat/common/aws/credentials.go b/x-pack/libbeat/common/aws/credentials.go index 3a3bf5b80e6..a438cc1f032 100644 --- a/x-pack/libbeat/common/aws/credentials.go +++ b/x-pack/libbeat/common/aws/credentials.go @@ -17,6 +17,7 @@ type ConfigAWS struct { SessionToken string `config:"session_token"` ProfileName string `config:"credential_profile_name"` SharedCredentialFile string `config:"shared_credential_file"` + Endpoint string `config:"endpoint"` } // GetAWSCredentials function gets aws credentials from the config. @@ -57,5 +58,23 @@ func GetAWSCredentials(config ConfigAWS) (awssdk.Config, error) { if config.SharedCredentialFile != "" { options = append(options, external.WithSharedConfigFiles([]string{config.SharedCredentialFile})) } - return external.LoadDefaultAWSConfig(options...) + + awsConfig, err := external.LoadDefaultAWSConfig(options...) + if err != nil { + return awsConfig, err + } + return awsConfig, nil +} + +// EnrichAWSConfigWithEndpoint function enabled endpoint resolver for AWS +// service clients when endpoint is given in config. +func EnrichAWSConfigWithEndpoint(endpoint string, serviceName string, regionName string, awsConfig awssdk.Config) awssdk.Config { + if endpoint != "" { + if regionName == "" { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://" + serviceName + "." + endpoint) + } else { + awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://" + serviceName + "." + regionName + "." + endpoint) + } + } + return awsConfig } diff --git a/x-pack/libbeat/common/aws/credentials_test.go b/x-pack/libbeat/common/aws/credentials_test.go index 8cab023f653..2e438033c9c 100644 --- a/x-pack/libbeat/common/aws/credentials_test.go +++ b/x-pack/libbeat/common/aws/credentials_test.go @@ -5,9 +5,9 @@ package aws import ( - "testing" - + awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/stretchr/testify/assert" + "testing" ) func TestGetAWSCredentials(t *testing.T) { @@ -26,3 +26,41 @@ func TestGetAWSCredentials(t *testing.T) { assert.Equal(t, inputConfig.SecretAccessKey, retrievedAWSConfig.SecretAccessKey) assert.Equal(t, inputConfig.SessionToken, retrievedAWSConfig.SessionToken) } + +func TestEnrichAWSConfigWithEndpoint(t *testing.T) { + cases := []struct { + title string + endpoint string + serviceName string + region string + awsConfig awssdk.Config + expectedAWSConfig awssdk.Config + }{ + { + "endpoint and serviceName given", + "amazonaws.com", + "ec2", + "", + awssdk.Config{}, + awssdk.Config{ + EndpointResolver: awssdk.ResolveWithEndpointURL("https://ec2.amazonaws.com"), + }, + }, + { + "endpoint, serviceName and region given", + "amazonaws.com", + "cloudwatch", + "us-west-1", + awssdk.Config{}, + awssdk.Config{ + EndpointResolver: awssdk.ResolveWithEndpointURL("https://cloudwatch.us-west-1.amazonaws.com"), + }, + }, + } + for _, c := range cases { + t.Run(c.title, func(t *testing.T) { + enrichedAWSConfig := EnrichAWSConfigWithEndpoint(c.endpoint, c.serviceName, c.region, c.awsConfig) + assert.Equal(t, c.expectedAWSConfig, enrichedAWSConfig) + }) + } +} diff --git a/x-pack/libbeat/docs/aws-credentials-config.asciidoc b/x-pack/libbeat/docs/aws-credentials-config.asciidoc index a83da149b06..33deb5a0941 100644 --- a/x-pack/libbeat/docs/aws-credentials-config.asciidoc +++ b/x-pack/libbeat/docs/aws-credentials-config.asciidoc @@ -2,6 +2,15 @@ === AWS Credentials Configuration To configure AWS credentials, either put the credentials into the {beatname_uc} configuration, or use a shared credentials file, as shown in the following examples. +[float] +==== Configuration parameters +* *access_key_id*: first part of access key. +* *secret_access_key*: second part of access key. +* *session_token*: required when using temporary security credentials. +* *credential_profile_name*: profile name in shared credentials file. +* *shared_credential_file*: directory of the shared credentials file. +* *endpoint*: URL of the entry point for an AWS web service. + [float] ==== Supported Formats * Use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or `AWS_SESSION_TOKEN` diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index a4bba395bc5..e1ef1f7c100 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -32,6 +32,7 @@ type Config struct { type MetricSet struct { mb.BaseMetricSet RegionsList []string + Endpoint string Period time.Duration AwsConfig *awssdk.Config AccountName string @@ -86,7 +87,9 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { } // Get IAM account name - svcIam := iam.New(awsConfig) + awsConfig.Region = "us-east-1" + svcIam := iam.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "iam", "", awsConfig)) req := svcIam.ListAccountAliasesRequest(&iam.ListAccountAliasesInput{}) output, err := req.Send(context.TODO()) if err != nil { @@ -100,7 +103,8 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { } // Get IAM account id - svcSts := sts.New(awsConfig) + svcSts := sts.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "sts", "", awsConfig)) reqIdentity := svcSts.GetCallerIdentityRequest(&sts.GetCallerIdentityInput{}) outputIdentity, err := reqIdentity.Send(context.TODO()) if err != nil { @@ -111,9 +115,8 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { // Construct MetricSet with a full regions list if config.Regions == nil { - // set default region to make initial aws api call - awsConfig.Region = "us-west-1" - svcEC2 := ec2.New(awsConfig) + svcEC2 := ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + config.AWSConfig.Endpoint, "ec2", "", awsConfig)) completeRegionsList, err := getRegions(svcEC2) if err != nil { return nil, err diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 40be8a846cd..43b021aa365 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -19,6 +19,7 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -136,8 +137,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(awsConfig) - svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) + + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) + + svcResourceAPI := resourcegroupstaggingapi.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "tagging", regionName, awsConfig)) eventsWithIdentifier, eventsNoIdentifier, err := m.createEvents(svcCloudwatch, svcResourceAPI, listMetricDetailTotal.metricsWithStats, listMetricDetailTotal.resourceTypeFilters, regionName, startTime, endTime) if err != nil { @@ -154,8 +159,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(awsConfig) - svcResourceAPI := resourcegroupstaggingapi.New(awsConfig) + + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) + + svcResourceAPI := resourcegroupstaggingapi.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "tagging", regionName, awsConfig)) // Create events based on namespaceDetailTotal from configuration for namespace, namespaceDetails := range namespaceDetailTotal { diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 41770c495ab..023788e905e 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -11,14 +11,14 @@ import ( "strings" "time" - "github.com/elastic/beats/libbeat/common" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/ec2iface" "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -91,7 +91,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - svcEC2 := ec2.New(awsConfig) + + svcEC2 := ec2.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "ec2", regionName, awsConfig)) + instanceIDs, instancesOutputs, err := getInstancesPerRegion(svcEC2) if err != nil { err = errors.Wrap(err, "getInstancesPerRegion failed, skipping region "+regionName) @@ -100,7 +103,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } - svcCloudwatch := cloudwatch.New(awsConfig) + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) + namespace := "AWS/EC2" listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { @@ -131,7 +136,6 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Create Cloudwatch Events for EC2 events, err := m.createCloudWatchEvents(metricDataOutput, instancesOutputs, regionName) - if err != nil { m.Logger().Error(err.Error()) report.Error(err) diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index e2f2d1eb7f5..34e00571b02 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -11,13 +11,13 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" - "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/rds" + "github.com/aws/aws-sdk-go-v2/service/rds/rdsiface" "github.com/pkg/errors" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -82,7 +82,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - svc := rds.New(awsConfig) + + svc := rds.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "rds", regionName, awsConfig)) + // Get DBInstance IDs per region dbInstanceIDs, dbDetailsMap, err := getDBInstancesPerRegion(svc) if err != nil { @@ -96,7 +99,9 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { continue } - svcCloudwatch := cloudwatch.New(awsConfig) + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) + namespace := "AWS/RDS" listMetricsOutput, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index de9d2574380..3c87f254c05 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -74,7 +75,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(awsConfig) + + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) + listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { err = errors.Wrap(err, "GetListMetricsOutput failed, skipping region "+regionName) diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index 5474cbc0117..34f3e189109 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -14,6 +14,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -74,7 +75,10 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(awsConfig) + + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) + listMetricsOutputs, err := aws.GetListMetricsOutput(namespace, regionName, svcCloudwatch) if err != nil { m.Logger().Error(err.Error()) diff --git a/x-pack/metricbeat/module/aws/sqs/sqs.go b/x-pack/metricbeat/module/aws/sqs/sqs.go index 5e70cdb0b1b..9bfe9bd9075 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs.go @@ -18,6 +18,7 @@ import ( s "github.com/elastic/beats/libbeat/common/schema" "github.com/elastic/beats/metricbeat/mb" + awscommon "github.com/elastic/beats/x-pack/libbeat/common/aws" "github.com/elastic/beats/x-pack/metricbeat/module/aws" ) @@ -71,8 +72,12 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() awsConfig.Region = regionName - svcCloudwatch := cloudwatch.New(awsConfig) - svcSQS := sqs.New(awsConfig) + + svcCloudwatch := cloudwatch.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "monitoring", regionName, awsConfig)) + + svcSQS := sqs.New(awscommon.EnrichAWSConfigWithEndpoint( + m.Endpoint, "sqs", regionName, awsConfig)) // Get queueUrls for each region queueURLs, err := getQueueUrls(svcSQS)