Add endpoint into AWS config

CHANGELOG.asciidoc

@@ -128,6 +128,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit
 - Libbeat HTTP's Server can listen to a unix socket using the `unix:///tmp/hello.sock` syntax. {pull}13655[13655]
 - Libbeat HTTP's Server can listen to a Windows named pipe using the `npipe:///hello` syntax. {pull}13655[13655]
 - Adding new `Enterprise` license type to the licenser. {issue}14246[14246]
+- Add endpoint config in AWS config to support using custom endpoint accessing AWS APIs. {issue}16245[16245] {pull}16263[16263]
 
 *Auditbeat*
 

filebeat/docs/modules/aws.asciidoc

@@ -33,18 +33,12 @@ Example config:
 - module: aws
   s3access:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
-
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
   elb:
     enabled: false
@@ -104,6 +98,19 @@ Filename of AWS credential file.
 
 AWS credential profile name.
 
+*`var.visibility_timeout`*::
+
+The duration that the received messages are hidden from ReceiveMessage request.
+Default to be 300 seconds.
+
+*`var.api_timeout`*::
+
+Maximum duration before AWS API request will be interrupted. Default to be 120 seconds.
+
+*`var.endpoint`*::
+
+Custom endpoint used to access AWS APIs.
+
 [float]
 === cloudtrail fileset
 

x-pack/filebeat/filebeat.reference.yml

@@ -119,6 +119,9 @@ filebeat.modules:
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
   elb:
     enabled: false
 
@@ -142,6 +145,9 @@ filebeat.modules:
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
   vpcflow:
     enabled: false
 
@@ -165,6 +171,9 @@ filebeat.modules:
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
   cloudtrail:
     enabled: false
 
@@ -188,6 +197,9 @@ filebeat.modules:
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
 #-------------------------------- Azure Module --------------------------------
 - module: azure
   # All logs

x-pack/filebeat/input/s3/input.go

@@ -181,7 +181,17 @@ func (p *s3Input) Run() {
 
 		awsConfig := p.awsConfig.Copy()
 		awsConfig.Region = regionName
+
+		// check if endpoint is given from configuration
+		if p.config.AwsConfig.Endpoint != "" {
+			awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://sqs." + regionName + "." + p.config.AwsConfig.Endpoint)
+		}
 		svcSQS := sqs.New(awsConfig)
+
+		// check if endpoint is given from configuration
+		if p.config.AwsConfig.Endpoint != "" {
+			awsConfig.EndpointResolver = awssdk.ResolveWithEndpointURL("https://s3." + regionName + "." + p.config.AwsConfig.Endpoint)
+		}
 		svcS3 := s3.New(awsConfig)
 
 		p.workerWg.Add(1)

x-pack/filebeat/module/aws/_meta/config.yml

@@ -22,6 +22,9 @@
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
   elb:
     enabled: false
 
@@ -45,6 +48,9 @@
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
   vpcflow:
     enabled: false
 
@@ -68,6 +74,9 @@
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
   cloudtrail:
     enabled: false
 
@@ -90,3 +99,6 @@
     # Maximum duration before AWS API request will be interrupted
     # Default to be 120s
     #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com

x-pack/filebeat/module/aws/_meta/docs.asciidoc

@@ -28,18 +28,12 @@ Example config:
 - module: aws
   s3access:
     enabled: false
-
-    # AWS SQS queue url
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
-
-    # Filename of AWS credential file
-    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
-    # "%UserProfile%\.aws\credentials" is used on Windows
-    # var.shared_credential_file: /etc/filebeat/aws_credentials
-
-    # Profile name for aws credential
-    # If not set the default profile is used
-    # var.credential_profile_name: fb-aws
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+    #var.credential_profile_name: fb-aws
+    #var.visibility_timeout: 300s
+    #var.api_timeout: 120s
+    #var.endpoint: amazonaws.com
 
   elb:
     enabled: false
@@ -99,6 +93,19 @@ Filename of AWS credential file.
 
 AWS credential profile name.
 
+*`var.visibility_timeout`*::
+
+The duration that the received messages are hidden from ReceiveMessage request.
+Default to be 300 seconds.
+
+*`var.api_timeout`*::
+
+Maximum duration before AWS API request will be interrupted. Default to be 120 seconds.
+
+*`var.endpoint`*::
+
+Custom endpoint used to access AWS APIs.
+
 [float]
 === cloudtrail fileset
 

x-pack/filebeat/module/aws/cloudtrail/config/cloudtrail.yml

@@ -20,6 +20,10 @@ visibility_timeout: {{ .visibility_timeout }}
 api_timeout: {{ .api_timeout }}
 {{ end }}
 
+{{ if .endpoint }}
+endpoint: {{ .endpoint }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 
 type: log

x-pack/filebeat/module/aws/cloudtrail/manifest.yml

@@ -7,6 +7,7 @@ var:
   - name: credential_profile_name
   - name: visibility_timeout
   - name: api_timeout
+  - name: endpoint
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/cloudtrail.yml

x-pack/filebeat/module/aws/elb/config/s3.yml

@@ -16,3 +16,7 @@ visibility_timeout: {{ .visibility_timeout }}
 {{ if .api_timeout }}
 api_timeout: {{ .api_timeout }}
 {{ end }}
+
+{{ if .endpoint }}
+endpoint: {{ .endpoint }}
+{{ end }}

x-pack/filebeat/module/aws/elb/manifest.yml

@@ -7,6 +7,7 @@ var:
   - name: credential_profile_name
   - name: visibility_timeout
   - name: api_timeout
+  - name: endpoint
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml

x-pack/filebeat/module/aws/s3access/config/s3.yml

@@ -16,3 +16,7 @@ visibility_timeout: {{ .visibility_timeout }}
 {{ if .api_timeout }}
 api_timeout: {{ .api_timeout }}
 {{ end }}
+
+{{ if .endpoint }}
+endpoint: {{ .endpoint }}
+{{ end }}

x-pack/filebeat/module/aws/s3access/manifest.yml

@@ -7,6 +7,7 @@ var:
   - name: credential_profile_name
   - name: visibility_timeout
   - name: api_timeout
+  - name: endpoint
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml

x-pack/filebeat/module/aws/vpcflow/config/input.yml

@@ -19,6 +19,10 @@ visibility_timeout: {{ .visibility_timeout }}
 api_timeout: {{ .api_timeout }}
 {{ end }}
 
+{{ if .endpoint }}
+endpoint: {{ .endpoint }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 
 type: log

x-pack/filebeat/module/aws/vpcflow/manifest.yml

@@ -7,6 +7,7 @@ var:
   - name: credential_profile_name
   - name: visibility_timeout
   - name: api_timeout
+  - name: endpoint
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/input.yml

x-pack/filebeat/modules.d/aws.yml.disabled

@@ -25,6 +25,9 @@
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
   elb:
     enabled: false
 
@@ -48,6 +51,9 @@
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
   vpcflow:
     enabled: false
 
@@ -71,6 +77,9 @@
     # Default to be 120s
     #var.api_timeout: 120s
 
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
   cloudtrail:
     enabled: false
 
@@ -93,3 +102,6 @@
     # Maximum duration before AWS API request will be interrupted
     # Default to be 120s
     #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com

x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gofrs/uuid"
 	"github.com/pkg/errors"
 
+	awssdk "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/elastic/beats/libbeat/autodiscover"
 	"github.com/elastic/beats/libbeat/autodiscover/template"
 	"github.com/elastic/beats/libbeat/common"
@@ -57,7 +58,13 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis
 	if config.Regions == nil {
 		// set default region to make initial aws api call
 		awsCfg.Region = "us-west-1"
+
+		// check if endpoint is given from configuration
+		if config.AWSConfig.Endpoint != "" {
+			awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + awsCfg.Region + "." + config.AWSConfig.Endpoint)
+		}
 		svcEC2 := ec2.New(awsCfg)
+
 		completeRegionsList, err := awsauto.GetRegions(svcEC2)
 		if err != nil {
 			return nil, err
@@ -72,6 +79,11 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis
 			logp.Error(errors.Wrap(err, "error loading AWS config for aws_ec2 autodiscover provider"))
 		}
 		awsCfg.Region = region
+
+		// check if endpoint is given from configuration
+		if config.AWSConfig.Endpoint != "" {
+			awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + region + "." + config.AWSConfig.Endpoint)
+		}
 		clients = append(clients, ec2.New(awsCfg))
 	}
 

x-pack/libbeat/autodiscover/providers/aws/elb/provider.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/elasticloadbalancingv2iface"
 	"github.com/gofrs/uuid"
 
+	awssdk "github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/elastic/beats/libbeat/autodiscover"
 	"github.com/elastic/beats/libbeat/autodiscover/template"
 	"github.com/elastic/beats/libbeat/common"
@@ -58,7 +59,13 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis
 	if config.Regions == nil {
 		// set default region to make initial aws api call
 		awsCfg.Region = "us-west-1"
+
+		// check if endpoint is given from configuration
+		if config.AWSConfig.Endpoint != "" {
+			awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://ec2." + awsCfg.Region + "." + config.AWSConfig.Endpoint)
+		}
 		svcEC2 := ec2.New(awsCfg)
+
 		completeRegionsList, err := awsauto.GetRegions(svcEC2)
 		if err != nil {
 			return nil, err
@@ -79,6 +86,11 @@ func AutodiscoverBuilder(bus bus.Bus, uuid uuid.UUID, c *common.Config) (autodis
 			logp.Err("error loading AWS config for aws_elb autodiscover provider: %s", err)
 		}
 		awsCfg.Region = region
+
+		// check if endpoint is given from configuration
+		if config.AWSConfig.Endpoint != "" {
+			awsCfg.EndpointResolver = awssdk.ResolveWithEndpointURL("https://elasticloadbalancing." + region + "." + config.AWSConfig.Endpoint)
+		}
 		clients = append(clients, elasticloadbalancingv2.New(awsCfg))
 	}