[8.0](backport #28899) Move fips_enabled setting to AWS Common (#29418

) (cherry picked from commit 377f97b) Co-authored-by: Alex Resnick <[email protected]> Co-authored-by: kaiyan-sheng <[email protected]>
elastic · Dec 15, 2021 · 518b482 · 518b482
1 parent bc117bc
commit 518b482
Show file tree

Hide file tree

Showing 39 changed files with 398 additions and 60 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -227,6 +227,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add http.pprof.enabled option to libbeat to allow http/pprof endpoints on the socket that libbeat creates for metrics. {issue}21965[21965]
 - Support custom analyzers in fields.yml. {issue}28540[28540] {pull}28926[28926]
 - Support self signed certificates on outputs {pull}29229[29229]
+- Add FIPS configuration option for all AWS API calls. {pull}[28899]
 
 *Auditbeat*
 

diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc
@@ -68,6 +68,24 @@ If endpoint is specified, `regions` config becomes required. For example:
     - ec2
 ----
 
+* *fips_enabled*
+
+Enforces the use of FIPS service endpoints. See <<aws-credentials-config,AWS credentials options>> for more information.
+
+[source,yaml]
+----
+- module: aws
+  period: 5m
+  fips_enabled: true
+  regions:
+    - us-east-1
+    - us-east-2
+    - us-west-1
+    - us-west-2
+  metricsets:
+    - ec2
+----
+
 The aws module comes with a predefined dashboard. For example:
 
 image::./images/metricbeat-aws-overview.png[]

diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc
@@ -182,9 +182,7 @@ file_selectors:
 [float]
 ==== `fips_enabled`
 
-Enabling this option changes the service name from `s3` to `s3-fips` for
-connecting to the correct service endpoint. For example:
-`s3-fips.us-gov-east-1.amazonaws.com`.
+Moved to <<aws-credentials-config,AWS credentials options>>.
 
 [id="input-{type}-include_s3_metadata"]
 [float]

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -160,6 +160,12 @@ filebeat.modules:
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   cloudwatch:
     enabled: false
 
@@ -212,6 +218,12 @@ filebeat.modules:
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   ec2:
     enabled: false
 
@@ -264,6 +276,12 @@ filebeat.modules:
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   elb:
     enabled: false
 
@@ -316,6 +334,12 @@ filebeat.modules:
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   s3access:
     enabled: false
 
@@ -368,6 +392,12 @@ filebeat.modules:
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   vpcflow:
     enabled: false
 
@@ -420,6 +450,12 @@ filebeat.modules:
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
 #----------------------------- AWS Fargate Module -----------------------------
 - module: awsfargate
   log:
@@ -476,6 +512,12 @@ filebeat.modules:
     # Default api_sleep is 200 ms
     #var.api_sleep: 200ms
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
 #-------------------------------- Azure Module --------------------------------
 - module: azure
   # All logs

diff --git a/x-pack/filebeat/input/awscloudwatch/input.go b/x-pack/filebeat/input/awscloudwatch/input.go
@@ -131,7 +131,8 @@ func NewInput(cfg *common.Config, connector channel.Connector, context input.Con
 // Run runs the input
 func (in *awsCloudWatchInput) Run() {
 	// Please see https://docs.aws.amazon.com/general/latest/gr/cwl_region.html for more info on Amazon CloudWatch Logs endpoints.
-	cwConfig := awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "logs", in.config.RegionName, in.awsConfig)
+	logsServiceName := awscommon.CreateServiceName("logs", in.config.AwsConfig.FIPSEnabled, in.config.RegionName)
+	cwConfig := awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, logsServiceName, in.config.RegionName, in.awsConfig)
 	svc := cloudwatchlogs.New(cwConfig)
 
 	var logGroupNames []string

diff --git a/x-pack/filebeat/input/awss3/config.go b/x-pack/filebeat/input/awss3/config.go
@@ -25,7 +25,6 @@ type config struct {
 	SQSWaitTime         time.Duration        `config:"sqs.wait_time"`         // The max duration for which the SQS ReceiveMessage call waits for a message to arrive in the queue before returning.
 	SQSMaxReceiveCount  int                  `config:"sqs.max_receive_count"` // The max number of times a message should be received (retried) before deleting it.
 	SQSScript           *scriptConfig        `config:"sqs.notification_parsing_script"`
-	FIPSEnabled         bool                 `config:"fips_enabled"`
 	MaxNumberOfMessages int                  `config:"max_number_of_messages"`
 	QueueURL            string               `config:"queue_url"`
 	BucketARN           string               `config:"bucket_arn"`
@@ -48,7 +47,6 @@ func defaultConfig() config {
 		BucketListPrefix:    "",
 		SQSWaitTime:         20 * time.Second,
 		SQSMaxReceiveCount:  5,
-		FIPSEnabled:         false,
 		MaxNumberOfMessages: 5,
 		PathStyle:           false,
 	}
@@ -99,7 +97,7 @@ func (c *config) Validate() error {
 			c.APITimeout, c.SQSWaitTime)
 	}
 
-	if c.FIPSEnabled && c.NonAWSBucketName != "" {
+	if c.AWSConfig.FIPSEnabled && c.NonAWSBucketName != "" {
 		return errors.New("fips_enabled cannot be used with a non-AWS S3 bucket.")
 	}
 	if c.PathStyle && c.NonAWSBucketName == "" {

diff --git a/x-pack/filebeat/input/awss3/config_test.go b/x-pack/filebeat/input/awss3/config_test.go
@@ -38,7 +38,6 @@ func TestConfig(t *testing.T) {
 			SQSWaitTime:         20 * time.Second,
 			BucketListInterval:  120 * time.Second,
 			BucketListPrefix:    "",
-			FIPSEnabled:         false,
 			PathStyle:           false,
 			MaxNumberOfMessages: 5,
 			ReaderConfig: readerConfig{

diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go
@@ -155,13 +155,11 @@ func (in *s3Input) Run(inputContext v2.Context, pipeline beat.Pipeline) error {
 }
 
 func (in *s3Input) createSQSReceiver(ctx v2.Context, client beat.Client) (*sqsReader, error) {
-	s3ServiceName := "s3"
-	if in.config.FIPSEnabled {
-		s3ServiceName = "s3-fips"
-	}
+	s3ServiceName := awscommon.CreateServiceName("s3", in.config.AWSConfig.FIPSEnabled, in.awsConfig.Region)
+	sqsServiceName := awscommon.CreateServiceName("sqs", in.config.AWSConfig.FIPSEnabled, in.awsConfig.Region)
 
 	sqsAPI := &awsSQSAPI{
-		client:            sqs.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, "sqs", in.awsConfig.Region, in.awsConfig)),
+		client:            sqs.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AWSConfig.Endpoint, sqsServiceName, in.awsConfig.Region, in.awsConfig)),
 		queueURL:          in.config.QueueURL,
 		apiTimeout:        in.config.APITimeout,
 		visibilityTimeout: in.config.VisibilityTimeout,
@@ -198,10 +196,7 @@ func (in *s3Input) createSQSReceiver(ctx v2.Context, client beat.Client) (*sqsRe
 }
 
 func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, client beat.Client, persistentStore *statestore.Store, states *states) (*s3Poller, error) {
-	s3ServiceName := "s3"
-	if in.config.FIPSEnabled {
-		s3ServiceName = "s3-fips"
-	}
+	s3ServiceName := awscommon.CreateServiceName("s3", in.config.AWSConfig.FIPSEnabled, in.awsConfig.Region)
 	var bucketName string
 	var bucketID string
 	if in.config.NonAWSBucketName != "" {

diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml
@@ -63,6 +63,12 @@
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   cloudwatch:
     enabled: false
 
@@ -115,6 +121,12 @@
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   ec2:
     enabled: false
 
@@ -167,6 +179,12 @@
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   elb:
     enabled: false
 
@@ -219,6 +237,12 @@
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   s3access:
     enabled: false
 
@@ -271,6 +295,12 @@
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
 
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
+
   vpcflow:
     enabled: false
 
@@ -322,3 +352,9 @@
 
     # The maximum number of messages to return from SQS. Valid values: 1 to 10.
     #var.max_number_of_messages: 5
+
+    # URL to proxy AWS API calls
+    #var.proxy_url: http://proxy:3128
+
+    # Configures the SSL settings, ie. set trusted CAs, ignore certificate verification....
+    #var.ssl:
diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/aws-s3.yml
@@ -81,6 +81,10 @@ max_number_of_messages: {{ .max_number_of_messages }}
 proxy_url: {{ .proxy_url }}
 {{ end }}
 
+{{ if .ssl }}
+ssl: {{ .ssl | tojson }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/cloudtrail/manifest.yml b/x-pack/filebeat/module/aws/cloudtrail/manifest.yml
@@ -28,6 +28,7 @@ var:
   - name: fips_enabled
   - name: proxy_url
   - name: max_number_of_messages
+  - name: ssl
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/aws-s3.yml
@@ -66,6 +66,10 @@ max_number_of_messages: {{ .max_number_of_messages }}
 proxy_url: {{ .proxy_url }}
 {{ end }}
 
+{{ if .ssl }}
+ssl: {{ .ssl | tojson }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml
@@ -22,6 +22,7 @@ var:
   - name: fips_enabled
   - name: proxy_url
   - name: max_number_of_messages
+  - name: ssl
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml b/x-pack/filebeat/module/aws/ec2/config/aws-s3.yml
@@ -66,6 +66,10 @@ max_number_of_messages: {{ .max_number_of_messages }}
 proxy_url: {{ .proxy_url }}
 {{ end }}
 
+{{ if .ssl }}
+ssl: {{ .ssl | tojson }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml
@@ -22,6 +22,7 @@ var:
   - name: fips_enabled
   - name: proxy_url
   - name: max_number_of_messages
+  - name: ssl
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/elb/config/aws-s3.yml b/x-pack/filebeat/module/aws/elb/config/aws-s3.yml
@@ -66,6 +66,10 @@ max_number_of_messages: {{ .max_number_of_messages }}
 proxy_url: {{ .proxy_url }}
 {{ end }}
 
+{{ if .ssl }}
+ssl: {{ .ssl | tojson }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/elb/manifest.yml b/x-pack/filebeat/module/aws/elb/manifest.yml
@@ -22,6 +22,7 @@ var:
   - name: fips_enabled
   - name: proxy_url
   - name: max_number_of_messages
+  - name: ssl
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml

diff --git a/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml b/x-pack/filebeat/module/aws/s3access/config/aws-s3.yml
@@ -66,6 +66,10 @@ max_number_of_messages: {{ .max_number_of_messages }}
 proxy_url: {{ .proxy_url }}
 {{ end }}
 
+{{ if .ssl }}
+ssl: {{ .ssl | tojson }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/aws/s3access/manifest.yml b/x-pack/filebeat/module/aws/s3access/manifest.yml
@@ -22,6 +22,7 @@ var:
   - name: fips_enabled
   - name: proxy_url
   - name: max_number_of_messages
+  - name: ssl
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/{{.input}}.yml
diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml
@@ -68,6 +68,10 @@ max_number_of_messages: {{ .max_number_of_messages }}
 proxy_url: {{ .proxy_url }}
 {{ end }}
 
+{{ if .ssl }}
+ssl: {{ .ssl | tojson }}
+{{ end }}
+
 {{ else if eq .input "file" }}
 
 type: log

diff --git a/x-pack/filebeat/module/aws/vpcflow/manifest.yml b/x-pack/filebeat/module/aws/vpcflow/manifest.yml
@@ -22,6 +22,7 @@ var:
   - name: fips_enabled
   - name: proxy_url
   - name: max_number_of_messages
+  - name: ssl
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/input.yml