Cherry-pick elastic#17456 to 7.7: [Filebeat] Add access_key_id, secre…

…t_access_key and session_token into aws module config (elastic#17484) * [Filebeat] Add access_key_id, secret_access_key and session_token into aws module config (elastic#17456) (cherry picked from commit 64aeb2d)
leweafan · Apr 6, 2020 · 39b2054 · 39b2054
1 parent 44302b3
commit 39b2054
Show file tree

Hide file tree

Showing 18 changed files with 351 additions and 20 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -299,6 +299,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve AWS cloudtrail field mappings {issue}16086[16086] {issue}16110[16110] {pull}17155[17155]
 - Move azure-eventhub input to GA. {issue}15671[15671] {pull}17313[17313]
 - Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275]
+- Added access_key_id, secret_access_key and session_token into aws module config. {pull}17456[17456]
 
 *Heartbeat*
 

diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc
@@ -5,6 +5,8 @@ This file is generated! See scripts/docs_collector.py
 [[filebeat-module-aws]]
 [role="xpack"]
 
+:libbeat-xpack-dir: ../../../x-pack/libbeat
+
 :modulename: aws
 :has-dashboards: true
 
@@ -23,6 +25,11 @@ from network interfaces in AWS VPC. ELB access logs captures detailed informatio
 about requests sent to the load balancer. CloudTrail logs contain events
 that represent actions taken by a user, role or AWS service.
 
+The `aws` module requires AWS credentials configuration in order to make AWS API calls.
+Users can either use `access_key_id`, `secret_access_key` and/or
+`session_token`, or use shared AWS credentials file.
+Please see <<aws-credentials-options,AWS credentials options>> for more details.
+
 include::../include/gs-link.asciidoc[]
 
 [float]
@@ -38,6 +45,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -47,6 +57,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -56,6 +69,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -65,6 +81,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -74,6 +93,9 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
@@ -83,23 +105,17 @@ Example config:
     #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
     #var.shared_credential_file: /etc/filebeat/aws_credentials
     #var.credential_profile_name: fb-aws
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
     #var.visibility_timeout: 300s
     #var.api_timeout: 120s
     #var.endpoint: amazonaws.com
-
 ----
 
 *`var.queue_url`*::
 
-AWS SQS queue url.
-
-*`var.shared_credential_file`*::
-
-Filename of AWS credential file.
-
-*`var.credential_profile_name`*::
-
-AWS credential profile name.
+(Required) AWS SQS queue url.
 
 *`var.visibility_timeout`*::
 
@@ -114,6 +130,23 @@ Maximum duration before AWS API request will be interrupted. Default to be 120 s
 
 Custom endpoint used to access AWS APIs.
 
+*`var.shared_credential_file`*::
+
+Filename of AWS credential file.
+
+*`var.credential_profile_name`*::
+
+AWS credential profile name.
+
+*`var.access_key_id`*::
+First part of access key.
+
+*`var.access_key_id`*::
+Second part of access key.
+
+*`var.access_key_id`*::
+Required when using temporary security credentials.
+
 [float]
 === cloudtrail fileset
 
@@ -188,6 +221,9 @@ This fileset comes with a predefined dashboard:
 [role="screenshot"]
 image::./images/filebeat-aws-vpcflow-overview.png[]
 
+[id="aws-credentials-options"]
+include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[]
+
 
 [float]
 === Fields

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -111,6 +111,11 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -137,6 +142,42 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
+    # The duration that the received messages are hidden from ReceiveMessage request
+    # Default to be 300s
+    #var.visibility_timeout: 300s
+
+    # Maximum duration before AWS API request will be interrupted
+    # Default to be 120s
+    #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
+  ec2:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    #var.credential_profile_name: fb-aws
+
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -163,6 +204,11 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -189,6 +235,11 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -215,6 +266,11 @@ filebeat.modules:
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s

diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml
@@ -14,6 +14,11 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -40,6 +45,42 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
+    # The duration that the received messages are hidden from ReceiveMessage request
+    # Default to be 300s
+    #var.visibility_timeout: 300s
+
+    # Maximum duration before AWS API request will be interrupted
+    # Default to be 120s
+    #var.api_timeout: 120s
+
+    # Custom endpoint used to access AWS APIs
+    #var.endpoint: amazonaws.com
+
+  ec2:
+    enabled: false
+
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue
+
+    # Filename of AWS credential file
+    # If not set "$HOME/.aws/credentials" is used on Linux/Mac
+    # "%UserProfile%\.aws\credentials" is used on Windows
+    #var.shared_credential_file: /etc/filebeat/aws_credentials
+
+    # Profile name for aws credential
+    # If not set the default profile is used
+    #var.credential_profile_name: fb-aws
+
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -66,6 +107,11 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -92,6 +138,11 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s
@@ -118,6 +169,11 @@
     # If not set the default profile is used
     #var.credential_profile_name: fb-aws
 
+    # Use access_key_id, secret_access_key and/or session_token instead of shared credential file
+    #var.access_key_id: access_key_id
+    #var.secret_access_key: secret_access_key
+    #var.session_token: session_token
+
     # The duration that the received messages are hidden from ReceiveMessage request
     # Default to be 300s
     #var.visibility_timeout: 300s