Allow role_arn work with access keys for AWS (#25446) (#25503)

(cherry picked from commit 0f50230)

CHANGELOG.next.asciidoc

@@ -152,6 +152,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix 'make setup' instructions for a new beat {pull}24944[24944]
 - Fix discovery of short-living and failing pods in Kubernetes autodiscover {issue}22718[22718] {pull}24742[24742]
 - Fix panic when overwriting metadata {pull}24741[24741]
+- Fix role_arn to work with access keys for AWS. {pull}25446[25446]
 
 *Auditbeat*
 

x-pack/libbeat/autodiscover/providers/aws/elb/provider.go

@@ -63,8 +63,6 @@ func AutodiscoverBuilder(
 
 	// Construct MetricSet with a full regions list if there is no region specified.
 	if config.Regions == nil {
-		// set default region to make initial aws api call
-		awsCfg.Region = "us-west-1"
 		svcEC2 := ec2.New(awscommon.EnrichAWSConfigWithEndpoint(
 			config.AWSConfig.Endpoint, "ec2", awsCfg.Region, awsCfg))
 

x-pack/libbeat/common/aws/credentials.go

@@ -29,37 +29,53 @@ type ConfigAWS struct {
 }
 
 // GetAWSCredentials function gets aws credentials from the config.
-// If access_key_id and secret_access_key are given, then use them as credentials.
-// If role_arn is given, assume the IAM role instead.
-// If none of the above is given, then load from aws config file. If credential_profile_name is not
-// given, then load default profile from the aws config file.
+// If access keys given, use them as credentials.
+// If access keys are not given, then load from AWS config file. If credential_profile_name is not
+// given, default profile will be used.
+// If role_arn is given, assume the IAM role either with access keys or default profile.
 func GetAWSCredentials(config ConfigAWS) (awssdk.Config, error) {
-	logger := logp.NewLogger("get_aws_credentials")
-
 	// Check if accessKeyID or secretAccessKey or sessionToken is given from configuration
 	if config.AccessKeyID != "" || config.SecretAccessKey != "" || config.SessionToken != "" {
-		logger.Debug("Using access_key_id, secret_access_key and/or session_token for AWS credential")
-		awsConfig := defaults.Config()
-		awsCredentials := awssdk.Credentials{
-			AccessKeyID:     config.AccessKeyID,
-			SecretAccessKey: config.SecretAccessKey,
-		}
+		return getAccessKeys(config), nil
+	}
+	return getSharedCredentialProfile(config)
+}
 
-		if config.SessionToken != "" {
-			awsCredentials.SessionToken = config.SessionToken
-		}
+func getAccessKeys(config ConfigAWS) awssdk.Config {
+	logger := logp.NewLogger("getAccessKeys")
+	awsConfig := defaults.Config()
+	awsCredentials := awssdk.Credentials{
+		AccessKeyID:     config.AccessKeyID,
+		SecretAccessKey: config.SecretAccessKey,
+	}
 
-		awsConfig.Credentials = awssdk.StaticCredentialsProvider{
-			Value: awsCredentials,
-		}
-		return awsConfig, nil
+	if config.SessionToken != "" {
+		awsCredentials.SessionToken = config.SessionToken
+	}
+
+	awsConfig.Credentials = awssdk.StaticCredentialsProvider{
+		Value: awsCredentials,
+	}
+
+	// Set default region to make initial aws api call
+	awsConfig.Region = "us-east-1"
+
+	// Assume IAM role if iam_role config parameter is given
+	if config.RoleArn != "" {
+		logger.Debug("Using role arn and access keys for AWS credential")
+		return getRoleArn(config, awsConfig)
 	}
 
+	logger.Debug("Using access keys for AWS credential")
+	return awsConfig
+}
+
+func getSharedCredentialProfile(config ConfigAWS) (awssdk.Config, error) {
 	// If accessKeyID, secretAccessKey or sessionToken is not given, iam_role is not given, then load from default config
 	// Please see https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html
 	// with more details.
 	// If credential_profile_name is empty, then default profile is used.
-	logger.Debug("Using shared credential profile for AWS credential")
+	logger := logp.NewLogger("getSharedCredentialProfile")
 	var options []external.Config
 	if config.ProfileName != "" {
 		options = append(options, external.WithSharedConfigProfile(config.ProfileName))
@@ -78,16 +94,24 @@ func GetAWSCredentials(config ConfigAWS) (awssdk.Config, error) {
 		return awsConfig, errors.Wrap(err, "external.LoadDefaultAWSConfig failed with shared credential profile given")
 	}
 
-	if config.RoleArn == "" {
-		return awsConfig, nil
-	}
+	// Set default region to make initial aws api call
+	awsConfig.Region = "us-east-1"
 
 	// Assume IAM role if iam_role config parameter is given
-	logger.Debug("Using role_arn for AWS credential")
+	if config.RoleArn != "" {
+		logger.Debug("Using role arn and shared credential profile for AWS credential")
+		return getRoleArn(config, awsConfig), nil
+	}
+
+	logger.Debug("Using shared credential profile for AWS credential")
+	return awsConfig, nil
+}
+
+func getRoleArn(config ConfigAWS, awsConfig awssdk.Config) awssdk.Config {
 	stsSvc := sts.New(awsConfig)
 	stsCredProvider := stscreds.NewAssumeRoleProvider(stsSvc, config.RoleArn)
 	awsConfig.Credentials = stsCredProvider
-	return awsConfig, nil
+	return awsConfig
 }
 
 // EnrichAWSConfigWithEndpoint function enabled endpoint resolver for AWS

x-pack/libbeat/docs/aws-credentials-config.asciidoc

@@ -56,12 +56,6 @@ AWS_SECRET_ACCESS_KEY=abcd
 $ docker run --env-file env.list -d --name=metricbeat --user=root --volume="$(pwd)/metricbeat.aws.yml:/usr/share/metricbeat/metricbeat.yml:ro" docker.elastic.co/beats/metricbeat:7.11.1 metricbeat -e -E cloud.auth=elastic:1234 -E cloud.id=test-aws:1234
 ----
 
-* Use `role_arn`
-
-If `access_key_id` and `secret_access_key` are not given, then {beatname_lc} will
-check for `role_arn`. `role_arn` is used to specify which AWS IAM role to assume
-for generating temporary credentials.
-
 * Use `credential_profile_name` and/or `shared_credential_file`
 
 If `access_key_id`, `secret_access_key` and `role_arn` are all not given, then
@@ -80,10 +74,11 @@ for more details.
 
 * Use `role_arn`
 
-If `access_key_id` and `secret_access_key`, `credential_profile_name` and/or
-`shared_credential_file` are not given, then {beatname_lc} will check for
-`role_arn`. `role_arn` is used to specify which AWS IAM role to assume
-for generating temporary credentials.
+`role_arn` is used to specify which AWS IAM role to assume for generating
+temporary credentials. If `role_arn` is given, {beatname_lc} will check if
+access keys are given. If not, {beatname_lc} will check for credential profile
+name. If neither is given, default credential profile will be used. Please make
+sure credentials are given under either a credential profile or access keys.
 
 If running on Docker, the credential file needs to be provided via a volume
 mount. For example, with Metricbeat: