From 6d933fda243591823ef180a28759eeb868de3b96 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 15:11:36 -0500 Subject: [PATCH 01/24] Respect when users explicitly set the profile --- src/AWSCredentials.jl | 10 ++++-- test/AWSCredentials.jl | 69 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index d6ca0588dd..2587e65584 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -110,12 +110,13 @@ Checks credential locations in the order: function AWSCredentials(; profile=nothing, throw_cred_error=true) creds = nothing credential_function = () -> nothing + explicit_profile = !isnothing(profile) profile = @something profile _aws_get_profile() # Define our search options, expected to be callable with no arguments. # Throw NoCredentials if none are found functions = [ - env_var_credentials, + () -> env_var_credentials(explicit_profile), () -> dot_aws_credentials(profile), () -> dot_aws_config(profile), credentials_from_webtoken, @@ -355,12 +356,15 @@ function ecs_instance_credentials() end """ - env_var_credentials() -> Union{AWSCredential, Nothing} + env_var_credentials(explicit_profile::Bool=false) -> Union{AWSCredential, Nothing} Use AWS environmental variables (e.g. AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, etc.) to create AWSCredentials. """ -function env_var_credentials() +function env_var_credentials(explicit_profile::Bool=false) + # Skip using environmental variables when a profile has been explicitly set + explicit_profile && return nothing + if haskey(ENV, "AWS_ACCESS_KEY_ID") && haskey(ENV, "AWS_SECRET_ACCESS_KEY") return AWSCredentials( ENV["AWS_ACCESS_KEY_ID"], diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 60c513b5d6..f283cfd395 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -399,6 +399,75 @@ end end end end + + @testset "Precedence" begin + mktempdir() do dir + config_file = joinpath(dir, "config") + creds_file = joinpath(dir, "creds") + + write( + creds_file, + """ + [profile1] + aws_access_key_id = AKI1 + aws_secret_access_key = SAK1 + + [profile2] + aws_access_key_id = AKI2 + aws_secret_access_key = SAK2 + """ + ) + + withenv( + [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., + "AWS_SHARED_CREDENTIALS_FILE" => creds_file, + "AWS_CONFIG_FILE" => config_file, + ) do + + @testset "explicit profile preferred" begin + withenv( + "AWS_PROFILE" => "profile1", + ) do + creds = AWSCredentials(profile="profile2") + @test creds.access_key_id == "AKI2" + end + + withenv( + "AWS_ACCESS_KEY_ID" => "AKI0", + "AWS_SECRET_ACCESS_KEY" => "SAK0", + ) do + creds = AWSCredentials(profile="profile2") + @test creds.access_key_id == "AKI2" + end + end + + @testset "AWS_ACCESS_KEY_ID preferred over AWS_PROFILE" begin + withenv( + "AWS_PROFILE" => "profile1", + "AWS_ACCESS_KEY_ID" => "AKI0", + "AWS_SECRET_ACCESS_KEY" => "SAK0", + ) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI0" + end + end + + # The AWS CLI used to use `AWS_DEFAULT_PROFILE` to set the AWS profile via the + # command line but this was deprecated in favor of `AWS_PROFILE`. We'll probably + # keeps support for this as long as AWS CLI continues to support it. + # https://github.com/aws/aws-cli/issues/2597 + @testset "AWS_PROFILE preferred over AWS_DEFAULT_PROFILE" begin + withenv( + "AWS_DEFAULT_PROFILE" => "profile1", + "AWS_PROFILE" => "profile2", + ) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI2" + end + end + end + end + end end @testset "Retrieving AWS Credentials" begin From 497dfbe2ea69611e8295e4288b93f8a9be89d42d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 15:32:32 -0500 Subject: [PATCH 02/24] Prefer SSO to credential files --- src/AWSCredentials.jl | 25 +++++++++++++++++++++++++ src/utilities/credentials.jl | 2 +- test/AWSCredentials.jl | 31 +++++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 2587e65584..a3e05c4de0 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -117,6 +117,7 @@ function AWSCredentials(; profile=nothing, throw_cred_error=true) # Throw NoCredentials if none are found functions = [ () -> env_var_credentials(explicit_profile), + () -> sso_credentials(profile), () -> dot_aws_credentials(profile), () -> dot_aws_config(profile), credentials_from_webtoken, @@ -408,6 +409,30 @@ function dot_aws_credentials_file() end end +function sso_credentials(profile=nothing) + config_file = @mock dot_aws_config_file() + + if isfile(config_file) + ini = read(Inifile(), config_file) + p = @something profile _aws_get_profile() + + # get all the fields for that profile + settings = _aws_profile_config(ini, p) + isempty(settings) && return nothing + + sso_start_url = get(settings, "sso_start_url", nothing) + + if !isnothing(sso_start_url) + access_key, secret_key, token, expiry = _aws_get_sso_credential_details(p, ini) + return AWSCredentials(access_key, secret_key, token; expiry=expiry) + else + return _aws_get_role(p, ini) + end + end + + return nothing +end + """ dot_aws_config(profile=nothing) -> Union{AWSCredential, Nothing} diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 599489cf49..6e2bc52407 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -71,7 +71,7 @@ function _aws_get_role(role::AbstractString, ini::Inifile) duration_seconds = get(settings, "duration_seconds", nothing) credentials = nothing - for f in (dot_aws_credentials, dot_aws_config) + for f in (sso_credentials, dot_aws_credentials, dot_aws_config) credentials = f(source_profile) credentials === nothing || break end diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index f283cfd395..c94572a652 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -405,9 +405,7 @@ end config_file = joinpath(dir, "config") creds_file = joinpath(dir, "creds") - write( - creds_file, - """ + basic_creds_content = """ [profile1] aws_access_key_id = AKI1 aws_secret_access_key = SAK1 @@ -416,7 +414,6 @@ end aws_access_key_id = AKI2 aws_secret_access_key = SAK2 """ - ) withenv( [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., @@ -425,6 +422,9 @@ end ) do @testset "explicit profile preferred" begin + isfile(config_file) && rm(config_file) + write(creds_file, basic_creds_content) + withenv( "AWS_PROFILE" => "profile1", ) do @@ -442,6 +442,9 @@ end end @testset "AWS_ACCESS_KEY_ID preferred over AWS_PROFILE" begin + isfile(config_file) && rm(config_file) + write(creds_file, basic_creds_content) + withenv( "AWS_PROFILE" => "profile1", "AWS_ACCESS_KEY_ID" => "AKI0", @@ -457,6 +460,9 @@ end # keeps support for this as long as AWS CLI continues to support it. # https://github.com/aws/aws-cli/issues/2597 @testset "AWS_PROFILE preferred over AWS_DEFAULT_PROFILE" begin + isfile(config_file) && rm(config_file) + write(creds_file, basic_creds_content) + withenv( "AWS_DEFAULT_PROFILE" => "profile1", "AWS_PROFILE" => "profile2", @@ -465,6 +471,23 @@ end @test creds.access_key_id == "AKI2" end end + + @testset "SSO preferred over credentials file" begin + write( + config_file, + """ + [profile profile1] + sso_start_url = https://my-sso-portal.awsapps.com/start + sso_role_name = role1 + """ + ) + write(creds_file, basic_creds_content) + + apply(Patches.sso_service_patches("AKI0", "SAK0")) do + creds = AWSCredentials(profile="profile1") + @test creds.access_key_id == "AKI0" + end + end end end end From 77d954862a5a7b555411e7bc63d3887f49e4e300 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 16:07:52 -0500 Subject: [PATCH 03/24] Test credential files are preferred over credential_process --- test/AWSCredentials.jl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index c94572a652..06b726e77d 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -488,6 +488,25 @@ end @test creds.access_key_id == "AKI0" end end + + @testset "Credential file over credential_process" begin + json = Dict( + "Version" => 1, + "AccessKeyId" => "AKI0", + "SecretAccessKey" => "SAK0", + ) + write( + config_file, + """ + [profile profile1] + credential_process = echo '$(JSON.json(json))' + """ + ) + write(creds_file, basic_creds_content) + + creds = AWSCredentials(profile="profile1") + @test creds.access_key_id == "AKI1" + end end end end From a4669539203ede3d3ff0023940e7ddcaf45958f0 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 16:12:18 -0500 Subject: [PATCH 04/24] Revise test ensuring credential_process is used over config creds --- test/AWSCredentials.jl | 72 ++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 06b726e77d..9f269d7475 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -507,6 +507,27 @@ end creds = AWSCredentials(profile="profile1") @test creds.access_key_id == "AKI1" end + + @testset "credential_process over config credentials" begin + json = Dict( + "Version" => 1, + "AccessKeyId" => "AKI0", + "SecretAccessKey" => "SAK0", + ) + write( + config_file, + """ + [profile profile1] + aws_access_key_id = AKI1 + aws_secret_access_key = SAK1 + credential_process = echo '$(JSON.json(json))' + """ + ) + isfile(creds_file) && rm(creds_file) + + creds = AWSCredentials(profile="profile1") + @test creds.access_key_id == "AKI0" + end end end end @@ -640,47 +661,22 @@ end chmod(credential_process_file, 0o700) withenv("AWS_CONFIG_FILE" => config_file) do - @testset "support" begin - open(config_file, "w") do io - write( - io, - """ - [profile $(test_values["Test-Config-Profile"])] - credential_process = $(abspath(credential_process_file)) - """, - ) - end - - result = dot_aws_config(test_values["Test-Config-Profile"]) - - @test result.access_key_id == test_values["Test-AccessKeyId"] - @test result.secret_key == test_values["Test-SecretAccessKey"] - @test isempty(result.token) - @test result.expiry == typemax(DateTime) + open(config_file, "w") do io + write( + io, + """ + [profile $(test_values["Test-Config-Profile"])] + credential_process = $(abspath(credential_process_file)) + """, + ) end - # The AWS CLI uses the config file `credential_process` setting over - # specifying the config file `aws_access_key_id`/`aws_secret_access_key`. - @testset "precedence" begin - open(config_file, "w") do io - write( - io, - """ - [profile $(test_values["Test-Config-Profile"])] - aws_access_key_id = invalid - aws_secret_access_key = invalid - credential_process = $(abspath(credential_process_file)) - """, - ) - end - - result = dot_aws_config(test_values["Test-Config-Profile"]) + result = dot_aws_config(test_values["Test-Config-Profile"]) - @test result.access_key_id == test_values["Test-AccessKeyId"] - @test result.secret_key == test_values["Test-SecretAccessKey"] - @test isempty(result.token) - @test result.expiry == typemax(DateTime) - end + @test result.access_key_id == test_values["Test-AccessKeyId"] + @test result.secret_key == test_values["Test-SecretAccessKey"] + @test isempty(result.token) + @test result.expiry == typemax(DateTime) end end end From 58e44c3e54061cbfa8ca62208fc37c70ed6eb575 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 16:19:55 -0500 Subject: [PATCH 05/24] Revise docstrings --- src/AWSCredentials.jl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index a3e05c4de0..478e7615c3 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -409,6 +409,15 @@ function dot_aws_credentials_file() end end +""" + sso_credentials(profile=nothing) -> Union{AWSCredential, Nothing} + +Retrieve credentials via AWS single sign-on settings defined in the `profile` within the AWS +config file. If no SSO settings are found for the `profile` `nothing` is returned. + +# Arguments +- `profile`: Specific profile used to get `AWSCredential`s, default is `nothing` +""" function sso_credentials(profile=nothing) config_file = @mock dot_aws_config_file() @@ -437,8 +446,8 @@ end dot_aws_config(profile=nothing) -> Union{AWSCredential, Nothing} Retrieve AWSCredentials for the default or specified profile from the `~/.aws/config` file. -Single sign-on profiles are also valid. If this fails, try to retrieve credentials from -`_aws_get_role()`, otherwise return `nothing` +If this fails, try to retrieve credentials from `_aws_get_role()`, otherwise return +`nothing`. # Arguments - `profile`: Specific profile used to get AWSCredentials, default is `nothing` From 9a40d2bda36108753a98e441eea808379849dc8f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 10:00:34 -0500 Subject: [PATCH 06/24] Comment on credential precedence testset --- test/AWSCredentials.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 9f269d7475..8de2660d17 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -400,7 +400,15 @@ end end end - @testset "Precedence" begin + # Verify that the search order for credentials mirrors the behavior of the AWS CLI + # (version 2.11.13). Whenever support is added for new credential types new tests should + # be added to this test set. To determine the credential preference order used by AWS + # CLI it is recommended you use a set of valid credentials and a set of invalid + # credentials to determine the precedence. + # + # Documentation on credential preference for the AWS SDK for .NET: + # https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/creds-assign.html + @testset "Credential Precedence" begin mktempdir() do dir config_file = joinpath(dir, "config") creds_file = joinpath(dir, "creds") From eab2126b6b4c6237f1586edccec623b6efc809f6 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 10:06:52 -0500 Subject: [PATCH 07/24] Remove SSO support from dot_aws_config --- src/AWSCredentials.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 478e7615c3..9f416968dc 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -465,7 +465,6 @@ function dot_aws_config(profile=nothing) credential_process = get(settings, "credential_process", nothing) access_key = get(settings, "aws_access_key_id", nothing) - sso_start_url = get(settings, "sso_start_url", nothing) if !isnothing(credential_process) cmd = Cmd(Base.shell_split(credential_process)) @@ -473,9 +472,6 @@ function dot_aws_config(profile=nothing) elseif !isnothing(access_key) access_key, secret_key, token = _aws_get_credential_details(p, ini) return AWSCredentials(access_key, secret_key, token) - elseif !isnothing(sso_start_url) - access_key, secret_key, token, expiry = _aws_get_sso_credential_details(p, ini) - return AWSCredentials(access_key, secret_key, token; expiry=expiry) else return _aws_get_role(p, ini) end From 5e873876b2c751086abe369568f4d5c83978475e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 16:20:45 -0500 Subject: [PATCH 08/24] Set project version to 1.86.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 9d152ec62f..28485f2c57 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AWS" uuid = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" license = "MIT" -version = "1.85.0" +version = "1.86.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From e13fca81e34f0815099ede9174d8f1dc0922801f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 10:40:11 -0500 Subject: [PATCH 09/24] Formatting --- test/AWSCredentials.jl | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 8de2660d17..9d5a22eef3 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -428,23 +428,21 @@ end "AWS_SHARED_CREDENTIALS_FILE" => creds_file, "AWS_CONFIG_FILE" => config_file, ) do - @testset "explicit profile preferred" begin isfile(config_file) && rm(config_file) write(creds_file, basic_creds_content) - withenv( - "AWS_PROFILE" => "profile1", - ) do - creds = AWSCredentials(profile="profile2") + withenv("AWS_PROFILE" => "profile1") do + creds = AWSCredentials(; profile="profile2") @test creds.access_key_id == "AKI2" end withenv( "AWS_ACCESS_KEY_ID" => "AKI0", "AWS_SECRET_ACCESS_KEY" => "SAK0", + # format trick: using this comment to force use of multiple lines ) do - creds = AWSCredentials(profile="profile2") + creds = AWSCredentials(; profile="profile2") @test creds.access_key_id == "AKI2" end end @@ -474,6 +472,7 @@ end withenv( "AWS_DEFAULT_PROFILE" => "profile1", "AWS_PROFILE" => "profile2", + # format trick: using this comment to force use of multiple lines ) do creds = AWSCredentials() @test creds.access_key_id == "AKI2" @@ -487,12 +486,12 @@ end [profile profile1] sso_start_url = https://my-sso-portal.awsapps.com/start sso_role_name = role1 - """ + """, ) write(creds_file, basic_creds_content) apply(Patches.sso_service_patches("AKI0", "SAK0")) do - creds = AWSCredentials(profile="profile1") + creds = AWSCredentials(; profile="profile1") @test creds.access_key_id == "AKI0" end end @@ -502,17 +501,18 @@ end "Version" => 1, "AccessKeyId" => "AKI0", "SecretAccessKey" => "SAK0", + # format trick: using this comment to force use of multiple lines ) write( config_file, """ [profile profile1] credential_process = echo '$(JSON.json(json))' - """ + """, ) write(creds_file, basic_creds_content) - creds = AWSCredentials(profile="profile1") + creds = AWSCredentials(; profile="profile1") @test creds.access_key_id == "AKI1" end @@ -521,6 +521,7 @@ end "Version" => 1, "AccessKeyId" => "AKI0", "SecretAccessKey" => "SAK0", + # format trick: using this comment to force use of multiple lines ) write( config_file, @@ -529,11 +530,11 @@ end aws_access_key_id = AKI1 aws_secret_access_key = SAK1 credential_process = echo '$(JSON.json(json))' - """ + """, ) isfile(creds_file) && rm(creds_file) - creds = AWSCredentials(profile="profile1") + creds = AWSCredentials(; profile="profile1") @test creds.access_key_id == "AKI0" end end From 962f34b5f2eb52bc10878e9ec7057e9a3ca97938 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 13:25:33 -0500 Subject: [PATCH 10/24] Add EC2 instance credential precedence --- src/AWSCredentials.jl | 4 +- test/AWSCredentials.jl | 86 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 9f416968dc..90743440ee 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -113,8 +113,8 @@ function AWSCredentials(; profile=nothing, throw_cred_error=true) explicit_profile = !isnothing(profile) profile = @something profile _aws_get_profile() - # Define our search options, expected to be callable with no arguments. - # Throw NoCredentials if none are found + # Define the credential preference order + # https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence functions = [ () -> env_var_credentials(explicit_profile), () -> sso_credentials(profile), diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 9d5a22eef3..a592f25635 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -406,8 +406,10 @@ end # CLI it is recommended you use a set of valid credentials and a set of invalid # credentials to determine the precedence. # - # Documentation on credential preference for the AWS SDK for .NET: - # https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/creds-assign.html + # Documentation on credential precedence: + # - https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence + # - https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/creds-assign.html + # - https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html @testset "Credential Precedence" begin mktempdir() do dir config_file = joinpath(dir, "config") @@ -423,6 +425,55 @@ end aws_secret_access_key = SAK2 """ + ec2_expiration = floor(now(UTC), Second) + ec2_json = Dict( + "AccessKeyId" => "AKI_EC2", + "SecretAccessKey" => "SAK_EC2", + "Token" => "TOK_EC2", + "Expiration" => Dates.format(ec2_expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + ) + + function ec2_metadata(url::AbstractString) + name = "local-credentials" + metadata_uri = "http://169.254.169.254/latest/meta-data" + if url == "$metadata_uri/iam/info" + return HTTP.Response(200, JSON.json("InstanceProfileArn" => "ARN0")) + elseif url == "$metadata_uri/iam/security-credentials/" + return HTTP.Response(200, name) + elseif url == "$metadata_uri/iam/security-credentials/$name" + return HTTP.Response(200, JSON.json(ec2_json)) + else + return HTTP.Response(404) + end + end + + ecs_expiration = floor(now(UTC), Second) + ecs_json = Dict( + "AccessKeyId" => "AKI_ECS", + "SecretAccessKey" => "SAK_ECS", + "Token" => "TOK_ECS", + "Expiration" => Dates.format(ecs_expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + ) + + function ecs_metadata(url::AbstractString) + if startswith(url, "http://169.254.170.2/") + return HTTP.Response(200, JSON.json(ecs_json)) + else + return HTTP.Response(404) + end + end + + function http_request_patcher(funcs) + @patch function HTTP.request(method, url, args...; kwargs...) + local r + for f in funcs + r = f(string(url)) + r.status != 404 && break + end + return r + end + end + withenv( [k => nothing for k in filter(startswith("AWS_"), keys(ENV))]..., "AWS_SHARED_CREDENTIALS_FILE" => creds_file, @@ -537,6 +588,37 @@ end creds = AWSCredentials(; profile="profile1") @test creds.access_key_id == "AKI0" end + + @testset "default config credentials over EC2 instance credentials" begin + write( + config_file, + """ + [default] + aws_access_key_id = AKI1 + aws_secret_access_key = SAK1 + """, + ) + isfile(creds_file) && rm(creds_file) + + apply(http_request_patcher([ecs_metadata])) do + @test isnothing(AWS._aws_get_profile(; default=nothing)) + + creds = AWSCredentials() + @test creds.access_key_id == "AKI1" + end + end + + # Note: The AWS CLI behavior was not tested here as this scenario is + # challenging to test for. + @testset "EC2 instance credentials over container credentials" begin + isfile(config_file) && rm(config_file) + isfile(creds_file) && rm(creds_file) + + apply(http_request_patcher([ec2_metadata, ecs_metadata])) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI_EC2" + end + end end end end From 8d65dbd49680a8288fe915cdfd8e010d7883839a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 13:30:36 -0500 Subject: [PATCH 11/24] SSO fix --- src/AWSCredentials.jl | 3 ++- test/AWSCredentials.jl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 90743440ee..a0ab383b54 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -23,7 +23,8 @@ export AWSCredentials, external_process_credentials, localhost_is_ec2, localhost_is_lambda, - localhost_maybe_ec2 + localhost_maybe_ec2, + sso_credentials function localhost_maybe_ec2() return localhost_is_ec2() || isfile("/sys/devices/virtual/dmi/id/product_uuid") diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index a592f25635..b154d367b5 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -725,7 +725,7 @@ end test_values["AccessKeyId"], test_values["SecretAccessKey"] ), ) do - specified_result = dot_aws_config(test_values["Test-SSO-Profile"]) + specified_result = sso_credentials(test_values["Test-SSO-Profile"]) @test specified_result.access_key_id == test_values["AccessKeyId"] @test specified_result.secret_key == test_values["SecretAccessKey"] From 9483fb8b4516c2bb8bb4e9cfd1644450dac7907b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 13:33:51 -0500 Subject: [PATCH 12/24] Expiration date format --- test/AWSCredentials.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index b154d367b5..e2cd2e9a50 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -13,6 +13,8 @@ macro test_ecode(error_codes, expr) end end +const EXPIRATION_FMT = dateformat"yyyy-mm-dd\THH:MM:SS\Z" + @testset "Load Credentials" begin user = aws_user_arn(aws) @test occursin(r"^arn:aws:(iam|sts)::[0-9]+:[^:]+$", user) @@ -425,12 +427,11 @@ end aws_secret_access_key = SAK2 """ - ec2_expiration = floor(now(UTC), Second) ec2_json = Dict( "AccessKeyId" => "AKI_EC2", "SecretAccessKey" => "SAK_EC2", "Token" => "TOK_EC2", - "Expiration" => Dates.format(ec2_expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + "Expiration" => Dates.format(now(UTC), EXPIRATION_FMT), ) function ec2_metadata(url::AbstractString) @@ -447,12 +448,11 @@ end end end - ecs_expiration = floor(now(UTC), Second) ecs_json = Dict( "AccessKeyId" => "AKI_ECS", "SecretAccessKey" => "SAK_ECS", "Token" => "TOK_ECS", - "Expiration" => Dates.format(ecs_expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + "Expiration" => Dates.format(now(UTC), EXPIRATION_FMT), ) function ecs_metadata(url::AbstractString) @@ -978,7 +978,7 @@ end "AccessKeyId" => "access-key", "SecretAccessKey" => "secret-key", "SessionToken" => "session-token", - "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + "Expiration" => Dates.format(expiration, EXPIRATION_FMT), ) creds = external_process_credentials(gen_process(temporary_resp)) @test creds.access_key_id == temporary_resp["AccessKeyId"] @@ -995,7 +995,7 @@ end "Version" => 1, "AccessKeyId" => "access-key", "SecretAccessKey" => "secret-key", - "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + "Expiration" => Dates.format(expiration, EXPIRATION_FMT), ) ex = KeyError("SessionToken") @test_throws ex external_process_credentials(gen_process(missing_token_resp)) From 67c7f3c8142d8b9082526a0acdb7449c41da4b90 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 13:57:00 -0500 Subject: [PATCH 13/24] Fix source_profile traversal --- src/AWSCredentials.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index a0ab383b54..2b843354e8 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -435,8 +435,6 @@ function sso_credentials(profile=nothing) if !isnothing(sso_start_url) access_key, secret_key, token, expiry = _aws_get_sso_credential_details(p, ini) return AWSCredentials(access_key, secret_key, token; expiry=expiry) - else - return _aws_get_role(p, ini) end end From 4d98c4ca64db07e96ff347a0c63e8808fa85ffe2 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 22:58:26 -0500 Subject: [PATCH 14/24] Fix mistake in precedence tests --- test/AWSCredentials.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index e2cd2e9a50..1fd10f709d 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -600,7 +600,7 @@ end ) isfile(creds_file) && rm(creds_file) - apply(http_request_patcher([ecs_metadata])) do + apply(http_request_patcher([ec2_metadata])) do @test isnothing(AWS._aws_get_profile(; default=nothing)) creds = AWSCredentials() @@ -610,7 +610,7 @@ end # Note: The AWS CLI behavior was not tested here as this scenario is # challenging to test for. - @testset "EC2 instance credentials over container credentials" begin + @testset "EC2 instance credentials over ECS container credentials" begin isfile(config_file) && rm(config_file) isfile(creds_file) && rm(creds_file) From 9ade651a35c6740b8044ecf46447be071ee0dde0 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 23:01:20 -0500 Subject: [PATCH 15/24] Implicit ECS credential support --- src/AWSCredentials.jl | 30 +++++++++++------ test/AWSCredentials.jl | 73 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 86 insertions(+), 17 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 2b843354e8..cc15c90e62 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -122,8 +122,9 @@ function AWSCredentials(; profile=nothing, throw_cred_error=true) () -> dot_aws_credentials(profile), () -> dot_aws_config(profile), credentials_from_webtoken, - ecs_instance_credentials, + () -> ecs_instance_credentials(; require_env=true), () -> ec2_instance_credentials(profile), + () -> ecs_instance_credentials(; require_env=false), ] # Loop through our search locations until we get credentials back @@ -317,13 +318,18 @@ function ec2_instance_credentials(profile::AbstractString) end """ - ecs_instance_credentials() -> Union{AWSCredential, Nothing} + ecs_instance_credentials(; require_env::Bool=true) -> Union{AWSCredential, Nothing} -Retrieve credentials from the local endpoint. Return `nothing` if not running on an ECS -instance. +Retrieve credentials from the ECS credential endpoint. Return `nothing` if the ECS +credential endpoint is not available. More information can be found at: -https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html +- https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html +- https://docs.aws.amazon.com/sdkref/latest/guide/feature-container-credentials.html + +# Keywords +- `require_env::Bool`: Only attempt to connect to the ECS credential endpoint when the + environmental variable `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` is set. # Returns - `AWSCredentials`: AWSCredentials from `ECS` credentials URI, `nothing` if the Env Var is @@ -333,14 +339,18 @@ https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html - `StatusError`: If the response status is >= 300 - `ParsingError`: Invalid HTTP request target """ -function ecs_instance_credentials() - if !haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") +function ecs_instance_credentials(; require_env::Bool=true) + if require_env && !haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") return nothing end - uri = ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] - - response = @mock HTTP.request("GET", "http://169.254.170.2$uri") + path = get(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "") + response = try + @mock HTTP.request("GET", "http://169.254.170.2$path"; retry=false, connect_timeout=5) + catch e + e isa HTTP.Exceptions.ConnectError && return nothing + rethrow() + end new_creds = String(response.body) new_creds = JSON.parse(new_creds) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 1fd10f709d..f6b8b9f86b 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -589,6 +589,27 @@ end @test creds.access_key_id == "AKI0" end + @testset "default config credentials over ECS container credentials ENV variables" begin + write( + config_file, + """ + [default] + aws_access_key_id = AKI1 + aws_secret_access_key = SAK1 + """, + ) + isfile(creds_file) && rm(creds_file) + + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-creds") do + apply(http_request_patcher([ecs_metadata])) do + @test isnothing(AWS._aws_get_profile(; default=nothing)) + + creds = AWSCredentials() + @test creds.access_key_id == "AKI1" + end + end + end + @testset "default config credentials over EC2 instance credentials" begin write( config_file, @@ -608,6 +629,18 @@ end end end + @testset "ECS container credentials ENV variables over EC2 instance credentials" begin + isfile(config_file) && rm(config_file) + isfile(creds_file) && rm(creds_file) + + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/get-creds") do + apply(http_request_patcher([ec2_metadata, ecs_metadata])) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI_ECS" + end + end + end + # Note: The AWS CLI behavior was not tested here as this scenario is # challenging to test for. @testset "EC2 instance credentials over ECS container credentials" begin @@ -649,14 +682,14 @@ end uri = test_values["URI"] url = string(url) - if url == "http://169.254.169.254/latest/meta-data/iam/info" + metadata_uri = "http://169.254.169.254/latest/meta-data" + if url == "$metadata_uri/iam/info" instance_profile_arn = test_values["InstanceProfileArn"] return HTTP.Response("{\"InstanceProfileArn\": \"$instance_profile_arn\"}") - elseif url == "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + elseif url == "$metadata_uri/iam/security-credentials/" return HTTP.Response(test_values["Security-Credentials"]) - elseif url == - "http://169.254.169.254/latest/meta-data/iam/security-credentials/$security_credentials" || - url == "http://169.254.170.2$uri" + elseif url == "$metadata_uri/iam/security-credentials/$security_credentials" || + startswith(url, "http://169.254.170.2") my_dict = JSON.json(test_values) response = HTTP.Response(my_dict) return response @@ -879,7 +912,17 @@ end @testset "Instance - ECS" begin withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => test_values["URI"]) do apply(_http_request_patch) do - result = ecs_instance_credentials() + result = ecs_instance_credentials(; require_env=true) + @test result.access_key_id == test_values["AccessKeyId"] + @test result.secret_key == test_values["SecretAccessKey"] + @test result.token == test_values["Token"] + @test result.user_arn == test_values["RoleArn"] + @test result.expiry == test_values["Expiration"] + @test result.renew == ecs_instance_credentials + + # TODO: If supported it would be better to just compare against compare + # against the last `result`. + result = ecs_instance_credentials(; require_env=false) @test result.access_key_id == test_values["AccessKeyId"] @test result.secret_key == test_values["SecretAccessKey"] @test result.token == test_values["Token"] @@ -888,6 +931,20 @@ end @test result.renew == ecs_instance_credentials end end + + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => nothing) do + apply(_http_request_patch) do + result = ecs_instance_credentials(; require_env=false) + @test result.access_key_id == test_values["AccessKeyId"] + @test result.secret_key == test_values["SecretAccessKey"] + @test result.token == test_values["Token"] + @test result.user_arn == test_values["RoleArn"] + @test result.expiry == test_values["Expiration"] + @test result.renew == ecs_instance_credentials + + @test ecs_instance_credentials(; require_env=true) === nothing + end + end end @testset "Web Identity File" begin @@ -1012,7 +1069,9 @@ end @testset "Credentials Not Found" begin patches = [ - @patch HTTP.request(method::String, url; kwargs...) = nothing + @patch function HTTP.request(method::String, url, args...; kwargs...) + throw(HTTP.Exceptions.ConnectError(url, "host is unreachable")) + end Patches._cred_file_patch Patches._config_file_patch ] From 0ad302007c0077b44a8fd1a712b6f4f5f617ee1e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 23:10:53 -0500 Subject: [PATCH 16/24] Only support ECS credentials when ENV vars are set --- src/AWSCredentials.jl | 25 ++++++++++++++----------- test/AWSCredentials.jl | 35 +++++++++++------------------------ 2 files changed, 25 insertions(+), 35 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index cc15c90e62..b04e10a913 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -114,17 +114,19 @@ function AWSCredentials(; profile=nothing, throw_cred_error=true) explicit_profile = !isnothing(profile) profile = @something profile _aws_get_profile() - # Define the credential preference order + # Define the credential preference order: # https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence + # + # Take note that the AWS CLI in practise preferse to use ECS instance credentials over + # EC2 credentials when the `AWS_CONTAINER_*` environmental variables are set. functions = [ () -> env_var_credentials(explicit_profile), () -> sso_credentials(profile), () -> dot_aws_credentials(profile), () -> dot_aws_config(profile), credentials_from_webtoken, - () -> ecs_instance_credentials(; require_env=true), + ecs_instance_credentials, () -> ec2_instance_credentials(profile), - () -> ecs_instance_credentials(; require_env=false), ] # Loop through our search locations until we get credentials back @@ -318,7 +320,7 @@ function ec2_instance_credentials(profile::AbstractString) end """ - ecs_instance_credentials(; require_env::Bool=true) -> Union{AWSCredential, Nothing} + ecs_instance_credentials() -> Union{AWSCredential, Nothing} Retrieve credentials from the ECS credential endpoint. Return `nothing` if the ECS credential endpoint is not available. @@ -327,10 +329,6 @@ More information can be found at: - https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html - https://docs.aws.amazon.com/sdkref/latest/guide/feature-container-credentials.html -# Keywords -- `require_env::Bool`: Only attempt to connect to the ECS credential endpoint when the - environmental variable `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` is set. - # Returns - `AWSCredentials`: AWSCredentials from `ECS` credentials URI, `nothing` if the Env Var is not set (not running on an ECS container instance) @@ -339,12 +337,17 @@ More information can be found at: - `StatusError`: If the response status is >= 300 - `ParsingError`: Invalid HTTP request target """ -function ecs_instance_credentials(; require_env::Bool=true) - if require_env && !haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") +function ecs_instance_credentials() + # > If your Amazon EC2 instance is using at least version `1.11.0` of the container agent + # > and a supported version of the AWS CLI or SDKs, then the SDK client will see that the + # > `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` variable is available, and it will use the + # > provided credentials to make calls to the AWS APIs + # – https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html + if !haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") return nothing end - path = get(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "") + path = ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] response = try @mock HTTP.request("GET", "http://169.254.170.2$path"; retry=false, connect_timeout=5) catch e diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index f6b8b9f86b..24e13b98f8 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -641,8 +641,10 @@ end end end - # Note: The AWS CLI behavior was not tested here as this scenario is - # challenging to test for. + # Note: It appears that the ECS container credentials are only used when + # a `AWS_CONTAINER_*` environmental variable is set. However, this test + # ensures that if we do add implicit support that the documented precedence + # order is not violated. @testset "EC2 instance credentials over ECS container credentials" begin isfile(config_file) && rm(config_file) isfile(creds_file) && rm(creds_file) @@ -689,7 +691,7 @@ end elseif url == "$metadata_uri/iam/security-credentials/" return HTTP.Response(test_values["Security-Credentials"]) elseif url == "$metadata_uri/iam/security-credentials/$security_credentials" || - startswith(url, "http://169.254.170.2") + url == "http://169.254.170.2$uri" my_dict = JSON.json(test_values) response = HTTP.Response(my_dict) return response @@ -912,17 +914,7 @@ end @testset "Instance - ECS" begin withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => test_values["URI"]) do apply(_http_request_patch) do - result = ecs_instance_credentials(; require_env=true) - @test result.access_key_id == test_values["AccessKeyId"] - @test result.secret_key == test_values["SecretAccessKey"] - @test result.token == test_values["Token"] - @test result.user_arn == test_values["RoleArn"] - @test result.expiry == test_values["Expiration"] - @test result.renew == ecs_instance_credentials - - # TODO: If supported it would be better to just compare against compare - # against the last `result`. - result = ecs_instance_credentials(; require_env=false) + result = ecs_instance_credentials() @test result.access_key_id == test_values["AccessKeyId"] @test result.secret_key == test_values["SecretAccessKey"] @test result.token == test_values["Token"] @@ -933,17 +925,12 @@ end end withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => nothing) do - apply(_http_request_patch) do - result = ecs_instance_credentials(; require_env=false) - @test result.access_key_id == test_values["AccessKeyId"] - @test result.secret_key == test_values["SecretAccessKey"] - @test result.token == test_values["Token"] - @test result.user_arn == test_values["RoleArn"] - @test result.expiry == test_values["Expiration"] - @test result.renew == ecs_instance_credentials + @test ecs_instance_credentials() === nothing + end - @test ecs_instance_credentials(; require_env=true) === nothing - end + withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/invalid") do + # Should internally generate a `ConnectError` + @test ecs_instance_credentials() === nothing end end From 16bbf81ad48313b92099733ccf71abdd80388d40 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 10 May 2023 23:23:58 -0500 Subject: [PATCH 17/24] Deprecate SSO support in dot_aws_config --- src/AWSCredentials.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index b04e10a913..621dba3017 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -477,6 +477,7 @@ function dot_aws_config(profile=nothing) credential_process = get(settings, "credential_process", nothing) access_key = get(settings, "aws_access_key_id", nothing) + sso_start_url = get(settings, "sso_start_url", nothing) if !isnothing(credential_process) cmd = Cmd(Base.shell_split(credential_process)) @@ -484,6 +485,13 @@ function dot_aws_config(profile=nothing) elseif !isnothing(access_key) access_key, secret_key, token = _aws_get_credential_details(p, ini) return AWSCredentials(access_key, secret_key, token) + elseif !isnothing(sso_start_url) + Base.depwarn( + "SSO support in `dot_aws_config` is deprecated, use `sso_credentials` instead.", + :dot_aws_config, + ) + access_key, secret_key, token, expiry = _aws_get_sso_credential_details(p, ini) + return AWSCredentials(access_key, secret_key, token; expiry=expiry) else return _aws_get_role(p, ini) end From a549e41d6a2e9ce5021560521e1494ac35d0d49f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 11 May 2023 08:55:04 -0500 Subject: [PATCH 18/24] Revise ECS explainer comment --- src/AWSCredentials.jl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 621dba3017..568bcd2f7a 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -338,18 +338,19 @@ More information can be found at: - `ParsingError`: Invalid HTTP request target """ function ecs_instance_credentials() - # > If your Amazon EC2 instance is using at least version `1.11.0` of the container agent - # > and a supported version of the AWS CLI or SDKs, then the SDK client will see that the - # > `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` variable is available, and it will use the - # > provided credentials to make calls to the AWS APIs + # The Amazon ECS agent will automatically populate the environmental variable + # `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` when running inside of an ECS task. We're + # interpreting this to mean than ECS credential provider should only be used if the + # `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` variable is set. # – https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html - if !haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + if haskey(ENV, "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + endpoint = "http://169.254.170.2" * ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] + else return nothing end - path = ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] response = try - @mock HTTP.request("GET", "http://169.254.170.2$path"; retry=false, connect_timeout=5) + @mock HTTP.request("GET", endpoint; retry=false, connect_timeout=5) catch e e isa HTTP.Exceptions.ConnectError && return nothing rethrow() From 92a2c0918602c6e94e0be2926e108a0ccdeeea5b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 11 May 2023 09:01:11 -0500 Subject: [PATCH 19/24] Handle URI in patch with ConnectError --- test/AWSCredentials.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 24e13b98f8..a4214a162d 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -1057,7 +1057,7 @@ end @testset "Credentials Not Found" begin patches = [ @patch function HTTP.request(method::String, url, args...; kwargs...) - throw(HTTP.Exceptions.ConnectError(url, "host is unreachable")) + throw(HTTP.Exceptions.ConnectError(string(url), "host is unreachable")) end Patches._cred_file_patch Patches._config_file_patch From 6fd5f61d3bc93fd9c2d49bb86526eb1b9a2cd015 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 11 May 2023 09:15:48 -0500 Subject: [PATCH 20/24] Update docstrings for dot_aws_config/dot_aws_credentials --- src/AWSCredentials.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 568bcd2f7a..7d4db970ba 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -397,7 +397,9 @@ end """ dot_aws_credentials(profile=nothing) -> Union{AWSCredential, Nothing} -Retrieve AWSCredentials from the `~/.aws/credentials` file +Retrieve `AWSCredentials` from the AWS CLI credentials file. The credential file defaults to +"~/.aws/credentials" but can be specified using the env variable +`AWS_SHARED_CREDENTIALS_FILE`. # Arguments - `profile`: Specific profile used to get AWSCredentials, default is `nothing` @@ -458,9 +460,11 @@ end """ dot_aws_config(profile=nothing) -> Union{AWSCredential, Nothing} -Retrieve AWSCredentials for the default or specified profile from the `~/.aws/config` file. -If this fails, try to retrieve credentials from `_aws_get_role()`, otherwise return -`nothing`. +Retrieve `AWSCredentials` from the AWS CLI configuration file. The configuration file +defaults to "~/.aws/config" but can be specified using the env variable `AWS_CONFIG_FILE`. +When no credentials can be found for the given `profile` then the associated +`source_profile` will be used to recursively look up credentials of parent profiles. If +still no credentials can be found then `nothing` will be returned. # Arguments - `profile`: Specific profile used to get AWSCredentials, default is `nothing` From 4f017a64c2a94323dbb22c2fa09f1c09b0704402 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 11 May 2023 09:23:03 -0500 Subject: [PATCH 21/24] Update documentation to use correct type --- src/AWSCredentials.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 7d4db970ba..c5650321a8 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -320,7 +320,7 @@ function ec2_instance_credentials(profile::AbstractString) end """ - ecs_instance_credentials() -> Union{AWSCredential, Nothing} + ecs_instance_credentials() -> Union{AWSCredentials, Nothing} Retrieve credentials from the ECS credential endpoint. Return `nothing` if the ECS credential endpoint is not available. @@ -372,7 +372,7 @@ function ecs_instance_credentials() end """ - env_var_credentials(explicit_profile::Bool=false) -> Union{AWSCredential, Nothing} + env_var_credentials(explicit_profile::Bool=false) -> Union{AWSCredentials, Nothing} Use AWS environmental variables (e.g. AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, etc.) to create AWSCredentials. @@ -395,7 +395,7 @@ function env_var_credentials(explicit_profile::Bool=false) end """ - dot_aws_credentials(profile=nothing) -> Union{AWSCredential, Nothing} + dot_aws_credentials(profile=nothing) -> Union{AWSCredentials, Nothing} Retrieve `AWSCredentials` from the AWS CLI credentials file. The credential file defaults to "~/.aws/credentials" but can be specified using the env variable @@ -427,13 +427,13 @@ function dot_aws_credentials_file() end """ - sso_credentials(profile=nothing) -> Union{AWSCredential, Nothing} + sso_credentials(profile=nothing) -> Union{AWSCredentials, Nothing} Retrieve credentials via AWS single sign-on settings defined in the `profile` within the AWS config file. If no SSO settings are found for the `profile` `nothing` is returned. # Arguments -- `profile`: Specific profile used to get `AWSCredential`s, default is `nothing` +- `profile`: Specific profile used to get `AWSCredentials`, default is `nothing` """ function sso_credentials(profile=nothing) config_file = @mock dot_aws_config_file() @@ -458,7 +458,7 @@ function sso_credentials(profile=nothing) end """ - dot_aws_config(profile=nothing) -> Union{AWSCredential, Nothing} + dot_aws_config(profile=nothing) -> Union{AWSCredentials, Nothing} Retrieve `AWSCredentials` from the AWS CLI configuration file. The configuration file defaults to "~/.aws/config" but can be specified using the env variable `AWS_CONFIG_FILE`. From 2d53308075858e462e3d6e876c204d8163a9c6e8 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 11 May 2023 09:32:35 -0500 Subject: [PATCH 22/24] Update docstring on AWS.jl credential precedence --- src/AWSCredentials.jl | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index c5650321a8..e0b957accc 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -42,20 +42,22 @@ The fields `access_key_id` and `secret_key` hold the access keys used to authent [Temporary Security Credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) require the extra session `token` field. The `user_arn` and `account_number` fields are used to cache the result of the [`aws_user_arn`](@ref) and [`aws_account_number`](@ref) functions. -AWS.jl searches for credentials in a series of possible locations and stops as soon as it finds credentials. -The order of precedence for this search is as follows: +AWS.jl searches for credentials in multiple locations and stops once credentials are found. +The credential preference order [mirrors the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence) +and is as follows: -1. Passing credentials directly to the `AWSCredentials` constructor +1. Credentials or a profile passed directly to the `AWSCredentials` 2. [Environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) -3. Shared credential file [(~/.aws/credentials)](http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) -4. AWS config file [(~/.aws/config)](http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html). - This includes [Single Sign-On (SSO)](http://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sso.html) credentials. - SSO users should follow the configuration instructions at the above link, and use `aws sso login` to log in. -5. Assume Role provider via the aws config file -6. Instance metadata service on an Amazon EC2 instance that has an IAM role configured +3. [AWS Single Sign-On (SSO)](http://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sso.html) provided via the AWS configuration file +4. [AWS credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) (e.g. "~/.aws/credentials") +5. [External process](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html) set via `credential_process` in the AWS configuration file +6. [AWS configuration file](http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) set via `aws_access_key_id` in the AWS configuration file +7. Assume Role provider via the aws config file +8. [Amazon EC2 instance metadata](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) +9. [Amazon ECS container credentials](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html) Once the credentials are found, the method by which they were accessed is stored in the `renew` field -and the DateTime at which they will expire is stored in the `expiry` field. +and the `DateTime` at which they will expire is stored in the `expiry` field. This allows the credentials to be refreshed as needed using [`check_credentials`](@ref). If `renew` is set to `nothing`, no attempt will be made to refresh the credentials. Any renewal function is expected to return `nothing` on failure or a populated `AWSCredentials` object on success. From 996821f57a43fd0e1a7e694728256d6e3210dcdc Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 11 May 2023 09:51:36 -0500 Subject: [PATCH 23/24] Increase precedence of web identity creds --- src/AWSCredentials.jl | 12 ++++++------ test/AWSCredentials.jl | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index e0b957accc..5287c992e0 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -48,11 +48,11 @@ and is as follows: 1. Credentials or a profile passed directly to the `AWSCredentials` 2. [Environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html) -3. [AWS Single Sign-On (SSO)](http://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sso.html) provided via the AWS configuration file -4. [AWS credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) (e.g. "~/.aws/credentials") -5. [External process](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html) set via `credential_process` in the AWS configuration file -6. [AWS configuration file](http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) set via `aws_access_key_id` in the AWS configuration file -7. Assume Role provider via the aws config file +3. [Web Identity](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html#cli-configure-role-oidc) +4. [AWS Single Sign-On (SSO)](http://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sso.html) provided via the AWS configuration file +5. [AWS credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) (e.g. "~/.aws/credentials") +6. [External process](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html) set via `credential_process` in the AWS configuration file +7. [AWS configuration file](http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) set via `aws_access_key_id` in the AWS configuration file 8. [Amazon EC2 instance metadata](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) 9. [Amazon ECS container credentials](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html) @@ -123,10 +123,10 @@ function AWSCredentials(; profile=nothing, throw_cred_error=true) # EC2 credentials when the `AWS_CONTAINER_*` environmental variables are set. functions = [ () -> env_var_credentials(explicit_profile), + credentials_from_webtoken, () -> sso_credentials(profile), () -> dot_aws_credentials(profile), () -> dot_aws_config(profile), - credentials_from_webtoken, ecs_instance_credentials, () -> ec2_instance_credentials(profile), ] diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index a4214a162d..677daebc0c 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -530,6 +530,41 @@ end end end + @testset "Web identity preferred over SSO" begin + write( + config_file, + """ + [default] + sso_start_url = https://my-sso-portal.awsapps.com/start + sso_role_name = role1 + """, + ) + isfile(creds_file) && rm(creds_file) + + web_identity_file = joinpath(dir, "web_identity") + write(web_identity_file, "webid") + + patches = [ + Patches._assume_role_patch( + "AssumeRoleWithWebIdentity"; + access_key="AKI_WEB", + secret_key="SAK_WEB", + session_token="TOK_WEB", + ), + Patches.sso_service_patches("AKI_SSO", "SAK_SSO"), + ] + + withenv( + "AWS_WEB_IDENTITY_TOKEN_FILE" => web_identity_file, + "AWS_ROLE_ARN" => "webid", + ) do + apply(patches) do + creds = AWSCredentials() + @test creds.access_key_id == "AKI_WEB" + end + end + end + @testset "SSO preferred over credentials file" begin write( config_file, @@ -541,9 +576,9 @@ end ) write(creds_file, basic_creds_content) - apply(Patches.sso_service_patches("AKI0", "SAK0")) do + apply(Patches.sso_service_patches("AKI_SSO", "SAK_SSO")) do creds = AWSCredentials(; profile="profile1") - @test creds.access_key_id == "AKI0" + @test creds.access_key_id == "AKI_SSO" end end From 7b1a425d3e0b91dc9dd9ccb8287a4c9fb127ddf3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 11 May 2023 10:15:18 -0500 Subject: [PATCH 24/24] Self-review --- src/AWSCredentials.jl | 29 ++++++++++++++++------------- test/AWSCredentials.jl | 6 +++++- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 5287c992e0..aca874d16b 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -42,8 +42,8 @@ The fields `access_key_id` and `secret_key` hold the access keys used to authent [Temporary Security Credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html) require the extra session `token` field. The `user_arn` and `account_number` fields are used to cache the result of the [`aws_user_arn`](@ref) and [`aws_account_number`](@ref) functions. -AWS.jl searches for credentials in multiple locations and stops once credentials are found. -The credential preference order [mirrors the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence) +AWS.jl searches for credentials in multiple locations and stops once any credentials are found. +The credential preference order mostly [mirrors the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence) and is as follows: 1. Credentials or a profile passed directly to the `AWSCredentials` @@ -53,8 +53,8 @@ and is as follows: 5. [AWS credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) (e.g. "~/.aws/credentials") 6. [External process](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html) set via `credential_process` in the AWS configuration file 7. [AWS configuration file](http://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) set via `aws_access_key_id` in the AWS configuration file -8. [Amazon EC2 instance metadata](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) -9. [Amazon ECS container credentials](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html) +8. [Amazon ECS container credentials](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html) +9. [Amazon EC2 instance metadata](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) Once the credentials are found, the method by which they were accessed is stored in the `renew` field and the `DateTime` at which they will expire is stored in the `expiry` field. @@ -119,8 +119,9 @@ function AWSCredentials(; profile=nothing, throw_cred_error=true) # Define the credential preference order: # https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence # - # Take note that the AWS CLI in practise preferse to use ECS instance credentials over - # EC2 credentials when the `AWS_CONTAINER_*` environmental variables are set. + # Note that the AWS CLI documentation states that EC2 instance credentials are preferred + # over ECS container credentials. However, in practice when `AWS_CONTAINER_*` + # environmental variables are set the ECS container credentials are prefered instead. functions = [ () -> env_var_credentials(explicit_profile), credentials_from_webtoken, @@ -324,8 +325,8 @@ end """ ecs_instance_credentials() -> Union{AWSCredentials, Nothing} -Retrieve credentials from the ECS credential endpoint. Return `nothing` if the ECS -credential endpoint is not available. +Retrieve credentials from the ECS credential endpoint. If the ECS credential endpoint is +unavailable then `nothing` will be returned. More information can be found at: - https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html @@ -431,8 +432,9 @@ end """ sso_credentials(profile=nothing) -> Union{AWSCredentials, Nothing} -Retrieve credentials via AWS single sign-on settings defined in the `profile` within the AWS -config file. If no SSO settings are found for the `profile` `nothing` is returned. +Retrieve credentials via AWS single sign-on (SSO) settings defined in the `profile` within +the AWS configuration file. If no SSO settings are found for the `profile` `nothing` is +returned. # Arguments - `profile`: Specific profile used to get `AWSCredentials`, default is `nothing` @@ -464,9 +466,9 @@ end Retrieve `AWSCredentials` from the AWS CLI configuration file. The configuration file defaults to "~/.aws/config" but can be specified using the env variable `AWS_CONFIG_FILE`. -When no credentials can be found for the given `profile` then the associated -`source_profile` will be used to recursively look up credentials of parent profiles. If -still no credentials can be found then `nothing` will be returned. +When no credentials are found for the given `profile` then the associated `source_profile` +will be used to recursively look up credentials of source profiles. If still no credentials +can be found then `nothing` will be returned. # Arguments - `profile`: Specific profile used to get AWSCredentials, default is `nothing` @@ -493,6 +495,7 @@ function dot_aws_config(profile=nothing) access_key, secret_key, token = _aws_get_credential_details(p, ini) return AWSCredentials(access_key, secret_key, token) elseif !isnothing(sso_start_url) + # Deprecation should only appear if `dot_aws_config` is called directly Base.depwarn( "SSO support in `dot_aws_config` is deprecated, use `sso_credentials` instead.", :dot_aws_config, diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 677daebc0c..4f278cbe97 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -959,12 +959,16 @@ end end end + # When the environmental variable isn't set then the ECS credential provider is + # unavailable. withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => nothing) do @test ecs_instance_credentials() === nothing end + # Specifying the environmental variable results in us attempting to connect to the + # ECS credential provider. withenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI" => "/invalid") do - # Should internally generate a `ConnectError` + # Internally throws a `ConnectError` exception @test ecs_instance_credentials() === nothing end end