Sourcing service config from the environment. (#3493)

## Motivation and Context   #2863 awslabs/aws-sdk-rust#1060 ## Description  This PR adds a new feature: the ability to source service-specific config from the environment. This is **only** supported when creating a service config from an `SdkConfig`. I've posted [a guide](#3537) to our discussions board. [This also adds support for setting an endpoint URL in environment config.](#2863) ## Testing    I have written several tests ensuring config is extracted with the correct precedence. ## Checklist  - [x] I have updated `CHANGELOG.next.toml` if I made changes to the AWS SDK, generated SDK code, or SDK runtime crates ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._ --------- Co-authored-by: John DiSanti <[email protected]> Co-authored-by: ysaito1001 <[email protected]>
smithy-lang · Apr 1, 2024 · 5f7113f · 5f7113f
1 parent 8af3449
commit 5f7113f
Show file tree

Hide file tree

Showing 50 changed files with 2,280 additions and 1,717 deletions.
diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml
@@ -22,3 +22,45 @@ message = "Make `BehaviorVersion` be future-proof by disallowing it to be constr
 references = ["aws-sdk-rust#1111", "smithy-rs#3513"]
 meta = { "breaking" = true, "tada" = false, "bug" = true, "target" = "client" }
 author = "Ten0"
+
+[[smithy-rs]]
+message = """
+Stalled stream protection now supports request upload streams. It is currently off by default, but will be enabled by default in a future release. To enable it now, you can do the following:
+
+```rust
+let config = my_service::Config::builder()
+    .stalled_stream_protection(StalledStreamProtectionConfig::enabled().build())
+    // ...
+    .build();
+```
+"""
+references = ["smithy-rs#3485"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+authors = ["jdisanti"]
+
+[[aws-sdk-rust]]
+message = """
+Stalled stream protection now supports request upload streams. It is currently off by default, but will be enabled by default in a future release. To enable it now, you can do the following:
+
+```rust
+let config = aws_config::defaults(BehaviorVersion::latest())
+    .stalled_stream_protection(StalledStreamProtectionConfig::enabled().build())
+    .load()
+    .await;
+```
+"""
+references = ["smithy-rs#3485"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+author = "jdisanti"
+
+[[smithy-rs]]
+message = "Stalled stream protection on downloads will now only trigger if the upstream source is too slow. Previously, stalled stream protection could be erroneously triggered if the user was slowly consuming the stream slower than the minimum speed limit."
+references = ["smithy-rs#3485"]
+meta = { "breaking" = false, "tada" = false, "bug" = true }
+authors = ["jdisanti"]
+
+[[aws-sdk-rust]]
+message = "Users may now set service-specific configuration in the environment. For more information, see [this discussion topic](https://github.com/smithy-lang/smithy-rs/discussions/3537)."
+references = ["smithy-rs#3493"]
+meta = { "breaking" = false, "tada" = true, "bug" = false }
+author = "Velfi"
diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml
@@ -35,6 +35,8 @@ aws-smithy-runtime = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime",
 aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["client"] }
 aws-smithy-types = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-types" }
 aws-types = { path = "../../sdk/build/aws-sdk/sdk/aws-types" }
+bytes = "1.1.0"
+http = "0.2.4"
 hyper = { version = "0.14.26", default-features = false }
 time = { version = "0.3.4", features = ["parsing"] }
 tokio = { version = "1.13.1", features = ["sync"] }
@@ -44,9 +46,6 @@ url = "2.3.1"
 # implementation detail of IMDS credentials provider
 fastrand = "2.0.0"
 
-bytes = "1.1.0"
-http = "0.2.4"
-
 # implementation detail of SSO credential caching
 aws-sdk-sso = { path = "../../sdk/build/aws-sdk/sdk/sso", default-features = false, optional = true }
 ring = { version = "0.17.5", optional = true }
@@ -62,12 +61,8 @@ aws-smithy-runtime-api = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-runtim
 futures-util = { version = "0.3.29", default-features = false }
 tracing-test = "0.2.4"
 tracing-subscriber = { version = "0.3.16", features = ["fmt", "json"] }
-
 tokio = { version = "1.23.1", features = ["full", "test-util"] }
 
-# used for fuzzing profile parsing
-arbitrary = "1.3"
-
 # used for test case deserialization
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"

diff --git a/aws/rust-runtime/aws-config/external-types.toml b/aws/rust-runtime/aws-config/external-types.toml
@@ -8,13 +8,21 @@ allowed_external_types = [
    "aws_credential_types::provider::credentials::Result",
    "aws_credential_types::provider::credentials::SharedCredentialsProvider",
    "aws_credential_types::provider::token::ProvideToken",
+   "aws_runtime::env_config::error::EnvConfigFileLoadError",
+   "aws_runtime::env_config::file::Builder",
+   "aws_runtime::env_config::file::EnvConfigFileKind",
+   "aws_runtime::env_config::file::EnvConfigFiles",
+   "aws_runtime::env_config::parse::EnvConfigParseError",
+   "aws_runtime::env_config::property::Property",
+   "aws_runtime::env_config::section::EnvConfigSections",
+   "aws_runtime::env_config::section::Profile",
    "aws_smithy_async::rt::sleep::AsyncSleep",
    "aws_smithy_async::rt::sleep::SharedAsyncSleep",
    "aws_smithy_async::time::SharedTimeSource",
    "aws_smithy_async::time::TimeSource",
-   "aws_smithy_runtime_api::box_error::BoxError",
    "aws_smithy_runtime::client::identity::cache::IdentityCache",
    "aws_smithy_runtime::client::identity::cache::lazy::LazyCacheBuilder",
+   "aws_smithy_runtime_api::box_error::BoxError",
    "aws_smithy_runtime_api::client::behavior_version::BehaviorVersion",
    "aws_smithy_runtime_api::client::dns::ResolveDns",
    "aws_smithy_runtime_api::client::dns::SharedDnsResolver",

diff --git a/aws/rust-runtime/aws-config/src/default_provider/app_name.rs b/aws/rust-runtime/aws-config/src/default_provider/app_name.rs
@@ -4,7 +4,7 @@
  */
 
 use crate::provider_config::ProviderConfig;
-use crate::standard_property::{PropertyResolutionError, StandardProperty};
+use aws_runtime::env_config::{EnvConfigError, EnvConfigValue};
 use aws_smithy_types::error::display::DisplayErrorContext;
 use aws_types::app_name::{AppName, InvalidAppName};
 
@@ -56,22 +56,24 @@ impl Builder {
         self
     }
 
-    async fn fallback_app_name(
-        &self,
-    ) -> Result<Option<AppName>, PropertyResolutionError<InvalidAppName>> {
-        StandardProperty::new()
+    async fn fallback_app_name(&self) -> Result<Option<AppName>, EnvConfigError<InvalidAppName>> {
+        let env = self.provider_config.env();
+        let profiles = self.provider_config.profile().await;
+
+        EnvConfigValue::new()
             .profile("sdk-ua-app-id")
-            .validate(&self.provider_config, |name| AppName::new(name.to_string()))
-            .await
+            .validate(&env, profiles, |name| AppName::new(name.to_string()))
     }
 
     /// Build an [`AppName`] from the default chain
     pub async fn app_name(self) -> Option<AppName> {
-        let standard = StandardProperty::new()
+        let env = self.provider_config.env();
+        let profiles = self.provider_config.profile().await;
+
+        let standard = EnvConfigValue::new()
             .env("AWS_SDK_UA_APP_ID")
             .profile("sdk_ua_app_id")
-            .validate(&self.provider_config, |name| AppName::new(name.to_string()))
-            .await;
+            .validate(&env, profiles, |name| AppName::new(name.to_string()));
         let with_fallback = match standard {
             Ok(None) => self.fallback_app_name().await,
             other => other,
@@ -87,6 +89,7 @@ impl Builder {
 #[cfg(test)]
 mod tests {
     use super::*;
+    #[allow(deprecated)]
     use crate::profile::profile_file::{ProfileFileKind, ProfileFiles};
     use crate::provider_config::ProviderConfig;
     use crate::test_case::{no_traffic_client, InstantSleep};
@@ -123,8 +126,13 @@ mod tests {
             .http_client(no_traffic_client())
             .profile_name("custom")
             .profile_files(
+                #[allow(deprecated)]
                 ProfileFiles::builder()
-                    .with_file(ProfileFileKind::Config, "test_config")
+                    .with_file(
+                        #[allow(deprecated)]
+                        ProfileFileKind::Config,
+                        "test_config",
+                    )
                     .build(),
             )
             .load()

diff --git a/aws/rust-runtime/aws-config/src/default_provider/endpoint_url.rs b/aws/rust-runtime/aws-config/src/default_provider/endpoint_url.rs
@@ -5,7 +5,7 @@
 
 use crate::environment::parse_url;
 use crate::provider_config::ProviderConfig;
-use crate::standard_property::StandardProperty;
+use aws_runtime::env_config::EnvConfigValue;
 use aws_smithy_types::error::display::DisplayErrorContext;
 
 mod env {
@@ -24,11 +24,13 @@ mod profile_key {
 ///
 /// If invalid values are found, the provider will return None and an error will be logged.
 pub async fn endpoint_url_provider(provider_config: &ProviderConfig) -> Option<String> {
-    StandardProperty::new()
+    let env = provider_config.env();
+    let profiles = provider_config.profile().await;
+
+    EnvConfigValue::new()
         .env(env::ENDPOINT_URL)
         .profile(profile_key::ENDPOINT_URL)
-        .validate(provider_config, parse_url)
-        .await
+        .validate(&env, profiles, parse_url)
         .map_err(
             |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for endpoint URL setting"),
         )
@@ -39,6 +41,7 @@ pub async fn endpoint_url_provider(provider_config: &ProviderConfig) -> Option<S
 mod test {
     use super::endpoint_url_provider;
     use super::env;
+    #[allow(deprecated)]
     use crate::profile::profile_file::{ProfileFileKind, ProfileFiles};
     use crate::provider_config::ProviderConfig;
     use aws_types::os_shim_internal::{Env, Fs};
@@ -61,8 +64,13 @@ mod test {
             .with_env(Env::from_slice(&[(env::ENDPOINT_URL, "http://localhost")]))
             .with_profile_config(
                 Some(
+                    #[allow(deprecated)]
                     ProfileFiles::builder()
-                        .with_file(ProfileFileKind::Config, "conf")
+                        .with_file(
+                            #[allow(deprecated)]
+                            ProfileFileKind::Config,
+                            "conf",
+                        )
                         .build(),
                 ),
                 None,

diff --git a/aws/rust-runtime/aws-config/src/default_provider/ignore_configured_endpoint_urls.rs b/aws/rust-runtime/aws-config/src/default_provider/ignore_configured_endpoint_urls.rs
@@ -5,7 +5,7 @@
 
 use crate::environment::parse_bool;
 use crate::provider_config::ProviderConfig;
-use crate::standard_property::StandardProperty;
+use aws_runtime::env_config::EnvConfigValue;
 use aws_smithy_types::error::display::DisplayErrorContext;
 
 mod env {
@@ -26,11 +26,13 @@ mod profile_key {
 pub async fn ignore_configured_endpoint_urls_provider(
     provider_config: &ProviderConfig,
 ) -> Option<bool> {
-    StandardProperty::new()
+    let env = provider_config.env();
+    let profiles = provider_config.profile().await;
+
+    EnvConfigValue::new()
         .env(env::IGNORE_CONFIGURED_ENDPOINT_URLS)
         .profile(profile_key::IGNORE_CONFIGURED_ENDPOINT_URLS)
-        .validate(provider_config, parse_bool)
-        .await
+        .validate(&env, profiles, parse_bool)
         .map_err(
             |err| tracing::warn!(err = %DisplayErrorContext(&err), "invalid value for 'ignore configured endpoint URLs' setting"),
         )
@@ -41,6 +43,7 @@ pub async fn ignore_configured_endpoint_urls_provider(
 mod test {
     use super::env;
     use super::ignore_configured_endpoint_urls_provider;
+    #[allow(deprecated)]
     use crate::profile::profile_file::{ProfileFileKind, ProfileFiles};
     use crate::provider_config::ProviderConfig;
     use aws_types::os_shim_internal::{Env, Fs};
@@ -70,8 +73,13 @@ mod test {
             )]))
             .with_profile_config(
                 Some(
+                    #[allow(deprecated)]
                     ProfileFiles::builder()
-                        .with_file(ProfileFileKind::Config, "conf")
+                        .with_file(
+                            #[allow(deprecated)]
+                            ProfileFileKind::Config,
+                            "conf",
+                        )
                         .build(),
                 ),
                 None,

diff --git a/aws/rust-runtime/aws-config/src/default_provider/retry_config.rs b/aws/rust-runtime/aws-config/src/default_provider/retry_config.rs
@@ -5,7 +5,7 @@
 
 use crate::provider_config::ProviderConfig;
 use crate::retry::error::{RetryConfigError, RetryConfigErrorKind};
-use crate::standard_property::{PropertyResolutionError, StandardProperty};
+use aws_runtime::env_config::{EnvConfigError, EnvConfigValue};
 use aws_smithy_types::error::display::DisplayErrorContext;
 use aws_smithy_types::retry::{RetryConfig, RetryMode};
 use std::str::FromStr;
@@ -101,29 +101,31 @@ impl Builder {
 
     pub(crate) async fn try_retry_config(
         self,
-    ) -> Result<RetryConfig, PropertyResolutionError<RetryConfigError>> {
-        // Both of these can return errors due to invalid config settings and we want to surface those as early as possible
+    ) -> Result<RetryConfig, EnvConfigError<RetryConfigError>> {
+        let env = self.provider_config.env();
+        let profiles = self.provider_config.profile().await;
+        // Both of these can return errors due to invalid config settings, and we want to surface those as early as possible
         // hence, we'll panic if any config values are invalid (missing values are OK though)
-        // We match this instead of unwrapping so we can print the error with the `Display` impl instead of the `Debug` impl that unwrap uses
+        // We match this instead of unwrapping, so we can print the error with the `Display` impl instead of the `Debug` impl that unwrap uses
         let mut retry_config = RetryConfig::standard();
-        let max_attempts = StandardProperty::new()
+        let max_attempts = EnvConfigValue::new()
             .env(env::MAX_ATTEMPTS)
             .profile(profile_keys::MAX_ATTEMPTS)
-            .validate(&self.provider_config, validate_max_attempts);
+            .validate(&env, profiles, validate_max_attempts);
 
-        let retry_mode = StandardProperty::new()
+        let retry_mode = EnvConfigValue::new()
             .env(env::RETRY_MODE)
             .profile(profile_keys::RETRY_MODE)
-            .validate(&self.provider_config, |s| {
+            .validate(&env, profiles, |s| {
                 RetryMode::from_str(s)
                     .map_err(|err| RetryConfigErrorKind::InvalidRetryMode { source: err }.into())
             });
 
-        if let Some(max_attempts) = max_attempts.await? {
+        if let Some(max_attempts) = max_attempts? {
             retry_config = retry_config.with_max_attempts(max_attempts);
         }
 
-        if let Some(retry_mode) = retry_mode.await? {
+        if let Some(retry_mode) = retry_mode? {
             retry_config = retry_config.with_retry_mode(retry_mode);
         }
 
@@ -146,12 +148,12 @@ mod test {
     use crate::retry::{
         error::RetryConfigError, error::RetryConfigErrorKind, RetryConfig, RetryMode,
     };
-    use crate::standard_property::PropertyResolutionError;
+    use aws_runtime::env_config::EnvConfigError;
     use aws_types::os_shim_internal::{Env, Fs};
 
     async fn test_provider(
         vars: &[(&str, &str)],
-    ) -> Result<RetryConfig, PropertyResolutionError<RetryConfigError>> {
+    ) -> Result<RetryConfig, EnvConfigError<RetryConfigError>> {
         super::Builder::default()
             .configure(&ProviderConfig::no_configuration().with_env(Env::from_slice(vars)))
             .try_retry_config()
@@ -296,7 +298,7 @@ max_attempts = potato
             test_provider(&[(env::MAX_ATTEMPTS, "not an integer")])
                 .await
                 .unwrap_err()
-                .err,
+                .err(),
             RetryConfigError {
                 kind: RetryConfigErrorKind::FailedToParseMaxAttempts { .. }
             }
@@ -327,8 +329,8 @@ max_attempts = potato
     async fn disallow_zero_max_attempts() {
         let err = test_provider(&[(env::MAX_ATTEMPTS, "0")])
             .await
-            .unwrap_err()
-            .err;
+            .unwrap_err();
+        let err = err.err();
         assert!(matches!(
             err,
             RetryConfigError {