diff --git a/.cargo-deny-config.toml b/.cargo-deny-config.toml index eb7a041b21..6e00f867c3 100644 --- a/.cargo-deny-config.toml +++ b/.cargo-deny-config.toml @@ -23,6 +23,7 @@ confidence-threshold = 1.0 exceptions = [ { allow = ["OpenSSL"], name = "ring", version = "*" }, { allow = ["OpenSSL"], name = "aws-lc-sys", version = "*" }, + { allow = ["OpenSSL"], name = "aws-lc-fips-sys", version = "*" }, ] [[licenses.clarify]] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73a6a58216..036e694fdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,24 +232,24 @@ jobs: - target: i686-unknown-linux-gnu build_smithy_rs_features: --all-features build_aws_exclude: '' - build_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript + build_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript --exclude aws-smithy-experimental test_smithy_rs_features: --all-features test_aws_exclude: '' - test_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript + test_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript --exclude aws-smithy-experimental - target: powerpc-unknown-linux-gnu build_smithy_rs_features: '' build_aws_exclude: --exclude aws-inlineable - build_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript + build_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript --exclude aws-smithy-experimental test_smithy_rs_features: '' test_aws_exclude: --exclude aws-inlineable test_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript - target: powerpc64-unknown-linux-gnu build_smithy_rs_features: '' build_aws_exclude: --exclude aws-inlineable - build_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript + build_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript --exclude aws-smithy-experimental test_smithy_rs_features: '' test_aws_exclude: --exclude aws-inlineable - test_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript + test_smithy_rs_exclude: --exclude aws-smithy-http-server-python --exclude aws-smithy-http-server-typescript --exclude aws-smithy-experimental env: CROSS_CONFIG: Cross.toml steps: diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index 7fae9074e1..a7210242cb 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -64,3 +64,25 @@ message = "Users may now set service-specific configuration in the environment. references = ["smithy-rs#3493"] meta = { "breaking" = false, "tada" = true, "bug" = false } author = "Velfi" + +[[smithy-rs]] +message = "Fix bug in Hyper 1.0 support where https URLs returned an error" +references = ["smithy-rs#3539"] +meta = { "breaking" = false, "tada" = false, "bug" = false } +author = "rcoh" + +[[smithy-rs]] +message = """Add FIPS support to our Hyper 1.0-based client. Customers can enable this mode by enabling the `crypto-aws-lc-fips` on `aws-smithy-experimental`. To construct a client using the new client, consult this [example](https://github.com/awslabs/aws-sdk-rust/blob/release-2024-03-29/sdk/s3/tests/hyper-10.rs). + +Please note that support for Hyper 1.0 remains experimental.""" +references = ["smithy-rs#3539"] +meta = { "breaking" = false, "tada" = true, "bug" = false } +author = "rcoh" + +[[aws-sdk-rust]] +message = """Add FIPS support to our Hyper 1.0-based client. Customers can enable this mode by enabling the `crypto-aws-lc-fips` on `aws-smithy-experimental`. To construct a client using the new client, consult this [example](https://github.com/awslabs/aws-sdk-rust/blob/release-2024-03-29/sdk/s3/tests/hyper-10.rs). + +Please note that support for Hyper 1.0 remains experimental.""" +references = ["smithy-rs#3539"] +meta = { "breaking" = false, "tada" = true, "bug" = false } +author = "rcoh" diff --git a/rust-runtime/aws-smithy-experimental/Cargo.toml b/rust-runtime/aws-smithy-experimental/Cargo.toml index 725abb590b..6933116aed 100644 --- a/rust-runtime/aws-smithy-experimental/Cargo.toml +++ b/rust-runtime/aws-smithy-experimental/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-experimental" -version = "0.1.0" +version = "0.1.1" authors = ["AWS Rust SDK Team "] description = "Experiments for the smithy-rs ecosystem" edition = "2021" @@ -9,7 +9,8 @@ repository = "https://github.com/smithy-lang/smithy-rs" [features] crypto-ring = ["rustls/ring"] -crypto-aws-lc = ["rustls/aws_lc_rs", "dep:fs_extra"] +crypto-aws-lc = ["rustls/aws_lc_rs"] +crypto-aws-lc-fips = ["rustls/fips"] [dependencies] aws-smithy-types = { path = "../aws-smithy-types", features = ["http-body-1-x"] } @@ -20,13 +21,12 @@ pin-project-lite = "0.2.13" hyper-util = "0.1.3" http = "1" tokio = "1" -hyper-rustls = { version = "0.26", features = ["http2", "http1"] } -rustls = { version = "0.22.2", default-features = false } +hyper-rustls = { version = "0.27", features = ["http2", "http1", "native-tokio", "tls12"], default-features = false } +rustls = { version = "0.23", default-features = false } h2 = "0.4" once_cell = "1.18.0" tracing = "0.1.40" tower = "0.4.1" -fs_extra = { version = "1.3.0", optional = true } # hack for cargo-minimal-versions [dev-dependencies] aws-smithy-async = { path = "../aws-smithy-async", features = ["rt-tokio", "test-util"] } @@ -40,7 +40,7 @@ doc-scrape-examples = true [[example]] name = "client-aws-lc" -required-features = ["crypto-aws-lc"] +required-features = ["crypto-aws-lc", "crypto-aws-lc-fips"] doc-scrape-examples = true [[example]] diff --git a/rust-runtime/aws-smithy-experimental/README.md b/rust-runtime/aws-smithy-experimental/README.md index 1c5b805597..8d800bcb8a 100644 --- a/rust-runtime/aws-smithy-experimental/README.md +++ b/rust-runtime/aws-smithy-experimental/README.md @@ -3,11 +3,13 @@ Staging ground for experimental new features in the smithy-rs ecosystem. ### Hyper 1.0 Support -This crate adds support for Hyper 1.0 (see [examples](./examples)). There a few blockers before stablization: -1. Moving to `rustls` 0.23 to take advantage of FIPS support. This is blocked on `hyper-rustls` being upgraded. -2. Expose an API for providing a custom connector. Currently that API is not exposed because a shim layer is needed to avoid taking a hard dependency on hyper-util. -3. Add support for poisoning connections in the connection pool. This API needs to be either backported into hyper-util or we need to establish our own client. +This crate allows customers to use Hyper 1.0. A valuable consequence of this is access to aws-lc-rs and its `FIPS` compliant crypto. This is available behind the `crypto-aws-lc-fips` feature. **Note**: FIPS support has somewhat [complex build requirements](https://github.com/aws/aws-lc/blob/main/BUILDING.md), namely CMake and Go. + +## Crate Stabilization +This crate adds support for Hyper 1.0 (see [examples](./examples)). There a few blockers before stablization: +1. Expose an API for providing a custom connector. Currently, that API is not exposed because a shim layer is needed to avoid taking a hard dependency on hyper-util. +2. Add support for poisoning connections in the connection pool. This API needs to be either backported into hyper-util or we need to establish our own client. This crate is part of the [AWS SDK for Rust](https://awslabs.github.io/aws-sdk-rust/) and the [smithy-rs](https://github.com/smithy-lang/smithy-rs) code generator. diff --git a/rust-runtime/aws-smithy-experimental/examples/client-aws-lc.rs b/rust-runtime/aws-smithy-experimental/examples/client-aws-lc.rs index 7897f6bacf..a4036e2ddf 100644 --- a/rust-runtime/aws-smithy-experimental/examples/client-aws-lc.rs +++ b/rust-runtime/aws-smithy-experimental/examples/client-aws-lc.rs @@ -4,9 +4,17 @@ */ use aws_smithy_experimental::hyper_1_0::{CryptoMode, HyperClientBuilder}; +#[tokio::main] -fn main() { +async fn main() { + // feature = crypto-aws-lc let _client = HyperClientBuilder::new() .crypto_mode(CryptoMode::AwsLc) .build_https(); + + // feature = crypto-aws-lc-fips + // A FIPS client can also be created. Note that this has a more complex build environment required. + let _client = HyperClientBuilder::new() + .crypto_mode(CryptoMode::AwsLcFips) + .build_https(); } diff --git a/rust-runtime/aws-smithy-experimental/src/hyper_1_0.rs b/rust-runtime/aws-smithy-experimental/src/hyper_1_0.rs index 55eb193dd5..65f95d3118 100644 --- a/rust-runtime/aws-smithy-experimental/src/hyper_1_0.rs +++ b/rust-runtime/aws-smithy-experimental/src/hyper_1_0.rs @@ -45,11 +45,14 @@ use aws_smithy_types::retry::ErrorKind; use crate::hyper_1_0::timeout_middleware::{ConnectTimeout, HttpTimeoutError}; #[derive(Debug, Eq, PartialEq, Clone, Copy)] +#[non_exhaustive] pub enum CryptoMode { #[cfg(feature = "crypto-ring")] Ring, #[cfg(feature = "crypto-aws-lc")] AwsLc, + #[cfg(feature = "crypto-aws-lc-fips")] + AwsLcFips, } impl CryptoMode { @@ -60,6 +63,16 @@ impl CryptoMode { #[cfg(feature = "crypto-ring")] CryptoMode::Ring => rustls::crypto::ring::default_provider(), + + #[cfg(feature = "crypto-aws-lc-fips")] + CryptoMode::AwsLcFips => { + let provider = rustls::crypto::default_fips_provider(); + assert!( + provider.fips(), + "FIPS was requested but the provider did not support FIPS" + ); + provider + } } } } @@ -113,12 +126,21 @@ mod cached_connectors { HttpsConnector, > = once_cell::sync::Lazy::new(|| make_tls(GaiResolver::new(), CryptoMode::AwsLc.provider())); + #[cfg(feature = "crypto-aws-lc-fips")] + pub(crate) static HTTPS_NATIVE_ROOTS_AWS_LC_FIPS: once_cell::sync::Lazy< + HttpsConnector, + > = once_cell::sync::Lazy::new(|| { + make_tls(GaiResolver::new(), CryptoMode::AwsLcFips.provider()) + }); + pub(super) fn cached_https(mode: Inner) -> hyper_rustls::HttpsConnector { match mode { #[cfg(feature = "crypto-ring")] Inner::Standard(CryptoMode::Ring) => HTTPS_NATIVE_ROOTS_RING.clone(), #[cfg(feature = "crypto-aws-lc")] Inner::Standard(CryptoMode::AwsLc) => HTTPS_NATIVE_ROOTS_AWS_LC.clone(), + #[cfg(feature = "crypto-aws-lc-fips")] + Inner::Standard(CryptoMode::AwsLcFips) => HTTPS_NATIVE_ROOTS_AWS_LC_FIPS.clone(), #[allow(unreachable_patterns)] Inner::Standard(_) => unreachable!("unexpected mode"), Inner::Custom(provider) => make_tls(GaiResolver::new(), provider), @@ -169,6 +191,8 @@ mod build_connector { crypto_provider: CryptoProvider, ) -> HttpsConnector> { use hyper_rustls::ConfigBuilderExt; + let mut base_connector = HttpConnector::new_with_resolver(resolver); + base_connector.enforce_http(false); hyper_rustls::HttpsConnectorBuilder::new() .with_tls_config( rustls::ClientConfig::builder_with_provider(Arc::new(restrict_ciphers(crypto_provider))) @@ -180,7 +204,7 @@ mod build_connector { .https_or_http() .enable_http1() .enable_http2() - .wrap_connector(HttpConnector::new_with_resolver(resolver)) + .wrap_connector(base_connector) } pub(super) fn https_with_resolver( diff --git a/rust-runtime/aws-smithy-experimental/tests/smoke_test_clients.rs b/rust-runtime/aws-smithy-experimental/tests/smoke_test_clients.rs new file mode 100644 index 0000000000..1e7cb72c27 --- /dev/null +++ b/rust-runtime/aws-smithy-experimental/tests/smoke_test_clients.rs @@ -0,0 +1,91 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_smithy_async::time::SystemTimeSource; +use aws_smithy_experimental::hyper_1_0::{CryptoMode, HyperClientBuilder}; +use aws_smithy_runtime_api::client::dns::{DnsFuture, ResolveDns, ResolveDnsError}; +use aws_smithy_runtime_api::client::http::{HttpClient, HttpConnector, HttpConnectorSettings}; +use aws_smithy_runtime_api::client::orchestrator::HttpRequest; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponentsBuilder; +use hyper_util::client::legacy::connect::dns::{GaiResolver, Name}; +use std::error::Error; +use std::str::FromStr; +use std::sync::Arc; +use tower::Service; + +#[cfg(feature = "crypto-ring")] +#[tokio::test] +async fn ring_client() { + let client = HyperClientBuilder::new() + .crypto_mode(CryptoMode::Ring) + .build_https(); + smoke_test_client(&client).await.unwrap(); +} + +#[cfg(feature = "crypto-aws-lc-fips")] +#[tokio::test] +async fn aws_lc_fips_client() { + let client = HyperClientBuilder::new() + .crypto_mode(CryptoMode::AwsLcFips) + .build_https(); + smoke_test_client(&client).await.unwrap(); +} + +#[cfg(feature = "crypto-aws-lc")] +#[tokio::test] +async fn aws_lc_client() { + let client = HyperClientBuilder::new() + .crypto_mode(CryptoMode::AwsLc) + .build_https(); + smoke_test_client(&client).await.unwrap(); +} + +#[cfg(feature = "crypto-ring")] +#[tokio::test] +async fn custom_dns_client() { + use std::sync::atomic::{AtomicUsize, Ordering}; + #[derive(Debug, Clone)] + struct PassThroughResolver { + inner: GaiResolver, + count: Arc, + } + impl ResolveDns for PassThroughResolver { + fn resolve_dns<'a>(&'a self, _name: &'a str) -> DnsFuture<'a> { + let mut inner = self.inner.clone(); + let name = Name::from_str(_name).unwrap(); + let count = self.count.clone(); + DnsFuture::new(async move { + count.fetch_add(1, Ordering::Relaxed); + let result = inner + .call(name) + .await + .map_err(|err| ResolveDnsError::new(err))?; + Ok(result.map(|addr| addr.ip()).collect::>()) + }) + } + } + let resolver = PassThroughResolver { + inner: GaiResolver::new(), + count: Default::default(), + }; + let client = HyperClientBuilder::new() + .crypto_mode(CryptoMode::Ring) + .build_with_resolver(resolver.clone()); + smoke_test_client(&client).await.unwrap(); + assert_eq!(resolver.count.load(Ordering::Relaxed), 1); +} + +async fn smoke_test_client(client: &dyn HttpClient) -> Result<(), Box> { + let connector_settings = HttpConnectorSettings::builder().build(); + let runtime_components = RuntimeComponentsBuilder::for_tests() + .with_time_source(Some(SystemTimeSource::new())) + .build() + .unwrap(); + let connector = client.http_connector(&connector_settings, &runtime_components); + let _response = connector + .call(HttpRequest::get("https://amazon.com").unwrap()) + .await?; + Ok(()) +} diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 1f77b41ed0..8abe689cd8 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -157,6 +157,7 @@ RUN set -eux; \ gcc \ git \ glibc-langpack-en \ + go \ java-11-amazon-corretto-headless \ make \ openssl-devel \ diff --git a/tools/ci-scripts/test-windows.sh b/tools/ci-scripts/test-windows.sh index 8a260bc95d..a862d2584f 100755 --- a/tools/ci-scripts/test-windows.sh +++ b/tools/ci-scripts/test-windows.sh @@ -8,6 +8,7 @@ set -eu -o pipefail exclusions=("--exclude" "aws-smithy-http-server-python" "--exclude" "aws-smithy-http-server-typescript" "--exclude" "aws-smithy-experimental") for runtime_path in "rust-runtime" "aws/rust-runtime"; do + echo "testing $runtime_path" pushd "${runtime_path}" &>/dev/null # aws-smithy-http-server-python cannot be compiled on Windows since it uses the `signal-hook` crate # which is not really yet fully supported on the platform. @@ -18,4 +19,5 @@ for runtime_path in "rust-runtime" "aws/rust-runtime"; do done # TODO(https://github.com/awslabs/aws-sdk-rust/issues/1117) We don't have a way to codegen the deps needed by the aws-config crate # (cd aws/rust-runtime/aws-config && cargo test --all-features) # aws-config is not part of the workspace so we have to test it separately +echo "Testing isolated features of aws-smithy-experimental" (cd rust-runtime && cargo test -p aws-smithy-experimental --features crypto-ring) # only ring works on windows