From 4e4fd4adfb3bff2e903d9becb54df7f74911a8d0 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Fri, 24 Mar 2023 23:13:07 +0100 Subject: [PATCH 01/72] feat: make outgoing request based on wasi-http proposal --- .../webassembly/src/adapter/http_client.rs | 156 +++++++++++++++--- 1 file changed, 135 insertions(+), 21 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/src/adapter/http_client.rs b/aws/sdk/integration-tests/webassembly/src/adapter/http_client.rs index 5ca84838c9..9012cde6e0 100644 --- a/aws/sdk/integration-tests/webassembly/src/adapter/http_client.rs +++ b/aws/sdk/integration-tests/webassembly/src/adapter/http_client.rs @@ -3,27 +3,141 @@ * SPDX-License-Identifier: Apache-2.0 */ +#![allow(unused_variables)] + use aws_smithy_http::body::SdkBody; +use http::header::{HeaderName, HeaderValue}; +use std::str::FromStr; +use wasi::{default_outgoing_http, poll, streams, types as http_types}; + +pub(crate) fn make_request(req: http::Request) -> anyhow::Result> { + let options = Some(http_types::RequestOptions { + connect_timeout_ms: None, + first_byte_timeout_ms: None, + between_bytes_timeout_ms: None, + }); + let req = HttpRequest::from(req).inner(); + // println!("HttpRequest: {:?}", req); + let res = default_outgoing_http::handle(req, options); + println!("{:?}", res); + let res: http::Response = HttpResponse(res).into(); + http_types::drop_outgoing_request(req); + Ok(res) +} + +pub struct HttpRequest(http_types::OutgoingRequest); + +impl HttpRequest { + pub fn inner(self) -> http_types::OutgoingRequest { + self.0 + } +} + +impl From> for HttpRequest { + fn from(req: http::Request) -> Self { + let (parts, sdk_body) = req.into_parts(); + // println!("request parts: {:?}", parts); + // println!("request body: {:?}", sdk_body); + let path = parts.uri.path(); + let query = parts.uri.query(); + let method = HttpMethod::from(parts.method); + let headers = HttpHeader::from(&parts.headers); + let scheme = match parts.uri.scheme_str().unwrap_or("") { + "http" => Some(http_types::SchemeParam::Http), + "https" => Some(http_types::SchemeParam::Https), + _ => None, + }; + Self(http_types::new_outgoing_request( + method.inner(), + path, + query.unwrap_or(""), + scheme, + parts.uri.authority().map(|a| a.as_str()).unwrap(), + headers.inner(), + )) + } +} + +pub struct HttpMethod<'a>(http_types::MethodParam<'a>); + +impl<'a> HttpMethod<'a> { + pub fn inner(self) -> http_types::MethodParam<'a> { + self.0 + } +} + +impl<'a> From for HttpMethod<'a> { + fn from(method: http::Method) -> Self { + Self(match method { + http::Method::GET => http_types::MethodParam::Get, + http::Method::POST => http_types::MethodParam::Post, + http::Method::PUT => http_types::MethodParam::Put, + http::Method::DELETE => http_types::MethodParam::Delete, + http::Method::PATCH => http_types::MethodParam::Patch, + http::Method::CONNECT => http_types::MethodParam::Connect, + http::Method::TRACE => http_types::MethodParam::Trace, + http::Method::HEAD => http_types::MethodParam::Head, + http::Method::OPTIONS => http_types::MethodParam::Options, + _ => panic!("failed due to unsupported method, currently supported methods are: GET, POST, PUT, DELETE, PATCH, CONNECT, TRACE, HEAD, and OPTIONS"), + }) + } +} + +struct HttpResponse(http_types::IncomingResponse); + +impl HttpResponse { + pub fn inner(self) -> http_types::IncomingResponse { + self.0 + } +} + +impl From for http::Response { + fn from(val: HttpResponse) -> Self { + let res_pointer = val.inner(); + poll::drop_pollable(res_pointer); + let status = http_types::incoming_response_status(res_pointer); + println!("status: {}", status); + let header_handle = http_types::incoming_response_headers(res_pointer); + let headers = http_types::fields_entries(header_handle); + println!("headers: {:?}", headers); + let stream: http_types::IncomingStream = + http_types::incoming_response_consume(res_pointer).unwrap(); + let len = 64 * 1024; + let (body, finished) = streams::read(stream, len).unwrap(); + let body = if body.is_empty() { + SdkBody::empty() + } else { + SdkBody::from(body) + }; + let mut res = http::Response::builder().status(status).body(body).unwrap(); + let headers_map = res.headers_mut(); + for (name, value) in headers { + headers_map.insert( + HeaderName::from_str(name.as_ref()).unwrap(), + HeaderValue::from_str(value.as_str()).unwrap(), + ); + } + streams::drop_input_stream(stream); + http_types::drop_incoming_response(res_pointer); + res + } +} + +pub struct HttpHeader(http_types::Fields); + +impl HttpHeader { + pub fn inner(self) -> http_types::Fields { + self.0 + } +} -pub(crate) fn make_request(_req: http::Request) -> Result, ()> { - // Consumers here would pass the HTTP request to - // the Wasm host in order to get the response back - let body = " - - - - 2023-01-23T11:59:03.575496Z - doc-example-bucket - - - 2023-01-23T23:32:13.125238Z - doc-example-bucket2 - - - - account-name - a3a42310-42d0-46d1-9745-0cee9f4fb851 - - "; - Ok(http::Response::new(SdkBody::from(body))) +impl<'a> From<&'a http::HeaderMap> for HttpHeader { + fn from(headers: &'a http::HeaderMap) -> Self { + Self(http_types::new_fields( + &headers + .iter() + .map(|(name, value)| (name.as_str(), value.to_str().unwrap())) + .collect::>(), + )) + } } From 04c76bda0bba307570f90840a42bfd46434805c3 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Fri, 31 Mar 2023 04:12:57 +0200 Subject: [PATCH 02/72] feat: support wasi in default configuration --- aws/rust-runtime/aws-config/Cargo.toml | 4 + aws/rust-runtime/aws-config/src/connector.rs | 29 +++- aws/rust-runtime/aws-config/src/lib.rs | 2 + .../aws-config/src/wasi_adapter/mod.rs | 49 ++++++ .../integration-tests/webassembly/Cargo.toml | 2 +- .../webassembly/src/adapter/http_client.rs | 143 ------------------ .../webassembly/src/adapter/mod.rs | 44 ------ .../webassembly/src/default_config.rs | 6 +- .../integration-tests/webassembly/src/lib.rs | 17 ++- .../webassembly/src/list_buckets.rs | 10 +- rust-runtime/aws-smithy-async/src/rt/sleep.rs | 36 ++++- 11 files changed, 139 insertions(+), 203 deletions(-) create mode 100644 aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs delete mode 100644 aws/sdk/integration-tests/webassembly/src/adapter/http_client.rs delete mode 100644 aws/sdk/integration-tests/webassembly/src/adapter/mod.rs diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index 2a350f3ef2..2ea0f83226 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -47,6 +47,10 @@ ring = { version = "0.16", optional = true } hex = { version = "0.4.3", optional = true } zeroize = { version = "1", optional = true } +[target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +anyhow = "1.0" +wasi-http = { git = "https://github.com/bytecodealliance/wasi", rev = "b5f853aef2be2058616605ef8ea525450c8dac35" } + [dev-dependencies] futures-util = { version = "0.3.16", default-features = false } tracing-test = "0.2.1" diff --git a/aws/rust-runtime/aws-config/src/connector.rs b/aws/rust-runtime/aws-config/src/connector.rs index 82bda6e02e..9c49d94c64 100644 --- a/aws/rust-runtime/aws-config/src/connector.rs +++ b/aws/rust-runtime/aws-config/src/connector.rs @@ -44,7 +44,34 @@ pub fn default_connector( } /// Given `ConnectorSettings` and an `AsyncSleep`, create a `DynConnector` from defaults depending on what cargo features are activated. -#[cfg(not(feature = "rustls"))] +#[cfg(all(not(feature = "rustls"), feature = "native-tls"))] +pub fn default_connector( + settings: &ConnectorSettings, + sleep: Option>, +) -> Option { + let hyper = base(settings, sleep).build(aws_smithy_client::conns::native_tls()); + Some(DynConnector::new(hyper)) +} + +/// Given `ConnectorSettings` and an `AsyncSleep`, create a `DynConnector` from defaults depending on what cargo features are activated. +#[cfg(all( + target_family = "wasm", + target_os = "wasi", + not(any(feature = "rustls", feature = "native-tls")) +))] +pub fn default_connector( + _settings: &ConnectorSettings, + _sleep: Option>, +) -> Option { + Some(DynConnector::new(crate::wasi_adapter::Adapter::default())) +} + +/// Given `ConnectorSettings` and an `AsyncSleep`, create a `DynConnector` from defaults depending on what cargo features are activated. +#[cfg(not(any( + all(target_family = "wasm", target_os = "wasi"), + feature = "rustls", + feature = "native-tls" +)))] pub fn default_connector( _settings: &ConnectorSettings, _sleep: Option>, diff --git a/aws/rust-runtime/aws-config/src/lib.rs b/aws/rust-runtime/aws-config/src/lib.rs index 2d237a5b8f..169a9410ef 100644 --- a/aws/rust-runtime/aws-config/src/lib.rs +++ b/aws/rust-runtime/aws-config/src/lib.rs @@ -127,6 +127,8 @@ pub mod sso; pub(crate) mod standard_property; pub mod sts; pub mod timeout; +#[cfg(all(target_family = "wasm", target_os = "wasi"))] +pub mod wasi_adapter; pub mod web_identity_token; /// Create an environment loader for AWS Configuration diff --git a/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs b/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs new file mode 100644 index 0000000000..cbb2b1c4b2 --- /dev/null +++ b/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! HTTP Service Adapter used in WASI environment + +use aws_smithy_http::body::SdkBody; +use aws_smithy_http::result::ConnectorError; +use http::{Request, Response}; +use std::task::{Context, Poll}; +use tower::Service; + +#[derive(Default, Debug, Clone)] +pub(crate) struct Adapter {} + +impl Service> for Adapter { + type Response = Response; + + type Error = ConnectorError; + + #[allow(clippy::type_complexity)] + type Future = std::pin::Pin< + Box> + Send + 'static>, + >; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, req: Request) -> Self::Future { + println!("Adapter: sending request..."); + let client = wasi_http::DefaultClient::new(None); + let (parts, body) = req.into_parts(); + let body = match body.bytes() { + Some(value) => value.to_vec(), + None => Vec::new(), + }; + let req = Request::from_parts(parts, body); + let res = client.handle(req).unwrap(); + let (parts, body) = res.into_parts(); + let body = if body.is_empty() { + SdkBody::empty() + } else { + SdkBody::from(body) + }; + Box::pin(async move { Ok(Response::::from_parts(parts, body)) }) + } +} diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index b66406e6f5..8e9e59777c 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -18,7 +18,7 @@ crate-type = ["cdylib"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false, features = ["rt-tokio"]} +aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false } aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["hardcoded-credentials"] } aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3", default-features = false } aws-smithy-client = { path = "../../build/aws-sdk/sdk/aws-smithy-client", default-features = false } diff --git a/aws/sdk/integration-tests/webassembly/src/adapter/http_client.rs b/aws/sdk/integration-tests/webassembly/src/adapter/http_client.rs deleted file mode 100644 index 9012cde6e0..0000000000 --- a/aws/sdk/integration-tests/webassembly/src/adapter/http_client.rs +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#![allow(unused_variables)] - -use aws_smithy_http::body::SdkBody; -use http::header::{HeaderName, HeaderValue}; -use std::str::FromStr; -use wasi::{default_outgoing_http, poll, streams, types as http_types}; - -pub(crate) fn make_request(req: http::Request) -> anyhow::Result> { - let options = Some(http_types::RequestOptions { - connect_timeout_ms: None, - first_byte_timeout_ms: None, - between_bytes_timeout_ms: None, - }); - let req = HttpRequest::from(req).inner(); - // println!("HttpRequest: {:?}", req); - let res = default_outgoing_http::handle(req, options); - println!("{:?}", res); - let res: http::Response = HttpResponse(res).into(); - http_types::drop_outgoing_request(req); - Ok(res) -} - -pub struct HttpRequest(http_types::OutgoingRequest); - -impl HttpRequest { - pub fn inner(self) -> http_types::OutgoingRequest { - self.0 - } -} - -impl From> for HttpRequest { - fn from(req: http::Request) -> Self { - let (parts, sdk_body) = req.into_parts(); - // println!("request parts: {:?}", parts); - // println!("request body: {:?}", sdk_body); - let path = parts.uri.path(); - let query = parts.uri.query(); - let method = HttpMethod::from(parts.method); - let headers = HttpHeader::from(&parts.headers); - let scheme = match parts.uri.scheme_str().unwrap_or("") { - "http" => Some(http_types::SchemeParam::Http), - "https" => Some(http_types::SchemeParam::Https), - _ => None, - }; - Self(http_types::new_outgoing_request( - method.inner(), - path, - query.unwrap_or(""), - scheme, - parts.uri.authority().map(|a| a.as_str()).unwrap(), - headers.inner(), - )) - } -} - -pub struct HttpMethod<'a>(http_types::MethodParam<'a>); - -impl<'a> HttpMethod<'a> { - pub fn inner(self) -> http_types::MethodParam<'a> { - self.0 - } -} - -impl<'a> From for HttpMethod<'a> { - fn from(method: http::Method) -> Self { - Self(match method { - http::Method::GET => http_types::MethodParam::Get, - http::Method::POST => http_types::MethodParam::Post, - http::Method::PUT => http_types::MethodParam::Put, - http::Method::DELETE => http_types::MethodParam::Delete, - http::Method::PATCH => http_types::MethodParam::Patch, - http::Method::CONNECT => http_types::MethodParam::Connect, - http::Method::TRACE => http_types::MethodParam::Trace, - http::Method::HEAD => http_types::MethodParam::Head, - http::Method::OPTIONS => http_types::MethodParam::Options, - _ => panic!("failed due to unsupported method, currently supported methods are: GET, POST, PUT, DELETE, PATCH, CONNECT, TRACE, HEAD, and OPTIONS"), - }) - } -} - -struct HttpResponse(http_types::IncomingResponse); - -impl HttpResponse { - pub fn inner(self) -> http_types::IncomingResponse { - self.0 - } -} - -impl From for http::Response { - fn from(val: HttpResponse) -> Self { - let res_pointer = val.inner(); - poll::drop_pollable(res_pointer); - let status = http_types::incoming_response_status(res_pointer); - println!("status: {}", status); - let header_handle = http_types::incoming_response_headers(res_pointer); - let headers = http_types::fields_entries(header_handle); - println!("headers: {:?}", headers); - let stream: http_types::IncomingStream = - http_types::incoming_response_consume(res_pointer).unwrap(); - let len = 64 * 1024; - let (body, finished) = streams::read(stream, len).unwrap(); - let body = if body.is_empty() { - SdkBody::empty() - } else { - SdkBody::from(body) - }; - let mut res = http::Response::builder().status(status).body(body).unwrap(); - let headers_map = res.headers_mut(); - for (name, value) in headers { - headers_map.insert( - HeaderName::from_str(name.as_ref()).unwrap(), - HeaderValue::from_str(value.as_str()).unwrap(), - ); - } - streams::drop_input_stream(stream); - http_types::drop_incoming_response(res_pointer); - res - } -} - -pub struct HttpHeader(http_types::Fields); - -impl HttpHeader { - pub fn inner(self) -> http_types::Fields { - self.0 - } -} - -impl<'a> From<&'a http::HeaderMap> for HttpHeader { - fn from(headers: &'a http::HeaderMap) -> Self { - Self(http_types::new_fields( - &headers - .iter() - .map(|(name, value)| (name.as_str(), value.to_str().unwrap())) - .collect::>(), - )) - } -} diff --git a/aws/sdk/integration-tests/webassembly/src/adapter/mod.rs b/aws/sdk/integration-tests/webassembly/src/adapter/mod.rs deleted file mode 100644 index b563eb097a..0000000000 --- a/aws/sdk/integration-tests/webassembly/src/adapter/mod.rs +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -mod http_client; - -use aws_smithy_client::erase::DynConnector; -use aws_smithy_client::http_connector::HttpConnector; -use aws_smithy_http::body::SdkBody; -use aws_smithy_http::result::ConnectorError; -use std::task::{Context, Poll}; -use tower::Service; - -#[derive(Default, Debug, Clone)] -pub(crate) struct Adapter {} - -impl Adapter { - pub fn to_http_connector() -> impl Into { - DynConnector::new(Adapter::default()) - } -} - -impl Service> for Adapter { - type Response = http::Response; - - type Error = ConnectorError; - - #[allow(clippy::type_complexity)] - type Future = std::pin::Pin< - Box> + Send + 'static>, - >; - - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn call(&mut self, req: http::Request) -> Self::Future { - println!("Adapter: sending request..."); - let res = http_client::make_request(req).unwrap(); - println!("{:?}", res); - Box::pin(async move { Ok(res) }) - } -} diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index fd8899cbe0..2780268018 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -3,14 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -use aws_config::retry::RetryConfig; use aws_credential_types::Credentials; -use aws_smithy_types::timeout::TimeoutConfig; +use aws_smithy_types::{retry::RetryConfig, timeout::TimeoutConfig}; use aws_types::region::Region; use std::future::Future; -use crate::adapter::Adapter; - pub(crate) fn get_default_config() -> impl Future { aws_config::from_env() .region(Region::from_static("us-west-2")) @@ -21,7 +18,6 @@ pub(crate) fn get_default_config() -> impl Future ListBucketsOutput { use aws_sdk_s3::Client; use crate::default_config::get_default_config; let shared_config = get_default_config().await; let client = Client::new(&shared_config); - let result = client.list_buckets().send().await.unwrap(); - assert_eq!(result.buckets().unwrap().len(), 2) + client.list_buckets().send().await.unwrap() } #[tokio::test] pub async fn test_s3_list_buckets() { - s3_list_buckets().await + let result = s3_list_buckets().await; + assert!(result.buckets().unwrap().len() > 0); } diff --git a/rust-runtime/aws-smithy-async/src/rt/sleep.rs b/rust-runtime/aws-smithy-async/src/rt/sleep.rs index 4ea9f70ed5..af06f3b3fe 100644 --- a/rust-runtime/aws-smithy-async/src/rt/sleep.rs +++ b/rust-runtime/aws-smithy-async/src/rt/sleep.rs @@ -45,7 +45,13 @@ pub fn default_async_sleep() -> Option> { Some(sleep_tokio()) } -#[cfg(not(feature = "rt-tokio"))] +#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +/// Returns a default sleep implementation based on the features enabled +pub fn default_async_sleep() -> Option> { + Some(sleep_wasi()) +} + +#[cfg(all(any(target_family = "wasm", target_os = "wasi"), feature = "rt-tokio"))] /// Returns a default sleep implementation based on the features enabled pub fn default_async_sleep() -> Option> { None @@ -104,3 +110,31 @@ impl AsyncSleep for TokioSleep { fn sleep_tokio() -> Arc { Arc::new(TokioSleep::new()) } + +/// Implementation of [`AsyncSleep`] for WASI. +#[non_exhaustive] +#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +#[derive(Debug, Default)] +pub struct WasiSleep; + +#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +impl WasiSleep { + /// Create a new [`AsyncSleep`] implementation using the WASI sleep implementation + pub fn new() -> WasiSleep { + Default::default() + } +} + +#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +impl AsyncSleep for WasiSleep { + fn sleep(&self, duration: std::time::Duration) -> Sleep { + Sleep::new(Box::pin(async move { + std::thread::sleep(duration); + })) + } +} + +#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +fn sleep_wasi() -> Arc { + Arc::new(WasiSleep::new()) +} From e3547cae1f55a157cf3eed1b1fc4ff9282ca8e19 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 4 Apr 2023 02:17:10 +0200 Subject: [PATCH 03/72] chore: use bytes body with wasi-http --- aws/rust-runtime/aws-config/Cargo.toml | 3 +-- .../aws-config/src/wasi_adapter/mod.rs | 25 ++++++++----------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index 2ea0f83226..6807790960 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -48,8 +48,7 @@ hex = { version = "0.4.3", optional = true } zeroize = { version = "1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] -anyhow = "1.0" -wasi-http = { git = "https://github.com/bytecodealliance/wasi", rev = "b5f853aef2be2058616605ef8ea525450c8dac35" } +wasi-http = { git = "https://github.com/bytecodealliance/wasi", rev = "43f1a2a45bbefdabe32551eac77885d31199b705" } [dev-dependencies] futures-util = { version = "0.3.16", default-features = false } diff --git a/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs b/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs index cbb2b1c4b2..2f6e6878ef 100644 --- a/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs +++ b/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs @@ -7,6 +7,7 @@ use aws_smithy_http::body::SdkBody; use aws_smithy_http::result::ConnectorError; +use bytes::Bytes; use http::{Request, Response}; use std::task::{Context, Poll}; use tower::Service; @@ -31,19 +32,15 @@ impl Service> for Adapter { fn call(&mut self, req: Request) -> Self::Future { println!("Adapter: sending request..."); let client = wasi_http::DefaultClient::new(None); - let (parts, body) = req.into_parts(); - let body = match body.bytes() { - Some(value) => value.to_vec(), - None => Vec::new(), - }; - let req = Request::from_parts(parts, body); - let res = client.handle(req).unwrap(); - let (parts, body) = res.into_parts(); - let body = if body.is_empty() { - SdkBody::empty() - } else { - SdkBody::from(body) - }; - Box::pin(async move { Ok(Response::::from_parts(parts, body)) }) + // Right now only synchronous calls can be made through WASI + let fut = client.handle(req.map(|body| match body.bytes() { + Some(value) => Bytes::copy_from_slice(value), + None => Bytes::new(), + })); + Box::pin(async move { + Ok(fut + .map_err(|err| ConnectorError::other(err.into(), None))? + .map(SdkBody::from)) + }) } } From 7c33972f361101790a07d39bf59e9fc55ae517b8 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 4 Apr 2023 02:18:19 +0200 Subject: [PATCH 04/72] chore: load credentials from environment variables --- .../integration-tests/webassembly/Cargo.toml | 2 + .../webassembly/src/default_config.rs | 44 +++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index 8e9e59777c..7963e7ff72 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -18,6 +18,7 @@ crate-type = ["cdylib"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = "1.0" aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false } aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["hardcoded-credentials"] } aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3", default-features = false } @@ -28,3 +29,4 @@ aws-types = { path = "../../build/aws-sdk/sdk/aws-types" } http = "0.2.8" tokio = { version = "1.24.2", features = ["macros", "rt"] } tower = "0.4.13" +wasi = "0.11.0" diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index 2780268018..92217822a6 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -3,18 +3,54 @@ * SPDX-License-Identifier: Apache-2.0 */ +use anyhow::{Context, Result}; use aws_credential_types::Credentials; use aws_smithy_types::{retry::RetryConfig, timeout::TimeoutConfig}; use aws_types::region::Region; -use std::future::Future; +use std::{ffi::CStr, future::Future}; + +// This can be replaced with `std::env::vars` as soon as the component model lands in wasmtime +// https://github.com/bytecodealliance/wasmtime/issues/4185 +fn read_environment() -> Result<(Option, Option, Option)> { + let (count, size) = unsafe { wasi::environ_sizes_get()? }; + let mut entries: Vec<*mut u8> = Vec::with_capacity(count); + + let mut buf: Vec = Vec::with_capacity(size); + unsafe { wasi::environ_get(entries.as_mut_ptr(), buf.as_mut_ptr())? }; + unsafe { entries.set_len(count) }; + + let mut access_key: Option = None; + let mut secret_key: Option = None; + let mut session_token: Option = None; + + for entry in entries { + let cstr = unsafe { CStr::from_ptr(entry as *const i8) }; + let (name, value) = cstr + .to_str()? + .split_once('=') + .context("missing = in environment variable")?; + if name == "AWS_ACCESS_KEY_ID" { + access_key = Some(value.to_string()); + } else if name == "AWS_SECRET_ACCESS_KEY" { + secret_key = Some(value.to_string()); + } else if name == "AWS_SESSION_TOKEN" { + session_token = Some(value.to_string()); + } + } + + Ok((access_key, secret_key, session_token)) +} pub(crate) fn get_default_config() -> impl Future { + let (access_key, secret_key, session_token) = + read_environment().expect("read aws credentials from environment variables"); + aws_config::from_env() .region(Region::from_static("us-west-2")) .credentials_provider(Credentials::from_keys( - "access_key", - "secret_key", - Some("session_token".to_string()), + access_key.expect("AWS_ACCESS_KEY_ID"), + secret_key.expect("AWS_SECRET_ACCESS_KEY"), + session_token, )) .timeout_config(TimeoutConfig::disabled()) .retry_config(RetryConfig::disabled()) From 528d0f8b21835e1ed581711934f2fd107caa35bc Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 4 Apr 2023 02:20:03 +0200 Subject: [PATCH 05/72] chore: run wasm integration tests --- tools/ci-scripts/check-aws-sdk-standalone-integration-tests | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests index ad626ec1fa..b48b689877 100755 --- a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests +++ b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests @@ -39,4 +39,10 @@ find "${tmp_dir}" pushd "${tmp_dir}/aws/sdk/integration-tests" cargo check --tests + +# Running WebAssembly (WASI) specific integration tests +pushd "${tmp_dir}/aws/sdk/integration-tests/webassembly" &>/dev/null +cargo wasi build +env_credentials="--env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} --env AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}" +CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime --wasi-modules=experimental-wasi-http ${env_credentials}" cargo wasi test -- --nocapture popd From 1ddbd459754ee71e24fdf86bf5cf3d3c6eb323b3 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 4 Apr 2023 21:08:28 +0200 Subject: [PATCH 06/72] chore: prepare git folder within cargo --- tools/ci-build/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index f5ed4869d9..7c957b4f17 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -86,6 +86,9 @@ RUN set -eux; \ cargo +${rust_nightly_version} -Z sparse-registry install --locked --path tools/ci-build/difftags; \ cargo +${rust_nightly_version} -Z sparse-registry install --locked --path tools/ci-build/sdk-lints; \ cargo +${rust_nightly_version} -Z sparse-registry install --locked --path tools/ci-build/sdk-versioner; \ + mkdir -p /opt/cargo/git/db; \ + chmod g+rw -R /opt/cargo/git; \ + chmod g+rw -R /opt/cargo/bin; \ chmod g+rw -R /opt/cargo/registry FROM install_rust AS cargo_deny From c4c306f41a71cd8d8715856a62d177cc830d0dc2 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Sun, 23 Apr 2023 21:37:37 +0200 Subject: [PATCH 07/72] chore(ci): update wasmtime to v8.0.0 --- tools/ci-build/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 7c957b4f17..1c22b962e3 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -127,8 +127,8 @@ ARG rust_nightly_version RUN cargo +${rust_nightly_version} -Z sparse-registry install wasm-pack --locked --version ${wasm_pack_version} FROM install_rust AS wasmtime -ARG wasmtime_precompiled_url=https://github.com/bytecodealliance/wasmtime/releases/download/v7.0.0/wasmtime-v7.0.0-x86_64-linux.tar.xz -ARG wasmtime_precompiled_sha256=b8a1c97f9107c885ea73a5c38677d0d340a7c26879d366e8a5f3dce84cffec99 +ARG wasmtime_precompiled_url=https://github.com/bytecodealliance/wasmtime/releases/download/v8.0.0/wasmtime-v8.0.0-x86_64-linux.tar.xz +ARG wasmtime_precompiled_sha256=0ba34842bbac8896dbdd860173b2c554d93d63ecc490f725fe3331db913714ad ARG rust_nightly_version RUN set -eux; \ curl "${wasmtime_precompiled_url}" -L -o wasmtime.xz; \ From 34827261db1aaa232c337048689b7c06213e60ea Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Sun, 23 Apr 2023 21:42:00 +0200 Subject: [PATCH 08/72] chore: update to latest commit in wasi-http crate --- aws/rust-runtime/aws-config/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index 6807790960..5b82bc70e2 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -48,7 +48,7 @@ hex = { version = "0.4.3", optional = true } zeroize = { version = "1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] -wasi-http = { git = "https://github.com/bytecodealliance/wasi", rev = "43f1a2a45bbefdabe32551eac77885d31199b705" } +wasi-http = { git = "https://github.com/bytecodealliance/wasi", rev = "b381fe3b35ba061a4f2ab7578c397424752e5e75" } [dev-dependencies] futures-util = { version = "0.3.16", default-features = false } From 17927d9294ce1b438b3bdd91dc42a4856d71696b Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Sun, 23 Apr 2023 21:45:08 +0200 Subject: [PATCH 09/72] chore: separate variable for listed buckets --- aws/sdk/integration-tests/webassembly/src/list_buckets.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aws/sdk/integration-tests/webassembly/src/list_buckets.rs b/aws/sdk/integration-tests/webassembly/src/list_buckets.rs index ec96a8c366..2c5c7e7945 100644 --- a/aws/sdk/integration-tests/webassembly/src/list_buckets.rs +++ b/aws/sdk/integration-tests/webassembly/src/list_buckets.rs @@ -18,5 +18,6 @@ pub async fn s3_list_buckets() -> ListBucketsOutput { #[tokio::test] pub async fn test_s3_list_buckets() { let result = s3_list_buckets().await; - assert!(result.buckets().unwrap().len() > 0); + let buckets = result.buckets().unwrap(); + assert!(buckets.len() > 0); } From 46919188a8521f4f362ecde82d88547b223b2398 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Sun, 23 Apr 2023 23:57:07 +0200 Subject: [PATCH 10/72] chore: set aws credentials env variables to empty string --- tools/ci-scripts/check-aws-sdk-standalone-integration-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests index b48b689877..034b05a005 100755 --- a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests +++ b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests @@ -43,6 +43,6 @@ cargo check --tests # Running WebAssembly (WASI) specific integration tests pushd "${tmp_dir}/aws/sdk/integration-tests/webassembly" &>/dev/null cargo wasi build -env_credentials="--env AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} --env AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}" +env_credentials="--env AWS_ACCESS_KEY_ID='' --env AWS_SECRET_ACCESS_KEY='' --env AWS_SESSION_TOKEN=''" CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime --wasi-modules=experimental-wasi-http ${env_credentials}" cargo wasi test -- --nocapture popd From 24a56f0f312503d4aaad707bf4ae0f9dd43638dd Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Thu, 27 Apr 2023 02:10:09 +0200 Subject: [PATCH 11/72] chore: update wasi-http crate --- aws/rust-runtime/aws-config/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index 5b82bc70e2..29bf1ae101 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -48,7 +48,7 @@ hex = { version = "0.4.3", optional = true } zeroize = { version = "1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] -wasi-http = { git = "https://github.com/bytecodealliance/wasi", rev = "b381fe3b35ba061a4f2ab7578c397424752e5e75" } +wasi-http = { git = "https://github.com/bytecodealliance/wasi", rev = "1bf637e10c84a5f93575bffd25e17f39d8589217" } [dev-dependencies] futures-util = { version = "0.3.16", default-features = false } From d91a56ba854a84e15cf386eba73bb6bbf2a184b5 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Thu, 27 Apr 2023 02:28:47 +0200 Subject: [PATCH 12/72] chore: set component-model feature flag --- .../webassembly/src/default_config.rs | 44 ++----------------- ...check-aws-sdk-standalone-integration-tests | 2 +- 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index 92217822a6..e172abbbd4 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -3,54 +3,18 @@ * SPDX-License-Identifier: Apache-2.0 */ -use anyhow::{Context, Result}; use aws_credential_types::Credentials; use aws_smithy_types::{retry::RetryConfig, timeout::TimeoutConfig}; use aws_types::region::Region; -use std::{ffi::CStr, future::Future}; - -// This can be replaced with `std::env::vars` as soon as the component model lands in wasmtime -// https://github.com/bytecodealliance/wasmtime/issues/4185 -fn read_environment() -> Result<(Option, Option, Option)> { - let (count, size) = unsafe { wasi::environ_sizes_get()? }; - let mut entries: Vec<*mut u8> = Vec::with_capacity(count); - - let mut buf: Vec = Vec::with_capacity(size); - unsafe { wasi::environ_get(entries.as_mut_ptr(), buf.as_mut_ptr())? }; - unsafe { entries.set_len(count) }; - - let mut access_key: Option = None; - let mut secret_key: Option = None; - let mut session_token: Option = None; - - for entry in entries { - let cstr = unsafe { CStr::from_ptr(entry as *const i8) }; - let (name, value) = cstr - .to_str()? - .split_once('=') - .context("missing = in environment variable")?; - if name == "AWS_ACCESS_KEY_ID" { - access_key = Some(value.to_string()); - } else if name == "AWS_SECRET_ACCESS_KEY" { - secret_key = Some(value.to_string()); - } else if name == "AWS_SESSION_TOKEN" { - session_token = Some(value.to_string()); - } - } - - Ok((access_key, secret_key, session_token)) -} +use std::{env, future::Future}; pub(crate) fn get_default_config() -> impl Future { - let (access_key, secret_key, session_token) = - read_environment().expect("read aws credentials from environment variables"); - aws_config::from_env() .region(Region::from_static("us-west-2")) .credentials_provider(Credentials::from_keys( - access_key.expect("AWS_ACCESS_KEY_ID"), - secret_key.expect("AWS_SECRET_ACCESS_KEY"), - session_token, + env::var("AWS_ACCESS_KEY_ID").expect("AWS_ACCESS_KEY_ID"), + env::var("AWS_SECRET_ACCESS_KEY").expect("AWS_SECRET_ACCESS_KEY"), + env::var("AWS_SESSION_TOKEN").ok(), )) .timeout_config(TimeoutConfig::disabled()) .retry_config(RetryConfig::disabled()) diff --git a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests index 034b05a005..0e6d9fb521 100755 --- a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests +++ b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests @@ -44,5 +44,5 @@ cargo check --tests pushd "${tmp_dir}/aws/sdk/integration-tests/webassembly" &>/dev/null cargo wasi build env_credentials="--env AWS_ACCESS_KEY_ID='' --env AWS_SECRET_ACCESS_KEY='' --env AWS_SESSION_TOKEN=''" -CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime --wasi-modules=experimental-wasi-http ${env_credentials}" cargo wasi test -- --nocapture +CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime --wasm-features=component-model --wasi-modules=experimental-wasi-http ${env_credentials}" cargo wasi test -- --nocapture popd From dfe6aa06610e46c9c84e4f8b3e35d76210530908 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Sat, 29 Apr 2023 11:00:34 +0200 Subject: [PATCH 13/72] chore: use wasi-preview2-prototype crate --- aws/rust-runtime/aws-config/Cargo.toml | 2 +- aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index 29bf1ae101..b91720d128 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -48,7 +48,7 @@ hex = { version = "0.4.3", optional = true } zeroize = { version = "1", optional = true } [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] -wasi-http = { git = "https://github.com/bytecodealliance/wasi", rev = "1bf637e10c84a5f93575bffd25e17f39d8589217" } +wasi-preview2-prototype = { version = "0.0.1", features = ["http-client"] } [dev-dependencies] futures-util = { version = "0.3.16", default-features = false } diff --git a/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs b/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs index 2f6e6878ef..ad426b98c0 100644 --- a/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs +++ b/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs @@ -11,6 +11,7 @@ use bytes::Bytes; use http::{Request, Response}; use std::task::{Context, Poll}; use tower::Service; +use wasi_preview2_prototype::http_client::DefaultClient; #[derive(Default, Debug, Clone)] pub(crate) struct Adapter {} @@ -31,7 +32,7 @@ impl Service> for Adapter { fn call(&mut self, req: Request) -> Self::Future { println!("Adapter: sending request..."); - let client = wasi_http::DefaultClient::new(None); + let client = DefaultClient::new(None); // Right now only synchronous calls can be made through WASI let fut = client.handle(req.map(|body| match body.bytes() { Some(value) => Bytes::copy_from_slice(value), From 904f002e8cd77db578c1c79d4866aa4ad3df6da5 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Sat, 29 Apr 2023 11:08:11 +0200 Subject: [PATCH 14/72] chore: remove wasi dependency --- aws/sdk/integration-tests/webassembly/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index 7963e7ff72..c0d2638af5 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -29,4 +29,3 @@ aws-types = { path = "../../build/aws-sdk/sdk/aws-types" } http = "0.2.8" tokio = { version = "1.24.2", features = ["macros", "rt"] } tower = "0.4.13" -wasi = "0.11.0" From cba7b09a84ad091568072dc9d6501e457822f4eb Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Wed, 3 May 2023 12:53:34 +0200 Subject: [PATCH 15/72] chore: install wasmtime with component model feature --- tools/ci-build/Dockerfile | 81 ++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 43 deletions(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 1c22b962e3..35d8995026 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -15,9 +15,9 @@ RUN yum -y updateinfo FROM bare_base_image as musl_toolchain RUN yum -y install tar gzip gcc make RUN curl https://musl.libc.org/releases/musl-1.2.3.tar.gz -o musl-1.2.3.tar.gz \ - && ls \ - && tar xvzf musl-1.2.3.tar.gz \ - && (cd musl-1.2.3 && ./configure && make install) + && ls \ + && tar xvzf musl-1.2.3.tar.gz \ + && (cd musl-1.2.3 && ./configure && make install) # # Rust & Tools Installation Stage @@ -31,27 +31,27 @@ ENV RUSTUP_HOME=/opt/rustup \ CARGO_INCREMENTAL=0 WORKDIR /root RUN yum -y install \ - autoconf \ - automake \ - binutils \ - ca-certificates \ - curl \ - gcc \ - gcc-c++ \ - git \ - make \ - openssl-devel \ - pkgconfig \ - tar \ - xz && \ + autoconf \ + automake \ + binutils \ + ca-certificates \ + curl \ + gcc \ + gcc-c++ \ + git \ + make \ + openssl-devel \ + pkgconfig \ + tar \ + xz && \ yum clean all RUN set -eux; \ if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then \ - curl https://static.rust-lang.org/rustup/archive/1.24.3/aarch64-unknown-linux-gnu/rustup-init --output rustup-init; \ - echo "32a1532f7cef072a667bac53f1a5542c99666c4071af0c9549795bbdb2069ec1 rustup-init" | sha256sum --check; \ + curl https://static.rust-lang.org/rustup/archive/1.24.3/aarch64-unknown-linux-gnu/rustup-init --output rustup-init; \ + echo "32a1532f7cef072a667bac53f1a5542c99666c4071af0c9549795bbdb2069ec1 rustup-init" | sha256sum --check; \ else \ - curl https://static.rust-lang.org/rustup/archive/1.24.3/x86_64-unknown-linux-gnu/rustup-init --output rustup-init; \ - echo "3dc5ef50861ee18657f9db2eeb7392f9c2a6c95c90ab41e45ab4ca71476b4338 rustup-init" | sha256sum --check; \ + curl https://static.rust-lang.org/rustup/archive/1.24.3/x86_64-unknown-linux-gnu/rustup-init --output rustup-init; \ + echo "3dc5ef50861ee18657f9db2eeb7392f9c2a6c95c90ab41e45ab4ca71476b4338 rustup-init" | sha256sum --check; \ fi; \ chmod +x rustup-init; \ ./rustup-init -y --no-modify-path --profile minimal --default-toolchain ${rust_stable_version}; \ @@ -76,9 +76,9 @@ ARG smithy_rs_commit_hash=main ARG checkout_smithy_rs_tools=false RUN set -eux; \ if [[ "${checkout_smithy_rs_tools}" == "true" ]]; then \ - git clone https://github.com/awslabs/smithy-rs.git; \ - cd smithy-rs; \ - git checkout ${smithy_rs_commit_hash}; \ + git clone https://github.com/awslabs/smithy-rs.git; \ + cd smithy-rs; \ + git checkout ${smithy_rs_commit_hash}; \ fi; \ cargo +${rust_nightly_version} -Z sparse-registry install --locked --path tools/ci-build/publisher; \ cargo +${rust_nightly_version} -Z sparse-registry install --locked --path tools/ci-build/changelogger; \ @@ -127,14 +127,9 @@ ARG rust_nightly_version RUN cargo +${rust_nightly_version} -Z sparse-registry install wasm-pack --locked --version ${wasm_pack_version} FROM install_rust AS wasmtime -ARG wasmtime_precompiled_url=https://github.com/bytecodealliance/wasmtime/releases/download/v8.0.0/wasmtime-v8.0.0-x86_64-linux.tar.xz -ARG wasmtime_precompiled_sha256=0ba34842bbac8896dbdd860173b2c554d93d63ecc490f725fe3331db913714ad +ARG cargo_wasmtime_version=9.0.1 ARG rust_nightly_version -RUN set -eux; \ - curl "${wasmtime_precompiled_url}" -L -o wasmtime.xz; \ - echo "${wasmtime_precompiled_sha256} wasmtime.xz" | sha256sum --check; \ - tar xf wasmtime.xz; \ - mv wasmtime-v*/wasmtime /opt; +RUN cargo +${rust_nightly_version} -Z sparse-registry install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} FROM install_rust AS cargo_wasi ARG cargo_wasi_version=0.1.27 @@ -153,18 +148,18 @@ ARG rust_stable_version ARG rust_nightly_version RUN set -eux; \ yum -y install \ - bc \ - ca-certificates \ - gcc \ - git \ - java-11-amazon-corretto-headless \ - make \ - openssl-devel \ - pkgconfig \ - python3 \ - python3-devel \ - python3-pip \ - shadow-utils; \ + bc \ + ca-certificates \ + gcc \ + git \ + java-11-amazon-corretto-headless \ + make \ + openssl-devel \ + pkgconfig \ + python3 \ + python3-devel \ + python3-pip \ + shadow-utils; \ yum clean all; \ rm -rf /var/cache/yum; \ groupadd build; \ @@ -178,7 +173,7 @@ COPY --chown=build:build --from=cargo_minimal_versions /opt/cargo/bin/cargo-mini COPY --chown=build:build --from=cargo_check_external_types /opt/cargo/bin/cargo-check-external-types /opt/cargo/bin/cargo-check-external-types COPY --chown=build:build --from=maturin /opt/cargo/bin/maturin /opt/cargo/bin/maturin COPY --chown=build:build --from=wasm_pack /opt/cargo/bin/wasm-pack /opt/cargo/bin/wasm-pack -COPY --chown=build:build --from=wasmtime /opt/wasmtime /opt/cargo/bin/wasmtime +COPY --chown=build:build --from=wasmtime /opt/cargo/bin/wasmtime /opt/cargo/bin/wasmtime COPY --chown=build:build --from=cargo_wasi /opt/cargo/bin/cargo-wasi /opt/cargo/bin/cargo-wasi COPY --chown=build:build --from=install_rust /opt/rustup /opt/rustup COPY --chown=build:build --from=cargo_semver_checks /opt/cargo/bin/cargo-semver-checks /opt/cargo/bin/cargo-semver-checks From f6cb13a2f715e4bf5cd445fb3e59cffd93fffa77 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Thu, 25 May 2023 22:33:40 +0200 Subject: [PATCH 16/72] chore: use default credentials provider chain --- .../integration-tests/webassembly/src/default_config.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index e172abbbd4..0580c3b4b3 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -3,19 +3,13 @@ * SPDX-License-Identifier: Apache-2.0 */ -use aws_credential_types::Credentials; use aws_smithy_types::{retry::RetryConfig, timeout::TimeoutConfig}; use aws_types::region::Region; -use std::{env, future::Future}; +use std::future::Future; pub(crate) fn get_default_config() -> impl Future { aws_config::from_env() .region(Region::from_static("us-west-2")) - .credentials_provider(Credentials::from_keys( - env::var("AWS_ACCESS_KEY_ID").expect("AWS_ACCESS_KEY_ID"), - env::var("AWS_SECRET_ACCESS_KEY").expect("AWS_SECRET_ACCESS_KEY"), - env::var("AWS_SESSION_TOKEN").ok(), - )) .timeout_config(TimeoutConfig::disabled()) .retry_config(RetryConfig::disabled()) .load() From 99418c19c0b48fd2e67c88b1404752fefc413904 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Thu, 25 May 2023 23:17:15 +0200 Subject: [PATCH 17/72] feat: move logic to aws-smithy-wasm crate --- buildSrc/src/main/kotlin/CrateSet.kt | 73 ++++---- rust-runtime/Cargo.toml | 1 + rust-runtime/aws-smithy-wasm/Cargo.toml | 26 +++ rust-runtime/aws-smithy-wasm/LICENSE | 175 ++++++++++++++++++ rust-runtime/aws-smithy-wasm/README.md | 7 + .../aws-smithy-wasm/external-types.toml | 2 + rust-runtime/aws-smithy-wasm/src/lib.rs | 17 ++ .../aws-smithy-wasm/src/wasi_adapter.rs | 48 +++++ 8 files changed, 315 insertions(+), 34 deletions(-) create mode 100644 rust-runtime/aws-smithy-wasm/Cargo.toml create mode 100644 rust-runtime/aws-smithy-wasm/LICENSE create mode 100644 rust-runtime/aws-smithy-wasm/README.md create mode 100644 rust-runtime/aws-smithy-wasm/external-types.toml create mode 100644 rust-runtime/aws-smithy-wasm/src/lib.rs create mode 100644 rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs diff --git a/buildSrc/src/main/kotlin/CrateSet.kt b/buildSrc/src/main/kotlin/CrateSet.kt index 132246cf39..8232ad7230 100644 --- a/buildSrc/src/main/kotlin/CrateSet.kt +++ b/buildSrc/src/main/kotlin/CrateSet.kt @@ -4,44 +4,49 @@ */ object CrateSet { - val AWS_SDK_RUNTIME = listOf( - "aws-config", - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-hyper", - "aws-runtime", - "aws-runtime-api", - "aws-sig-auth", - "aws-sigv4", - "aws-types", - ) + val AWS_SDK_RUNTIME = + listOf( + "aws-config", + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-hyper", + "aws-runtime", + "aws-runtime-api", + "aws-sig-auth", + "aws-sigv4", + "aws-types", + ) - private val SMITHY_RUNTIME_COMMON = listOf( - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-client", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-http-auth", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-protocol-test", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-types-convert", - "aws-smithy-xml", - ) + private val SMITHY_RUNTIME_COMMON = + listOf( + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-auth", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-protocol-test", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-types-convert", + "aws-smithy-wasm", + "aws-smithy-xml", + ) val AWS_SDK_SMITHY_RUNTIME = SMITHY_RUNTIME_COMMON - val SERVER_SMITHY_RUNTIME = SMITHY_RUNTIME_COMMON + listOf( - "aws-smithy-http-server", - "aws-smithy-http-server-python", - "aws-smithy-http-server-typescript", - ) + val SERVER_SMITHY_RUNTIME = + SMITHY_RUNTIME_COMMON + + listOf( + "aws-smithy-http-server", + "aws-smithy-http-server-python", + "aws-smithy-http-server-typescript", + ) val ENTIRE_SMITHY_RUNTIME = (AWS_SDK_SMITHY_RUNTIME + SERVER_SMITHY_RUNTIME).toSortedSet() } diff --git a/rust-runtime/Cargo.toml b/rust-runtime/Cargo.toml index 4f1555345a..58925218d9 100644 --- a/rust-runtime/Cargo.toml +++ b/rust-runtime/Cargo.toml @@ -19,5 +19,6 @@ members = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-types-convert", + "aws-smithy-wasm", "aws-smithy-xml", ] diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml new file mode 100644 index 0000000000..de6cb50460 --- /dev/null +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "aws-smithy-wasm" +version = "0.0.0-smithy-rs-head" +authors = [ + "AWS Rust SDK Team ", + "Eduardo Rodrigues <16357187+eduardomourar@users.noreply.github.com>", +] +description = "Smithy WebAssembly configuration for smithy-rs." +edition = "2021" +license = "Apache-2.0" +repository = "https://github.com/awslabs/smithy-rs" + +[dependencies] +aws-smithy-http = { path = "../aws-smithy-http" } + +[target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +bytes = "1" +http = "0.2.4" +tower = { version = "0.4.8" } +wasi-preview2-prototype = { version = "0.0.1", features = ["http-client"] } + +[package.metadata.docs.rs] +all-features = true +targets = ["x86_64-unknown-linux-gnu"] +rustdoc-args = ["--cfg", "docsrs"] +# End of docs.rs metadata diff --git a/rust-runtime/aws-smithy-wasm/LICENSE b/rust-runtime/aws-smithy-wasm/LICENSE new file mode 100644 index 0000000000..67db858821 --- /dev/null +++ b/rust-runtime/aws-smithy-wasm/LICENSE @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/rust-runtime/aws-smithy-wasm/README.md b/rust-runtime/aws-smithy-wasm/README.md new file mode 100644 index 0000000000..277c7e75d1 --- /dev/null +++ b/rust-runtime/aws-smithy-wasm/README.md @@ -0,0 +1,7 @@ +# aws-smithy-wasm + +WebAssembly related configuration for service clients generated by [smithy-rs](https://github.com/awslabs/smithy-rs). + + +This crate is part of the [AWS SDK for Rust](https://awslabs.github.io/aws-sdk-rust/) and the [smithy-rs](https://github.com/awslabs/smithy-rs) code generator. In most cases, it should not be used directly. + diff --git a/rust-runtime/aws-smithy-wasm/external-types.toml b/rust-runtime/aws-smithy-wasm/external-types.toml new file mode 100644 index 0000000000..ff30ccf5ad --- /dev/null +++ b/rust-runtime/aws-smithy-wasm/external-types.toml @@ -0,0 +1,2 @@ +allowed_external_types = [ +] diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs new file mode 100644 index 0000000000..a0c4bfaf8f --- /dev/null +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -0,0 +1,17 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#![allow(clippy::derive_partial_eq_without_eq)] +#![warn( + missing_docs, + rustdoc::missing_crate_level_docs, + unreachable_pub, + rust_2018_idioms +)] + +//! Smithy WebAssembly + +#[cfg(all(target_family = "wasm", target_os = "wasi"))] +pub mod wasi_adapter; diff --git a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs new file mode 100644 index 0000000000..afb64feab7 --- /dev/null +++ b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs @@ -0,0 +1,48 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! HTTP WASI Adapter + +use aws_smithy_http::body::SdkBody; +use aws_smithy_http::result::ConnectorError; +use bytes::Bytes; +use http::{Request, Response}; +use std::task::{Context, Poll}; +use tower::Service; +use wasi_preview2_prototype::http_client::DefaultClient; + +#[derive(Default, Debug, Clone)] +/// HTTP Service Adapter used in WASI environment +pub struct Adapter {} + +impl Service> for Adapter { + type Response = Response; + + type Error = ConnectorError; + + #[allow(clippy::type_complexity)] + type Future = std::pin::Pin< + Box> + Send + 'static>, + >; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, req: Request) -> Self::Future { + println!("Adapter: sending request..."); + let client = DefaultClient::new(None); + // Right now only synchronous calls can be made through WASI + let fut = client.handle(req.map(|body| match body.bytes() { + Some(value) => Bytes::copy_from_slice(value), + None => Bytes::new(), + })); + Box::pin(async move { + Ok(fut + .map_err(|err| ConnectorError::other(err.into(), None))? + .map(SdkBody::from)) + }) + } +} From 77a4ac860d72e1c968bd5b21216e901fb4fcb85c Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Thu, 25 May 2023 23:18:49 +0200 Subject: [PATCH 18/72] chore: use wasm crate in aws config --- aws/rust-runtime/aws-config/Cargo.toml | 4 +- aws/rust-runtime/aws-config/src/connector.rs | 4 +- aws/rust-runtime/aws-config/src/lib.rs | 2 - .../aws-config/src/wasi_adapter/mod.rs | 47 ------------------- 4 files changed, 5 insertions(+), 52 deletions(-) delete mode 100644 aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index b91720d128..e906863ea4 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -47,8 +47,8 @@ ring = { version = "0.16", optional = true } hex = { version = "0.4.3", optional = true } zeroize = { version = "1", optional = true } -[target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] -wasi-preview2-prototype = { version = "0.0.1", features = ["http-client"] } +[target.'cfg(target_family = "wasm")'.dependencies] +aws-smithy-wasm = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-wasm" } [dev-dependencies] futures-util = { version = "0.3.16", default-features = false } diff --git a/aws/rust-runtime/aws-config/src/connector.rs b/aws/rust-runtime/aws-config/src/connector.rs index 9c49d94c64..aba5913f63 100644 --- a/aws/rust-runtime/aws-config/src/connector.rs +++ b/aws/rust-runtime/aws-config/src/connector.rs @@ -63,7 +63,9 @@ pub fn default_connector( _settings: &ConnectorSettings, _sleep: Option>, ) -> Option { - Some(DynConnector::new(crate::wasi_adapter::Adapter::default())) + Some(DynConnector::new( + aws_smithy_wasm::wasi_adapter::Adapter::default(), + )) } /// Given `ConnectorSettings` and an `AsyncSleep`, create a `DynConnector` from defaults depending on what cargo features are activated. diff --git a/aws/rust-runtime/aws-config/src/lib.rs b/aws/rust-runtime/aws-config/src/lib.rs index 169a9410ef..2d237a5b8f 100644 --- a/aws/rust-runtime/aws-config/src/lib.rs +++ b/aws/rust-runtime/aws-config/src/lib.rs @@ -127,8 +127,6 @@ pub mod sso; pub(crate) mod standard_property; pub mod sts; pub mod timeout; -#[cfg(all(target_family = "wasm", target_os = "wasi"))] -pub mod wasi_adapter; pub mod web_identity_token; /// Create an environment loader for AWS Configuration diff --git a/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs b/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs deleted file mode 100644 index ad426b98c0..0000000000 --- a/aws/rust-runtime/aws-config/src/wasi_adapter/mod.rs +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -//! HTTP Service Adapter used in WASI environment - -use aws_smithy_http::body::SdkBody; -use aws_smithy_http::result::ConnectorError; -use bytes::Bytes; -use http::{Request, Response}; -use std::task::{Context, Poll}; -use tower::Service; -use wasi_preview2_prototype::http_client::DefaultClient; - -#[derive(Default, Debug, Clone)] -pub(crate) struct Adapter {} - -impl Service> for Adapter { - type Response = Response; - - type Error = ConnectorError; - - #[allow(clippy::type_complexity)] - type Future = std::pin::Pin< - Box> + Send + 'static>, - >; - - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn call(&mut self, req: Request) -> Self::Future { - println!("Adapter: sending request..."); - let client = DefaultClient::new(None); - // Right now only synchronous calls can be made through WASI - let fut = client.handle(req.map(|body| match body.bytes() { - Some(value) => Bytes::copy_from_slice(value), - None => Bytes::new(), - })); - Box::pin(async move { - Ok(fut - .map_err(|err| ConnectorError::other(err.into(), None))? - .map(SdkBody::from)) - }) - } -} From ee489fee9332cbf544f692cf4453e27059139c24 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Thu, 25 May 2023 23:26:05 +0200 Subject: [PATCH 19/72] chore: re-enable timeout and retry config --- aws/sdk/integration-tests/webassembly/src/default_config.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index 0580c3b4b3..af098e5db1 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -3,15 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ -use aws_smithy_types::{retry::RetryConfig, timeout::TimeoutConfig}; use aws_types::region::Region; use std::future::Future; pub(crate) fn get_default_config() -> impl Future { aws_config::from_env() .region(Region::from_static("us-west-2")) - .timeout_config(TimeoutConfig::disabled()) - .retry_config(RetryConfig::disabled()) .load() } From ad014619b61a7fd6629dbb98ac28dc44c9c24409 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Fri, 26 May 2023 00:13:39 +0200 Subject: [PATCH 20/72] chore(smithy-async): compile to wasi with rt-tokio feature chore: fix formatting --- aws/rust-runtime/aws-config/Cargo.toml | 6 +- .../aws-config/external-types.toml | 1 + aws/rust-runtime/aws-config/src/connector.rs | 3 +- .../integration-tests/webassembly/Cargo.toml | 7 +- .../webassembly/src/default_config.rs | 5 +- buildSrc/src/main/kotlin/CrateSet.kt | 76 +++++++++---------- rust-runtime/aws-smithy-async/src/rt/sleep.rs | 37 ++++++--- rust-runtime/aws-smithy-wasm/Cargo.toml | 4 +- 8 files changed, 72 insertions(+), 67 deletions(-) diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index e906863ea4..2b9998e2e9 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -16,6 +16,8 @@ allow-compilation = [] # our tests use `cargo test --all-features` and native-tl rt-tokio = ["aws-smithy-async/rt-tokio", "tokio/rt"] credentials-sso = ["dep:aws-sdk-sso", "dep:ring", "dep:hex", "dep:zeroize"] +wasm = ["dep:aws-smithy-wasm"] + default = ["client-hyper", "rustls", "rt-tokio", "credentials-sso"] [dependencies] @@ -47,8 +49,8 @@ ring = { version = "0.16", optional = true } hex = { version = "0.4.3", optional = true } zeroize = { version = "1", optional = true } -[target.'cfg(target_family = "wasm")'.dependencies] -aws-smithy-wasm = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-wasm" } +# used for WebAssembly target family +aws-smithy-wasm = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-wasm", optional = true } [dev-dependencies] futures-util = { version = "0.3.16", default-features = false } diff --git a/aws/rust-runtime/aws-config/external-types.toml b/aws/rust-runtime/aws-config/external-types.toml index 7e0257f46d..76ae158d82 100644 --- a/aws/rust-runtime/aws-config/external-types.toml +++ b/aws/rust-runtime/aws-config/external-types.toml @@ -24,6 +24,7 @@ allowed_external_types = [ "aws_smithy_types::timeout::OperationTimeoutConfig", "aws_smithy_types::timeout::TimeoutConfig", "aws_smithy_types::timeout::TimeoutConfigBuilder", + "aws_smithy_wasm::wasi_adapter::*", "aws_types::*", "http::response::Response", "http::uri::Uri", diff --git a/aws/rust-runtime/aws-config/src/connector.rs b/aws/rust-runtime/aws-config/src/connector.rs index aba5913f63..c4a257e939 100644 --- a/aws/rust-runtime/aws-config/src/connector.rs +++ b/aws/rust-runtime/aws-config/src/connector.rs @@ -55,6 +55,7 @@ pub fn default_connector( /// Given `ConnectorSettings` and an `AsyncSleep`, create a `DynConnector` from defaults depending on what cargo features are activated. #[cfg(all( + feature = "wasm", target_family = "wasm", target_os = "wasi", not(any(feature = "rustls", feature = "native-tls")) @@ -70,7 +71,7 @@ pub fn default_connector( /// Given `ConnectorSettings` and an `AsyncSleep`, create a `DynConnector` from defaults depending on what cargo features are activated. #[cfg(not(any( - all(target_family = "wasm", target_os = "wasi"), + all(feature = "wasm", target_family = "wasm", target_os = "wasi"), feature = "rustls", feature = "native-tls" )))] diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index c0d2638af5..fcd563d683 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -19,13 +19,8 @@ crate-type = ["cdylib"] [dependencies] anyhow = "1.0" -aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false } -aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["hardcoded-credentials"] } +aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false, features = ["wasm"] } aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3", default-features = false } -aws-smithy-client = { path = "../../build/aws-sdk/sdk/aws-smithy-client", default-features = false } -aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" } -aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types" } -aws-types = { path = "../../build/aws-sdk/sdk/aws-types" } http = "0.2.8" tokio = { version = "1.24.2", features = ["macros", "rt"] } tower = "0.4.13" diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index af098e5db1..31b7825739 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -3,13 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ -use aws_types::region::Region; use std::future::Future; pub(crate) fn get_default_config() -> impl Future { - aws_config::from_env() - .region(Region::from_static("us-west-2")) - .load() + aws_config::from_env().region("us-west-2").load() } #[tokio::test] diff --git a/buildSrc/src/main/kotlin/CrateSet.kt b/buildSrc/src/main/kotlin/CrateSet.kt index 8232ad7230..d34b920891 100644 --- a/buildSrc/src/main/kotlin/CrateSet.kt +++ b/buildSrc/src/main/kotlin/CrateSet.kt @@ -3,50 +3,46 @@ * SPDX-License-Identifier: Apache-2.0 */ -object CrateSet { - val AWS_SDK_RUNTIME = - listOf( - "aws-config", - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-hyper", - "aws-runtime", - "aws-runtime-api", - "aws-sig-auth", - "aws-sigv4", - "aws-types", - ) + object CrateSet { + val AWS_SDK_RUNTIME = listOf( + "aws-config", + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-hyper", + "aws-runtime", + "aws-runtime-api", + "aws-sig-auth", + "aws-sigv4", + "aws-types", + ) - private val SMITHY_RUNTIME_COMMON = - listOf( - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-client", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-http-auth", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-protocol-test", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-types-convert", - "aws-smithy-wasm", - "aws-smithy-xml", - ) + private val SMITHY_RUNTIME_COMMON = listOf( + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-auth", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-protocol-test", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-types-convert", + "aws-smithy-wasm", + "aws-smithy-xml", + ) val AWS_SDK_SMITHY_RUNTIME = SMITHY_RUNTIME_COMMON - val SERVER_SMITHY_RUNTIME = - SMITHY_RUNTIME_COMMON + - listOf( - "aws-smithy-http-server", - "aws-smithy-http-server-python", - "aws-smithy-http-server-typescript", - ) + val SERVER_SMITHY_RUNTIME = SMITHY_RUNTIME_COMMON + listOf( + "aws-smithy-http-server", + "aws-smithy-http-server-python", + "aws-smithy-http-server-typescript", + ) val ENTIRE_SMITHY_RUNTIME = (AWS_SDK_SMITHY_RUNTIME + SERVER_SMITHY_RUNTIME).toSortedSet() } diff --git a/rust-runtime/aws-smithy-async/src/rt/sleep.rs b/rust-runtime/aws-smithy-async/src/rt/sleep.rs index af06f3b3fe..1bc0c51cc0 100644 --- a/rust-runtime/aws-smithy-async/src/rt/sleep.rs +++ b/rust-runtime/aws-smithy-async/src/rt/sleep.rs @@ -39,19 +39,22 @@ where } } -#[cfg(feature = "rt-tokio")] +#[cfg(all( + feature = "rt-tokio", + not(all(target_family = "wasm", target_os = "wasi")) +))] /// Returns a default sleep implementation based on the features enabled pub fn default_async_sleep() -> Option> { Some(sleep_tokio()) } -#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +#[cfg(all(target_family = "wasm", target_os = "wasi"))] /// Returns a default sleep implementation based on the features enabled pub fn default_async_sleep() -> Option> { Some(sleep_wasi()) } -#[cfg(all(any(target_family = "wasm", target_os = "wasi"), feature = "rt-tokio"))] +#[cfg(not(any(all(target_family = "wasm", target_os = "wasi"), feature = "rt-tokio")))] /// Returns a default sleep implementation based on the features enabled pub fn default_async_sleep() -> Option> { None @@ -87,11 +90,17 @@ impl Future for Sleep { /// Implementation of [`AsyncSleep`] for Tokio. #[non_exhaustive] -#[cfg(feature = "rt-tokio")] +#[cfg(all( + feature = "rt-tokio", + not(all(target_family = "wasm", target_os = "wasi")) +))] #[derive(Debug, Default)] pub struct TokioSleep; -#[cfg(feature = "rt-tokio")] +#[cfg(all( + feature = "rt-tokio", + not(all(target_family = "wasm", target_os = "wasi")) +))] impl TokioSleep { /// Create a new [`AsyncSleep`] implementation using the Tokio hashed wheel sleep implementation pub fn new() -> TokioSleep { @@ -99,25 +108,31 @@ impl TokioSleep { } } -#[cfg(feature = "rt-tokio")] +#[cfg(all( + feature = "rt-tokio", + not(all(target_family = "wasm", target_os = "wasi")) +))] impl AsyncSleep for TokioSleep { fn sleep(&self, duration: Duration) -> Sleep { Sleep::new(tokio::time::sleep(duration)) } } -#[cfg(feature = "rt-tokio")] +#[cfg(all( + feature = "rt-tokio", + not(all(target_family = "wasm", target_os = "wasi")) +))] fn sleep_tokio() -> Arc { Arc::new(TokioSleep::new()) } /// Implementation of [`AsyncSleep`] for WASI. #[non_exhaustive] -#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +#[cfg(all(target_family = "wasm", target_os = "wasi"))] #[derive(Debug, Default)] pub struct WasiSleep; -#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +#[cfg(all(target_family = "wasm", target_os = "wasi"))] impl WasiSleep { /// Create a new [`AsyncSleep`] implementation using the WASI sleep implementation pub fn new() -> WasiSleep { @@ -125,7 +140,7 @@ impl WasiSleep { } } -#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +#[cfg(all(target_family = "wasm", target_os = "wasi"))] impl AsyncSleep for WasiSleep { fn sleep(&self, duration: std::time::Duration) -> Sleep { Sleep::new(Box::pin(async move { @@ -134,7 +149,7 @@ impl AsyncSleep for WasiSleep { } } -#[cfg(all(target_family = "wasm", target_os = "wasi", not(feature = "rt-tokio")))] +#[cfg(all(target_family = "wasm", target_os = "wasi"))] fn sleep_wasi() -> Arc { Arc::new(WasiSleep::new()) } diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index de6cb50460..4064204cce 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -10,10 +10,8 @@ edition = "2021" license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" -[dependencies] -aws-smithy-http = { path = "../aws-smithy-http" } - [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +aws-smithy-http = { path = "../aws-smithy-http" } bytes = "1" http = "0.2.4" tower = { version = "0.4.8" } From 0e93edb0a5415ccb5e70b3277ba251192a507950 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Sat, 19 Aug 2023 15:15:46 +0200 Subject: [PATCH 21/72] chore: update wasi-preview2-prototype to v0.0.2 --- rust-runtime/aws-smithy-wasm/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index 4064204cce..a5b7649112 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -15,7 +15,7 @@ aws-smithy-http = { path = "../aws-smithy-http" } bytes = "1" http = "0.2.4" tower = { version = "0.4.8" } -wasi-preview2-prototype = { version = "0.0.1", features = ["http-client"] } +wasi-preview2-prototype = { version = "0.0.2", features = ["http-client"] } [package.metadata.docs.rs] all-features = true From a5e858fda6b6eb1f049cb905295a008473d78f55 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Sun, 20 Aug 2023 00:57:31 +0200 Subject: [PATCH 22/72] chore: run list objects instead --- .../integration-tests/webassembly/Cargo.toml | 5 ++ .../webassembly/src/default_config.rs | 23 +++++++-- .../integration-tests/webassembly/src/lib.rs | 4 +- .../webassembly/src/list_buckets.rs | 23 --------- .../webassembly/src/list_objects.rs | 47 +++++++++++++++++++ 5 files changed, 74 insertions(+), 28 deletions(-) delete mode 100644 aws/sdk/integration-tests/webassembly/src/list_buckets.rs create mode 100644 aws/sdk/integration-tests/webassembly/src/list_objects.rs diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index fcd563d683..df5aa827ca 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -20,7 +20,12 @@ crate-type = ["cdylib"] [dependencies] anyhow = "1.0" aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false, features = ["wasm"] } +aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types" } aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3", default-features = false } +aws-sig-auth = { path = "../../build/aws-sdk/sdk/aws-sig-auth" } +aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async", default-features = false } +aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" } +aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types" } http = "0.2.8" tokio = { version = "1.24.2", features = ["macros", "rt"] } tower = "0.4.13" diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index 31b7825739..dbd1f8b3ba 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -3,10 +3,27 @@ * SPDX-License-Identifier: Apache-2.0 */ -use std::future::Future; +use aws_config::environment::EnvironmentVariableCredentialsProvider; +use aws_credential_types::cache::CredentialsCache; +use aws_smithy_async::rt::sleep::WasiSleep; +use aws_smithy_types::{retry::RetryConfig, timeout::TimeoutConfig}; -pub(crate) fn get_default_config() -> impl Future { - aws_config::from_env().region("us-west-2").load() +fn credentials_cache() -> CredentialsCache { + CredentialsCache::lazy_builder() + .sleep(std::sync::Arc::new(WasiSleep::default())) + .into_credentials_cache() +} + +pub(crate) async fn get_default_config() -> aws_config::SdkConfig { + aws_config::from_env() + .region("us-west-2") + .timeout_config(TimeoutConfig::disabled()) + .retry_config(RetryConfig::disabled()) + .sleep_impl(WasiSleep::default()) + .credentials_cache(credentials_cache()) + .credentials_provider(EnvironmentVariableCredentialsProvider::default()) + .load() + .await } #[tokio::test] diff --git a/aws/sdk/integration-tests/webassembly/src/lib.rs b/aws/sdk/integration-tests/webassembly/src/lib.rs index 089279eb16..89b0440d69 100644 --- a/aws/sdk/integration-tests/webassembly/src/lib.rs +++ b/aws/sdk/integration-tests/webassembly/src/lib.rs @@ -4,7 +4,7 @@ */ mod default_config; -mod list_buckets; +mod list_objects; #[no_mangle] pub extern "C" fn run() { @@ -16,7 +16,7 @@ pub extern "C" fn run() { .build() .unwrap(); rt.block_on(async move { - let result = crate::list_buckets::s3_list_buckets().await; + let result = crate::list_objects::s3_list_objects().await; println!("result: {:?}", result); }); } diff --git a/aws/sdk/integration-tests/webassembly/src/list_buckets.rs b/aws/sdk/integration-tests/webassembly/src/list_buckets.rs deleted file mode 100644 index 2c5c7e7945..0000000000 --- a/aws/sdk/integration-tests/webassembly/src/list_buckets.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -use aws_sdk_s3::operation::list_buckets::ListBucketsOutput; - -pub async fn s3_list_buckets() -> ListBucketsOutput { - use aws_sdk_s3::Client; - - use crate::default_config::get_default_config; - - let shared_config = get_default_config().await; - let client = Client::new(&shared_config); - client.list_buckets().send().await.unwrap() -} - -#[tokio::test] -pub async fn test_s3_list_buckets() { - let result = s3_list_buckets().await; - let buckets = result.buckets().unwrap(); - assert!(buckets.len() > 0); -} diff --git a/aws/sdk/integration-tests/webassembly/src/list_objects.rs b/aws/sdk/integration-tests/webassembly/src/list_objects.rs new file mode 100644 index 0000000000..c22792f14f --- /dev/null +++ b/aws/sdk/integration-tests/webassembly/src/list_objects.rs @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output; + +pub async fn s3_list_objects() -> ListObjectsV2Output { + use aws_sdk_s3::Client; + + use crate::default_config::get_default_config; + + let shared_config = get_default_config().await; + let client = Client::new(&shared_config); + let mut operation = client + .list_objects_v2() + .bucket("smithsonian-open-access") + .delimiter("/") + .prefix("media/") + .max_keys(5) + .customize() + .await + .unwrap(); + operation.map_operation(make_unsigned).unwrap().send().await.unwrap() +} + +fn make_unsigned( + mut operation: aws_smithy_http::operation::Operation, +) -> Result, std::convert::Infallible> { + { + let mut props = operation.properties_mut(); + let mut signing_config = props + .get_mut::() + .expect("has signing_config"); + signing_config.signing_requirements = aws_sig_auth::signer::SigningRequirements::Disabled; + } + + Ok(operation) +} + +#[tokio::test] +pub async fn test_s3_list_objects() { + let result = s3_list_objects().await; + println!("result: {:?}", result); + let objects = result.contents().unwrap(); + assert!(objects.len() > 1); +} From 53e84df268b310868a76ca8d56335e5b31c3cfc1 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 02:58:47 +0200 Subject: [PATCH 23/72] chore: fix misconfigured imports --- aws/rust-runtime/aws-config/src/connector.rs | 4 ++-- rust-runtime/aws-smithy-async/src/rt/sleep.rs | 4 ++-- .../aws-smithy-wasm/src/wasi_adapter.rs | 19 +++++++++++++------ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/aws/rust-runtime/aws-config/src/connector.rs b/aws/rust-runtime/aws-config/src/connector.rs index 27040ca860..5a0ebf1176 100644 --- a/aws/rust-runtime/aws-config/src/connector.rs +++ b/aws/rust-runtime/aws-config/src/connector.rs @@ -30,8 +30,8 @@ compile_error!("Feature native-tls has been removed. For upgrade instructions, s not(any(feature = "rustls", feature = "native-tls")) ))] pub fn default_connector( - _settings: &ConnectorSettings, - _sleep: Option>, + _settings: &aws_smithy_client::http_connector::ConnectorSettings, + _sleep: Option, ) -> Option { Some(DynConnector::new( aws_smithy_wasm::wasi_adapter::Adapter::default(), diff --git a/rust-runtime/aws-smithy-async/src/rt/sleep.rs b/rust-runtime/aws-smithy-async/src/rt/sleep.rs index c1f16bee04..f3f2847714 100644 --- a/rust-runtime/aws-smithy-async/src/rt/sleep.rs +++ b/rust-runtime/aws-smithy-async/src/rt/sleep.rs @@ -79,8 +79,8 @@ pub fn default_async_sleep() -> Option { #[cfg(all(target_family = "wasm", target_os = "wasi"))] /// Returns a default sleep implementation based on the features enabled -pub fn default_async_sleep() -> Option> { - Some(sleep_wasi()) +pub fn default_async_sleep() -> Option { + Some(SharedAsyncSleep::from(sleep_wasi())) } #[cfg(not(any(all(target_family = "wasm", target_os = "wasi"), feature = "rt-tokio")))] diff --git a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs index afb64feab7..8f084d2dfd 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs @@ -5,8 +5,7 @@ //! HTTP WASI Adapter -use aws_smithy_http::body::SdkBody; -use aws_smithy_http::result::ConnectorError; +use aws_smithy_http::{body::SdkBody, byte_stream::ByteStream, result::ConnectorError}; use bytes::Bytes; use http::{Request, Response}; use std::task::{Context, Poll}; @@ -31,7 +30,7 @@ impl Service> for Adapter { Poll::Ready(Ok(())) } - fn call(&mut self, req: Request) -> Self::Future { + fn call(&mut self, req: http::Request) -> Self::Future { println!("Adapter: sending request..."); let client = DefaultClient::new(None); // Right now only synchronous calls can be made through WASI @@ -40,9 +39,17 @@ impl Service> for Adapter { None => Bytes::new(), })); Box::pin(async move { - Ok(fut - .map_err(|err| ConnectorError::other(err.into(), None))? - .map(SdkBody::from)) + let res = fut + .map_err(|err| ConnectorError::other(err.into(), None)) + .expect("response from adapter"); + + let (parts, body) = res.into_parts(); + let loaded_body = if body.is_empty() { + SdkBody::empty() + } else { + SdkBody::from(body) + }; + Ok(http::Response::from_parts(parts, loaded_body)) }) } } From ea29d562edf2fa12517073bd4f9699b83f4dfb5d Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 03:01:02 +0200 Subject: [PATCH 24/72] chore: simplify wasm tests --- .../webassembly/.cargo/config.toml | 6 ++++ .../integration-tests/webassembly/Cargo.toml | 20 +++++++---- .../webassembly/src/default_config.rs | 17 ++------- .../integration-tests/webassembly/src/lib.rs | 36 ++++++++++++------- .../webassembly/src/list_objects.rs | 25 +++---------- ...check-aws-sdk-standalone-integration-tests | 5 ++- 6 files changed, 52 insertions(+), 57 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/.cargo/config.toml b/aws/sdk/integration-tests/webassembly/.cargo/config.toml index e6e321b297..7897ba239c 100644 --- a/aws/sdk/integration-tests/webassembly/.cargo/config.toml +++ b/aws/sdk/integration-tests/webassembly/.cargo/config.toml @@ -3,6 +3,12 @@ target = "wasm32-wasi" [target.wasm32-wasi] rustflags = ["-C", "opt-level=1"] +runner = [ + "wasmtime", + "run", + "--preview2", + "--wasi-modules=experimental-wasi-http", +] # TODO(Rust 1.70): The sparse registry config can be removed when upgrading to Rust 1.70 [registries.crates-io] diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index df5aa827ca..01ab44577c 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -2,7 +2,9 @@ [package] name = "webassembly" version = "0.1.0" -authors = ["Eduardo Rodrigues <16357187+eduardomourar@users.noreply.github.com>"] +authors = [ + "Eduardo Rodrigues <16357187+eduardomourar@users.noreply.github.com>", +] description = """ These tests ensure that things will fail (or not fail) as expected when target is set to wasm32-wasi for all SDK and runtime crates. @@ -12,20 +14,24 @@ license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" publish = false -[lib] -crate-type = ["cdylib"] - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] anyhow = "1.0" -aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false, features = ["wasm"] } +aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false, features = [ + "wasm", +] } aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types" } aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3", default-features = false } aws-sig-auth = { path = "../../build/aws-sdk/sdk/aws-sig-auth" } -aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async", default-features = false } +aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async", default-features = false, features = [ + "rt-tokio", +] } aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" } +aws-smithy-runtime-api = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime-api" } aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types" } +futures = "0.3.28" http = "0.2.8" -tokio = { version = "1.24.2", features = ["macros", "rt"] } +tokio = { version = "1.32.0", features = ["macros", "rt"] } tower = "0.4.13" +wit-bindgen = { version = "0.11.0", features = ["macros", "realloc"] } diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index dbd1f8b3ba..d84195c959 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -3,25 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -use aws_config::environment::EnvironmentVariableCredentialsProvider; -use aws_credential_types::cache::CredentialsCache; -use aws_smithy_async::rt::sleep::WasiSleep; use aws_smithy_types::{retry::RetryConfig, timeout::TimeoutConfig}; -fn credentials_cache() -> CredentialsCache { - CredentialsCache::lazy_builder() - .sleep(std::sync::Arc::new(WasiSleep::default())) - .into_credentials_cache() -} - pub(crate) async fn get_default_config() -> aws_config::SdkConfig { aws_config::from_env() - .region("us-west-2") + .region("us-east-2") .timeout_config(TimeoutConfig::disabled()) .retry_config(RetryConfig::disabled()) - .sleep_impl(WasiSleep::default()) - .credentials_cache(credentials_cache()) - .credentials_provider(EnvironmentVariableCredentialsProvider::default()) + .no_credentials() .load() .await } @@ -30,5 +19,5 @@ pub(crate) async fn get_default_config() -> aws_config::SdkConfig { pub async fn test_default_config() { let shared_config = get_default_config().await; let client = aws_sdk_s3::Client::new(&shared_config); - assert_eq!(client.conf().region().unwrap().to_string(), "us-west-2") + assert_eq!(client.conf().region().unwrap().to_string(), "us-east-2") } diff --git a/aws/sdk/integration-tests/webassembly/src/lib.rs b/aws/sdk/integration-tests/webassembly/src/lib.rs index 89b0440d69..801a5fbd3f 100644 --- a/aws/sdk/integration-tests/webassembly/src/lib.rs +++ b/aws/sdk/integration-tests/webassembly/src/lib.rs @@ -6,17 +6,27 @@ mod default_config; mod list_objects; -#[no_mangle] -pub extern "C" fn run() { - std::panic::set_hook(Box::new(move |panic_info| { - println!("Internal unhandled panic:\n{:?}!", panic_info); - std::process::exit(1); - })); - let rt = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); - rt.block_on(async move { - let result = crate::list_objects::s3_list_objects().await; - println!("result: {:?}", result); - }); +wit_bindgen::generate!({ + inline: " + package aws:component + + interface run { + run: func() -> result + } + + world main { + export run + } + ", + exports: { + "aws:component/run": Component + } +}); + +struct Component; + +impl exports::aws::component::run::Guest for Component { + fn run() -> Result<(), ()> { + Ok(()) + } } diff --git a/aws/sdk/integration-tests/webassembly/src/list_objects.rs b/aws/sdk/integration-tests/webassembly/src/list_objects.rs index c22792f14f..7d3747f990 100644 --- a/aws/sdk/integration-tests/webassembly/src/list_objects.rs +++ b/aws/sdk/integration-tests/webassembly/src/list_objects.rs @@ -5,37 +5,22 @@ use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output; -pub async fn s3_list_objects() -> ListObjectsV2Output { +async fn s3_list_objects() -> ListObjectsV2Output { use aws_sdk_s3::Client; - use crate::default_config::get_default_config; let shared_config = get_default_config().await; let client = Client::new(&shared_config); - let mut operation = client + let operation = client .list_objects_v2() - .bucket("smithsonian-open-access") + .bucket("nara-national-archives-catalog") .delimiter("/") - .prefix("media/") + .prefix("authority-records/organization/") .max_keys(5) .customize() .await .unwrap(); - operation.map_operation(make_unsigned).unwrap().send().await.unwrap() -} - -fn make_unsigned( - mut operation: aws_smithy_http::operation::Operation, -) -> Result, std::convert::Infallible> { - { - let mut props = operation.properties_mut(); - let mut signing_config = props - .get_mut::() - .expect("has signing_config"); - signing_config.signing_requirements = aws_sig_auth::signer::SigningRequirements::Disabled; - } - - Ok(operation) + operation.send().await.unwrap() } #[tokio::test] diff --git a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests index 0e6d9fb521..d5d1feb6f0 100755 --- a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests +++ b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests @@ -42,7 +42,6 @@ cargo check --tests # Running WebAssembly (WASI) specific integration tests pushd "${tmp_dir}/aws/sdk/integration-tests/webassembly" &>/dev/null -cargo wasi build -env_credentials="--env AWS_ACCESS_KEY_ID='' --env AWS_SECRET_ACCESS_KEY='' --env AWS_SESSION_TOKEN=''" -CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime --wasm-features=component-model --wasi-modules=experimental-wasi-http ${env_credentials}" cargo wasi test -- --nocapture +cargo build +cargo test -- --nocapture popd From a16bcc1b659240231d98c06d41b87f4d31fd90e2 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 10:30:50 +0200 Subject: [PATCH 25/72] chore: remove cargo-wasi references --- tools/ci-build/Dockerfile | 5 ----- tools/ci-scripts/check-aws-config | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 0ab34cdd61..d6308f58bb 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -125,10 +125,6 @@ ARG cargo_wasmtime_version=9.0.1 ARG rust_nightly_version RUN cargo +${rust_nightly_version} -Z sparse-registry install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} -FROM install_rust AS cargo_wasi -ARG cargo_wasi_version=0.1.27 -RUN cargo install cargo-wasi --locked --version ${cargo_wasi_version} - FROM install_rust AS cargo_semver_checks ARG cargo_semver_checks_version=0.20.0 ARG rust_nightly_version @@ -180,7 +176,6 @@ COPY --chown=build:build --from=cargo_check_external_types /opt/cargo/bin/cargo- COPY --chown=build:build --from=maturin /opt/cargo/bin/maturin /opt/cargo/bin/maturin COPY --chown=build:build --from=wasm_pack /opt/cargo/bin/wasm-pack /opt/cargo/bin/wasm-pack COPY --chown=build:build --from=wasmtime /opt/cargo/bin/wasmtime /opt/cargo/bin/wasmtime -COPY --chown=build:build --from=cargo_wasi /opt/cargo/bin/cargo-wasi /opt/cargo/bin/cargo-wasi COPY --chown=build:build --from=install_rust /opt/rustup /opt/rustup COPY --chown=build:build --from=cargo_semver_checks /opt/cargo/bin/cargo-semver-checks /opt/cargo/bin/cargo-semver-checks COPY --chown=build:build --from=cargo_mdbook /opt/cargo/bin/mdbook /opt/cargo/bin/mdbook diff --git a/tools/ci-scripts/check-aws-config b/tools/ci-scripts/check-aws-config index 5c6d910fe2..3957438c48 100755 --- a/tools/ci-scripts/check-aws-config +++ b/tools/ci-scripts/check-aws-config @@ -45,6 +45,6 @@ cargo check --target wasm32-wasi --no-default-features # TODO(https://github.com/awslabs/smithy-rs/issues/2499): Uncomment the following once aws-config tests compile for WASM # echo "${C_YELLOW}## Testing the wasm32-unknown-unknown and wasm32-wasi targets${C_RESET}" # wasm-pack test --node -- --no-default-features -# cargo wasi test --no-default-features +# cargo test --target wasm32-wasi --no-default-features popd &>/dev/null From 7d8638cdb1a36c6b7da53a024645e55dee05422a Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 10:32:03 +0200 Subject: [PATCH 26/72] chore: remove unnecessary sparse registry config --- .../integration-tests/webassembly/.cargo/config.toml | 4 ---- tools/.cargo/config.toml | 4 ---- tools/ci-build/Dockerfile | 10 ++++------ 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/.cargo/config.toml b/aws/sdk/integration-tests/webassembly/.cargo/config.toml index 7897ba239c..c454cff49f 100644 --- a/aws/sdk/integration-tests/webassembly/.cargo/config.toml +++ b/aws/sdk/integration-tests/webassembly/.cargo/config.toml @@ -9,7 +9,3 @@ runner = [ "--preview2", "--wasi-modules=experimental-wasi-http", ] - -# TODO(Rust 1.70): The sparse registry config can be removed when upgrading to Rust 1.70 -[registries.crates-io] -protocol = "sparse" diff --git a/tools/.cargo/config.toml b/tools/.cargo/config.toml index 6636645498..2bf2a2e3e4 100644 --- a/tools/.cargo/config.toml +++ b/tools/.cargo/config.toml @@ -2,7 +2,3 @@ # Tools shouldn't share `target` with the rest of the project to # avoid thrash from differing compiler flags target-dir = "target" - -# TODO(Rust 1.70): The sparse registry config can be removed when upgrading to Rust 1.70 -[registries.crates-io] -protocol = "sparse" diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index d6308f58bb..646cf83903 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -123,22 +123,22 @@ RUN cargo install wasm-pack --locked --version ${wasm_pack_version} FROM install_rust AS wasmtime ARG cargo_wasmtime_version=9.0.1 ARG rust_nightly_version -RUN cargo +${rust_nightly_version} -Z sparse-registry install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} +RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} FROM install_rust AS cargo_semver_checks ARG cargo_semver_checks_version=0.20.0 ARG rust_nightly_version -RUN cargo +${rust_nightly_version} -Z sparse-registry install cargo-semver-checks --locked --version ${cargo_semver_checks_version} +RUN cargo +${rust_nightly_version} install cargo-semver-checks --locked --version ${cargo_semver_checks_version} FROM install_rust AS cargo_mdbook ARG cargo_mdbook_version=0.4.30 ARG rust_nightly_version -RUN cargo +${rust_nightly_version} -Z sparse-registry install mdbook --locked --version ${cargo_mdbook_version} +RUN cargo +${rust_nightly_version} install mdbook --locked --version ${cargo_mdbook_version} FROM install_rust AS cargo_mdbook_mermaid ARG cargo_mdbook_mermaid_version=0.12.6 ARG rust_nightly_version -RUN cargo +${rust_nightly_version} -Z sparse-registry install mdbook-mermaid --locked --version ${cargo_mdbook_mermaid_version} +RUN cargo +${rust_nightly_version} install mdbook-mermaid --locked --version ${cargo_mdbook_mermaid_version} # # Final image @@ -182,7 +182,6 @@ COPY --chown=build:build --from=cargo_mdbook /opt/cargo/bin/mdbook /opt/cargo/bi COPY --chown=build:build --from=cargo_mdbook_mermaid /opt/cargo/bin/mdbook-mermaid /opt/cargo/bin/mdbook-mermaid COPY --chown=build:build --from=musl_toolchain /usr/local/musl/ /usr/local/musl/ ENV PATH=$PATH:/usr/local/musl/bin/ -# TODO(Rust 1.70): The sparse registry config (`CARGO_REGISTRIES_CRATES_IO_PROTOCOL`) can be removed when upgrading to Rust 1.70 ENV PATH=/opt/cargo/bin:$PATH \ CARGO_HOME=/opt/cargo \ RUSTUP_HOME=/opt/rustup \ @@ -190,7 +189,6 @@ ENV PATH=/opt/cargo/bin:$PATH \ GRADLE_USER_HOME=/home/build/.gradle \ RUST_STABLE_VERSION=${rust_stable_version} \ RUST_NIGHTLY_VERSION=${rust_nightly_version} \ - CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse \ CARGO_INCREMENTAL=0 \ RUSTDOCFLAGS="-D warnings" \ RUSTFLAGS="-D warnings" \ From b9028f309413f7efb2eb4467cadf9b868e5508d3 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 10:41:04 +0200 Subject: [PATCH 27/72] chore: use temporarily wasmtime from git --- tools/ci-build/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 646cf83903..3742e8ed8d 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -121,9 +121,10 @@ ARG wasm_pack_version=0.11.0 RUN cargo install wasm-pack --locked --version ${wasm_pack_version} FROM install_rust AS wasmtime -ARG cargo_wasmtime_version=9.0.1 +# should be changed to 13.0.0 when released +ARG cargo_wasmtime_version=d1d8893d2e66409662bad7f9ec65050a218be681 ARG rust_nightly_version -RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} +RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-model" --locked --git https://github.com/bytecodealliance/wasmtime --rev ${cargo_wasmtime_version} FROM install_rust AS cargo_semver_checks ARG cargo_semver_checks_version=0.20.0 From 69f50fe3453de138567a1c7bebdc7c01456171ee Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 10:59:18 +0200 Subject: [PATCH 28/72] chore: remove wasi sleep --- rust-runtime/aws-smithy-async/src/rt/sleep.rs | 61 ++----------------- 1 file changed, 6 insertions(+), 55 deletions(-) diff --git a/rust-runtime/aws-smithy-async/src/rt/sleep.rs b/rust-runtime/aws-smithy-async/src/rt/sleep.rs index f3f2847714..076db95c97 100644 --- a/rust-runtime/aws-smithy-async/src/rt/sleep.rs +++ b/rust-runtime/aws-smithy-async/src/rt/sleep.rs @@ -68,22 +68,13 @@ impl AsyncSleep for SharedAsyncSleep { } } -#[cfg(all( - feature = "rt-tokio", - not(all(target_family = "wasm", target_os = "wasi")) -))] +#[cfg(feature = "rt-tokio")] /// Returns a default sleep implementation based on the features enabled pub fn default_async_sleep() -> Option { Some(SharedAsyncSleep::from(sleep_tokio())) } -#[cfg(all(target_family = "wasm", target_os = "wasi"))] -/// Returns a default sleep implementation based on the features enabled -pub fn default_async_sleep() -> Option { - Some(SharedAsyncSleep::from(sleep_wasi())) -} - -#[cfg(not(any(all(target_family = "wasm", target_os = "wasi"), feature = "rt-tokio")))] +#[cfg(not(feature = "rt-tokio"))] /// Returns a default sleep implementation based on the features enabled pub fn default_async_sleep() -> Option { None @@ -119,17 +110,11 @@ impl Future for Sleep { /// Implementation of [`AsyncSleep`] for Tokio. #[non_exhaustive] -#[cfg(all( - feature = "rt-tokio", - not(all(target_family = "wasm", target_os = "wasi")) -))] +#[cfg(feature = "rt-tokio")] #[derive(Debug, Default)] pub struct TokioSleep; -#[cfg(all( - feature = "rt-tokio", - not(all(target_family = "wasm", target_os = "wasi")) -))] +#[cfg(feature = "rt-tokio")] impl TokioSleep { /// Create a new [`AsyncSleep`] implementation using the Tokio hashed wheel sleep implementation pub fn new() -> TokioSleep { @@ -137,48 +122,14 @@ impl TokioSleep { } } -#[cfg(all( - feature = "rt-tokio", - not(all(target_family = "wasm", target_os = "wasi")) -))] +#[cfg(feature = "rt-tokio")] impl AsyncSleep for TokioSleep { fn sleep(&self, duration: Duration) -> Sleep { Sleep::new(tokio::time::sleep(duration)) } } -#[cfg(all( - feature = "rt-tokio", - not(all(target_family = "wasm", target_os = "wasi")) -))] +#[cfg(feature = "rt-tokio")] fn sleep_tokio() -> Arc { Arc::new(TokioSleep::new()) } - -/// Implementation of [`AsyncSleep`] for WASI. -#[non_exhaustive] -#[cfg(all(target_family = "wasm", target_os = "wasi"))] -#[derive(Debug, Default)] -pub struct WasiSleep; - -#[cfg(all(target_family = "wasm", target_os = "wasi"))] -impl WasiSleep { - /// Create a new [`AsyncSleep`] implementation using the WASI sleep implementation - pub fn new() -> WasiSleep { - Default::default() - } -} - -#[cfg(all(target_family = "wasm", target_os = "wasi"))] -impl AsyncSleep for WasiSleep { - fn sleep(&self, duration: std::time::Duration) -> Sleep { - Sleep::new(Box::pin(async move { - std::thread::sleep(duration); - })) - } -} - -#[cfg(all(target_family = "wasm", target_os = "wasi"))] -fn sleep_wasi() -> Arc { - Arc::new(WasiSleep::new()) -} From f8d14afb8a97756fd6fef3706a6c729d1eb38e5e Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 11:08:57 +0200 Subject: [PATCH 29/72] feat(wasm): define top level setter for connector --- rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs index 8f084d2dfd..f86e03a574 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs @@ -5,13 +5,21 @@ //! HTTP WASI Adapter -use aws_smithy_http::{body::SdkBody, byte_stream::ByteStream, result::ConnectorError}; +use aws_smithy_client::{erase::DynConnector, http_connector::HttpConnector}; +use aws_smithy_http::{body::SdkBody, result::ConnectorError}; use bytes::Bytes; use http::{Request, Response}; -use std::task::{Context, Poll}; +use std::{ + sync::Arc, + task::{Context, Poll}, +}; use tower::Service; use wasi_preview2_prototype::http_client::DefaultClient; +pub fn wasi_connector() -> HttpConnector { + HttpConnector::ConnectorFn(Arc::new(|_, _| Some(DynConnector::new(Adapter::default())))) +} + #[derive(Default, Debug, Clone)] /// HTTP Service Adapter used in WASI environment pub struct Adapter {} From 0ac58eadb6578515299a35d3a338861342ed8b9a Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 11:11:14 +0200 Subject: [PATCH 30/72] chore: remove default wasi connector from aws-config --- aws/rust-runtime/aws-config/Cargo.toml | 33 ++++++++++++------- .../aws-config/external-types.toml | 1 - aws/rust-runtime/aws-config/src/connector.rs | 24 ++------------ 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/aws/rust-runtime/aws-config/Cargo.toml b/aws/rust-runtime/aws-config/Cargo.toml index b962200859..c4613c0465 100644 --- a/aws/rust-runtime/aws-config/Cargo.toml +++ b/aws/rust-runtime/aws-config/Cargo.toml @@ -1,7 +1,10 @@ [package] name = "aws-config" version = "0.0.0-smithy-rs-head" -authors = ["AWS Rust SDK Team ", "Russell Cohen "] +authors = [ + "AWS Rust SDK Team ", + "Russell Cohen ", +] description = "AWS SDK config and credential provider implementations." edition = "2021" exclude = ["test-data/*", "integration-tests/*"] @@ -12,12 +15,11 @@ repository = "https://github.com/awslabs/smithy-rs" client-hyper = ["aws-smithy-client/client-hyper"] rustls = ["aws-smithy-client/rustls", "client-hyper"] native-tls = [] -allow-compilation = [] # our tests use `cargo test --all-features` and native-tls breaks CI +allow-compilation = [ +] # our tests use `cargo test --all-features` and native-tls breaks CI rt-tokio = ["aws-smithy-async/rt-tokio", "tokio/rt"] credentials-sso = ["dep:aws-sdk-sso", "dep:ring", "dep:hex", "dep:zeroize"] -wasm = ["dep:aws-smithy-wasm"] - default = ["client-hyper", "rustls", "rt-tokio", "credentials-sso"] [dependencies] @@ -49,9 +51,6 @@ ring = { version = "0.16", optional = true } hex = { version = "0.4.3", optional = true } zeroize = { version = "1", optional = true } -# used for WebAssembly target family -aws-smithy-wasm = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-wasm", optional = true } - [dev-dependencies] futures-util = { version = "0.3.16", default-features = false } tracing-test = "0.2.1" @@ -66,12 +65,24 @@ arbitrary = "1.3" serde = { version = "1", features = ["derive"] } serde_json = "1" -aws-credential-types = { path = "../../sdk/build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] } -aws-smithy-client = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-client", features = ["test-util", "rt-tokio", "client-hyper"] } +aws-credential-types = { path = "../../sdk/build/aws-sdk/sdk/aws-credential-types", features = [ + "test-util", +] } +aws-smithy-client = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-client", features = [ + "test-util", + "rt-tokio", + "client-hyper", +] } # used for a usage example -hyper-rustls = { version = "0.24", features = ["webpki-tokio", "http2", "http1"] } -aws-smithy-async = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-async", features = ["test-util"] } +hyper-rustls = { version = "0.24", features = [ + "webpki-tokio", + "http2", + "http1", +] } +aws-smithy-async = { path = "../../sdk/build/aws-sdk/sdk/aws-smithy-async", features = [ + "test-util", +] } [package.metadata.docs.rs] all-features = true diff --git a/aws/rust-runtime/aws-config/external-types.toml b/aws/rust-runtime/aws-config/external-types.toml index 6837b0b4d5..b90a84885c 100644 --- a/aws/rust-runtime/aws-config/external-types.toml +++ b/aws/rust-runtime/aws-config/external-types.toml @@ -26,7 +26,6 @@ allowed_external_types = [ "aws_smithy_types::timeout::OperationTimeoutConfig", "aws_smithy_types::timeout::TimeoutConfig", "aws_smithy_types::timeout::TimeoutConfigBuilder", - "aws_smithy_wasm::wasi_adapter::*", "aws_types::*", "http::response::Response", "http::uri::Uri", diff --git a/aws/rust-runtime/aws-config/src/connector.rs b/aws/rust-runtime/aws-config/src/connector.rs index 5a0ebf1176..33c820261f 100644 --- a/aws/rust-runtime/aws-config/src/connector.rs +++ b/aws/rust-runtime/aws-config/src/connector.rs @@ -22,28 +22,8 @@ pub use aws_smithy_client::conns::default_connector; #[cfg(all(feature = "native-tls", not(feature = "allow-compilation")))] compile_error!("Feature native-tls has been removed. For upgrade instructions, see: https://awslabs.github.io/smithy-rs/design/transport/connector.html"); -/// Given `ConnectorSettings` and an `AsyncSleep`, create a `DynConnector` from defaults depending on what cargo features are activated. -#[cfg(all( - feature = "wasm", - target_family = "wasm", - target_os = "wasi", - not(any(feature = "rustls", feature = "native-tls")) -))] -pub fn default_connector( - _settings: &aws_smithy_client::http_connector::ConnectorSettings, - _sleep: Option, -) -> Option { - Some(DynConnector::new( - aws_smithy_wasm::wasi_adapter::Adapter::default(), - )) -} - -/// Given `ConnectorSettings` and an `AsyncSleep`, create a `DynConnector` from defaults depending on what cargo features are activated. -#[cfg(not(any( - all(feature = "wasm", target_family = "wasm", target_os = "wasi"), - feature = "rustls", - feature = "native-tls" -)))] +/// Given `ConnectorSettings` and a [`SharedAsyncSleep`](aws_smithy_async::rt::sleep::SharedAsyncSleep), create a `DynConnector` from defaults depending on what cargo features are activated. +#[cfg(not(feature = "client-hyper"))] pub fn default_connector( _settings: &aws_smithy_client::http_connector::ConnectorSettings, _sleep: Option, From f37119347751437fefaaa2fe4876cf99a595449c Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 11:46:05 +0200 Subject: [PATCH 31/72] chore: update wasm integration tests --- aws/sdk/integration-tests/webassembly/Cargo.toml | 10 ++-------- .../webassembly/src/default_config.rs | 2 ++ aws/sdk/integration-tests/webassembly/src/lib.rs | 2 ++ rust-runtime/aws-smithy-wasm/Cargo.toml | 1 + 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index 01ab44577c..1de3d037cf 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -19,17 +19,11 @@ publish = false [dependencies] anyhow = "1.0" aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false, features = [ - "wasm", -] } -aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types" } -aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3", default-features = false } -aws-sig-auth = { path = "../../build/aws-sdk/sdk/aws-sig-auth" } -aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async", default-features = false, features = [ "rt-tokio", ] } -aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" } -aws-smithy-runtime-api = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime-api" } +aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3", default-features = false } aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types" } +aws-smithy-wasm = { path = "../../build/aws-sdk/sdk/aws-smithy-wasm" } futures = "0.3.28" http = "0.2.8" tokio = { version = "1.32.0", features = ["macros", "rt"] } diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index 3869d05a0a..1e8f075a74 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -4,12 +4,14 @@ */ use aws_smithy_types::{retry::RetryConfig, timeout::TimeoutConfig}; +use aws_smithy_wasm::wasi_adapter::wasi_connector; pub(crate) async fn get_default_config() -> aws_config::SdkConfig { aws_config::from_env() .region("us-east-2") .timeout_config(TimeoutConfig::disabled()) .retry_config(RetryConfig::disabled()) + .http_connector(wasi_connector()) .no_credentials() .load() .await diff --git a/aws/sdk/integration-tests/webassembly/src/lib.rs b/aws/sdk/integration-tests/webassembly/src/lib.rs index 801a5fbd3f..86770409f1 100644 --- a/aws/sdk/integration-tests/webassembly/src/lib.rs +++ b/aws/sdk/integration-tests/webassembly/src/lib.rs @@ -3,6 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +#![allow(dead_code)] + mod default_config; mod list_objects; diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index a5b7649112..e7f528de7f 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -11,6 +11,7 @@ license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +aws-smithy-client = { path = "../aws-smithy-client" } aws-smithy-http = { path = "../aws-smithy-http" } bytes = "1" http = "0.2.4" From 1318f9a0959c2405380ba4dcd1945f12cac2af0a Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 16:31:35 +0200 Subject: [PATCH 32/72] chore: setup tracing in aws-smithy-wasm --- rust-runtime/aws-smithy-wasm/Cargo.toml | 1 + rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index e7f528de7f..20c15134e2 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -16,6 +16,7 @@ aws-smithy-http = { path = "../aws-smithy-http" } bytes = "1" http = "0.2.4" tower = { version = "0.4.8" } +tracing = "0.1" wasi-preview2-prototype = { version = "0.0.2", features = ["http-client"] } [package.metadata.docs.rs] diff --git a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs index f86e03a574..107c0a8b6b 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs @@ -39,7 +39,8 @@ impl Service> for Adapter { } fn call(&mut self, req: http::Request) -> Self::Future { - println!("Adapter: sending request..."); + tracing::debug!("adapter: sending request"); + tracing::trace!("request details {:?}", req); let client = DefaultClient::new(None); // Right now only synchronous calls can be made through WASI let fut = client.handle(req.map(|body| match body.bytes() { @@ -50,6 +51,8 @@ impl Service> for Adapter { let res = fut .map_err(|err| ConnectorError::other(err.into(), None)) .expect("response from adapter"); + tracing::debug!("adapter: response received"); + tracing::trace!("response details {:?}", res); let (parts, body) = res.into_parts(); let loaded_body = if body.is_empty() { From 5ca61082c397ef79bcc2d95996ffd2564a9bf8c8 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Tue, 29 Aug 2023 16:32:00 +0200 Subject: [PATCH 33/72] chore: some extra clean up --- aws/sdk/integration-tests/webassembly/src/list_objects.rs | 1 - buildSrc/src/main/kotlin/CrateSet.kt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/src/list_objects.rs b/aws/sdk/integration-tests/webassembly/src/list_objects.rs index 7d3747f990..870e787f4d 100644 --- a/aws/sdk/integration-tests/webassembly/src/list_objects.rs +++ b/aws/sdk/integration-tests/webassembly/src/list_objects.rs @@ -26,7 +26,6 @@ async fn s3_list_objects() -> ListObjectsV2Output { #[tokio::test] pub async fn test_s3_list_objects() { let result = s3_list_objects().await; - println!("result: {:?}", result); let objects = result.contents().unwrap(); assert!(objects.len() > 1); } diff --git a/buildSrc/src/main/kotlin/CrateSet.kt b/buildSrc/src/main/kotlin/CrateSet.kt index d34b920891..824b4ff8f8 100644 --- a/buildSrc/src/main/kotlin/CrateSet.kt +++ b/buildSrc/src/main/kotlin/CrateSet.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ - object CrateSet { +object CrateSet { val AWS_SDK_RUNTIME = listOf( "aws-config", "aws-credential-types", From 044d37ab5dae48d090991ebf01d8565b04fc41d9 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Wed, 30 Aug 2023 11:42:59 +0200 Subject: [PATCH 34/72] chore: document wasi connector function --- rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs index 107c0a8b6b..fe1baf7868 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs @@ -16,6 +16,9 @@ use std::{ use tower::Service; use wasi_preview2_prototype::http_client::DefaultClient; +/// Creates a connector function that can be used during instantiation of the client SDK +/// in order to route the HTTP requests through the WebAssembly host. The host must +/// support the WASI HTTP proposal as defined in the Preview 2 specification. pub fn wasi_connector() -> HttpConnector { HttpConnector::ConnectorFn(Arc::new(|_, _| Some(DynConnector::new(Adapter::default())))) } From 917e0011f40928e295f6a8c4feecc6e4109db53d Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Wed, 30 Aug 2023 16:32:17 +0200 Subject: [PATCH 35/72] chore: ensure webassembly test can compile to non-wasm target --- .../integration-tests/webassembly/Cargo.toml | 8 ++--- .../integration-tests/webassembly/src/lib.rs | 29 +++-------------- .../webassembly/src/list_objects.rs | 2 +- .../integration-tests/webassembly/src/wasi.rs | 32 +++++++++++++++++++ ...check-aws-sdk-standalone-integration-tests | 2 +- 5 files changed, 41 insertions(+), 32 deletions(-) create mode 100644 aws/sdk/integration-tests/webassembly/src/wasi.rs diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index 1de3d037cf..dd16d3c9a2 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -16,16 +16,14 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[dependencies] -anyhow = "1.0" +[target.'cfg(target_family = "wasm")'.dependencies] aws-config = { path = "../../build/aws-sdk/sdk/aws-config", default-features = false, features = [ "rt-tokio", ] } aws-sdk-s3 = { path = "../../build/aws-sdk/sdk/s3", default-features = false } aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types" } aws-smithy-wasm = { path = "../../build/aws-sdk/sdk/aws-smithy-wasm" } -futures = "0.3.28" -http = "0.2.8" tokio = { version = "1.32.0", features = ["macros", "rt"] } -tower = "0.4.13" + +[target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] wit-bindgen = { version = "0.11.0", features = ["macros", "realloc"] } diff --git a/aws/sdk/integration-tests/webassembly/src/lib.rs b/aws/sdk/integration-tests/webassembly/src/lib.rs index 86770409f1..27c892cbc1 100644 --- a/aws/sdk/integration-tests/webassembly/src/lib.rs +++ b/aws/sdk/integration-tests/webassembly/src/lib.rs @@ -5,30 +5,9 @@ #![allow(dead_code)] +#[cfg(target_family = "wasm")] mod default_config; +#[cfg(target_family = "wasm")] mod list_objects; - -wit_bindgen::generate!({ - inline: " - package aws:component - - interface run { - run: func() -> result - } - - world main { - export run - } - ", - exports: { - "aws:component/run": Component - } -}); - -struct Component; - -impl exports::aws::component::run::Guest for Component { - fn run() -> Result<(), ()> { - Ok(()) - } -} +#[cfg(all(target_family = "wasm", target_os = "wasi"))] +mod wasi; diff --git a/aws/sdk/integration-tests/webassembly/src/list_objects.rs b/aws/sdk/integration-tests/webassembly/src/list_objects.rs index 870e787f4d..bf815c7225 100644 --- a/aws/sdk/integration-tests/webassembly/src/list_objects.rs +++ b/aws/sdk/integration-tests/webassembly/src/list_objects.rs @@ -6,8 +6,8 @@ use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output; async fn s3_list_objects() -> ListObjectsV2Output { - use aws_sdk_s3::Client; use crate::default_config::get_default_config; + use aws_sdk_s3::Client; let shared_config = get_default_config().await; let client = Client::new(&shared_config); diff --git a/aws/sdk/integration-tests/webassembly/src/wasi.rs b/aws/sdk/integration-tests/webassembly/src/wasi.rs new file mode 100644 index 0000000000..e85a7c07b0 --- /dev/null +++ b/aws/sdk/integration-tests/webassembly/src/wasi.rs @@ -0,0 +1,32 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// Needed for WASI-compliant environment as it expects specific functions +// to be exported such as `cabi_realloc`, `_start`, etc. + +wit_bindgen::generate!({ + inline: " + package aws:component + + interface run { + run: func() -> result + } + + world main { + export run + } + ", + exports: { + "aws:component/run": Component + } +}); + +struct Component; + +impl exports::aws::component::run::Guest for Component { + fn run() -> Result<(), ()> { + Ok(()) + } +} diff --git a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests index d5d1feb6f0..dd6dc1c8fc 100755 --- a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests +++ b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests @@ -42,6 +42,6 @@ cargo check --tests # Running WebAssembly (WASI) specific integration tests pushd "${tmp_dir}/aws/sdk/integration-tests/webassembly" &>/dev/null -cargo build +cargo check --tests cargo test -- --nocapture popd From 6d6cfdb41f86c8494e96b802ac015f90ef5cef4b Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Fri, 8 Sep 2023 11:47:20 +0200 Subject: [PATCH 36/72] chore: remove unnecessary tracing in adapter --- rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs index fe1baf7868..3e7a5775ff 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs @@ -42,8 +42,7 @@ impl Service> for Adapter { } fn call(&mut self, req: http::Request) -> Self::Future { - tracing::debug!("adapter: sending request"); - tracing::trace!("request details {:?}", req); + tracing::trace!("adapter: sending request"); let client = DefaultClient::new(None); // Right now only synchronous calls can be made through WASI let fut = client.handle(req.map(|body| match body.bytes() { @@ -54,9 +53,7 @@ impl Service> for Adapter { let res = fut .map_err(|err| ConnectorError::other(err.into(), None)) .expect("response from adapter"); - tracing::debug!("adapter: response received"); - tracing::trace!("response details {:?}", res); - + tracing::trace!("adapter: response received"); let (parts, body) = res.into_parts(); let loaded_body = if body.is_empty() { SdkBody::empty() From 93f7ffcdc46d4fe414284f9524f27dec2178d9d7 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <16357187+eduardomourar@users.noreply.github.com> Date: Thu, 21 Sep 2023 00:31:45 +0200 Subject: [PATCH 37/72] chore: update wasmtime to v13 --- aws/sdk/integration-tests/webassembly/.cargo/config.toml | 2 +- rust-runtime/aws-smithy-wasm/Cargo.toml | 2 +- tools/ci-build/Dockerfile | 5 ++--- tools/ci-scripts/check-aws-sdk-standalone-integration-tests | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/.cargo/config.toml b/aws/sdk/integration-tests/webassembly/.cargo/config.toml index c454cff49f..22b92b4be0 100644 --- a/aws/sdk/integration-tests/webassembly/.cargo/config.toml +++ b/aws/sdk/integration-tests/webassembly/.cargo/config.toml @@ -5,7 +5,7 @@ target = "wasm32-wasi" rustflags = ["-C", "opt-level=1"] runner = [ "wasmtime", - "run", + "--disable-cache", "--preview2", "--wasi-modules=experimental-wasi-http", ] diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index 20c15134e2..e2ecd0d158 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -17,7 +17,7 @@ bytes = "1" http = "0.2.4" tower = { version = "0.4.8" } tracing = "0.1" -wasi-preview2-prototype = { version = "0.0.2", features = ["http-client"] } +wasi-preview2-prototype = { version = "0.0.3", features = ["http-client"] } [package.metadata.docs.rs] all-features = true diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 0ca6259d2a..255f96eeef 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -122,10 +122,9 @@ ARG wasm_pack_version=0.11.0 RUN cargo install wasm-pack --locked --version ${wasm_pack_version} FROM install_rust AS wasmtime -# should be changed to 13.0.0 when released -ARG cargo_wasmtime_version=d1d8893d2e66409662bad7f9ec65050a218be681 +ARG cargo_wasmtime_version=13.0.0 ARG rust_nightly_version -RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-model" --locked --git https://github.com/bytecodealliance/wasmtime --rev ${cargo_wasmtime_version} +RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} FROM install_rust AS cargo_semver_checks ARG cargo_semver_checks_version=0.20.0 diff --git a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests index dd6dc1c8fc..6cef5e071c 100755 --- a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests +++ b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests @@ -43,5 +43,5 @@ cargo check --tests # Running WebAssembly (WASI) specific integration tests pushd "${tmp_dir}/aws/sdk/integration-tests/webassembly" &>/dev/null cargo check --tests -cargo test -- --nocapture +RUST_TEST_NOCAPTURE=1 cargo test popd From 3e5ad291095676a771ce1589b5c91a71e5c128c5 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues <16357187+eduardomourar@users.noreply.github.com> Date: Thu, 21 Sep 2023 21:39:10 +0200 Subject: [PATCH 38/72] Revert "chore: prepare git folder within cargo" This reverts commit 1ddbd459754ee71e24fdf86bf5cf3d3c6eb323b3. --- tools/ci-build/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 255f96eeef..fcf46320dc 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -87,9 +87,6 @@ RUN set -eux; \ cargo install --locked --path tools/ci-build/difftags; \ cargo install --locked --path tools/ci-build/sdk-lints; \ cargo install --locked --path tools/ci-build/sdk-versioner; \ - mkdir -p /opt/cargo/git/db; \ - chmod g+rw -R /opt/cargo/git; \ - chmod g+rw -R /opt/cargo/bin; \ chmod g+rw -R /opt/cargo/registry FROM install_rust AS cargo_deny From 843a4d722e577b1e6f31f938d7dd08732cc0139e Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Tue, 10 Oct 2023 10:59:24 -0700 Subject: [PATCH 39/72] Apply fixes for breaking changes introduced by the HttpClient refactor --- .../integration-tests/webassembly/Cargo.toml | 1 + .../webassembly/src/default_config.rs | 5 +- rust-runtime/aws-smithy-wasm/Cargo.toml | 2 +- rust-runtime/aws-smithy-wasm/src/lib.rs | 2 +- rust-runtime/aws-smithy-wasm/src/wasi.rs | 90 +++++++++++++++++++ .../aws-smithy-wasm/src/wasi_adapter.rs | 66 -------------- 6 files changed, 94 insertions(+), 72 deletions(-) create mode 100644 rust-runtime/aws-smithy-wasm/src/wasi.rs delete mode 100644 rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index 2b71367aff..7632325fcd 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -26,6 +26,7 @@ aws-smithy-runtime = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime", feat aws-smithy-runtime-api = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["client"] } aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types" } aws-smithy-wasm = { path = "../../build/aws-sdk/sdk/aws-smithy-wasm" } +http = "0.2.9" tokio = { version = "1.32.0", features = ["macros", "rt"] } [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index 5117ae95e8..a2435c5d74 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -5,12 +5,9 @@ use crate::http::WasmHttpConnector; use aws_config::retry::RetryConfig; -use aws_credential_types::Credentials; use aws_smithy_types::timeout::TimeoutConfig; -use aws_types::region::Region; -use std::future::Future; -pub(crate) fn get_default_config() -> impl Future { +pub(crate) async fn get_default_config() -> aws_config::SdkConfig { aws_config::from_env() .region("us-east-2") .timeout_config(TimeoutConfig::disabled()) diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index e2ecd0d158..52a34135e0 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -11,7 +11,7 @@ license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] -aws-smithy-client = { path = "../aws-smithy-client" } +aws-smithy-runtime-api = { path = "../aws-smithy-runtime-api" } aws-smithy-http = { path = "../aws-smithy-http" } bytes = "1" http = "0.2.4" diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs index a0c4bfaf8f..140872a9fe 100644 --- a/rust-runtime/aws-smithy-wasm/src/lib.rs +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -14,4 +14,4 @@ //! Smithy WebAssembly #[cfg(all(target_family = "wasm", target_os = "wasi"))] -pub mod wasi_adapter; +pub mod wasi; diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs new file mode 100644 index 0000000000..e7bcd66a9a --- /dev/null +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -0,0 +1,90 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! HTTP WASI Adapter + +use aws_smithy_http::{body::SdkBody, result::ConnectorError}; +use aws_smithy_runtime_api::{ + client::{ + http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, + SharedHttpClient, SharedHttpConnector, + }, + orchestrator::HttpRequest, + runtime_components::RuntimeComponents, + }, + shared::IntoShared, +}; +use bytes::Bytes; +use wasi_preview2_prototype::http_client::DefaultClient; + +/// Creates a connector function that can be used during instantiation of the client SDK +/// in order to route the HTTP requests through the WebAssembly host. The host must +/// support the WASI HTTP proposal as defined in the Preview 2 specification. +pub fn wasi_http_client() -> SharedHttpClient { + WasiHttpClient::new().into_shared() +} + +/// HTTP client used in WASI environment +#[derive(Debug, Clone)] +pub struct WasiHttpClient { + connector: SharedHttpConnector, +} + +impl WasiHttpClient { + /// Create a new Wasi HTTP client. + pub fn new() -> Self { + Default::default() + } +} + +impl Default for WasiHttpClient { + fn default() -> Self { + Self { + connector: WasiHttpConnector.into_shared(), + } + } +} + +impl HttpClient for WasiHttpClient { + fn http_connector( + &self, + _settings: &HttpConnectorSettings, + _components: &RuntimeComponents, + ) -> SharedHttpConnector { + // TODO(wasi): add connect/read timeouts + self.connector.clone() + } +} + +/// HTTP connector used in WASI environment +#[non_exhaustive] +#[derive(Debug)] +pub struct WasiHttpConnector; + +impl HttpConnector for WasiHttpConnector { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + tracing::trace!("WasiHttpConnector: sending request {request:?}"); + let client = DefaultClient::new(None); + // Right now only synchronous calls can be made through WASI + let fut = client.handle(request.map(|body| match body.bytes() { + Some(value) => Bytes::copy_from_slice(value), + None => Bytes::new(), + })); + HttpConnectorFuture::new(async move { + let response = fut + .map_err(|err| ConnectorError::other(err.into(), None)) + .expect("response from adapter"); + tracing::trace!("WasiHttpConnector: response received {response:?}"); + let (parts, body) = response.into_parts(); + let loaded_body = if body.is_empty() { + SdkBody::empty() + } else { + SdkBody::from(body) + }; + Ok(http::Response::from_parts(parts, loaded_body)) + }) + } +} diff --git a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs b/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs deleted file mode 100644 index 3e7a5775ff..0000000000 --- a/rust-runtime/aws-smithy-wasm/src/wasi_adapter.rs +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -//! HTTP WASI Adapter - -use aws_smithy_client::{erase::DynConnector, http_connector::HttpConnector}; -use aws_smithy_http::{body::SdkBody, result::ConnectorError}; -use bytes::Bytes; -use http::{Request, Response}; -use std::{ - sync::Arc, - task::{Context, Poll}, -}; -use tower::Service; -use wasi_preview2_prototype::http_client::DefaultClient; - -/// Creates a connector function that can be used during instantiation of the client SDK -/// in order to route the HTTP requests through the WebAssembly host. The host must -/// support the WASI HTTP proposal as defined in the Preview 2 specification. -pub fn wasi_connector() -> HttpConnector { - HttpConnector::ConnectorFn(Arc::new(|_, _| Some(DynConnector::new(Adapter::default())))) -} - -#[derive(Default, Debug, Clone)] -/// HTTP Service Adapter used in WASI environment -pub struct Adapter {} - -impl Service> for Adapter { - type Response = Response; - - type Error = ConnectorError; - - #[allow(clippy::type_complexity)] - type Future = std::pin::Pin< - Box> + Send + 'static>, - >; - - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { - Poll::Ready(Ok(())) - } - - fn call(&mut self, req: http::Request) -> Self::Future { - tracing::trace!("adapter: sending request"); - let client = DefaultClient::new(None); - // Right now only synchronous calls can be made through WASI - let fut = client.handle(req.map(|body| match body.bytes() { - Some(value) => Bytes::copy_from_slice(value), - None => Bytes::new(), - })); - Box::pin(async move { - let res = fut - .map_err(|err| ConnectorError::other(err.into(), None)) - .expect("response from adapter"); - tracing::trace!("adapter: response received"); - let (parts, body) = res.into_parts(); - let loaded_body = if body.is_empty() { - SdkBody::empty() - } else { - SdkBody::from(body) - }; - Ok(http::Response::from_parts(parts, loaded_body)) - }) - } -} From 8a32aff84daf2cde7f832326510906195f23df74 Mon Sep 17 00:00:00 2001 From: Landon James Date: Sat, 10 Feb 2024 20:20:16 -0800 Subject: [PATCH 40/72] Reimplemented SDK's HttpConnector with WASI types --- rust-runtime/aws-smithy-wasm/Cargo.toml | 12 +- .../aws-smithy-wasm/external-types.toml | 1 + rust-runtime/aws-smithy-wasm/src/lib.rs | 2 +- rust-runtime/aws-smithy-wasm/src/wasi.rs | 292 +++++++++++++++++- 4 files changed, 284 insertions(+), 23 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index 9cc15d75d0..302412779c 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -10,14 +10,16 @@ edition = "2021" license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" -[target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +# [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +[dependencies] aws-smithy-runtime-api = { path = "../aws-smithy-runtime-api" } aws-smithy-http = { path = "../aws-smithy-http" } +aws-smithy-types = { path = "../aws-smithy-types" } bytes = "1" -http = "0.2.4" -tower = { version = "0.4.8" } -tracing = "0.1" -wasi-preview2-prototype = { version = "0.0.3", features = ["http-client"] } +http = "1.0.0" +tower = "0.4.13" +tracing = "0.1.40" +wasi = "0.12.0+wasi-0.2.0" [package.metadata.docs.rs] all-features = true diff --git a/rust-runtime/aws-smithy-wasm/external-types.toml b/rust-runtime/aws-smithy-wasm/external-types.toml index ff30ccf5ad..8a2e2ce81a 100644 --- a/rust-runtime/aws-smithy-wasm/external-types.toml +++ b/rust-runtime/aws-smithy-wasm/external-types.toml @@ -1,2 +1,3 @@ allowed_external_types = [ + "aws_smithy_types::*", ] diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs index 4c423f3eb1..ff702663d7 100644 --- a/rust-runtime/aws-smithy-wasm/src/lib.rs +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -16,5 +16,5 @@ //! Smithy WebAssembly -#[cfg(all(target_family = "wasm", target_os = "wasi"))] +// #[cfg(all(target_family = "wasm", target_os = "wasi"))] pub mod wasi; diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index e7bcd66a9a..e35952557a 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -5,7 +5,9 @@ //! HTTP WASI Adapter -use aws_smithy_http::{body::SdkBody, result::ConnectorError}; +use std::ops::Deref; + +use aws_smithy_http::header::ParseError; use aws_smithy_runtime_api::{ client::{ http::{ @@ -13,13 +15,20 @@ use aws_smithy_runtime_api::{ SharedHttpClient, SharedHttpConnector, }, orchestrator::HttpRequest, + result::{ConnectorError, SdkError}, runtime_components::RuntimeComponents, }, + http::Response, shared::IntoShared, }; +use aws_smithy_types::body::SdkBody; use bytes::Bytes; -use wasi_preview2_prototype::http_client::DefaultClient; - +// use crate::wasi::{outgoing_handler, types}; +use http::{HeaderName, HeaderValue}; +use wasi::http::{outgoing_handler, types as http_types}; +use wasi::io::poll; +use wasi::io::streams; +// use wasi_preview2_prototype::http_client::DefaultClient; /// Creates a connector function that can be used during instantiation of the client SDK /// in order to route the HTTP requests through the WebAssembly host. The host must /// support the WASI HTTP proposal as defined in the Preview 2 specification. @@ -67,24 +76,273 @@ pub struct WasiHttpConnector; impl HttpConnector for WasiHttpConnector { fn call(&self, request: HttpRequest) -> HttpConnectorFuture { tracing::trace!("WasiHttpConnector: sending request {request:?}"); - let client = DefaultClient::new(None); - // Right now only synchronous calls can be made through WASI - let fut = client.handle(request.map(|body| match body.bytes() { + let client = WasiDefaultClient::new(None); + let http_req = request.try_into_http1x().expect("http request invalid"); + let converted_req = http_req.map(|body| match body.bytes() { Some(value) => Bytes::copy_from_slice(value), None => Bytes::new(), - })); + }); + + // Right now only synchronous calls can be made through WASI + let fut_result = client.handle(converted_req); + HttpConnectorFuture::new(async move { - let response = fut - .map_err(|err| ConnectorError::other(err.into(), None)) - .expect("response from adapter"); + let fut = fut_result?; + let response = fut.map(|body| { + if body.is_empty() { + SdkBody::empty() + } else { + SdkBody::from(body) + } + }); tracing::trace!("WasiHttpConnector: response received {response:?}"); - let (parts, body) = response.into_parts(); - let loaded_body = if body.is_empty() { - SdkBody::empty() - } else { - SdkBody::from(body) - }; - Ok(http::Response::from_parts(parts, loaded_body)) + + let sdk_res = Response::try_from(response) + .map_err(|err| ConnectorError::other(err.into(), None))?; + // .expect("response from adapter"); + + Ok(sdk_res) }) } } + +pub struct WasiDefaultClient { + options: Option, +} + +impl WasiDefaultClient { + pub fn new(options: Option) -> Self { + Self { options } + } + + pub fn handle( + &self, + req: http::Request, + ) -> Result, ConnectorError> { + let req = WasiRequest::try_from(req).expect("Converting http request"); + + let res = outgoing_handler::handle(req.id, self.options).expect("Http response"); + let subscription = res.subscribe(); + subscription.block(); + + //This is pretty ugly because the FutureIncomingResponse .get() method returns a + //Option, ()>>. + //The outer Option is the readiness which we know is Some because we .block() waiting for it + //The outer Result is just a singleton enforcer so we can only get the response once + //The inner Result indicates whether the HTTP call was sent/received successfully (not the 200 succes of the call) + let incoming_res = res + .get() + .unwrap() + .unwrap() + .map_err(|err| ConnectorError::other(err.into(), None))?; + + let response = + http::Response::try_from(WasiResponse(incoming_res)).expect("Converting http response"); + + Ok(response) + } +} + +#[derive(Debug)] +pub struct WasiRequest { + id: outgoing_handler::OutgoingRequest, + body: http_types::OutputStream, +} + +impl WasiRequest { + pub fn new(id: outgoing_handler::OutgoingRequest, body: http_types::OutputStream) -> Self { + Self { id, body } + } +} + +impl TryFrom> for WasiRequest { + type Error = ParseError; + + fn try_from(value: http::Request) -> Result { + let (parts, body) = value.into_parts(); + let method = WasiMethod::try_from(parts.method) + .map_err(|_| ParseError::new("Invalid http Method"))?; + let path_with_query = parts.uri.path_and_query(); + let headers = WasiHeaders::from(&parts.headers); + let scheme = match parts.uri.scheme_str().unwrap_or("") { + "http" => Some(&http_types::Scheme::Http), + "https" => Some(&http_types::Scheme::Https), + _ => None, + }; + let request = http_types::new_outgoing_request( + &method, + path_with_query.map(|q| q.as_str()), + scheme, + parts.uri.authority().map(|a| a.as_str()), + headers.to_owned(), + ); + + let request2 = http_types::OutgoingRequest::new(&parts.headers); + + let request_body = http_types::outgoing_request_write(request) + .map_err(|_| ParseError::new("outgoing request write failed"))?; + + if body.is_empty() { + let pollable = streams::subscribe_to_output_stream(request_body); + let mut buf = body.as_ref(); + while !buf.is_empty() { + poll::poll_oneoff(&[pollable]); + + let permit = match streams::check_write(request_body) { + Ok(n) => usize::try_from(n)?, + Err(_) => return Err(ParseError::new("Output stream error")), + }; + + let len = buf.len().min(permit); + let (chunk, rest) = buf.split_at(len); + buf = rest; + + if streams::write(request_body, chunk).is_err() { + return Err(ParseError::new("Output stream error")); + } + } + + if streams::flush(request_body).is_err() { + return Err(ParseError::new("Output stream error")); + } + + poll::poll_oneoff(&[sub.pollable]); + + if streams::check_write(request_body).is_err() { + return Err(ParseError::new("Output stream error")); + } + } + + Ok(WasiRequest::new(request, request_body)) + } +} + +pub struct WasiMethod(http_types::Method); + +impl Deref for WasiMethod { + type Target = http_types::Method; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl TryFrom for WasiMethod { + type Error = ParseError; + + fn try_from(method: http::Method) -> Result { + Ok(Self(match method { + http::Method::GET => http_types::Method::Get, + http::Method::POST => http_types::Method::Post, + http::Method::PUT => http_types::Method::Put, + http::Method::DELETE => http_types::Method::Delete, + http::Method::PATCH => http_types::Method::Patch, + http::Method::CONNECT => http_types::Method::Connect, + http::Method::TRACE => http_types::Method::Trace, + http::Method::HEAD => http_types::Method::Head, + http::Method::OPTIONS => http_types::Method::Options, + _ => return Err(ParseError::new("failed due to unsupported method, currently supported methods are: GET, POST, PUT, DELETE, PATCH, CONNECT, TRACE, HEAD, and OPTIONS")), + })) + } +} + +pub struct WasiResponse(http_types::IncomingResponse); + +impl Deref for WasiResponse { + type Target = http_types::IncomingResponse; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl TryFrom for http::Response { + type Error = ParseError; + + fn try_from(value: WasiResponse) -> Result { + let future_response = value; + + let incoming_response = match http_types::future_incoming_response_get(future_response) { + Some(result) => result, + None => { + let pollable = http_types::listen_to_future_incoming_response(future_response); + let _ = poll::poll_oneoff(&[pollable]); + http_types::future_incoming_response_get(future_response) + .expect("incoming response available") + } + } + .map_err(|e| ParseError::new("incoming response error: {e:?}"))?; + + let status = http_types::incoming_response_status(incoming_response); + + let body_stream = http_types::incoming_response_consume(incoming_response) + .map_err(|_| ParseError::new("consuming incoming response"))?; + + let mut body = BytesMut::new(); + { + let pollable = streams::subscribe_to_input_stream(body_stream); + poll::poll_oneoff(&[pollable]); + let mut eof = streams::StreamStatus::Open; + while eof != streams::StreamStatus::Ended { + let (body_chunk, stream_status) = streams::read(body_stream, u64::MAX) + .map_err(|e| ParseError::new("reading response body: {e:?}"))?; + eof = if body_chunk.is_empty() { + streams::StreamStatus::Ended + } else { + stream_status + }; + body.put(body_chunk.as_slice()); + } + } + + let mut res = http::Response::builder() + .status(status) + .body(body.freeze()) + .map_err(|_| ParseError::new("building http response"))?; + + let headers_handle = http_types::incoming_response_headers(incoming_response); + if headers_handle > 0 { + let headers_map = res.headers_mut(); + for (name, value) in http_types::fields_entries(headers_handle) { + headers_map.insert( + HeaderName::from_bytes(name.as_bytes()) + .map_err(|_| ParseError::new("converting response header name"))?, + HeaderValue::from_bytes(value.as_slice()) + .map_err(|_| ParseError::new("converting response header value"))?, + ); + } + } + + Ok(res) + } +} + +pub struct WasiHeaders(http_types::Fields); + +impl Deref for WasiHeaders { + type Target = http_types::Fields; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a> From<&'a http::HeaderMap> for WasiHeaders { + fn from(headers: &'a http::HeaderMap) -> Self { + let entries = headers + .iter() + .map(|(name, value)| { + ( + name.to_string(), + value.to_str().unwrap().as_bytes().to_vec(), + ) + }) + .collect::>() + .as_slice(); + + //The fields should come from the SDK so we trust that they are valid + let fields = http_types::Fields::from_list(entries).expect("Invalid http headers."); + + Self(fields) + } +} From 83f079ae1d38acd9c31d04b7705d79c369e17318 Mon Sep 17 00:00:00 2001 From: Landon James Date: Sun, 11 Feb 2024 20:40:19 -0800 Subject: [PATCH 41/72] Wasm connector is building now Still need to do testing --- rust-runtime/aws-smithy-wasm/src/wasi.rs | 183 +++++++++-------------- 1 file changed, 74 insertions(+), 109 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index e35952557a..43745aef27 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -15,20 +15,19 @@ use aws_smithy_runtime_api::{ SharedHttpClient, SharedHttpConnector, }, orchestrator::HttpRequest, - result::{ConnectorError, SdkError}, + result::ConnectorError, runtime_components::RuntimeComponents, }, http::Response, shared::IntoShared, }; use aws_smithy_types::body::SdkBody; -use bytes::Bytes; -// use crate::wasi::{outgoing_handler, types}; -use http::{HeaderName, HeaderValue}; -use wasi::http::{outgoing_handler, types as http_types}; -use wasi::io::poll; -use wasi::io::streams; -// use wasi_preview2_prototype::http_client::DefaultClient; +use bytes::{Bytes, BytesMut}; +use wasi::http::{ + outgoing_handler, + types::{self as http_types, RequestOptions}, +}; + /// Creates a connector function that can be used during instantiation of the client SDK /// in order to route the HTTP requests through the WebAssembly host. The host must /// support the WASI HTTP proposal as defined in the Preview 2 specification. @@ -83,7 +82,6 @@ impl HttpConnector for WasiHttpConnector { None => Bytes::new(), }); - // Right now only synchronous calls can be made through WASI let fut_result = client.handle(converted_req); HttpConnectorFuture::new(async move { @@ -99,7 +97,6 @@ impl HttpConnector for WasiHttpConnector { let sdk_res = Response::try_from(response) .map_err(|err| ConnectorError::other(err.into(), None))?; - // .expect("response from adapter"); Ok(sdk_res) }) @@ -121,19 +118,34 @@ impl WasiDefaultClient { ) -> Result, ConnectorError> { let req = WasiRequest::try_from(req).expect("Converting http request"); - let res = outgoing_handler::handle(req.id, self.options).expect("Http response"); + //RequestOptions doesn't impl Clone or Copy and outgoing_handler::handle takes ownership, + //so we need to recreate it + let options = if let Some(opts) = &self.options { + let new_opts = RequestOptions::new(); + new_opts.set_between_bytes_timeout(opts.between_bytes_timeout()); + new_opts.set_connect_timeout(opts.connect_timeout()); + new_opts.set_first_byte_timeout(opts.first_byte_timeout()); + + Some(new_opts) + } else { + None + }; + + let res = outgoing_handler::handle(req.id, options).expect("Http response"); + // Right now only synchronous calls can be made through WASI, so we subscribe and + // block on the FutureIncomingResponse let subscription = res.subscribe(); subscription.block(); - //This is pretty ugly because the FutureIncomingResponse .get() method returns a + //The FutureIncomingResponse .get() method returns a //Option, ()>>. - //The outer Option is the readiness which we know is Some because we .block() waiting for it + //The outer Option ensures readiness which we know is Some because we .block() waiting for it //The outer Result is just a singleton enforcer so we can only get the response once //The inner Result indicates whether the HTTP call was sent/received successfully (not the 200 succes of the call) let incoming_res = res .get() - .unwrap() - .unwrap() + .expect("http response not ready") + .expect("http response accessed more than once") .map_err(|err| ConnectorError::other(err.into(), None))?; let response = @@ -146,11 +158,11 @@ impl WasiDefaultClient { #[derive(Debug)] pub struct WasiRequest { id: outgoing_handler::OutgoingRequest, - body: http_types::OutputStream, + body: http_types::OutgoingBody, } impl WasiRequest { - pub fn new(id: outgoing_handler::OutgoingRequest, body: http_types::OutputStream) -> Self { + pub fn new(id: outgoing_handler::OutgoingRequest, body: http_types::OutgoingBody) -> Self { Self { id, body } } } @@ -162,56 +174,35 @@ impl TryFrom> for WasiRequest { let (parts, body) = value.into_parts(); let method = WasiMethod::try_from(parts.method) .map_err(|_| ParseError::new("Invalid http Method"))?; - let path_with_query = parts.uri.path_and_query(); - let headers = WasiHeaders::from(&parts.headers); + let path_with_query = parts.uri.path_and_query().map(|path| path.as_str()); + let headers = WasiHeaders::from(parts.headers); let scheme = match parts.uri.scheme_str().unwrap_or("") { "http" => Some(&http_types::Scheme::Http), "https" => Some(&http_types::Scheme::Https), _ => None, }; - let request = http_types::new_outgoing_request( - &method, - path_with_query.map(|q| q.as_str()), - scheme, - parts.uri.authority().map(|a| a.as_str()), - headers.to_owned(), - ); - - let request2 = http_types::OutgoingRequest::new(&parts.headers); - - let request_body = http_types::outgoing_request_write(request) - .map_err(|_| ParseError::new("outgoing request write failed"))?; - - if body.is_empty() { - let pollable = streams::subscribe_to_output_stream(request_body); - let mut buf = body.as_ref(); - while !buf.is_empty() { - poll::poll_oneoff(&[pollable]); - - let permit = match streams::check_write(request_body) { - Ok(n) => usize::try_from(n)?, - Err(_) => return Err(ParseError::new("Output stream error")), - }; - - let len = buf.len().min(permit); - let (chunk, rest) = buf.split_at(len); - buf = rest; - - if streams::write(request_body, chunk).is_err() { - return Err(ParseError::new("Output stream error")); - } - } + let authority = parts.uri.authority().map(|auth| auth.as_str()); - if streams::flush(request_body).is_err() { - return Err(ParseError::new("Output stream error")); - } + let request = http_types::OutgoingRequest::new(headers.0); + request.set_scheme(scheme); + request.set_method(&method); + request.set_path_with_query(path_with_query); + request.set_authority(authority); - poll::poll_oneoff(&[sub.pollable]); + let request_body = request.body().expect("body accessed more than once"); - if streams::check_write(request_body).is_err() { - return Err(ParseError::new("Output stream error")); - } - } + let request_stream = request_body + .write() + .expect("output stream accessed more than once"); + + request_stream + .blocking_write_and_flush(&body) + .expect("failed to write body"); + + //The OutputStream is a child resource: it must be dropped + //before the parent OutgoingBody resource is dropped (or finished), + //otherwise the OutgoingBody drop or finish will trap. + drop(request_stream); Ok(WasiRequest::new(request, request_body)) } @@ -260,59 +251,35 @@ impl TryFrom for http::Response { type Error = ParseError; fn try_from(value: WasiResponse) -> Result { - let future_response = value; - - let incoming_response = match http_types::future_incoming_response_get(future_response) { - Some(result) => result, - None => { - let pollable = http_types::listen_to_future_incoming_response(future_response); - let _ = poll::poll_oneoff(&[pollable]); - http_types::future_incoming_response_get(future_response) - .expect("incoming response available") - } - } - .map_err(|e| ParseError::new("incoming response error: {e:?}"))?; + let response = value.0; - let status = http_types::incoming_response_status(incoming_response); + let status = response.status(); - let body_stream = http_types::incoming_response_consume(incoming_response) - .map_err(|_| ParseError::new("consuming incoming response"))?; + let body_stream = response + .consume() + .expect("consume called more than once") + .stream() + .expect("stream accessed more than once"); let mut body = BytesMut::new(); - { - let pollable = streams::subscribe_to_input_stream(body_stream); - poll::poll_oneoff(&[pollable]); - let mut eof = streams::StreamStatus::Open; - while eof != streams::StreamStatus::Ended { - let (body_chunk, stream_status) = streams::read(body_stream, u64::MAX) - .map_err(|e| ParseError::new("reading response body: {e:?}"))?; - eof = if body_chunk.is_empty() { - streams::StreamStatus::Ended - } else { - stream_status - }; - body.put(body_chunk.as_slice()); - } + + //blocking_read blocks until at least one byte is available to read + while let Ok(stream_bytes) = body_stream.blocking_read(u64::MAX) { + body.extend_from_slice(stream_bytes.as_slice()) } - let mut res = http::Response::builder() - .status(status) + let headers = response.headers().entries(); + + let res_build = headers + .into_iter() + .fold(http::Response::builder().status(status), |rb, header| { + rb.header(header.0, header.1) + }); + + let res = res_build .body(body.freeze()) .map_err(|_| ParseError::new("building http response"))?; - let headers_handle = http_types::incoming_response_headers(incoming_response); - if headers_handle > 0 { - let headers_map = res.headers_mut(); - for (name, value) in http_types::fields_entries(headers_handle) { - headers_map.insert( - HeaderName::from_bytes(name.as_bytes()) - .map_err(|_| ParseError::new("converting response header name"))?, - HeaderValue::from_bytes(value.as_slice()) - .map_err(|_| ParseError::new("converting response header value"))?, - ); - } - } - Ok(res) } } @@ -327,8 +294,8 @@ impl Deref for WasiHeaders { } } -impl<'a> From<&'a http::HeaderMap> for WasiHeaders { - fn from(headers: &'a http::HeaderMap) -> Self { +impl From for WasiHeaders { + fn from(headers: http::HeaderMap) -> Self { let entries = headers .iter() .map(|(name, value)| { @@ -337,11 +304,9 @@ impl<'a> From<&'a http::HeaderMap> for WasiHeaders { value.to_str().unwrap().as_bytes().to_vec(), ) }) - .collect::>() - .as_slice(); + .collect::>(); - //The fields should come from the SDK so we trust that they are valid - let fields = http_types::Fields::from_list(entries).expect("Invalid http headers."); + let fields = http_types::Fields::from_list(&entries).expect("Invalid http headers."); Self(fields) } From a6cdb9f207bcf93952d996ad42ca860d31324aa7 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 12 Feb 2024 11:14:16 -0800 Subject: [PATCH 42/72] Adding http-1x feature to runtime-api dependency --- rust-runtime/aws-smithy-wasm/Cargo.toml | 2 +- rust-runtime/aws-smithy-wasm/src/wasi.rs | 176 +++++++++++------------ 2 files changed, 86 insertions(+), 92 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index 302412779c..e97d09cea2 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -12,7 +12,7 @@ repository = "https://github.com/awslabs/smithy-rs" # [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] [dependencies] -aws-smithy-runtime-api = { path = "../aws-smithy-runtime-api" } +aws-smithy-runtime-api = { path = "../aws-smithy-runtime-api", features = ["http-1x"]} aws-smithy-http = { path = "../aws-smithy-http" } aws-smithy-types = { path = "../aws-smithy-types" } bytes = "1" diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index 43745aef27..a9eb991f30 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -4,9 +4,6 @@ */ //! HTTP WASI Adapter - -use std::ops::Deref; - use aws_smithy_http::header::ParseError; use aws_smithy_runtime_api::{ client::{ @@ -25,7 +22,7 @@ use aws_smithy_types::body::SdkBody; use bytes::{Bytes, BytesMut}; use wasi::http::{ outgoing_handler, - types::{self as http_types, RequestOptions}, + types::{self as wasi_http, OutgoingBody, RequestOptions}, }; /// Creates a connector function that can be used during instantiation of the client SDK @@ -62,7 +59,6 @@ impl HttpClient for WasiHttpClient { _settings: &HttpConnectorSettings, _components: &RuntimeComponents, ) -> SharedHttpConnector { - // TODO(wasi): add connect/read timeouts self.connector.clone() } } @@ -75,8 +71,9 @@ pub struct WasiHttpConnector; impl HttpConnector for WasiHttpConnector { fn call(&self, request: HttpRequest) -> HttpConnectorFuture { tracing::trace!("WasiHttpConnector: sending request {request:?}"); - let client = WasiDefaultClient::new(None); - let http_req = request.try_into_http1x().expect("http request invalid"); + // TODO(wasi): add connect/read timeouts + let client = WasiDefaultClient::new(WasiRequestOptions(None)); + let http_req = request.try_into_http1x().expect("Http request invalid"); let converted_req = http_req.map(|body| match body.bytes() { Some(value) => Bytes::copy_from_slice(value), None => Bytes::new(), @@ -103,35 +100,23 @@ impl HttpConnector for WasiHttpConnector { } } -pub struct WasiDefaultClient { - options: Option, +/// Wasi http client containing the options to pass to the +struct WasiDefaultClient { + options: WasiRequestOptions, } impl WasiDefaultClient { - pub fn new(options: Option) -> Self { + /// Create a new Wasi HTTP client. + fn new(options: WasiRequestOptions) -> Self { Self { options } } - pub fn handle( - &self, - req: http::Request, - ) -> Result, ConnectorError> { + /// Make outgoing http request in a Wasi environment + fn handle(&self, req: http::Request) -> Result, ConnectorError> { let req = WasiRequest::try_from(req).expect("Converting http request"); - //RequestOptions doesn't impl Clone or Copy and outgoing_handler::handle takes ownership, - //so we need to recreate it - let options = if let Some(opts) = &self.options { - let new_opts = RequestOptions::new(); - new_opts.set_between_bytes_timeout(opts.between_bytes_timeout()); - new_opts.set_connect_timeout(opts.connect_timeout()); - new_opts.set_first_byte_timeout(opts.first_byte_timeout()); + let res = outgoing_handler::handle(req.0, self.options.clone().0).expect("Http response"); - Some(new_opts) - } else { - None - }; - - let res = outgoing_handler::handle(req.id, options).expect("Http response"); // Right now only synchronous calls can be made through WASI, so we subscribe and // block on the FutureIncomingResponse let subscription = res.subscribe(); @@ -144,26 +129,52 @@ impl WasiDefaultClient { //The inner Result indicates whether the HTTP call was sent/received successfully (not the 200 succes of the call) let incoming_res = res .get() - .expect("http response not ready") - .expect("http response accessed more than once") + .expect("Http response not ready") + .expect("Http response accessed more than once") .map_err(|err| ConnectorError::other(err.into(), None))?; - let response = - http::Response::try_from(WasiResponse(incoming_res)).expect("Converting http response"); + let response = http::Response::try_from(WasiResponse(incoming_res)) + .expect("Converting to http response"); Ok(response) } } -#[derive(Debug)] -pub struct WasiRequest { - id: outgoing_handler::OutgoingRequest, - body: http_types::OutgoingBody, +/// Wrapper for the Wasi RequestOptions type to allow us to impl Clone +struct WasiRequestOptions(Option); + +//The Wasi RequestOptions type doesn't impl copy or clone but the outgoing_handler::handle method +//takes ownership, so we impl it on this wrapper type +impl Clone for WasiRequestOptions { + fn clone(&self) -> Self { + let new_opts = if let Some(opts) = &self.0 { + let new_opts = RequestOptions::new(); + new_opts + .set_between_bytes_timeout(opts.between_bytes_timeout()) + .expect("Between bytes timeout"); + new_opts + .set_connect_timeout(opts.connect_timeout()) + .expect("Connect timeout"); + new_opts + .set_first_byte_timeout(opts.first_byte_timeout()) + .expect("First byte timeout"); + + Some(new_opts) + } else { + None + }; + + Self(new_opts) + } } +/// Wrapper to allow converting between http Request types and Wasi Request types +#[derive(Debug)] +struct WasiRequest(outgoing_handler::OutgoingRequest); + impl WasiRequest { - pub fn new(id: outgoing_handler::OutgoingRequest, body: http_types::OutgoingBody) -> Self { - Self { id, body } + fn new(req: outgoing_handler::OutgoingRequest) -> Self { + Self(req) } } @@ -172,80 +183,70 @@ impl TryFrom> for WasiRequest { fn try_from(value: http::Request) -> Result { let (parts, body) = value.into_parts(); - let method = WasiMethod::try_from(parts.method) - .map_err(|_| ParseError::new("Invalid http Method"))?; + let method = + WasiMethod::try_from(parts.method).map_err(|err| ParseError::new(err.to_string()))?; let path_with_query = parts.uri.path_and_query().map(|path| path.as_str()); let headers = WasiHeaders::from(parts.headers); let scheme = match parts.uri.scheme_str().unwrap_or("") { - "http" => Some(&http_types::Scheme::Http), - "https" => Some(&http_types::Scheme::Https), + "http" => Some(&wasi_http::Scheme::Http), + "https" => Some(&wasi_http::Scheme::Https), _ => None, }; let authority = parts.uri.authority().map(|auth| auth.as_str()); - let request = http_types::OutgoingRequest::new(headers.0); - request.set_scheme(scheme); - request.set_method(&method); - request.set_path_with_query(path_with_query); - request.set_authority(authority); + let request = wasi_http::OutgoingRequest::new(headers.0); + request.set_scheme(scheme).expect("Set scheme"); + request.set_method(&method.0).expect("Set method"); + request + .set_path_with_query(path_with_query) + .expect("Set path with query"); + request.set_authority(authority).expect("Set authority"); - let request_body = request.body().expect("body accessed more than once"); + let request_body = request.body().expect("Body accessed more than once"); let request_stream = request_body .write() - .expect("output stream accessed more than once"); + .expect("Output stream accessed more than once"); request_stream .blocking_write_and_flush(&body) - .expect("failed to write body"); + .expect("Failed to write body"); //The OutputStream is a child resource: it must be dropped //before the parent OutgoingBody resource is dropped (or finished), //otherwise the OutgoingBody drop or finish will trap. drop(request_stream); - Ok(WasiRequest::new(request, request_body)) - } -} - -pub struct WasiMethod(http_types::Method); - -impl Deref for WasiMethod { - type Target = http_types::Method; + OutgoingBody::finish(request_body, None).expect("Http body finished"); - fn deref(&self) -> &Self::Target { - &self.0 + Ok(WasiRequest::new(request)) } } +/// Wrapper to allow converting between http Methods and Wasi Methods +struct WasiMethod(wasi_http::Method); + impl TryFrom for WasiMethod { type Error = ParseError; fn try_from(method: http::Method) -> Result { Ok(Self(match method { - http::Method::GET => http_types::Method::Get, - http::Method::POST => http_types::Method::Post, - http::Method::PUT => http_types::Method::Put, - http::Method::DELETE => http_types::Method::Delete, - http::Method::PATCH => http_types::Method::Patch, - http::Method::CONNECT => http_types::Method::Connect, - http::Method::TRACE => http_types::Method::Trace, - http::Method::HEAD => http_types::Method::Head, - http::Method::OPTIONS => http_types::Method::Options, + http::Method::GET => wasi_http::Method::Get, + http::Method::POST => wasi_http::Method::Post, + http::Method::PUT => wasi_http::Method::Put, + http::Method::DELETE => wasi_http::Method::Delete, + http::Method::PATCH => wasi_http::Method::Patch, + http::Method::CONNECT => wasi_http::Method::Connect, + http::Method::TRACE => wasi_http::Method::Trace, + http::Method::HEAD => wasi_http::Method::Head, + http::Method::OPTIONS => wasi_http::Method::Options, _ => return Err(ParseError::new("failed due to unsupported method, currently supported methods are: GET, POST, PUT, DELETE, PATCH, CONNECT, TRACE, HEAD, and OPTIONS")), })) } } -pub struct WasiResponse(http_types::IncomingResponse); - -impl Deref for WasiResponse { - type Target = http_types::IncomingResponse; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} +/// Wrapper to allow converting between http Response types and Wasi Response types +struct WasiResponse(wasi_http::IncomingResponse); impl TryFrom for http::Response { type Error = ParseError; @@ -257,13 +258,13 @@ impl TryFrom for http::Response { let body_stream = response .consume() - .expect("consume called more than once") + .expect("Consume called more than once") .stream() - .expect("stream accessed more than once"); + .expect("Stream accessed more than once"); let mut body = BytesMut::new(); - //blocking_read blocks until at least one byte is available to read + //blocking_read blocks until at least one byte is available while let Ok(stream_bytes) = body_stream.blocking_read(u64::MAX) { body.extend_from_slice(stream_bytes.as_slice()) } @@ -278,21 +279,14 @@ impl TryFrom for http::Response { let res = res_build .body(body.freeze()) - .map_err(|_| ParseError::new("building http response"))?; + .map_err(|err| ParseError::new(err.to_string()))?; Ok(res) } } -pub struct WasiHeaders(http_types::Fields); - -impl Deref for WasiHeaders { - type Target = http_types::Fields; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} +/// Wrapper to allow converting between http headers and Wasi headers +struct WasiHeaders(wasi_http::Fields); impl From for WasiHeaders { fn from(headers: http::HeaderMap) -> Self { @@ -306,7 +300,7 @@ impl From for WasiHeaders { }) .collect::>(); - let fields = http_types::Fields::from_list(&entries).expect("Invalid http headers."); + let fields = wasi_http::Fields::from_list(&entries).expect("Invalid http headers."); Self(fields) } From e5feef2520d642dff6a8b645e3b56ccb447cbc12 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 12 Feb 2024 15:14:48 -0800 Subject: [PATCH 43/72] Updating tests for wasi http connector --- .../webassembly/.cargo/config.toml | 6 +++--- aws/sdk/integration-tests/webassembly/Cargo.toml | 6 +++++- .../webassembly/src/default_config.rs | 3 ++- .../webassembly/src/list_objects.rs | 13 ++++++++----- aws/sdk/integration-tests/webassembly/src/wasi.rs | 6 +++--- tools/ci-build/Dockerfile | 2 +- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/.cargo/config.toml b/aws/sdk/integration-tests/webassembly/.cargo/config.toml index 22b92b4be0..06b75c8e09 100644 --- a/aws/sdk/integration-tests/webassembly/.cargo/config.toml +++ b/aws/sdk/integration-tests/webassembly/.cargo/config.toml @@ -5,7 +5,7 @@ target = "wasm32-wasi" rustflags = ["-C", "opt-level=1"] runner = [ "wasmtime", - "--disable-cache", - "--preview2", - "--wasi-modules=experimental-wasi-http", + "-C", "cache=n", + "-S", "preview2=y", + "-S", "http=y" ] diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index 24611e4c45..15418e204f 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -14,6 +14,10 @@ license = "Apache-2.0" repository = "https://github.com/smithy-lang/smithy-rs" publish = false + +[features] +default = ["test-util"] +test-util = [] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [target.'cfg(target_family = "wasm")'.dependencies] @@ -32,4 +36,4 @@ http = "0.2.9" tokio = { version = "1.32.0", features = ["macros", "rt"] } [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] -wit-bindgen = { version = "0.11.0", features = ["macros", "realloc"] } +wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index a2435c5d74..63528829ad 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -6,6 +6,7 @@ use crate::http::WasmHttpConnector; use aws_config::retry::RetryConfig; use aws_smithy_types::timeout::TimeoutConfig; +use aws_smithy_wasm::wasi::wasi_http_client; pub(crate) async fn get_default_config() -> aws_config::SdkConfig { aws_config::from_env() @@ -13,7 +14,7 @@ pub(crate) async fn get_default_config() -> aws_config::SdkConfig { .timeout_config(TimeoutConfig::disabled()) .retry_config(RetryConfig::disabled()) .no_credentials() - .http_client(WasmHttpConnector::new()) + .http_client(wasi_http_client()) .load() .await } diff --git a/aws/sdk/integration-tests/webassembly/src/list_objects.rs b/aws/sdk/integration-tests/webassembly/src/list_objects.rs index ec577ac879..948e8e861a 100644 --- a/aws/sdk/integration-tests/webassembly/src/list_objects.rs +++ b/aws/sdk/integration-tests/webassembly/src/list_objects.rs @@ -4,6 +4,7 @@ */ use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output; +use aws_smithy_wasm::wasi::wasi_http_client; async fn s3_list_objects() -> ListObjectsV2Output { use crate::default_config::get_default_config; @@ -16,11 +17,13 @@ async fn s3_list_objects() -> ListObjectsV2Output { .bucket("nara-national-archives-catalog") .delimiter("/") .prefix("authority-records/organization/") - .max_keys(5) - .customize() - .await - .unwrap(); - operation.send().await.unwrap() + .max_keys(5); + // .customize() + // .await + // .unwrap(); + let blah = operation.send().await; + println!("{blah:#?}"); + blah.expect("successful call") } #[tokio::test] diff --git a/aws/sdk/integration-tests/webassembly/src/wasi.rs b/aws/sdk/integration-tests/webassembly/src/wasi.rs index e85a7c07b0..fc131bc69b 100644 --- a/aws/sdk/integration-tests/webassembly/src/wasi.rs +++ b/aws/sdk/integration-tests/webassembly/src/wasi.rs @@ -8,14 +8,14 @@ wit_bindgen::generate!({ inline: " - package aws:component + package aws:component; interface run { - run: func() -> result + run: func() -> result; } world main { - export run + export run; } ", exports: { diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 30bea313e1..744c618575 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -121,7 +121,7 @@ ARG wasm_pack_version=0.11.0 RUN cargo install wasm-pack --locked --version ${wasm_pack_version} FROM install_rust AS wasmtime -ARG cargo_wasmtime_version=13.0.0 +ARG cargo_wasmtime_version=17.0.0 ARG rust_nightly_version RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} From f621e611cd77e6ba12de0ecc8eec854809aedbe5 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 13 Feb 2024 12:16:07 -0800 Subject: [PATCH 44/72] Updating wasi http to drop child resources --- rust-runtime/aws-smithy-wasm/src/wasi.rs | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index a9eb991f30..9235e69884 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -256,9 +256,21 @@ impl TryFrom for http::Response { let status = response.status(); - let body_stream = response - .consume() - .expect("Consume called more than once") + let headers = response.headers().entries(); + + let res_build = headers + .iter() + .fold(http::Response::builder().status(status), |rb, header| { + rb.header(header.0.clone(), header.1.clone()) + }); + + //The returned headers resource is immutable: set, append, and delete operations will + //fail with header-error.immutable. + drop(headers); + + let body_incoming = response.consume().expect("Consume called more than once"); + + let body_stream = body_incoming .stream() .expect("Stream accessed more than once"); @@ -269,13 +281,9 @@ impl TryFrom for http::Response { body.extend_from_slice(stream_bytes.as_slice()) } - let headers = response.headers().entries(); - - let res_build = headers - .into_iter() - .fold(http::Response::builder().status(status), |rb, header| { - rb.header(header.0, header.1) - }); + //The input-stream resource is a child: it must be dropped before the parent + //incoming-body is dropped, or consumed by incoming-body.finish. + drop(body_stream); let res = res_build .body(body.freeze()) From b102e2b7cd582ef3bd8f7b46d3815968da725b6c Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 13 Feb 2024 15:45:12 -0800 Subject: [PATCH 45/72] Basic test setup for wasi should work in CI Remove unnecessary instances of clone Updating cfg/target --- aws/sdk/integration-tests/.gitignore | 1 + aws/sdk/integration-tests/test.sh | 8 ++- .../integration-tests/webassembly/Cargo.toml | 8 +++ .../webassembly/src/default_config.rs | 5 +- .../integration-tests/webassembly/src/http.rs | 62 ------------------- .../integration-tests/webassembly/src/lib.rs | 2 - .../webassembly/src/list_objects.rs | 13 ++-- rust-runtime/aws-smithy-wasm/Cargo.toml | 4 +- rust-runtime/aws-smithy-wasm/README.md | 4 +- rust-runtime/aws-smithy-wasm/src/lib.rs | 2 +- rust-runtime/aws-smithy-wasm/src/wasi.rs | 15 +++-- tools/ci-build/Dockerfile | 6 ++ 12 files changed, 41 insertions(+), 89 deletions(-) delete mode 100644 aws/sdk/integration-tests/webassembly/src/http.rs diff --git a/aws/sdk/integration-tests/.gitignore b/aws/sdk/integration-tests/.gitignore index 1e7caa9ea8..5f5fe6daf2 100644 --- a/aws/sdk/integration-tests/.gitignore +++ b/aws/sdk/integration-tests/.gitignore @@ -1,2 +1,3 @@ Cargo.lock target/ +webassembly/src/bindings.rs diff --git a/aws/sdk/integration-tests/test.sh b/aws/sdk/integration-tests/test.sh index d26a34a116..664e4427a2 100755 --- a/aws/sdk/integration-tests/test.sh +++ b/aws/sdk/integration-tests/test.sh @@ -11,6 +11,12 @@ for f in *; do echo echo "Testing ${f}..." echo "###############" - cargo test --manifest-path "${f}/Cargo.toml" --all-features + if [ "$f" != "webassembly" ]; then + cargo test --manifest-path "${f}/Cargo.toml" --all-features + else + # The webassembly tests use a custom runner set in config.toml that + # is not picked up when running the tests outside of the package + cd webassembly && cargo component test && cd .. + fi fi done diff --git a/aws/sdk/integration-tests/webassembly/Cargo.toml b/aws/sdk/integration-tests/webassembly/Cargo.toml index 15418e204f..66c06fd06e 100644 --- a/aws/sdk/integration-tests/webassembly/Cargo.toml +++ b/aws/sdk/integration-tests/webassembly/Cargo.toml @@ -37,3 +37,11 @@ tokio = { version = "1.32.0", features = ["macros", "rt"] } [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } + + +[lib] +crate-type = ["cdylib"] + +# metadata used by cargo-component to identify which wit world to embed in the binary +[package.metadata.component] +package = "aws:component" diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs index 63528829ad..005ffe967f 100644 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ b/aws/sdk/integration-tests/webassembly/src/default_config.rs @@ -3,12 +3,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -use crate::http::WasmHttpConnector; use aws_config::retry::RetryConfig; use aws_smithy_types::timeout::TimeoutConfig; use aws_smithy_wasm::wasi::wasi_http_client; -pub(crate) async fn get_default_config() -> aws_config::SdkConfig { +pub(crate) async fn get_default_wasi_config() -> aws_config::SdkConfig { aws_config::from_env() .region("us-east-2") .timeout_config(TimeoutConfig::disabled()) @@ -21,7 +20,7 @@ pub(crate) async fn get_default_config() -> aws_config::SdkConfig { #[tokio::test] pub async fn test_default_config() { - let shared_config = get_default_config().await; + let shared_config = get_default_wasi_config().await; let client = aws_sdk_s3::Client::new(&shared_config); assert_eq!(client.config().region().unwrap().to_string(), "us-east-2") } diff --git a/aws/sdk/integration-tests/webassembly/src/http.rs b/aws/sdk/integration-tests/webassembly/src/http.rs deleted file mode 100644 index 19354d0606..0000000000 --- a/aws/sdk/integration-tests/webassembly/src/http.rs +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -use aws_smithy_runtime_api::client::http::{ - HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, -}; -use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; -use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; -use aws_smithy_runtime_api::shared::IntoShared; -use aws_smithy_types::body::SdkBody; - -pub(crate) fn make_request(_req: HttpRequest) -> Result { - // Consumers here would pass the HTTP request to - // the Wasm host in order to get the response back - let body = " - - - - 2023-01-23T11:59:03.575496Z - doc-example-bucket - - - 2023-01-23T23:32:13.125238Z - doc-example-bucket2 - - - - account-name - a3a42310-42d0-46d1-9745-0cee9f4fb851 - - "; - Ok(HttpResponse::try_from(http::Response::new(SdkBody::from(body))).unwrap()) -} - -#[derive(Default, Debug, Clone)] -pub(crate) struct WasmHttpConnector; -impl WasmHttpConnector { - pub fn new() -> Self { - Self - } -} - -impl HttpConnector for WasmHttpConnector { - fn call(&self, request: HttpRequest) -> HttpConnectorFuture { - println!("Adapter: sending request..."); - let res = make_request(request).unwrap(); - println!("{:?}", res); - HttpConnectorFuture::new(async move { Ok(res) }) - } -} - -impl HttpClient for WasmHttpConnector { - fn http_connector( - &self, - _settings: &HttpConnectorSettings, - _components: &RuntimeComponents, - ) -> SharedHttpConnector { - self.clone().into_shared() - } -} diff --git a/aws/sdk/integration-tests/webassembly/src/lib.rs b/aws/sdk/integration-tests/webassembly/src/lib.rs index 41cd16a3c6..27c892cbc1 100644 --- a/aws/sdk/integration-tests/webassembly/src/lib.rs +++ b/aws/sdk/integration-tests/webassembly/src/lib.rs @@ -8,8 +8,6 @@ #[cfg(target_family = "wasm")] mod default_config; #[cfg(target_family = "wasm")] -mod http; -#[cfg(target_family = "wasm")] mod list_objects; #[cfg(all(target_family = "wasm", target_os = "wasi"))] mod wasi; diff --git a/aws/sdk/integration-tests/webassembly/src/list_objects.rs b/aws/sdk/integration-tests/webassembly/src/list_objects.rs index 948e8e861a..dd55921673 100644 --- a/aws/sdk/integration-tests/webassembly/src/list_objects.rs +++ b/aws/sdk/integration-tests/webassembly/src/list_objects.rs @@ -4,13 +4,12 @@ */ use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output; -use aws_smithy_wasm::wasi::wasi_http_client; async fn s3_list_objects() -> ListObjectsV2Output { - use crate::default_config::get_default_config; + use crate::default_config::get_default_wasi_config; use aws_sdk_s3::Client; - let shared_config = get_default_config().await; + let shared_config = get_default_wasi_config().await; let client = Client::new(&shared_config); let operation = client .list_objects_v2() @@ -18,12 +17,8 @@ async fn s3_list_objects() -> ListObjectsV2Output { .delimiter("/") .prefix("authority-records/organization/") .max_keys(5); - // .customize() - // .await - // .unwrap(); - let blah = operation.send().await; - println!("{blah:#?}"); - blah.expect("successful call") + + operation.send().await.expect("successful call") } #[tokio::test] diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index e97d09cea2..a261e2c1ad 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -10,8 +10,8 @@ edition = "2021" license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" -# [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] -[dependencies] +[target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +# [dependencies] aws-smithy-runtime-api = { path = "../aws-smithy-runtime-api", features = ["http-1x"]} aws-smithy-http = { path = "../aws-smithy-http" } aws-smithy-types = { path = "../aws-smithy-types" } diff --git a/rust-runtime/aws-smithy-wasm/README.md b/rust-runtime/aws-smithy-wasm/README.md index ec5b5df079..0b496a2df5 100644 --- a/rust-runtime/aws-smithy-wasm/README.md +++ b/rust-runtime/aws-smithy-wasm/README.md @@ -1,7 +1,9 @@ # aws-smithy-wasm -WebAssembly related configuration for service clients generated by [smithy-rs](https://github.com/awslabs/smithy-rs). +WebAssembly and WASI related configuration for service clients generated by [smithy-rs](https://github.com/awslabs/smithy-rs). + This crate is part of the [AWS SDK for Rust](https://awslabs.github.io/aws-sdk-rust/) and the [smithy-rs](https://github.com/smithy-lang/smithy-rs) code generator. In most cases, it should not be used directly. + diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs index ff702663d7..4c423f3eb1 100644 --- a/rust-runtime/aws-smithy-wasm/src/lib.rs +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -16,5 +16,5 @@ //! Smithy WebAssembly -// #[cfg(all(target_family = "wasm", target_os = "wasi"))] +#[cfg(all(target_family = "wasm", target_os = "wasi"))] pub mod wasi; diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index 9235e69884..a17007a538 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -256,20 +256,21 @@ impl TryFrom for http::Response { let status = response.status(); + //This headers resource is a child: it must be dropped before the parent incoming-response is dropped. + //The drop happens via the consuming iterator used below let headers = response.headers().entries(); let res_build = headers - .iter() + .into_iter() .fold(http::Response::builder().status(status), |rb, header| { - rb.header(header.0.clone(), header.1.clone()) + rb.header(header.0, header.1) }); - //The returned headers resource is immutable: set, append, and delete operations will - //fail with header-error.immutable. - drop(headers); - let body_incoming = response.consume().expect("Consume called more than once"); + //The input-stream resource is a child: it must be dropped before the parent + //incoming-body is dropped, or consumed by incoming-body.finish. + //That drop is done explicitly below let body_stream = body_incoming .stream() .expect("Stream accessed more than once"); @@ -281,8 +282,6 @@ impl TryFrom for http::Response { body.extend_from_slice(stream_bytes.as_slice()) } - //The input-stream resource is a child: it must be dropped before the parent - //incoming-body is dropped, or consumed by incoming-body.finish. drop(body_stream); let res = res_build diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 744c618575..5eeeb17f66 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -125,6 +125,11 @@ ARG cargo_wasmtime_version=17.0.0 ARG rust_nightly_version RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} +FROM install_rust AS cargo_component +ARG cargo_component_version=0.7.1 +ARG rust_nightly_version +RUN cargo install cargo-component --version ${cargo_component_version} + FROM install_rust AS cargo_semver_checks ARG cargo_semver_checks_version=0.24.1 RUN cargo install cargo-semver-checks --locked --version ${cargo_semver_checks_version} @@ -175,6 +180,7 @@ COPY --chown=build:build --from=cargo_check_external_types /opt/cargo/bin/cargo- COPY --chown=build:build --from=maturin /opt/cargo/bin/maturin /opt/cargo/bin/maturin COPY --chown=build:build --from=wasm_pack /opt/cargo/bin/wasm-pack /opt/cargo/bin/wasm-pack COPY --chown=build:build --from=wasmtime /opt/cargo/bin/wasmtime /opt/cargo/bin/wasmtime +COPY --chown=build:build --from=cargo_component /opt/cargo/bin/cargo-component /opt/cargo/bin/cargo-component COPY --chown=build:build --from=install_rust /opt/rustup /opt/rustup COPY --chown=build:build --from=cargo_semver_checks /opt/cargo/bin/cargo-semver-checks /opt/cargo/bin/cargo-semver-checks COPY --chown=build:build --from=cargo_mdbook /opt/cargo/bin/mdbook /opt/cargo/bin/mdbook From bb8bb4dc8bfcc8501495177c61f6fe6fbe8d64ba Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 13 Feb 2024 16:30:57 -0800 Subject: [PATCH 46/72] Update CHANGELOG.next.toml --- CHANGELOG.next.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index d7d27b7e25..13affdfdc3 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -46,3 +46,9 @@ message = "Added impl `Display` to Enums." references = ["smithy-rs#3336", "smithy-rs#3391"] meta = { "breaking" = false, "tada" = false, "bug" = false} author = "iampkmone" + +[[smithy-rs]] +message = "Add aws-smithy-wasm crate to enable SDK use in wasi compliant environments" +references = ["smithy-rs#2087", "smithy-rs#2520"] +meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client"} +author = "landonxjames" From d0feecbc69d0a17c7236040c6e53bcef0100ff4e Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 13 Feb 2024 16:53:45 -0800 Subject: [PATCH 47/72] Update cargo-component to install nightly for MSRV --- tools/ci-build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 5eeeb17f66..2fd8c65fbd 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -128,7 +128,7 @@ RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-mo FROM install_rust AS cargo_component ARG cargo_component_version=0.7.1 ARG rust_nightly_version -RUN cargo install cargo-component --version ${cargo_component_version} +RUN cargo +${rust_nightly_version} install cargo-component --version ${cargo_component_version} FROM install_rust AS cargo_semver_checks ARG cargo_semver_checks_version=0.24.1 From 9a653b388bdc4a319850fa3b4ea9551c83c73e95 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 13 Feb 2024 18:07:39 -0800 Subject: [PATCH 48/72] Lint updates --- CHANGELOG.next.toml | 2 +- rust-runtime/aws-smithy-wasm/README.md | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index f08a282a36..214219c66d 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -125,6 +125,6 @@ author = "rcoh" [[smithy-rs]] message = "Add aws-smithy-wasm crate to enable SDK use in wasi compliant environments" -references = ["smithy-rs#2087", "smithy-rs#2520", "smithy-rs#3409] +references = ["smithy-rs#2087", "smithy-rs#2520", "smithy-rs#3409"] meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client"} author = "landonxjames" diff --git a/rust-runtime/aws-smithy-wasm/README.md b/rust-runtime/aws-smithy-wasm/README.md index 0b496a2df5..5190145609 100644 --- a/rust-runtime/aws-smithy-wasm/README.md +++ b/rust-runtime/aws-smithy-wasm/README.md @@ -3,7 +3,5 @@ WebAssembly and WASI related configuration for service clients generated by [smithy-rs](https://github.com/awslabs/smithy-rs). - This crate is part of the [AWS SDK for Rust](https://awslabs.github.io/aws-sdk-rust/) and the [smithy-rs](https://github.com/smithy-lang/smithy-rs) code generator. In most cases, it should not be used directly. - From 27012977a3e7bc5241b70be57fdcc86bd1f15eeb Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 13 Feb 2024 19:15:34 -0800 Subject: [PATCH 49/72] CI updates --- aws/sdk/integration-tests/webassembly/src/list_objects.rs | 2 +- tools/ci-scripts/check-aws-sdk-standalone-integration-tests | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/sdk/integration-tests/webassembly/src/list_objects.rs b/aws/sdk/integration-tests/webassembly/src/list_objects.rs index dd55921673..6da8bee1c5 100644 --- a/aws/sdk/integration-tests/webassembly/src/list_objects.rs +++ b/aws/sdk/integration-tests/webassembly/src/list_objects.rs @@ -17,7 +17,7 @@ async fn s3_list_objects() -> ListObjectsV2Output { .delimiter("/") .prefix("authority-records/organization/") .max_keys(5); - + operation.send().await.expect("successful call") } diff --git a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests index beb03c8f86..d10a172297 100755 --- a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests +++ b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests @@ -43,7 +43,7 @@ cargo check --tests --all-features # Running WebAssembly (WASI) specific integration tests pushd "${tmp_dir}/aws/sdk/integration-tests/webassembly" &>/dev/null cargo check --tests -RUST_TEST_NOCAPTURE=1 cargo test +RUST_TEST_NOCAPTURE=1 cargo component test popd popd From 02632f2fa0863c64a6d4af0791cdf07b28a2e40e Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 14 Feb 2024 12:50:32 -0800 Subject: [PATCH 50/72] Update CHANGELOG.next.toml Co-authored-by: John DiSanti --- CHANGELOG.next.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index 214219c66d..0e1f48a3fe 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -126,5 +126,5 @@ author = "rcoh" [[smithy-rs]] message = "Add aws-smithy-wasm crate to enable SDK use in wasi compliant environments" references = ["smithy-rs#2087", "smithy-rs#2520", "smithy-rs#3409"] -meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client"} +meta = { "breaking" = false, "tada" = true, "bug" = false, "target" = "client"} author = "landonxjames" From 1df0bf6158330e69c82071a89de732ec631c481b Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 14 Feb 2024 12:56:28 -0800 Subject: [PATCH 51/72] Update rust-runtime/aws-smithy-wasm/Cargo.toml Co-authored-by: John DiSanti --- rust-runtime/aws-smithy-wasm/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index a261e2c1ad..8e7fda03f9 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "aws-smithy-wasm" -version = "0.0.0-smithy-rs-head" +version = "0.1.0" authors = [ "AWS Rust SDK Team ", "Eduardo Rodrigues <16357187+eduardomourar@users.noreply.github.com>", From 23db87e3d752a4b7a2beb366a41d8b283787c55d Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 14 Feb 2024 12:56:47 -0800 Subject: [PATCH 52/72] Update rust-runtime/aws-smithy-wasm/Cargo.toml Co-authored-by: John DiSanti --- rust-runtime/aws-smithy-wasm/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index 8e7fda03f9..8c261890b1 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -11,7 +11,6 @@ license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] -# [dependencies] aws-smithy-runtime-api = { path = "../aws-smithy-runtime-api", features = ["http-1x"]} aws-smithy-http = { path = "../aws-smithy-http" } aws-smithy-types = { path = "../aws-smithy-types" } From b2134c726aa63452503502812aa9ec216b1c94cc Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 14 Feb 2024 12:57:43 -0800 Subject: [PATCH 53/72] Update rust-runtime/aws-smithy-wasm/src/wasi.rs Co-authored-by: John DiSanti --- rust-runtime/aws-smithy-wasm/src/wasi.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index a17007a538..2257f26b74 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -25,7 +25,7 @@ use wasi::http::{ types::{self as wasi_http, OutgoingBody, RequestOptions}, }; -/// Creates a connector function that can be used during instantiation of the client SDK +/// Creates a HTTP client that can be used during instantiation of the client SDK /// in order to route the HTTP requests through the WebAssembly host. The host must /// support the WASI HTTP proposal as defined in the Preview 2 specification. pub fn wasi_http_client() -> SharedHttpClient { From 100586527e660c1ec752b3ae3195997c001da664 Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 14 Feb 2024 12:57:56 -0800 Subject: [PATCH 54/72] Update rust-runtime/aws-smithy-wasm/src/wasi.rs Co-authored-by: John DiSanti --- rust-runtime/aws-smithy-wasm/src/wasi.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index 2257f26b74..44227fc379 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -39,7 +39,7 @@ pub struct WasiHttpClient { } impl WasiHttpClient { - /// Create a new Wasi HTTP client. + /// Create a new WASI HTTP client. pub fn new() -> Self { Default::default() } From 68ae686a9daa54644349d008dabfa23cffe775ff Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 14 Feb 2024 12:58:40 -0800 Subject: [PATCH 55/72] Apply suggestions from code review Co-authored-by: John DiSanti --- rust-runtime/aws-smithy-wasm/src/wasi.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index 44227fc379..eb511d01b3 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -100,18 +100,18 @@ impl HttpConnector for WasiHttpConnector { } } -/// Wasi http client containing the options to pass to the +/// WASI HTTP client containing the options to pass to the struct WasiDefaultClient { options: WasiRequestOptions, } impl WasiDefaultClient { - /// Create a new Wasi HTTP client. + /// Create a new WASI HTTP client. fn new(options: WasiRequestOptions) -> Self { Self { options } } - /// Make outgoing http request in a Wasi environment + /// Make outgoing HTTP request in a WASI environment fn handle(&self, req: http::Request) -> Result, ConnectorError> { let req = WasiRequest::try_from(req).expect("Converting http request"); From a38ef6ba579ea4dbcc018c2a125de7f8f7f2b8a9 Mon Sep 17 00:00:00 2001 From: Landon James Date: Wed, 14 Feb 2024 14:36:26 -0800 Subject: [PATCH 56/72] Updating build scripts with PR feedback --- aws/sdk/integration-tests/test.sh | 2 +- tools/ci-scripts/check-aws-sdk-standalone-integration-tests | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/aws/sdk/integration-tests/test.sh b/aws/sdk/integration-tests/test.sh index 664e4427a2..88026f6d7c 100755 --- a/aws/sdk/integration-tests/test.sh +++ b/aws/sdk/integration-tests/test.sh @@ -16,7 +16,7 @@ for f in *; do else # The webassembly tests use a custom runner set in config.toml that # is not picked up when running the tests outside of the package - cd webassembly && cargo component test && cd .. + cd webassembly && cargo component test --all-features --all-targets && cd .. fi fi done diff --git a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests index d10a172297..9ba41631dc 100755 --- a/tools/ci-scripts/check-aws-sdk-standalone-integration-tests +++ b/tools/ci-scripts/check-aws-sdk-standalone-integration-tests @@ -42,8 +42,7 @@ cargo check --tests --all-features # Running WebAssembly (WASI) specific integration tests pushd "${tmp_dir}/aws/sdk/integration-tests/webassembly" &>/dev/null -cargo check --tests -RUST_TEST_NOCAPTURE=1 cargo component test +cargo check --tests --all-features popd popd From b2ff222aaa85382b6a19fd7fd3f451fd6cdee186 Mon Sep 17 00:00:00 2001 From: Landon James Date: Thu, 15 Feb 2024 10:47:50 -0800 Subject: [PATCH 57/72] Updating error handling --- rust-runtime/aws-smithy-wasm/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs index 4c423f3eb1..ff702663d7 100644 --- a/rust-runtime/aws-smithy-wasm/src/lib.rs +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -16,5 +16,5 @@ //! Smithy WebAssembly -#[cfg(all(target_family = "wasm", target_os = "wasi"))] +// #[cfg(all(target_family = "wasm", target_os = "wasi"))] pub mod wasi; From bf14237bfb3f53f9756317aab33b3705b98daf9c Mon Sep 17 00:00:00 2001 From: Landon James Date: Thu, 15 Feb 2024 18:33:53 -0800 Subject: [PATCH 58/72] Update WASI HTTP client Added a Builder to allow for future configuration to be baked in in a backwards compatible manner. Also updated some of the error handling. Updated the associated tests to use the new Builder and to not send a network call. Sending the actual network call will be moved to the canary in another commit. Undoing some autoformatting in the Dockerfile Lint fixes --- .../webassembly/src/default_config.rs | 26 ---- .../webassembly/src/http_client.rs | 54 +++++++ .../integration-tests/webassembly/src/lib.rs | 4 +- .../webassembly/src/list_objects.rs | 29 ---- rust-runtime/aws-smithy-wasm/src/lib.rs | 2 +- rust-runtime/aws-smithy-wasm/src/wasi.rs | 147 +++++++++++------- tools/ci-build/Dockerfile | 10 +- 7 files changed, 150 insertions(+), 122 deletions(-) delete mode 100644 aws/sdk/integration-tests/webassembly/src/default_config.rs create mode 100644 aws/sdk/integration-tests/webassembly/src/http_client.rs delete mode 100644 aws/sdk/integration-tests/webassembly/src/list_objects.rs diff --git a/aws/sdk/integration-tests/webassembly/src/default_config.rs b/aws/sdk/integration-tests/webassembly/src/default_config.rs deleted file mode 100644 index 005ffe967f..0000000000 --- a/aws/sdk/integration-tests/webassembly/src/default_config.rs +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -use aws_config::retry::RetryConfig; -use aws_smithy_types::timeout::TimeoutConfig; -use aws_smithy_wasm::wasi::wasi_http_client; - -pub(crate) async fn get_default_wasi_config() -> aws_config::SdkConfig { - aws_config::from_env() - .region("us-east-2") - .timeout_config(TimeoutConfig::disabled()) - .retry_config(RetryConfig::disabled()) - .no_credentials() - .http_client(wasi_http_client()) - .load() - .await -} - -#[tokio::test] -pub async fn test_default_config() { - let shared_config = get_default_wasi_config().await; - let client = aws_sdk_s3::Client::new(&shared_config); - assert_eq!(client.config().region().unwrap().to_string(), "us-east-2") -} diff --git a/aws/sdk/integration-tests/webassembly/src/http_client.rs b/aws/sdk/integration-tests/webassembly/src/http_client.rs new file mode 100644 index 0000000000..5737b0fa3a --- /dev/null +++ b/aws/sdk/integration-tests/webassembly/src/http_client.rs @@ -0,0 +1,54 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_config::retry::RetryConfig; +use aws_sdk_s3::operation::list_objects_v2::builders::ListObjectsV2FluentBuilder; +use aws_sdk_s3::Client; +use aws_smithy_types::timeout::TimeoutConfig; +use aws_smithy_wasm::wasi::WasiHttpClientBuilder; + +pub(crate) async fn get_default_wasi_config() -> aws_config::SdkConfig { + let http_client = WasiHttpClientBuilder::new().build(); + aws_config::from_env() + .region("us-east-2") + .timeout_config(TimeoutConfig::disabled()) + .retry_config(RetryConfig::disabled()) + .no_credentials() + .http_client(http_client) + .load() + .await +} + +#[tokio::test] +pub async fn test_default_config() { + let shared_config = get_default_wasi_config().await; + let client = aws_sdk_s3::Client::new(&shared_config); + assert_eq!(client.config().region().unwrap().to_string(), "us-east-2") +} + +async fn s3_list_objects_operation() -> ListObjectsV2FluentBuilder { + let shared_config = get_default_wasi_config().await; + let client = Client::new(&shared_config); + let operation = client + .list_objects_v2() + .bucket("nara-national-archives-catalog") + .delimiter("/") + .prefix("authority-records/organization/") + .max_keys(5); + + operation +} + +// Test constructing an operation using an SdkConfig with a WASI http client +// We do not send the request to keep these tests sandboxable, a full test of +// the client is in the SDK canary. +#[tokio::test] +pub async fn test_operation_construction() { + let operation = s3_list_objects_operation().await; + assert_eq!( + operation.get_bucket(), + &Some("nara-national-archives-catalog".to_string()) + ); +} diff --git a/aws/sdk/integration-tests/webassembly/src/lib.rs b/aws/sdk/integration-tests/webassembly/src/lib.rs index 27c892cbc1..3c29305c06 100644 --- a/aws/sdk/integration-tests/webassembly/src/lib.rs +++ b/aws/sdk/integration-tests/webassembly/src/lib.rs @@ -6,8 +6,6 @@ #![allow(dead_code)] #[cfg(target_family = "wasm")] -mod default_config; -#[cfg(target_family = "wasm")] -mod list_objects; +mod http_client; #[cfg(all(target_family = "wasm", target_os = "wasi"))] mod wasi; diff --git a/aws/sdk/integration-tests/webassembly/src/list_objects.rs b/aws/sdk/integration-tests/webassembly/src/list_objects.rs deleted file mode 100644 index 6da8bee1c5..0000000000 --- a/aws/sdk/integration-tests/webassembly/src/list_objects.rs +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output; - -async fn s3_list_objects() -> ListObjectsV2Output { - use crate::default_config::get_default_wasi_config; - use aws_sdk_s3::Client; - - let shared_config = get_default_wasi_config().await; - let client = Client::new(&shared_config); - let operation = client - .list_objects_v2() - .bucket("nara-national-archives-catalog") - .delimiter("/") - .prefix("authority-records/organization/") - .max_keys(5); - - operation.send().await.expect("successful call") -} - -#[tokio::test] -pub async fn test_s3_list_objects() { - let result = s3_list_objects().await; - let objects = result.contents(); - assert!(objects.len() > 1); -} diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs index ff702663d7..4c423f3eb1 100644 --- a/rust-runtime/aws-smithy-wasm/src/lib.rs +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -16,5 +16,5 @@ //! Smithy WebAssembly -// #[cfg(all(target_family = "wasm", target_os = "wasi"))] +#[cfg(all(target_family = "wasm", target_os = "wasi"))] pub mod wasi; diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index eb511d01b3..e8bc4ddf97 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -//! HTTP WASI Adapter +//! WASI HTTP Adapter use aws_smithy_http::header::ParseError; use aws_smithy_runtime_api::{ client::{ @@ -25,54 +25,54 @@ use wasi::http::{ types::{self as wasi_http, OutgoingBody, RequestOptions}, }; -/// Creates a HTTP client that can be used during instantiation of the client SDK -/// in order to route the HTTP requests through the WebAssembly host. The host must -/// support the WASI HTTP proposal as defined in the Preview 2 specification. -pub fn wasi_http_client() -> SharedHttpClient { - WasiHttpClient::new().into_shared() -} - -/// HTTP client used in WASI environment -#[derive(Debug, Clone)] -pub struct WasiHttpClient { - connector: SharedHttpConnector, -} +/// Builder for [`WasiHttpClient`]. Currently empty, but allows for future +/// config options to be added in a backwards compatible manner. +#[derive(Default, Debug)] +pub struct WasiHttpClientBuilder {} -impl WasiHttpClient { - /// Create a new WASI HTTP client. +impl WasiHttpClientBuilder { + /// Creates a new builder. pub fn new() -> Self { Default::default() } -} -impl Default for WasiHttpClient { - fn default() -> Self { - Self { - connector: WasiHttpConnector.into_shared(), - } + /// Builds the [`WasiHttpClient`]. + pub fn build(self) -> SharedHttpClient { + let client = WasiHttpClient {}; + client.into_shared() } } +/// An HTTP client that can be used during instantiation of the client SDK in +/// order to route the HTTP requests through the WebAssembly host. The host must +/// support the WASI HTTP proposal as defined in the Preview 2 specification. +#[derive(Debug, Clone)] +pub struct WasiHttpClient {} + impl HttpClient for WasiHttpClient { fn http_connector( &self, - _settings: &HttpConnectorSettings, + settings: &HttpConnectorSettings, _components: &RuntimeComponents, ) -> SharedHttpConnector { - self.connector.clone() + let options = WasiRequestOptions::from(settings); + let connector = WasiHttpConnector { options }; + + connector.into_shared() } } /// HTTP connector used in WASI environment -#[non_exhaustive] -#[derive(Debug)] -pub struct WasiHttpConnector; +#[derive(Debug, Clone)] +pub struct WasiHttpConnector { + options: WasiRequestOptions, +} impl HttpConnector for WasiHttpConnector { fn call(&self, request: HttpRequest) -> HttpConnectorFuture { tracing::trace!("WasiHttpConnector: sending request {request:?}"); - // TODO(wasi): add connect/read timeouts - let client = WasiDefaultClient::new(WasiRequestOptions(None)); + + let client = WasiDefaultClient::new(self.options.clone()); let http_req = request.try_into_http1x().expect("Http request invalid"); let converted_req = http_req.map(|body| match body.bytes() { Some(value) => Bytes::copy_from_slice(value), @@ -100,7 +100,7 @@ impl HttpConnector for WasiHttpConnector { } } -/// WASI HTTP client containing the options to pass to the +/// WASI HTTP client containing the options passed to the outgoing_handler struct WasiDefaultClient { options: WasiRequestOptions, } @@ -113,9 +113,11 @@ impl WasiDefaultClient { /// Make outgoing HTTP request in a WASI environment fn handle(&self, req: http::Request) -> Result, ConnectorError> { - let req = WasiRequest::try_from(req).expect("Converting http request"); + let req = + WasiRequest::try_from(req).map_err(|err| ConnectorError::other(err.into(), None))?; - let res = outgoing_handler::handle(req.0, self.options.clone().0).expect("Http response"); + let res = outgoing_handler::handle(req.0, self.options.clone().0) + .map_err(|err| ConnectorError::other(err.into(), None))?; // Right now only synchronous calls can be made through WASI, so we subscribe and // block on the FutureIncomingResponse @@ -134,19 +136,45 @@ impl WasiDefaultClient { .map_err(|err| ConnectorError::other(err.into(), None))?; let response = http::Response::try_from(WasiResponse(incoming_res)) - .expect("Converting to http response"); + .map_err(|err| ConnectorError::other(err.into(), None))?; Ok(response) } } -/// Wrapper for the Wasi RequestOptions type to allow us to impl Clone +/// Wrapper for the WASI RequestOptions type to allow us to impl Clone +#[derive(Debug)] struct WasiRequestOptions(Option); -//The Wasi RequestOptions type doesn't impl copy or clone but the outgoing_handler::handle method +impl From<&HttpConnectorSettings> for WasiRequestOptions { + fn from(value: &HttpConnectorSettings) -> Self { + //The WASI Duration is nanoseconds represented as u64 + let connect_timeout = value + .connect_timeout() + .map(|dur| u64::try_from(dur.as_nanos()).unwrap_or(u64::MAX)); + let read_timeout = value + .read_timeout() + .map(|dur| u64::try_from(dur.as_nanos()).unwrap_or(u64::MAX)); + + //Note: unable to find any documentation about what timeout values are not supported + //so not sure under what circumstances these set operations would actually fail + let wasi_http_opts = wasi_http::RequestOptions::new(); + wasi_http_opts + .set_connect_timeout(connect_timeout) + .expect("Connect timeout not supported"); + wasi_http_opts + .set_first_byte_timeout(read_timeout) + .expect("Read timeout not supported"); + + WasiRequestOptions(Some(wasi_http_opts)) + } +} +//The WASI RequestOptions type doesn't impl copy or clone but the outgoing_handler::handle method //takes ownership, so we impl it on this wrapper type impl Clone for WasiRequestOptions { fn clone(&self) -> Self { + //Note none of the expects here should ever trigger since all of the values passed in are from + //the existing RequestOptions that is being cloned and should be valid let new_opts = if let Some(opts) = &self.0 { let new_opts = RequestOptions::new(); new_opts @@ -168,25 +196,18 @@ impl Clone for WasiRequestOptions { } } -/// Wrapper to allow converting between http Request types and Wasi Request types +/// Wrapper to allow converting between HTTP Request types and WASI Request types #[derive(Debug)] struct WasiRequest(outgoing_handler::OutgoingRequest); -impl WasiRequest { - fn new(req: outgoing_handler::OutgoingRequest) -> Self { - Self(req) - } -} - impl TryFrom> for WasiRequest { type Error = ParseError; fn try_from(value: http::Request) -> Result { let (parts, body) = value.into_parts(); - let method = - WasiMethod::try_from(parts.method).map_err(|err| ParseError::new(err.to_string()))?; + let method = WasiMethod::try_from(parts.method)?; let path_with_query = parts.uri.path_and_query().map(|path| path.as_str()); - let headers = WasiHeaders::from(parts.headers); + let headers = WasiHeaders::try_from(parts.headers)?; let scheme = match parts.uri.scheme_str().unwrap_or("") { "http" => Some(&wasi_http::Scheme::Http), "https" => Some(&wasi_http::Scheme::Https), @@ -195,12 +216,18 @@ impl TryFrom> for WasiRequest { let authority = parts.uri.authority().map(|auth| auth.as_str()); let request = wasi_http::OutgoingRequest::new(headers.0); - request.set_scheme(scheme).expect("Set scheme"); - request.set_method(&method.0).expect("Set method"); + request + .set_scheme(scheme) + .map_err(|_| ParseError::new("Failed to set HTTP scheme"))?; + request + .set_method(&method.0) + .map_err(|_| ParseError::new("Failed to set HTTP method"))?; request .set_path_with_query(path_with_query) - .expect("Set path with query"); - request.set_authority(authority).expect("Set authority"); + .map_err(|_| ParseError::new("Failed to set HTTP path"))?; + request + .set_authority(authority) + .map_err(|_| ParseError::new("Failed to set HTTP authority"))?; let request_body = request.body().expect("Body accessed more than once"); @@ -210,20 +237,21 @@ impl TryFrom> for WasiRequest { request_stream .blocking_write_and_flush(&body) - .expect("Failed to write body"); + .map_err(|_| ParseError::new("Failed to write HTTP body"))?; //The OutputStream is a child resource: it must be dropped //before the parent OutgoingBody resource is dropped (or finished), //otherwise the OutgoingBody drop or finish will trap. drop(request_stream); - OutgoingBody::finish(request_body, None).expect("Http body finished"); + OutgoingBody::finish(request_body, None) + .map_err(|_| ParseError::new("Failed to finalize HTTP body"))?; - Ok(WasiRequest::new(request)) + Ok(WasiRequest(request)) } } -/// Wrapper to allow converting between http Methods and Wasi Methods +/// Wrapper to allow converting between HTTP Methods and WASI Methods struct WasiMethod(wasi_http::Method); impl TryFrom for WasiMethod { @@ -245,7 +273,7 @@ impl TryFrom for WasiMethod { } } -/// Wrapper to allow converting between http Response types and Wasi Response types +/// Wrapper to allow converting between HTTP Response types and WASI Response types struct WasiResponse(wasi_http::IncomingResponse); impl TryFrom for http::Response { @@ -292,11 +320,13 @@ impl TryFrom for http::Response { } } -/// Wrapper to allow converting between http headers and Wasi headers +/// Wrapper to allow converting between HTTP headers and WASI headers struct WasiHeaders(wasi_http::Fields); -impl From for WasiHeaders { - fn from(headers: http::HeaderMap) -> Self { +impl TryFrom for WasiHeaders { + type Error = ParseError; + + fn try_from(headers: http::HeaderMap) -> Result { let entries = headers .iter() .map(|(name, value)| { @@ -307,8 +337,9 @@ impl From for WasiHeaders { }) .collect::>(); - let fields = wasi_http::Fields::from_list(&entries).expect("Invalid http headers."); + let fields = wasi_http::Fields::from_list(&entries) + .map_err(|err| ParseError::new(err.to_string()))?; - Self(fields) + Ok(Self(fields)) } } diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 2fd8c65fbd..347b04ed1a 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -49,11 +49,11 @@ RUN yum -y install --allowerasing \ yum clean all RUN set -eux; \ if [[ "$(uname -m)" == "aarch64" || "$(uname -m)" == "arm64" ]]; then \ - curl https://static.rust-lang.org/rustup/archive/1.24.3/aarch64-unknown-linux-gnu/rustup-init --output rustup-init; \ - echo "32a1532f7cef072a667bac53f1a5542c99666c4071af0c9549795bbdb2069ec1 rustup-init" | sha256sum --check; \ + curl https://static.rust-lang.org/rustup/archive/1.24.3/aarch64-unknown-linux-gnu/rustup-init --output rustup-init; \ + echo "32a1532f7cef072a667bac53f1a5542c99666c4071af0c9549795bbdb2069ec1 rustup-init" | sha256sum --check; \ else \ - curl https://static.rust-lang.org/rustup/archive/1.24.3/x86_64-unknown-linux-gnu/rustup-init --output rustup-init; \ - echo "3dc5ef50861ee18657f9db2eeb7392f9c2a6c95c90ab41e45ab4ca71476b4338 rustup-init" | sha256sum --check; \ + curl https://static.rust-lang.org/rustup/archive/1.24.3/x86_64-unknown-linux-gnu/rustup-init --output rustup-init; \ + echo "3dc5ef50861ee18657f9db2eeb7392f9c2a6c95c90ab41e45ab4ca71476b4338 rustup-init" | sha256sum --check; \ fi; \ chmod +x rustup-init; \ ./rustup-init -y --no-modify-path --profile minimal --default-toolchain ${rust_stable_version}; \ @@ -128,7 +128,7 @@ RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-mo FROM install_rust AS cargo_component ARG cargo_component_version=0.7.1 ARG rust_nightly_version -RUN cargo +${rust_nightly_version} install cargo-component --version ${cargo_component_version} +RUN cargo +${rust_nightly_version} install cargo-component --locked --version ${cargo_component_version} FROM install_rust AS cargo_semver_checks ARG cargo_semver_checks_version=0.24.1 From cd091fb592158ceadfc9e0281a8794e9421c0bbc Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 19 Feb 2024 11:44:08 -0800 Subject: [PATCH 59/72] First pass at a wasm canary It compiles and the wasm runs, although unsuccessfully due to some missing WASI imports. Still trying to figure that out. --- tools/ci-cdk/canary-lambda/src/latest.rs | 1 + .../canary-lambda/src/latest/wasm_canary.rs | 74 + .../ci-cdk/canary-runner/src/build_bundle.rs | 42 + tools/ci-cdk/canary-wasm/Cargo.lock | 1283 +++++++++++++++++ tools/ci-cdk/canary-wasm/Cargo.toml | 24 + tools/ci-cdk/canary-wasm/src/bindings.rs | 11 + tools/ci-cdk/canary-wasm/src/lib.rs | 57 + 7 files changed, 1492 insertions(+) create mode 100644 tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs create mode 100644 tools/ci-cdk/canary-wasm/Cargo.lock create mode 100644 tools/ci-cdk/canary-wasm/Cargo.toml create mode 100644 tools/ci-cdk/canary-wasm/src/bindings.rs create mode 100644 tools/ci-cdk/canary-wasm/src/lib.rs diff --git a/tools/ci-cdk/canary-lambda/src/latest.rs b/tools/ci-cdk/canary-lambda/src/latest.rs index 238c361166..f034a5578a 100644 --- a/tools/ci-cdk/canary-lambda/src/latest.rs +++ b/tools/ci-cdk/canary-lambda/src/latest.rs @@ -6,3 +6,4 @@ pub(crate) mod paginator_canary; pub(crate) mod s3_canary; pub(crate) mod transcribe_canary; +pub(crate) mod wasm_canary; diff --git a/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs new file mode 100644 index 0000000000..91477adc7f --- /dev/null +++ b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs @@ -0,0 +1,74 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use crate::{mk_canary, CanaryEnv}; + +use wasmtime::component::{bindgen, Component, Linker}; +use wasmtime::{Config, Engine, Store}; +use wasmtime_wasi::sync::WasiCtxBuilder; + +// mk_canary!("wasm", |sdk_config: &SdkConfig, env: &CanaryEnv| { +// wasm_canary() +// }); + +//This macro creates bindings to call the wasm functions in Rust +//TODO: this should point to a wit file in canary-wasm +bindgen!({ + inline: " + package aws:component; + + interface canary-interface { + run-canary: func() -> result; + } + + world canary-world { + export canary-interface; + } +", +}); + +pub async fn wasm_canary() -> anyhow::Result { + let cur_dir = std::env::current_dir().expect("Current dir"); + println!("CURRENT DIR: {cur_dir:#?}"); + + // Create a Wasmtime Engine configured to run Components + let mut engine_config = wasmtime::Config::new(); + engine_config.wasm_component_model(true); + let engine = Engine::new(&engine_config).expect("Failed to create wasm engine"); + + // Create our component from the wasm file + let component = Component::from_file( + &engine, + cur_dir.join("aws_sdk_rust_lambda_canary_wasm.wasm"), + ) + .expect("Failed to load wasm"); + let linker = Linker::new(&engine); + // wasmtime_wasi::add_to_linker(&mut linker, |cx| cx)?; + + // Configure and create a `WasiCtx`, which WASI functions need access to + // through the host state of the store (which in this case is the host state + // of the store) + let wasi_ctx = WasiCtxBuilder::new().inherit_stdio().build(); + let mut store = Store::new(&engine, wasi_ctx); + + // Instantiate our module with the bindgen! bindings + let (bindings, _) = CanaryWorld::instantiate(&mut store, &component, &linker)?; + + println!("CALLING WASM FUNC"); + let canary_interface = bindings.aws_component_canary_interface(); + let api_result = canary_interface + .call_run_canary(store)? + .map_err(|err| anyhow::Error::msg(err))?; + println!("{api_result}"); + Ok(api_result) +} + +// #[ignore] +#[cfg(test)] +#[tokio::test] +async fn test_wasm_canary() { + let res = wasm_canary().await; + println!("{res:#?}"); +} diff --git a/tools/ci-cdk/canary-runner/src/build_bundle.rs b/tools/ci-cdk/canary-runner/src/build_bundle.rs index eaea69835e..1077b356b7 100644 --- a/tools/ci-cdk/canary-runner/src/build_bundle.rs +++ b/tools/ci-cdk/canary-runner/src/build_bundle.rs @@ -55,6 +55,9 @@ tokio-stream = "0" tracing-texray = "0.1.1" reqwest = { version = "0.11.14", features = ["rustls-tls"], default-features = false } edit-distance = "2" +wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } +wasmtime = { version = "16.0.0", features = ["component-model"] } +wasmtime-wasi = "16.0.0" "#; const REQUIRED_SDK_CRATES: &[&str] = &[ @@ -62,6 +65,7 @@ const REQUIRED_SDK_CRATES: &[&str] = &[ "aws-sdk-s3", "aws-sdk-ec2", "aws-sdk-transcribestreaming", + "aws-smithy-wasm", ]; // The elements in this `Vec` should be sorted in an ascending order by the release date. @@ -258,6 +262,11 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { let crate_manifest_content = generate_crate_manifest(crate_source)?; fs::write(&manifest_path, crate_manifest_content).context("failed to write Cargo.toml")?; + let wasm_manifest_path = std::env::current_dir() + .expect("Current dir") + .join("../canary-wasm/Cargo.toml"); + + println!("MAINFEST PATH: {wasm_manifest_path:#?}"); if !opt.manifest_only { // Compile the canary Lambda let mut command = Command::new("cargo"); @@ -271,6 +280,16 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { } handle_failure("cargo build", &command.output()?)?; + // Compile the wasm canary to a .wasm binary + let mut wasm_command = Command::new("cargo"); + wasm_command + .arg("component") + .arg("build") + .arg("--release") + .arg("--manifest-path") + .arg(&wasm_manifest_path); + handle_failure("cargo build", &wasm_command.output()?)?; + // Bundle the Lambda let repository_root = find_git_repository_root("smithy-rs", canary_path)?; let target_path = { @@ -280,6 +299,14 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { } path.join("release") }; + let wasm_bin_path = { + repository_root + .join("tools") + .join("target") + .join("wasm32-wasi") + .join("release") + .join("aws_sdk_rust_lambda_canary_wasm.wasm") + }; let bin_path = target_path.join("bootstrap"); let bundle_path = target_path.join(name_bundle( &bin_path, @@ -289,6 +316,7 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { let zip_file = fs::File::create(&bundle_path).context(here!())?; let mut zip = zip::ZipWriter::new(zip_file); + //Write the canary bin to the zip zip.start_file( "bootstrap", zip::write::FileOptions::default().unix_permissions(0o755), @@ -296,6 +324,15 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { .context(here!())?; zip.write_all(&fs::read(&bin_path).context(here!("read target"))?) .context(here!())?; + + // Write the wasm bin to the zip + zip.start_file( + "aws_sdk_rust_lambda_canary_wasm.wasm", + zip::write::FileOptions::default().unix_permissions(0o755), + ) + .context(here!())?; + zip.write_all(&fs::read(&wasm_bin_path).context(here!("read wasm bin"))?) + .context(here!())?; zip.finish().context(here!())?; println!( @@ -453,10 +490,12 @@ tokio-stream = "0" tracing-texray = "0.1.1" reqwest = { version = "0.11.14", features = ["rustls-tls"], default-features = false } edit-distance = "2" +wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } aws-config = { path = "some/sdk/path/aws-config", features = ["behavior-version-latest"] } aws-sdk-s3 = { path = "some/sdk/path/s3" } aws-sdk-ec2 = { path = "some/sdk/path/ec2" } aws-sdk-transcribestreaming = { path = "some/sdk/path/transcribestreaming" } +aws-smithy-wasm = { path = "some/sdk/path/aws-smithy-wasm" } [features] latest = [] @@ -518,10 +557,12 @@ tokio-stream = "0" tracing-texray = "0.1.1" reqwest = { version = "0.11.14", features = ["rustls-tls"], default-features = false } edit-distance = "2" +wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } aws-config = { version = "0.46.0", features = ["behavior-version-latest"] } aws-sdk-s3 = "0.20.0" aws-sdk-ec2 = "0.19.0" aws-sdk-transcribestreaming = "0.16.0" +aws-smithy-wasm = "0.1.0" [features] latest = [] @@ -538,6 +579,7 @@ default = ["latest"] crate_version("aws-sdk-s3", "0.20.0"), crate_version("aws-sdk-ec2", "0.19.0"), crate_version("aws-sdk-transcribestreaming", "0.16.0"), + crate_version("aws-smithy-wasm", "0.1.0"), ] .into_iter() .collect(), diff --git a/tools/ci-cdk/canary-wasm/Cargo.lock b/tools/ci-cdk/canary-wasm/Cargo.lock new file mode 100644 index 0000000000..e50ad881ce --- /dev/null +++ b/tools/ci-cdk/canary-wasm/Cargo.lock @@ -0,0 +1,1283 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "anyhow" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "aws-config" +version = "1.1.5" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.11", + "hyper", + "time", + "tokio", + "tracing", +] + +[[package]] +name = "aws-credential-types" +version = "1.1.5" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.1.5" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.11", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-rust-lambda-canary-wasm" +version = "0.1.0" +dependencies = [ + "aws-config", + "aws-sdk-s3", + "aws-smithy-wasm", + "tokio", + "wit-bindgen", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.0.0-local" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http 0.2.11", + "http-body", + "once_cell", + "percent-encoding", + "regex-lite", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.0.0-local" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.11", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.1.5" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.11", + "http 1.0.0", + "once_cell", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.1.5" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.5" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.11", + "http-body", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.4" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.5" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.11", + "http-body", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.5" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.5" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.1.5" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.11", + "http-body", + "once_cell", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.1.5" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.11", + "http 1.0.0", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-types" +version = "1.1.5" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.11", + "http-body", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", +] + +[[package]] +name = "aws-smithy-wasm" +version = "0.60.5" +dependencies = [ + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "http 1.0.0", + "tower", + "tracing", + "wasi", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.5" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.1.5" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 0.2.11", + "rustc_version", + "tracing", +] + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32c" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89254598aa9b9fa608de44b3ae54c810f0f06d755e24c50177f1f8f31ff50ce2" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 0.2.11", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" +dependencies = [ + "equivalent", + "hashbrown", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex-lite" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" + +[[package]] +name = "semver" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" + +[[package]] +name = "serde" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + +[[package]] +name = "spdx" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bde1398b09b9f93fc2fc9b9da86e362693e999d3a54a8ac47a99a5a73f638b" +dependencies = [ + "smallvec", +] + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "2.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915aea9e586f80826ee59f8453c1101f9d1c4b3964cd2460185ee8e299ada496" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "time" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "uuid" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.12.0+wasi-0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "457dd7e321b36e7d9d275997825e17ff9dcba2f6c8c0ff656e2f46db6c8579ef" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-encoder" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad2b51884de9c7f4fe2fd1043fccb8dcad4b1e29558146ee57a144d15779f3f" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-encoder" +version = "0.41.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972f97a5d8318f908dded23594188a90bcd09365986b1163e66d70170e5287ae" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-metadata" +version = "0.10.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18ebaa7bd0f9e7a5e5dd29b9a998acf21c4abed74265524dd7e85934597bfb10" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder 0.41.2", + "wasmparser 0.121.2", +] + +[[package]] +name = "wasmparser" +version = "0.118.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f1154f1ab868e2a01d9834a805faca7bf8b50d041b4ca714d005d0dab1c50c" +dependencies = [ + "indexmap", + "semver", +] + +[[package]] +name = "wasmparser" +version = "0.121.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dbe55c8f9d0dbd25d9447a5a889ff90c0cc3feaa7395310d3d826b2c703eaab" +dependencies = [ + "bitflags", + "indexmap", + "semver", +] + +[[package]] +name = "wit-bindgen" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b76f1d099678b4f69402a421e888bbe71bf20320c2f3f3565d0e7484dbe5bc20" +dependencies = [ + "bitflags", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75d55e1a488af2981fb0edac80d8d20a51ac36897a1bdef4abde33c29c1b6d0d" +dependencies = [ + "anyhow", + "wit-component", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01ff9cae7bf5736750d94d91eb8a49f5e3a04aff1d1a3218287d9b2964510f8" +dependencies = [ + "anyhow", + "heck", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804a98e2538393d47aa7da65a7348116d6ff403b426665152b70a168c0146d49" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", + "wit-component", +] + +[[package]] +name = "wit-component" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a35a2a9992898c9d27f1664001860595a4bc99d32dd3599d547412e17d7e2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.38.1", + "wasm-metadata", + "wasmparser 0.118.2", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "316b36a9f0005f5aa4b03c39bc3728d045df136f8c13a73b7db4510dec725e08" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", +] + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/tools/ci-cdk/canary-wasm/Cargo.toml b/tools/ci-cdk/canary-wasm/Cargo.toml new file mode 100644 index 0000000000..cf9efed87d --- /dev/null +++ b/tools/ci-cdk/canary-wasm/Cargo.toml @@ -0,0 +1,24 @@ + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# IMPORTANT: Don't edit this file directly! Run `canary-runner build-bundle` to modify this file instead. +[package] +name = "aws-sdk-rust-lambda-canary-wasm" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[dependencies] +tokio = { version = "1.36.0", features = ["macros", "rt"] } +wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } +aws-config = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-config", default-features = false, features = ["behavior-version-latest"] } +aws-sdk-s3 = { path = "../../../aws/sdk/build/aws-sdk/sdk/s3", default-features = false } +aws-smithy-wasm = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-smithy-wasm" } + +[lib] +crate-type = ["cdylib"] + +# metadata used by cargo-component to identify which wit world to embed in the binary +[package.metadata.component] +package = "aws:component" diff --git a/tools/ci-cdk/canary-wasm/src/bindings.rs b/tools/ci-cdk/canary-wasm/src/bindings.rs new file mode 100644 index 0000000000..b190912a80 --- /dev/null +++ b/tools/ci-cdk/canary-wasm/src/bindings.rs @@ -0,0 +1,11 @@ +// Generated by `wit-bindgen` 0.16.0. DO NOT EDIT! + +#[cfg(target_arch = "wasm32")] +#[link_section = "component-type:aws-sdk-rust-lambda-canary-wasm"] +#[doc(hidden)] +pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 257] = [3, 0, 31, 97, 119, 115, 45, 115, 100, 107, 45, 114, 117, 115, 116, 45, 108, 97, 109, 98, 100, 97, 45, 99, 97, 110, 97, 114, 121, 45, 119, 97, 115, 109, 0, 97, 115, 109, 13, 0, 1, 0, 7, 84, 1, 65, 2, 1, 65, 0, 4, 1, 73, 99, 111, 109, 112, 111, 110, 101, 110, 116, 58, 97, 119, 115, 45, 115, 100, 107, 45, 114, 117, 115, 116, 45, 108, 97, 109, 98, 100, 97, 45, 99, 97, 110, 97, 114, 121, 45, 119, 97, 115, 109, 47, 97, 119, 115, 45, 115, 100, 107, 45, 114, 117, 115, 116, 45, 108, 97, 109, 98, 100, 97, 45, 99, 97, 110, 97, 114, 121, 45, 119, 97, 115, 109, 4, 0, 11, 37, 1, 0, 31, 97, 119, 115, 45, 115, 100, 107, 45, 114, 117, 115, 116, 45, 108, 97, 109, 98, 100, 97, 45, 99, 97, 110, 97, 114, 121, 45, 119, 97, 115, 109, 3, 0, 0, 0, 16, 12, 112, 97, 99, 107, 97, 103, 101, 45, 100, 111, 99, 115, 0, 123, 125, 0, 70, 9, 112, 114, 111, 100, 117, 99, 101, 114, 115, 1, 12, 112, 114, 111, 99, 101, 115, 115, 101, 100, 45, 98, 121, 2, 13, 119, 105, 116, 45, 99, 111, 109, 112, 111, 110, 101, 110, 116, 6, 48, 46, 49, 56, 46, 50, 16, 119, 105, 116, 45, 98, 105, 110, 100, 103, 101, 110, 45, 114, 117, 115, 116, 6, 48, 46, 49, 54, 46, 48]; + +#[inline(never)] +#[doc(hidden)] +#[cfg(target_arch = "wasm32")] +pub fn __link_section() {} diff --git a/tools/ci-cdk/canary-wasm/src/lib.rs b/tools/ci-cdk/canary-wasm/src/lib.rs new file mode 100644 index 0000000000..1a7a753bdb --- /dev/null +++ b/tools/ci-cdk/canary-wasm/src/lib.rs @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +use aws_sdk_s3 as s3; +use aws_smithy_wasm::wasi::WasiHttpClientBuilder; + +// Needed for WASI-compliant environment as it expects specific functions +// to be exported such as `cabi_realloc`, `_start`, etc. +wit_bindgen::generate!({ + inline: " + package aws:component; + + interface canary-interface { + run-canary: func() -> result; + } + + world canary-world { + export canary-interface; + } + ", + exports: { + "aws:component/canary-interface": Component + } +}); + +struct Component; + +impl exports::aws::component::canary_interface::Guest for Component { + fn run_canary() -> Result { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .expect("Failed to generate runtime"); + let res = rt.block_on(run_canary())?; + Ok(res) + } +} + +async fn run_canary() -> Result { + let http_client = WasiHttpClientBuilder::new().build(); + let config = aws_config::from_env().http_client(http_client).load().await; + + let client = s3::Client::new(&config); + let result = client + .list_objects_v2() + .bucket("nara-national-archives-catalog") + .delimiter("/") + .prefix("authority-records/organization/") + .max_keys(5) + .send() + .await + .expect("Failed to ListObjects"); + + println!("WASM CANARY RESULT: {result:#?}"); + Ok(format!("{result:?}")) +} From 42c3b4f1a48abf7125de59590b11a01203916c28 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 19 Feb 2024 15:08:19 -0800 Subject: [PATCH 60/72] Slightly farther in getting wasmtime to run --- .../canary-lambda/src/latest/wasm_canary.rs | 65 +++++++++++++++++-- .../ci-cdk/canary-runner/src/build_bundle.rs | 6 +- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs index 91477adc7f..33a1b1b1c1 100644 --- a/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs +++ b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs @@ -6,8 +6,9 @@ use crate::{mk_canary, CanaryEnv}; use wasmtime::component::{bindgen, Component, Linker}; -use wasmtime::{Config, Engine, Store}; -use wasmtime_wasi::sync::WasiCtxBuilder; +use wasmtime::{Engine, Store}; +use wasmtime_wasi::preview2::WasiCtxBuilder; +use wasmtime_wasi::WasiCtx; // mk_canary!("wasm", |sdk_config: &SdkConfig, env: &CanaryEnv| { // wasm_canary() @@ -27,8 +28,47 @@ bindgen!({ export canary-interface; } ", +with: { + "wasi:io/error": wasmtime_wasi::preview2::bindings::io::error, + "wasi:io/streams": wasmtime_wasi::preview2::bindings::io::streams, + "wasi:io/poll": wasmtime_wasi::preview2::bindings::io::poll, +} }); +struct WasiHostCtx { + preview2_ctx: wasmtime_wasi::preview2::WasiCtx, + preview2_table: wasmtime::component::ResourceTable, + preview1_adapter: wasmtime_wasi::preview2::preview1::WasiPreview1Adapter, +} + +impl wasmtime_wasi::preview2::WasiView for WasiHostCtx { + fn table(&self) -> &wasmtime::component::ResourceTable { + &self.preview2_table + } + + fn ctx(&self) -> &wasmtime_wasi::preview2::WasiCtx { + &self.preview2_ctx + } + + fn table_mut(&mut self) -> &mut wasmtime::component::ResourceTable { + &mut self.preview2_table + } + + fn ctx_mut(&mut self) -> &mut wasmtime_wasi::preview2::WasiCtx { + &mut self.preview2_ctx + } +} + +impl wasmtime_wasi::preview2::preview1::WasiPreview1View for WasiHostCtx { + fn adapter(&self) -> &wasmtime_wasi::preview2::preview1::WasiPreview1Adapter { + &self.preview1_adapter + } + + fn adapter_mut(&mut self) -> &mut wasmtime_wasi::preview2::preview1::WasiPreview1Adapter { + &mut self.preview1_adapter + } +} + pub async fn wasm_canary() -> anyhow::Result { let cur_dir = std::env::current_dir().expect("Current dir"); println!("CURRENT DIR: {cur_dir:#?}"); @@ -44,14 +84,27 @@ pub async fn wasm_canary() -> anyhow::Result { cur_dir.join("aws_sdk_rust_lambda_canary_wasm.wasm"), ) .expect("Failed to load wasm"); - let linker = Linker::new(&engine); - // wasmtime_wasi::add_to_linker(&mut linker, |cx| cx)?; + let mut linker: Linker = Linker::new(&engine); + // wasmtime_wasi::preview2::preview1::add_to_linker_async(&mut linker); + // wasmtime_wasi::preview2::bindings::io::poll::add_to_linker(&mut linker, |cx| cx); // Configure and create a `WasiCtx`, which WASI functions need access to // through the host state of the store (which in this case is the host state // of the store) - let wasi_ctx = WasiCtxBuilder::new().inherit_stdio().build(); - let mut store = Store::new(&engine, wasi_ctx); + let wasi_ctx = WasiCtxBuilder::new() + .inherit_stdio() + .inherit_stderr() + .inherit_stdout() + .inherit_network() + .build(); + + let host_ctx = WasiHostCtx { + preview2_ctx: wasi_ctx, + preview2_table: wasmtime_wasi::preview2::ResourceTable::new(), + preview1_adapter: wasmtime_wasi::preview2::preview1::WasiPreview1Adapter::new(), + }; + + let mut store: Store = Store::new(&engine, host_ctx); // Instantiate our module with the bindgen! bindings let (bindings, _) = CanaryWorld::instantiate(&mut store, &component, &linker)?; diff --git a/tools/ci-cdk/canary-runner/src/build_bundle.rs b/tools/ci-cdk/canary-runner/src/build_bundle.rs index 1077b356b7..e2a2e67e10 100644 --- a/tools/ci-cdk/canary-runner/src/build_bundle.rs +++ b/tools/ci-cdk/canary-runner/src/build_bundle.rs @@ -56,8 +56,8 @@ tracing-texray = "0.1.1" reqwest = { version = "0.11.14", features = ["rustls-tls"], default-features = false } edit-distance = "2" wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } -wasmtime = { version = "16.0.0", features = ["component-model"] } -wasmtime-wasi = "16.0.0" +wasmtime = { version = "17.0.1", features = ["component-model"] } +wasmtime-wasi = "17.0.1" "#; const REQUIRED_SDK_CRATES: &[&str] = &[ @@ -324,7 +324,7 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { .context(here!())?; zip.write_all(&fs::read(&bin_path).context(here!("read target"))?) .context(here!())?; - + // Write the wasm bin to the zip zip.start_file( "aws_sdk_rust_lambda_canary_wasm.wasm", From 3fc1aa810321c78076e5759a19a3032abb8f4e79 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 19 Feb 2024 22:50:38 -0800 Subject: [PATCH 61/72] Wasm canary test works locally --- .../canary-lambda/src/latest/wasm_canary.rs | 122 ++++++++++++++++-- .../ci-cdk/canary-runner/src/build_bundle.rs | 1 + tools/ci-cdk/canary-wasm/Cargo.lock | 1 + tools/ci-cdk/canary-wasm/Cargo.toml | 3 +- tools/ci-cdk/canary-wasm/src/lib.rs | 21 ++- 5 files changed, 133 insertions(+), 15 deletions(-) diff --git a/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs index 33a1b1b1c1..19cff94ba3 100644 --- a/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs +++ b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs @@ -7,8 +7,7 @@ use crate::{mk_canary, CanaryEnv}; use wasmtime::component::{bindgen, Component, Linker}; use wasmtime::{Engine, Store}; -use wasmtime_wasi::preview2::WasiCtxBuilder; -use wasmtime_wasi::WasiCtx; +use wasmtime_wasi::preview2::{WasiCtxBuilder, WasiView}; // mk_canary!("wasm", |sdk_config: &SdkConfig, env: &CanaryEnv| { // wasm_canary() @@ -39,6 +38,7 @@ struct WasiHostCtx { preview2_ctx: wasmtime_wasi::preview2::WasiCtx, preview2_table: wasmtime::component::ResourceTable, preview1_adapter: wasmtime_wasi::preview2::preview1::WasiPreview1Adapter, + wasi_http_ctx: wasmtime_wasi_http::WasiHttpCtx, } impl wasmtime_wasi::preview2::WasiView for WasiHostCtx { @@ -69,14 +69,46 @@ impl wasmtime_wasi::preview2::preview1::WasiPreview1View for WasiHostCtx { } } -pub async fn wasm_canary() -> anyhow::Result { +impl wasmtime_wasi::preview2::bindings::sync_io::io::error::HostError for WasiHostCtx { + fn to_debug_string( + &mut self, + err: wasmtime::component::Resource< + wasmtime_wasi::preview2::bindings::sync_io::io::error::Error, + >, + ) -> wasmtime::Result { + Ok(format!("{:?}", self.table().get(&err)?)) + } + + fn drop( + &mut self, + err: wasmtime::component::Resource< + wasmtime_wasi::preview2::bindings::sync_io::io::error::Error, + >, + ) -> wasmtime::Result<()> { + self.table_mut().delete(err)?; + Ok(()) + } +} + +impl wasmtime_wasi::preview2::bindings::sync_io::io::error::Host for WasiHostCtx {} + +impl wasmtime_wasi_http::types::WasiHttpView for WasiHostCtx { + fn ctx(&mut self) -> &mut wasmtime_wasi_http::WasiHttpCtx { + &mut self.wasi_http_ctx + } + + fn table(&mut self) -> &mut wasmtime::component::ResourceTable { + &mut self.preview2_table + } +} + +pub fn wasm_canary() -> anyhow::Result { let cur_dir = std::env::current_dir().expect("Current dir"); println!("CURRENT DIR: {cur_dir:#?}"); // Create a Wasmtime Engine configured to run Components - let mut engine_config = wasmtime::Config::new(); - engine_config.wasm_component_model(true); - let engine = Engine::new(&engine_config).expect("Failed to create wasm engine"); + let engine = Engine::new(&wasmtime::Config::new().wasm_component_model(true)) + .expect("Failed to create wasm engine"); // Create our component from the wasm file let component = Component::from_file( @@ -84,24 +116,87 @@ pub async fn wasm_canary() -> anyhow::Result { cur_dir.join("aws_sdk_rust_lambda_canary_wasm.wasm"), ) .expect("Failed to load wasm"); + + // Create the linker and link in the necessary WASI bindings let mut linker: Linker = Linker::new(&engine); - // wasmtime_wasi::preview2::preview1::add_to_linker_async(&mut linker); - // wasmtime_wasi::preview2::bindings::io::poll::add_to_linker(&mut linker, |cx| cx); + wasmtime_wasi::preview2::bindings::sync_io::io::poll::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Poll"); + wasmtime_wasi::preview2::bindings::sync_io::io::error::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Error"); + wasmtime_wasi::preview2::bindings::sync_io::io::streams::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Streams"); + wasmtime_wasi::preview2::bindings::random::random::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Random"); + wasmtime_wasi::preview2::bindings::wasi::clocks::monotonic_clock::add_to_linker( + &mut linker, + |cx| cx, + ) + .expect("Failed to link Clock"); + wasmtime_wasi::preview2::bindings::sync_io::filesystem::types::add_to_linker( + &mut linker, + |cx| cx, + ) + .expect("Failed to link Filesystem Types"); + wasmtime_wasi::preview2::bindings::filesystem::preopens::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Filesystem Preopen"); + wasmtime_wasi::preview2::bindings::wasi::clocks::wall_clock::add_to_linker(&mut linker, |cx| { + cx + }) + .expect("Failed to link Wall Clock"); + wasmtime_wasi::preview2::bindings::wasi::cli::environment::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Environment"); + wasmtime_wasi::preview2::bindings::wasi::cli::exit::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Environment"); + wasmtime_wasi::preview2::bindings::wasi::cli::stdin::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Stdin"); + wasmtime_wasi::preview2::bindings::wasi::cli::stdout::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Stdout"); + wasmtime_wasi::preview2::bindings::wasi::cli::stderr::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Stderr"); + // wasmtime_wasi::preview2::command::add_to_linker(&mut linker); + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_input::add_to_linker( + &mut linker, + |cx| cx, + ) + .expect("Failed to link Terminal Input"); + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_output::add_to_linker( + &mut linker, + |cx| cx, + ) + .expect("Failed to link Terminal Output"); + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_stdin::add_to_linker( + &mut linker, + |cx| cx, + ) + .expect("Failed to link Terminal Stdin"); + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_stdout::add_to_linker( + &mut linker, + |cx| cx, + ) + .expect("Failed to link Terminal Stdout"); + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_stderr::add_to_linker( + &mut linker, + |cx| cx, + ) + .expect("Failed to link Terminal Stderr"); + wasmtime_wasi_http::bindings::http::types::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link HTTP Types"); + wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link HTTP Outgoing Handler"); // Configure and create a `WasiCtx`, which WASI functions need access to // through the host state of the store (which in this case is the host state // of the store) let wasi_ctx = WasiCtxBuilder::new() - .inherit_stdio() .inherit_stderr() .inherit_stdout() - .inherit_network() .build(); let host_ctx = WasiHostCtx { preview2_ctx: wasi_ctx, preview2_table: wasmtime_wasi::preview2::ResourceTable::new(), preview1_adapter: wasmtime_wasi::preview2::preview1::WasiPreview1Adapter::new(), + wasi_http_ctx: wasmtime_wasi_http::WasiHttpCtx {}, }; let mut store: Store = Store::new(&engine, host_ctx); @@ -120,8 +215,9 @@ pub async fn wasm_canary() -> anyhow::Result { // #[ignore] #[cfg(test)] -#[tokio::test] -async fn test_wasm_canary() { - let res = wasm_canary().await; +// #[tokio::test] +#[test] +fn test_wasm_canary() { + let res = wasm_canary(); println!("{res:#?}"); } diff --git a/tools/ci-cdk/canary-runner/src/build_bundle.rs b/tools/ci-cdk/canary-runner/src/build_bundle.rs index e2a2e67e10..81ecef7408 100644 --- a/tools/ci-cdk/canary-runner/src/build_bundle.rs +++ b/tools/ci-cdk/canary-runner/src/build_bundle.rs @@ -58,6 +58,7 @@ edit-distance = "2" wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } wasmtime = { version = "17.0.1", features = ["component-model"] } wasmtime-wasi = "17.0.1" +wasmtime-wasi-http = "17.0.1" "#; const REQUIRED_SDK_CRATES: &[&str] = &[ diff --git a/tools/ci-cdk/canary-wasm/Cargo.lock b/tools/ci-cdk/canary-wasm/Cargo.lock index e50ad881ce..b30f4f2af1 100644 --- a/tools/ci-cdk/canary-wasm/Cargo.lock +++ b/tools/ci-cdk/canary-wasm/Cargo.lock @@ -90,6 +90,7 @@ version = "0.1.0" dependencies = [ "aws-config", "aws-sdk-s3", + "aws-smithy-async", "aws-smithy-wasm", "tokio", "wit-bindgen", diff --git a/tools/ci-cdk/canary-wasm/Cargo.toml b/tools/ci-cdk/canary-wasm/Cargo.toml index cf9efed87d..8985208e2f 100644 --- a/tools/ci-cdk/canary-wasm/Cargo.toml +++ b/tools/ci-cdk/canary-wasm/Cargo.toml @@ -10,11 +10,12 @@ edition = "2021" license = "Apache-2.0" [dependencies] -tokio = { version = "1.36.0", features = ["macros", "rt"] } +tokio = { version = "1.36.0", features = ["macros", "rt", "time"] } wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } aws-config = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-config", default-features = false, features = ["behavior-version-latest"] } aws-sdk-s3 = { path = "../../../aws/sdk/build/aws-sdk/sdk/s3", default-features = false } aws-smithy-wasm = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-smithy-wasm" } +aws-smithy-async = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-smithy-async" } [lib] crate-type = ["cdylib"] diff --git a/tools/ci-cdk/canary-wasm/src/lib.rs b/tools/ci-cdk/canary-wasm/src/lib.rs index 1a7a753bdb..e59f65f43b 100644 --- a/tools/ci-cdk/canary-wasm/src/lib.rs +++ b/tools/ci-cdk/canary-wasm/src/lib.rs @@ -3,7 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ +use aws_config::Region; use aws_sdk_s3 as s3; +use aws_smithy_async::rt::sleep::{AsyncSleep, Sleep}; use aws_smithy_wasm::wasi::WasiHttpClientBuilder; // Needed for WASI-compliant environment as it expects specific functions @@ -30,6 +32,7 @@ struct Component; impl exports::aws::component::canary_interface::Guest for Component { fn run_canary() -> Result { let rt = tokio::runtime::Builder::new_current_thread() + .enable_time() .build() .expect("Failed to generate runtime"); let res = rt.block_on(run_canary())?; @@ -39,7 +42,13 @@ impl exports::aws::component::canary_interface::Guest for Component { async fn run_canary() -> Result { let http_client = WasiHttpClientBuilder::new().build(); - let config = aws_config::from_env().http_client(http_client).load().await; + let config = aws_config::from_env() + .region(Region::new("us-east-2")) + .no_credentials() + .http_client(http_client) + .sleep_impl(WasmSleep) + .load() + .await; let client = s3::Client::new(&config); let result = client @@ -55,3 +64,13 @@ async fn run_canary() -> Result { println!("WASM CANARY RESULT: {result:#?}"); Ok(format!("{result:?}")) } + +#[derive(Debug, Clone)] +struct WasmSleep; +impl AsyncSleep for WasmSleep { + fn sleep(&self, duration: std::time::Duration) -> Sleep { + Sleep::new(Box::pin(async move { + tokio::time::sleep(duration).await; + })) + } +} From 352cdfdc8c4ef81d25150e2a7338d8295025db87 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 20 Feb 2024 09:50:09 -0800 Subject: [PATCH 62/72] Add WasmSleep struct to wasm crate --- rust-runtime/aws-smithy-wasm/Cargo.toml | 2 ++ rust-runtime/aws-smithy-wasm/src/lib.rs | 3 +++ rust-runtime/aws-smithy-wasm/src/wasm.rs | 13 +++++++++++++ 3 files changed, 18 insertions(+) create mode 100644 rust-runtime/aws-smithy-wasm/src/wasm.rs diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index 8c261890b1..ea3f89f9b4 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -11,11 +11,13 @@ license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" [target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +aws-smithy-async = { path = "../aws-smithy-async" } aws-smithy-runtime-api = { path = "../aws-smithy-runtime-api", features = ["http-1x"]} aws-smithy-http = { path = "../aws-smithy-http" } aws-smithy-types = { path = "../aws-smithy-types" } bytes = "1" http = "1.0.0" +tokio = { version = "1.36.0", features = ["rt", "macros", "time"] } tower = "0.4.13" tracing = "0.1.40" wasi = "0.12.0+wasi-0.2.0" diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs index 4c423f3eb1..58d128866c 100644 --- a/rust-runtime/aws-smithy-wasm/src/lib.rs +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -18,3 +18,6 @@ #[cfg(all(target_family = "wasm", target_os = "wasi"))] pub mod wasi; + +#[cfg(all(target_family = "wasm"))] +pub mod wasm; diff --git a/rust-runtime/aws-smithy-wasm/src/wasm.rs b/rust-runtime/aws-smithy-wasm/src/wasm.rs new file mode 100644 index 0000000000..5678acb907 --- /dev/null +++ b/rust-runtime/aws-smithy-wasm/src/wasm.rs @@ -0,0 +1,13 @@ +use aws_smithy_async::rt::sleep::{AsyncSleep, Sleep}; + +/// A struct implementing the AsyncSleep trait that can be used in +/// Wasm environments +#[derive(Debug, Clone)] +pub struct WasmSleep; +impl AsyncSleep for WasmSleep { + fn sleep(&self, duration: std::time::Duration) -> Sleep { + Sleep::new(Box::pin(async move { + tokio::time::sleep(duration).await; + })) + } +} From ade60da07894f76c685b86bd0d3fd9c02825979b Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 20 Feb 2024 11:08:35 -0800 Subject: [PATCH 63/72] Cleaning up some of the canary committed Unfortunately having trouble making the canary run `async` because there is a tokio error. --- .../canary-lambda/src/latest/wasm_canary.rs | 98 ++++++------- tools/ci-cdk/canary-wasm/Cargo.lock | 23 ++-- tools/ci-cdk/canary-wasm/Cargo.toml | 1 - tools/ci-cdk/canary-wasm/src/bindings.rs | 130 +++++++++++++++++- tools/ci-cdk/canary-wasm/src/lib.rs | 41 ++---- tools/ci-cdk/canary-wasm/wit/component.wit | 15 ++ 6 files changed, 213 insertions(+), 95 deletions(-) create mode 100644 tools/ci-cdk/canary-wasm/wit/component.wit diff --git a/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs index 19cff94ba3..2ca4cad026 100644 --- a/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs +++ b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs @@ -5,6 +5,7 @@ use crate::{mk_canary, CanaryEnv}; +use aws_config::SdkConfig; use wasmtime::component::{bindgen, Component, Linker}; use wasmtime::{Engine, Store}; use wasmtime_wasi::preview2::{WasiCtxBuilder, WasiView}; @@ -16,22 +17,8 @@ use wasmtime_wasi::preview2::{WasiCtxBuilder, WasiView}; //This macro creates bindings to call the wasm functions in Rust //TODO: this should point to a wit file in canary-wasm bindgen!({ - inline: " - package aws:component; - - interface canary-interface { - run-canary: func() -> result; - } - - world canary-world { - export canary-interface; - } -", -with: { - "wasi:io/error": wasmtime_wasi::preview2::bindings::io::error, - "wasi:io/streams": wasmtime_wasi::preview2::bindings::io::streams, - "wasi:io/poll": wasmtime_wasi::preview2::bindings::io::poll, -} + world: "canary-world", + path: "../canary-wasm/wit/component.wit", }); struct WasiHostCtx { @@ -102,23 +89,53 @@ impl wasmtime_wasi_http::types::WasiHttpView for WasiHostCtx { } } -pub fn wasm_canary() -> anyhow::Result { - let cur_dir = std::env::current_dir().expect("Current dir"); - println!("CURRENT DIR: {cur_dir:#?}"); +pub fn wasm_canary() -> anyhow::Result> { + let wasm_bin_path = std::env::current_dir() + .expect("Current dir") + .join("aws_sdk_rust_lambda_canary_wasm.wasm"); // Create a Wasmtime Engine configured to run Components let engine = Engine::new(&wasmtime::Config::new().wasm_component_model(true)) .expect("Failed to create wasm engine"); // Create our component from the wasm file - let component = Component::from_file( - &engine, - cur_dir.join("aws_sdk_rust_lambda_canary_wasm.wasm"), - ) - .expect("Failed to load wasm"); + let component = Component::from_file(&engine, wasm_bin_path).expect("Failed to load wasm"); // Create the linker and link in the necessary WASI bindings let mut linker: Linker = Linker::new(&engine); + link_all_the_things(&mut linker); + + // Configure and create a `WasiCtx`, which WASI functions need access to + // through the host state of the store (which in this case is the host state + // of the store) + let wasi_ctx = WasiCtxBuilder::new() + .inherit_stderr() + .inherit_stdout() + .build(); + + let host_ctx = WasiHostCtx { + preview2_ctx: wasi_ctx, + preview2_table: wasmtime_wasi::preview2::ResourceTable::new(), + preview1_adapter: wasmtime_wasi::preview2::preview1::WasiPreview1Adapter::new(), + wasi_http_ctx: wasmtime_wasi_http::WasiHttpCtx {}, + }; + + let mut store: Store = Store::new(&engine, host_ctx); + + // Instantiate our module with the bindgen! bindings + let (bindings, _) = CanaryWorld::instantiate(&mut store, &component, &linker)?; + + println!("CALLING WASM FUNC"); + let canary_interface = bindings.aws_component_canary_interface(); + let api_result = canary_interface + .call_run_canary(store)? + .map_err(|err| anyhow::Error::msg(err))?; + + Ok(api_result) +} + +/// This function adds all of the WASI bindings to the linker +fn link_all_the_things(mut linker: &mut Linker) { wasmtime_wasi::preview2::bindings::sync_io::io::poll::add_to_linker(&mut linker, |cx| cx) .expect("Failed to link Poll"); wasmtime_wasi::preview2::bindings::sync_io::io::error::add_to_linker(&mut linker, |cx| cx) @@ -183,34 +200,6 @@ pub fn wasm_canary() -> anyhow::Result { .expect("Failed to link HTTP Types"); wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker(&mut linker, |cx| cx) .expect("Failed to link HTTP Outgoing Handler"); - - // Configure and create a `WasiCtx`, which WASI functions need access to - // through the host state of the store (which in this case is the host state - // of the store) - let wasi_ctx = WasiCtxBuilder::new() - .inherit_stderr() - .inherit_stdout() - .build(); - - let host_ctx = WasiHostCtx { - preview2_ctx: wasi_ctx, - preview2_table: wasmtime_wasi::preview2::ResourceTable::new(), - preview1_adapter: wasmtime_wasi::preview2::preview1::WasiPreview1Adapter::new(), - wasi_http_ctx: wasmtime_wasi_http::WasiHttpCtx {}, - }; - - let mut store: Store = Store::new(&engine, host_ctx); - - // Instantiate our module with the bindgen! bindings - let (bindings, _) = CanaryWorld::instantiate(&mut store, &component, &linker)?; - - println!("CALLING WASM FUNC"); - let canary_interface = bindings.aws_component_canary_interface(); - let api_result = canary_interface - .call_run_canary(store)? - .map_err(|err| anyhow::Error::msg(err))?; - println!("{api_result}"); - Ok(api_result) } // #[ignore] @@ -218,6 +207,7 @@ pub fn wasm_canary() -> anyhow::Result { // #[tokio::test] #[test] fn test_wasm_canary() { - let res = wasm_canary(); - println!("{res:#?}"); + let res = wasm_canary().expect("Wasm return"); + + assert!(res.len() > 0) } diff --git a/tools/ci-cdk/canary-wasm/Cargo.lock b/tools/ci-cdk/canary-wasm/Cargo.lock index b30f4f2af1..aac8afab10 100644 --- a/tools/ci-cdk/canary-wasm/Cargo.lock +++ b/tools/ci-cdk/canary-wasm/Cargo.lock @@ -31,7 +31,7 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "1.1.5" +version = "1.1.6" dependencies = [ "aws-credential-types", "aws-runtime", @@ -54,7 +54,7 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.1.5" +version = "1.1.6" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -64,7 +64,7 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.1.5" +version = "1.1.6" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -90,7 +90,6 @@ version = "0.1.0" dependencies = [ "aws-config", "aws-sdk-s3", - "aws-smithy-async", "aws-smithy-wasm", "tokio", "wit-bindgen", @@ -146,7 +145,7 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.1.5" +version = "1.1.6" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -168,7 +167,7 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.1.5" +version = "1.1.6" dependencies = [ "futures-util", "pin-project-lite", @@ -239,7 +238,7 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.1.5" +version = "1.1.6" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -258,7 +257,7 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.1.5" +version = "1.1.6" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -272,7 +271,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.1.5" +version = "1.1.6" dependencies = [ "base64-simd", "bytes", @@ -291,13 +290,15 @@ dependencies = [ [[package]] name = "aws-smithy-wasm" -version = "0.60.5" +version = "0.1.0" dependencies = [ + "aws-smithy-async", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "http 1.0.0", + "tokio", "tower", "tracing", "wasi", @@ -312,7 +313,7 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.1.5" +version = "1.1.6" dependencies = [ "aws-credential-types", "aws-smithy-async", diff --git a/tools/ci-cdk/canary-wasm/Cargo.toml b/tools/ci-cdk/canary-wasm/Cargo.toml index 8985208e2f..32e2ad4f42 100644 --- a/tools/ci-cdk/canary-wasm/Cargo.toml +++ b/tools/ci-cdk/canary-wasm/Cargo.toml @@ -15,7 +15,6 @@ wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } aws-config = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-config", default-features = false, features = ["behavior-version-latest"] } aws-sdk-s3 = { path = "../../../aws/sdk/build/aws-sdk/sdk/s3", default-features = false } aws-smithy-wasm = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-smithy-wasm" } -aws-smithy-async = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-smithy-async" } [lib] crate-type = ["cdylib"] diff --git a/tools/ci-cdk/canary-wasm/src/bindings.rs b/tools/ci-cdk/canary-wasm/src/bindings.rs index b190912a80..74af2df502 100644 --- a/tools/ci-cdk/canary-wasm/src/bindings.rs +++ b/tools/ci-cdk/canary-wasm/src/bindings.rs @@ -1,9 +1,135 @@ // Generated by `wit-bindgen` 0.16.0. DO NOT EDIT! +pub mod exports { + pub mod aws { + pub mod component { + + #[allow(clippy::all)] + pub mod canary_interface { + #[used] + #[doc(hidden)] + #[cfg(target_arch = "wasm32")] + static __FORCE_SECTION_REF: fn() = super::super::super::super::__link_section; + const _: () = { + + #[doc(hidden)] + #[export_name = "aws:component/canary-interface#run-canary"] + #[allow(non_snake_case)] + unsafe extern "C" fn __export_run_canary() -> i32 { + #[allow(unused_imports)] + use wit_bindgen::rt::{alloc, vec::Vec, string::String}; + + // Before executing any other code, use this function to run all static + // constructors, if they have not yet been run. This is a hack required + // to work around wasi-libc ctors calling import functions to initialize + // the environment. + // + // This functionality will be removed once rust 1.69.0 is stable, at which + // point wasi-libc will no longer have this behavior. + // + // See + // https://github.com/bytecodealliance/preview2-prototyping/issues/99 + // for more details. + #[cfg(target_arch="wasm32")] + wit_bindgen::rt::run_ctors_once(); + + let result0 = <_GuestImpl as Guest>::run_canary(); + let ptr1 = _RET_AREA.0.as_mut_ptr() as i32; + match result0 { + Ok(e) => { { + *((ptr1 + 0) as *mut u8) = (0i32) as u8; + let vec3 = e; + let len3 = vec3.len() as i32; + let layout3 = alloc::Layout::from_size_align_unchecked(vec3.len() * 8, 4); + let result3 = if layout3.size() != 0 + { + let ptr = alloc::alloc(layout3); + if ptr.is_null() + { + alloc::handle_alloc_error(layout3); + } + ptr + }else {{ + ::core::ptr::null_mut() + }}; + for (i, e) in vec3.into_iter().enumerate() { + let base = result3 as i32 + (i as i32) * 8; + { + let vec2 = (e.into_bytes()).into_boxed_slice(); + let ptr2 = vec2.as_ptr() as i32; + let len2 = vec2.len() as i32; + ::core::mem::forget(vec2); + *((base + 4) as *mut i32) = len2; + *((base + 0) as *mut i32) = ptr2; + } + } + *((ptr1 + 8) as *mut i32) = len3; + *((ptr1 + 4) as *mut i32) = result3 as i32; + } }, + Err(e) => { { + *((ptr1 + 0) as *mut u8) = (1i32) as u8; + let vec4 = (e.into_bytes()).into_boxed_slice(); + let ptr4 = vec4.as_ptr() as i32; + let len4 = vec4.len() as i32; + ::core::mem::forget(vec4); + *((ptr1 + 8) as *mut i32) = len4; + *((ptr1 + 4) as *mut i32) = ptr4; + } }, + };ptr1 + } + + const _: () = { + #[doc(hidden)] + #[export_name = "cabi_post_aws:component/canary-interface#run-canary"] + #[allow(non_snake_case)] + unsafe extern "C" fn __post_return_run_canary(arg0: i32,) { + let l0 = i32::from(*((arg0 + 0) as *const u8)); + match l0 { + 0 => { + let l3 = *((arg0 + 4) as *const i32); + let l4 = *((arg0 + 8) as *const i32); + let base5 = l3; + let len5 = l4; + for i in 0..len5 { + let base = base5 + i *8; + { + let l1 = *((base + 0) as *const i32); + let l2 = *((base + 4) as *const i32); + wit_bindgen::rt::dealloc(l1, (l2) as usize, 1); + } + } + wit_bindgen::rt::dealloc(base5, (len5 as usize) * 8, 4); + }, + _ => { + let l6 = *((arg0 + 4) as *const i32); + let l7 = *((arg0 + 8) as *const i32); + wit_bindgen::rt::dealloc(l6, (l7) as usize, 1); + }, + } + } + }; + }; + use super::super::super::super::super::Component as _GuestImpl; + pub trait Guest { + fn run_canary() -> Result,wit_bindgen::rt::string::String>; + } + + #[allow(unused_imports)] + use wit_bindgen::rt::{alloc, vec::Vec, string::String}; + + #[repr(align(4))] + struct _RetArea([u8; 12]); + static mut _RET_AREA: _RetArea = _RetArea([0; 12]); + + } + + } + } +} #[cfg(target_arch = "wasm32")] -#[link_section = "component-type:aws-sdk-rust-lambda-canary-wasm"] +#[link_section = "component-type:canary-world"] #[doc(hidden)] -pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 257] = [3, 0, 31, 97, 119, 115, 45, 115, 100, 107, 45, 114, 117, 115, 116, 45, 108, 97, 109, 98, 100, 97, 45, 99, 97, 110, 97, 114, 121, 45, 119, 97, 115, 109, 0, 97, 115, 109, 13, 0, 1, 0, 7, 84, 1, 65, 2, 1, 65, 0, 4, 1, 73, 99, 111, 109, 112, 111, 110, 101, 110, 116, 58, 97, 119, 115, 45, 115, 100, 107, 45, 114, 117, 115, 116, 45, 108, 97, 109, 98, 100, 97, 45, 99, 97, 110, 97, 114, 121, 45, 119, 97, 115, 109, 47, 97, 119, 115, 45, 115, 100, 107, 45, 114, 117, 115, 116, 45, 108, 97, 109, 98, 100, 97, 45, 99, 97, 110, 97, 114, 121, 45, 119, 97, 115, 109, 4, 0, 11, 37, 1, 0, 31, 97, 119, 115, 45, 115, 100, 107, 45, 114, 117, 115, 116, 45, 108, 97, 109, 98, 100, 97, 45, 99, 97, 110, 97, 114, 121, 45, 119, 97, 115, 109, 3, 0, 0, 0, 16, 12, 112, 97, 99, 107, 97, 103, 101, 45, 100, 111, 99, 115, 0, 123, 125, 0, 70, 9, 112, 114, 111, 100, 117, 99, 101, 114, 115, 1, 12, 112, 114, 111, 99, 101, 115, 115, 101, 100, 45, 98, 121, 2, 13, 119, 105, 116, 45, 99, 111, 109, 112, 111, 110, 101, 110, 116, 6, 48, 46, 49, 56, 46, 50, 16, 119, 105, 116, 45, 98, 105, 110, 100, 103, 101, 110, 45, 114, 117, 115, 116, 6, 48, 46, 49, 54, 46, 48]; +pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 462] = [3, 0, 12, 99, 97, 110, 97, 114, 121, 45, 119, 111, 114, 108, 100, 0, 97, 115, 109, 13, 0, 1, 0, 7, 70, 1, 65, 2, 1, 66, 4, 1, 112, 115, 1, 106, 1, 0, 1, 115, 1, 64, 0, 0, 1, 4, 0, 10, 114, 117, 110, 45, 99, 97, 110, 97, 114, 121, 1, 2, 4, 1, 30, 97, 119, 115, 58, 99, 111, 109, 112, 111, 110, 101, 110, 116, 47, 99, 97, 110, 97, 114, 121, 45, 105, 110, 116, 101, 114, 102, 97, 99, 101, 5, 0, 11, 22, 1, 0, 16, 99, 97, 110, 97, 114, 121, 45, 105, 110, 116, 101, 114, 102, 97, 99, 101, 3, 0, 0, 7, 104, 1, 65, 2, 1, 65, 2, 1, 66, 4, 1, 112, 115, 1, 106, 1, 0, 1, 115, 1, 64, 0, 0, 1, 4, 0, 10, 114, 117, 110, 45, 99, 97, 110, 97, 114, 121, 1, 2, 4, 1, 30, 97, 119, 115, 58, 99, 111, 109, 112, 111, 110, 101, 110, 116, 47, 99, 97, 110, 97, 114, 121, 45, 105, 110, 116, 101, 114, 102, 97, 99, 101, 5, 0, 4, 1, 26, 97, 119, 115, 58, 99, 111, 109, 112, 111, 110, 101, 110, 116, 47, 99, 97, 110, 97, 114, 121, 45, 119, 111, 114, 108, 100, 4, 0, 11, 18, 1, 0, 12, 99, 97, 110, 97, 114, 121, 45, 119, 111, 114, 108, 100, 3, 2, 0, 0, 142, 1, 12, 112, 97, 99, 107, 97, 103, 101, 45, 100, 111, 99, 115, 0, 123, 34, 100, 111, 99, 115, 34, 58, 34, 42, 92, 110, 32, 42, 32, 67, 111, 112, 121, 114, 105, 103, 104, 116, 32, 65, 109, 97, 122, 111, 110, 46, 99, 111, 109, 44, 32, 73, 110, 99, 46, 32, 111, 114, 32, 105, 116, 115, 32, 97, 102, 102, 105, 108, 105, 97, 116, 101, 115, 46, 32, 65, 108, 108, 32, 82, 105, 103, 104, 116, 115, 32, 82, 101, 115, 101, 114, 118, 101, 100, 46, 92, 110, 32, 42, 32, 83, 80, 68, 88, 45, 76, 105, 99, 101, 110, 115, 101, 45, 73, 100, 101, 110, 116, 105, 102, 105, 101, 114, 58, 32, 65, 112, 97, 99, 104, 101, 45, 50, 46, 48, 92, 110, 32, 42, 47, 34, 125, 0, 70, 9, 112, 114, 111, 100, 117, 99, 101, 114, 115, 1, 12, 112, 114, 111, 99, 101, 115, 115, 101, 100, 45, 98, 121, 2, 13, 119, 105, 116, 45, 99, 111, 109, 112, 111, 110, 101, 110, 116, 6, 48, 46, 49, 56, 46, 50, 16, 119, 105, 116, 45, 98, 105, 110, 100, 103, 101, 110, 45, 114, 117, 115, 116, 6, 48, 46, 49, 54, 46, 48]; #[inline(never)] #[doc(hidden)] diff --git a/tools/ci-cdk/canary-wasm/src/lib.rs b/tools/ci-cdk/canary-wasm/src/lib.rs index e59f65f43b..7e48031354 100644 --- a/tools/ci-cdk/canary-wasm/src/lib.rs +++ b/tools/ci-cdk/canary-wasm/src/lib.rs @@ -5,23 +5,12 @@ use aws_config::Region; use aws_sdk_s3 as s3; -use aws_smithy_async::rt::sleep::{AsyncSleep, Sleep}; use aws_smithy_wasm::wasi::WasiHttpClientBuilder; +use aws_smithy_wasm::wasm::WasmSleep; -// Needed for WASI-compliant environment as it expects specific functions -// to be exported such as `cabi_realloc`, `_start`, etc. +//Generates the Rust bindings from the wit file wit_bindgen::generate!({ - inline: " - package aws:component; - - interface canary-interface { - run-canary: func() -> result; - } - - world canary-world { - export canary-interface; - } - ", + world: "canary-world", exports: { "aws:component/canary-interface": Component } @@ -30,7 +19,7 @@ wit_bindgen::generate!({ struct Component; impl exports::aws::component::canary_interface::Guest for Component { - fn run_canary() -> Result { + fn run_canary() -> Result, String> { let rt = tokio::runtime::Builder::new_current_thread() .enable_time() .build() @@ -40,7 +29,7 @@ impl exports::aws::component::canary_interface::Guest for Component { } } -async fn run_canary() -> Result { +async fn run_canary() -> Result, String> { let http_client = WasiHttpClientBuilder::new().build(); let config = aws_config::from_env() .region(Region::new("us-east-2")) @@ -61,16 +50,14 @@ async fn run_canary() -> Result { .await .expect("Failed to ListObjects"); - println!("WASM CANARY RESULT: {result:#?}"); - Ok(format!("{result:?}")) -} + //For ease of modeling the return we just extract the keys from the objects + let object_names: Vec = result + .contents + .expect("No S3 Objects") + .iter() + .map(|obj| obj.key().expect("Object has no name").to_string()) + .collect(); -#[derive(Debug, Clone)] -struct WasmSleep; -impl AsyncSleep for WasmSleep { - fn sleep(&self, duration: std::time::Duration) -> Sleep { - Sleep::new(Box::pin(async move { - tokio::time::sleep(duration).await; - })) - } + println!("WASM CANARY RESULT: {object_names:#?}"); + Ok(object_names) } diff --git a/tools/ci-cdk/canary-wasm/wit/component.wit b/tools/ci-cdk/canary-wasm/wit/component.wit new file mode 100644 index 0000000000..a23ee0aadf --- /dev/null +++ b/tools/ci-cdk/canary-wasm/wit/component.wit @@ -0,0 +1,15 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws:component; + +interface canary-interface { + run-canary: func() -> result, string>; +} + +world canary-world { + export canary-interface; +} + \ No newline at end of file From 7eeb60dacf3c736a9e77daa7b8a5d83276f364f0 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 20 Feb 2024 14:50:07 -0800 Subject: [PATCH 64/72] Added wasm_canary to the canary runner Updating some lints --- rust-runtime/aws-smithy-wasm/src/lib.rs | 4 +- rust-runtime/aws-smithy-wasm/src/wasm.rs | 7 + tools/ci-cdk/README.md | 4 +- tools/ci-cdk/canary-lambda/src/canary.rs | 3 +- .../canary-lambda/src/latest/wasm_canary.rs | 173 +++++++----------- .../ci-cdk/canary-runner/src/build_bundle.rs | 2 +- tools/ci-cdk/canary-wasm/src/lib.rs | 1 - 7 files changed, 81 insertions(+), 113 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs index 58d128866c..caf6310972 100644 --- a/rust-runtime/aws-smithy-wasm/src/lib.rs +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -16,8 +16,10 @@ //! Smithy WebAssembly +/// Tools for using Smithy SDKs in WASI environments #[cfg(all(target_family = "wasm", target_os = "wasi"))] pub mod wasi; -#[cfg(all(target_family = "wasm"))] +/// Tools for using Smithy SDKs in WASM environments +#[cfg(target_family = "wasm")] pub mod wasm; diff --git a/rust-runtime/aws-smithy-wasm/src/wasm.rs b/rust-runtime/aws-smithy-wasm/src/wasm.rs index 5678acb907..c438aa1acc 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasm.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasm.rs @@ -1,9 +1,16 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +//! WASM Tools for the SDK use aws_smithy_async::rt::sleep::{AsyncSleep, Sleep}; /// A struct implementing the AsyncSleep trait that can be used in /// Wasm environments #[derive(Debug, Clone)] pub struct WasmSleep; + impl AsyncSleep for WasmSleep { fn sleep(&self, duration: std::time::Duration) -> Sleep { Sleep::new(Box::pin(async move { diff --git a/tools/ci-cdk/README.md b/tools/ci-cdk/README.md index cbbd304293..b47bfc0ada 100644 --- a/tools/ci-cdk/README.md +++ b/tools/ci-cdk/README.md @@ -21,10 +21,10 @@ From there, you can just point the `canary-runner` to the `cdk-outputs.json` to ```bash cd canary-runner -cargo run -- run --sdk-release-tag --musl --cdk-output ../cdk-outputs.json +cargo run -- --sdk-release-tag --musl --cdk-outputs ../cdk-outputs.json ``` -__NOTE:__ You may want to add a `--profile` to the `deploy` command to select a specific credential +**NOTE:** You may want to add a `--profile` to the deploy command to select a specific credential profile to deploy to if you don't want to use the default. Also, if this is a new test AWS account, be sure it CDK bootstrap it before attempting to deploy. diff --git a/tools/ci-cdk/canary-lambda/src/canary.rs b/tools/ci-cdk/canary-lambda/src/canary.rs index 0f3a986a09..6b60811d4e 100644 --- a/tools/ci-cdk/canary-lambda/src/canary.rs +++ b/tools/ci-cdk/canary-lambda/src/canary.rs @@ -12,7 +12,7 @@ use aws_config::SdkConfig; use tracing::{info_span, Instrument}; use crate::current_canary::paginator_canary; -use crate::current_canary::{s3_canary, transcribe_canary}; +use crate::current_canary::{s3_canary, transcribe_canary, wasm_canary}; #[macro_export] macro_rules! mk_canary { @@ -35,6 +35,7 @@ pub fn get_canaries_to_run( paginator_canary::mk_canary(&sdk_config, &env), s3_canary::mk_canary(&sdk_config, &env), transcribe_canary::mk_canary(&sdk_config, &env), + wasm_canary::mk_canary(&sdk_config, &env), ]; canaries diff --git a/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs index 2ca4cad026..4832a58746 100644 --- a/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs +++ b/tools/ci-cdk/canary-lambda/src/latest/wasm_canary.rs @@ -8,23 +8,22 @@ use crate::{mk_canary, CanaryEnv}; use aws_config::SdkConfig; use wasmtime::component::{bindgen, Component, Linker}; use wasmtime::{Engine, Store}; -use wasmtime_wasi::preview2::{WasiCtxBuilder, WasiView}; +use wasmtime_wasi::preview2::WasiCtxBuilder; -// mk_canary!("wasm", |sdk_config: &SdkConfig, env: &CanaryEnv| { -// wasm_canary() -// }); +mk_canary!("wasm", |_sdk_config: &SdkConfig, _env: &CanaryEnv| { + wasm_canary() +}); //This macro creates bindings to call the wasm functions in Rust -//TODO: this should point to a wit file in canary-wasm bindgen!({ world: "canary-world", path: "../canary-wasm/wit/component.wit", + async: true }); struct WasiHostCtx { preview2_ctx: wasmtime_wasi::preview2::WasiCtx, preview2_table: wasmtime::component::ResourceTable, - preview1_adapter: wasmtime_wasi::preview2::preview1::WasiPreview1Adapter, wasi_http_ctx: wasmtime_wasi_http::WasiHttpCtx, } @@ -46,39 +45,6 @@ impl wasmtime_wasi::preview2::WasiView for WasiHostCtx { } } -impl wasmtime_wasi::preview2::preview1::WasiPreview1View for WasiHostCtx { - fn adapter(&self) -> &wasmtime_wasi::preview2::preview1::WasiPreview1Adapter { - &self.preview1_adapter - } - - fn adapter_mut(&mut self) -> &mut wasmtime_wasi::preview2::preview1::WasiPreview1Adapter { - &mut self.preview1_adapter - } -} - -impl wasmtime_wasi::preview2::bindings::sync_io::io::error::HostError for WasiHostCtx { - fn to_debug_string( - &mut self, - err: wasmtime::component::Resource< - wasmtime_wasi::preview2::bindings::sync_io::io::error::Error, - >, - ) -> wasmtime::Result { - Ok(format!("{:?}", self.table().get(&err)?)) - } - - fn drop( - &mut self, - err: wasmtime::component::Resource< - wasmtime_wasi::preview2::bindings::sync_io::io::error::Error, - >, - ) -> wasmtime::Result<()> { - self.table_mut().delete(err)?; - Ok(()) - } -} - -impl wasmtime_wasi::preview2::bindings::sync_io::io::error::Host for WasiHostCtx {} - impl wasmtime_wasi_http::types::WasiHttpView for WasiHostCtx { fn ctx(&mut self) -> &mut wasmtime_wasi_http::WasiHttpCtx { &mut self.wasi_http_ctx @@ -89,17 +55,20 @@ impl wasmtime_wasi_http::types::WasiHttpView for WasiHostCtx { } } -pub fn wasm_canary() -> anyhow::Result> { +pub async fn wasm_canary() -> anyhow::Result<()> { let wasm_bin_path = std::env::current_dir() .expect("Current dir") .join("aws_sdk_rust_lambda_canary_wasm.wasm"); // Create a Wasmtime Engine configured to run Components - let engine = Engine::new(&wasmtime::Config::new().wasm_component_model(true)) - .expect("Failed to create wasm engine"); + let engine = Engine::new( + wasmtime::Config::new() + .wasm_component_model(true) + .async_support(true), + )?; // Create our component from the wasm file - let component = Component::from_file(&engine, wasm_bin_path).expect("Failed to load wasm"); + let component = Component::from_file(&engine, wasm_bin_path)?; // Create the linker and link in the necessary WASI bindings let mut linker: Linker = Linker::new(&engine); @@ -116,98 +85,88 @@ pub fn wasm_canary() -> anyhow::Result> { let host_ctx = WasiHostCtx { preview2_ctx: wasi_ctx, preview2_table: wasmtime_wasi::preview2::ResourceTable::new(), - preview1_adapter: wasmtime_wasi::preview2::preview1::WasiPreview1Adapter::new(), wasi_http_ctx: wasmtime_wasi_http::WasiHttpCtx {}, }; let mut store: Store = Store::new(&engine, host_ctx); // Instantiate our module with the bindgen! bindings - let (bindings, _) = CanaryWorld::instantiate(&mut store, &component, &linker)?; + let (bindings, _) = CanaryWorld::instantiate_async(&mut store, &component, &linker).await?; - println!("CALLING WASM FUNC"); let canary_interface = bindings.aws_component_canary_interface(); let api_result = canary_interface - .call_run_canary(store)? - .map_err(|err| anyhow::Error::msg(err))?; + .call_run_canary(store) + .await? + .map_err(anyhow::Error::msg)?; - Ok(api_result) + // Asserting on the post FFI result to confirm everything in the wasm module worked + assert!(!api_result.is_empty()); + + Ok(()) } /// This function adds all of the WASI bindings to the linker -fn link_all_the_things(mut linker: &mut Linker) { - wasmtime_wasi::preview2::bindings::sync_io::io::poll::add_to_linker(&mut linker, |cx| cx) +fn link_all_the_things(linker: &mut Linker) { + //IO + wasmtime_wasi::preview2::bindings::io::poll::add_to_linker(linker, |cx| cx) .expect("Failed to link Poll"); - wasmtime_wasi::preview2::bindings::sync_io::io::error::add_to_linker(&mut linker, |cx| cx) + wasmtime_wasi::preview2::bindings::io::error::add_to_linker(linker, |cx| cx) .expect("Failed to link Error"); - wasmtime_wasi::preview2::bindings::sync_io::io::streams::add_to_linker(&mut linker, |cx| cx) + wasmtime_wasi::preview2::bindings::io::streams::add_to_linker(linker, |cx| cx) .expect("Failed to link Streams"); - wasmtime_wasi::preview2::bindings::random::random::add_to_linker(&mut linker, |cx| cx) + + //Random + wasmtime_wasi::preview2::bindings::random::random::add_to_linker(linker, |cx| cx) .expect("Failed to link Random"); - wasmtime_wasi::preview2::bindings::wasi::clocks::monotonic_clock::add_to_linker( - &mut linker, - |cx| cx, - ) - .expect("Failed to link Clock"); - wasmtime_wasi::preview2::bindings::sync_io::filesystem::types::add_to_linker( - &mut linker, - |cx| cx, - ) - .expect("Failed to link Filesystem Types"); - wasmtime_wasi::preview2::bindings::filesystem::preopens::add_to_linker(&mut linker, |cx| cx) - .expect("Failed to link Filesystem Preopen"); - wasmtime_wasi::preview2::bindings::wasi::clocks::wall_clock::add_to_linker(&mut linker, |cx| { + + //Clocks + wasmtime_wasi::preview2::bindings::wasi::clocks::monotonic_clock::add_to_linker(linker, |cx| { cx }) - .expect("Failed to link Wall Clock"); - wasmtime_wasi::preview2::bindings::wasi::cli::environment::add_to_linker(&mut linker, |cx| cx) + .expect("Failed to link Clock"); + wasmtime_wasi::preview2::bindings::wasi::clocks::wall_clock::add_to_linker(linker, |cx| cx) + .expect("Failed to link Wall Clock"); + + //Filesystem + wasmtime_wasi::preview2::bindings::filesystem::types::add_to_linker(linker, |cx| cx) + .expect("Failed to link Filesystem Types"); + wasmtime_wasi::preview2::bindings::filesystem::preopens::add_to_linker(linker, |cx| cx) + .expect("Failed to link Filesystem Preopen"); + + //CLI + wasmtime_wasi::preview2::bindings::wasi::cli::environment::add_to_linker(linker, |cx| cx) .expect("Failed to link Environment"); - wasmtime_wasi::preview2::bindings::wasi::cli::exit::add_to_linker(&mut linker, |cx| cx) + wasmtime_wasi::preview2::bindings::wasi::cli::exit::add_to_linker(linker, |cx| cx) .expect("Failed to link Environment"); - wasmtime_wasi::preview2::bindings::wasi::cli::stdin::add_to_linker(&mut linker, |cx| cx) + wasmtime_wasi::preview2::bindings::wasi::cli::stdin::add_to_linker(linker, |cx| cx) .expect("Failed to link Stdin"); - wasmtime_wasi::preview2::bindings::wasi::cli::stdout::add_to_linker(&mut linker, |cx| cx) + wasmtime_wasi::preview2::bindings::wasi::cli::stdout::add_to_linker(linker, |cx| cx) .expect("Failed to link Stdout"); - wasmtime_wasi::preview2::bindings::wasi::cli::stderr::add_to_linker(&mut linker, |cx| cx) + wasmtime_wasi::preview2::bindings::wasi::cli::stderr::add_to_linker(linker, |cx| cx) .expect("Failed to link Stderr"); - // wasmtime_wasi::preview2::command::add_to_linker(&mut linker); - wasmtime_wasi::preview2::bindings::wasi::cli::terminal_input::add_to_linker( - &mut linker, - |cx| cx, - ) - .expect("Failed to link Terminal Input"); - wasmtime_wasi::preview2::bindings::wasi::cli::terminal_output::add_to_linker( - &mut linker, - |cx| cx, - ) - .expect("Failed to link Terminal Output"); - wasmtime_wasi::preview2::bindings::wasi::cli::terminal_stdin::add_to_linker( - &mut linker, - |cx| cx, - ) - .expect("Failed to link Terminal Stdin"); - wasmtime_wasi::preview2::bindings::wasi::cli::terminal_stdout::add_to_linker( - &mut linker, - |cx| cx, - ) - .expect("Failed to link Terminal Stdout"); - wasmtime_wasi::preview2::bindings::wasi::cli::terminal_stderr::add_to_linker( - &mut linker, - |cx| cx, - ) - .expect("Failed to link Terminal Stderr"); - wasmtime_wasi_http::bindings::http::types::add_to_linker(&mut linker, |cx| cx) + + // CLI Terminal + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_input::add_to_linker(linker, |cx| cx) + .expect("Failed to link Terminal Input"); + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_output::add_to_linker(linker, |cx| cx) + .expect("Failed to link Terminal Output"); + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_stdin::add_to_linker(linker, |cx| cx) + .expect("Failed to link Terminal Stdin"); + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_stdout::add_to_linker(linker, |cx| cx) + .expect("Failed to link Terminal Stdout"); + wasmtime_wasi::preview2::bindings::wasi::cli::terminal_stderr::add_to_linker(linker, |cx| cx) + .expect("Failed to link Terminal Stderr"); + + //HTTP + wasmtime_wasi_http::bindings::http::types::add_to_linker(linker, |cx| cx) .expect("Failed to link HTTP Types"); - wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker(&mut linker, |cx| cx) + wasmtime_wasi_http::bindings::http::outgoing_handler::add_to_linker(linker, |cx| cx) .expect("Failed to link HTTP Outgoing Handler"); } // #[ignore] #[cfg(test)] -// #[tokio::test] -#[test] -fn test_wasm_canary() { - let res = wasm_canary().expect("Wasm return"); - - assert!(res.len() > 0) +#[tokio::test] +async fn test_wasm_canary() { + wasm_canary().await.expect("Wasm return") } diff --git a/tools/ci-cdk/canary-runner/src/build_bundle.rs b/tools/ci-cdk/canary-runner/src/build_bundle.rs index 81ecef7408..f4441289d0 100644 --- a/tools/ci-cdk/canary-runner/src/build_bundle.rs +++ b/tools/ci-cdk/canary-runner/src/build_bundle.rs @@ -332,7 +332,7 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { zip::write::FileOptions::default().unix_permissions(0o755), ) .context(here!())?; - zip.write_all(&fs::read(&wasm_bin_path).context(here!("read wasm bin"))?) + zip.write_all(&fs::read(wasm_bin_path).context(here!("read wasm bin"))?) .context(here!())?; zip.finish().context(here!())?; diff --git a/tools/ci-cdk/canary-wasm/src/lib.rs b/tools/ci-cdk/canary-wasm/src/lib.rs index 7e48031354..d2c7ed365a 100644 --- a/tools/ci-cdk/canary-wasm/src/lib.rs +++ b/tools/ci-cdk/canary-wasm/src/lib.rs @@ -58,6 +58,5 @@ async fn run_canary() -> Result, String> { .map(|obj| obj.key().expect("Object has no name").to_string()) .collect(); - println!("WASM CANARY RESULT: {object_names:#?}"); Ok(object_names) } From 3d1b9b4f31af71e908a36d85f2eaf67e73d13e77 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 20 Feb 2024 16:40:55 -0800 Subject: [PATCH 65/72] Fixing canary builder tests --- tools/ci-cdk/.gitignore | 3 + tools/ci-cdk/README.md | 2 +- .../ci-cdk/canary-runner/src/build_bundle.rs | 6 + tools/ci-cdk/canary-wasm/src/bindings.rs | 137 ------------------ 4 files changed, 10 insertions(+), 138 deletions(-) delete mode 100644 tools/ci-cdk/canary-wasm/src/bindings.rs diff --git a/tools/ci-cdk/.gitignore b/tools/ci-cdk/.gitignore index f6764db4c1..135a9d8508 100644 --- a/tools/ci-cdk/.gitignore +++ b/tools/ci-cdk/.gitignore @@ -8,3 +8,6 @@ build .cdk.staging cdk.out cdk-outputs.json + +# Generated Rust file +canary-wasm/src/bindings.rs diff --git a/tools/ci-cdk/README.md b/tools/ci-cdk/README.md index b47bfc0ada..34c78d6ea2 100644 --- a/tools/ci-cdk/README.md +++ b/tools/ci-cdk/README.md @@ -21,7 +21,7 @@ From there, you can just point the `canary-runner` to the `cdk-outputs.json` to ```bash cd canary-runner -cargo run -- --sdk-release-tag --musl --cdk-outputs ../cdk-outputs.json +cargo run -- run --sdk-release-tag --musl --cdk-output ../cdk-outputs.json ``` **NOTE:** You may want to add a `--profile` to the deploy command to select a specific credential diff --git a/tools/ci-cdk/canary-runner/src/build_bundle.rs b/tools/ci-cdk/canary-runner/src/build_bundle.rs index f4441289d0..59979b04b4 100644 --- a/tools/ci-cdk/canary-runner/src/build_bundle.rs +++ b/tools/ci-cdk/canary-runner/src/build_bundle.rs @@ -492,6 +492,9 @@ tracing-texray = "0.1.1" reqwest = { version = "0.11.14", features = ["rustls-tls"], default-features = false } edit-distance = "2" wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } +wasmtime = { version = "17.0.1", features = ["component-model"] } +wasmtime-wasi = "17.0.1" +wasmtime-wasi-http = "17.0.1" aws-config = { path = "some/sdk/path/aws-config", features = ["behavior-version-latest"] } aws-sdk-s3 = { path = "some/sdk/path/s3" } aws-sdk-ec2 = { path = "some/sdk/path/ec2" } @@ -559,6 +562,9 @@ tracing-texray = "0.1.1" reqwest = { version = "0.11.14", features = ["rustls-tls"], default-features = false } edit-distance = "2" wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } +wasmtime = { version = "17.0.1", features = ["component-model"] } +wasmtime-wasi = "17.0.1" +wasmtime-wasi-http = "17.0.1" aws-config = { version = "0.46.0", features = ["behavior-version-latest"] } aws-sdk-s3 = "0.20.0" aws-sdk-ec2 = "0.19.0" diff --git a/tools/ci-cdk/canary-wasm/src/bindings.rs b/tools/ci-cdk/canary-wasm/src/bindings.rs deleted file mode 100644 index 74af2df502..0000000000 --- a/tools/ci-cdk/canary-wasm/src/bindings.rs +++ /dev/null @@ -1,137 +0,0 @@ -// Generated by `wit-bindgen` 0.16.0. DO NOT EDIT! -pub mod exports { - pub mod aws { - pub mod component { - - #[allow(clippy::all)] - pub mod canary_interface { - #[used] - #[doc(hidden)] - #[cfg(target_arch = "wasm32")] - static __FORCE_SECTION_REF: fn() = super::super::super::super::__link_section; - const _: () = { - - #[doc(hidden)] - #[export_name = "aws:component/canary-interface#run-canary"] - #[allow(non_snake_case)] - unsafe extern "C" fn __export_run_canary() -> i32 { - #[allow(unused_imports)] - use wit_bindgen::rt::{alloc, vec::Vec, string::String}; - - // Before executing any other code, use this function to run all static - // constructors, if they have not yet been run. This is a hack required - // to work around wasi-libc ctors calling import functions to initialize - // the environment. - // - // This functionality will be removed once rust 1.69.0 is stable, at which - // point wasi-libc will no longer have this behavior. - // - // See - // https://github.com/bytecodealliance/preview2-prototyping/issues/99 - // for more details. - #[cfg(target_arch="wasm32")] - wit_bindgen::rt::run_ctors_once(); - - let result0 = <_GuestImpl as Guest>::run_canary(); - let ptr1 = _RET_AREA.0.as_mut_ptr() as i32; - match result0 { - Ok(e) => { { - *((ptr1 + 0) as *mut u8) = (0i32) as u8; - let vec3 = e; - let len3 = vec3.len() as i32; - let layout3 = alloc::Layout::from_size_align_unchecked(vec3.len() * 8, 4); - let result3 = if layout3.size() != 0 - { - let ptr = alloc::alloc(layout3); - if ptr.is_null() - { - alloc::handle_alloc_error(layout3); - } - ptr - }else {{ - ::core::ptr::null_mut() - }}; - for (i, e) in vec3.into_iter().enumerate() { - let base = result3 as i32 + (i as i32) * 8; - { - let vec2 = (e.into_bytes()).into_boxed_slice(); - let ptr2 = vec2.as_ptr() as i32; - let len2 = vec2.len() as i32; - ::core::mem::forget(vec2); - *((base + 4) as *mut i32) = len2; - *((base + 0) as *mut i32) = ptr2; - } - } - *((ptr1 + 8) as *mut i32) = len3; - *((ptr1 + 4) as *mut i32) = result3 as i32; - } }, - Err(e) => { { - *((ptr1 + 0) as *mut u8) = (1i32) as u8; - let vec4 = (e.into_bytes()).into_boxed_slice(); - let ptr4 = vec4.as_ptr() as i32; - let len4 = vec4.len() as i32; - ::core::mem::forget(vec4); - *((ptr1 + 8) as *mut i32) = len4; - *((ptr1 + 4) as *mut i32) = ptr4; - } }, - };ptr1 - } - - const _: () = { - #[doc(hidden)] - #[export_name = "cabi_post_aws:component/canary-interface#run-canary"] - #[allow(non_snake_case)] - unsafe extern "C" fn __post_return_run_canary(arg0: i32,) { - let l0 = i32::from(*((arg0 + 0) as *const u8)); - match l0 { - 0 => { - let l3 = *((arg0 + 4) as *const i32); - let l4 = *((arg0 + 8) as *const i32); - let base5 = l3; - let len5 = l4; - for i in 0..len5 { - let base = base5 + i *8; - { - let l1 = *((base + 0) as *const i32); - let l2 = *((base + 4) as *const i32); - wit_bindgen::rt::dealloc(l1, (l2) as usize, 1); - } - } - wit_bindgen::rt::dealloc(base5, (len5 as usize) * 8, 4); - }, - _ => { - let l6 = *((arg0 + 4) as *const i32); - let l7 = *((arg0 + 8) as *const i32); - wit_bindgen::rt::dealloc(l6, (l7) as usize, 1); - }, - } - } - }; - }; - use super::super::super::super::super::Component as _GuestImpl; - pub trait Guest { - fn run_canary() -> Result,wit_bindgen::rt::string::String>; - } - - #[allow(unused_imports)] - use wit_bindgen::rt::{alloc, vec::Vec, string::String}; - - #[repr(align(4))] - struct _RetArea([u8; 12]); - static mut _RET_AREA: _RetArea = _RetArea([0; 12]); - - } - - } - } -} - -#[cfg(target_arch = "wasm32")] -#[link_section = "component-type:canary-world"] -#[doc(hidden)] -pub static __WIT_BINDGEN_COMPONENT_TYPE: [u8; 462] = [3, 0, 12, 99, 97, 110, 97, 114, 121, 45, 119, 111, 114, 108, 100, 0, 97, 115, 109, 13, 0, 1, 0, 7, 70, 1, 65, 2, 1, 66, 4, 1, 112, 115, 1, 106, 1, 0, 1, 115, 1, 64, 0, 0, 1, 4, 0, 10, 114, 117, 110, 45, 99, 97, 110, 97, 114, 121, 1, 2, 4, 1, 30, 97, 119, 115, 58, 99, 111, 109, 112, 111, 110, 101, 110, 116, 47, 99, 97, 110, 97, 114, 121, 45, 105, 110, 116, 101, 114, 102, 97, 99, 101, 5, 0, 11, 22, 1, 0, 16, 99, 97, 110, 97, 114, 121, 45, 105, 110, 116, 101, 114, 102, 97, 99, 101, 3, 0, 0, 7, 104, 1, 65, 2, 1, 65, 2, 1, 66, 4, 1, 112, 115, 1, 106, 1, 0, 1, 115, 1, 64, 0, 0, 1, 4, 0, 10, 114, 117, 110, 45, 99, 97, 110, 97, 114, 121, 1, 2, 4, 1, 30, 97, 119, 115, 58, 99, 111, 109, 112, 111, 110, 101, 110, 116, 47, 99, 97, 110, 97, 114, 121, 45, 105, 110, 116, 101, 114, 102, 97, 99, 101, 5, 0, 4, 1, 26, 97, 119, 115, 58, 99, 111, 109, 112, 111, 110, 101, 110, 116, 47, 99, 97, 110, 97, 114, 121, 45, 119, 111, 114, 108, 100, 4, 0, 11, 18, 1, 0, 12, 99, 97, 110, 97, 114, 121, 45, 119, 111, 114, 108, 100, 3, 2, 0, 0, 142, 1, 12, 112, 97, 99, 107, 97, 103, 101, 45, 100, 111, 99, 115, 0, 123, 34, 100, 111, 99, 115, 34, 58, 34, 42, 92, 110, 32, 42, 32, 67, 111, 112, 121, 114, 105, 103, 104, 116, 32, 65, 109, 97, 122, 111, 110, 46, 99, 111, 109, 44, 32, 73, 110, 99, 46, 32, 111, 114, 32, 105, 116, 115, 32, 97, 102, 102, 105, 108, 105, 97, 116, 101, 115, 46, 32, 65, 108, 108, 32, 82, 105, 103, 104, 116, 115, 32, 82, 101, 115, 101, 114, 118, 101, 100, 46, 92, 110, 32, 42, 32, 83, 80, 68, 88, 45, 76, 105, 99, 101, 110, 115, 101, 45, 73, 100, 101, 110, 116, 105, 102, 105, 101, 114, 58, 32, 65, 112, 97, 99, 104, 101, 45, 50, 46, 48, 92, 110, 32, 42, 47, 34, 125, 0, 70, 9, 112, 114, 111, 100, 117, 99, 101, 114, 115, 1, 12, 112, 114, 111, 99, 101, 115, 115, 101, 100, 45, 98, 121, 2, 13, 119, 105, 116, 45, 99, 111, 109, 112, 111, 110, 101, 110, 116, 6, 48, 46, 49, 56, 46, 50, 16, 119, 105, 116, 45, 98, 105, 110, 100, 103, 101, 110, 45, 114, 117, 115, 116, 6, 48, 46, 49, 54, 46, 48]; - -#[inline(never)] -#[doc(hidden)] -#[cfg(target_arch = "wasm32")] -pub fn __link_section() {} From 147a26a3845dec4606f91c8efea1041c357e20d3 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 20 Feb 2024 23:38:31 -0800 Subject: [PATCH 66/72] Increase canary timeouts wasm canary is slow --- tools/ci-cdk/canary-lambda/src/main.rs | 2 +- tools/ci-cdk/canary-runner/src/run.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci-cdk/canary-lambda/src/main.rs b/tools/ci-cdk/canary-lambda/src/main.rs index df0b36c45a..df6a3d5c6a 100644 --- a/tools/ci-cdk/canary-lambda/src/main.rs +++ b/tools/ci-cdk/canary-lambda/src/main.rs @@ -123,7 +123,7 @@ async fn lambda_main(sdk_config: SdkConfig) -> Result { } async fn canary_result(handle: JoinHandle>) -> Result<(), String> { - match timeout(Duration::from_secs(20), handle).await { + match timeout(Duration::from_secs(180), handle).await { Err(_timeout) => Err("canary timed out".into()), Ok(Ok(result)) => match result { Ok(_) => Ok(()), diff --git a/tools/ci-cdk/canary-runner/src/run.rs b/tools/ci-cdk/canary-runner/src/run.rs index 8fd8cb4064..6d6d4e9f63 100644 --- a/tools/ci-cdk/canary-runner/src/run.rs +++ b/tools/ci-cdk/canary-runner/src/run.rs @@ -388,7 +388,7 @@ async fn create_lambda_fn( ) .publish(true) .environment(env_builder.build()) - .timeout(60) + .timeout(180) .send() .await .context(here!("failed to create canary Lambda function"))?; From 0c4144092dec6fb76c13f21a781915d5ba20c925 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 26 Feb 2024 11:58:54 -0800 Subject: [PATCH 67/72] Apply suggestions from code review Co-authored-by: John DiSanti --- rust-runtime/aws-smithy-wasm/src/wasi.rs | 2 ++ tools/ci-cdk/canary-runner/src/build_bundle.rs | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index e8bc4ddf97..d7e6198040 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -28,6 +28,7 @@ use wasi::http::{ /// Builder for [`WasiHttpClient`]. Currently empty, but allows for future /// config options to be added in a backwards compatible manner. #[derive(Default, Debug)] +#[non_exhaustive] pub struct WasiHttpClientBuilder {} impl WasiHttpClientBuilder { @@ -47,6 +48,7 @@ impl WasiHttpClientBuilder { /// order to route the HTTP requests through the WebAssembly host. The host must /// support the WASI HTTP proposal as defined in the Preview 2 specification. #[derive(Debug, Clone)] +#[non_exhaustive] pub struct WasiHttpClient {} impl HttpClient for WasiHttpClient { diff --git a/tools/ci-cdk/canary-runner/src/build_bundle.rs b/tools/ci-cdk/canary-runner/src/build_bundle.rs index 59979b04b4..15ec5c3cbf 100644 --- a/tools/ci-cdk/canary-runner/src/build_bundle.rs +++ b/tools/ci-cdk/canary-runner/src/build_bundle.rs @@ -267,7 +267,6 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { .expect("Current dir") .join("../canary-wasm/Cargo.toml"); - println!("MAINFEST PATH: {wasm_manifest_path:#?}"); if !opt.manifest_only { // Compile the canary Lambda let mut command = Command::new("cargo"); @@ -289,7 +288,7 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { .arg("--release") .arg("--manifest-path") .arg(&wasm_manifest_path); - handle_failure("cargo build", &wasm_command.output()?)?; + handle_failure("cargo component build", &wasm_command.output()?)?; // Bundle the Lambda let repository_root = find_git_repository_root("smithy-rs", canary_path)?; @@ -329,7 +328,7 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result> { // Write the wasm bin to the zip zip.start_file( "aws_sdk_rust_lambda_canary_wasm.wasm", - zip::write::FileOptions::default().unix_permissions(0o755), + zip::write::FileOptions::default().unix_permissions(0o644), ) .context(here!())?; zip.write_all(&fs::read(wasm_bin_path).context(here!("read wasm bin"))?) From c883aad007e0f5929794be8f5370854679a5f213 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 26 Feb 2024 13:18:41 -0800 Subject: [PATCH 68/72] Updates from PR feedback Remove WasmSleep in favor of existing TokioSleep Update the targeting in wasm crates dependencies Updating comments around setting timeouts --- rust-runtime/aws-smithy-wasm/Cargo.toml | 4 +++- rust-runtime/aws-smithy-wasm/src/lib.rs | 4 ---- rust-runtime/aws-smithy-wasm/src/wasi.rs | 7 +++++-- rust-runtime/aws-smithy-wasm/src/wasm.rs | 20 -------------------- tools/ci-cdk/canary-wasm/Cargo.lock | 1 + tools/ci-cdk/canary-wasm/Cargo.toml | 1 + tools/ci-cdk/canary-wasm/src/lib.rs | 5 +++-- 7 files changed, 13 insertions(+), 29 deletions(-) delete mode 100644 rust-runtime/aws-smithy-wasm/src/wasm.rs diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index ea3f89f9b4..fc06b70d20 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -10,7 +10,7 @@ edition = "2021" license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" -[target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +[dependencies] aws-smithy-async = { path = "../aws-smithy-async" } aws-smithy-runtime-api = { path = "../aws-smithy-runtime-api", features = ["http-1x"]} aws-smithy-http = { path = "../aws-smithy-http" } @@ -20,6 +20,8 @@ http = "1.0.0" tokio = { version = "1.36.0", features = ["rt", "macros", "time"] } tower = "0.4.13" tracing = "0.1.40" + +[target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] wasi = "0.12.0+wasi-0.2.0" [package.metadata.docs.rs] diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs index caf6310972..c2e1ae05f6 100644 --- a/rust-runtime/aws-smithy-wasm/src/lib.rs +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -19,7 +19,3 @@ /// Tools for using Smithy SDKs in WASI environments #[cfg(all(target_family = "wasm", target_os = "wasi"))] pub mod wasi; - -/// Tools for using Smithy SDKs in WASM environments -#[cfg(target_family = "wasm")] -pub mod wasm; diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index d7e6198040..a72c8b7004 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -151,6 +151,8 @@ struct WasiRequestOptions(Option); impl From<&HttpConnectorSettings> for WasiRequestOptions { fn from(value: &HttpConnectorSettings) -> Self { //The WASI Duration is nanoseconds represented as u64 + //Note: that the HttpConnectorSettings provides nanoseconds as u128 + //so here we are clamping to u64::MAX if the value is above that let connect_timeout = value .connect_timeout() .map(|dur| u64::try_from(dur.as_nanos()).unwrap_or(u64::MAX)); @@ -158,8 +160,9 @@ impl From<&HttpConnectorSettings> for WasiRequestOptions { .read_timeout() .map(|dur| u64::try_from(dur.as_nanos()).unwrap_or(u64::MAX)); - //Note: unable to find any documentation about what timeout values are not supported - //so not sure under what circumstances these set operations would actually fail + //Note: these only fail if setting this particular type of timeout is not + //supported. Spec compliant runtimes should always support these so it is + //unlikely to be an issue. let wasi_http_opts = wasi_http::RequestOptions::new(); wasi_http_opts .set_connect_timeout(connect_timeout) diff --git a/rust-runtime/aws-smithy-wasm/src/wasm.rs b/rust-runtime/aws-smithy-wasm/src/wasm.rs deleted file mode 100644 index c438aa1acc..0000000000 --- a/rust-runtime/aws-smithy-wasm/src/wasm.rs +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -//! WASM Tools for the SDK -use aws_smithy_async::rt::sleep::{AsyncSleep, Sleep}; - -/// A struct implementing the AsyncSleep trait that can be used in -/// Wasm environments -#[derive(Debug, Clone)] -pub struct WasmSleep; - -impl AsyncSleep for WasmSleep { - fn sleep(&self, duration: std::time::Duration) -> Sleep { - Sleep::new(Box::pin(async move { - tokio::time::sleep(duration).await; - })) - } -} diff --git a/tools/ci-cdk/canary-wasm/Cargo.lock b/tools/ci-cdk/canary-wasm/Cargo.lock index aac8afab10..4601b3024d 100644 --- a/tools/ci-cdk/canary-wasm/Cargo.lock +++ b/tools/ci-cdk/canary-wasm/Cargo.lock @@ -90,6 +90,7 @@ version = "0.1.0" dependencies = [ "aws-config", "aws-sdk-s3", + "aws-smithy-async", "aws-smithy-wasm", "tokio", "wit-bindgen", diff --git a/tools/ci-cdk/canary-wasm/Cargo.toml b/tools/ci-cdk/canary-wasm/Cargo.toml index 32e2ad4f42..de883a420c 100644 --- a/tools/ci-cdk/canary-wasm/Cargo.toml +++ b/tools/ci-cdk/canary-wasm/Cargo.toml @@ -15,6 +15,7 @@ wit-bindgen = { version = "0.16.0", features = ["macros", "realloc"] } aws-config = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-config", default-features = false, features = ["behavior-version-latest"] } aws-sdk-s3 = { path = "../../../aws/sdk/build/aws-sdk/sdk/s3", default-features = false } aws-smithy-wasm = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-smithy-wasm" } +aws-smithy-async = { path = "../../../aws/sdk/build/aws-sdk/sdk/aws-smithy-async", default-features = false, features = ["rt-tokio"]} [lib] crate-type = ["cdylib"] diff --git a/tools/ci-cdk/canary-wasm/src/lib.rs b/tools/ci-cdk/canary-wasm/src/lib.rs index d2c7ed365a..3a56a67fb8 100644 --- a/tools/ci-cdk/canary-wasm/src/lib.rs +++ b/tools/ci-cdk/canary-wasm/src/lib.rs @@ -6,7 +6,7 @@ use aws_config::Region; use aws_sdk_s3 as s3; use aws_smithy_wasm::wasi::WasiHttpClientBuilder; -use aws_smithy_wasm::wasm::WasmSleep; +use aws_smithy_async::rt::sleep::TokioSleep; //Generates the Rust bindings from the wit file wit_bindgen::generate!({ @@ -31,11 +31,12 @@ impl exports::aws::component::canary_interface::Guest for Component { async fn run_canary() -> Result, String> { let http_client = WasiHttpClientBuilder::new().build(); + let sleep = TokioSleep::new(); let config = aws_config::from_env() .region(Region::new("us-east-2")) .no_credentials() .http_client(http_client) - .sleep_impl(WasmSleep) + .sleep_impl(sleep) .load() .await; From a778a18d223af46e4b96db88028284c2655c4a84 Mon Sep 17 00:00:00 2001 From: Landon James Date: Mon, 26 Feb 2024 15:00:15 -0800 Subject: [PATCH 69/72] Bump version of wasmtime-cli in Dockerfile Moving wasmtime-cli off nightly since MSRV bump Removing all dependency targeting to appease tests Removing the target on the deps in aws-smithy-wasm's Cargo.toml caused the cargo udeps tests to fail indicating those were unused dependencies. This was caused by the target clause on pub mod wasi in the crate's lib.rs, so that also had to be removed. --- rust-runtime/aws-smithy-wasm/Cargo.toml | 7 ++----- rust-runtime/aws-smithy-wasm/src/lib.rs | 1 - tools/ci-build/Dockerfile | 4 ++-- tools/ci-cdk/canary-wasm/Cargo.lock | 28 ++++++++++++------------- tools/ci-cdk/canary-wasm/src/lib.rs | 2 +- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/rust-runtime/aws-smithy-wasm/Cargo.toml b/rust-runtime/aws-smithy-wasm/Cargo.toml index fc06b70d20..7dc1dd4cc1 100644 --- a/rust-runtime/aws-smithy-wasm/Cargo.toml +++ b/rust-runtime/aws-smithy-wasm/Cargo.toml @@ -11,17 +11,14 @@ license = "Apache-2.0" repository = "https://github.com/awslabs/smithy-rs" [dependencies] -aws-smithy-async = { path = "../aws-smithy-async" } aws-smithy-runtime-api = { path = "../aws-smithy-runtime-api", features = ["http-1x"]} aws-smithy-http = { path = "../aws-smithy-http" } aws-smithy-types = { path = "../aws-smithy-types" } bytes = "1" http = "1.0.0" -tokio = { version = "1.36.0", features = ["rt", "macros", "time"] } -tower = "0.4.13" tracing = "0.1.40" - -[target.'cfg(all(target_family = "wasm", target_os = "wasi"))'.dependencies] +# Note the wasi crate will only build for target wasm32-wasi, but having a target +# statement here breaks some of the CI tests, so we leave it with the rest of the deps wasi = "0.12.0+wasi-0.2.0" [package.metadata.docs.rs] diff --git a/rust-runtime/aws-smithy-wasm/src/lib.rs b/rust-runtime/aws-smithy-wasm/src/lib.rs index c2e1ae05f6..84e9b7dd9f 100644 --- a/rust-runtime/aws-smithy-wasm/src/lib.rs +++ b/rust-runtime/aws-smithy-wasm/src/lib.rs @@ -17,5 +17,4 @@ //! Smithy WebAssembly /// Tools for using Smithy SDKs in WASI environments -#[cfg(all(target_family = "wasm", target_os = "wasi"))] pub mod wasi; diff --git a/tools/ci-build/Dockerfile b/tools/ci-build/Dockerfile index 29de537696..a6bc44128d 100644 --- a/tools/ci-build/Dockerfile +++ b/tools/ci-build/Dockerfile @@ -121,9 +121,9 @@ ARG wasm_pack_version=0.11.0 RUN cargo install wasm-pack --locked --version ${wasm_pack_version} FROM install_rust AS wasmtime -ARG cargo_wasmtime_version=17.0.0 +ARG cargo_wasmtime_version=18.0.1 ARG rust_nightly_version -RUN cargo +${rust_nightly_version} install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} +RUN cargo install wasmtime-cli --features="component-model" --locked --version ${cargo_wasmtime_version} FROM install_rust AS cargo_component ARG cargo_component_version=0.7.1 diff --git a/tools/ci-cdk/canary-wasm/Cargo.lock b/tools/ci-cdk/canary-wasm/Cargo.lock index 4601b3024d..112f72a1dc 100644 --- a/tools/ci-cdk/canary-wasm/Cargo.lock +++ b/tools/ci-cdk/canary-wasm/Cargo.lock @@ -31,7 +31,7 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "1.1.6" +version = "1.1.7" dependencies = [ "aws-credential-types", "aws-runtime", @@ -54,7 +54,7 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.1.6" +version = "1.1.7" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -64,7 +64,7 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.1.6" +version = "1.1.7" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -146,7 +146,7 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.1.6" +version = "1.1.7" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -168,7 +168,7 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.1.6" +version = "1.1.7" dependencies = [ "futures-util", "pin-project-lite", @@ -177,7 +177,7 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.5" +version = "0.60.6" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -205,7 +205,7 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.5" +version = "0.60.6" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -224,14 +224,14 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.60.5" +version = "0.60.6" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.60.5" +version = "0.60.6" dependencies = [ "aws-smithy-types", "urlencoding", @@ -239,7 +239,7 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.1.6" +version = "1.1.7" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -258,7 +258,7 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.1.6" +version = "1.1.7" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -272,7 +272,7 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.1.6" +version = "1.1.7" dependencies = [ "base64-simd", "bytes", @@ -307,14 +307,14 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.5" +version = "0.60.6" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.1.6" +version = "1.1.7" dependencies = [ "aws-credential-types", "aws-smithy-async", diff --git a/tools/ci-cdk/canary-wasm/src/lib.rs b/tools/ci-cdk/canary-wasm/src/lib.rs index 3a56a67fb8..6d90399ec0 100644 --- a/tools/ci-cdk/canary-wasm/src/lib.rs +++ b/tools/ci-cdk/canary-wasm/src/lib.rs @@ -5,8 +5,8 @@ use aws_config::Region; use aws_sdk_s3 as s3; -use aws_smithy_wasm::wasi::WasiHttpClientBuilder; use aws_smithy_async::rt::sleep::TokioSleep; +use aws_smithy_wasm::wasi::WasiHttpClientBuilder; //Generates the Rust bindings from the wit file wit_bindgen::generate!({ From 6a174b3fb80eae92bfd68f39a90945f1bc3f867c Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 27 Feb 2024 09:57:01 -0800 Subject: [PATCH 70/72] Adding allowed external types to aws-smity-wasm --- rust-runtime/aws-smithy-wasm/external-types.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust-runtime/aws-smithy-wasm/external-types.toml b/rust-runtime/aws-smithy-wasm/external-types.toml index 8a2e2ce81a..25256eeaba 100644 --- a/rust-runtime/aws-smithy-wasm/external-types.toml +++ b/rust-runtime/aws-smithy-wasm/external-types.toml @@ -1,3 +1,5 @@ allowed_external_types = [ - "aws_smithy_types::*", + "aws_smithy_runtime_api::client::http::HttpClient", + "aws_smithy_runtime_api::client::http::SharedHttpClient", + "aws_smithy_runtime_api::client::http::HttpConnector" ] From f6720a397d4daf00b9367ae51aecbb36797e1268 Mon Sep 17 00:00:00 2001 From: Landon James Date: Tue, 27 Feb 2024 14:01:56 -0800 Subject: [PATCH 71/72] Update rust-runtime/aws-smithy-wasm/src/wasi.rs Co-authored-by: John DiSanti --- rust-runtime/aws-smithy-wasm/src/wasi.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-runtime/aws-smithy-wasm/src/wasi.rs b/rust-runtime/aws-smithy-wasm/src/wasi.rs index a72c8b7004..1c32d3ee94 100644 --- a/rust-runtime/aws-smithy-wasm/src/wasi.rs +++ b/rust-runtime/aws-smithy-wasm/src/wasi.rs @@ -66,7 +66,7 @@ impl HttpClient for WasiHttpClient { /// HTTP connector used in WASI environment #[derive(Debug, Clone)] -pub struct WasiHttpConnector { +struct WasiHttpConnector { options: WasiRequestOptions, } From 778bfb096fe6f1c36acb0a25566535d6b012e386 Mon Sep 17 00:00:00 2001 From: John DiSanti Date: Tue, 27 Feb 2024 14:34:58 -0800 Subject: [PATCH 72/72] Update CHANGELOG.next.toml --- CHANGELOG.next.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.toml b/CHANGELOG.next.toml index 360d9322d7..a3f0176ff0 100644 --- a/CHANGELOG.next.toml +++ b/CHANGELOG.next.toml @@ -11,20 +11,20 @@ # meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client | server | all"} # author = "rcoh" -[[aws-sdk-rust]] -message = "The MSRV has been increase to 1.74.1" -references = ["smithy-rs#3410"] -meta = { "bug" = false, "breaking" = false, tada = false } -author = "rcoh" - [[aws-sdk-rust]] message = "EKS Pod Identity is now supported as part of the default ECS credential provider." references = ["smithy-rs#3416"] meta = { "breaking" = false, "bug" = false, "tada" = true } author = "jackkleeman" +[[aws-sdk-rust]] +message = "Added aws-smithy-wasm crate to enable SDK use in WASI compliant environments" +references = ["smithy-rs#2087", "smithy-rs#2520", "smithy-rs#3409", "aws-sdk-rust#59"] +meta = { "breaking" = false, "tada" = true, "bug" = false } +author = "landonxjames" + [[smithy-rs]] -message = "Add aws-smithy-wasm crate to enable SDK use in wasi compliant environments" +message = "Added aws-smithy-wasm crate to enable SDK use in WASI compliant environments" references = ["smithy-rs#2087", "smithy-rs#2520", "smithy-rs#3409"] meta = { "breaking" = false, "tada" = true, "bug" = false, "target" = "client"} author = "landonxjames"