add support for nullable struct members when generating AWS SDKs

CHANGELOG.next.toml

@@ -34,6 +34,21 @@ references = ["smithy-rs#2911"]
 meta = { "breaking" = true, "tada" = false, "bug" = false }
 author = "Velfi"
 
+[[aws-sdk-rust]]
+message = "Struct members modeled as required are no longer wrapped in `Option`s [when possible](https://smithy.io/2.0/spec/aggregate-types.html#structure-member-optionality). For upgrade guidance and more info, see [here](https://github.com/awslabs/smithy-rs/discussions/2929)."
+references = ["smithy-rs#2916", "aws-sdk-rust#536"]
+meta = { "breaking" = true, "tada" = true, "bug" = false }
+author = "Velfi"
+
+[[smithy-rs]]
+message = """
+Support for Smithy IDLv2 nullability is now enabled by default. You can maintain the old behavior by setting `nullabilityCheckMode: "CLIENT_ZERO_VALUE_V1" in your codegen config.
+For upgrade guidance and more info, see [here](https://github.com/awslabs/smithy-rs/discussions/2929).
+"""
+references = ["smithy-rs#2916", "smithy-rs#1767"]
+meta = { "breaking" = false, "tada" = true, "bug" = false, "target" = "client"}
+author = "Velfi"
+
 [[aws-sdk-rust]]
 message = """
 All versions of SigningParams have been updated to contain an [`Identity`](https://docs.rs/aws-smithy-runtime-api/latest/aws_smithy_runtime_api/client/identity/struct.Identity.html)

aws/rust-runtime/aws-config/src/sts/util.rs

@@ -17,24 +17,15 @@ pub(crate) fn into_credentials(
 ) -> provider::Result {
     let sts_credentials = sts_credentials
         .ok_or_else(|| CredentialsError::unhandled("STS credentials must be defined"))?;
-    let expiration = SystemTime::try_from(
-        sts_credentials
-            .expiration
-            .ok_or_else(|| CredentialsError::unhandled("missing expiration"))?,
-    )
-    .map_err(|_| {
+    let expiration = SystemTime::try_from(sts_credentials.expiration).map_err(|_| {
         CredentialsError::unhandled(
             "credential expiration time cannot be represented by a SystemTime",
         )
     })?;
     Ok(AwsCredentials::new(
-        sts_credentials
-            .access_key_id
-            .ok_or_else(|| CredentialsError::unhandled("access key id missing from result"))?,
-        sts_credentials
-            .secret_access_key
-            .ok_or_else(|| CredentialsError::unhandled("secret access token missing"))?,
-        sts_credentials.session_token,
+        sts_credentials.access_key_id,
+        sts_credentials.secret_access_key,
+        Some(sts_credentials.session_token),
         Some(expiration),
         provider_name,
     ))

aws/sdk-adhoc-test/build.gradle.kts

@@ -35,40 +35,46 @@ dependencies {
     implementation("software.amazon.smithy:smithy-aws-protocol-tests:$smithyVersion")
     implementation("software.amazon.smithy:smithy-protocol-test-traits:$smithyVersion")
     implementation("software.amazon.smithy:smithy-aws-traits:$smithyVersion")
+    implementation("software.amazon.smithy:smithy-model:$smithyVersion")
 }
 
-val allCodegenTests = listOf(
-    CodegenTest(
-        "com.amazonaws.apigateway#BackplaneControlService",
-        "apigateway",
-        imports = listOf("models/apigateway-rules.smithy"),
+fun getNullabilityCheckMode(): String = properties.get("nullability.check.mode") ?: "CLIENT_CAREFUL"
+
+fun baseTest(service: String, module: String, imports: List<String> = listOf()): CodegenTest {
+    return CodegenTest(
+        service = service,
+        module = module,
+        imports = imports,
+        extraCodegenConfig = """
+            "includeFluentClient": false,
+            "nullabilityCheckMode": "${getNullabilityCheckMode()}"
+        """,
         extraConfig = """
-            ,
-            "codegen": {
-                "includeFluentClient": false
-            },
-            "customizationConfig": {
+            , "customizationConfig": {
                 "awsSdk": {
-                    "generateReadme": false
+                    "generateReadme": false,
+                    "requireEndpointResolver": false
                 }
             }
         """,
+    )
+}
+
+val allCodegenTests = listOf(
+    baseTest(
+        "com.amazonaws.apigateway#BackplaneControlService",
+        "apigateway",
+        imports = listOf("models/apigateway-rules.smithy"),
     ),
-    CodegenTest(
+    baseTest(
         "com.amazonaws.testservice#TestService",
         "endpoint-test-service",
         imports = listOf("models/single-static-endpoint.smithy"),
-        extraConfig = """
-            ,
-            "codegen": {
-                "includeFluentClient": false
-            },
-            "customizationConfig": {
-                "awsSdk": {
-                    "generateReadme": false
-                }
-            }
-        """,
+    ),
+    baseTest(
+        "com.amazonaws.testservice#RequiredValues",
+        "required-values",
+        imports = listOf("models/required-value-test.smithy"),
     ),
 )
 

aws/sdk-adhoc-test/models/required-value-test.smithy

@@ -0,0 +1,28 @@
+$version: "1.0"
+
+namespace com.amazonaws.testservice
+
+use aws.api#service
+use aws.protocols#restJson1
+
+@restJson1
+@title("Test Service")
+@service(sdkId: "Test")
+@aws.auth#sigv4(name: "test-service")
+service RequiredValues {
+    operations: [TestOperation]
+}
+
+@http(method: "GET", uri: "/")
+operation TestOperation {
+    errors: [Error]
+}
+
+@error("client")
+structure Error {
+    @required
+    requestId: String
+
+    @required
+    message: String
+}

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/BaseRequestIdDecorator.kt

@@ -5,7 +5,9 @@
 
 package software.amazon.smithy.rustsdk
 
+import software.amazon.smithy.model.shapes.MemberShape
 import software.amazon.smithy.model.shapes.OperationShape
+import software.amazon.smithy.model.shapes.StructureShape
 import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext
 import software.amazon.smithy.rust.codegen.client.smithy.ClientRustModule
 import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator
@@ -19,13 +21,15 @@ import software.amazon.smithy.rust.codegen.core.rustlang.rustBlock
 import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate
 import software.amazon.smithy.rust.codegen.core.rustlang.writable
 import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType
+import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope
 import software.amazon.smithy.rust.codegen.core.smithy.RustCrate
 import software.amazon.smithy.rust.codegen.core.smithy.generators.BuilderCustomization
 import software.amazon.smithy.rust.codegen.core.smithy.generators.BuilderSection
 import software.amazon.smithy.rust.codegen.core.smithy.generators.StructureCustomization
 import software.amazon.smithy.rust.codegen.core.smithy.generators.StructureSection
 import software.amazon.smithy.rust.codegen.core.smithy.generators.error.ErrorImplCustomization
 import software.amazon.smithy.rust.codegen.core.smithy.generators.error.ErrorImplSection
+import software.amazon.smithy.rust.codegen.core.smithy.isOptional
 import software.amazon.smithy.rust.codegen.core.smithy.traits.SyntheticOutputTrait
 import software.amazon.smithy.rust.codegen.core.util.hasTrait
 
@@ -72,6 +76,10 @@ abstract class BaseRequestIdDecorator : ClientCodegenDecorator {
         }
     }
 
+    open fun asMemberShape(container: StructureShape): MemberShape? {
+        return container.members().firstOrNull { member -> member.memberName.lowercase() == "requestid" }
+    }
+
     private inner class RequestIdOperationCustomization(private val codegenContext: ClientCodegenContext) :
         OperationCustomization() {
         override fun section(section: OperationSection): Writable = writable {
@@ -82,19 +90,22 @@ abstract class BaseRequestIdDecorator : ClientCodegenDecorator {
                         "apply_to_error" to applyToError(codegenContext),
                     )
                 }
+
                 is OperationSection.MutateOutput -> {
                     rust(
                         "output._set_$fieldName(#T::$accessorFunctionName(${section.responseHeadersName}).map(str::to_string));",
                         accessorTrait(codegenContext),
                     )
                 }
+
                 is OperationSection.BeforeParseResponse -> {
                     rustTemplate(
                         "#{tracing}::debug!($fieldName = ?#{trait}::$accessorFunctionName(${section.responseName}));",
                         "tracing" to RuntimeType.Tracing,
                         "trait" to accessorTrait(codegenContext),
                     )
                 }
+
                 else -> {}
             }
         }
@@ -123,8 +134,17 @@ abstract class BaseRequestIdDecorator : ClientCodegenDecorator {
                         rustBlock("fn $accessorFunctionName(&self) -> Option<&str>") {
                             rustBlock("match self") {
                                 section.allErrors.forEach { error ->
+                                    val optional = asMemberShape(error)?.let { member ->
+                                        codegenContext.symbolProvider.toSymbol(member).isOptional()
+                                    } ?: true
+                                    val wrapped = writable {
+                                        when (optional) {
+                                            false -> rustTemplate("#{Some}(e.$accessorFunctionName())", *preludeScope)
+                                            true -> rustTemplate("e.$accessorFunctionName()")
+                                        }
+                                    }
                                     val sym = codegenContext.symbolProvider.toSymbol(error)
-                                    rust("Self::${sym.name}(e) => e.$accessorFunctionName(),")
+                                    rust("Self::${sym.name}(e) => #T,", wrapped)
                                 }
                                 rust("Self::Unhandled(e) => e.$accessorFunctionName(),")
                             }

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExtendedRequestIdDecorator.kt

@@ -5,6 +5,8 @@
 
 package software.amazon.smithy.rustsdk.customize.s3
 
+import software.amazon.smithy.model.shapes.MemberShape
+import software.amazon.smithy.model.shapes.StructureShape
 import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext
 import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType
 import software.amazon.smithy.rustsdk.BaseRequestIdDecorator
@@ -17,6 +19,10 @@ class S3ExtendedRequestIdDecorator : BaseRequestIdDecorator() {
     override val fieldName: String = "extended_request_id"
     override val accessorFunctionName: String = "extended_request_id"
 
+    override fun asMemberShape(container: StructureShape): MemberShape? {
+        return null
+    }
+
     private val requestIdModule: RuntimeType =
         RuntimeType.forInlineDependency(InlineAwsDependency.forRustFile("s3_request_id"))
 

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/timestream/TimestreamDecorator.kt

@@ -60,12 +60,12 @@ class TimestreamDecorator : ClientCodegenDecorator {
                         client.describe_endpoints().send().await.map_err(|e| {
                             #{ResolveEndpointError}::from_source("failed to call describe_endpoints", e)
                         })?;
-                    let endpoint = describe_endpoints.endpoints().unwrap().get(0).unwrap();
+                    let endpoint = describe_endpoints.endpoints().get(0).unwrap();
                     let expiry = client.config().time_source().expect("checked when ep discovery was enabled").now()
                         + #{Duration}::from_secs(endpoint.cache_period_in_minutes() as u64 * 60);
                     Ok((
                         #{Endpoint}::builder()
-                            .url(format!("https://{}", endpoint.address().unwrap()))
+                            .url(format!("https://{}", endpoint.address()))
                             .build(),
                         expiry,
                     ))

aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/customize/ec2/EC2MakePrimitivesOptionalTest.kt

@@ -6,15 +6,22 @@
 package software.amazon.smithy.rustsdk.customize.ec2
 
 import io.kotest.matchers.shouldBe
-import org.junit.jupiter.api.Test
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.CsvSource
 import software.amazon.smithy.model.knowledge.NullableIndex
 import software.amazon.smithy.model.shapes.StructureShape
 import software.amazon.smithy.rust.codegen.core.testutil.asSmithyModel
 import software.amazon.smithy.rust.codegen.core.util.lookup
 
 internal class EC2MakePrimitivesOptionalTest {
-    @Test
-    fun `primitive shapes are boxed`() {
+    @ParameterizedTest
+    @CsvSource(
+        "CLIENT",
+        "CLIENT_CAREFUL",
+        "CLIENT_ZERO_VALUE_V1",
+        "CLIENT_ZERO_VALUE_V1_NO_INPUT",
+    )
+    fun `primitive shapes are boxed`(nullabilityCheckMode: NullableIndex.CheckMode) {
         val baseModel = """
             namespace test
             structure Primitives {
@@ -36,7 +43,7 @@ internal class EC2MakePrimitivesOptionalTest {
         val nullableIndex = NullableIndex(model)
         val struct = model.lookup<StructureShape>("test#Primitives")
         struct.members().forEach {
-            nullableIndex.isMemberNullable(it, NullableIndex.CheckMode.CLIENT_ZERO_VALUE_V1) shouldBe true
+            nullableIndex.isMemberNullable(it, nullabilityCheckMode) shouldBe true
         }
     }
 }

aws/sdk/build.gradle.kts

@@ -60,6 +60,7 @@ val crateVersioner by lazy { aws.sdk.CrateVersioner.defaultFor(rootProject, prop
 
 fun getRustMSRV(): String = properties.get("rust.msrv") ?: throw Exception("Rust MSRV missing")
 fun getPreviousReleaseVersionManifestPath(): String? = properties.get("aws.sdk.previous.release.versions.manifest")
+fun getNullabilityCheckMode(): String = properties.get("nullability.check.mode") ?: "CLIENT_CAREFUL"
 
 fun loadServiceMembership(): Membership {
     val membershipOverride = properties.get("aws.services")?.let { parseMembership(it) }
@@ -103,7 +104,8 @@ fun generateSmithyBuild(services: AwsServices): String {
                             "renameErrors": false,
                             "debugMode": $debugMode,
                             "eventStreamAllowList": [$eventStreamAllowListMembers],
-                            "enableUserConfigurableRuntimePlugins": false
+                            "enableUserConfigurableRuntimePlugins": false,
+                            "nullabilityCheckMode": "${getNullabilityCheckMode()}"
                         },
                         "service": "${service.service}",
                         "module": "$moduleName",

aws/sdk/integration-tests/Cargo.toml

@@ -14,7 +14,7 @@ members = [
     "s3",
     "s3control",
     "sts",
-    "transcribestreaming",
     "timestreamquery",
+    "transcribestreaming",
     "webassembly",
 ]

aws/sdk/integration-tests/dynamodb/tests/movies.rs

@@ -28,31 +28,36 @@ async fn create_table(client: &Client, table_name: &str) {
             KeySchemaElement::builder()
                 .attribute_name("year")
                 .key_type(KeyType::Hash)
-                .build(),
+                .build()
+                .unwrap(),
         )
         .key_schema(
             KeySchemaElement::builder()
                 .attribute_name("title")
                 .key_type(KeyType::Range)
-                .build(),
+                .build()
+                .unwrap(),
         )
         .attribute_definitions(
             AttributeDefinition::builder()
                 .attribute_name("year")
                 .attribute_type(ScalarAttributeType::N)
-                .build(),
+                .build()
+                .unwrap(),
         )
         .attribute_definitions(
             AttributeDefinition::builder()
                 .attribute_name("title")
                 .attribute_type(ScalarAttributeType::S)
-                .build(),
+                .build()
+                .unwrap(),
         )
         .provisioned_throughput(
             ProvisionedThroughput::builder()
                 .read_capacity_units(10)
                 .write_capacity_units(10)
-                .build(),
+                .build()
+                .unwrap(),
         )
         .send()
         .await