Skip to content

Commit

Permalink
Fix S3 optional auth (#2907)
Browse files Browse the repository at this point in the history 
## Motivation and Context
This PR implements a short-term solution for aws-sdk-rust#864 while a
long-term solution is worked out.

## Testing
- Tested manually against S3.
- Added DVR tests.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
  • Loading branch information
@jdisanti
jdisanti authored Aug 10, 2023
1 parent d7ffa89 commit 0286b9f
Show file tree
Hide file tree
Showing 9 changed files with 539 additions and 2 deletions.
6 changes: 6 additions & 0 deletions CHANGELOG.next.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,9 @@ message = "`RuntimeComponents` and `RuntimeComponentsBuilder` are now re-exporte
references = ["smithy-rs#2904"]
meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client"}
author = "jdisanti"

[[aws-sdk-rust]]
message = "Fix requests to S3 with `no_credentials` set."
references = ["smithy-rs#2907", "aws-sdk-rust#864"]
meta = { "breaking" = false, "tada" = false, "bug" = true }
author = "jdisanti"
24 changes: 24 additions & 0 deletions aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import software.amazon.smithy.model.shapes.ServiceShape
import software.amazon.smithy.model.shapes.Shape
import software.amazon.smithy.model.shapes.ShapeId
import software.amazon.smithy.model.shapes.StructureShape
import software.amazon.smithy.model.traits.OptionalAuthTrait
import software.amazon.smithy.model.transform.ModelTransformer
import software.amazon.smithy.rulesengine.traits.EndpointTestCase
import software.amazon.smithy.rulesengine.traits.EndpointTestOperationInput
Expand All @@ -34,6 +35,7 @@ import software.amazon.smithy.rust.codegen.core.smithy.protocols.ProtocolFunctio
import software.amazon.smithy.rust.codegen.core.smithy.protocols.ProtocolMap
import software.amazon.smithy.rust.codegen.core.smithy.protocols.RestXml
import software.amazon.smithy.rust.codegen.core.smithy.traits.AllowInvalidXmlRoot
import software.amazon.smithy.rust.codegen.core.util.hasTrait
import software.amazon.smithy.rust.codegen.core.util.letIf
import software.amazon.smithy.rustsdk.getBuiltIn
import software.amazon.smithy.rustsdk.toWritable
Expand Down Expand Up @@ -82,6 +84,8 @@ class S3Decorator : ClientCodegenDecorator {
},
)::transform,
)
// enable optional auth for operations commonly used with public buckets
.let(AddOptionalAuth()::transform)

override fun endpointCustomizations(codegenContext: ClientCodegenContext): List<EndpointCustomization> {
return listOf(
Expand Down Expand Up @@ -129,6 +133,26 @@ class FilterEndpointTests(
}
}

// TODO(P96049742): This model transform may need to change depending on if and how the S3 model is updated.
private class AddOptionalAuth {
private val s3OptionalAuthOperations = listOf(
ShapeId.from("com.amazonaws.s3#ListObjects"),
ShapeId.from("com.amazonaws.s3#ListObjectsV2"),
ShapeId.from("com.amazonaws.s3#HeadObject"),
ShapeId.from("com.amazonaws.s3#GetObject"),
)

fun transform(model: Model) = ModelTransformer.create().mapShapes(model) { shape ->
if (shape is OperationShape && s3OptionalAuthOperations.contains(shape.id) && !shape.hasTrait<OptionalAuthTrait>()) {
shape.toBuilder()
.addTrait(OptionalAuthTrait())
.build()
} else {
shape
}
}
}

class S3ProtocolOverride(codegenContext: CodegenContext) : RestXml(codegenContext) {
private val runtimeConfig = codegenContext.runtimeConfig
private val errorScope = arrayOf(
Expand Down
2 changes: 1 addition & 1 deletion aws/sdk/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -338,7 +338,7 @@ tasks.register("generateCargoWorkspace") {
doFirst {
outputDir.mkdirs()
outputDir.resolve("Cargo.toml").writeText(generateCargoWorkspace(awsServices))
rootProject.rootDir.resolve("clippy-root.toml").copyTo(outputDir.resolve("clippy.toml"))
rootProject.rootDir.resolve("clippy-root.toml").copyTo(outputDir.resolve("clippy.toml"), overwrite = true)
}
inputs.property("servicelist", awsServices.moduleNames.toString())
if (awsServices.examples.isNotEmpty()) {
Expand Down
85 changes: 85 additions & 0 deletions aws/sdk/integration-tests/s3/tests/data/no_auth/get-object.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
{
"events": [
{
"connection_id": 0,
"action": {
"Request": {
"request": {
"uri": "https://gdc-organoid-pancreatic-phs001611-2-open.s3.us-east-1.amazonaws.com/0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz?x-id=GetObject",
"headers": {
"amz-sdk-request": [
"attempt=1; max=3"
],
"user-agent": [
"aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0"
],
"x-amz-user-agent": [
"aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0"
]
},
"method": "GET"
}
}
}
},
{
"connection_id": 0,
"action": {
"Eof": {
"ok": true,
"direction": "Request"
}
}
},
{
"connection_id": 0,
"action": {
"Response": {
"response": {
"Ok": {
"status": 200,
"version": "HTTP/1.1",
"headers": {
"content-type": [
"binary/octet-stream"
],
"x-amz-id-2": [
"mO5q2ZSztYdEU923Zi5sHNctHwRRzOyngQEWsZWHwOJEgxrj9dw0KH0IVovTxu2Y8V0ps5z4KMQ="
],
"content-length": [
"386910"
],
"accept-ranges": [
"bytes"
],
"x-amz-server-side-encryption": [
"AES256"
],
"x-amz-request-id": [
"EGGB3A7GXR9YWDYM"
],
"last-modified": [
"Mon, 27 Jan 2020 20:56:51 GMT"
],
"date": [
"Mon, 07 Aug 2023 20:44:42 GMT"
],
"x-amz-meta-description": [
"{\"url\": \"s3://cleversafe.service.consul/stage-submission-5/ORGANOID-PANCREATIC/0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz\", \"node_id\": \"0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz\"}"
],
"server": [
"AmazonS3"
],
"etag": [
"\"446fc665f99183cd0540d7656a79d3ed\""
]
}
}
}
}
}
}
],
"docs": "traffic recording of optional auth (no Authorization header is included)",
"version": "V0"
}
94 changes: 94 additions & 0 deletions aws/sdk/integration-tests/s3/tests/data/no_auth/head-object.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
{
"events": [
{
"connection_id": 0,
"action": {
"Request": {
"request": {
"uri": "https://gdc-organoid-pancreatic-phs001611-2-open.s3.us-east-1.amazonaws.com/0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz",
"headers": {
"user-agent": [
"aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0"
],
"amz-sdk-request": [
"attempt=1; max=3"
],
"x-amz-user-agent": [
"aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0"
]
},
"method": "HEAD"
}
}
}
},
{
"connection_id": 0,
"action": {
"Eof": {
"ok": true,
"direction": "Request"
}
}
},
{
"connection_id": 0,
"action": {
"Response": {
"response": {
"Ok": {
"status": 200,
"version": "HTTP/1.1",
"headers": {
"last-modified": [
"Mon, 27 Jan 2020 20:56:51 GMT"
],
"content-type": [
"binary/octet-stream"
],
"date": [
"Mon, 07 Aug 2023 20:44:42 GMT"
],
"server": [
"AmazonS3"
],
"content-length": [
"386910"
],
"accept-ranges": [
"bytes"
],
"x-amz-server-side-encryption": [
"AES256"
],
"x-amz-id-2": [
"+d6tSM3krTTrvY+y6PFHnkw9OhAtJhQy8RzFrPO6vnUOIuvqViB9gFZvfJCcVMj7gX+dpIvZ3HI="
],
"x-amz-request-id": [
"EGGF3G9KFMFHZ3E0"
],
"etag": [
"\"446fc665f99183cd0540d7656a79d3ed\""
],
"x-amz-meta-description": [
"{\"url\": \"s3://cleversafe.service.consul/stage-submission-5/ORGANOID-PANCREATIC/0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz\", \"node_id\": \"0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz\"}"
]
}
}
}
}
}
},
{
"connection_id": 0,
"action": {
"Eof": {
"ok": true,
"direction": "Response"
}
}
}
],
"docs": "traffic recording of optional auth (no Authorization header is included)",
"version": "V0"
}
93 changes: 93 additions & 0 deletions aws/sdk/integration-tests/s3/tests/data/no_auth/list-objects-v2.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
{
"events": [
{
"connection_id": 0,
"action": {
"Request": {
"request": {
"uri": "https://gdc-organoid-pancreatic-phs001611-2-open.s3.us-east-1.amazonaws.com/?list-type=2&max-keys=3",
"headers": {
"amz-sdk-request": [
"attempt=1; max=3"
],
"x-amz-user-agent": [
"aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0"
],
"user-agent": [
"aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0"
]
},
"method": "GET"
}
}
}
},
{
"connection_id": 0,
"action": {
"Eof": {
"ok": true,
"direction": "Request"
}
}
},
{
"connection_id": 0,
"action": {
"Response": {
"response": {
"Ok": {
"status": 200,
"version": "HTTP/1.1",
"headers": {
"x-amz-id-2": [
"InRlbSiDTNSjIiYuGbkpnrz0TIgFVsDu8bnzVwF2UvZiOuiwhvdA3oltBT1ILZqNyFzSIkShTKk="
],
"x-amz-request-id": [
"H8T96AN5TTDT3SSQ"
],
"server": [
"AmazonS3"
],
"x-amz-bucket-region": [
"us-east-1"
],
"date": [
"Mon, 07 Aug 2023 20:44:41 GMT"
],
"content-type": [
"application/xml"
],
"transfer-encoding": [
"chunked"
]
}
}
}
}
}
},
{
"connection_id": 0,
"action": {
"Data": {
"data": {
"Utf8": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>gdc-organoid-pancreatic-phs001611-2-open</Name><Prefix></Prefix><NextContinuationToken>1SL9nYFaimMAwnR9dJnF4M5NMfm3Em6/ClPUVLEH3GOSw5yjeI+wCBLj3THB8DuJSUFhKNy5cGK5QBb/SvE+MKMZurarr0ZhOhQae2SQ8B4QQPkqQHKp9MeJXsYe4UH8/okpqJUZNS2AQt7gXrz7mFdIJXPuSckj02e06tvxZAOkHu7ER4xTJ+odI774K2xB+pcD3H0pqTUt+TebzB83BzA==</NextContinuationToken><KeyCount>3</KeyCount><MaxKeys>3</MaxKeys><IsTruncated>true</IsTruncated><Contents><Key>0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz</Key><LastModified>2020-01-27T20:56:51.000Z</LastModified><ETag>&quot;446fc665f99183cd0540d7656a79d3ed&quot;</ETag><Size>386910</Size><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>04a0a508-459a-4758-ac40-c3e8cb966683/30520ecd-c6bd-4039-9b1a-d3f999235598.FPKM-UQ.txt.gz</Key><LastModified>2020-01-27T20:56:50.000Z</LastModified><ETag>&quot;a2c6997aa0c6a9fd697af3e0517d96be&quot;</ETag><Size>388349</Size><StorageClass>STANDARD</StorageClass></Contents><Contents><Key>0541851c-ac0c-496e-93d2-3c03921fa6bd/c92f3dc4-24ea-457b-b90a-d6d599b14a73.rna_seq.star_gene_counts.tsv.gz</Key><LastModified>2020-01-27T20:56:51.000Z</LastModified><ETag>&quot;f2c4e159c9b2f4233c4c0c27f4c25472&quot;</ETag><Size>396626</Size><StorageClass>STANDARD</StorageClass></Contents></ListBucketResult>"
},
"direction": "Response"
}
}
},
{
"connection_id": 0,
"action": {
"Eof": {
"ok": true,
"direction": "Response"
}
}
}
],
"docs": "traffic recording of optional auth (no Authorization header is included)",
"version": "V0"
}
Loading

0 comments on commit 0286b9f

Please sign in to comment.