Skip to content

Commit

Permalink
Support trailing checksum with no signing (#459)
Browse files Browse the repository at this point in the history
  • Loading branch information
@TingDaoK
TingDaoK authored Nov 27, 2024
1 parent e373ef4 commit 0b006a6
Show file tree
Hide file tree
Showing 6 changed files with 246 additions and 21 deletions.
4 changes: 4 additions & 0 deletions include/aws/s3/private/s3_client_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,10 @@ struct aws_s3_client_vtable {

struct aws_parallel_input_stream *(
*parallel_input_stream_new_from_file)(struct aws_allocator *allocator, struct aws_byte_cursor file_name);

struct aws_http_stream *(*http_connection_make_request)(
struct aws_http_connection *client_connection,
const struct aws_http_make_request_options *options);
};

struct aws_s3_upload_part_timeout_stats {
Expand Down
3 changes: 3 additions & 0 deletions include/aws/s3/private/s3_request.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,9 @@ struct aws_s3_request {

/* The metrics for the request telemetry */
struct aws_s3_request_metrics *metrics;

/* The request is required to have the unsigned payload */
uint32_t require_streaming_unsigned_payload_header : 1;
} send_data;

/* When true, response headers from the request will be stored in the request's response_headers variable. */
Expand Down
1 change: 1 addition & 0 deletions source/s3_client.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ static struct aws_s3_client_vtable s_s3_client_default_vtable = {
.endpoint_shutdown_callback = s_s3_client_endpoint_shutdown_callback,
.finish_destroy = s_s3_client_finish_destroy_default,
.parallel_input_stream_new_from_file = aws_parallel_input_stream_new_from_file,
.http_connection_make_request = aws_http_connection_make_request,
};

void aws_s3_set_dns_ttl(size_t ttl) {
Expand Down
54 changes: 33 additions & 21 deletions source/s3_meta_request.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -864,7 +864,7 @@ static void s_get_original_credentials_callback(struct aws_credentials *credenti
}
}

static int s_meta_request_resolve_signing_config(
static void s_meta_request_resolve_signing_config(
struct aws_signing_config_aws *out_signing_config,
struct aws_s3_request *request,
struct aws_s3_meta_request *meta_request) {
Expand Down Expand Up @@ -897,7 +897,12 @@ static int s_meta_request_resolve_signing_config(
&out_signing_config->signed_body_value, &g_aws_signed_body_value_streaming_unsigned_payload_trailer)) {
out_signing_config->signed_body_value = g_aws_signed_body_value_unsigned_payload;
}
return AWS_OP_SUCCESS;

/**
* In case of the signing was skipped for anonymous credentials, or presigned URL.
*/
request->send_data.require_streaming_unsigned_payload_header = aws_byte_cursor_eq(
&out_signing_config->signed_body_value, &g_aws_signed_body_value_streaming_unsigned_payload_trailer);
}

void aws_s3_meta_request_sign_request_default_impl(
Expand All @@ -915,19 +920,19 @@ void aws_s3_meta_request_sign_request_default_impl(

struct aws_signing_config_aws signing_config;

if (s_meta_request_resolve_signing_config(&signing_config, request, meta_request)) {
AWS_LOGF_DEBUG(
s_meta_request_resolve_signing_config(&signing_config, request, meta_request);

request->send_data.signable = aws_signable_new_http_request(meta_request->allocator, request->send_data.message);
if (request->send_data.signable == NULL) {
AWS_LOGF_ERROR(
AWS_LS_S3_META_REQUEST,
"id=%p: No signing config present. Not signing request %p.",
"id=%p: Could not allocate signable for request %p",
(void *)meta_request,
(void *)request);

on_signing_complete(NULL, AWS_ERROR_SUCCESS, user_data);
on_signing_complete(NULL, aws_last_error_or_unknown(), user_data);
return;
}

request->send_data.signable = aws_signable_new_http_request(meta_request->allocator, request->send_data.message);

AWS_LOGF_TRACE(
AWS_LS_S3_META_REQUEST,
"id=%p Created signable %p for request %p with message %p",
Expand All @@ -936,17 +941,6 @@ void aws_s3_meta_request_sign_request_default_impl(
(void *)request,
(void *)request->send_data.message);

if (request->send_data.signable == NULL) {
AWS_LOGF_ERROR(
AWS_LS_S3_META_REQUEST,
"id=%p: Could not allocate signable for request %p",
(void *)meta_request,
(void *)request);

on_signing_complete(NULL, aws_last_error_or_unknown(), user_data);
return;
}

if (signing_config.algorithm == AWS_SIGNING_ALGORITHM_V4_S3EXPRESS && !disable_s3_express_signing) {
/* Fetch credentials from S3 Express provider. */
struct aws_get_s3express_credentials_user_data *context =
Expand Down Expand Up @@ -1054,6 +1048,21 @@ static void s_s3_meta_request_request_on_signed(
goto finish;
}

/**
* Add "x-amz-content-sha256: STREAMING-UNSIGNED-PAYLOAD-TRAILER" header to support trailing checksum.
*/
if (request->send_data.require_streaming_unsigned_payload_header) {
struct aws_http_headers *headers = aws_http_message_get_headers(request->send_data.message);
AWS_ASSERT(headers != NULL);
if (aws_http_headers_set(
headers,
aws_byte_cursor_from_c_str("x-amz-content-sha256"),
g_aws_signed_body_value_streaming_unsigned_payload_trailer)) {
error_code = aws_last_error_or_unknown();
goto finish;
}
}

if (request->send_data.metrics) {
struct aws_s3_request_metrics *metric = request->send_data.metrics;
aws_high_res_clock_get_ticks((uint64_t *)&metric->time_metrics.sign_end_timestamp_ns);
Expand Down Expand Up @@ -1082,6 +1091,8 @@ void aws_s3_meta_request_send_request(struct aws_s3_meta_request *meta_request,
AWS_PRECONDITION(connection);
AWS_PRECONDITION(connection->http_connection);

struct aws_s3_client *client = meta_request->client;
AWS_PRECONDITION(client);
struct aws_s3_request *request = connection->request;
AWS_PRECONDITION(request);

Expand All @@ -1104,7 +1115,8 @@ void aws_s3_meta_request_send_request(struct aws_s3_meta_request *meta_request,
request->upload_timeout_ms = (size_t)options.response_first_byte_timeout_ms;
}

struct aws_http_stream *stream = aws_http_connection_make_request(connection->http_connection, &options);
struct aws_http_stream *stream =
client->vtable->http_connection_make_request(connection->http_connection, &options);

if (stream == NULL) {
AWS_LOGF_ERROR(
Expand Down
2 changes: 2 additions & 0 deletions tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,8 @@ add_net_test_case(test_s3_list_bucket_valid)
# Tests against local mock server
if(ENABLE_MOCK_SERVER_TESTS)
add_net_test_case(multipart_upload_mock_server)
add_net_test_case(multipart_upload_unsigned_with_trailer_checksum_mock_server)
add_net_test_case(single_upload_unsigned_with_trailer_checksum_mock_server)
add_net_test_case(multipart_upload_with_network_interface_names_mock_server)
add_net_test_case(multipart_upload_checksum_with_retry_mock_server)
add_net_test_case(multipart_download_checksum_with_retry_mock_server)
Expand Down
203 changes: 203 additions & 0 deletions tests/s3_mock_server_tests.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,209 @@ TEST_CASE(multipart_upload_mock_server) {
return AWS_OP_SUCCESS;
}

/* Singleton used by tests in this file */
static struct get_requests_header_tester {
struct aws_allocator *alloc;

/* Store the requests headers in the array. Array of struct aws_http_headers * */
struct aws_array_list headers_array;
struct aws_mutex lock;
} s_get_requests_header_tester;

static int s_get_requests_header_tester_init(struct aws_allocator *alloc) {
ASSERT_SUCCESS(aws_array_list_init_dynamic(
&s_get_requests_header_tester.headers_array, alloc, 1, sizeof(struct aws_http_headers *)));
ASSERT_SUCCESS(aws_mutex_init(&s_get_requests_header_tester.lock));
return AWS_OP_SUCCESS;
}

static void s_get_requests_header_tester_clean_up(void) {
/* iterate thought the headers array to clean up the headers */
for (size_t i = 0; i < aws_array_list_length(&s_get_requests_header_tester.headers_array); ++i) {
struct aws_http_headers *headers = NULL;
aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, i);
aws_http_headers_release(headers);
}
aws_mutex_clean_up(&s_get_requests_header_tester.lock);
aws_array_list_clean_up(&s_get_requests_header_tester.headers_array);
}

struct aws_http_stream *s_get_requests_header_make_request(
struct aws_http_connection *client_connection,
const struct aws_http_make_request_options *options) {
/**
* Record the headers in the array.
*/
aws_mutex_lock(&s_get_requests_header_tester.lock);
struct aws_http_headers *headers = aws_http_message_get_headers(options->request);
/* Keep the headers alive until we clean up the tester. */
aws_http_headers_acquire(headers);
aws_array_list_push_back(&s_get_requests_header_tester.headers_array, &headers);
aws_mutex_unlock(&s_get_requests_header_tester.lock);

struct aws_http_stream *stream = aws_http_connection_make_request(client_connection, options);
return stream;
}

/**
* Note: currently (Nov, 2024), S3 don't support create multipart upload anonymously.
*/
TEST_CASE(multipart_upload_unsigned_with_trailer_checksum_mock_server) {
(void)ctx;

struct aws_s3_tester tester;
ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
ASSERT_SUCCESS(s_get_requests_header_tester_init(allocator));

struct aws_s3_client_config client_config = {
.tls_mode = AWS_MR_TLS_DISABLED,
};

ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION));
struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);
ASSERT_NOT_NULL(client);

/* Patch the client vtable to record the request header */
struct aws_s3_client_vtable *s3_client_get_requests_header_vtable = client->vtable;
s3_client_get_requests_header_vtable->http_connection_make_request = s_get_requests_header_make_request;

struct aws_byte_cursor object_path = aws_byte_cursor_from_c_str("/default");

struct aws_s3_tester_meta_request_options put_options = {
.allocator = allocator,
.meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
.client = client,
.checksum_algorithm = AWS_SCA_CRC32,
.validate_get_response_checksum = false,
.put_options =
{
.object_size_mb = 10,
.object_path_override = object_path,
},
.mock_server = true,
};
struct aws_s3_meta_request_test_results out_results;
aws_s3_meta_request_test_results_init(&out_results, allocator);
ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &put_options, &out_results));
ASSERT_SUCCESS(s_validate_mpu_mock_server_metrics(&out_results.synced_data.metrics));

/**
* Check the recorded headers.
* 4 requests should be made:
* - Create MPU
* - 2 Upload Part
* - Complete MPU
*/
ASSERT_UINT_EQUALS(4, aws_array_list_length(&s_get_requests_header_tester.headers_array));
struct aws_byte_cursor content_sha256_header = aws_byte_cursor_from_c_str("x-amz-content-sha256");
struct aws_byte_cursor content_sha256_header_val;
AWS_ZERO_STRUCT(content_sha256_header_val);
struct aws_byte_cursor authorization_header = aws_byte_cursor_from_c_str("Authorization");
struct aws_byte_cursor authorization_header_val;
AWS_ZERO_STRUCT(authorization_header_val);
/* The first request should be Create MPU, and it should not have x-amz-content-sha256 header. */
struct aws_http_headers *headers = NULL;
ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 0));
/* No x-amz-content-sha256 header should be found. */
ASSERT_FAILS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));
/* The second and third requests should be Upload Part, and it should have x-amz-content-sha256 header with
* STREAMING-UNSIGNED-PAYLOAD-TRAILER. */
ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 1));
/* x-amz-content-sha256 header should be found. */
ASSERT_SUCCESS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));
ASSERT_TRUE(
aws_byte_cursor_eq(&content_sha256_header_val, &g_aws_signed_body_value_streaming_unsigned_payload_trailer));
/* But the Authorization header should not be found, since we are not signing the request. */
ASSERT_FAILS(aws_http_headers_get(headers, authorization_header, &authorization_header_val));
ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 2));
/* x-amz-content-sha256 header should be found. */
ASSERT_SUCCESS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));
ASSERT_TRUE(
aws_byte_cursor_eq(&content_sha256_header_val, &g_aws_signed_body_value_streaming_unsigned_payload_trailer));
/* But the Authorization header should not be found, since we are not signing the request. */
ASSERT_FAILS(aws_http_headers_get(headers, authorization_header, &authorization_header_val));
/* The last request should be Complete MPU, and it should not have x-amz-content-sha256 header. */
ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 3));
/* No x-amz-content-sha256 header should be found. */
ASSERT_FAILS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));

aws_s3_meta_request_test_results_clean_up(&out_results);
aws_s3_client_release(client);
aws_s3_tester_clean_up(&tester);
s_get_requests_header_tester_clean_up();

return AWS_OP_SUCCESS;
}

TEST_CASE(single_upload_unsigned_with_trailer_checksum_mock_server) {
(void)ctx;

struct aws_s3_tester tester;
ASSERT_SUCCESS(aws_s3_tester_init(allocator, &tester));
ASSERT_SUCCESS(s_get_requests_header_tester_init(allocator));

struct aws_s3_client_config client_config = {
.tls_mode = AWS_MR_TLS_DISABLED,
.part_size = 20 * 1024 * 1024,
};

ASSERT_SUCCESS(aws_s3_tester_bind_client(&tester, &client_config, AWS_S3_TESTER_BIND_CLIENT_REGION));
struct aws_s3_client *client = aws_s3_client_new(allocator, &client_config);
ASSERT_NOT_NULL(client);

/* Patch the client vtable to record the request header */
struct aws_s3_client_vtable *s3_client_get_requests_header_vtable = client->vtable;
s3_client_get_requests_header_vtable->http_connection_make_request = s_get_requests_header_make_request;

struct aws_byte_cursor object_path = aws_byte_cursor_from_c_str("/default");

struct aws_s3_tester_meta_request_options put_options = {
.allocator = allocator,
.meta_request_type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT,
.client = client,
.checksum_algorithm = AWS_SCA_CRC32,
.validate_get_response_checksum = false,
.put_options =
{
.object_size_mb = 10,
.object_path_override = object_path,
},
.mock_server = true,
};
struct aws_s3_meta_request_test_results out_results;
aws_s3_meta_request_test_results_init(&out_results, allocator);
ASSERT_SUCCESS(aws_s3_tester_send_meta_request_with_options(&tester, &put_options, &out_results));

/**
* Check the recorded headers.
* 1 request should be made:
* - Put Object
*/
ASSERT_UINT_EQUALS(1, aws_array_list_length(&s_get_requests_header_tester.headers_array));
struct aws_byte_cursor content_sha256_header = aws_byte_cursor_from_c_str("x-amz-content-sha256");
struct aws_byte_cursor content_sha256_header_val;
AWS_ZERO_STRUCT(content_sha256_header_val);
struct aws_byte_cursor authorization_header = aws_byte_cursor_from_c_str("Authorization");
struct aws_byte_cursor authorization_header_val;
AWS_ZERO_STRUCT(authorization_header_val);
/* The request should be Put Object, and it should have x-amz-content-sha256 header and not Authorization header. */
struct aws_http_headers *headers = NULL;
ASSERT_SUCCESS(aws_array_list_get_at(&s_get_requests_header_tester.headers_array, &headers, 0));
/* x-amz-content-sha256 header should be found. */
ASSERT_SUCCESS(aws_http_headers_get(headers, content_sha256_header, &content_sha256_header_val));
ASSERT_TRUE(
aws_byte_cursor_eq(&content_sha256_header_val, &g_aws_signed_body_value_streaming_unsigned_payload_trailer));
/* But the Authorization header should not be found, since we are not signing the request. */
ASSERT_FAILS(aws_http_headers_get(headers, authorization_header, &authorization_header_val));

aws_s3_meta_request_test_results_clean_up(&out_results);
aws_s3_client_release(client);
aws_s3_tester_clean_up(&tester);
s_get_requests_header_tester_clean_up();

return AWS_OP_SUCCESS;
}

TEST_CASE(multipart_upload_with_network_interface_names_mock_server) {
(void)ctx;
#if defined(AWS_OS_WINDOWS)
Expand Down

0 comments on commit 0b006a6

Please sign in to comment.