Merge branch 'main' into 32bit-port

include/aws/s3/private/s3_auto_ranged_get.h

@@ -10,14 +10,25 @@
 
 enum aws_s3_auto_ranged_get_request_type {
     AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_HEAD_OBJECT,
-    AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_PART,
-    AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_INITIAL_MESSAGE,
+    AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_GET_OBJECT_WITH_RANGE,
+    AWS_S3_AUTO_RANGE_GET_REQUEST_TYPE_GET_OBJECT_WITH_PART_NUMBER_1,
 };
 
 struct aws_s3_auto_ranged_get {
     struct aws_s3_meta_request base;
 
     enum aws_s3_checksum_algorithm validation_algorithm;
+
+    struct aws_string *etag;
+
+    bool initial_message_has_start_range;
+    bool initial_message_has_end_range;
+    uint64_t initial_range_start;
+    uint64_t initial_range_end;
+
+    uint64_t object_size_hint;
+    bool object_size_hint_available;
+
     /* Members to only be used when the mutex in the base type is locked. */
     struct {
         /* The starting byte of the data that we will be retrieved from the object.
@@ -30,6 +41,8 @@ struct aws_s3_auto_ranged_get {
          * So if begin=0 and end=0 then 1 byte is being downloaded. */
         uint64_t object_range_end;
 
+        uint64_t first_part_size;
+
         /* The total number of parts that are being used in downloading the object range. Note that "part" here
          * currently refers to a range-get, and does not require a "part" on the service side. */
         uint32_t total_num_parts;
@@ -47,15 +60,11 @@ struct aws_s3_auto_ranged_get {
         uint32_t object_range_empty : 1;
         uint32_t head_object_sent : 1;
         uint32_t head_object_completed : 1;
-        uint32_t get_without_range_sent : 1;
-        uint32_t get_without_range_completed : 1;
         uint32_t read_window_warning_issued : 1;
     } synced_data;
 
     uint32_t initial_message_has_range_header : 1;
     uint32_t initial_message_has_if_match_header : 1;
-
-    struct aws_string *etag;
 };
 
 AWS_EXTERN_C_BEGIN

include/aws/s3/private/s3_meta_request_impl.h

@@ -59,7 +59,7 @@ struct aws_s3_meta_request_event {
     enum aws_s3_meta_request_event_type {
         AWS_S3_META_REQUEST_EVENT_RESPONSE_BODY, /* body_callback */
         AWS_S3_META_REQUEST_EVENT_PROGRESS,      /* progress_callback */
-        /* TODO: AWS_S3_META_REQUEST_EVENT_TELEMETRY */
+        AWS_S3_META_REQUEST_EVENT_TELEMETRY,     /* telemetry_callback */
     } type;
 
     union {
@@ -72,6 +72,11 @@ struct aws_s3_meta_request_event {
         struct {
             struct aws_s3_meta_request_progress info;
         } progress;
+
+        /* data for AWS_S3_META_REQUEST_EVENT_TELEMETRY */
+        struct {
+            struct aws_s3_request_metrics *metrics;
+        } telemetry;
     } u;
 };
 
@@ -221,6 +226,9 @@ struct aws_s3_meta_request {
         /* True if the finish result has been set. */
         uint32_t finish_result_set : 1;
 
+        /* To track aws_s3_requests with cancellable HTTP streams */
+        struct aws_linked_list cancellable_http_streams_list;
+
     } synced_data;
 
     /* Anything in this structure should only ever be accessed by the client on its process work event loop task. */
@@ -359,6 +367,9 @@ void aws_s3_meta_request_add_event_for_delivery_synced(
  * The meta-request's finish callback must not be invoked until this returns false. */
 bool aws_s3_meta_request_are_events_out_for_delivery_synced(struct aws_s3_meta_request *meta_request);
 
+/* Cancel the requests with cancellable HTTP stream for the meta request */
+void aws_s3_meta_request_cancel_cancellable_requests_synced(struct aws_s3_meta_request *meta_request, int error_code);
+
 /* Asynchronously read from the meta request's input stream. Should always be done outside of any mutex,
  * as reading from the stream could cause user code to call back into aws-c-s3.
  * This will fill the buffer to capacity, unless end of stream is reached.

include/aws/s3/private/s3_request.h

@@ -113,6 +113,15 @@ struct aws_s3_request {
     /* Linked list node used for queuing. */
     struct aws_linked_list_node node;
 
+    /* Linked list node used for tracking the request is active from HTTP level. */
+    struct aws_linked_list_node cancellable_http_streams_list_node;
+
+    /* The meta request lock must be held to access the data */
+    struct {
+        /* The underlying http stream, only valid when the request is active from HTTP level */
+        struct aws_http_stream *cancellable_http_stream;
+    } synced_data;
+
     /* TODO Ref count on the request is no longer needed--only one part of code should ever be holding onto a request,
      * and we can just transfer ownership.*/
     struct aws_ref_count ref_count;

include/aws/s3/private/s3_util.h

@@ -34,6 +34,8 @@ struct aws_http_stream;
 struct aws_http_headers;
 struct aws_http_message;
 struct aws_s3_client;
+struct aws_s3_request;
+struct aws_s3_meta_request;
 
 struct aws_cached_signing_config_aws {
     struct aws_allocator *allocator;
@@ -152,6 +154,9 @@ extern const struct aws_byte_cursor g_content_range_header_name;
 AWS_S3_API
 extern const struct aws_byte_cursor g_accept_ranges_header_name;
 
+AWS_S3_API
+extern const struct aws_byte_cursor g_mp_parts_count_header_name;
+
 AWS_S3_API
 extern const struct aws_byte_cursor g_post_method;
 
@@ -240,11 +245,25 @@ int aws_s3_parse_content_length_response_header(
     struct aws_http_headers *response_headers,
     uint64_t *out_content_length);
 
-/* Calculate the number of parts based on overall object-range and part_size. This takes into account aligning
- * part-ranges on part_size. (ie: if object_range_start is not evenly divisible by part_size, it is considered in the
- * middle of a contiguous part, and that first part will be smaller than part_size.) */
+/*
+ * Given the request headers list, finds the Range header and parses the range-start and range-end. All arguments are
+ * required.
+ * */
+AWS_S3_API
+int aws_s3_parse_request_range_header(
+    struct aws_http_headers *request_headers,
+    bool *out_has_start_range,
+    bool *out_has_end_range,
+    uint64_t *out_start_range,
+    uint64_t *out_end_range);
+
+/* Calculate the number of parts based on overall object-range and part_size. */
 AWS_S3_API
-uint32_t aws_s3_get_num_parts(size_t part_size, uint64_t object_range_start, uint64_t object_range_end);
+uint32_t aws_s3_calculate_auto_ranged_get_num_parts(
+    size_t part_size,
+    uint64_t first_part_size,
+    uint64_t object_range_start,
+    uint64_t object_range_end);
 
 /**
  * Calculates the optimal part size and num parts given the 'content_length' and 'client_part_size'.
@@ -261,13 +280,15 @@ int aws_s3_calculate_optimal_mpu_part_size_and_num_parts(
     uint32_t *out_num_parts);
 
 /* Calculates the part range for a part given overall object range, size of each part, and the part's number. Note: part
- * numbers begin at one. This takes into account aligning part-ranges on part_size. Intended to be used in conjunction
- * with aws_s3_get_num_parts. part_number should be less than or equal to the result of aws_s3_get_num_parts. */
+ * numbers begin at one. Intended to be used in conjunction
+ * with aws_s3_calculate_auto_ranged_get_num_parts. part_number should be less than or equal to the result of
+ * aws_s3_calculate_auto_ranged_get_num_parts. */
 AWS_S3_API
-void aws_s3_get_part_range(
+void aws_s3_calculate_auto_ranged_get_part_range(
     uint64_t object_range_start,
     uint64_t object_range_end,
     size_t part_size,
+    uint64_t first_part_size,
     uint32_t part_number,
     uint64_t *out_part_range_start,
     uint64_t *out_part_range_end);
@@ -276,6 +297,9 @@ void aws_s3_get_part_range(
 AWS_S3_API
 int aws_s3_crt_error_code_from_server_error_code_string(struct aws_byte_cursor error_code_string);
 
+AWS_S3_API
+void aws_s3_request_finish_up_metrics_synced(struct aws_s3_request *request, struct aws_s3_meta_request *meta_request);
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_S3_UTIL_H */

include/aws/s3/s3.h

@@ -44,6 +44,8 @@ enum aws_s3_errors {
     AWS_ERROR_S3_EXCEEDS_MEMORY_LIMIT,
     AWS_ERROR_S3_INVALID_MEMORY_LIMIT_CONFIG,
     AWS_ERROR_S3EXPRESS_CREATE_SESSION_FAILED,
+    AWS_ERROR_S3_INTERNAL_PART_SIZE_MISMATCH_RETRYING_WITH_RANGE,
+
     AWS_ERROR_S3_END_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_S3_PACKAGE_ID)
 };
 

include/aws/s3/s3_client.h

@@ -209,8 +209,7 @@ typedef void(aws_s3_meta_request_progress_fn)(
     void *user_data);
 
 /**
- * Invoked to report the telemetry of the meta request once a single request finishes. Invoked from the thread of the
- * connection that request made from.
+ * Invoked to report the telemetry of the meta request once a single request finishes.
  * Note: *metrics is only valid for the duration of the callback. If you need to keep it around, use
  * `aws_s3_request_metrics_acquire`
  */
@@ -690,9 +689,6 @@ struct aws_s3_meta_request_options {
      * If set the request will keep track of the metrics from `aws_s3_request_metrics`, and fire the callback when the
      * request finishes receiving response.
      * See `aws_s3_meta_request_telemetry_fn`
-     *
-     * Notes:
-     * - The callback will be invoked multiple times from different threads.
      */
     aws_s3_meta_request_telemetry_fn *telemetry_callback;
 
@@ -727,6 +723,15 @@ struct aws_s3_meta_request_options {
      * from the buffer and compare them them to previously uploaded part checksums.
      */
     struct aws_s3_meta_request_resume_token *resume_token;
+
+    /*
+     * Optional.
+     * Total object size hint, in bytes.
+     * The optimal strategy for downloading a file depends on its size.
+     * Set this hint to help the S3 client choose the best strategy for this particular file.
+     * This is just used as an estimate, so it's okay to provide an approximate value if the exact size is unknown.
+     */
+    uint64_t *object_size_hint;
 };
 
 /* Result details of a meta request.

source/s3.c

@@ -45,6 +45,8 @@ static struct aws_error_info s_errors[] = {
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_INVALID_MEMORY_LIMIT_CONFIG, "Specified memory configuration is invalid for the system. "
         "Memory limit should be at least 1GiB. Part size and max part size should be smaller than memory limit."),
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3EXPRESS_CREATE_SESSION_FAILED, "CreateSession call failed when signing with S3 Express."),
+    AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_INTERNAL_PART_SIZE_MISMATCH_RETRYING_WITH_RANGE, "part_size mismatch, possibly due to wrong object_size_hint. Retrying with Range instead of partNumber."),
+
 };
 /* clang-format on */