add the auto-compression support during the data transition

oss_c_sdk/aos_define.h

@@ -107,7 +107,7 @@ typedef apr_array_header_t aos_array_header_t;
 #define AOS_CONNECT_TIMEOUT 10
 #define AOS_DNS_CACHE_TIMOUT 60
 #define AOS_MIN_SPEED_LIMIT 1024
-#define AOS_MIN_SPEED_TIME 15
+#define AOS_MIN_SPEED_TIME 500
 #define AOS_MAX_MEMORY_SIZE 1024*1024*1024L
 #define AOS_MAX_PART_SIZE 512*1024*1024L
 #define AOS_DEFAULT_PART_SIZE 1024*1024L

oss_c_sdk/aos_status.c

@@ -4,6 +4,7 @@
 
 const char AOS_XML_PARSE_ERROR_CODE[] = "ParseXmlError";
 const char AOS_OPEN_FILE_ERROR_CODE[] = "OpenFileFail";
+const char AOS_SELECT_INPUT_ERROR_CODE[] = "SelectInputError";
 const char AOS_WRITE_FILE_ERROR_CODE[] = "WriteFileFail";
 const char AOS_RENAME_FILE_ERROR_CODE[] = "RenameFileFail";
 const char AOS_HTTP_IO_ERROR_CODE[] = "HttpIoError";

oss_c_sdk/aos_status.h

@@ -43,6 +43,7 @@ aos_status_t *aos_status_parse_from_body(aos_pool_t *p, aos_list_t *bc, int code
 
 extern const char AOS_XML_PARSE_ERROR_CODE[];
 extern const char AOS_OPEN_FILE_ERROR_CODE[];
+extern const char AOS_SELECT_INPUT_ERROR_CODE[];
 extern const char AOS_WRITE_FILE_ERROR_CODE[];
 extern const char AOS_RENAME_FILE_ERROR_CODE[];
 extern const char AOS_HTTP_IO_ERROR_CODE[];

oss_c_sdk/aos_transport.c

@@ -241,8 +241,8 @@ size_t aos_curl_default_write_callback(char *ptr, size_t size, size_t nmemb, voi
         aos_move_transport_state(t, TRANS_STATE_BODY_IN);
         return bytes;
     }
-
-    if (t->resp->type == BODY_IN_MEMORY && t->resp->body_len >= (int64_t)t->controller->options->max_memory_size) {
+    
+    if (t->resp->type == BODY_IN_MEMORY && t->resp->body_len >= (int64_t)t->controller->options->max_memory_size) { 
         t->controller->reason = apr_psprintf(t->pool,
              "receive body too big, current body size: %" APR_INT64_T_FMT ", max memory size: %" APR_INT64_T_FMT,
               t->resp->body_len, t->controller->options->max_memory_size);
@@ -384,6 +384,12 @@ int aos_curl_transport_setup(aos_curl_http_transport_t *t)
     curl_easy_setopt_safe(CURLOPT_CONNECTTIMEOUT, t->controller->options->connect_timeout);
     curl_easy_setopt_safe(CURLOPT_LOW_SPEED_LIMIT, t->controller->options->speed_limit);
     curl_easy_setopt_safe(CURLOPT_LOW_SPEED_TIME, t->controller->options->speed_time);
+    //curl_easy_setopt_safe(CURLOPT_TIMEOUT, 500); // max timeout 500s
+
+    /*if (t->controller->options->enable_accept_encoding)
+    {
+        curl_easy_setopt_safe(CURLOPT_ACCEPT_ENCODING, "");
+    }*/
 
     aos_init_curl_headers(t);
     curl_easy_setopt_safe(CURLOPT_HTTPHEADER, t->headers);

oss_c_sdk/aos_transport.h

@@ -34,6 +34,7 @@ struct aos_http_request_options_s {
     int enable_crc;
     char *proxy_host;
     char *proxy_auth;
+    int enable_accept_encoding;  //enable accept-encoding
 };
 
 struct aos_http_transport_options_s {

oss_c_sdk/oss_api.h

@@ -206,11 +206,18 @@ aos_status_t *oss_do_put_object_from_file(const oss_request_options_t *options,
 aos_status_t *oss_get_object_to_buffer(const oss_request_options_t *options, 
                                        const aos_string_t *bucket, 
                                        const aos_string_t *object,
-                                       aos_table_t *headers, 
+                                       aos_table_t *headers,
                                        aos_table_t *params,
                                        aos_list_t *buffer, 
                                        aos_table_t **resp_headers);
 
+aos_status_t *oss_select_object_to_buffer(const oss_request_options_t *options, 
+                                       const aos_string_t *bucket, 
+                                       const aos_string_t *object,
+                                       const oss_select_option *select_option,
+                                       aos_table_t *headers,
+                                       aos_list_t *buffer, 
+                                       aos_table_t **resp_headers);
 /*
  * @brief  get oss object to buffer
  * @param[in]   options             the oss request options
@@ -232,6 +239,15 @@ aos_status_t *oss_do_get_object_to_buffer(const oss_request_options_t *options,
                                           oss_progress_callback progress_callback, 
                                           aos_table_t **resp_headers);
 
+aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *options, 
+                                          const aos_string_t *bucket, 
+                                          const aos_string_t *object,
+                                          const oss_select_option *select_option,
+                                          aos_table_t *headers, 
+                                          aos_list_t *buffer,
+                                          oss_progress_callback progress_callback, 
+                                          aos_table_t **resp_headers);
+
 /*
  * @brief  get oss object to file
  * @param[in]   options             the oss request options
@@ -287,6 +303,13 @@ aos_status_t *oss_head_object(const oss_request_options_t *options,
                               aos_table_t *headers, 
                               aos_table_t **resp_headers);
 
+aos_status_t *oss_create_select_object_metadata(const oss_request_options_t *options,
+                              const aos_string_t *bucket, 
+                              const aos_string_t *object,
+                              const oss_select_metadata_option *select_metadata_option,
+                              aos_table_t *headers, 
+                              aos_table_t **resp_headers);
+
 /*
  * @brief  delete oss object
  * @param[in]   options             the oss request options

oss_c_sdk/oss_define.c

@@ -1,4 +1,5 @@
 #include "oss_define.h"
+#include "aos_string.h"
 
 const char OSS_CANNONICALIZED_HEADER_PREFIX[] = "x-oss-";
 const char OSS_CANNONICALIZED_HEADER_DATE[] = "x-oss-date";
@@ -59,8 +60,32 @@ const char LIVE_CHANNEL_STATUS_IDLE[] = "idle";
 const char LIVE_CHANNEL_STATUS_LIVE[] = "live";
 const char LIVE_CHANNEL_DEFAULT_TYPE[] = "HLS";
 const char LIVE_CHANNEL_DEFAULT_PLAYLIST[] = "playlist.m3u8";
+const char OSS_SELECT_CSV[] = "csv/select";
+const char OSS_CSV_META[] = "csv/meta";
+const char OSS_SELECT_CSV_ROWS[] = "x-oss-select-csv-rows";
+const char OSS_SELECT_CSV_SPLITS[] = "x-oss-select-csv-splits";
 const int  LIVE_CHANNEL_DEFAULT_FRAG_DURATION = 5;
 const int  LIVE_CHANNEL_DEFAULT_FRAG_COUNT = 3;
 const int OSS_MAX_PART_NUM = 10000;
 const int OSS_PER_RET_NUM = 1000;
 const int MAX_SUFFIX_LEN = 1024;
+
+const csv_format_option csv_format_option_default = {
+        .field_delimiter = ',',
+        .field_quote = '"',
+        .comment = '#',
+        .record_delimiter.data = "\n",
+        .record_delimiter.len = 1,
+        .header_info = CSV_HEADER_IGNORE
+};
+
+const oss_select_metadata_option oss_select_metadata_option_default = {
+        .overwrite = 0,
+        .input_serialization.compression_info = NONE,
+        .input_serialization.csv_format.field_delimiter = ',',
+        .input_serialization.csv_format.field_quote = '"',
+        .input_serialization.csv_format.comment = '#',
+        .input_serialization.csv_format.record_delimiter.data = "\n",
+        .input_serialization.csv_format.record_delimiter.len = 1,
+        .input_serialization.csv_format.header_info = CSV_HEADER_IGNORE
+};

oss_c_sdk/oss_define.h

@@ -87,6 +87,10 @@ extern const char LIVE_CHANNEL_DEFAULT_TYPE[];
 extern const char LIVE_CHANNEL_DEFAULT_PLAYLIST[];
 extern const int  LIVE_CHANNEL_DEFAULT_FRAG_DURATION;
 extern const int  LIVE_CHANNEL_DEFAULT_FRAG_COUNT;
+extern const char OSS_SELECT_CSV[];
+extern const char OSS_CSV_META[];
+extern const char OSS_SELECT_CSV_ROWS[];
+extern const char OSS_SELECT_CSV_SPLITS[];
 extern const int OSS_MAX_PART_NUM;
 extern const int OSS_PER_RET_NUM;
 extern const int MAX_SUFFIX_LEN;
@@ -315,4 +319,60 @@ typedef struct {
     aos_string_t remote_addr;
 } oss_live_record_content_t;
 
+typedef enum{
+    CSV_HEADER_IGNORE = 0,
+    CSV_HEADER_NONE = 1,
+    CSV_HEADER_USE = 2
+} csv_header_info;
+
+typedef enum {
+    NONE = 1,
+    GZIP = 2
+} select_compression_info;
+
+typedef struct {
+    char field_delimiter;
+    char field_quote;
+    char comment;
+    aos_string_t record_delimiter;
+    csv_header_info header_info;
+} csv_format_option;
+
+extern const csv_format_option csv_format_option_default;
+
+typedef struct {
+    select_compression_info compression_info;
+    csv_format_option csv_format;
+} select_input_serialization;
+
+typedef struct {
+    int raw_output; //always output raw data now
+    int keep_all_columns;
+    csv_format_option csv_format;
+} select_output_serialization;
+
+typedef struct {
+    int overwrite;
+    select_input_serialization input_serialization;
+} oss_select_metadata_option;
+
+extern const oss_select_metadata_option oss_select_metadata_option_default;
+
+typedef enum {
+    NO_RANGE = 0,
+    LINE = 1,
+    SPLIT = 2
+} select_range_option;
+
+typedef struct {
+    select_range_option range_option;
+    int range[2];
+    aos_string_t expression;
+    select_input_serialization input_serialization;
+    select_output_serialization output_serialization;
+} oss_select_option;
+
+#define aos_select_input_error_status_set(STATUS, RES, MSG) do {\
+        aos_status_set(STATUS, RES, AOS_SELECT_INPUT_ERROR_CODE, MSG); \
+    } while(0)
 #endif

oss_c_sdk/oss_object.c

@@ -6,6 +6,7 @@
 #include "oss_util.h"
 #include "oss_xml.h"
 #include "oss_api.h"
+#include "oss_define.h"
 
 char *oss_gen_signed_url(const oss_request_options_t *options,
                          const aos_string_t *bucket, 
@@ -142,6 +143,18 @@ aos_status_t *oss_get_object_to_buffer(const oss_request_options_t *options,
                                        params, buffer, NULL, resp_headers);
 }
 
+aos_status_t *oss_select_object_to_buffer(const oss_request_options_t *options, 
+                                       const aos_string_t *bucket, 
+                                       const aos_string_t *object,
+                                       const oss_select_option *select_option,
+                                       aos_table_t *headers, 
+                                       aos_list_t *buffer, 
+                                       aos_table_t **resp_headers)
+{
+    return oss_do_select_object_to_buffer(options, bucket, object, select_option, headers,
+                                       buffer, NULL, resp_headers);
+}
+
 aos_status_t *oss_do_get_object_to_buffer(const oss_request_options_t *options, 
                                           const aos_string_t *bucket, 
                                           const aos_string_t *object,
@@ -173,6 +186,50 @@ aos_status_t *oss_do_get_object_to_buffer(const oss_request_options_t *options,
     return s;
 }
 
+aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *options, 
+                                          const aos_string_t *bucket, 
+                                          const aos_string_t *object,
+                                          const oss_select_option *select_option,
+                                          aos_table_t *headers, 
+                                          aos_list_t *buffer,
+                                          oss_progress_callback progress_callback, 
+                                          aos_table_t **resp_headers)
+{
+    aos_status_t *s = NULL;
+    aos_http_request_t *req = NULL;
+    aos_http_response_t *resp = NULL;
+    aos_table_t*params = NULL;
+    aos_list_t body;
+
+    headers = aos_table_create_if_null(options, headers, 0);
+    params = aos_table_create_if_null(options, params, 0);
+
+    apr_table_add(params, OSS_PROCESS, OSS_SELECT_CSV);
+    apr_table_set(headers, OSS_CONTENT_TYPE, "");
+
+    if (select_option == NULL) {
+        s = aos_status_create(options->pool);
+        aos_select_input_error_status_set(s, 400, "select option is null");
+        return s;
+    }
+    oss_init_object_request(options, bucket, object, HTTP_POST,
+                            &req, params, headers, progress_callback, 0, &resp);
+
+    build_select_object_metadata_body(options->pool, select_option, &body);
+    oss_write_request_body_from_buffer(&body, req);
+
+    s = oss_process_request(options, req, resp);
+    oss_fill_read_response_body(resp, buffer);
+    oss_fill_read_response_header(resp, resp_headers);
+
+    if (is_enable_crc(options) && has_crc_in_response(resp) &&  
+        !has_range_or_process_in_request(req)) {
+        oss_check_crc_consistent(resp->crc64, resp->headers, s);
+    }
+
+    return s;
+}
+
 aos_status_t *oss_get_object_to_file(const oss_request_options_t *options,
                                      const aos_string_t *bucket, 
                                      const aos_string_t *object,
@@ -252,6 +309,43 @@ aos_status_t *oss_head_object(const oss_request_options_t *options,
     return s;
 }
 
+aos_status_t *oss_create_select_object_metadata(const oss_request_options_t *options,
+                              const aos_string_t *bucket, 
+                              const aos_string_t *object,
+                              const oss_select_metadata_option *select_metadata_option,
+                              aos_table_t *headers, 
+                              aos_table_t **resp_headers)
+{
+    aos_status_t *s = NULL;
+    aos_http_request_t *req = NULL;
+    aos_http_response_t *resp = NULL;
+    aos_table_t *query_params = NULL;
+    aos_list_t body;
+
+    headers = aos_table_create_if_null(options, headers, 0);    
+
+    query_params = aos_table_create_if_null(options, query_params, 0);
+
+    apr_table_set(query_params, OSS_PROCESS, OSS_CSV_META);
+    apr_table_set(headers, OSS_CONTENT_TYPE, "");
+
+    if (select_metadata_option == NULL) {
+        select_metadata_option = &oss_select_metadata_option_default;
+    }
+
+    oss_init_object_request(options, bucket, object, HTTP_POST,
+                            &req, query_params, headers, NULL, 0, &resp);
+
+    build_create_select_object_metadata_body(options->pool, select_metadata_option, &body);
+    oss_write_request_body_from_buffer(&body, req);
+
+    s = oss_process_request(options, req, resp);
+    oss_fill_read_response_header(resp, resp_headers);
+
+    return s;
+}
+
+
 aos_status_t *oss_delete_object(const oss_request_options_t *options,
                                 const aos_string_t *bucket, 
                                 const aos_string_t *object, 

oss_c_sdk/oss_util.c

@@ -147,7 +147,20 @@ static void generate_rtmp_proto(const oss_request_options_t *options,
 
 int is_valid_ip(const char *str)
 {
-    if (INADDR_NONE == inet_addr(str) || INADDR_ANY == inet_addr(str)) {
+    char ip[24];
+    int i = 0;
+    for(i = 0;i < strlen(str)&& i <24;i++){
+        if (str[i] != ':'){
+            ip[i]=str[i];
+        }
+        else
+        {
+            ip[i]=0;
+            break;
+        }
+    }
+
+    if (INADDR_NONE == inet_addr(ip) || INADDR_ANY == inet_addr(ip)) {
         return 0;
     }
     return 1;
@@ -924,3 +937,20 @@ int oss_temp_file_rename(aos_status_t *s, const char *from_path, const char *to_
     return res;
 }
 
+char* range_to_string(const char* prefix, int start, int end, char* range_str)
+{
+    if (start >= 0 && end >= 0){
+        sprintf(range_str, "%s%d-%d", prefix, start, end);
+    }
+    else if (start >= 0){
+        sprintf(range_str, "%s%d-", prefix, start);
+    }
+    else if (end >= 0){
+        sprintf(range_str, "%s-%d", prefix, end);
+    }
+    else{
+        return NULL;
+    }
+
+    return range_str;
+}

oss_c_sdk/oss_util.h

@@ -293,6 +293,5 @@ int oss_get_temporary_file_name(aos_pool_t *p, const aos_string_t *filename, aos
 
 int oss_temp_file_rename(aos_status_t *s, const char *from_path, const char *to_path, apr_pool_t *pool);
 
-OSS_CPP_END
-
+char* range_to_string(const char* prefix, int start, int end, char* range_str);
 #endif