From 60679fde7de5dcd988ada4d1d5800dfaaf80a879 Mon Sep 17 00:00:00 2001 From: tonyxuqqi Date: Thu, 18 Jan 2018 10:17:13 -0800 Subject: [PATCH 1/8] Add accept-encoding support for automatic compression in data transition from oss server. --- oss_c_sdk/aos_transport.c | 9 ++++-- oss_c_sdk/aos_transport.h | 1 + oss_c_sdk_sample/oss_get_object_sample.c | 10 ++++--- oss_c_sdk_test/test_oss_object.c | 37 ++++++++++++++++++++---- 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/oss_c_sdk/aos_transport.c b/oss_c_sdk/aos_transport.c index 1050fa48..1fb52d4b 100644 --- a/oss_c_sdk/aos_transport.c +++ b/oss_c_sdk/aos_transport.c @@ -241,8 +241,8 @@ size_t aos_curl_default_write_callback(char *ptr, size_t size, size_t nmemb, voi aos_move_transport_state(t, TRANS_STATE_BODY_IN); return bytes; } - - if (t->resp->type == BODY_IN_MEMORY && t->resp->body_len >= (int64_t)t->controller->options->max_memory_size) { + + if (t->resp->type == BODY_IN_MEMORY && t->resp->body_len >= (int64_t)t->controller->options->max_memory_size) { t->controller->reason = apr_psprintf(t->pool, "receive body too big, current body size: %" APR_INT64_T_FMT ", max memory size: %" APR_INT64_T_FMT, t->resp->body_len, t->controller->options->max_memory_size); @@ -385,6 +385,11 @@ int aos_curl_transport_setup(aos_curl_http_transport_t *t) curl_easy_setopt_safe(CURLOPT_LOW_SPEED_LIMIT, t->controller->options->speed_limit); curl_easy_setopt_safe(CURLOPT_LOW_SPEED_TIME, t->controller->options->speed_time); + if (t->controller->options->enable_accept_encoding) + { + curl_easy_setopt_safe(CURLOPT_ACCEPT_ENCODING, ""); + } + aos_init_curl_headers(t); curl_easy_setopt_safe(CURLOPT_HTTPHEADER, t->headers); diff --git a/oss_c_sdk/aos_transport.h b/oss_c_sdk/aos_transport.h index da05068f..aaf25eb8 100644 --- a/oss_c_sdk/aos_transport.h +++ b/oss_c_sdk/aos_transport.h @@ -34,6 +34,7 @@ struct aos_http_request_options_s { int enable_crc; char *proxy_host; char *proxy_auth; + int enable_accept_encoding; //enable accept-encoding }; struct aos_http_transport_options_s { diff --git a/oss_c_sdk_sample/oss_get_object_sample.c b/oss_c_sdk_sample/oss_get_object_sample.c index 866d82bd..035630ef 100644 --- a/oss_c_sdk_sample/oss_get_object_sample.c +++ b/oss_c_sdk_sample/oss_get_object_sample.c @@ -61,7 +61,7 @@ void get_object_to_buffer() aos_pool_destroy(p); } -void get_object_to_local_file() +void get_object_to_local_file(int enable_ae) { aos_pool_t *p = NULL; aos_string_t bucket; @@ -83,6 +83,7 @@ void get_object_to_local_file() headers = aos_table_make(p, 0); aos_str_set(&file, download_filename); + options->ctl->options->enable_accept_encoding = enable_ae; s = oss_get_object_to_file(options, &bucket, &object, headers, params, &file, &resp_headers); if (aos_status_is_ok(s)) { @@ -120,7 +121,7 @@ void get_object_to_buffer_with_range() aos_list_init(&buffer); headers = aos_table_make(p, 1); - /* ����Range����ȡ�ļ���ָ����Χ��bytes=20-100������20�͵�100���ַ� */ + /* ����Range����ȡ�ļ���ָ����Χ��bytes=20-100������20�͵�100���ַ� */ apr_table_set(headers, "Range", "bytes=20-100"); s = oss_get_object_to_buffer(options, &bucket, &object, @@ -173,7 +174,7 @@ void get_object_to_local_file_with_range() aos_str_set(&file, download_filename); headers = aos_table_make(p, 1); - /* ����Range����ȡ�ļ���ָ����Χ��bytes=20-100������20�͵�100���ַ� */ + /* ����Range����ȡ�ļ���ָ����Χ��bytes=20-100������20�͵�100���ַ� */ apr_table_set(headers, "Range", "bytes=20-100"); s = oss_get_object_to_file(options, &bucket, &object, headers, @@ -324,7 +325,8 @@ void get_oss_dir_to_local_dir() void get_object_sample() { get_object_to_buffer(); - get_object_to_local_file(); + get_object_to_local_file(0); + get_object_to_local_file(1); get_object_to_buffer_with_range(); get_object_to_local_file_with_range(); diff --git a/oss_c_sdk_test/test_oss_object.c b/oss_c_sdk_test/test_oss_object.c index ffffcc3f..5353b326 100644 --- a/oss_c_sdk_test/test_oss_object.c +++ b/oss_c_sdk_test/test_oss_object.c @@ -199,8 +199,9 @@ void test_put_object_from_buffer_with_specified(CuTest *tc) printf("test_put_object_from_buffer_with_specified ok\n"); } -void test_put_object_from_file(CuTest *tc) +void test_put_object_from_file(CuTest *tc, int enable_ae) { + printf("start test_put_object_from_file %d",enable_ae); aos_pool_t *p = NULL; char *object_name = "video_1.ts"; char *filename = __FILE__; @@ -217,6 +218,7 @@ void test_put_object_from_file(CuTest *tc) aos_pool_create(&p, NULL); options = oss_request_options_create(p); init_test_request_options(options, is_cname); + options->ctl->options->enable_accept_encoding = enable_ae; headers = aos_table_make(p, 5); s = create_test_object_from_file(options, TEST_BUCKET_NAME, object_name, filename, headers); @@ -239,7 +241,17 @@ void test_put_object_from_file(CuTest *tc) content_type = (char*)(apr_table_get(head_resp_headers, OSS_CONTENT_TYPE)); CuAssertStrEquals(tc, "application/octet-stream", content_type); - printf("test_put_object_from_file ok\n"); + printf("test_put_object_from_file %d ok\n", enable_ae); +} + +void test_put_object_from_file_with_te(CuTest *tc) +{ + test_put_object_from_file(tc, 1); +} + +void test_put_object_from_file_without_te(CuTest *tc) +{ + test_put_object_from_file(tc, 0); } void test_put_object_with_large_length_header(CuTest *tc) @@ -432,7 +444,7 @@ void test_get_object_to_buffer_with_range(CuTest *tc) printf("test_get_object_to_buffer_with_range ok\n"); } -void test_get_object_to_file(CuTest *tc) +void test_get_object_to_file(CuTest *tc, int enable_ae) { aos_pool_t *p = NULL; aos_string_t bucket; @@ -452,6 +464,7 @@ void test_get_object_to_file(CuTest *tc) aos_pool_create(&p, NULL); options = oss_request_options_create(p); init_test_request_options(options, is_cname); + options->ctl->options->enable_accept_encoding = enable_ae; aos_str_set(&bucket, TEST_BUCKET_NAME); aos_str_set(&object, object_name); aos_str_set(&file, filename); @@ -468,7 +481,17 @@ void test_get_object_to_file(CuTest *tc) remove(filename); aos_pool_destroy(p); - printf("test_get_object_to_file ok\n"); + printf("test_get_object_to_file %d ok\n", enable_ae); +} + +void test_get_object_to_file_with_te(CuTest *tc) +{ + test_get_object_to_file(tc,1); +} + +void test_get_object_to_file_without_te(CuTest *tc) +{ + test_get_object_to_file(tc,0); } void test_head_object(CuTest *tc) @@ -933,14 +956,16 @@ CuSuite *test_oss_object() SUITE_ADD_TEST(suite, test_object_setup); SUITE_ADD_TEST(suite, test_put_object_from_buffer); - SUITE_ADD_TEST(suite, test_put_object_from_file); + SUITE_ADD_TEST(suite, test_put_object_from_file_with_te); + SUITE_ADD_TEST(suite, test_put_object_from_file_without_te); SUITE_ADD_TEST(suite, test_put_object_from_buffer_with_specified); SUITE_ADD_TEST(suite, test_get_object_to_buffer); SUITE_ADD_TEST(suite, test_get_object_to_buffer_with_range); SUITE_ADD_TEST(suite, test_put_object_from_file_with_content_type); SUITE_ADD_TEST(suite, test_put_object_from_buffer_with_default_content_type); SUITE_ADD_TEST(suite, test_put_object_with_large_length_header); - SUITE_ADD_TEST(suite, test_get_object_to_file); + SUITE_ADD_TEST(suite, test_get_object_to_file_with_te); + SUITE_ADD_TEST(suite, test_get_object_to_file_without_te); SUITE_ADD_TEST(suite, test_head_object); SUITE_ADD_TEST(suite, test_head_object_with_not_exist); SUITE_ADD_TEST(suite, test_copy_object); From f6d7e387e96e9cb6ca30cb8170b983dbd7e89ef8 Mon Sep 17 00:00:00 2001 From: tonyxuqqi Date: Thu, 18 Jan 2018 10:23:59 -0800 Subject: [PATCH 2/8] fix a commment in test code --- oss_c_sdk_sample/oss_get_object_sample.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oss_c_sdk_sample/oss_get_object_sample.c b/oss_c_sdk_sample/oss_get_object_sample.c index 035630ef..362b955d 100644 --- a/oss_c_sdk_sample/oss_get_object_sample.c +++ b/oss_c_sdk_sample/oss_get_object_sample.c @@ -121,7 +121,7 @@ void get_object_to_buffer_with_range() aos_list_init(&buffer); headers = aos_table_make(p, 1); - /* ����Range����ȡ�ļ���ָ����Χ��bytes=20-100������20�͵�100���ַ� */ + /* Set range for reading the data. bytes=20-100 means reading the data from the offset 20 to 100 of the file */ apr_table_set(headers, "Range", "bytes=20-100"); s = oss_get_object_to_buffer(options, &bucket, &object, @@ -174,7 +174,7 @@ void get_object_to_local_file_with_range() aos_str_set(&file, download_filename); headers = aos_table_make(p, 1); - /* ����Range����ȡ�ļ���ָ����Χ��bytes=20-100������20�͵�100���ַ� */ + /* Set range for reading the data. bytes=20-100 means reading the data from the offset 20 to 100 of the file */ apr_table_set(headers, "Range", "bytes=20-100"); s = oss_get_object_to_file(options, &bucket, &object, headers, From 5efa1263f83e3e1a31ec7d9d6d2bd801aa3674bd Mon Sep 17 00:00:00 2001 From: tonyxuqqi Date: Fri, 6 Apr 2018 11:03:03 -0700 Subject: [PATCH 3/8] head csv objcet --- oss_c_sdk/oss_api.h | 24 +++++++- oss_c_sdk/oss_object.c | 75 +++++++++++++++++++++++ oss_c_sdk_sample/main.c | 10 +-- oss_c_sdk_sample/oss_head_object_sample.c | 2 +- 4 files changed, 104 insertions(+), 7 deletions(-) diff --git a/oss_c_sdk/oss_api.h b/oss_c_sdk/oss_api.h index 3fec2d86..7da3489a 100644 --- a/oss_c_sdk/oss_api.h +++ b/oss_c_sdk/oss_api.h @@ -211,6 +211,14 @@ aos_status_t *oss_get_object_to_buffer(const oss_request_options_t *options, aos_list_t *buffer, aos_table_t **resp_headers); +aos_status_t *oss_select_object_to_buffer(const oss_request_options_t *options, + const aos_string_t *bucket, + const aos_string_t *object, + const aos_string_t *sql, + aos_table_t *headers, + aos_table_t *params, + aos_list_t *buffer, + aos_table_t **resp_headers); /* * @brief get oss object to buffer * @param[in] options the oss request options @@ -227,7 +235,15 @@ aos_status_t *oss_do_get_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, aos_table_t *headers, - aos_table_t *params, + aos_list_t *buffer, + oss_progress_callback progress_callback, + aos_table_t **resp_headers); + +aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *options, + const aos_string_t *bucket, + const aos_string_t *object, + const aos_string_t *sql, + aos_table_t *headers, aos_list_t *buffer, oss_progress_callback progress_callback, aos_table_t **resp_headers); @@ -287,6 +303,12 @@ aos_status_t *oss_head_object(const oss_request_options_t *options, aos_table_t *headers, aos_table_t **resp_headers); +aos_status_t *oss_head_csv_object(const oss_request_options_t *options, + const aos_string_t *bucket, + const aos_string_t *object, + aos_table_t *headers, + aos_table_t **resp_headers); + /* * @brief delete oss object * @param[in] options the oss request options diff --git a/oss_c_sdk/oss_object.c b/oss_c_sdk/oss_object.c index 57c488ae..4f13ea8d 100644 --- a/oss_c_sdk/oss_object.c +++ b/oss_c_sdk/oss_object.c @@ -142,6 +142,18 @@ aos_status_t *oss_get_object_to_buffer(const oss_request_options_t *options, params, buffer, NULL, resp_headers); } +aos_status_t *oss_select_object_to_buffer(const oss_request_options_t *options, + const aos_string_t *bucket, + const aos_string_t *object, + const aos_string_t *sql, + aos_table_t *headers, + aos_list_t *buffer, + aos_table_t **resp_headers) +{ + return oss_do_select_object_to_buffer(options, bucket, object, sql, headers, + buffer, NULL, resp_headers); +} + aos_status_t *oss_do_get_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, @@ -173,6 +185,42 @@ aos_status_t *oss_do_get_object_to_buffer(const oss_request_options_t *options, return s; } +aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *options, + const aos_string_t *bucket, + const aos_string_t *object, + const aos_string_t *sql, + aos_table_t *headers, + aos_list_t *buffer, + oss_progress_callback progress_callback, + aos_table_t **resp_headers) +{ + aos_status_t *s = NULL; + aos_http_request_t *req = NULL; + aos_http_response_t *resp = NULL; + aos_table_t*params = NULL; + headers = aos_table_create_if_null(options, headers, 0); + params = aos_table_create_if_null(options, params, 0); + + apr_table_add(query_params, "x-oss-process", "csv/selcet"); + apr_table_add(query_params, "sql", sql); + + oss_init_object_request(options, bucket, object, HTTP_GET, + &req, params, headers, progress_callback, 0, &resp); + + + + s = oss_process_request(options, req, resp); + oss_fill_read_response_body(resp, buffer); + oss_fill_read_response_header(resp, resp_headers); + + if (is_enable_crc(options) && has_crc_in_response(resp) && + !has_range_or_process_in_request(req)) { + oss_check_crc_consistent(resp->crc64, resp->headers, s); + } + + return s; +} + aos_status_t *oss_get_object_to_file(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, @@ -252,6 +300,33 @@ aos_status_t *oss_head_object(const oss_request_options_t *options, return s; } +aos_status_t *oss_head_csv_object(const oss_request_options_t *options, + const aos_string_t *bucket, + const aos_string_t *object, + aos_table_t *headers, + aos_table_t **resp_headers) +{ + aos_status_t *s = NULL; + aos_http_request_t *req = NULL; + aos_http_response_t *resp = NULL; + aos_table_t *query_params = NULL; + + headers = aos_table_create_if_null(options, headers, 0); + + query_params = aos_table_create_if_null(options, query_params, 0); + + apr_table_add(query_params, "csv", ""); + + oss_init_object_request(options, bucket, object, HTTP_HEAD, + &req, query_params, headers, NULL, 0, &resp); + + s = oss_process_request(options, req, resp); + oss_fill_read_response_header(resp, resp_headers); + + return s; +} + + aos_status_t *oss_delete_object(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, diff --git a/oss_c_sdk_sample/main.c b/oss_c_sdk_sample/main.c index 854a9283..b8382177 100644 --- a/oss_c_sdk_sample/main.c +++ b/oss_c_sdk_sample/main.c @@ -28,11 +28,11 @@ int main(int argc, char *argv[]) exit(1); } - put_object_sample(); - append_object_sample(); - get_object_sample(); + //put_object_sample(); + //append_object_sample(); + //get_object_sample(); head_object_sample(); - list_object_sample(); + /*list_object_sample(); multipart_object_sample(); delete_object_sample(); callback_sample(); @@ -40,7 +40,7 @@ int main(int argc, char *argv[]) crc_sample(); image_sample(); resumable_upload_sample(); - resumable_download_sample(); + resumable_download_sample();*/ aos_http_io_deinitialize(); diff --git a/oss_c_sdk_sample/oss_head_object_sample.c b/oss_c_sdk_sample/oss_head_object_sample.c index cce76f6b..2aa7f058 100644 --- a/oss_c_sdk_sample/oss_head_object_sample.c +++ b/oss_c_sdk_sample/oss_head_object_sample.c @@ -29,7 +29,7 @@ void head_object() aos_str_set(&object, OBJECT_NAME); headers = aos_table_make(p, 0); - s = oss_head_object(options, &bucket, &object, headers, &resp_headers); + s = oss_head_csv_object(options, &bucket, &object, headers, &resp_headers); if (aos_status_is_ok(s)) { content_length_str = (char*)apr_table_get(resp_headers, OSS_CONTENT_LENGTH); From 187c6a8109ee0d012764eb381095913952706aea Mon Sep 17 00:00:00 2001 From: "qi.xu" Date: Fri, 6 Apr 2018 16:55:32 -0700 Subject: [PATCH 4/8] compile fix --- oss_c_sdk/oss_api.h | 6 +++--- oss_c_sdk/oss_object.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/oss_c_sdk/oss_api.h b/oss_c_sdk/oss_api.h index 7da3489a..20b80f36 100644 --- a/oss_c_sdk/oss_api.h +++ b/oss_c_sdk/oss_api.h @@ -206,7 +206,7 @@ aos_status_t *oss_do_put_object_from_file(const oss_request_options_t *options, aos_status_t *oss_get_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, - aos_table_t *headers, + aos_table_t *headers, aos_table_t *params, aos_list_t *buffer, aos_table_t **resp_headers); @@ -215,8 +215,7 @@ aos_status_t *oss_select_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, const aos_string_t *sql, - aos_table_t *headers, - aos_table_t *params, + aos_table_t *headers, aos_list_t *buffer, aos_table_t **resp_headers); /* @@ -235,6 +234,7 @@ aos_status_t *oss_do_get_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, aos_table_t *headers, + aos_table_t *params, aos_list_t *buffer, oss_progress_callback progress_callback, aos_table_t **resp_headers); diff --git a/oss_c_sdk/oss_object.c b/oss_c_sdk/oss_object.c index 4f13ea8d..613dffc5 100644 --- a/oss_c_sdk/oss_object.c +++ b/oss_c_sdk/oss_object.c @@ -201,8 +201,8 @@ aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *option headers = aos_table_create_if_null(options, headers, 0); params = aos_table_create_if_null(options, params, 0); - apr_table_add(query_params, "x-oss-process", "csv/selcet"); - apr_table_add(query_params, "sql", sql); + apr_table_add(params, "x-oss-process", "csv/selcet"); + apr_table_add(params, "sql", sql); oss_init_object_request(options, bucket, object, HTTP_GET, &req, params, headers, progress_callback, 0, &resp); From d0555188a28688f15d2c125deb74aa6b7ef20055 Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 9 Apr 2018 16:50:55 -0700 Subject: [PATCH 5/8] Add oss select APIs and two samples --- oss_c_sdk/aos_define.h | 2 +- oss_c_sdk/aos_transport.c | 5 +- oss_c_sdk/oss_api.h | 5 + oss_c_sdk/oss_auth.c | 2 + oss_c_sdk/oss_define.c | 10 ++ oss_c_sdk/oss_define.h | 29 +++++ oss_c_sdk/oss_object.c | 57 +++++++- oss_c_sdk/oss_util.c | 95 +++++++++++++- oss_c_sdk/oss_util.h | 5 +- oss_c_sdk_sample/CMakeLists.txt | 2 +- oss_c_sdk_sample/main.c | 9 +- oss_c_sdk_sample/oss_head_object_sample.c | 2 +- oss_c_sdk_sample/oss_select_sample.c | 151 ++++++++++++++++++++++ 13 files changed, 359 insertions(+), 15 deletions(-) create mode 100644 oss_c_sdk_sample/oss_select_sample.c diff --git a/oss_c_sdk/aos_define.h b/oss_c_sdk/aos_define.h index 97f0fa81..f0e54531 100644 --- a/oss_c_sdk/aos_define.h +++ b/oss_c_sdk/aos_define.h @@ -107,7 +107,7 @@ typedef apr_array_header_t aos_array_header_t; #define AOS_CONNECT_TIMEOUT 10 #define AOS_DNS_CACHE_TIMOUT 60 #define AOS_MIN_SPEED_LIMIT 1024 -#define AOS_MIN_SPEED_TIME 15 +#define AOS_MIN_SPEED_TIME 500 #define AOS_MAX_MEMORY_SIZE 1024*1024*1024L #define AOS_MAX_PART_SIZE 512*1024*1024L #define AOS_DEFAULT_PART_SIZE 1024*1024L diff --git a/oss_c_sdk/aos_transport.c b/oss_c_sdk/aos_transport.c index 1fb52d4b..7928b314 100644 --- a/oss_c_sdk/aos_transport.c +++ b/oss_c_sdk/aos_transport.c @@ -384,11 +384,12 @@ int aos_curl_transport_setup(aos_curl_http_transport_t *t) curl_easy_setopt_safe(CURLOPT_CONNECTTIMEOUT, t->controller->options->connect_timeout); curl_easy_setopt_safe(CURLOPT_LOW_SPEED_LIMIT, t->controller->options->speed_limit); curl_easy_setopt_safe(CURLOPT_LOW_SPEED_TIME, t->controller->options->speed_time); + //curl_easy_setopt_safe(CURLOPT_TIMEOUT, 500); // max timeout 500s - if (t->controller->options->enable_accept_encoding) + /*if (t->controller->options->enable_accept_encoding) { curl_easy_setopt_safe(CURLOPT_ACCEPT_ENCODING, ""); - } + }*/ aos_init_curl_headers(t); curl_easy_setopt_safe(CURLOPT_HTTPHEADER, t->headers); diff --git a/oss_c_sdk/oss_api.h b/oss_c_sdk/oss_api.h index 20b80f36..35601560 100644 --- a/oss_c_sdk/oss_api.h +++ b/oss_c_sdk/oss_api.h @@ -215,6 +215,8 @@ aos_status_t *oss_select_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, const aos_string_t *sql, + const csv_format_option *csv_format, + const oss_select_option *select_option, aos_table_t *headers, aos_list_t *buffer, aos_table_t **resp_headers); @@ -243,6 +245,8 @@ aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *option const aos_string_t *bucket, const aos_string_t *object, const aos_string_t *sql, + const csv_format_option *csv_format, + const oss_select_option *select_option, aos_table_t *headers, aos_list_t *buffer, oss_progress_callback progress_callback, @@ -306,6 +310,7 @@ aos_status_t *oss_head_object(const oss_request_options_t *options, aos_status_t *oss_head_csv_object(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, + const csv_format_option *csv_format, aos_table_t *headers, aos_table_t **resp_headers); diff --git a/oss_c_sdk/oss_auth.c b/oss_c_sdk/oss_auth.c index ee246908..1fb12768 100644 --- a/oss_c_sdk/oss_auth.c +++ b/oss_c_sdk/oss_auth.c @@ -25,6 +25,8 @@ static const char *g_s_oss_sub_resource_list[] = { "endTime", "x-oss-process", "security-token", + "csv", + "sql", NULL, }; diff --git a/oss_c_sdk/oss_define.c b/oss_c_sdk/oss_define.c index 4cb91937..e537c6db 100644 --- a/oss_c_sdk/oss_define.c +++ b/oss_c_sdk/oss_define.c @@ -59,6 +59,16 @@ const char LIVE_CHANNEL_STATUS_IDLE[] = "idle"; const char LIVE_CHANNEL_STATUS_LIVE[] = "live"; const char LIVE_CHANNEL_DEFAULT_TYPE[] = "HLS"; const char LIVE_CHANNEL_DEFAULT_PLAYLIST[] = "playlist.m3u8"; +const char OSS_SELECT_CSV[] = "csv/select"; +const char OSS_SELECT_SQL[] = "sql"; +const char OSS_SELECT_INPUT_FIELD_DELIMITER[] = "x-oss-select-input-field-delimiter"; +const char OSS_SELECT_INPUT_QUOTE_CHARACTER[] = "x-oss-select-input-quote-character"; +const char OSS_SELECT_INPUT_RECORD_DELIMITER[] = "x-oss-select-input-record-delimiter"; +const char OSS_SELECT_INPUT_FILE_HEADER[] = "x-oss-select-input-file-header"; +const char OSS_SELECT_LINE_RANGE[] = "x-oss-select-line-range"; +const char OSS_SELECT_OUTPUT_KEEP_ALL_COLUMNS[] = "x-oss-select-keep-all-columns"; +const char OSS_SELECT_OUTPUT_RAW[] = "x-oss-select-output-raw-data"; +const char OSS_SELECT_CSV_ROWS[] = "x-oss-select-csv-rows"; const int LIVE_CHANNEL_DEFAULT_FRAG_DURATION = 5; const int LIVE_CHANNEL_DEFAULT_FRAG_COUNT = 3; const int OSS_MAX_PART_NUM = 10000; diff --git a/oss_c_sdk/oss_define.h b/oss_c_sdk/oss_define.h index 7afccc69..0df64dfd 100644 --- a/oss_c_sdk/oss_define.h +++ b/oss_c_sdk/oss_define.h @@ -87,6 +87,16 @@ extern const char LIVE_CHANNEL_DEFAULT_TYPE[]; extern const char LIVE_CHANNEL_DEFAULT_PLAYLIST[]; extern const int LIVE_CHANNEL_DEFAULT_FRAG_DURATION; extern const int LIVE_CHANNEL_DEFAULT_FRAG_COUNT; +extern const char OSS_SELECT_CSV[]; +extern const char OSS_SELECT_SQL[]; +extern const char OSS_SELECT_INPUT_FIELD_DELIMITER[]; +extern const char OSS_SELECT_INPUT_QUOTE_CHARACTER[]; +extern const char OSS_SELECT_INPUT_RECORD_DELIMITER[]; +extern const char OSS_SELECT_INPUT_FILE_HEADER[]; +extern const char OSS_SELECT_LINE_RANGE[]; +extern const char OSS_SELECT_OUTPUT_KEEP_ALL_COLUMNS[]; +extern const char OSS_SELECT_OUTPUT_RAW[]; +extern const char OSS_SELECT_CSV_ROWS[]; extern const int OSS_MAX_PART_NUM; extern const int OSS_PER_RET_NUM; extern const int MAX_SUFFIX_LEN; @@ -315,4 +325,23 @@ typedef struct { aos_string_t remote_addr; } oss_live_record_content_t; +typedef enum{ + CSV_HEADER_IGNORE = 0, + CSV_HEADER_NONE = 1, + CSV_HEADER_USE = 2 +} csv_header_info; + +typedef struct { + char field_delimiter; + char field_quote; + aos_string_t new_line; + csv_header_info header_info; +} csv_format_option; + +typedef struct { + int raw_output; + int keep_all_columns; + int start_line; + int end_line; +} oss_select_option; #endif diff --git a/oss_c_sdk/oss_object.c b/oss_c_sdk/oss_object.c index 613dffc5..e8bb35f6 100644 --- a/oss_c_sdk/oss_object.c +++ b/oss_c_sdk/oss_object.c @@ -146,11 +146,13 @@ aos_status_t *oss_select_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, const aos_string_t *sql, + const csv_format_option *csv_format, + const oss_select_option *select_option, aos_table_t *headers, aos_list_t *buffer, aos_table_t **resp_headers) { - return oss_do_select_object_to_buffer(options, bucket, object, sql, headers, + return oss_do_select_object_to_buffer(options, bucket, object, sql, csv_format, select_option, headers, buffer, NULL, resp_headers); } @@ -189,6 +191,8 @@ aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *option const aos_string_t *bucket, const aos_string_t *object, const aos_string_t *sql, + const csv_format_option *csv_format, + const oss_select_option *select_option, aos_table_t *headers, aos_list_t *buffer, oss_progress_callback progress_callback, @@ -201,14 +205,40 @@ aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *option headers = aos_table_create_if_null(options, headers, 0); params = aos_table_create_if_null(options, params, 0); - apr_table_add(params, "x-oss-process", "csv/selcet"); - apr_table_add(params, "sql", sql); + apr_table_add(params, OSS_PROCESS, OSS_SELECT_CSV); + apr_table_add(params, OSS_SELECT_SQL, sql->data); + + if (csv_format != NULL){ + char field_delimiter[4]; + apr_table_set(headers, OSS_SELECT_INPUT_FIELD_DELIMITER, delimiter_to_string(csv_format->field_delimiter, field_delimiter)); + + char newline[5]; + apr_table_set(headers, OSS_SELECT_INPUT_RECORD_DELIMITER, newline_to_string(&(csv_format->new_line), newline)); + + char quote[2]; + quote[0] = csv_format->field_quote; + quote[1] = 0; + apr_table_set(headers, OSS_SELECT_INPUT_QUOTE_CHARACTER,quote); + + char file_header[10]; + apr_table_set(headers, OSS_SELECT_INPUT_FILE_HEADER, file_header_to_string(csv_format->header_info, file_header)); + + } + + if (select_option) + { + apr_table_set(headers, OSS_SELECT_OUTPUT_KEEP_ALL_COLUMNS, select_option->keep_all_columns ? "true" : "false"); + apr_table_set(headers, OSS_SELECT_OUTPUT_RAW, select_option->raw_output ? "true" : "false"); + + char range_str[64]; + if (range_to_string(select_option->start_line, select_option->end_line, range_str) != NULL){ + apr_table_set(headers, OSS_SELECT_LINE_RANGE, range_str); + } + } oss_init_object_request(options, bucket, object, HTTP_GET, &req, params, headers, progress_callback, 0, &resp); - - s = oss_process_request(options, req, resp); oss_fill_read_response_body(resp, buffer); oss_fill_read_response_header(resp, resp_headers); @@ -303,6 +333,7 @@ aos_status_t *oss_head_object(const oss_request_options_t *options, aos_status_t *oss_head_csv_object(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, + const csv_format_option *csv_format, aos_table_t *headers, aos_table_t **resp_headers) { @@ -317,6 +348,22 @@ aos_status_t *oss_head_csv_object(const oss_request_options_t *options, apr_table_add(query_params, "csv", ""); + if (csv_format != NULL){ + char field_delimiter[4]; + apr_table_set(headers, OSS_SELECT_INPUT_FIELD_DELIMITER, delimiter_to_string(csv_format->field_delimiter, field_delimiter)); + + char newline[5]; + apr_table_set(headers, OSS_SELECT_INPUT_RECORD_DELIMITER, newline_to_string(&(csv_format->new_line), newline)); + + char quote[2]; + quote[0] = csv_format->field_quote; + quote[1] = 0; + apr_table_set(headers, OSS_SELECT_INPUT_QUOTE_CHARACTER,quote); + + char file_header[10]; + apr_table_set(headers, OSS_SELECT_INPUT_FILE_HEADER, file_header_to_string(csv_format->header_info, file_header)); + } + oss_init_object_request(options, bucket, object, HTTP_HEAD, &req, query_params, headers, NULL, 0, &resp); diff --git a/oss_c_sdk/oss_util.c b/oss_c_sdk/oss_util.c index da95c8b4..6ba1affc 100644 --- a/oss_c_sdk/oss_util.c +++ b/oss_c_sdk/oss_util.c @@ -147,7 +147,19 @@ static void generate_rtmp_proto(const oss_request_options_t *options, int is_valid_ip(const char *str) { - if (INADDR_NONE == inet_addr(str) || INADDR_ANY == inet_addr(str)) { + char ip[24]; + for(int i = 0;i < strlen(str)&& i <24;i++){ + if (str[i] != ':'){ + ip[i]=str[i]; + } + else + { + ip[i]=0; + break; + } + } + + if (INADDR_NONE == inet_addr(ip) || INADDR_ANY == inet_addr(ip)) { return 0; } return 1; @@ -924,3 +936,84 @@ int oss_temp_file_rename(aos_status_t *s, const char *from_path, const char *to_ return res; } +char* delimiter_to_string(char c, char* str) +{ + switch(c) + { + case '\t': + *str = '\\'; + *(str+1) = 't'; + *(str+2) = 0; + break; + case '\v': + *str = '\\'; + *(str+1) = 'v'; + *(str+2) = 0; + break; + default: + *str = c; + *(str+1) = 0; + break; + } + + return str; +} + +char* newline_to_string(const aos_string_t *newline, char* newline_str) +{ + int j = 0; + for(int i=0; i < newline->len; i++, j++){ + if (newline->data[i] == '\n'){ + newline_str[j] = '\\'; + newline_str[j+1] = 'n'; + j++; + } + else if (newline->data[i] == '\r'){ + newline_str[j] = '\\'; + newline_str[j+1] = 'r'; + j++; + } + else{ + newline_str[j] = newline->data[i]; + } + } + + newline_str[j] = '\0'; + return newline_str; +} + +char* file_header_to_string(const csv_header_info header, char* file_header_str) +{ + if (header == CSV_HEADER_IGNORE) + { + strcpy(file_header_str, "Ignore"); + } + else if (header == CSV_HEADER_USE) + { + strcpy(file_header_str, "Use"); + } + else if(header == CSV_HEADER_NONE) + { + strcpy(file_header_str, "None"); + } + + return file_header_str; +} + +char* range_to_string(int start, int end, char* range_str) +{ + if (start >= 0 && end >= 0){ + sprintf(range_str, "%d-%d", start, end); + } + else if (start >= 0){ + sprintf(range_str, "%d-", start); + } + else if (end >= 0){ + sprintf(range_str, "-%d", end); + } + else{ + return NULL; + } + + return range_str; +} \ No newline at end of file diff --git a/oss_c_sdk/oss_util.h b/oss_c_sdk/oss_util.h index 713ad713..aaf9b802 100644 --- a/oss_c_sdk/oss_util.h +++ b/oss_c_sdk/oss_util.h @@ -293,6 +293,9 @@ int oss_get_temporary_file_name(aos_pool_t *p, const aos_string_t *filename, aos int oss_temp_file_rename(aos_status_t *s, const char *from_path, const char *to_path, apr_pool_t *pool); -OSS_CPP_END +char* delimiter_to_string(char c, char* str); +char* newline_to_string(const aos_string_t *newline, char* newline_str); +char* file_header_to_string(const csv_header_info header, char* file_header_str); +char* range_to_string(int start, int end, char* range_str); #endif diff --git a/oss_c_sdk_sample/CMakeLists.txt b/oss_c_sdk_sample/CMakeLists.txt index de762900..b6263b4e 100644 --- a/oss_c_sdk_sample/CMakeLists.txt +++ b/oss_c_sdk_sample/CMakeLists.txt @@ -6,7 +6,7 @@ set(OSS_SAMPLE_SOURCE_FILES oss_sample_util.c oss_config.c main.c oss_put_object_sample.c oss_append_object_sample.c oss_get_object_sample.c oss_head_object_sample.c oss_delete_object_sample.c oss_multipart_upload_sample.c oss_crc_sample.c oss_image_sample.c oss_progress_sample.c oss_callback_sample.c - oss_list_object_sample.c oss_resumable_sample.c) + oss_list_object_sample.c oss_resumable_sample.c oss_select_sample.c) include_directories (${APR_INCLUDE_DIR}) include_directories (${APR_UTIL_INCLUDE_DIR}) diff --git a/oss_c_sdk_sample/main.c b/oss_c_sdk_sample/main.c index b8382177..9b8fc174 100644 --- a/oss_c_sdk_sample/main.c +++ b/oss_c_sdk_sample/main.c @@ -21,17 +21,20 @@ extern void crc_sample(); extern void image_sample(); extern void resumable_upload_sample(); extern void resumable_download_sample(); - +extern void select_object_to_buffer_sample(); int main(int argc, char *argv[]) { if (aos_http_io_initialize(NULL, 0) != AOSE_OK) { exit(1); } + select_object_to_buffer_sample(); + printf("select_object_to_buffer_big_file_sample\n"); + select_object_to_buffer_big_file_sample(); //put_object_sample(); //append_object_sample(); //get_object_sample(); - head_object_sample(); + //head_object_sample(); /*list_object_sample(); multipart_object_sample(); delete_object_sample(); @@ -44,7 +47,7 @@ int main(int argc, char *argv[]) aos_http_io_deinitialize(); - system("pause"); + //system("pause"); return 0; } diff --git a/oss_c_sdk_sample/oss_head_object_sample.c b/oss_c_sdk_sample/oss_head_object_sample.c index 2aa7f058..cce76f6b 100644 --- a/oss_c_sdk_sample/oss_head_object_sample.c +++ b/oss_c_sdk_sample/oss_head_object_sample.c @@ -29,7 +29,7 @@ void head_object() aos_str_set(&object, OBJECT_NAME); headers = aos_table_make(p, 0); - s = oss_head_csv_object(options, &bucket, &object, headers, &resp_headers); + s = oss_head_object(options, &bucket, &object, headers, &resp_headers); if (aos_status_is_ok(s)) { content_length_str = (char*)apr_table_get(resp_headers, OSS_CONTENT_LENGTH); diff --git a/oss_c_sdk_sample/oss_select_sample.c b/oss_c_sdk_sample/oss_select_sample.c new file mode 100644 index 00000000..99ad8037 --- /dev/null +++ b/oss_c_sdk_sample/oss_select_sample.c @@ -0,0 +1,151 @@ +#include "aos_log.h" +#include "aos_util.h" +#include "aos_string.h" +#include "aos_status.h" +#include "oss_auth.h" +#include "oss_util.h" +#include "oss_api.h" +#include "oss_config.h" +#include "oss_sample_util.h" + +int head_csv_object() +{ + aos_pool_t *p = NULL; + aos_string_t bucket; + aos_string_t object; + int is_cname = 0; + oss_request_options_t *options = NULL; + aos_table_t *headers = NULL; + aos_table_t *resp_headers = NULL; + aos_status_t *s = NULL; + char *rows_str = NULL; + char *object_type = NULL; + int64_t rows = 0; + + aos_pool_create(&p, NULL); + options = oss_request_options_create(p); + init_sample_request_options(options, is_cname); + aos_str_set(&bucket, BUCKET_NAME); + aos_str_set(&object, OBJECT_NAME); + headers = aos_table_make(p, 0); + + csv_format_option csv_format; + csv_format.field_delimiter = ','; + csv_format.field_quote = '"'; + aos_str_set(&(csv_format.new_line), "\r\n"); + csv_format.header_info = CSV_HEADER_IGNORE; + + s = oss_head_csv_object(options, &bucket, &object, &csv_format, headers, &resp_headers); + + if (aos_status_is_ok(s)) { + rows_str = (char*)apr_table_get(resp_headers, OSS_SELECT_CSV_ROWS); + if (rows_str != NULL) { + rows = atol(rows_str); + } + + object_type = (char*)apr_table_get(resp_headers, OSS_OBJECT_TYPE); + + printf("csv head object succeeded, object type:%s, csv rows:%ld\n", + object_type, rows); + } else { + printf("csv head object failed\n"); + } + + aos_pool_destroy(p); + return rows; +} + +char* select_object_to_buffer(aos_pool_t *p, const char* sql_str, int start_line, int end_line, int64_t* plen) +{ + aos_string_t bucket; + aos_string_t object; + int is_cname = 0; + oss_request_options_t *options = NULL; + aos_table_t *headers = NULL; + aos_table_t *params = NULL; + aos_table_t *resp_headers = NULL; + aos_status_t *s = NULL; + aos_list_t buffer; + aos_buf_t *content = NULL; + char *buf = NULL; + int64_t len = 0; + int64_t size = 0; + int64_t pos = 0; + + options = oss_request_options_create(p); + init_sample_request_options(options, is_cname); + aos_str_set(&bucket, BUCKET_NAME); + aos_str_set(&object, OBJECT_NAME); + aos_list_init(&buffer); + + csv_format_option csv_format; + csv_format.field_delimiter = ','; + csv_format.field_quote = '"'; + aos_str_set(&(csv_format.new_line), "\r\n"); + csv_format.header_info = CSV_HEADER_IGNORE; + + oss_select_option select_option; + select_option.keep_all_columns = 1; + select_option.raw_output = 1; + select_option.start_line = start_line; + select_option.end_line = end_line; + + aos_string_t sql; + aos_str_set(&sql, sql_str); + s = oss_select_object_to_buffer(options, &bucket, &object, &sql, &csv_format, &select_option, + headers, &buffer, &resp_headers); + + if (aos_status_is_ok(s)) { + printf("get object to buffer succeeded\n"); + } + else { + printf("get object to buffer failed\n"); + } + + //get buffer len + aos_list_for_each_entry(aos_buf_t, content, &buffer, node) { + len += aos_buf_size(content); + } + + buf = aos_pcalloc(p, (apr_size_t)(len + 1)); + buf[len] = '\0'; + + //copy buffer content to memory + aos_list_for_each_entry(aos_buf_t, content, &buffer, node) { + size = aos_buf_size(content); + memcpy(buf + pos, content->pos, (size_t)size); + pos += size; + } + + *plen = len; + return buf; +} + +void select_object_to_buffer_sample() +{ + int64_t result_len = 0; + aos_pool_t *p=NULL; + aos_pool_create(&p, NULL); + char *buf = select_object_to_buffer(p, "select count(*) from ossobject where _4 > 60 and _1 like 'Tom%'", -1, -1, &result_len); + printf(buf); + buf = select_object_to_buffer(p, "select _1,_4 from ossobject where _4 > 60 and _1 like 'Tom%' limit 100", -1, -1, &result_len); + printf("\n\n%s", buf); + aos_pool_destroy(p); +} + +void select_object_to_buffer_big_file_sample() +{ + aos_pool_t *p=NULL; + aos_pool_create(&p, NULL); + int lines = head_csv_object(); + int blockSize = 8; + int64_t result_len = 0; + for(int i = 0; i < blockSize; i++){ + int start_line = i * lines/blockSize; + int end_line = i == blockSize -1 ? lines - 1 : (i+1) * lines/blockSize - 1; + char *buf = select_object_to_buffer(p, "select count(*) from ossobject where _4 > 45 and _1 like 'Tom%'", start_line, end_line, &result_len); + printf(buf); + } + + aos_pool_destroy(p); +} \ No newline at end of file From dec1fa7d7658a9abbf00cb47ae17b38d0b0739fb Mon Sep 17 00:00:00 2001 From: Unknown Date: Mon, 9 Apr 2018 17:13:52 -0700 Subject: [PATCH 6/8] fix a bug: the header name should be x-oss-select-output-keep-all-columns --- oss_c_sdk/oss_define.c | 2 +- oss_c_sdk_sample/oss_select_sample.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oss_c_sdk/oss_define.c b/oss_c_sdk/oss_define.c index e537c6db..ddc34ca8 100644 --- a/oss_c_sdk/oss_define.c +++ b/oss_c_sdk/oss_define.c @@ -66,7 +66,7 @@ const char OSS_SELECT_INPUT_QUOTE_CHARACTER[] = "x-oss-select-input-quote-charac const char OSS_SELECT_INPUT_RECORD_DELIMITER[] = "x-oss-select-input-record-delimiter"; const char OSS_SELECT_INPUT_FILE_HEADER[] = "x-oss-select-input-file-header"; const char OSS_SELECT_LINE_RANGE[] = "x-oss-select-line-range"; -const char OSS_SELECT_OUTPUT_KEEP_ALL_COLUMNS[] = "x-oss-select-keep-all-columns"; +const char OSS_SELECT_OUTPUT_KEEP_ALL_COLUMNS[] = "x-oss-select-output-keep-all-columns"; const char OSS_SELECT_OUTPUT_RAW[] = "x-oss-select-output-raw-data"; const char OSS_SELECT_CSV_ROWS[] = "x-oss-select-csv-rows"; const int LIVE_CHANNEL_DEFAULT_FRAG_DURATION = 5; diff --git a/oss_c_sdk_sample/oss_select_sample.c b/oss_c_sdk_sample/oss_select_sample.c index 99ad8037..6d8776b3 100644 --- a/oss_c_sdk_sample/oss_select_sample.c +++ b/oss_c_sdk_sample/oss_select_sample.c @@ -143,7 +143,7 @@ void select_object_to_buffer_big_file_sample() for(int i = 0; i < blockSize; i++){ int start_line = i * lines/blockSize; int end_line = i == blockSize -1 ? lines - 1 : (i+1) * lines/blockSize - 1; - char *buf = select_object_to_buffer(p, "select count(*) from ossobject where _4 > 45 and _1 like 'Tom%'", start_line, end_line, &result_len); + char *buf = select_object_to_buffer(p, "select count(*) from ossobject where _4 > 60 and _1 like 'Tom%'", start_line, end_line, &result_len); printf(buf); } From 270f1805bb5854161338a108edce3f0dad47f9a4 Mon Sep 17 00:00:00 2001 From: "qi.xu" Date: Fri, 27 Apr 2018 02:27:48 +0800 Subject: [PATCH 7/8] update head csv object API --- oss_c_sdk/oss_object.c | 4 ++-- oss_c_sdk/oss_util.c | 8 +++++--- oss_c_sdk_sample/CMakeLists.txt | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/oss_c_sdk/oss_object.c b/oss_c_sdk/oss_object.c index e8bb35f6..d6ceff10 100644 --- a/oss_c_sdk/oss_object.c +++ b/oss_c_sdk/oss_object.c @@ -346,7 +346,7 @@ aos_status_t *oss_head_csv_object(const oss_request_options_t *options, query_params = aos_table_create_if_null(options, query_params, 0); - apr_table_add(query_params, "csv", ""); + apr_table_add(query_params, "x-oss-process", "csv/meta"); if (csv_format != NULL){ char field_delimiter[4]; @@ -364,7 +364,7 @@ aos_status_t *oss_head_csv_object(const oss_request_options_t *options, apr_table_set(headers, OSS_SELECT_INPUT_FILE_HEADER, file_header_to_string(csv_format->header_info, file_header)); } - oss_init_object_request(options, bucket, object, HTTP_HEAD, + oss_init_object_request(options, bucket, object, HTTP_GET, &req, query_params, headers, NULL, 0, &resp); s = oss_process_request(options, req, resp); diff --git a/oss_c_sdk/oss_util.c b/oss_c_sdk/oss_util.c index 6ba1affc..4fd1fa61 100644 --- a/oss_c_sdk/oss_util.c +++ b/oss_c_sdk/oss_util.c @@ -148,7 +148,8 @@ static void generate_rtmp_proto(const oss_request_options_t *options, int is_valid_ip(const char *str) { char ip[24]; - for(int i = 0;i < strlen(str)&& i <24;i++){ + int i = 0; + for(i = 0;i < strlen(str)&& i <24;i++){ if (str[i] != ':'){ ip[i]=str[i]; } @@ -962,7 +963,8 @@ char* delimiter_to_string(char c, char* str) char* newline_to_string(const aos_string_t *newline, char* newline_str) { int j = 0; - for(int i=0; i < newline->len; i++, j++){ + int i = 0; + for(i=0; i < newline->len; i++, j++){ if (newline->data[i] == '\n'){ newline_str[j] = '\\'; newline_str[j+1] = 'n'; @@ -1016,4 +1018,4 @@ char* range_to_string(int start, int end, char* range_str) } return range_str; -} \ No newline at end of file +} diff --git a/oss_c_sdk_sample/CMakeLists.txt b/oss_c_sdk_sample/CMakeLists.txt index b6263b4e..c716d114 100644 --- a/oss_c_sdk_sample/CMakeLists.txt +++ b/oss_c_sdk_sample/CMakeLists.txt @@ -32,6 +32,7 @@ function(_TARGET_SAMPLE_LIBRARIES SAMPLE_BIN_NAME SOURCE_FILES) target_link_libraries(${SAMPLE_BIN_NAME} ${PTHREAD_LIBRARY}) target_link_libraries(${SAMPLE_BIN_NAME} ${RT_LIBRARY}) target_link_libraries(${SAMPLE_BIN_NAME} ${M_LIBRARY}) + target_link_libraries(${SAMPLE_BIN_NAME} expat) endfunction() _TARGET_SAMPLE_LIBRARIES(oss_c_sdk_sample "${OSS_SAMPLE_SOURCE_FILES}") From 901c2179638d1c3eaaf995a514b732ed32943dc6 Mon Sep 17 00:00:00 2001 From: wujinhu Date: Thu, 7 Jun 2018 16:49:37 +0800 Subject: [PATCH 8/8] change select to post --- oss_c_sdk/aos_status.c | 1 + oss_c_sdk/aos_status.h | 1 + oss_c_sdk/oss_api.h | 8 +- oss_c_sdk/oss_auth.c | 2 - oss_c_sdk/oss_define.c | 31 ++-- oss_c_sdk/oss_define.h | 55 +++++-- oss_c_sdk/oss_object.c | 76 ++++------ oss_c_sdk/oss_util.c | 75 +--------- oss_c_sdk/oss_util.h | 6 +- oss_c_sdk/oss_xml.c | 206 +++++++++++++++++++++++++++ oss_c_sdk/oss_xml.h | 12 ++ oss_c_sdk_sample/main.c | 1 + oss_c_sdk_sample/oss_select_sample.c | 78 ++++++---- 13 files changed, 371 insertions(+), 181 deletions(-) diff --git a/oss_c_sdk/aos_status.c b/oss_c_sdk/aos_status.c index adf589dd..13f767ee 100644 --- a/oss_c_sdk/aos_status.c +++ b/oss_c_sdk/aos_status.c @@ -4,6 +4,7 @@ const char AOS_XML_PARSE_ERROR_CODE[] = "ParseXmlError"; const char AOS_OPEN_FILE_ERROR_CODE[] = "OpenFileFail"; +const char AOS_SELECT_INPUT_ERROR_CODE[] = "SelectInputError"; const char AOS_WRITE_FILE_ERROR_CODE[] = "WriteFileFail"; const char AOS_RENAME_FILE_ERROR_CODE[] = "RenameFileFail"; const char AOS_HTTP_IO_ERROR_CODE[] = "HttpIoError"; diff --git a/oss_c_sdk/aos_status.h b/oss_c_sdk/aos_status.h index 16de2741..4b4bd00f 100644 --- a/oss_c_sdk/aos_status.h +++ b/oss_c_sdk/aos_status.h @@ -43,6 +43,7 @@ aos_status_t *aos_status_parse_from_body(aos_pool_t *p, aos_list_t *bc, int code extern const char AOS_XML_PARSE_ERROR_CODE[]; extern const char AOS_OPEN_FILE_ERROR_CODE[]; +extern const char AOS_SELECT_INPUT_ERROR_CODE[]; extern const char AOS_WRITE_FILE_ERROR_CODE[]; extern const char AOS_RENAME_FILE_ERROR_CODE[]; extern const char AOS_HTTP_IO_ERROR_CODE[]; diff --git a/oss_c_sdk/oss_api.h b/oss_c_sdk/oss_api.h index 35601560..740f2ab7 100644 --- a/oss_c_sdk/oss_api.h +++ b/oss_c_sdk/oss_api.h @@ -214,8 +214,6 @@ aos_status_t *oss_get_object_to_buffer(const oss_request_options_t *options, aos_status_t *oss_select_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, - const aos_string_t *sql, - const csv_format_option *csv_format, const oss_select_option *select_option, aos_table_t *headers, aos_list_t *buffer, @@ -244,8 +242,6 @@ aos_status_t *oss_do_get_object_to_buffer(const oss_request_options_t *options, aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, - const aos_string_t *sql, - const csv_format_option *csv_format, const oss_select_option *select_option, aos_table_t *headers, aos_list_t *buffer, @@ -307,10 +303,10 @@ aos_status_t *oss_head_object(const oss_request_options_t *options, aos_table_t *headers, aos_table_t **resp_headers); -aos_status_t *oss_head_csv_object(const oss_request_options_t *options, +aos_status_t *oss_create_select_object_metadata(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, - const csv_format_option *csv_format, + const oss_select_metadata_option *select_metadata_option, aos_table_t *headers, aos_table_t **resp_headers); diff --git a/oss_c_sdk/oss_auth.c b/oss_c_sdk/oss_auth.c index 1fb12768..ee246908 100644 --- a/oss_c_sdk/oss_auth.c +++ b/oss_c_sdk/oss_auth.c @@ -25,8 +25,6 @@ static const char *g_s_oss_sub_resource_list[] = { "endTime", "x-oss-process", "security-token", - "csv", - "sql", NULL, }; diff --git a/oss_c_sdk/oss_define.c b/oss_c_sdk/oss_define.c index ddc34ca8..ae47e385 100644 --- a/oss_c_sdk/oss_define.c +++ b/oss_c_sdk/oss_define.c @@ -1,4 +1,5 @@ #include "oss_define.h" +#include "aos_string.h" const char OSS_CANNONICALIZED_HEADER_PREFIX[] = "x-oss-"; const char OSS_CANNONICALIZED_HEADER_DATE[] = "x-oss-date"; @@ -60,17 +61,31 @@ const char LIVE_CHANNEL_STATUS_LIVE[] = "live"; const char LIVE_CHANNEL_DEFAULT_TYPE[] = "HLS"; const char LIVE_CHANNEL_DEFAULT_PLAYLIST[] = "playlist.m3u8"; const char OSS_SELECT_CSV[] = "csv/select"; -const char OSS_SELECT_SQL[] = "sql"; -const char OSS_SELECT_INPUT_FIELD_DELIMITER[] = "x-oss-select-input-field-delimiter"; -const char OSS_SELECT_INPUT_QUOTE_CHARACTER[] = "x-oss-select-input-quote-character"; -const char OSS_SELECT_INPUT_RECORD_DELIMITER[] = "x-oss-select-input-record-delimiter"; -const char OSS_SELECT_INPUT_FILE_HEADER[] = "x-oss-select-input-file-header"; -const char OSS_SELECT_LINE_RANGE[] = "x-oss-select-line-range"; -const char OSS_SELECT_OUTPUT_KEEP_ALL_COLUMNS[] = "x-oss-select-output-keep-all-columns"; -const char OSS_SELECT_OUTPUT_RAW[] = "x-oss-select-output-raw-data"; +const char OSS_CSV_META[] = "csv/meta"; const char OSS_SELECT_CSV_ROWS[] = "x-oss-select-csv-rows"; +const char OSS_SELECT_CSV_SPLITS[] = "x-oss-select-csv-splits"; const int LIVE_CHANNEL_DEFAULT_FRAG_DURATION = 5; const int LIVE_CHANNEL_DEFAULT_FRAG_COUNT = 3; const int OSS_MAX_PART_NUM = 10000; const int OSS_PER_RET_NUM = 1000; const int MAX_SUFFIX_LEN = 1024; + +const csv_format_option csv_format_option_default = { + .field_delimiter = ',', + .field_quote = '"', + .comment = '#', + .record_delimiter.data = "\n", + .record_delimiter.len = 1, + .header_info = CSV_HEADER_IGNORE +}; + +const oss_select_metadata_option oss_select_metadata_option_default = { + .overwrite = 0, + .input_serialization.compression_info = NONE, + .input_serialization.csv_format.field_delimiter = ',', + .input_serialization.csv_format.field_quote = '"', + .input_serialization.csv_format.comment = '#', + .input_serialization.csv_format.record_delimiter.data = "\n", + .input_serialization.csv_format.record_delimiter.len = 1, + .input_serialization.csv_format.header_info = CSV_HEADER_IGNORE +}; diff --git a/oss_c_sdk/oss_define.h b/oss_c_sdk/oss_define.h index 0df64dfd..8ef8174e 100644 --- a/oss_c_sdk/oss_define.h +++ b/oss_c_sdk/oss_define.h @@ -88,15 +88,9 @@ extern const char LIVE_CHANNEL_DEFAULT_PLAYLIST[]; extern const int LIVE_CHANNEL_DEFAULT_FRAG_DURATION; extern const int LIVE_CHANNEL_DEFAULT_FRAG_COUNT; extern const char OSS_SELECT_CSV[]; -extern const char OSS_SELECT_SQL[]; -extern const char OSS_SELECT_INPUT_FIELD_DELIMITER[]; -extern const char OSS_SELECT_INPUT_QUOTE_CHARACTER[]; -extern const char OSS_SELECT_INPUT_RECORD_DELIMITER[]; -extern const char OSS_SELECT_INPUT_FILE_HEADER[]; -extern const char OSS_SELECT_LINE_RANGE[]; -extern const char OSS_SELECT_OUTPUT_KEEP_ALL_COLUMNS[]; -extern const char OSS_SELECT_OUTPUT_RAW[]; +extern const char OSS_CSV_META[]; extern const char OSS_SELECT_CSV_ROWS[]; +extern const char OSS_SELECT_CSV_SPLITS[]; extern const int OSS_MAX_PART_NUM; extern const int OSS_PER_RET_NUM; extern const int MAX_SUFFIX_LEN; @@ -331,17 +325,54 @@ typedef enum{ CSV_HEADER_USE = 2 } csv_header_info; +typedef enum { + NONE = 1, + GZIP = 2 +} select_compression_info; + typedef struct { char field_delimiter; char field_quote; - aos_string_t new_line; + char comment; + aos_string_t record_delimiter; csv_header_info header_info; } csv_format_option; +extern const csv_format_option csv_format_option_default; + typedef struct { - int raw_output; + select_compression_info compression_info; + csv_format_option csv_format; +} select_input_serialization; + +typedef struct { + int raw_output; //always output raw data now int keep_all_columns; - int start_line; - int end_line; + csv_format_option csv_format; +} select_output_serialization; + +typedef struct { + int overwrite; + select_input_serialization input_serialization; +} oss_select_metadata_option; + +extern const oss_select_metadata_option oss_select_metadata_option_default; + +typedef enum { + NO_RANGE = 0, + LINE = 1, + SPLIT = 2 +} select_range_option; + +typedef struct { + select_range_option range_option; + int range[2]; + aos_string_t expression; + select_input_serialization input_serialization; + select_output_serialization output_serialization; } oss_select_option; + +#define aos_select_input_error_status_set(STATUS, RES, MSG) do {\ + aos_status_set(STATUS, RES, AOS_SELECT_INPUT_ERROR_CODE, MSG); \ + } while(0) #endif diff --git a/oss_c_sdk/oss_object.c b/oss_c_sdk/oss_object.c index d6ceff10..51ebea02 100644 --- a/oss_c_sdk/oss_object.c +++ b/oss_c_sdk/oss_object.c @@ -6,6 +6,7 @@ #include "oss_util.h" #include "oss_xml.h" #include "oss_api.h" +#include "oss_define.h" char *oss_gen_signed_url(const oss_request_options_t *options, const aos_string_t *bucket, @@ -145,14 +146,12 @@ aos_status_t *oss_get_object_to_buffer(const oss_request_options_t *options, aos_status_t *oss_select_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, - const aos_string_t *sql, - const csv_format_option *csv_format, const oss_select_option *select_option, aos_table_t *headers, aos_list_t *buffer, aos_table_t **resp_headers) { - return oss_do_select_object_to_buffer(options, bucket, object, sql, csv_format, select_option, headers, + return oss_do_select_object_to_buffer(options, bucket, object, select_option, headers, buffer, NULL, resp_headers); } @@ -190,8 +189,6 @@ aos_status_t *oss_do_get_object_to_buffer(const oss_request_options_t *options, aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, - const aos_string_t *sql, - const csv_format_option *csv_format, const oss_select_option *select_option, aos_table_t *headers, aos_list_t *buffer, @@ -202,43 +199,25 @@ aos_status_t *oss_do_select_object_to_buffer(const oss_request_options_t *option aos_http_request_t *req = NULL; aos_http_response_t *resp = NULL; aos_table_t*params = NULL; + aos_list_t body; + headers = aos_table_create_if_null(options, headers, 0); params = aos_table_create_if_null(options, params, 0); apr_table_add(params, OSS_PROCESS, OSS_SELECT_CSV); - apr_table_add(params, OSS_SELECT_SQL, sql->data); - - if (csv_format != NULL){ - char field_delimiter[4]; - apr_table_set(headers, OSS_SELECT_INPUT_FIELD_DELIMITER, delimiter_to_string(csv_format->field_delimiter, field_delimiter)); - - char newline[5]; - apr_table_set(headers, OSS_SELECT_INPUT_RECORD_DELIMITER, newline_to_string(&(csv_format->new_line), newline)); - - char quote[2]; - quote[0] = csv_format->field_quote; - quote[1] = 0; - apr_table_set(headers, OSS_SELECT_INPUT_QUOTE_CHARACTER,quote); - - char file_header[10]; - apr_table_set(headers, OSS_SELECT_INPUT_FILE_HEADER, file_header_to_string(csv_format->header_info, file_header)); + apr_table_set(headers, OSS_CONTENT_TYPE, ""); + if (select_option == NULL) { + s = aos_status_create(options->pool); + aos_select_input_error_status_set(s, 400, "select option is null"); + return s; } - - if (select_option) - { - apr_table_set(headers, OSS_SELECT_OUTPUT_KEEP_ALL_COLUMNS, select_option->keep_all_columns ? "true" : "false"); - apr_table_set(headers, OSS_SELECT_OUTPUT_RAW, select_option->raw_output ? "true" : "false"); - - char range_str[64]; - if (range_to_string(select_option->start_line, select_option->end_line, range_str) != NULL){ - apr_table_set(headers, OSS_SELECT_LINE_RANGE, range_str); - } - } - - oss_init_object_request(options, bucket, object, HTTP_GET, + oss_init_object_request(options, bucket, object, HTTP_POST, &req, params, headers, progress_callback, 0, &resp); + build_select_object_metadata_body(options->pool, select_option, &body); + oss_write_request_body_from_buffer(&body, req); + s = oss_process_request(options, req, resp); oss_fill_read_response_body(resp, buffer); oss_fill_read_response_header(resp, resp_headers); @@ -330,10 +309,10 @@ aos_status_t *oss_head_object(const oss_request_options_t *options, return s; } -aos_status_t *oss_head_csv_object(const oss_request_options_t *options, +aos_status_t *oss_create_select_object_metadata(const oss_request_options_t *options, const aos_string_t *bucket, const aos_string_t *object, - const csv_format_option *csv_format, + const oss_select_metadata_option *select_metadata_option, aos_table_t *headers, aos_table_t **resp_headers) { @@ -341,32 +320,25 @@ aos_status_t *oss_head_csv_object(const oss_request_options_t *options, aos_http_request_t *req = NULL; aos_http_response_t *resp = NULL; aos_table_t *query_params = NULL; + aos_list_t body; headers = aos_table_create_if_null(options, headers, 0); query_params = aos_table_create_if_null(options, query_params, 0); - apr_table_add(query_params, "x-oss-process", "csv/meta"); - - if (csv_format != NULL){ - char field_delimiter[4]; - apr_table_set(headers, OSS_SELECT_INPUT_FIELD_DELIMITER, delimiter_to_string(csv_format->field_delimiter, field_delimiter)); + apr_table_set(query_params, OSS_PROCESS, OSS_CSV_META); + apr_table_set(headers, OSS_CONTENT_TYPE, ""); - char newline[5]; - apr_table_set(headers, OSS_SELECT_INPUT_RECORD_DELIMITER, newline_to_string(&(csv_format->new_line), newline)); - - char quote[2]; - quote[0] = csv_format->field_quote; - quote[1] = 0; - apr_table_set(headers, OSS_SELECT_INPUT_QUOTE_CHARACTER,quote); - - char file_header[10]; - apr_table_set(headers, OSS_SELECT_INPUT_FILE_HEADER, file_header_to_string(csv_format->header_info, file_header)); + if (select_metadata_option == NULL) { + select_metadata_option = &oss_select_metadata_option_default; } - oss_init_object_request(options, bucket, object, HTTP_GET, + oss_init_object_request(options, bucket, object, HTTP_POST, &req, query_params, headers, NULL, 0, &resp); + build_create_select_object_metadata_body(options->pool, select_metadata_option, &body); + oss_write_request_body_from_buffer(&body, req); + s = oss_process_request(options, req, resp); oss_fill_read_response_header(resp, resp_headers); diff --git a/oss_c_sdk/oss_util.c b/oss_c_sdk/oss_util.c index 4fd1fa61..acea42f1 100644 --- a/oss_c_sdk/oss_util.c +++ b/oss_c_sdk/oss_util.c @@ -937,85 +937,20 @@ int oss_temp_file_rename(aos_status_t *s, const char *from_path, const char *to_ return res; } -char* delimiter_to_string(char c, char* str) -{ - switch(c) - { - case '\t': - *str = '\\'; - *(str+1) = 't'; - *(str+2) = 0; - break; - case '\v': - *str = '\\'; - *(str+1) = 'v'; - *(str+2) = 0; - break; - default: - *str = c; - *(str+1) = 0; - break; - } - - return str; -} - -char* newline_to_string(const aos_string_t *newline, char* newline_str) -{ - int j = 0; - int i = 0; - for(i=0; i < newline->len; i++, j++){ - if (newline->data[i] == '\n'){ - newline_str[j] = '\\'; - newline_str[j+1] = 'n'; - j++; - } - else if (newline->data[i] == '\r'){ - newline_str[j] = '\\'; - newline_str[j+1] = 'r'; - j++; - } - else{ - newline_str[j] = newline->data[i]; - } - } - - newline_str[j] = '\0'; - return newline_str; -} - -char* file_header_to_string(const csv_header_info header, char* file_header_str) -{ - if (header == CSV_HEADER_IGNORE) - { - strcpy(file_header_str, "Ignore"); - } - else if (header == CSV_HEADER_USE) - { - strcpy(file_header_str, "Use"); - } - else if(header == CSV_HEADER_NONE) - { - strcpy(file_header_str, "None"); - } - - return file_header_str; -} - -char* range_to_string(int start, int end, char* range_str) +char* range_to_string(const char* prefix, int start, int end, char* range_str) { if (start >= 0 && end >= 0){ - sprintf(range_str, "%d-%d", start, end); + sprintf(range_str, "%s%d-%d", prefix, start, end); } else if (start >= 0){ - sprintf(range_str, "%d-", start); + sprintf(range_str, "%s%d-", prefix, start); } else if (end >= 0){ - sprintf(range_str, "-%d", end); + sprintf(range_str, "%s-%d", prefix, end); } else{ return NULL; } return range_str; -} +} \ No newline at end of file diff --git a/oss_c_sdk/oss_util.h b/oss_c_sdk/oss_util.h index aaf9b802..1c3e0f51 100644 --- a/oss_c_sdk/oss_util.h +++ b/oss_c_sdk/oss_util.h @@ -293,9 +293,5 @@ int oss_get_temporary_file_name(aos_pool_t *p, const aos_string_t *filename, aos int oss_temp_file_rename(aos_status_t *s, const char *from_path, const char *to_path, apr_pool_t *pool); -char* delimiter_to_string(char c, char* str); -char* newline_to_string(const aos_string_t *newline, char* newline_str); -char* file_header_to_string(const csv_header_info header, char* file_header_str); -char* range_to_string(int start, int end, char* range_str); - +char* range_to_string(const char* prefix, int start, int end, char* range_str); #endif diff --git a/oss_c_sdk/oss_xml.c b/oss_c_sdk/oss_xml.c index ea301825..19278135 100644 --- a/oss_c_sdk/oss_xml.c +++ b/oss_c_sdk/oss_xml.c @@ -802,6 +802,212 @@ void build_create_live_channel_body(aos_pool_t *p, oss_live_channel_configuratio aos_list_add_tail(&b->node, body); } +char *build_create_select_object_metadata_xml(aos_pool_t *p, const oss_select_metadata_option *option) { + char *xml_buff; + char *create_select_object_metadata_xml; + aos_string_t xml_doc; + mxml_node_t *doc; + mxml_node_t *root_node; + mxml_node_t *overwrite_node; + mxml_node_t *input_serialization_node; + mxml_node_t *compression_type_node; + mxml_node_t *csv_node; + mxml_node_t *record_delimiter_node; + mxml_node_t *field_delimiter_node; + mxml_node_t *quote_char_node; + int encoded_len; + + doc = mxmlNewXML("1.0"); + root_node = mxmlNewElement(doc, "CsvMetaRequest"); + + overwrite_node = mxmlNewElement(root_node, "OverwriteIfExists"); + mxmlNewText(overwrite_node, 0, option->overwrite == 0 ? "false" : "true"); + + input_serialization_node = mxmlNewElement(root_node, "InputSerialization"); + compression_type_node = mxmlNewElement(input_serialization_node, "CompressionType"); + + mxmlNewText(compression_type_node, 0, option->input_serialization.compression_info == NONE ? "NONE" : "GZIP"); + csv_node = mxmlNewElement(input_serialization_node, "CSV"); + record_delimiter_node = mxmlNewElement(csv_node, "RecordDelimiter"); + + // encode and add record_delimiter + char encoded[128]; + encoded_len = aos_base64_encode((const unsigned char*)option->input_serialization.csv_format.record_delimiter.data, + option->input_serialization.csv_format.record_delimiter.len, encoded); + encoded[encoded_len] = '\0'; + mxmlNewText(record_delimiter_node, 0, encoded); + + // encode and add field_delimiter + encoded_len = aos_base64_encode((const unsigned char*)&(option->input_serialization.csv_format.field_delimiter), + 1, encoded); + encoded[encoded_len] = '\0'; + field_delimiter_node = mxmlNewElement(csv_node, "FieldDelimiter"); + mxmlNewText(field_delimiter_node, 0, encoded); + + // encode and add field_quote + encoded_len = aos_base64_encode((const unsigned char*)&(option->input_serialization.csv_format.field_quote), + 1, encoded); + encoded[encoded_len] = '\0'; + quote_char_node = mxmlNewElement(csv_node, "QuoteCharacter"); + mxmlNewText(quote_char_node, 0, encoded); + + xml_buff = new_xml_buff(doc); + if (xml_buff == NULL) { + return NULL; + } + aos_str_set(&xml_doc, xml_buff); + create_select_object_metadata_xml = aos_pstrdup(p, &xml_doc); + + free(xml_buff); + mxmlDelete(doc); + + return create_select_object_metadata_xml; +} + +void build_create_select_object_metadata_body(aos_pool_t *p, const oss_select_metadata_option *option, aos_list_t *body) { + char *create_select_object_metadata_xml; + aos_buf_t *b; + + create_select_object_metadata_xml = build_create_select_object_metadata_xml(p, option); + aos_list_init(body); + b = aos_buf_pack(p, create_select_object_metadata_xml, strlen(create_select_object_metadata_xml)); + aos_list_add_tail(&b->node, body); +} + +char *build_select_object_metadata_xml(aos_pool_t *p, const oss_select_option *option) { + char *xml_buff; + char *select_object_xml; + aos_string_t xml_doc; + mxml_node_t *doc; + mxml_node_t *root_node; + mxml_node_t *expression_node; + mxml_node_t *input_serialization_node; + mxml_node_t *compression_type_node; + mxml_node_t *csv_node; + mxml_node_t *range_node; + mxml_node_t *record_delimiter_node; + mxml_node_t *field_delimiter_node; + mxml_node_t *quote_char_node; + mxml_node_t *comment_char_node; + + mxml_node_t *output_serialization_node; + mxml_node_t *output_csv_node; + mxml_node_t *output_record_delimiter_node; + mxml_node_t *output_field_delimiter_node; + mxml_node_t *output_quote_char_node; + mxml_node_t *output_keep_all_columns_node; + mxml_node_t *output_raw_data_node; + + int encoded_len; + //TODO, make sure buffer is enough + char buff[1024 * 1024]; + + doc = mxmlNewXML("1.0"); + root_node = mxmlNewElement(doc, "SelectRequest"); + + //expression + expression_node = mxmlNewElement(root_node, "Expression"); + encoded_len = aos_base64_encode((const unsigned char*)option->expression.data, option->expression.len, buff); + buff[encoded_len] = '\0'; + mxmlNewText(expression_node, 0, buff); + + input_serialization_node = mxmlNewElement(root_node, "InputSerialization"); + compression_type_node = mxmlNewElement(input_serialization_node, "CompressionType"); + mxmlNewText(compression_type_node, 0, option->input_serialization.compression_info == NONE ? "NONE" : "GZIP"); + + csv_node = mxmlNewElement(input_serialization_node, "CSV"); + record_delimiter_node = mxmlNewElement(csv_node, "RecordDelimiter"); + + // encode and add record_delimiter + encoded_len = aos_base64_encode((const unsigned char*)option->input_serialization.csv_format.record_delimiter.data, + option->input_serialization.csv_format.record_delimiter.len, buff); + buff[encoded_len] = '\0'; + mxmlNewText(record_delimiter_node, 0, buff); + // encode and add field_delimiter + encoded_len = aos_base64_encode((const unsigned char*)&(option->input_serialization.csv_format.field_delimiter), + 1, buff); + buff[encoded_len] = '\0'; + field_delimiter_node = mxmlNewElement(csv_node, "FieldDelimiter"); + mxmlNewText(field_delimiter_node, 0, buff); + + // encode and add field_quote + encoded_len = aos_base64_encode((const unsigned char*)&(option->input_serialization.csv_format.field_quote), + 1, buff); + buff[encoded_len] = '\0'; + quote_char_node = mxmlNewElement(csv_node, "QuoteCharacter"); + mxmlNewText(quote_char_node, 0, buff); + + // encode and add comment + encoded_len = aos_base64_encode((const unsigned char*)&(option->input_serialization.csv_format.comment), + 1, buff); + buff[encoded_len] = '\0'; + comment_char_node = mxmlNewElement(csv_node, "Comments"); + mxmlNewText(comment_char_node, 0, buff); + // add range + char* range_str = NULL; + if (option->range_option == LINE) { + range_str = range_to_string("line-range=", option->range[0], option->range[1], buff); + } else if (option->range_option == SPLIT) { + range_str = range_to_string("split-range=", option->range[0], option->range[1], buff); + } + + if (range_str != NULL) { + range_node = mxmlNewElement(csv_node, "Range"); + mxmlNewText(range_node, 0, range_str); + } + + output_serialization_node = mxmlNewElement(root_node, "OutputSerialization"); + //output raw data + output_raw_data_node = mxmlNewElement(output_serialization_node, "OutputRawData"); + mxmlNewText(output_raw_data_node, 0, "true"); + + output_csv_node = mxmlNewElement(output_serialization_node, "CSV"); + output_record_delimiter_node = mxmlNewElement(output_csv_node, "RecordDelimiter"); + + // encode and add record_delimiter + encoded_len = aos_base64_encode((const unsigned char*)option->output_serialization.csv_format.record_delimiter.data, + option->output_serialization.csv_format.record_delimiter.len, buff); + buff[encoded_len] = '\0'; + mxmlNewText(output_record_delimiter_node, 0, buff); + // encode and add field_delimiter + encoded_len = aos_base64_encode((const unsigned char*)&(option->output_serialization.csv_format.field_delimiter), + 1, buff); + buff[encoded_len] = '\0'; + output_field_delimiter_node = mxmlNewElement(output_csv_node, "FieldDelimiter"); + mxmlNewText(output_field_delimiter_node, 0, buff); + + // encode and add field_quote + encoded_len = aos_base64_encode((const unsigned char*)&(option->output_serialization.csv_format.field_quote), + 1, buff); + buff[encoded_len] = '\0'; + output_quote_char_node = mxmlNewElement(output_csv_node, "QuoteCharacter"); + mxmlNewText(output_quote_char_node, 0, buff); + + output_keep_all_columns_node = mxmlNewElement(output_csv_node, "KeepAllColumns"); + mxmlNewText(output_keep_all_columns_node, 0, option->output_serialization.keep_all_columns == 0 ? "false" : "true"); + + xml_buff = new_xml_buff(doc); + if (xml_buff == NULL) { + return NULL; + } + aos_str_set(&xml_doc, xml_buff); + select_object_xml = aos_pstrdup(p, &xml_doc); + + free(xml_buff); + mxmlDelete(doc); + + return select_object_xml; +} + +void build_select_object_metadata_body(aos_pool_t *p, const oss_select_option *option, aos_list_t *body) { + char *select_object_xml; + aos_buf_t *b; + + select_object_xml = build_select_object_metadata_xml(p, option); + aos_list_init(body); + b = aos_buf_pack(p, select_object_xml, strlen(select_object_xml)); + aos_list_add_tail(&b->node, body); +} void oss_live_channel_info_target_content_parse(aos_pool_t *p, mxml_node_t *xml_node, oss_live_channel_target_t *target) { diff --git a/oss_c_sdk/oss_xml.h b/oss_c_sdk/oss_xml.h index 702018a5..d079eaff 100644 --- a/oss_c_sdk/oss_xml.h +++ b/oss_c_sdk/oss_xml.h @@ -136,6 +136,18 @@ int oss_delete_objects_parse_from_body(aos_pool_t *p, aos_list_t *bc, aos_list_t char *build_create_live_channel_xml(aos_pool_t *p, oss_live_channel_configuration_t *config); void build_create_live_channel_body(aos_pool_t *p, oss_live_channel_configuration_t *config, aos_list_t *body); +/** + * @brief build body for create select object metadata + */ +char *build_create_select_object_metadata_xml(aos_pool_t *p, const oss_select_metadata_option *option); +void build_create_select_object_metadata_body(aos_pool_t *p, const oss_select_metadata_option *option, aos_list_t *body); + +/** + * @brief build body for select object + */ +char *build_select_object_metadata_xml(aos_pool_t *p, const oss_select_option *option); +void build_select_object_metadata_body(aos_pool_t *p, const oss_select_option *option, aos_list_t *body); + /** * @brief parse create live channel contents from xml body **/ diff --git a/oss_c_sdk_sample/main.c b/oss_c_sdk_sample/main.c index 9b8fc174..e5a55449 100644 --- a/oss_c_sdk_sample/main.c +++ b/oss_c_sdk_sample/main.c @@ -22,6 +22,7 @@ extern void image_sample(); extern void resumable_upload_sample(); extern void resumable_download_sample(); extern void select_object_to_buffer_sample(); +extern void select_object_to_buffer_big_file_sample(); int main(int argc, char *argv[]) { if (aos_http_io_initialize(NULL, 0) != AOSE_OK) { diff --git a/oss_c_sdk_sample/oss_select_sample.c b/oss_c_sdk_sample/oss_select_sample.c index 6d8776b3..baf86bd7 100644 --- a/oss_c_sdk_sample/oss_select_sample.c +++ b/oss_c_sdk_sample/oss_select_sample.c @@ -1,3 +1,4 @@ +#include #include "aos_log.h" #include "aos_util.h" #include "aos_string.h" @@ -8,7 +9,7 @@ #include "oss_config.h" #include "oss_sample_util.h" -int head_csv_object() +void create_select_object_metadata(int* rows, int* splits) { aos_pool_t *p = NULL; aos_string_t bucket; @@ -19,8 +20,8 @@ int head_csv_object() aos_table_t *resp_headers = NULL; aos_status_t *s = NULL; char *rows_str = NULL; + char *splits_str = NULL; char *object_type = NULL; - int64_t rows = 0; aos_pool_create(&p, NULL); options = oss_request_options_create(p); @@ -32,37 +33,47 @@ int head_csv_object() csv_format_option csv_format; csv_format.field_delimiter = ','; csv_format.field_quote = '"'; - aos_str_set(&(csv_format.new_line), "\r\n"); + aos_str_set(&(csv_format.record_delimiter), "\n"); csv_format.header_info = CSV_HEADER_IGNORE; - s = oss_head_csv_object(options, &bucket, &object, &csv_format, headers, &resp_headers); + select_input_serialization input_serialization; + input_serialization.csv_format = csv_format; + input_serialization.compression_info = NONE; + + oss_select_metadata_option option; + option.overwrite = 1; + option.input_serialization = input_serialization; + s = oss_create_select_object_metadata(options, &bucket, &object, &option, headers, &resp_headers); if (aos_status_is_ok(s)) { rows_str = (char*)apr_table_get(resp_headers, OSS_SELECT_CSV_ROWS); if (rows_str != NULL) { - rows = atol(rows_str); + *rows = atol(rows_str); + } + + splits_str = (char*)apr_table_get(resp_headers, OSS_SELECT_CSV_SPLITS); + if (splits_str != NULL) { + *splits = atol(splits_str); } object_type = (char*)apr_table_get(resp_headers, OSS_OBJECT_TYPE); - printf("csv head object succeeded, object type:%s, csv rows:%ld\n", - object_type, rows); + printf("csv head object succeeded, object type:%s, csv rows:%ld\n, splits:%ld\n", + object_type, *rows, *splits); } else { printf("csv head object failed\n"); } aos_pool_destroy(p); - return rows; } -char* select_object_to_buffer(aos_pool_t *p, const char* sql_str, int start_line, int end_line, int64_t* plen) +char* select_object_to_buffer(aos_pool_t *p, const char* sql_str, int start, int end, select_range_option option, int64_t* plen) { aos_string_t bucket; aos_string_t object; int is_cname = 0; oss_request_options_t *options = NULL; aos_table_t *headers = NULL; - aos_table_t *params = NULL; aos_table_t *resp_headers = NULL; aos_status_t *s = NULL; aos_list_t buffer; @@ -78,21 +89,26 @@ char* select_object_to_buffer(aos_pool_t *p, const char* sql_str, int start_line aos_str_set(&object, OBJECT_NAME); aos_list_init(&buffer); - csv_format_option csv_format; - csv_format.field_delimiter = ','; - csv_format.field_quote = '"'; - aos_str_set(&(csv_format.new_line), "\r\n"); - csv_format.header_info = CSV_HEADER_IGNORE; + select_input_serialization input_serialization; + input_serialization.csv_format = csv_format_option_default; + input_serialization.compression_info = NONE; + + select_output_serialization output_serialization; + output_serialization.csv_format = csv_format_option_default; + output_serialization.keep_all_columns = 1; oss_select_option select_option; - select_option.keep_all_columns = 1; - select_option.raw_output = 1; - select_option.start_line = start_line; - select_option.end_line = end_line; + select_option.input_serialization = input_serialization; + select_option.output_serialization = output_serialization; + select_option.range_option = option; + select_option.range[0] = start; + select_option.range[1] = end; aos_string_t sql; aos_str_set(&sql, sql_str); - s = oss_select_object_to_buffer(options, &bucket, &object, &sql, &csv_format, &select_option, + select_option.expression = sql; + + s = oss_select_object_to_buffer(options, &bucket, &object, &select_option, headers, &buffer, &resp_headers); if (aos_status_is_ok(s)) { @@ -126,9 +142,9 @@ void select_object_to_buffer_sample() int64_t result_len = 0; aos_pool_t *p=NULL; aos_pool_create(&p, NULL); - char *buf = select_object_to_buffer(p, "select count(*) from ossobject where _4 > 60 and _1 like 'Tom%'", -1, -1, &result_len); + char *buf = select_object_to_buffer(p, "select count(*) from ossobject where _4 > 60 and _1 like 'Tom%'", -1, -1, NO_RANGE, &result_len); printf(buf); - buf = select_object_to_buffer(p, "select _1,_4 from ossobject where _4 > 60 and _1 like 'Tom%' limit 100", -1, -1, &result_len); + buf = select_object_to_buffer(p, "select _1,_4 from ossobject where _4 > 60 and _1 like 'Tom%' limit 100", -1, -1, NO_RANGE, &result_len); printf("\n\n%s", buf); aos_pool_destroy(p); } @@ -137,15 +153,25 @@ void select_object_to_buffer_big_file_sample() { aos_pool_t *p=NULL; aos_pool_create(&p, NULL); - int lines = head_csv_object(); - int blockSize = 8; + int splits; + int lines; + create_select_object_metadata(&lines, &splits); + int blockSize = 3; int64_t result_len = 0; + printf("------------line range sample-----------\n"); for(int i = 0; i < blockSize; i++){ int start_line = i * lines/blockSize; int end_line = i == blockSize -1 ? lines - 1 : (i+1) * lines/blockSize - 1; - char *buf = select_object_to_buffer(p, "select count(*) from ossobject where _4 > 60 and _1 like 'Tom%'", start_line, end_line, &result_len); - printf(buf); + char *buf = select_object_to_buffer(p, "select count(*) from ossobject where _4 > 50 and _1 like 'Tom%'", start_line, end_line, LINE, &result_len); + printf("%ld\n", atol(buf)); } + printf("------------split range sample-----------\n"); + for(int i = 0; i < blockSize; i++){ + int start_split = i * splits/blockSize; + int end_split = i == blockSize -1 ? splits - 1 : (i+1) * splits/blockSize - 1; + char *buf = select_object_to_buffer(p, "select count(*) from ossobject where _4 > 50 and _1 like 'Tom%'", start_split, end_split, SPLIT, &result_len); + printf("%ld\n", atol(buf)); + } aos_pool_destroy(p); } \ No newline at end of file