From 11db99ba28043598b72041fcbdc785932e937ca7 Mon Sep 17 00:00:00 2001 From: heyuchen Date: Thu, 17 Dec 2020 15:29:24 +0800 Subject: [PATCH] feat(bulk_load): add remote_file_root for start_bulk_load_request --- rdsn | 2 +- src/server/config.ini | 1 - src/server/config.min.ini | 1 - src/shell/commands/bulk_load.cpp | 15 +++++++++++++-- src/shell/main.cpp | 3 ++- src/test/function_test/test_bulk_load.cpp | 2 +- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/rdsn b/rdsn index 65edf61e37..67e06d2c50 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit 65edf61e376b692f41845ab3deb71cb686ea5575 +Subproject commit 67e06d2c5022e82984b0c7e6651876b5b0e7410d diff --git a/src/server/config.ini b/src/server/config.ini index 02813ae4e4..eb6e56020e 100644 --- a/src/server/config.ini +++ b/src/server/config.ini @@ -266,7 +266,6 @@ cold_backup_root = %{cluster.name} max_concurrent_uploading_file_count = 10 - bulk_load_provider_root = bulk_load_root max_concurrent_bulk_load_downloading_count = 5 [pegasus.server] diff --git a/src/server/config.min.ini b/src/server/config.min.ini index a12fe8d0af..0094879713 100644 --- a/src/server/config.min.ini +++ b/src/server/config.min.ini @@ -117,7 +117,6 @@ duplication_disabled = true cluster_name = onebox cold_backup_checkpoint_reserve_minutes = 10 - bulk_load_provider_root = bulk_load_root [meta_server.apps.@APP_NAME@] app_name = @APP_NAME@ diff --git a/src/shell/commands/bulk_load.cpp b/src/shell/commands/bulk_load.cpp index 19a0c357a9..dd8fc930e4 100644 --- a/src/shell/commands/bulk_load.cpp +++ b/src/shell/commands/bulk_load.cpp @@ -24,16 +24,18 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args) static struct option long_options[] = {{"app_name", required_argument, 0, 'a'}, {"cluster_name", required_argument, 0, 'c'}, {"file_provider_type", required_argument, 0, 'p'}, + {"root_path", required_argument, 0, 'r'}, {0, 0, 0, 0}}; std::string app_name; std::string cluster_name; std::string file_provider_type; + std::string remote_root_path; optind = 0; while (true) { int option_index = 0; int c; - c = getopt_long(args.argc, args.argv, "a:c:p:", long_options, &option_index); + c = getopt_long(args.argc, args.argv, "a:c:p:r:", long_options, &option_index); if (c == -1) break; switch (c) { @@ -46,6 +48,9 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args) case 'p': file_provider_type = optarg; break; + case 'r': + remote_root_path = optarg; + break; default: return false; } @@ -65,7 +70,13 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args) return false; } - auto err_resp = sc->ddl_client->start_bulk_load(app_name, cluster_name, file_provider_type); + if (remote_root_path.empty()) { + fprintf(stderr, "remote_root_path should not be empty\n"); + return false; + } + + auto err_resp = sc->ddl_client->start_bulk_load( + app_name, cluster_name, file_provider_type, remote_root_path); dsn::error_s err = err_resp.get_error(); std::string hint_msg; if (err.is_ok()) { diff --git a/src/shell/main.cpp b/src/shell/main.cpp index 76610cb1e5..9a02ab33aa 100644 --- a/src/shell/main.cpp +++ b/src/shell/main.cpp @@ -460,7 +460,8 @@ static command_executor commands[] = { { "start_bulk_load", "start app bulk load", - "<-a --app_name str> <-c --cluster_name str> <-p --file_provider_type str>", + "<-a --app_name str> <-c --cluster_name str> <-p --file_provider_type str> <-r " + "--root_path>", start_bulk_load, }, { diff --git a/src/test/function_test/test_bulk_load.cpp b/src/test/function_test/test_bulk_load.cpp index 963d419e20..7f5c11b26e 100644 --- a/src/test/function_test/test_bulk_load.cpp +++ b/src/test/function_test/test_bulk_load.cpp @@ -103,7 +103,7 @@ class bulk_load_test : public testing::Test error_code start_bulk_load() { - auto err_resp = ddl_client->start_bulk_load(APP_NAME, CLUSTER, PROVIDER); + auto err_resp = ddl_client->start_bulk_load(APP_NAME, CLUSTER, PROVIDER, LOCAL_ROOT); return err_resp.get_value().err; }