Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(bulk_load): add remote_file_root for start_bulk_load_request #660

Merged
merged 1 commit into from
Dec 21, 2020
Merged

feat(bulk_load): add remote_file_root for start_bulk_load_request #660

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion rdsn
Open in desktop
Submodule rdsn updated 33 files
+1 −0 bin/dsn.cmake
+2 −1 include/dsn/dist/replication/replication_ddl_client.h
+30 −6 include/dsn/dist/replication/replication_types.h
+0 −4 include/dsn/tool-api/aio_task.h
+2 −0 include/dsn/utility/defer.h
+2 −5 src/aio/aio_provider.cpp
+1 −1 src/aio/aio_provider.h
+2 −9 src/aio/disk_engine.cpp
+1 −1 src/aio/disk_engine.h
+1 −1 src/aio/sim_aio_provider.cpp
+3 −1 src/client/replication_ddl_client.cpp
+0 −5 src/common/replication_common.cpp
+0 −1 src/common/replication_common.h
+72 −0 src/common/replication_types.cpp
+9 −3 src/meta/meta_bulk_load_service.cpp
+15 −6 src/meta/meta_bulk_load_service.h
+35 −0 src/meta/meta_http_service.cpp
+7 −0 src/meta/meta_http_service.h
+4 −1 src/meta/test/meta_bulk_load_service_test.cpp
+82 −0 src/meta/test/meta_http_service_test.cpp
+22 −19 src/replica/bulk_load/replica_bulk_loader.cpp
+9 −12 src/replica/bulk_load/replica_bulk_loader.h
+5 −1 src/replica/bulk_load/test/replica_bulk_loader_test.cpp
+9 −0 src/replica/replica_2pc.cpp
+85 −0 src/replica/split/replica_split_manager.cpp
+13 −0 src/replica/split/replica_split_manager.h
+6 −3 src/replication.thrift
+9 −1 src/runtime/security/client_negotiation.cpp
+5 −1 src/runtime/security/replica_access_controller.cpp
+1 −1 src/runtime/test/replica_access_controller_test.cpp
+0 −4 src/utils/test/time_utils_test.cpp
+14 −0 src/utils/test/utils.cpp
+1 −1 thirdparty/CMakeLists.txt
1 change: 0 additions & 1 deletion src/server/config.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,6 @@
cold_backup_root = %{cluster.name}
max_concurrent_uploading_file_count = 10

bulk_load_provider_root = bulk_load_root
max_concurrent_bulk_load_downloading_count = 5

[pegasus.server]
Expand Down
1 change: 0 additions & 1 deletion src/server/config.min.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,6 @@
duplication_disabled = true
cluster_name = onebox
cold_backup_checkpoint_reserve_minutes = 10
bulk_load_provider_root = bulk_load_root

[meta_server.apps.@APP_NAME@]
app_name = @APP_NAME@
Expand Down
15 changes: 13 additions & 2 deletions src/shell/commands/bulk_load.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,18 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args)
static struct option long_options[] = {{"app_name", required_argument, 0, 'a'},
{"cluster_name", required_argument, 0, 'c'},
{"file_provider_type", required_argument, 0, 'p'},
{"root_path", required_argument, 0, 'r'},
{0, 0, 0, 0}};
std::string app_name;
std::string cluster_name;
std::string file_provider_type;
std::string remote_root_path;

optind = 0;
while (true) {
int option_index = 0;
int c;
c = getopt_long(args.argc, args.argv, "a:c:p:", long_options, &option_index);
c = getopt_long(args.argc, args.argv, "a:c:p:r:", long_options, &option_index);
if (c == -1)
break;
switch (c) {
Expand All @@ -46,6 +48,9 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args)
case 'p':
file_provider_type = optarg;
break;
case 'r':
remote_root_path = optarg;
break;
default:
return false;
}
Expand All @@ -65,7 +70,13 @@ bool start_bulk_load(command_executor *e, shell_context *sc, arguments args)
return false;
}

auto err_resp = sc->ddl_client->start_bulk_load(app_name, cluster_name, file_provider_type);
if (remote_root_path.empty()) {
fprintf(stderr, "remote_root_path should not be empty\n");
return false;
}

auto err_resp = sc->ddl_client->start_bulk_load(
app_name, cluster_name, file_provider_type, remote_root_path);
dsn::error_s err = err_resp.get_error();
std::string hint_msg;
if (err.is_ok()) {
Expand Down
3 changes: 2 additions & 1 deletion src/shell/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,8 @@ static command_executor commands[] = {
{
"start_bulk_load",
"start app bulk load",
"<-a --app_name str> <-c --cluster_name str> <-p --file_provider_type str>",
"<-a --app_name str> <-c --cluster_name str> <-p --file_provider_type str> <-r "
"--root_path>",
start_bulk_load,
},
{
Expand Down
2 changes: 1 addition & 1 deletion src/test/function_test/test_bulk_load.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ class bulk_load_test : public testing::Test

error_code start_bulk_load()
{
auto err_resp = ddl_client->start_bulk_load(APP_NAME, CLUSTER, PROVIDER);
auto err_resp = ddl_client->start_bulk_load(APP_NAME, CLUSTER, PROVIDER, LOCAL_ROOT);
return err_resp.get_value().err;
}

Expand Down