Skip to content

Commit

Permalink
support only load reverse_edge (PaddlePaddle#199)
Browse files Browse the repository at this point in the history 
Co-authored-by: root <[email protected]>
  • Loading branch information
@huwei02
huwei02 and root authored Jan 12, 2023
1 parent 74839ce commit ca67b11
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 19 deletions.
48 changes: 38 additions & 10 deletions paddle/fluid/distributed/ps/table/common_graph_table.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1262,7 +1262,8 @@ int32_t GraphTable::parse_type_to_typepath(
int32_t GraphTable::parse_edge_and_load(std::string etype2files,
std::string graph_data_local_path,
int part_num,
bool reverse) {
bool reverse,
const std::vector<bool>& is_reverse_edge_map) {
std::vector<std::string> etypes;
std::unordered_map<std::string, std::string> edge_to_edgedir;
int res = parse_type_to_typepath(
Expand All @@ -1282,6 +1283,15 @@ int32_t GraphTable::parse_edge_and_load(std::string etype2files,
tasks.push_back(
_shards_task_pool[i % task_pool_size_]->enqueue([&, i, this]() -> int {
std::string etype_path = edge_to_edgedir[etypes[i]];
bool only_load_reverse_edge = false;
if (!reverse) {
only_load_reverse_edge = is_reverse_edge_map[i];
}
if (only_load_reverse_edge) {
VLOG(1) << "only_load_reverse_edge is True, etype[" << etypes[i] << "], file_path[" << etype_path << "]";
} else {
VLOG(1) << "only_load_reverse_edge is False, etype[" << etypes[i] << "], file_path[" << etype_path << "]";
}
auto etype_path_list = paddle::framework::localfs_list(etype_path);
std::string etype_path_str;
if (part_num > 0 && part_num < (int)etype_path_list.size()) {
Expand All @@ -1293,10 +1303,14 @@ int32_t GraphTable::parse_edge_and_load(std::string etype2files,
etype_path_str =
paddle::string::join_strings(etype_path_list, delim);
}
this->load_edges(etype_path_str, false, etypes[i]);
if (reverse) {
std::string r_etype = get_inverse_etype(etypes[i]);
this->load_edges(etype_path_str, true, r_etype);
if (!only_load_reverse_edge) {
this->load_edges(etype_path_str, false, etypes[i]);
if (reverse) {
std::string r_etype = get_inverse_etype(etypes[i]);
this->load_edges(etype_path_str, true, r_etype);
}
} else {
this->load_edges(etype_path_str, true, etypes[i]);
}
return 0;
}));
Expand Down Expand Up @@ -1354,7 +1368,8 @@ int32_t GraphTable::load_node_and_edge_file(std::string etype2files,
std::string ntype2files,
std::string graph_data_local_path,
int part_num,
bool reverse) {
bool reverse,
const std::vector<bool>& is_reverse_edge_map) {
std::vector<std::string> etypes;
std::unordered_map<std::string, std::string> edge_to_edgedir;
int res = parse_type_to_typepath(
Expand Down Expand Up @@ -1384,6 +1399,15 @@ int32_t GraphTable::load_node_and_edge_file(std::string etype2files,
_shards_task_pool[i % task_pool_size_]->enqueue([&, i, this]() -> int {
if (i < etypes.size()) {
std::string etype_path = edge_to_edgedir[etypes[i]];
bool only_load_reverse_edge = false;
if (!reverse) {
only_load_reverse_edge = is_reverse_edge_map[i];
}
if (only_load_reverse_edge) {
VLOG(1) << "only_load_reverse_edge is True, etype[" << etypes[i] << "], file_path[" << etype_path << "]";
} else {
VLOG(1) << "only_load_reverse_edge is False, etype[" << etypes[i] << "], file_path[" << etype_path << "]";
}
auto etype_path_list = paddle::framework::localfs_list(etype_path);
std::string etype_path_str;
if (part_num > 0 && part_num < (int)etype_path_list.size()) {
Expand All @@ -1395,10 +1419,14 @@ int32_t GraphTable::load_node_and_edge_file(std::string etype2files,
etype_path_str =
paddle::string::join_strings(etype_path_list, delim);
}
this->load_edges(etype_path_str, false, etypes[i]);
if (reverse) {
std::string r_etype = get_inverse_etype(etypes[i]);
this->load_edges(etype_path_str, true, r_etype);
if (!only_load_reverse_edge) {
this->load_edges(etype_path_str, false, etypes[i]);
if (reverse) {
std::string r_etype = get_inverse_etype(etypes[i]);
this->load_edges(etype_path_str, true, r_etype);
}
} else {
this->load_edges(etype_path_str, true, etypes[i]);
}
} else {
std::string npath = node_to_nodedir[ntypes[0]];
Expand Down
6 changes: 4 additions & 2 deletions paddle/fluid/distributed/ps/table/common_graph_table.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,11 +562,13 @@ class GraphTable : public Table {
std::string ntype2files,
std::string graph_data_local_path,
int part_num,
bool reverse);
bool reverse,
const std::vector<bool>& is_reverse_edge_map);
int32_t parse_edge_and_load(std::string etype2files,
std::string graph_data_local_path,
int part_num,
bool reverse);
bool reverse,
const std::vector<bool>& is_reverse_edge_map);
int32_t parse_node_and_load(std::string ntype2files,
std::string graph_data_local_path,
int part_num);
Expand Down
11 changes: 7 additions & 4 deletions paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu
View file
Original file line number Diff line number Diff line change
Expand Up @@ -447,10 +447,11 @@ void GraphGpuWrapper::load_edge_file(std::string name,
void GraphGpuWrapper::load_edge_file(std::string etype2files,
std::string graph_data_local_path,
int part_num,
bool reverse) {
bool reverse,
const std::vector<bool>& is_reverse_edge_map) {
((GpuPsGraphTable *)graph_table)
->cpu_graph_table_->parse_edge_and_load(
etype2files, graph_data_local_path, part_num, reverse);
etype2files, graph_data_local_path, part_num, reverse, is_reverse_edge_map);
}

int GraphGpuWrapper::load_node_file(std::string name, std::string filepath) {
Expand All @@ -477,10 +478,12 @@ void GraphGpuWrapper::load_node_and_edge(std::string etype2files,
std::string ntype2files,
std::string graph_data_local_path,
int part_num,
bool reverse) {
bool reverse,
const std::vector<bool>& is_reverse_edge_map) {
((GpuPsGraphTable *)graph_table)
->cpu_graph_table_->load_node_and_edge_file(
etype2files, ntype2files, graph_data_local_path, part_num, reverse);
etype2files, ntype2files, graph_data_local_path, part_num, reverse,
is_reverse_edge_map);
}

void GraphGpuWrapper::add_table_feat_conf(std::string table_name,
Expand Down
6 changes: 4 additions & 2 deletions paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ class GraphGpuWrapper {
void load_edge_file(std::string etype2files,
std::string graph_data_local_path,
int part_num,
bool reverse);
bool reverse,
const std::vector<bool>& is_reverse_edge_map);

int load_node_file(std::string name, std::string filepath);
int load_node_file(std::string ntype2files,
Expand All @@ -77,7 +78,8 @@ class GraphGpuWrapper {
std::string ntype2files,
std::string graph_data_local_path,
int part_num,
bool reverse);
bool reverse,
const std::vector<bool>& is_reverse_edge_map);
int32_t load_next_partition(int idx);
int32_t get_partition_num(int idx);
void load_node_weight(int type_id, int idx, std::string path);
Expand Down
2 changes: 1 addition & 1 deletion paddle/fluid/pybind/fleet_py.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,7 @@ void BindGraphGpuWrapper(py::module* m) {
py::overload_cast<std::string, std::string, bool>(
&GraphGpuWrapper::load_edge_file))
.def("load_edge_file",
py::overload_cast<std::string, std::string, int, bool>(
py::overload_cast<std::string, std::string, int, bool, const std::vector<bool>&>(
&GraphGpuWrapper::load_edge_file))
.def("load_node_and_edge", &GraphGpuWrapper::load_node_and_edge)
.def("upload_batch",
Expand Down

0 comments on commit ca67b11

Please sign in to comment.