Merge commits from Jun 6-15

.github/ops-bot.yaml

@@ -5,4 +5,5 @@ auto_merger: true
 branch_checker: true
 label_checker: true
 release_drafter: true
-external_contributors: false
+copy_prs: true
+rerun_tests: true

conda/recipes/cugraph/meta.yaml

@@ -49,8 +49,8 @@ requirements:
     - cudf={{ minor_version }}
     - dask-cudf {{ minor_version }}
     - dask-cuda {{ minor_version }}
-    - dask==2022.05.2
-    - distributed==2022.05.2
+    - dask>=2022.05.2
+    - distributed>=2022.05.2
     - ucx-py {{ ucx_py_version }}
     - ucx-proc=*=gpu
     - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}

cpp/CMakeLists.txt

@@ -170,7 +170,6 @@ add_library(cugraph
     src/structure/legacy/graph.cu
     src/linear_assignment/hungarian.cu
     src/traversal/legacy/bfs.cu
-    src/traversal/legacy/sssp.cu
     src/link_prediction/jaccard.cu
     src/link_prediction/overlap.cu
     src/layout/force_atlas2.cu
@@ -239,11 +238,6 @@ add_library(cugraph
     src/structure/create_graph_from_edgelist_mg.cu
     src/structure/symmetrize_edgelist_sg.cu
     src/structure/symmetrize_edgelist_mg.cu
-    src/visitors/graph_envelope.cpp
-    src/visitors/visitors_factory.cpp
-    src/visitors/bfs_visitor.cpp
-    src/visitors/rw_visitor.cpp
-    src/visitors/graph_make_visitor.cpp
     src/community/triangle_count_sg.cu
     src/community/triangle_count_mg.cu
 )
@@ -360,7 +354,6 @@ endif()
 # - C-API library --------------------------------------------------------------
 
 add_library(cugraph_c
-        src/c_api/cugraph_api.cpp
         src/c_api/resource_handle.cpp
         src/c_api/array.cpp
         src/c_api/error.cpp

cpp/include/cugraph/algorithms.hpp

@@ -428,36 +428,6 @@ template <typename VT, typename ET, typename WT>
 std::unique_ptr<legacy::GraphCOO<VT, ET, WT>> get_two_hop_neighbors(
   legacy::GraphCSRView<VT, ET, WT> const& graph);
 
-/**
- * @Synopsis   Performs a single source shortest path traversal of a graph starting from a vertex.
- *
- * @throws     cugraph::logic_error with a custom message when an error occurs.
- *
- * @tparam VT                        Type of vertex identifiers. Supported value : int (signed,
- * 32-bit)
- * @tparam ET                        Type of edge identifiers.  Supported value : int (signed,
- * 32-bit)
- * @tparam WT                        Type of edge weights. Supported values : float or double.
- *
- * @param[in] graph                  cuGraph graph descriptor, should contain the connectivity
- * information as a CSR
- *
- * @param[out] distances            If set to a valid pointer, array of size V populated by distance
- * of every vertex in the graph from the starting vertex. Memory is provided and owned by the
- * caller.
- *
- * @param[out] predecessors         If set to a valid pointer, array of size V populated by the SSSP
- * predecessor of every vertex. Memory is provided and owned by the caller.
- *
- * @param[in] start_vertex           The starting vertex for SSSP
- *
- */
-template <typename VT, typename ET, typename WT>
-void sssp(legacy::GraphCSRView<VT, ET, WT> const& graph,
-          WT* distances,
-          VT* predecessors,
-          const VT source_vertex);
-
 // FIXME: Internally distances is of int (signed 32-bit) data type, but current
 // template uses data from VT, ET, WT from the legacy::GraphCSR View even if weights
 // are not considered

cpp/include/cugraph/graph.hpp

@@ -19,6 +19,7 @@
 #include <cugraph/utilities/error.hpp>
 
 #include <raft/handle.hpp>
+#include <raft/span.hpp>
 #include <rmm/device_uvector.hpp>
 
 #include <cstddef>
@@ -181,36 +182,111 @@ class graph_t<vertex_t, edge_t, weight_t, store_transposed, multi_gpu, std::enab
       }
     }
 
+    std::conditional_t<store_transposed,
+                       std::optional<raft::device_span<vertex_t const>>,
+                       std::optional<std::vector<raft::device_span<vertex_t const>>>>
+      local_sorted_unique_edge_srcs{std::nullopt};
+    std::conditional_t<store_transposed,
+                       std::optional<raft::device_span<vertex_t const>>,
+                       std::optional<std::vector<raft::device_span<vertex_t const>>>>
+      local_sorted_unique_edge_src_chunk_start_offsets{std::nullopt};
+    std::conditional_t<store_transposed,
+                       std::optional<raft::host_span<vertex_t const>>,
+                       std::optional<std::byte>>
+      local_sorted_unique_edge_src_vertex_partition_offsets{std::nullopt};
+
+    std::conditional_t<store_transposed,
+                       std::optional<std::vector<raft::device_span<vertex_t const>>>,
+                       std::optional<raft::device_span<vertex_t const>>>
+      local_sorted_unique_edge_dsts{std::nullopt};
+    std::conditional_t<store_transposed,
+                       std::optional<std::vector<raft::device_span<vertex_t const>>>,
+                       std::optional<raft::device_span<vertex_t const>>>
+      local_sorted_unique_edge_dst_chunk_start_offsets{std::nullopt};
+    std::conditional_t<!store_transposed,
+                       std::optional<raft::host_span<vertex_t const>>,
+                       std::optional<std::byte>>
+      local_sorted_unique_edge_dst_vertex_partition_offsets{std::nullopt};
+
+    if (local_sorted_unique_edge_srcs_) {
+      if constexpr (store_transposed) {  // minor
+        local_sorted_unique_edge_srcs = raft::device_span<vertex_t const>(
+          (*local_sorted_unique_edge_srcs_).begin(), (*local_sorted_unique_edge_srcs_).end());
+        local_sorted_unique_edge_src_chunk_start_offsets = raft::device_span<vertex_t const>(
+          (*local_sorted_unique_edge_src_chunk_start_offsets_).begin(),
+          (*local_sorted_unique_edge_src_chunk_start_offsets_).end());
+        local_sorted_unique_edge_src_vertex_partition_offsets = raft::host_span<vertex_t const>(
+          (*local_sorted_unique_edge_src_vertex_partition_offsets_).data(),
+          (*local_sorted_unique_edge_src_vertex_partition_offsets_).data() +
+            (*local_sorted_unique_edge_src_vertex_partition_offsets_).size());
+      } else {  // major
+        local_sorted_unique_edge_srcs =
+          std::vector<raft::device_span<vertex_t const>>((*local_sorted_unique_edge_srcs_).size());
+        local_sorted_unique_edge_src_chunk_start_offsets =
+          std::vector<raft::device_span<vertex_t const>>(
+            (*local_sorted_unique_edge_src_chunk_start_offsets_).size());
+        for (size_t i = 0; (*local_sorted_unique_edge_srcs).size(); ++i) {
+          (*local_sorted_unique_edge_srcs)[i] =
+            raft::device_span<vertex_t const>((*local_sorted_unique_edge_srcs_)[i].begin(),
+                                              (*local_sorted_unique_edge_srcs_)[i].end());
+          (*local_sorted_unique_edge_src_chunk_start_offsets)[i] =
+            raft::device_span<vertex_t const>(
+              (*local_sorted_unique_edge_src_chunk_start_offsets_)[i].begin(),
+              (*local_sorted_unique_edge_src_chunk_start_offsets_)[i].end());
+        }
+      }
+    }
+
+    if (local_sorted_unique_edge_dsts_) {
+      if constexpr (store_transposed) {  // major
+        local_sorted_unique_edge_dsts =
+          std::vector<raft::device_span<vertex_t const>>((*local_sorted_unique_edge_dsts_).size());
+        local_sorted_unique_edge_dst_chunk_start_offsets =
+          std::vector<raft::device_span<vertex_t const>>(
+            (*local_sorted_unique_edge_dst_chunk_start_offsets_).size());
+        for (size_t i = 0; (*local_sorted_unique_edge_dsts).size(); ++i) {
+          (*local_sorted_unique_edge_dsts)[i] =
+            raft::device_span<vertex_t const>((*local_sorted_unique_edge_dsts_)[i].begin(),
+                                              (*local_sorted_unique_edge_dsts_)[i].end());
+          (*local_sorted_unique_edge_dst_chunk_start_offsets)[i] =
+            raft::device_span<vertex_t const>(
+              (*local_sorted_unique_edge_dst_chunk_start_offsets_)[i].begin(),
+              (*local_sorted_unique_edge_dst_chunk_start_offsets_)[i].end());
+        }
+      } else {  // minor
+        local_sorted_unique_edge_dsts = raft::device_span<vertex_t const>(
+          (*local_sorted_unique_edge_dsts_).begin(), (*local_sorted_unique_edge_dsts_).end());
+        local_sorted_unique_edge_dst_chunk_start_offsets = raft::device_span<vertex_t const>(
+          (*local_sorted_unique_edge_dst_chunk_start_offsets_).begin(),
+          (*local_sorted_unique_edge_dst_chunk_start_offsets_).end());
+        local_sorted_unique_edge_dst_vertex_partition_offsets = raft::host_span<vertex_t const>(
+          (*local_sorted_unique_edge_dst_vertex_partition_offsets_).data(),
+          (*local_sorted_unique_edge_dst_vertex_partition_offsets_).data() +
+            (*local_sorted_unique_edge_dst_vertex_partition_offsets_).size());
+      }
+    }
+
     return graph_view_t<vertex_t, edge_t, weight_t, store_transposed, multi_gpu>(
       *(this->handle_ptr()),
       offsets,
       indices,
       weights,
       dcs_nzd_vertices,
       dcs_nzd_vertex_counts,
-      graph_view_meta_t<vertex_t, edge_t, multi_gpu>{
+      graph_view_meta_t<vertex_t, edge_t, store_transposed, multi_gpu>{
         this->number_of_vertices(),
         this->number_of_edges(),
         this->graph_properties(),
         partition_,
         edge_partition_segment_offsets_,
-        local_sorted_unique_edge_srcs_
-          ? std::optional<vertex_t const*>{(*local_sorted_unique_edge_srcs_).data()}
-          : std::nullopt,
-        local_sorted_unique_edge_srcs_
-          ? std::optional<vertex_t const*>{(*local_sorted_unique_edge_srcs_).data() +
-                                           (*local_sorted_unique_edge_srcs_).size()}
-          : std::nullopt,
-        local_sorted_unique_edge_src_offsets_,
-        local_sorted_unique_edge_dsts_
-          ? std::optional<vertex_t const*>{(*local_sorted_unique_edge_dsts_).data()}
-          : std::nullopt,
-        local_sorted_unique_edge_dsts_
-          ? std::optional<vertex_t const*>{(*local_sorted_unique_edge_dsts_).data() +
-                                           (*local_sorted_unique_edge_dsts_).size()}
-          : std::nullopt,
-        local_sorted_unique_edge_dst_offsets_,
-      });
+        local_sorted_unique_edge_srcs,
+        local_sorted_unique_edge_src_chunk_start_offsets,
+        local_sorted_unique_edge_src_chunk_size_,
+        local_sorted_unique_edge_src_vertex_partition_offsets,
+        local_sorted_unique_edge_dsts,
+        local_sorted_unique_edge_dst_chunk_start_offsets,
+        local_sorted_unique_edge_dst_chunk_size_,
+        local_sorted_unique_edge_dst_vertex_partition_offsets});
   }
 
   std::tuple<rmm::device_uvector<vertex_t>,
@@ -237,10 +313,34 @@ class graph_t<vertex_t, edge_t, weight_t, store_transposed, multi_gpu, std::enab
 
   // if valid, store row/column properties in key/value pairs (this saves memory if # unique edge
   // rows/cols << V / row_comm_size|col_comm_size).
-  std::optional<rmm::device_uvector<vertex_t>> local_sorted_unique_edge_srcs_{std::nullopt};
-  std::optional<rmm::device_uvector<vertex_t>> local_sorted_unique_edge_dsts_{std::nullopt};
-  std::optional<std::vector<vertex_t>> local_sorted_unique_edge_src_offsets_{std::nullopt};
-  std::optional<std::vector<vertex_t>> local_sorted_unique_edge_dst_offsets_{std::nullopt};
+
+  std::conditional_t<store_transposed,
+                     std::optional<rmm::device_uvector<vertex_t>>,
+                     std::optional<std::vector<rmm::device_uvector<vertex_t>>>>
+    local_sorted_unique_edge_srcs_{std::nullopt};
+  std::conditional_t<store_transposed,
+                     std::optional<rmm::device_uvector<vertex_t>>,
+                     std::optional<std::vector<rmm::device_uvector<vertex_t>>>>
+    local_sorted_unique_edge_src_chunk_start_offsets_{std::nullopt};
+  std::optional<vertex_t> local_sorted_unique_edge_src_chunk_size_{std::nullopt};
+  std::conditional_t<store_transposed,
+                     std::optional<std::vector<vertex_t>>,
+                     std::optional<std::byte> /* dummy */>
+    local_sorted_unique_edge_src_vertex_partition_offsets_{std::nullopt};
+
+  std::conditional_t<store_transposed,
+                     std::optional<std::vector<rmm::device_uvector<vertex_t>>>,
+                     std::optional<rmm::device_uvector<vertex_t>>>
+    local_sorted_unique_edge_dsts_{std::nullopt};
+  std::conditional_t<store_transposed,
+                     std::optional<std::vector<rmm::device_uvector<vertex_t>>>,
+                     std::optional<rmm::device_uvector<vertex_t>>>
+    local_sorted_unique_edge_dst_chunk_start_offsets_{std::nullopt};
+  std::optional<vertex_t> local_sorted_unique_edge_dst_chunk_size_{std::nullopt};
+  std::conditional_t<!store_transposed,
+                     std::optional<std::vector<vertex_t>>,
+                     std::optional<std::byte> /* dummy */>
+    local_sorted_unique_edge_dst_vertex_partition_offsets_{std::nullopt};
 };
 
 // single-GPU version
@@ -339,10 +439,10 @@ class graph_t<vertex_t, edge_t, weight_t, store_transposed, multi_gpu, std::enab
       offsets_.data(),
       indices_.data(),
       weights_ ? std::optional<weight_t const*>{(*weights_).data()} : std::nullopt,
-      graph_view_meta_t<vertex_t, edge_t, multi_gpu>{this->number_of_vertices(),
-                                                     this->number_of_edges(),
-                                                     this->graph_properties(),
-                                                     segment_offsets_});
+      graph_view_meta_t<vertex_t, edge_t, store_transposed, multi_gpu>{this->number_of_vertices(),
+                                                                       this->number_of_edges(),
+                                                                       this->graph_properties(),
+                                                                       segment_offsets_});
   }
 
   // FIXME: possibley to be added later;