Rob's ICM fix with file merged

src/c4/Invert_Comm_Map.cc

@@ -10,28 +10,28 @@
 //---------------------------------------------------------------------------//
 
 #include "Invert_Comm_Map.hh"
+#include "MPI_Traits.hh"
 #include "ds++/Assert.hh"
+#include <vector>
 
 namespace rtt_c4 {
 
 //---------------------------------------------------------------------------//
 // MPI version of invert_comm_map
 #ifdef C4_MPI
-void invert_comm_map(std::vector<int> const &to_values,
-                     std::vector<int> &from_values) {
+void invert_comm_map(Invert_Comm_Map_t const &to_map,
+                     Invert_Comm_Map_t &from_map) {
   const int myproc = rtt_c4::node();
   const int numprocs = rtt_c4::nodes();
 
-  // value to indicate a proc will be communicating with myproc.
-  int flag = 1;
-
-  // The vector that the other procs will set the flag value, if they
-  // are writing to the current proc.
-  std::vector<int> proc_flag(numprocs, 0); // initially, all zero
+  // The local vector that the other procs will set the size they are sending,
+  // at the index of their processor number.  Zero indicates no comm from that
+  // proc.
+  std::vector<size_t> proc_flag(numprocs, 0); // initially, all zero
 
   // Create the RMA memory window of the vector.
   MPI_Win win;
-  MPI_Win_create(&proc_flag[0], numprocs * sizeof(int), sizeof(int),
+  MPI_Win_create(&proc_flag[0], numprocs * sizeof(size_t), sizeof(size_t),
                  MPI_INFO_NULL, MPI_COMM_WORLD, &win);
 
   // Assertion value for fences.  Currently, we effectively don't set
@@ -40,25 +40,26 @@ void invert_comm_map(std::vector<int> const &to_values,
 
   // Set the local and remote vector values
   MPI_Win_fence(fence_assert, win);
-  for (auto it = to_values.begin(); it != to_values.end(); ++it) {
-    Require(*it >= 0);
-    Require(*it < numprocs);
-    if (*it == myproc) {
+  for (auto it = to_map.begin(); it != to_map.end(); ++it) {
+    Require(it->first >= 0);
+    Require(it->first < numprocs);
+    Require(it->second > 0);
+    if (it->first == myproc) {
       // ... set our local value
-      proc_flag[myproc] = 1;
+      proc_flag[myproc] = it->second;
     } else {
       // ... set the value on the remote proc
-      MPI_Put(&flag, 1, MPI_INT, *it, myproc, 1, MPI_INT, win);
+      MPI_Put(&(it->second), 1, MPI_Traits<size_t>::element_type(), it->first,
+              myproc, 1, MPI_Traits<size_t>::element_type(), win);
     }
   }
   MPI_Win_fence(fence_assert, win);
 
-  // Back out the from_values from the full flags vector
-  from_values.clear();
+  // Back out the map from the vector
+  from_map.clear();
   for (int i = 0; i < numprocs; ++i) {
-    Check(proc_flag[i] == 0 || proc_flag[i] == flag);
-    if (proc_flag[i] == flag)
-      from_values.push_back(i);
+    if (proc_flag[i] > 0)
+      from_map[i] = proc_flag[i];
   }
 
   MPI_Win_free(&win);
@@ -67,18 +68,19 @@ void invert_comm_map(std::vector<int> const &to_values,
 //---------------------------------------------------------------------------//
 // SCALAR version of invert_comm_map
 #elif defined(C4_SCALAR)
-void invert_comm_map(std::vector<int> const &to_values,
-                     std::vector<int> &from_values) {
-  Require(to_values.size() <= 1);
-  from_values.clear();
-  if (to_values.size() > 0 && to_values[0] == 0) {
-    from_values.push_back(0);
+void invert_comm_map(Invert_Comm_Map_t const &to_map,
+                     Invert_Comm_Map_t &from_map) {
+  Require(to_map.size() == 0u || (to_map.size() == 1u && to_map[0] > 0));
+  from_map.clear();
+  auto it = to_map.find(0);
+  if (it != to_map.end()) {
+    from_map.push_back(it->second);
   }
 }
 #else
 //---------------------------------------------------------------------------//
 // Default version of invert_comm_map, which throws an error.
-void invert_comm_map(std::vector<int> const &, std::vector<int> &) {
+void invert_comm_map(Invert_Comm_Map_t const &, Invert_Comm_Map_t &) {
   Insist(0, "invert_comm_map not implemented for this communication type!");
 }
 #endif // ifdef C4_MPI

src/c4/Invert_Comm_Map.hh

@@ -13,27 +13,30 @@
 #define c4_Invert_Comm_Map_hh
 
 #include "C4_Functions.hh"
-#include <vector>
+#include <map>
 
 namespace rtt_c4 {
 
+//! Map type for invert_comm_map
+typedef std::map<int, size_t> Invert_Comm_Map_t;
+
 //---------------------------------------------------------------------------//
 /**
  * \brief Invert the contents of a one-to-many mapping between nodes.
  *
- * \param[in]  to_values A vector of node numbers that this node communicates
- *             with.
- * \param[out] from_values On output, the vector of node numbers that correspond
- *             to \a to_values.
+ * \param[in] to_map On input, a map from processor number to the size of
+ *        information to be sent to (or received from) that processor
+ *        by the current processor.
+ * \param[out] from_map On output, a map from processor number to the size of
+ *        information to be received from (or sent to) that processor by
+ *        the current processor.  On input, ignored and deleted.
  *
- * So if the argument \a to_values contains "send to" node values, then the
- * result \a from_values contains the "receive from" node values.  But this
- * routine can also be used as the argument \a to_values contains "receive from"
- * node values, then the result \a from_values contains the "send to" node
- * values.
+ * Here, the units of the "size of information" is up to the caller.  For
+ * example, it might be the number of bytes, or the number of elements in an
+ * array.  The size must be positive (specifically, nonzero).
  */
-DLL_PUBLIC_c4 void invert_comm_map(std::vector<int> const &to_values,
-                                   std::vector<int> &from_values);
+DLL_PUBLIC_c4 void invert_comm_map(Invert_Comm_Map_t const &to_map,
+                                   Invert_Comm_Map_t &from_map);
 
 } // end namespace rtt_c4
 

src/c4/test/tstInvert_Comm_Map.cc

@@ -21,30 +21,25 @@ using namespace rtt_c4;
 //---------------------------------------------------------------------------//
 void test2(rtt_c4::ParallelUnitTest &ut) {
   size_t const node = rtt_c4::node();
-  std::vector<int> to_nodes;
+  Invert_Comm_Map_t to_map;
 
   if (node == 0) {
-    to_nodes.resize(1);
-    to_nodes[0] = 1;
-  }
-
-  if (node == 1) {
+    to_map[1] = 15;
+  } else if (node != 1) {
     // No communication from node 1.
-    to_nodes.resize(0);
+    FAILMSG("Incorrect node for test2.");
   }
 
-  std::vector<int> from_nodes(0);
-  invert_comm_map(to_nodes, from_nodes);
+  Invert_Comm_Map_t from_map;
+  invert_comm_map(to_map, from_map);
 
   if (node == 0) {
-    if (from_nodes.size() != 0u)
+    if (from_map.size() != 0u)
       FAILMSG("Incorrect map size on node 0.");
-  }
-
-  if (node == 1) {
-    if (from_nodes.size() != 1u)
+  } else if (node == 1) {
+    if (from_map.size() != 1u)
       FAILMSG("Incorrect size of map on node 1.");
-    if (from_nodes[0] != 0)
+    if (from_map[0] != 15)
       FAILMSG("Incorrect map contents on node 1.");
   }
 
@@ -59,38 +54,36 @@ void test2(rtt_c4::ParallelUnitTest &ut) {
 //----------------------------------------------------------------------------//
 void test4(rtt_c4::ParallelUnitTest &ut) {
   size_t const node = rtt_c4::node();
-  std::vector<int> to_nodes;
+  Invert_Comm_Map_t to_map;
 
   if (node == 0) {
-    to_nodes.push_back(1);
-    to_nodes.push_back(2);
-    to_nodes.push_back(3);
-  }
-  if (node == 1) {
-    to_nodes.push_back(0);
-  }
-  if (node == 2) {
-    to_nodes.push_back(0);
-  }
-  if (node == 3) {
-    to_nodes.push_back(0);
-  }
+    to_map[1] = 10;
+    to_map[2] = 20;
+    to_map[3] = 30;
+  } else if (node == 1)
+    to_map[0] = 110;
+  else if (node == 2)
+    to_map[0] = 120;
+  else if (node == 3)
+    to_map[0] = 130;
+  else
+    FAILMSG("Incorrect node for test4.");
 
-  std::vector<int> from_nodes(0);
+  Invert_Comm_Map_t from_map;
 
-  invert_comm_map(to_nodes, from_nodes);
+  invert_comm_map(to_map, from_map);
 
   if (node == 0) {
-    if (from_nodes.size() != 3u)
+    if (from_map.size() != 3u)
       FAILMSG("Incorrect map size on node 0");
     for (int i = 0; i < 3; ++i) {
-      if (from_nodes[i] != i + 1)
+      if (from_map[i + 1] != 100u + 10u * (i + 1))
         FAILMSG("Incorrent map contents on node 0");
     }
   } else {
-    if (from_nodes.size() != 1u)
+    if (from_map.size() != 1u)
       FAILMSG("Incorrect map size.");
-    if (from_nodes[0] != 0)
+    if (from_map[0] != 10u * node)
       FAILMSG("Incorrect map contents.");
   }
 
@@ -105,20 +98,21 @@ void test4(rtt_c4::ParallelUnitTest &ut) {
 //----------------------------------------------------------------------------//
 void test_n_to_n(rtt_c4::ParallelUnitTest &ut) {
 
+  const int node = rtt_c4::node();
   const int nodes = rtt_c4::nodes();
 
-  std::vector<int> to_nodes;
+  Invert_Comm_Map_t to_map;
   for (int i = 0; i < nodes; ++i)
-    to_nodes.push_back(i);
+    to_map[i] = 10u * node + 1u;
 
-  std::vector<int> from_nodes;
-  invert_comm_map(to_nodes, from_nodes);
+  Invert_Comm_Map_t from_map;
+  invert_comm_map(to_map, from_map);
 
-  if (static_cast<int>(from_nodes.size()) != nodes)
-    FAILMSG("Incorrect from_nodes size.");
+  if (static_cast<int>(from_map.size()) != nodes)
+    FAILMSG("Incorrect from_map size.");
 
   for (int i = 0; i < nodes; ++i) {
-    if (to_nodes[i] != from_nodes[i])
+    if (from_map[i] != 10u * i + 1u)
       FAILMSG("Incorrect data in map.");
   }
 
@@ -136,15 +130,15 @@ void test_cyclic(rtt_c4::ParallelUnitTest &ut) {
   const int node = rtt_c4::node();
   const int nodes = rtt_c4::nodes();
 
-  std::vector<int> to_nodes(1);
-  to_nodes[0] = (node + 1) % nodes;
+  Invert_Comm_Map_t to_map;
+  to_map[(node + 1) % nodes] = 10u;
 
-  std::vector<int> from_nodes;
-  invert_comm_map(to_nodes, from_nodes);
+  Invert_Comm_Map_t from_map;
+  invert_comm_map(to_map, from_map);
 
-  if (from_nodes.size() != 1u)
+  if (from_map.size() != 1u)
     FAILMSG("Incorrect map size.");
-  if (from_nodes[0] != (node + nodes - 1) % nodes)
+  if (from_map[(node + nodes - 1) % nodes] != 10u)
     FAILMSG("Incorrect map contents in cyclc test.");
 
   if (ut.numFails == 0)
@@ -158,12 +152,12 @@ void test_cyclic(rtt_c4::ParallelUnitTest &ut) {
 //----------------------------------------------------------------------------//
 void test_empty(rtt_c4::ParallelUnitTest &ut) {
 
-  std::vector<int> to_nodes;
-  std::vector<int> from_nodes;
+  Invert_Comm_Map_t to_map;
+  Invert_Comm_Map_t from_map;
 
-  invert_comm_map(to_nodes, from_nodes);
+  invert_comm_map(to_map, from_map);
 
-  if (from_nodes.size() != 0u)
+  if (from_map.size() != 0u)
     FAILMSG("Incorrect map size in empty test.");
 
   if (ut.numFails == 0)