Skip to content

Commit

Permalink
Merge pull request #113 from losalamos/size_ICM
Browse files Browse the repository at this point in the history 
Rob's ICM fix with file merged
  • Loading branch information
@KineticTheory
KineticTheory authored Nov 14, 2016
2 parents 896ecd4 + ee9772f commit ea953c5
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 91 deletions.
56 changes: 29 additions & 27 deletions src/c4/Invert_Comm_Map.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,28 +10,28 @@
//---------------------------------------------------------------------------//

#include "Invert_Comm_Map.hh"
#include "MPI_Traits.hh"
#include "ds++/Assert.hh"
#include <vector>

namespace rtt_c4 {

//---------------------------------------------------------------------------//
// MPI version of invert_comm_map
#ifdef C4_MPI
void invert_comm_map(std::vector<int> const &to_values,
std::vector<int> &from_values) {
void invert_comm_map(Invert_Comm_Map_t const &to_map,
Invert_Comm_Map_t &from_map) {
const int myproc = rtt_c4::node();
const int numprocs = rtt_c4::nodes();

// value to indicate a proc will be communicating with myproc.
int flag = 1;

// The vector that the other procs will set the flag value, if they
// are writing to the current proc.
std::vector<int> proc_flag(numprocs, 0); // initially, all zero
// The local vector that the other procs will set the size they are sending,
// at the index of their processor number. Zero indicates no comm from that
// proc.
std::vector<size_t> proc_flag(numprocs, 0); // initially, all zero

// Create the RMA memory window of the vector.
MPI_Win win;
MPI_Win_create(&proc_flag[0], numprocs * sizeof(int), sizeof(int),
MPI_Win_create(&proc_flag[0], numprocs * sizeof(size_t), sizeof(size_t),
MPI_INFO_NULL, MPI_COMM_WORLD, &win);

// Assertion value for fences. Currently, we effectively don't set
Expand All @@ -40,25 +40,26 @@ void invert_comm_map(std::vector<int> const &to_values,

// Set the local and remote vector values
MPI_Win_fence(fence_assert, win);
for (auto it = to_values.begin(); it != to_values.end(); ++it) {
Require(*it >= 0);
Require(*it < numprocs);
if (*it == myproc) {
for (auto it = to_map.begin(); it != to_map.end(); ++it) {
Require(it->first >= 0);
Require(it->first < numprocs);
Require(it->second > 0);
if (it->first == myproc) {
// ... set our local value
proc_flag[myproc] = 1;
proc_flag[myproc] = it->second;
} else {
// ... set the value on the remote proc
MPI_Put(&flag, 1, MPI_INT, *it, myproc, 1, MPI_INT, win);
MPI_Put(&(it->second), 1, MPI_Traits<size_t>::element_type(), it->first,
myproc, 1, MPI_Traits<size_t>::element_type(), win);
}
}
MPI_Win_fence(fence_assert, win);

// Back out the from_values from the full flags vector
from_values.clear();
// Back out the map from the vector
from_map.clear();
for (int i = 0; i < numprocs; ++i) {
Check(proc_flag[i] == 0 || proc_flag[i] == flag);
if (proc_flag[i] == flag)
from_values.push_back(i);
if (proc_flag[i] > 0)
from_map[i] = proc_flag[i];
}

MPI_Win_free(&win);
Expand All @@ -67,18 +68,19 @@ void invert_comm_map(std::vector<int> const &to_values,
//---------------------------------------------------------------------------//
// SCALAR version of invert_comm_map
#elif defined(C4_SCALAR)
void invert_comm_map(std::vector<int> const &to_values,
std::vector<int> &from_values) {
Require(to_values.size() <= 1);
from_values.clear();
if (to_values.size() > 0 && to_values[0] == 0) {
from_values.push_back(0);
void invert_comm_map(Invert_Comm_Map_t const &to_map,
Invert_Comm_Map_t &from_map) {
Require(to_map.size() == 0u || (to_map.size() == 1u && to_map[0] > 0));
from_map.clear();
auto it = to_map.find(0);
if (it != to_map.end()) {
from_map.push_back(it->second);
}
}
#else
//---------------------------------------------------------------------------//
// Default version of invert_comm_map, which throws an error.
void invert_comm_map(std::vector<int> const &, std::vector<int> &) {
void invert_comm_map(Invert_Comm_Map_t const &, Invert_Comm_Map_t &) {
Insist(0, "invert_comm_map not implemented for this communication type!");
}
#endif // ifdef C4_MPI
Expand Down
27 changes: 15 additions & 12 deletions src/c4/Invert_Comm_Map.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,30 @@
#define c4_Invert_Comm_Map_hh

#include "C4_Functions.hh"
#include <vector>
#include <map>

namespace rtt_c4 {

//! Map type for invert_comm_map
typedef std::map<int, size_t> Invert_Comm_Map_t;

//---------------------------------------------------------------------------//
/**
* \brief Invert the contents of a one-to-many mapping between nodes.
*
* \param[in] to_values A vector of node numbers that this node communicates
* with.
* \param[out] from_values On output, the vector of node numbers that correspond
* to \a to_values.
* \param[in] to_map On input, a map from processor number to the size of
* information to be sent to (or received from) that processor
* by the current processor.
* \param[out] from_map On output, a map from processor number to the size of
* information to be received from (or sent to) that processor by
* the current processor. On input, ignored and deleted.
*
* So if the argument \a to_values contains "send to" node values, then the
* result \a from_values contains the "receive from" node values. But this
* routine can also be used as the argument \a to_values contains "receive from"
* node values, then the result \a from_values contains the "send to" node
* values.
* Here, the units of the "size of information" is up to the caller. For
* example, it might be the number of bytes, or the number of elements in an
* array. The size must be positive (specifically, nonzero).
*/
DLL_PUBLIC_c4 void invert_comm_map(std::vector<int> const &to_values,
std::vector<int> &from_values);
DLL_PUBLIC_c4 void invert_comm_map(Invert_Comm_Map_t const &to_map,
Invert_Comm_Map_t &from_map);

} // end namespace rtt_c4

Expand Down
98 changes: 46 additions & 52 deletions src/c4/test/tstInvert_Comm_Map.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,30 +21,25 @@ using namespace rtt_c4;
//---------------------------------------------------------------------------//
void test2(rtt_c4::ParallelUnitTest &ut) {
size_t const node = rtt_c4::node();
std::vector<int> to_nodes;
Invert_Comm_Map_t to_map;

if (node == 0) {
to_nodes.resize(1);
to_nodes[0] = 1;
}

if (node == 1) {
to_map[1] = 15;
} else if (node != 1) {
// No communication from node 1.
to_nodes.resize(0);
FAILMSG("Incorrect node for test2.");
}

std::vector<int> from_nodes(0);
invert_comm_map(to_nodes, from_nodes);
Invert_Comm_Map_t from_map;
invert_comm_map(to_map, from_map);

if (node == 0) {
if (from_nodes.size() != 0u)
if (from_map.size() != 0u)
FAILMSG("Incorrect map size on node 0.");
}

if (node == 1) {
if (from_nodes.size() != 1u)
} else if (node == 1) {
if (from_map.size() != 1u)
FAILMSG("Incorrect size of map on node 1.");
if (from_nodes[0] != 0)
if (from_map[0] != 15)
FAILMSG("Incorrect map contents on node 1.");
}

Expand All @@ -59,38 +54,36 @@ void test2(rtt_c4::ParallelUnitTest &ut) {
//----------------------------------------------------------------------------//
void test4(rtt_c4::ParallelUnitTest &ut) {
size_t const node = rtt_c4::node();
std::vector<int> to_nodes;
Invert_Comm_Map_t to_map;

if (node == 0) {
to_nodes.push_back(1);
to_nodes.push_back(2);
to_nodes.push_back(3);
}
if (node == 1) {
to_nodes.push_back(0);
}
if (node == 2) {
to_nodes.push_back(0);
}
if (node == 3) {
to_nodes.push_back(0);
}
to_map[1] = 10;
to_map[2] = 20;
to_map[3] = 30;
} else if (node == 1)
to_map[0] = 110;
else if (node == 2)
to_map[0] = 120;
else if (node == 3)
to_map[0] = 130;
else
FAILMSG("Incorrect node for test4.");

std::vector<int> from_nodes(0);
Invert_Comm_Map_t from_map;

invert_comm_map(to_nodes, from_nodes);
invert_comm_map(to_map, from_map);

if (node == 0) {
if (from_nodes.size() != 3u)
if (from_map.size() != 3u)
FAILMSG("Incorrect map size on node 0");
for (int i = 0; i < 3; ++i) {
if (from_nodes[i] != i + 1)
if (from_map[i + 1] != 100u + 10u * (i + 1))
FAILMSG("Incorrent map contents on node 0");
}
} else {
if (from_nodes.size() != 1u)
if (from_map.size() != 1u)
FAILMSG("Incorrect map size.");
if (from_nodes[0] != 0)
if (from_map[0] != 10u * node)
FAILMSG("Incorrect map contents.");
}

Expand All @@ -105,20 +98,21 @@ void test4(rtt_c4::ParallelUnitTest &ut) {
//----------------------------------------------------------------------------//
void test_n_to_n(rtt_c4::ParallelUnitTest &ut) {

const int node = rtt_c4::node();
const int nodes = rtt_c4::nodes();

std::vector<int> to_nodes;
Invert_Comm_Map_t to_map;
for (int i = 0; i < nodes; ++i)
to_nodes.push_back(i);
to_map[i] = 10u * node + 1u;

std::vector<int> from_nodes;
invert_comm_map(to_nodes, from_nodes);
Invert_Comm_Map_t from_map;
invert_comm_map(to_map, from_map);

if (static_cast<int>(from_nodes.size()) != nodes)
FAILMSG("Incorrect from_nodes size.");
if (static_cast<int>(from_map.size()) != nodes)
FAILMSG("Incorrect from_map size.");

for (int i = 0; i < nodes; ++i) {
if (to_nodes[i] != from_nodes[i])
if (from_map[i] != 10u * i + 1u)
FAILMSG("Incorrect data in map.");
}

Expand All @@ -136,15 +130,15 @@ void test_cyclic(rtt_c4::ParallelUnitTest &ut) {
const int node = rtt_c4::node();
const int nodes = rtt_c4::nodes();

std::vector<int> to_nodes(1);
to_nodes[0] = (node + 1) % nodes;
Invert_Comm_Map_t to_map;
to_map[(node + 1) % nodes] = 10u;

std::vector<int> from_nodes;
invert_comm_map(to_nodes, from_nodes);
Invert_Comm_Map_t from_map;
invert_comm_map(to_map, from_map);

if (from_nodes.size() != 1u)
if (from_map.size() != 1u)
FAILMSG("Incorrect map size.");
if (from_nodes[0] != (node + nodes - 1) % nodes)
if (from_map[(node + nodes - 1) % nodes] != 10u)
FAILMSG("Incorrect map contents in cyclc test.");

if (ut.numFails == 0)
Expand All @@ -158,12 +152,12 @@ void test_cyclic(rtt_c4::ParallelUnitTest &ut) {
//----------------------------------------------------------------------------//
void test_empty(rtt_c4::ParallelUnitTest &ut) {

std::vector<int> to_nodes;
std::vector<int> from_nodes;
Invert_Comm_Map_t to_map;
Invert_Comm_Map_t from_map;

invert_comm_map(to_nodes, from_nodes);
invert_comm_map(to_map, from_map);

if (from_nodes.size() != 0u)
if (from_map.size() != 0u)
FAILMSG("Incorrect map size in empty test.");

if (ut.numFails == 0)
Expand Down

0 comments on commit ea953c5

Please sign in to comment.