Skip to content

Commit

Permalink
core: Reduce forces from ghosts as doubles.
Browse files Browse the repository at this point in the history
  • Loading branch information
@fweik
fweik committed Mar 29, 2018
1 parent a69f018 commit a538ebb
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 70 deletions.
129 changes: 64 additions & 65 deletions src/core/ghosts.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,6 @@ static char *r_buffer = nullptr;

std::vector<int> r_bondbuffer;

static MPI_Op MPI_FORCES_SUM;

/** whether the ghosts should also have velocity information, e. g. for DPD or RATTLE.
You need this whenever you need the relative velocity of two particles.
NO CHANGES OF THIS VALUE OUTSIDE OF \ref on_ghost_flags_change !!!!
Expand Down Expand Up @@ -142,9 +140,9 @@ int calc_transmit_size(GhostCommunication *gc, int data_parts)
return n_buffer_new;
}

void prepare_send_buffer(GhostCommunication *gc, int data_parts)
{
GHOST_TRACE(fprintf(stderr, "%d: prepare sending to/bcast from %d\n", this_node, gc->node));
void prepare_send_buffer(GhostCommunication *gc, int data_parts) {
GHOST_TRACE(fprintf(stderr, "%d: prepare sending to/bcast from %d\n",
this_node, gc->node));

/* reallocate send buffer */
n_s_buffer = calc_transmit_size(gc, data_parts);
Expand All @@ -159,84 +157,84 @@ void prepare_send_buffer(GhostCommunication *gc, int data_parts)
/* put in data */
char *insert = s_buffer;
for (int pl = 0; pl < gc->n_part_lists; pl++) {
int np = gc->part_lists[pl]->n;
int np = gc->part_lists[pl]->n;
if (data_parts & GHOSTTRANS_PARTNUM) {
*(int *)insert = np;
insert += sizeof(int);
GHOST_TRACE(fprintf(stderr, "%d: %d particles assigned\n",
this_node, np));
}
else {
GHOST_TRACE(
fprintf(stderr, "%d: %d particles assigned\n", this_node, np));
} else {
Particle *part = gc->part_lists[pl]->part;
for (int p = 0; p < np; p++) {
Particle *pt = &part[p];
if (data_parts & GHOSTTRANS_PROPRTS) {
memmove(insert, &pt->p, sizeof(ParticleProperties));
insert += sizeof(ParticleProperties);
Particle *pt = &part[p];
if (data_parts & GHOSTTRANS_PROPRTS) {
memmove(insert, &pt->p, sizeof(ParticleProperties));
insert += sizeof(ParticleProperties);
#ifdef GHOSTS_HAVE_BONDS
*(int *)insert = pt->bl.n;
insert += sizeof(int);
insert += sizeof(int);
if (pt->bl.n) {
s_bondbuffer.insert(s_bondbuffer.end(), pt->bl.e, pt->bl.e + pt->bl.n);
s_bondbuffer.insert(s_bondbuffer.end(), pt->bl.e,
pt->bl.e + pt->bl.n);
}
#ifdef EXCLUSIONS
*(int *)insert = pt->el.n;
insert += sizeof(int);
insert += sizeof(int);
if (pt->el.n) {
s_bondbuffer.insert(s_bondbuffer.end(), pt->el.e, pt->el.e + pt->el.n);
s_bondbuffer.insert(s_bondbuffer.end(), pt->el.e,
pt->el.e + pt->el.n);
}
#endif
#endif
}
if (data_parts & GHOSTTRANS_POSSHFTD) {
/* ok, this is not nice, but perhaps fast */
ParticlePosition *pp = (ParticlePosition *)insert;
int i;
memmove(pp, &pt->r, sizeof(ParticlePosition));
for (i = 0; i < 3; i++)
pp->p[i] += gc->shift[i];
/* No special wrapping for Lees-Edwards here:
* LE wrap-on-receive instead, for convenience in
* mapping to local cell geometry. */
insert += sizeof(ParticlePosition);
}
else if (data_parts & GHOSTTRANS_POSITION) {
memmove(insert, &pt->r, sizeof(ParticlePosition));
insert += sizeof(ParticlePosition);
}
if (data_parts & GHOSTTRANS_MOMENTUM) {
memmove(insert, &pt->m, sizeof(ParticleMomentum));
insert += sizeof(ParticleMomentum);
}
if (data_parts & GHOSTTRANS_FORCE) {
memmove(insert, &pt->f, sizeof(ParticleForce));
insert += sizeof(ParticleForce);
}
}
if (data_parts & GHOSTTRANS_POSSHFTD) {
/* ok, this is not nice, but perhaps fast */
ParticlePosition *pp = (ParticlePosition *)insert;
int i;
memmove(pp, &pt->r, sizeof(ParticlePosition));
for (i = 0; i < 3; i++)
pp->p[i] += gc->shift[i];
/* No special wrapping for Lees-Edwards here:
* LE wrap-on-receive instead, for convenience in
* mapping to local cell geometry. */
insert += sizeof(ParticlePosition);
} else if (data_parts & GHOSTTRANS_POSITION) {
memmove(insert, &pt->r, sizeof(ParticlePosition));
insert += sizeof(ParticlePosition);
}
if (data_parts & GHOSTTRANS_MOMENTUM) {
memmove(insert, &pt->m, sizeof(ParticleMomentum));
insert += sizeof(ParticleMomentum);
}
if (data_parts & GHOSTTRANS_FORCE) {
memmove(insert, &pt->f, sizeof(ParticleForce));
insert += sizeof(ParticleForce);
}
#ifdef LB
if (data_parts & GHOSTTRANS_COUPLING) {
memmove(insert, &pt->lc, sizeof(ParticleLatticeCoupling));
insert += sizeof(ParticleLatticeCoupling);
}
if (data_parts & GHOSTTRANS_COUPLING) {
memmove(insert, &pt->lc, sizeof(ParticleLatticeCoupling));
insert += sizeof(ParticleLatticeCoupling);
}
#endif
#ifdef ENGINE
if (data_parts & GHOSTTRANS_SWIMMING) {
if (data_parts & GHOSTTRANS_SWIMMING) {
memmove(insert, &pt->swim, sizeof(ParticleParametersSwimming));
insert += sizeof(ParticleParametersSwimming);
insert += sizeof(ParticleParametersSwimming);
}
#endif
}
}
}
if (data_parts & GHOSTTRANS_PROPRTS) {
GHOST_TRACE(fprintf(stderr, "%d: bond buffer size is %ld\n",
this_node, s_bondbuffer.size()));
GHOST_TRACE(fprintf(stderr, "%d: bond buffer size is %ld\n", this_node,
s_bondbuffer.size()));
*(int *)insert = int(s_bondbuffer.size());
insert += sizeof(int);
}

if (insert - s_buffer != n_s_buffer) {
fprintf(stderr, "%d: INTERNAL ERROR: send buffer size %d "
"differs from what I put in (%ld)\n",
"differs from what I put in (%ld)\n",
this_node, n_s_buffer, insert - s_buffer);
errexit();
}
Expand Down Expand Up @@ -643,15 +641,21 @@ void ghost_communicator(GhostCommunicator *gc)
}
}
break;
case GHOST_RDCE:
GHOST_TRACE(fprintf(stderr, "%d: ghost_comm reduce to %d (%d bytes)\n", this_node, node, n_s_buffer));
if (node == this_node)
MPI_Reduce(s_buffer, r_buffer, n_s_buffer, MPI_BYTE, MPI_FORCES_SUM, node, comm_cart);
else
MPI_Reduce(s_buffer, nullptr, n_s_buffer, MPI_BYTE, MPI_FORCES_SUM, node, comm_cart);
break;
case GHOST_RDCE: {
GHOST_TRACE(fprintf(stderr, "%d: ghost_comm reduce to %d (%d bytes)\n",
this_node, node, n_s_buffer));

if (node == this_node)
MPI_Reduce(reinterpret_cast<double *>(s_buffer),
reinterpret_cast<double *>(r_buffer),
n_s_buffer / sizeof(double), MPI_DOUBLE, MPI_SUM, node,
comm_cart);
else
MPI_Reduce(reinterpret_cast<double *>(s_buffer), nullptr,
n_s_buffer / sizeof(double), MPI_DOUBLE, MPI_SUM, node,
comm_cart);
} break;
}
//GHOST_TRACE(MPI_Barrier(comm_cart));
GHOST_TRACE(fprintf(stderr, "%d: ghost_comm done\n", this_node));

/* recv op; write back data directly, if no PSTSTORE delay is requested. */
Expand Down Expand Up @@ -699,11 +703,6 @@ void ghost_communicator(GhostCommunicator *gc)
}
}

void ghost_init()
{
MPI_Op_create(reduce_forces_sum, 1, &MPI_FORCES_SUM);
}

/** Go through \ref ghost_cells and remove the ghost entries from \ref
local_particles. Part of \ref dd_exchange_and_sort_particles.*/
void invalidate_ghosts()
Expand Down
3 changes: 0 additions & 3 deletions src/core/ghosts.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,6 @@ void prepare_comm(GhostCommunicator *comm, int data_parts, int num);
/** Free a communicator. */
void free_comm(GhostCommunicator *comm);

/** Initialize ghosts. */
void ghost_init();

/** do a ghost communication */
void ghost_communicator(GhostCommunicator *gc);

Expand Down
2 changes: 0 additions & 2 deletions src/core/initialize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,6 @@ void on_program_start() {
/* initially go for domain decomposition */
topology_init(CELL_STRUCTURE_DOMDEC, &local_cells);

ghost_init();

#ifdef P3M
p3m_pre_init();
#endif
Expand Down

0 comments on commit a538ebb

Please sign in to comment.