Skip to content

Commit

Permalink
some more added features, homogenized stuff and so on.
Browse files Browse the repository at this point in the history
  • Loading branch information
Stefan Kesselheim authored and Stefan Kesselheim committed Jul 19, 2010
1 parent 42dffbc commit 7b1a2b9
Show file tree
Hide file tree
Showing 15 changed files with 501 additions and 134 deletions.
8 changes: 4 additions & 4 deletions communication.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2483,7 +2483,7 @@ void mpi_send_exclusion_slave(int part1, int part2)
void mpi_send_fluid(int node, int index, double rho, double *j, double *pi) {
#ifdef LB
if (node==this_node) {
lb_set_local_fields(index, rho, j, pi);
lb_calc_n_equilibrium(index, rho, j, pi);
} else {
double data[10] = { rho, j[0], j[1], j[2], pi[0], pi[1], pi[2], pi[3], pi[4], pi[5] };
mpi_issue(REQ_SET_FLUID, node, index);
Expand All @@ -2498,7 +2498,7 @@ void mpi_send_fluid_slave(int node, int index) {
double data[10];
MPI_Status status;
MPI_Recv(data, 10, MPI_DOUBLE, 0, REQ_SET_FLUID, MPI_COMM_WORLD, &status);
lb_set_local_fields(index, data[0], &data[1], &data[4]);
lb_calc_n_equilibrium(index, data[0], &data[1], &data[4]);
}
#endif
}
Expand All @@ -2507,7 +2507,7 @@ void mpi_send_fluid_slave(int node, int index) {
void mpi_recv_fluid(int node, int index, double *rho, double *j, double *pi) {
#ifdef LB
if (node==this_node) {
lb_get_local_fields(index, rho, j, pi);
lb_calc_local_fields(index, rho, j, pi);
} else {
double data[10];
mpi_issue(REQ_GET_FLUID, node, index);
Expand All @@ -2532,7 +2532,7 @@ void mpi_recv_fluid_slave(int node, int index) {
#ifdef LB
if (node==this_node) {
double data[10];
lb_get_local_fields(index, &data[0], &data[1], &data[4]);
lb_calc_local_fields(index, &data[0], &data[1], &data[4]);
MPI_Send(data, 10, MPI_DOUBLE, 0, REQ_GET_FLUID, MPI_COMM_WORLD);
}
#endif
Expand Down
3 changes: 0 additions & 3 deletions config.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,6 @@
#error MODES requires the fftw
#endif

#ifdef LB
#error LB requires the fftw
#endif

#endif

Expand Down
137 changes: 98 additions & 39 deletions halo.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* You should have received a copy of that license along with this program;
* if not, refer to http://www.espresso.mpg.de/license.html where its current version can be found, or
* write to Max-Planck-Institute for Polymer Research, Theory Group, PO Box 3148, 55021 Mainz, Germany.
* Copyright (c) 2002-2009; all rights reserved unless otherwise stated.
* Copyright (c) 2002-2006; all rights reserved unless otherwise stated.
*/

/** \file halo.c
Expand All @@ -24,6 +24,9 @@

#ifdef LATTICE

/** Primitive fieldtypes and their initializers */
struct _Fieldtype fieldtype_double = { 0, NULL, NULL, sizeof(double), 0, 0, 0, 0, NULL };

/** Creates a fieldtype describing the data layout
* @param count number of subtypes (Input)
* @param lengths array of lenghts of the subtytpes (Input)
Expand All @@ -36,6 +39,9 @@ void halo_create_fieldtype(int count, int* lengths, int *disps, int extent, Fiel

Fieldtype ntype = *newtype = malloc(sizeof(*ntype));

ntype->subtype = NULL;
ntype->vflag = 0;

ntype->vblocks = 1;
ntype->vstride = 1;
ntype->vskip = 1;
Expand Down Expand Up @@ -69,17 +75,45 @@ void halo_create_field_vector(int vblocks, int vstride, int vskip, Fieldtype old
int i;

Fieldtype ntype = *newtype = malloc(sizeof(*ntype));

ntype->subtype = oldtype;
ntype->vflag = 1;

ntype->vblocks = vblocks;
ntype->vstride = vstride;
ntype->vskip = vskip;

ntype->extent = oldtype->extent;
ntype->vskip = vskip;

ntype->extent = oldtype->extent * ((vblocks-1)*vskip + vstride);

int count = ntype->count = oldtype->count;
ntype->lengths = malloc(count*2*sizeof(int));
ntype->disps = (int *)((char *)ntype->lengths + count*sizeof(int));

for (i=0;i<count;i++) {
ntype->disps[i] = oldtype->disps[i];
ntype->lengths[i] = oldtype->lengths[i];
}

}

void halo_create_field_hvector(int vblocks, int vstride, int vskip, Fieldtype oldtype, Fieldtype *newtype) {
int i;

Fieldtype ntype = *newtype = malloc(sizeof(*ntype));

ntype->subtype = oldtype;
ntype->vflag = 0;

ntype->vblocks = vblocks;
ntype->vstride = vstride;
ntype->vskip = vskip;

ntype->extent = oldtype->extent*vstride + (vblocks-1)*vskip;

int count = ntype->count = oldtype->count;
ntype->lengths = malloc(count*2*sizeof(int));
ntype->disps = (int *)((char *)ntype->lengths + count*sizeof(int));

for (i=0;i<count;i++) {
ntype->disps[i] = oldtype->disps[i];
ntype->lengths[i] = oldtype->lengths[i];
Expand Down Expand Up @@ -125,31 +159,58 @@ MDINLINE void halo_dtset(void *dest, int value, Fieldtype type) {

}

MDINLINE void halo_dtcopy(void *r_buffer, void *s_buffer, int count, Fieldtype type);

MDINLINE void halo_copy_vector(void *r_buffer, void *s_buffer, int count, Fieldtype type, int vflag) {
int i, j;
void *dest, *src;

int vblocks = type->vblocks;
int vstride = type->vstride;
int vskip = type->vskip;
int extent = type->extent;

HALO_TRACE(fprintf(stderr, "%d: halo_copy_vector %p %p vblocks=%d vstride=%d vskip=%d extent=%d subtype_extent=%d\n",this_node,r_buffer,s_buffer,vblocks,vstride,vskip,extent,type->subtype->extent));

if (vflag){
vskip *= type->subtype->extent;
}

for (i=0; i<count; i++, s_buffer+=extent, r_buffer+=extent) {
for (j=0, dest=r_buffer, src=s_buffer; j<vblocks; j++, dest+=vskip, src+=vskip) {
halo_dtcopy(dest,src,vstride,type->subtype);
}
}

}

/** Copy lattice data with layout described by fieldtype.
* @param r_buffer data destination
* @param s_buffer data source
* @param type field layout type
*/
MDINLINE void halo_dtcopy(void *r_buffer, void *s_buffer, Fieldtype type) {
int i, j, k;
void *dest, *src;

int vblocks = type->vblocks;
int vstride = type->vstride;
int vskip = type->vskip;
int count = type->count;
int *lens = type->lengths;
int *disps = type->disps;
int extent = type->extent;

HALO_TRACE(fprintf(stderr, "%d: halo comm local copy r_buffer=%p s_buffer=%p\n",this_node,r_buffer,s_buffer));

for (i=0; i<vblocks; i++, r_buffer+=vskip*extent, s_buffer+=vskip*extent) {
for (j=0, dest=r_buffer, src=s_buffer; j<vstride; j++, dest+=extent, src+=extent) {
for (k=0; k<count; k++) {
memcpy(dest+disps[k],src+disps[k],lens[k]);
}
MDINLINE void halo_dtcopy(void *r_buffer, void *s_buffer, int count, Fieldtype type) {
int i, j;

HALO_TRACE(fprintf(stderr, "%d: halo_dtcopy r_buffer=%p s_buffer=%p blocks=%d stride=%d skip=%d\n",this_node,r_buffer,s_buffer,type->vblocks,type->vstride,type->vskip));

if (type->subtype) {
halo_copy_vector(r_buffer, s_buffer, count, type, type->vflag);
} else {

for (i=0; i<count; i++, s_buffer+=type->extent, r_buffer+=type->extent) {
if (!type->count) {
memcpy(r_buffer,s_buffer,type->extent);
} else {

for (j=0; j<type->count; j++) {
memcpy(r_buffer+type->disps[j],s_buffer+type->disps[j],type->lengths[j]);
}

}

}

}

}
Expand All @@ -165,13 +226,12 @@ void prepare_halo_communication(HaloCommunicator *hc, Lattice *lattice, Fieldtyp
int k, n, dir, lr, cnt, num = 0 ;
int *grid = lattice->grid ;
int *period = lattice->halo_grid ;
void *data = lattice->data;

for (n=0; n<hc->num; n++) {
MPI_Type_free(&(hc->halo_info[n].datatype));
}

num = 2*3; // two communications in each space direction
num = 2*3; /* two communications in each space direction */

hc->num = num ;
hc->halo_info = realloc(hc->halo_info,num*sizeof(HaloInfo)) ;
Expand Down Expand Up @@ -199,12 +259,13 @@ void prepare_halo_communication(HaloCommunicator *hc, Lattice *lattice, Fieldtyp

if (lr==0) {
/* send to left, recv from right */
hinfo->send_buffer = &(((char *)data)[extent * stride * 1]);
hinfo->recv_buffer = &(((char *)data)[extent * stride * (grid[dir]+1)]);
hinfo->s_offset = extent * stride * 1;
hinfo->r_offset = extent * stride * (grid[dir]+1);
} else {
/* send to right, recv from left */
hinfo->send_buffer = &(((char *)data)[extent * stride * grid[dir]]);
hinfo->recv_buffer = &(((char *)data)[extent * stride * 0]);
hinfo->s_offset = extent * stride * grid[dir];
hinfo->r_offset = extent * stride * 0;

}

hinfo->source_node = node_neighbors[2*dir+1-lr];
Expand All @@ -214,7 +275,7 @@ void prepare_halo_communication(HaloCommunicator *hc, Lattice *lattice, Fieldtyp

MPI_Type_vector(nblocks, stride, skip, datatype, &hinfo->datatype);
MPI_Type_commit(&hinfo->datatype);

#ifdef PARTIAL_PERIODIC
if ( !PERIODIC(dir) && (boundary[2*dir+lr] != 0 || boundary[2*dir+1-lr] != 0) ) {
if (node_grid[dir] == 1) {
Expand Down Expand Up @@ -245,7 +306,7 @@ void prepare_halo_communication(HaloCommunicator *hc, Lattice *lattice, Fieldtyp
}
}

HALO_TRACE(fprintf(stderr,"%d: prepare_halo_communication dir=%d lr=%d s_buffer=%p r_buffer=%p, s_node=%d d_node=%d extent=%d type=%d\n",this_node,dir,lr,hinfo->send_buffer,hinfo->recv_buffer,hinfo->source_node,hinfo->dest_node,(int)extent,hinfo->type)) ;
HALO_TRACE(fprintf(stderr,"%d: prepare_halo_communication dir=%d lr=%d s_offset=%ld r_offset=%ld s_node=%d d_node=%d type=%d\n",this_node,dir,lr,hinfo->s_offset,hinfo->r_offset,hinfo->source_node,hinfo->dest_node,hinfo->type)) ;

cnt++;

Expand All @@ -261,7 +322,6 @@ void release_halo_communication(HaloCommunicator *hc) {
int n;

for (n=0; n<hc->num; n++) {
HALO_TRACE(fprintf(stderr,"%d: freeing %p\n",this_node,&(hc->halo_info[n].datatype)));
MPI_Type_free(&(hc->halo_info[n].datatype));
}

Expand All @@ -273,7 +333,7 @@ void release_halo_communication(HaloCommunicator *hc) {
* described by the halo communicator
* @param hc halo communicator describing the parallelization scheme
*/
void halo_communication(HaloCommunicator *hc) {
void halo_communication(HaloCommunicator *hc, void *base) {
int n, comm_type, s_node, r_node;
void *s_buffer, *r_buffer ;

Expand All @@ -282,21 +342,21 @@ void halo_communication(HaloCommunicator *hc) {
MPI_Request request;
MPI_Status status;

HALO_TRACE(fprintf(stderr, "%d: halo_comm %p (num=%d)\n", this_node, hc, hc->num)) ;
HALO_TRACE(fprintf(stderr, "%d: halo_comm base=%p num=%d\n", this_node, base, hc->num)) ;

for (n = 0; n < hc->num; n++) {

HALO_TRACE(fprintf(stderr, "%d: halo_comm round %d\n", this_node, n)) ;

comm_type = hc->halo_info[n].type ;
s_buffer = hc->halo_info[n].send_buffer ;
r_buffer = hc->halo_info[n].recv_buffer ;
s_buffer = (char *)base + hc->halo_info[n].s_offset;
r_buffer = (char *)base + hc->halo_info[n].r_offset;

switch (comm_type) {

case HALO_LOCL:
fieldtype = hc->halo_info[n].fieldtype;
halo_dtcopy(r_buffer,s_buffer,fieldtype);
halo_dtcopy(r_buffer,s_buffer,1,fieldtype);
break ;

case HALO_SENDRECV:
Expand Down Expand Up @@ -340,15 +400,14 @@ void halo_communication(HaloCommunicator *hc) {

HALO_TRACE(fprintf(stderr,"%d: halo_comm open boundaries\n",this_node));

/* \todo this does not work for the n_i - <n_i> */
halo_dtset(r_buffer,0,fieldtype);
break;

}

}

HALO_TRACE(fprintf(stderr, "%d: halo_comm %p finished\n", this_node, hc));

}

#endif /* LATTICE */
33 changes: 22 additions & 11 deletions halo.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* You should have received a copy of that license along with this program;
* if not, refer to http://www.espresso.mpg.de/license.html where its current version can be found, or
* write to Max-Planck-Institute for Polymer Research, Theory Group, PO Box 3148, 55021 Mainz, Germany.
* Copyright (c) 2002-2009; all rights reserved unless otherwise stated.
* Copyright (c) 2002-2006; all rights reserved unless otherwise stated.
*/

/** \file halo.h
Expand Down Expand Up @@ -57,29 +57,39 @@
* is similar to MPI datatypes but a bit more compact. See \ref
* halo_create_fieldtype, \ref halo_create_field_vector and \ref
* halo_dtcopy to understand how it works. */
typedef struct _Fieldtype {
typedef struct _Fieldtype *Fieldtype;
struct _Fieldtype {
int count; /**< number of subtypes in fieldtype */
int *disps; /**< displacements of the subtypes */
int *lengths; /**< lengths of the subtypes */
int extent; /**< extent of the complete fieldtype including gaps */
int vblocks; /**< number of blocks in field vectors */
int vstride; /**< size of strides in field vectors */
int vskip; /**< displacement between strides in field vectors */
} *Fieldtype;
int vflag;
Fieldtype subtype;
};

/** Predefined fieldtypes */
extern struct _Fieldtype fieldtype_double;
#define FIELDTYPE_DOUBLE (&fieldtype_double)

/** Structure describing a Halo region */
typedef struct {

int type; /**< type of halo communication */
int type; /**< type of halo communication */

int source_node; /**< index of processor which sends halo data */
int dest_node; /**< index of processor receiving halo data */

int source_node; /**< index of processor which sends halo data */
int dest_node; /**< index of processor receiving halo data */
//void *send_buffer; /**< pointer to data being sent */
//void *recv_buffer; /**< pointer to data being received */

void *send_buffer; /**< pointer to data being sent */
void *recv_buffer; /**< pointer to data being received */
unsigned long s_offset; /**< offset for send buffer */
unsigned long r_offset; /**< offset for receive buffer */

Fieldtype fieldtype; /**< type layout of the data beeing exchanged */
MPI_Datatype datatype; /**< MPI datatype of data beeing communicated */
Fieldtype fieldtype; /**< type layout of the data beeing exchanged */
MPI_Datatype datatype; /**< MPI datatype of data beeing communicated */

} HaloInfo ;

Expand Down Expand Up @@ -110,6 +120,7 @@ void halo_create_fieldtype(int count, int *lens, int *disps, int extent, Fieldty
* @param newtype newly created fieldtype (Input/Output)
*/
void halo_create_field_vector(int vblocks, int vstride, int vskip, Fieldtype oldtype, Fieldtype *newtype);
void halo_create_field_hvector(int vblocks, int vstride, int vskip, Fieldtype oldtype, Fieldtype *newtype);

/** Frees a fieldtype
* @param ftype pointer to the type to be freed (Input)
Expand All @@ -134,7 +145,7 @@ void release_halo_communication(HaloCommunicator *hc);
* described by the halo communicator
* @param hc halo communicator describing the parallelization scheme
*/
void halo_communication(HaloCommunicator *hc);
void halo_communication(HaloCommunicator *hc, void *base);

#endif /* LATTICE */

Expand Down
Loading

0 comments on commit 7b1a2b9

Please sign in to comment.