Skip to content

Commit

Permalink
Merge pull request open-mpi#627 from bureddy/v1.10
Browse files Browse the repository at this point in the history 
HCOLL: Add alltoallv interface
  • Loading branch information
@jsquyres
jsquyres committed Oct 3, 2015
2 parents 4834a14 + da0b1b6 commit e8ae71b
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 9 deletions.
39 changes: 35 additions & 4 deletions ompi/mca/coll/hcoll/coll_hcoll.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,10 @@ struct mca_coll_hcoll_module_t {
mca_coll_base_module_t *previous_iallreduce_module;
mca_coll_base_module_igatherv_fn_t previous_igatherv;
mca_coll_base_module_t *previous_igatherv_module;
mca_coll_base_module_ialltoall_fn_t previous_ialltoall;
mca_coll_base_module_t *previous_ialltoall_module;
mca_coll_base_module_ialltoallv_fn_t previous_ialltoallv;
mca_coll_base_module_t *previous_ialltoallv_module;
};
typedef struct mca_coll_hcoll_module_t mca_coll_hcoll_module_t;

Expand Down Expand Up @@ -186,6 +190,15 @@ int mca_coll_hcoll_alltoall(void *sbuf, int scount,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module);

int mca_coll_hcoll_alltoallv(void *sbuf, int *scounts,
int *sdisps,
struct ompi_datatype_t *sdtype,
void *rbuf, int *rcounts,
int *rdisps,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module);

int mca_coll_hcoll_gatherv(void* sbuf, int scount,
struct ompi_datatype_t *sdtype,
void* rbuf, int *rcounts, int *displs,
Expand All @@ -199,10 +212,10 @@ int mca_coll_hcoll_ibarrier(struct ompi_communicator_t *comm,
mca_coll_base_module_t *module);

int mca_coll_hcoll_ibcast(void *buff, int count,
struct ompi_datatype_t *datatype, int root,
struct ompi_communicator_t *comm,
ompi_request_t** request,
mca_coll_base_module_t *module);
struct ompi_datatype_t *datatype, int root,
struct ompi_communicator_t *comm,
ompi_request_t** request,
mca_coll_base_module_t *module);

int mca_coll_hcoll_iallgather(void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
Expand All @@ -219,6 +232,24 @@ int mca_coll_hcoll_iallreduce(void *sbuf, void *rbuf, int count,
ompi_request_t** request,
mca_coll_base_module_t *module);

int mca_coll_hcoll_ialltoall(void *sbuf, int scount,
struct ompi_datatype_t *sdtype,
void* rbuf, int rcount,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
ompi_request_t **req,
mca_coll_base_module_t *module);

int mca_coll_hcoll_ialltoallv(void *sbuf, int *scounts,
int *sdisps,
struct ompi_datatype_t *sdtype,
void *rbuf, int *rcounts,
int *rdisps,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
ompi_request_t **req,
mca_coll_base_module_t *module);

int mca_coll_hcoll_igatherv(void* sbuf, int scount,
struct ompi_datatype_t *sdtype,
void* rbuf, int *rcounts, int *displs,
Expand Down
16 changes: 11 additions & 5 deletions ompi/mca/coll/hcoll/coll_hcoll_module.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,19 +81,21 @@ static void mca_coll_hcoll_module_destruct(mca_coll_hcoll_module_t *hcoll_module
OBJ_RELEASE(hcoll_module->previous_allreduce_module);
OBJ_RELEASE(hcoll_module->previous_allgather_module);
OBJ_RELEASE(hcoll_module->previous_gatherv_module);
OBJ_RELEASE(hcoll_module->previous_alltoall_module);
OBJ_RELEASE(hcoll_module->previous_alltoallv_module);

OBJ_RELEASE(hcoll_module->previous_ibarrier_module);
OBJ_RELEASE(hcoll_module->previous_ibcast_module);
OBJ_RELEASE(hcoll_module->previous_iallreduce_module);
OBJ_RELEASE(hcoll_module->previous_iallgather_module);
OBJ_RELEASE(hcoll_module->previous_igatherv_module);
OBJ_RELEASE(hcoll_module->previous_ialltoall_module);
OBJ_RELEASE(hcoll_module->previous_ialltoallv_module);

/*
OBJ_RELEASE(hcoll_module->previous_allgatherv_module);
OBJ_RELEASE(hcoll_module->previous_gather_module);
OBJ_RELEASE(hcoll_module->previous_gatherv_module);
OBJ_RELEASE(hcoll_module->previous_alltoall_module);
OBJ_RELEASE(hcoll_module->previous_alltoallv_module);
OBJ_RELEASE(hcoll_module->previous_alltoallw_module);
OBJ_RELEASE(hcoll_module->previous_reduce_scatter_module);
OBJ_RELEASE(hcoll_module->previous_reduce_module);
Expand Down Expand Up @@ -127,12 +129,16 @@ static int mca_coll_hcoll_save_coll_handlers(mca_coll_hcoll_module_t *hcoll_modu
HCOL_SAVE_PREV_COLL_API(allreduce);
HCOL_SAVE_PREV_COLL_API(allgather);
HCOL_SAVE_PREV_COLL_API(gatherv);
HCOL_SAVE_PREV_COLL_API(alltoall);
HCOL_SAVE_PREV_COLL_API(alltoallv);

HCOL_SAVE_PREV_COLL_API(ibarrier);
HCOL_SAVE_PREV_COLL_API(ibcast);
HCOL_SAVE_PREV_COLL_API(iallreduce);
HCOL_SAVE_PREV_COLL_API(iallgather);
HCOL_SAVE_PREV_COLL_API(igatherv);
HCOL_SAVE_PREV_COLL_API(ialltoall);
HCOL_SAVE_PREV_COLL_API(ialltoallv);

/*
These collectives are not yet part of hcoll, so
Expand All @@ -141,9 +147,6 @@ static int mca_coll_hcoll_save_coll_handlers(mca_coll_hcoll_module_t *hcoll_modu
HCOL_SAVE_PREV_COLL_API(gather);
HCOL_SAVE_PREV_COLL_API(reduce);
HCOL_SAVE_PREV_COLL_API(allgatherv);
HCOL_SAVE_PREV_COLL_API(gatherv);
HCOL_SAVE_PREV_COLL_API(alltoall);
HCOL_SAVE_PREV_COLL_API(alltoallv);
HCOL_SAVE_PREV_COLL_API(alltoallw);
*/
return OMPI_SUCCESS;
Expand Down Expand Up @@ -311,13 +314,16 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
hcoll_module->super.coll_allgather = hcoll_collectives.coll_allgather ? mca_coll_hcoll_allgather : NULL;
hcoll_module->super.coll_allreduce = hcoll_collectives.coll_allreduce ? mca_coll_hcoll_allreduce : NULL;
hcoll_module->super.coll_alltoall = /*hcoll_collectives.coll_alltoall ? mca_coll_hcoll_alltoall : */ NULL;
hcoll_module->super.coll_alltoallv = hcoll_collectives.coll_alltoallv ? mca_coll_hcoll_alltoallv : NULL;
hcoll_module->super.coll_gatherv = hcoll_collectives.coll_gatherv ? mca_coll_hcoll_gatherv : NULL;
hcoll_module->super.coll_ibarrier = hcoll_collectives.coll_ibarrier ? mca_coll_hcoll_ibarrier : NULL;
hcoll_module->super.coll_ibcast = hcoll_collectives.coll_ibcast ? mca_coll_hcoll_ibcast : NULL;
hcoll_module->super.coll_iallgather = hcoll_collectives.coll_iallgather ? mca_coll_hcoll_iallgather : NULL;
hcoll_module->super.coll_iallreduce = hcoll_collectives.coll_iallreduce ? mca_coll_hcoll_iallreduce : NULL;
hcoll_module->super.coll_gather = /*hcoll_collectives.coll_gather ? mca_coll_hcoll_gather :*/ NULL;
hcoll_module->super.coll_igatherv = hcoll_collectives.coll_igatherv ? mca_coll_hcoll_igatherv : NULL;
hcoll_module->super.coll_ialltoall = /*hcoll_collectives.coll_ialltoall ? mca_coll_hcoll_ialltoall : */ NULL;
hcoll_module->super.coll_ialltoallv = /*hcoll_collectives.coll_ialltoallv ? mca_coll_hcoll_ialltoallv : */ NULL;

*priority = cm->hcoll_priority;
module = &hcoll_module->super;
Expand Down
37 changes: 37 additions & 0 deletions ompi/mca/coll/hcoll/coll_hcoll_ops.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,43 @@ int mca_coll_hcoll_alltoall(void *sbuf, int scount,
return rc;
}

int mca_coll_hcoll_alltoallv(void *sbuf, int *scounts, int *sdisps,
struct ompi_datatype_t *sdtype,
void *rbuf, int *rcounts, int *rdisps,
struct ompi_datatype_t *rdtype,
struct ompi_communicator_t *comm,
mca_coll_base_module_t *module)
{
dte_data_representation_t stype;
dte_data_representation_t rtype;
int rc;
HCOL_VERBOSE(20,"RUNNING HCOL ALLTOALLV");
mca_coll_hcoll_module_t *hcoll_module = (mca_coll_hcoll_module_t*)module;
stype = ompi_dtype_2_dte_dtype(sdtype);
rtype = ompi_dtype_2_dte_dtype(rdtype);
if (OPAL_UNLIKELY((HCOL_DTE_IS_ZERO(stype) || HCOL_DTE_IS_ZERO(rtype)
|| HCOL_DTE_IS_COMPLEX(stype) || HCOL_DTE_IS_COMPLEX(rtype)))
&& mca_coll_hcoll_component.hcoll_datatype_fallback){
HCOL_VERBOSE(20,"Ompi_datatype is not supported: sdtype = %s, rdtype = %s; calling fallback alltoallv;",
sdtype->super.name,
rdtype->super.name);
rc = hcoll_module->previous_alltoallv(sbuf, scounts, sdisps, sdtype,
rbuf, rcounts, rdisps, rdtype,
comm, hcoll_module->previous_alltoallv_module);
return rc;
}
rc = hcoll_collectives.coll_alltoallv(sbuf, scounts, sdisps, stype,
rbuf, rcounts, rdisps, rtype,
hcoll_module->hcoll_context);
if (HCOLL_SUCCESS != rc){
HCOL_VERBOSE(20,"RUNNING FALLBACK ALLTOALLV");
rc = hcoll_module->previous_alltoallv(sbuf, scounts, sdisps, sdtype,
rbuf, rcounts, rdisps, rdtype,
comm, hcoll_module->previous_alltoallv_module);
}
return rc;
}

int mca_coll_hcoll_gatherv(void* sbuf, int scount,
struct ompi_datatype_t *sdtype,
void* rbuf, int *rcounts, int *displs,
Expand Down

0 comments on commit e8ae71b

Please sign in to comment.