diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index cf43617eedf..531a41e0b4f 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -2153,3 +2153,51 @@ int pio_sorted_copy(const void *array, void *sortedarray, io_desc_t *iodesc, int } return PIO_NOERR; } + +/** + * Compute the maximum aggregate number of bytes. This is called by + * subset_rearrange_create() and box_rearrange_create(). + * + * @param ios pointer to the IO system structure. + * @param iodesc a pointer to decomposition description. + * @returns 0 for success, error code otherwise. + * @author Jim Edwards + */ +int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc) +{ + int maxbytesoniotask = INT_MAX; + int maxbytesoncomputetask = INT_MAX; + int maxbytes; + int mpierr; /* Return code from MPI functions. */ + + /* Check inputs. */ + pioassert(iodesc, "invalid input", __FILE__, __LINE__); + + LOG((2, "compute_maxaggregate_bytes iodesc->maxiobuflen = %d iodesc->ndof = %d", + iodesc->maxiobuflen, iodesc->ndof)); + + /* Determine the max bytes that can be held on IO task. */ + if (ios->ioproc && iodesc->maxiobuflen > 0) + maxbytesoniotask = pio_buffer_size_limit / iodesc->maxiobuflen; + + /* Determine the max bytes that can be held on computation task. */ + if (ios->comp_rank >= 0 && iodesc->ndof > 0) + maxbytesoncomputetask = pio_cnbuffer_limit / iodesc->ndof; + + /* Take the min of the max IO and max comp bytes. */ + maxbytes = min(maxbytesoniotask, maxbytesoncomputetask); + LOG((2, "compute_maxaggregate_bytes maxbytesoniotask = %d maxbytesoncomputetask = %d", + maxbytesoniotask, maxbytesoncomputetask)); + + /* Get the min value of this on all tasks. */ + LOG((3, "before allreaduce maxbytes = %d", maxbytes)); + if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN, + ios->union_comm))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + LOG((3, "after allreaduce maxbytes = %d", maxbytes)); + + /* Remember the result. */ + iodesc->maxbytes = maxbytes; + + return PIO_NOERR; +} diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h index 1724d07b2f4..f087b0083d3 100644 --- a/src/clib/pio_internal.h +++ b/src/clib/pio_internal.h @@ -249,6 +249,8 @@ extern "C" { /* Flush contents of multi-buffer to disk. */ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize); + int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc); + /* Compute the size that the IO tasks will need to hold the data. */ int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc); diff --git a/src/clib/pio_rearrange.c b/src/clib/pio_rearrange.c index 134a237f0a5..cebf72d849d 100644 --- a/src/clib/pio_rearrange.c +++ b/src/clib/pio_rearrange.c @@ -7,6 +7,12 @@ #include #include + +#if PIO_USE_MPISERIAL +# define MPI_Type_create_hvector MPI_Type_hvector +#endif + + /** * Convert a 1-D index into a coordinate value in an arbitrary * dimension space. E.g., for index 4 into a array defined as a[3][2], @@ -542,7 +548,8 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int ierr; /* Check inputs. */ - pioassert(ios && iodesc && dest_ioproc && dest_ioindex && + pioassert(ios && iodesc && (iodesc->ndof == 0 || + (iodesc->ndof > 0 && dest_ioproc && dest_ioindex)) && iodesc->rearranger == PIO_REARR_BOX && ios->num_uniontasks > 0, "invalid input", __FILE__, __LINE__); LOG((1, "compute_counts ios->num_uniontasks = %d ios->compproc %d ios->ioproc %d", @@ -556,11 +563,12 @@ int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, int recv_displs[ios->num_uniontasks]; /* The list of indeces on each compute task */ - PIO_Offset *s2rindex; - - if (!(s2rindex = malloc(sizeof(PIO_Offset) * iodesc->ndof))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - + PIO_Offset *s2rindex = NULL; + if (iodesc->ndof > 0) + { + if (!(s2rindex = malloc(sizeof(PIO_Offset) * iodesc->ndof))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + } /* Allocate memory for the array of counts and init to zero. */ if (!(iodesc->scount = calloc(ios->num_iotasks, sizeof(int)))) return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); @@ -876,15 +884,10 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, * spaced blocks of the same size. The block size * is 1, the stride here is the length of the * collected array (llen). */ -#if PIO_USE_MPISERIAL - if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->mpitype_size, - iodesc->rtype[i], &recvtypes[i]))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); -#else if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->mpitype_size, iodesc->rtype[i], &recvtypes[i]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); -#endif /* PIO_USE_MPISERIAL */ + pioassert(recvtypes[i] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__); if ((mpierr = MPI_Type_commit(&recvtypes[i]))) @@ -897,15 +900,10 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, "recvcounts[iodesc->rfrom[i]] = %d", i, iodesc->rfrom[i], recvcounts[iodesc->rfrom[i]])); -#if PIO_USE_MPISERIAL - if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->mpitype_size, - iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]]))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); -#else if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->mpitype_size, iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]]))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); -#endif /* PIO_USE_MPISERIAL */ + pioassert(recvtypes[iodesc->rfrom[i]] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__); @@ -920,35 +918,32 @@ int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, /* On compute tasks loop over iotasks and create a data type for * each exchange. */ - for (int i = 0; i < niotasks; i++) + if(!ios->async || ios->compproc) { - int io_comprank = ios->ioranks[i]; - LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i])); - if (iodesc->rearranger == PIO_REARR_SUBSET) - io_comprank = 0; - - LOG((3, "i = %d iodesc->scount[i] = %d", i, iodesc->scount[i])); - if (iodesc->scount[i] > 0 && sbuf) - { - LOG((3, "io task %d creating sendtypes[%d]", i, io_comprank)); - sendcounts[io_comprank] = 1; -#if PIO_USE_MPISERIAL - if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->mpitype_size, - iodesc->stype[i], &sendtypes[io_comprank]))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); -#else - if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->mpitype_size, - iodesc->stype[i], &sendtypes[io_comprank]))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); -#endif /* PIO_USE_MPISERIAL */ - pioassert(sendtypes[io_comprank] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__); - - if ((mpierr = MPI_Type_commit(&sendtypes[io_comprank]))) - return check_mpi(NULL, mpierr, __FILE__, __LINE__); - } - else + for (int i = 0; i < niotasks; i++) { - sendcounts[io_comprank] = 0; + int io_comprank = ios->ioranks[i]; + LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i])); + if (iodesc->rearranger == PIO_REARR_SUBSET) + io_comprank = 0; + + LOG((3, "i = %d iodesc->scount[i] = %d", i, iodesc->scount[i])); + if (iodesc->scount[i] > 0 && sbuf) + { + LOG((3, "io task %d creating sendtypes[%d]", i, io_comprank)); + sendcounts[io_comprank] = 1; + if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->mpitype_size, + iodesc->stype[i], &sendtypes[io_comprank]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + pioassert(sendtypes[io_comprank] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__); + + if ((mpierr = MPI_Type_commit(&sendtypes[io_comprank]))) + return check_mpi(NULL, mpierr, __FILE__, __LINE__); + } + else + { + sendcounts[io_comprank] = 0; + } } } @@ -1204,6 +1199,9 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com { int ret; +#ifdef TIMING + GPTLstart("PIO:box_rearrange_create"); +#endif /* Check inputs. */ pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims > 0 && iodesc, "invalid input", __FILE__, __LINE__); @@ -1211,8 +1209,8 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com "ios->num_iotasks = %d", maplen, ndims, ios->num_comptasks, ios->num_iotasks)); /* Allocate arrays needed for this function. */ - int *dest_ioproc; /* Destination IO task for each data element on compute task. */ - PIO_Offset *dest_ioindex; /* Offset into IO task array for each data element. */ + int *dest_ioproc = NULL; /* Destination IO task for each data element on compute task. */ + PIO_Offset *dest_ioindex = NULL; /* Offset into IO task array for each data element. */ int sendcounts[ios->num_uniontasks]; /* Send counts for swapm call. */ int sdispls[ios->num_uniontasks]; /* Send displacements for swapm. */ int recvcounts[ios->num_uniontasks]; /* Receive counts for swapm. */ @@ -1220,11 +1218,12 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com MPI_Datatype dtypes[ios->num_uniontasks]; /* Array of MPI_OFFSET types for swapm. */ PIO_Offset iomaplen[ios->num_iotasks]; /* Gets the llen of all IO tasks. */ - /* Allocate storage. */ - if (!(dest_ioproc = malloc(sizeof(int) * maplen))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); - if (!(dest_ioindex = malloc(sizeof(PIO_Offset) * maplen))) - return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + /* sc_info msg = [iomaplen, starts_for_all_dims, count_for_all_dims] */ + int sc_info_msg_maplen_sz = 1; /* The iomaplen, == 0 implies start/count are invalid */ + int sc_info_msg_sc_sz = 2 * ndims; /* The (start + count) for all dims */ + int sc_info_msg_sz = sc_info_msg_maplen_sz + sc_info_msg_sc_sz; + PIO_Offset sc_info_msg_send[sc_info_msg_sz]; + PIO_Offset sc_info_msg_recv[ios->num_iotasks * sc_info_msg_sz]; /* This is the box rearranger. */ iodesc->rearranger = PIO_REARR_BOX; @@ -1232,6 +1231,30 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com /* Number of elements of data on compute node. */ iodesc->ndof = maplen; + if (maplen > 0) + { + if (!(dest_ioproc = malloc(maplen * sizeof(int)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + + if (!(dest_ioindex = malloc(maplen * sizeof(PIO_Offset)))) + return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__); + } + + /* Initialize the sc_info send and recv messages */ + for(int i=0; iioranks[i]). Each + * sc_info message contains [iomaplen, start_for_all_dims, count_for_all_dims] + */ + for(int i=0; inum_iotasks * sc_info_msg_sz; i++) + { + sc_info_msg_recv[i] = 0; + } + /* Initialize array values. */ for (int i = 0; i < maplen; i++) { @@ -1258,14 +1281,7 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com pioassert(iodesc->llen == 0, "error", __FILE__, __LINE__); if (ios->ioproc) { - /* Set up send counts for sending llen in all to all - * gather. We are sending to all tasks, IO and computation. */ - for (int i = 0; i < ios->num_comptasks; i++) - sendcounts[ios->compranks[i]] = 1; - for (int i = 0; i < ios->num_iotasks; i++) - sendcounts[ios->ioranks[i]] = 1; - - /* Determine llen, the length of the data array on this IO + /* Determine llen, the lenght of the data array on this IO * node, by multipliying the counts in the * iodesc->firstregion. */ iodesc->llen = 1; @@ -1284,76 +1300,83 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com LOG((2, "iodesc->needsfill = %d ios->num_iotasks = %d", iodesc->needsfill, ios->num_iotasks)); - /* Set up receive counts and displacements to for an AllToAll - * gather of llen. */ + /* Set the iomaplen in the sc_info msg */ + sc_info_msg_send[0] = iodesc->llen; + + /* start/count array to be sent: 1st half for start, 2nd half for count */ + for (int j = 0; j < ndims; j++) + { + /* The first data in sc_info_msg_send[] is the iomaplen */ + sc_info_msg_send[j + 1] = iodesc->firstregion->start[j]; + sc_info_msg_send[ndims + j + 1] = iodesc->firstregion->count[j]; + } + + /* Set the recvcounts/recv displs for the sc_info msg from each io task */ for (int i = 0; i < ios->num_iotasks; i++) { - recvcounts[ios->ioranks[i]] = 1; - rdispls[ios->ioranks[i]] = i * SIZEOF_MPI_OFFSET; - LOG((3, "i = %d ios->ioranks[%d] = %d recvcounts[%d] = %d rdispls[%d] = %d", - i, i, ios->ioranks[i], ios->ioranks[i], recvcounts[ios->ioranks[i]], - ios->ioranks[i], rdispls[ios->ioranks[i]])); + /* From each iotask all procs (compute and I/O procs) receive an + * sc_info message containing [iomaplen, start_for_all_dims, + * count_for_all_dims] and the size of this message is + * [sizeof(MPI_OFFSET) + ndims * sizeof(MPI_OFFSET) + ndims * + * sizeof(MPI_OFFSET)] + * Note: The displacements are in bytes + */ + recvcounts[ios->ioranks[i]] = sc_info_msg_sz; + rdispls[ios->ioranks[i]] = i * sc_info_msg_sz * SIZEOF_MPI_OFFSET; + } + + /* Set the sendcounts/send displs for the sc_info msg sent from each + * I/O task + */ + for(int i=0; inum_uniontasks; i++){ + sendcounts[i] = 0; + sdispls[i] = 0; + } + if(ios->ioproc){ + /* Only I/O procs send sc_info messages */ + int nmsgs = ios->num_iotasks; + for (int i = 0; i < ios->num_comptasks; i++) + { + sendcounts[ios->compranks[i]] = sc_info_msg_sz; + sdispls[ios->compranks[i]] = 0; + } + for (int i = 0; i < ios->num_iotasks; i++) + { + sendcounts[ios->ioranks[i]] = sc_info_msg_sz; + sdispls[ios->ioranks[i]] = 0; + } } - /* All-gather the llen to all tasks into array iomaplen. */ - LOG((3, "calling pio_swapm to allgather llen into array iomaplen, ndims = %d dtypes[0] = %d", - ndims, dtypes)); - if ((ret = pio_swapm(&iodesc->llen, sendcounts, sdispls, dtypes, iomaplen, recvcounts, - rdispls, dtypes, ios->union_comm, &iodesc->rearr_opts.io2comp))) + /* Send sc_info msg from iotasks (all iotasks) to all procs(compute and I/O procs)*/ + LOG((3, "about to call pio_swapm with start/count from iotask ndims = %d", + ndims)); + if ((ret = pio_swapm(sc_info_msg_send, sendcounts, sdispls, dtypes, sc_info_msg_recv, + recvcounts, rdispls, dtypes, ios->union_comm, + &iodesc->rearr_opts.io2comp))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); - LOG((3, "iodesc->llen = %d", iodesc->llen)); + #if PIO_ENABLE_LOGGING + /* First entry in the sc_info msg for each iorank is the iomaplen */ for (int i = 0; i < ios->num_iotasks; i++) - LOG((3, "iomaplen[%d] = %d", i, iomaplen[i])); + LOG((3, "iomaplen[%d] = %d", i, sc_info_msg_recv[i * sc_info_msg_sz])); #endif /* PIO_ENABLE_LOGGING */ - /* For each IO task send starts/counts to all compute tasks. */ for (int i = 0; i < ios->num_iotasks; i++) { - /* The ipmaplen contains the llen (number of data elements) - * for this IO task. */ - LOG((2, "iomaplen[%d] = %d", i, iomaplen[i])); - - /* If there is data for this IO task, send start/count to all - * compute tasks. */ - if (iomaplen[i] > 0) + /* First entry in the sc_info msg is the iomaplen */ + iomaplen[i] = sc_info_msg_recv[i * sc_info_msg_sz]; + if(iomaplen[i] > 0) { - PIO_Offset start[ndims]; - PIO_Offset count[ndims]; - - /* Set up send/recv parameters for all to all gather of - * counts and starts. */ - for (int j = 0; j < ios->num_uniontasks; j++) - { - sendcounts[j] = 0; - sdispls[j] = 0; - rdispls[j] = 0; - recvcounts[j] = 0; - if (ios->union_rank == ios->ioranks[i]) - sendcounts[j] = ndims; - } - recvcounts[ios->ioranks[i]] = ndims; - - /* The count array from iotask i is sent to all compute tasks. */ - LOG((3, "about to call pio_swapm with count from iotask %d ndims = %d", - i, ndims)); - if ((ret = pio_swapm(iodesc->firstregion->count, sendcounts, sdispls, dtypes, count, - recvcounts, rdispls, dtypes, ios->union_comm, - &iodesc->rearr_opts.io2comp))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); + /* The rest of the entries in the sc_info msg are the start and + * count arrays + */ + PIO_Offset *start = &(sc_info_msg_recv[i * sc_info_msg_sz + 1]); + PIO_Offset *count = &(sc_info_msg_recv[i * sc_info_msg_sz + 1 + ndims]); - /* The start array from iotask i is sent to all compute tasks. */ - LOG((3, "about to call pio_swapm with start from iotask %d ndims = %d", - i, ndims)); - if ((ret = pio_swapm(iodesc->firstregion->start, sendcounts, sdispls, dtypes, - start, recvcounts, rdispls, dtypes, ios->union_comm, - &iodesc->rearr_opts.io2comp))) - return pio_err(ios, NULL, ret, __FILE__, __LINE__); - -#if PIO_ENABLE_LOGGING + #if PIO_ENABLE_LOGGING for (int d = 0; d < ndims; d++) LOG((3, "start[%d] = %lld count[%d] = %lld", d, start[d], d, count[d])); -#endif /* PIO_ENABLE_LOGGING */ + #endif /* PIO_ENABLE_LOGGING */ /* For each element of the data array on the compute task, * find the IO task to send the data element to, and its @@ -1366,10 +1389,10 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com /* The compmap array is 1 based but calculations are 0 based */ LOG((3, "about to call idx_to_dim_list ndims = %d ", ndims)); idx_to_dim_list(ndims, gdimlen, compmap[k] - 1, gcoord); -#if PIO_ENABLE_LOGGING + #if PIO_ENABLE_LOGGING for (int d = 0; d < ndims; d++) LOG((3, "gcoord[%d] = %lld", d, gcoord[d])); -#endif /* PIO_ENABLE_LOGGING */ + #endif /* PIO_ENABLE_LOGGING */ /* Find a destination for each entry in the compmap. */ for (int j = 0; j < ndims; j++) @@ -1403,16 +1426,20 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com /* Check that a destination is found for each compmap entry. */ for (int k = 0; k < maplen; k++) if (dest_ioproc[k] < 0 && compmap[k] > 0) + { + LOG((1, "Error: Found dest_ioproc[%d] = %d and compmap[%d] = %lld", k, dest_ioproc[k], k, compmap[k])); return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__); + } /* Completes the mapping for the box rearranger. */ LOG((2, "calling compute_counts maplen = %d", maplen)); if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex))) return pio_err(ios, NULL, ret, __FILE__, __LINE__); - /* Free resources. */ free(dest_ioproc); free(dest_ioindex); + dest_ioproc = NULL; + dest_ioindex = NULL; /* Compute the max io buffer size needed for an iodesc. */ if (ios->ioproc) @@ -1422,6 +1449,15 @@ int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *com LOG((3, "iodesc->maxiobuflen = %d", iodesc->maxiobuflen)); } + /* Using maxiobuflen compute the maximum number of bytes that the + * io task buffer can handle. */ + if ((ret = compute_maxaggregate_bytes(ios, iodesc))) + return pio_err(ios, NULL, ret, __FILE__, __LINE__); + LOG((3, "iodesc->maxbytes = %d", iodesc->maxbytes)); + +#ifdef TIMING + GPTLstop("PIO:box_rearrange_create"); +#endif return PIO_NOERR; } diff --git a/src/clib/pioc_sc.c b/src/clib/pioc_sc.c index b9e21b7442a..4a85a00b76a 100644 --- a/src/clib/pioc_sc.c +++ b/src/clib/pioc_sc.c @@ -23,7 +23,6 @@ int blocksize = DEFAULT_BLOCKSIZE; * @param a * @param b * @returns greatest common divisor. - * @author Jim Edwards */ int gcd(int a, int b ) { @@ -39,7 +38,6 @@ int gcd(int a, int b ) * @param a * @param b * @returns greates common divisor. - * @author Jim Edwards */ long long lgcd(long long a, long long b) { @@ -48,13 +46,39 @@ long long lgcd(long long a, long long b) return lgcd(b % a, a); } +/** + * Return the gcd of elements in an int array. + * + * @param nain length of the array + * @param ain an array of length nain + * @returns greatest common divisor. + */ +int gcd_array(int nain, int *ain) +{ + int i; + int bsize = 1; + + for (i = 0; i < nain; i++) + if (ain[i] <= 1) + return bsize; + + bsize = ain[0]; + i = 1; + while (i < nain && bsize > 1) + { + bsize = gcd(bsize, ain[i]); + i++; + } + + return bsize; +} + /** * Return the greatest common devisor of array ain as int_64. * * @param nain number of elements in ain. * @param ain array of length nain. * @returns GCD of elements in ain. - * @author Jim Edwards */ long long lgcd_array(int nain, long long *ain) { @@ -85,7 +109,6 @@ long long lgcd_array(int nain, long long *ain) * @param rank IO rank of this task. * @param start pointer to PIO_Offset that will get the start value. * @param count pointer to PIO_Offset that will get the count value. - * @author Jim Edwards */ void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start, PIO_Offset *count) @@ -126,14 +149,13 @@ void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start, /** * Look for the largest block of data for io which can be expressed in - * terms of start and count. + * terms of start and count (account for gaps). * * @param arrlen * @param arr_in * @returns the size of the block - * @author Jim Edwards, Ed Hartnett */ -PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) +PIO_Offset GCDblocksize_gaps(int arrlen, const PIO_Offset *arr_in) { int numblks = 0; /* Number of blocks. */ int numtimes = 0; /* Number of times adjacent arr_in elements differ by != 1. */ @@ -144,19 +166,15 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) PIO_Offset bsize; /* Size of the block. */ PIO_Offset bsizeg; /* Size of gap block. */ PIO_Offset blklensum; /* Sum of all block lengths. */ - PIO_Offset *del_arr; /* Array of deltas between adjacent elements in arr_in. */ - PIO_Offset *loc_arr; + PIO_Offset *del_arr = NULL; /* Array of deltas between adjacent elements in arr_in. */ /* Check inputs. */ pioassert(arrlen > 0 && arr_in, "invalid input", __FILE__, __LINE__); - /* Allocate arrays. */ - if (!(loc_arr = malloc(sizeof(PIO_Offset) * (arrlen - 1)))) - return PIO_ENOMEM; - if (!(del_arr = malloc(sizeof(PIO_Offset) * (arrlen - 1)))) + if (arrlen > 1) { - free(loc_arr); - return PIO_ENOMEM; + if (!(del_arr = malloc((arrlen - 1) * sizeof(PIO_Offset)))) + return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); } /* Count the number of contiguous blocks in arr_in. If any if @@ -170,9 +188,9 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) numtimes++; if ( i > 0 && del_arr[i - 1] > 1) { - free(loc_arr); free(del_arr); - return 1; + del_arr = NULL; + return(1); } } } @@ -204,7 +222,13 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) numgaps = ii; } + /* If numblks > 1 then arrlen must be > 1 */ + PIO_Offset *loc_arr = calloc(arrlen - 1, sizeof(PIO_Offset)); + if (!loc_arr) + return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); + j = 0; + /* If numblks > 1 then n must be <= (arrlen - 1) */ for (int i = 0; i < n; i++) loc_arr[i] = 1; @@ -212,13 +236,23 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) if(del_arr[i] != 1) loc_arr[j++] = i; - blk_len[0] = loc_arr[0]; + /* This is handled differently from the Fortran version in PIO1, + * since array index is 1-based in Fortran and 0-based in C. + * Original Fortran code: blk_len(1) = loc_arr(1) + * Converted C code (incorrect): blk_len[0] = loc_arr[0]; + * Converted C code (correct): blk_len[0] = loc_arr[0] + 1; + * For example, if loc_arr[0] is 2, the first block actually + * has 3 elements with indices 0, 1 and 2. */ + blk_len[0] = loc_arr[0] + 1; blklensum = blk_len[0]; + /* If numblks > 1 then numblks must be <= arrlen */ for(int i = 1; i < numblks - 1; i++) { blk_len[i] = loc_arr[i] - loc_arr[i - 1]; blklensum += blk_len[i]; } + free(loc_arr); + loc_arr = NULL; blk_len[numblks - 1] = arrlen - blklensum; /* Get the GCD in blk_len array. */ @@ -239,8 +273,66 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) bsize = lgcd(bsize, arr_in[0]); } - free(loc_arr); free(del_arr); + del_arr = NULL; + + return bsize; +} + +/** + * Look for the largest block of data for io which can be expressed in + * terms of start and count (ignore gaps). + * + * @param arrlen + * @param arr_in + * @returns the size of the block + */ +PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) +{ + /* Check inputs. */ + pioassert(arrlen > 0 && arr_in && arr_in[0] >= 0, "invalid input", __FILE__, __LINE__); + + /* If theres is only one contiguous block with length 1, + * the result must be 1 and we can return. */ + if (arrlen == 1) + return 1; + + /* We can use the array length as the initial value. + * Suppose we have n contiguous blocks with lengths + * b1, b2, ..., bn, then gcd(b1, b2, ..., bn) = + * gcd(b1 + b2 + ... + bn, b1, b2, ..., bn) = + * gcd(arrlen, b1, b2, ..., bn) */ + PIO_Offset bsize = arrlen; + + /* The minimum length of a block is 1. */ + PIO_Offset blk_len = 1; + + for (int i = 0; i < arrlen - 1; i++) + { + pioassert(arr_in[i + 1] >= 0, "invalid input", __FILE__, __LINE__); + + if ((arr_in[i + 1] - arr_in[i]) == 1) + { + /* Still in a contiguous block. */ + blk_len++; + } + else + { + /* The end of a block has been reached. */ + if (blk_len == 1) + return 1; + + bsize = lgcd(bsize, blk_len); + if (bsize == 1) + return 1; + + /* Continue to find next block. */ + blk_len = 1; + } + } + + /* Handle the last block. */ + bsize = lgcd(bsize, blk_len); return bsize; } @@ -259,12 +351,11 @@ PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in) * @param count array of length ndims with data count values. * @param num_aiotasks the number of IO tasks used(?) * @returns 0 for success, error code otherwise. - * @author Jim Edwards */ int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_procs, int myiorank, PIO_Offset *start, PIO_Offset *count, int *num_aiotasks) { - int minbytes; + int minbytes; int maxbytes; int minblocksize; /* Like minbytes, but in data elements. */ int basesize; /* Size in bytes of base data type. */ @@ -431,6 +522,6 @@ int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_proc /* Return the number of IO procs used to the caller. */ *num_aiotasks = use_io_procs; - + return PIO_NOERR; }