diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c index 4827dea6a7e..5a1f7bc845b 100644 --- a/src/clib/pio_darray_int.c +++ b/src/clib/pio_darray_int.c @@ -13,6 +13,12 @@ #include #include +#if USE_VARD +#define USE_VARD_READ 1 +#define USE_VARD_WRITE 1 +#endif + + /* 10MB default limit. */ extern PIO_Offset pio_buffer_size_limit; @@ -100,16 +106,14 @@ int get_vard_mpidatatype(io_desc_t *iodesc, MPI_Offset gdim0, PIO_Offset unlimdi { int sa_ndims; - int sacount[fndims]; - int sastart[fndims]; int gdims[fndims]; int dim_offset; int mpierr; MPI_Aint displacements[rrcnt]; int blocklengths[rrcnt]; - if(gdim0) - sacount[0] = 1; + if(rrcnt == 0) + return PIO_NOERR; for ( int rc=0; rc 0) { + unlimdimoffset = gdim0; sastart[0] = max(0, frame); displacements[rc]=0; } else + { + sacount[0] = 1; displacements[rc] = unlimdimoffset * max(0, frame); + } + for (int i=dim_offset; i< fndims; i++) + { + sacount[i-dim_offset] = (int) countlist[rc][i]; + sastart[i-dim_offset] = (int) startlist[rc][i]; + } + #if PIO_ENABLE_LOGGING for (int i=0; i< sa_ndims; i++) - LOG((3, "vard: sastart[%d]=%d sacount[%d]=%d gdims[%d]=%d %ld %ld", - i,sastart[i], i,sacount[i], i, gdims[i], startlist[rc][i], countlist[rc][i])); + LOG((3, "vard: sastart[%d]=%d sacount[%d]=%d gdims[%d]=%d %ld %ld displacement = %ld un %d", + i,sastart[i], i,sacount[i], i, gdims[i], startlist[rc][i], countlist[rc][i], displacements[rc], unlimdimoffset)); #endif if((mpierr = MPI_Type_create_subarray(sa_ndims, gdims, sacount, sastart,MPI_ORDER_C @@ -190,7 +201,7 @@ int get_vard_mpidatatype(io_desc_t *iodesc, MPI_Offset gdim0, PIO_Offset unlimdi if((mpierr = MPI_Type_commit(filetype))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); - + return PIO_NOERR; } #endif @@ -296,8 +307,9 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * var_desc_t *vdesc; /* Pointer to var info struct. */ int dsize; /* Data size (for one region). */ int ierr = PIO_NOERR; -#if USE_VARD +#if USE_VARD_WRITE PIO_Offset gdim0; /* global size of first dimension if no unlimited dimension and ndimsiosystem && varids && varids[0] >= 0 && varids[0] <= PIO_MAX_VARS && @@ -325,8 +337,9 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * PIO_Offset llen = fill ? iodesc->holegridsize : iodesc->llen; void *iobuf = fill ? vdesc->fillbuf : file->iobuf; -#if USE_VARD - ierr = get_gdim0(file, iodesc, varids[0], fndims, &gdim0); +#if USE_VARD_WRITE + if (!ios->async || !ios->ioproc) + ierr = get_gdim0(file, iodesc, varids[0], fndims, &gdim0); #endif /* If this is an IO task write the data. */ @@ -339,22 +352,6 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * int ndims = iodesc->ndims; PIO_Offset *startlist[num_regions]; /* Array of start arrays for ncmpi_iput_varn(). */ PIO_Offset *countlist[num_regions]; /* Array of count arrays for ncmpi_iput_varn(). */ -#if USE_VARD - PIO_Offset unlimdimoffset; - int mpierr; - MPI_Datatype filetype; - MPI_Datatype subarray[num_regions]; - filetype = MPI_DATATYPE_NULL; - - if (file->iotype == PIO_IOTYPE_PNETCDF && gdim0 == 0) /* if there is an unlimited dimension get the offset between records of a variable */ - { -#ifdef _PNETCDF - if((ierr = ncmpi_inq_recsize(file->fh, &unlimdimoffset))) - return pio_err(NULL, file, ierr, __FILE__, __LINE__); -#endif - } -#endif - LOG((3, "num_regions = %d", num_regions)); /* Process each region of data to be written. */ for (int regioncnt = 0; regioncnt < num_regions; regioncnt++) @@ -423,6 +420,10 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * /* Do this when we reach the last region. */ if (regioncnt == num_regions - 1) { +#ifdef USE_VARD_WRITE + MPI_Datatype filetype = MPI_DATATYPE_NULL; + MPI_Datatype subarray[rrcnt]; +#endif /* For each variable to be written. */ for (int nv = 0; nv < nvars; nv++) { @@ -430,10 +431,18 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * if ((ierr = get_var_desc(varids[nv], &file->varlist, &vdesc))) return pio_err(NULL, file, ierr, __FILE__, __LINE__); -#if USE_VARD +#if USE_VARD_WRITE /* If this is the first variable or the frame has changed between variables (this should be rare) */ if(nv==0 || (nv > 0 && frame != NULL && frame[nv] != frame[nv-1])){ int thisframe; + PIO_Offset unlimdimoffset; + if (gdim0 == 0) /* if there is an unlimited dimension get the offset between records of a variable */ + { + if((ierr = ncmpi_inq_recsize(file->fh, &unlimdimoffset))) + return pio_err(NULL, file, ierr, __FILE__, __LINE__); + LOG((3, "num_regions = %d unlimdimoffset %ld", num_regions, unlimdimoffset)); + }else + unlimdimoffset = gdim0; if (frame) thisframe = frame[nv]; else @@ -461,10 +470,20 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int * vdesc->request[i] = NC_REQ_NULL; } -#if USE_VARD - LOG((3, "vard: call ncmpi_put_vard llen = %d %d ", llen, iodesc->mpitype_size )); +#if USE_VARD_WRITE + LOG((3, "vard: call ncmpi_put_vard llen = %d %d", llen, iodesc->mpitype_size )); ierr = ncmpi_put_vard_all(file->fh, varids[nv], filetype, bufptr, llen, iodesc->mpitype); LOG((3, "vard: return ncmpi_put_vard ierr = %d", ierr)); + if(filetype != MPI_DATATYPE_NULL) + { + int mpierr; + for(int i=0; inext; } /* next regioncnt */ -#if USE_VARD - if(filetype != MPI_DATATYPE_NULL) - { - int mpierr; - for(int i=0; iioproc) */ /* Check the return code from the netCDF/pnetcdf call. */ @@ -946,6 +953,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu int ierr; /* Return code from netCDF functions. */ #ifdef USE_VARD_READ MPI_Offset gdim0; + gdim0 = 0; #endif /* Check inputs. */ @@ -971,7 +979,8 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims))) return pio_err(ios, file, ierr, __FILE__, __LINE__); #if USE_VARD_READ - ierr = get_gdim0(file, iodesc, vid, fndims, &gdim0); + if(!ios->async || !ios->ioproc) + ierr = get_gdim0(file, iodesc, vid, fndims, &gdim0); #endif /* IO procs will read the data. */ @@ -986,20 +995,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu PIO_Offset *startlist[iodesc->maxregions]; PIO_Offset *countlist[iodesc->maxregions]; -#if USE_VARD_READ - PIO_Offset unlimdimoffset; - int mpierr; - MPI_Datatype filetype; - MPI_Datatype subarray[iodesc->maxregions]; - filetype = MPI_DATATYPE_NULL; - - if (gdim0 == 0) /* if there is an unlimited dimension get the offset between records of a variable */ - { - if((ierr = ncmpi_inq_recsize(file->fh, &unlimdimoffset))) - return pio_err(NULL, file, ierr, __FILE__, __LINE__); - } -#endif - /* buffer is incremented by byte and loffset is in terms of the iodessc->mpitype so we need to multiply by the size of the mpitype. */ @@ -1146,6 +1141,16 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobu { #if USE_VARD_READ MPI_Datatype filetype, subarray[rrlen]; + PIO_Offset unlimdimoffset; + int mpierr; + if (gdim0 == 0) /* if there is an unlimited dimension get the offset between records of a variable */ + { + if((ierr = ncmpi_inq_recsize(file->fh, &unlimdimoffset))) + return pio_err(NULL, file, ierr, __FILE__, __LINE__); + } + else + unlimdimoffset = gdim0; + filetype = MPI_DATATYPE_NULL; for(int i=0; ifh, &usage))) /* allow the buffer to be undefined */ @@ -1549,9 +1554,10 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) if (usage > maxusage) maxusage = usage; + LOG((2, "flush_output_buffer usage=%ld force=%d",usage, force)); /* If the user forces it, or the buffer has exceeded the size * limit, then flush to disk. */ - if (force || usage >= pio_buffer_size_limit) + if (force || (usage >= pio_buffer_size_limit)) { int rcnt; int maxreq; diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c index 250d77b8781..dfa269df57c 100644 --- a/src/clib/pio_getput_int.c +++ b/src/clib/pio_getput_int.c @@ -544,7 +544,7 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off * num_elem will remain 1). */ for (int vd = 0; vd < ndims; vd++) num_elem *= count[vd]; - LOG((2, "PIOc_get_vars_tc num_elem = %d", num_elem)); + LOG((1, "PIOc_get_vars_tc num_elem = %d", num_elem)); } /* If async is in use, and this is not an IO task, bcast the parameters. */ @@ -583,7 +583,7 @@ int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); if (!mpierr) mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm); - LOG((2, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d start_present = %d " + LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d start_present = %d " "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid, ndims, start_present, count_present, stride_present, xtype, num_elem)); } @@ -937,13 +937,17 @@ int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Off return check_netcdf(file, ierr, __FILE__, __LINE__); } - LOG((2, "ndims = %d typelen = %d", ndims, typelen)); - /* How many elements of data? If no count array was passed, * this is a scalar. */ if (count) for (int vd = 0; vd < ndims; vd++) num_elem *= count[vd]; + + LOG((1, "ndims = %d typelen = %d num_elem %d", ndims, typelen, num_elem)); + if (count) + for (int vd = 0; vd < ndims; vd++) + LOG((1, "count[%d] = %d",vd,count[vd])); + } /* If async is in use, and this is not an IO task, bcast the parameters. */ diff --git a/src/clib/pioc.c b/src/clib/pioc.c index 1f83c597e02..f375621408e 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -121,7 +121,7 @@ int PIOc_advanceframe(int ncid, int varid) int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ int ret; - LOG((1, "PIOc_advanceframe ncid = %d varid = %d")); + LOG((1, "PIOc_advanceframe ncid = %d varid = %d", ncid, varid)); /* Get the file info. */ if ((ret = pio_get_file(ncid, &file))) @@ -877,7 +877,7 @@ int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int bas /* Create a group for the IO tasks. */ if ((mpierr = MPI_Group_incl(compgroup, ios->num_iotasks, ios->ioranks, &iogroup))) - return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); + return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); /* Create an MPI communicator for the IO tasks. */ if ((mpierr = MPI_Comm_create(ios->comp_comm, iogroup, &ios->io_comm))) @@ -1287,7 +1287,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, /* Determine which tasks to use for each computational component. */ if ((ret = determine_procs(num_io_procs, component_count, num_procs_per_comp, proc_list, my_proc_list))) - return pio_err(NULL, NULL, ret, __FILE__, __LINE__); + return pio_err(NULL, NULL, ret, __FILE__, __LINE__); /* Get rank of this task in world. */ if ((ret = MPI_Comm_rank(world, &my_rank))) @@ -1519,7 +1519,7 @@ int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, if ((ret = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank))) return check_mpi(NULL, ret, __FILE__, __LINE__); LOG((3, "my_iosys->union_rank %d", my_iosys->union_rank)); - + /* Set my_comm to union_comm for async. */ my_iosys->my_comm = my_iosys->union_comm; LOG((3, "intracomm created for union cmp = %d union_rank = %d union_comm = %d", diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index 3c8832ee7e9..dde1cf3a678 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -228,6 +228,12 @@ void pio_log(int severity, const char *fmt, ...) ptr += strlen(rank_str); rem_len -= strlen(rank_str); + /* Show the severity. */ + snprintf(rank_str, MAX_RANK_STR, ":%d ", severity); + strncpy(ptr, rank_str, (rem_len > 0) ? rem_len : 0); + ptr += strlen(rank_str); + rem_len -= strlen(rank_str); + /* Print out the variable list of args with vprintf. */ va_start(argp, fmt); vsnprintf(ptr, ((rem_len > 0) ? rem_len : 0), fmt, argp); diff --git a/tests/cunit/test_darray.c b/tests/cunit/test_darray.c index aaca7d88d61..0b6c9a033f9 100644 --- a/tests/cunit/test_darray.c +++ b/tests/cunit/test_darray.c @@ -341,7 +341,7 @@ int main(int argc, char **argv) /* Initialize test. */ if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, - MIN_NTASKS, -1, &test_comm))) + MIN_NTASKS, 3, &test_comm))) ERR(ERR_INIT); if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) diff --git a/tests/cunit/test_darray_async.c b/tests/cunit/test_darray_async.c index 9d493fc51eb..199ccad6e91 100644 --- a/tests/cunit/test_darray_async.c +++ b/tests/cunit/test_darray_async.c @@ -104,19 +104,19 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, switch (piotype) { case PIO_BYTE: - if (((signed char *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + if (((signed char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) ERR(ret); break; case PIO_CHAR: - if (((char *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + if (((char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) ERR(ret); break; case PIO_SHORT: - if (((short *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + if (((short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) ERR(ret); break; case PIO_INT: - if (((int *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + if (((int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) ERR(ret); break; case PIO_FLOAT: @@ -129,23 +129,23 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, break; #ifdef _NETCDF4 case PIO_UBYTE: - if (((unsigned char *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + if (((unsigned char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) ERR(ret); break; case PIO_USHORT: - if (((unsigned short *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + if (((unsigned short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) ERR(ret); break; case PIO_UINT: - if (((unsigned int *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + if (((unsigned int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) ERR(ret); break; case PIO_INT64: - if (((long long *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + if (((long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) ERR(ret); break; case PIO_UINT64: - if (((unsigned long long *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + if (((unsigned long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) ERR(ret); break; #endif /* _NETCDF4 */