Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support decompositions with extra outermost dimensions #113

Merged
merged 2 commits into from
Aug 1, 2018
Merged

Support decompositions with extra outermost dimensions #113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
603d2f7
Adding tests for extra dimensions in the decomposition
dqwu Jun 13, 2018
d0163d2
Handling extra dimensions in the decomposition
dqwu Jun 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
159 changes: 90 additions & 69 deletions src/clib/pio_darray_int.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,13 @@ int compute_buffer_init(iosystem_desc_t *ios)

/**
* Fill start/count arrays for write_darray_multi_par(). This is an
* internal funciton.
* internal function.
*
* @param ndims the number of dims in the decomposition.
* @param dimlen the lengths of dims in the decomposition.
* @param fndims the number of dims in the file.
* @param vdesc pointer to the var_desc_t info.
* @param region pointer to a region.
* @param frame array of record values.
* @param start an already-allocated array which gets the start
* values.
* @param count an already-allocated array which gets the count
Expand All @@ -90,9 +90,8 @@ int compute_buffer_init(iosystem_desc_t *ios)
* @ingroup PIO_write_darray
* @author Ed Hartnett
*/
int find_start_count(int ndims, int fndims, var_desc_t *vdesc,
io_region *region, const int *frame, size_t *start,
size_t *count)
int find_start_count(int ndims, const int *dimlen, int fndims, var_desc_t *vdesc,
io_region *region, size_t *start, size_t *count)
{
/* Init start/count arrays to zero. */
for (int i = 0; i < fndims; i++)
Expand All @@ -103,42 +102,44 @@ int find_start_count(int ndims, int fndims, var_desc_t *vdesc,

if (region)
{
if (vdesc->record >= 0)
/* Allow extra outermost dimensions in the decomposition */
int num_extra_dims = (vdesc->record >= 0 && fndims > 1)? (ndims - (fndims - 1)) : (ndims - fndims);
pioassert(num_extra_dims >= 0, "Unexpected num_extra_dims", __FILE__, __LINE__);
if (num_extra_dims > 0)
{
for (int d = 0; d < num_extra_dims; d++)
pioassert(dimlen[d] == 1, "Extra outermost dimensions must have lengths of 1",
__FILE__, __LINE__);
}

if (vdesc->record >= 0 && fndims > 1)
{
/* This is a record based multidimensional
* array. Figure out start/count for all but the
* record dimension (dimid 0). */
for (int i = fndims - ndims; i < fndims; i++)
for (int i = 1; i < fndims; i++)
{
start[i] = region->start[i - (fndims - ndims)];
count[i] = region->count[i - (fndims - ndims)];
start[i] = region->start[num_extra_dims + (i - 1)];
count[i] = region->count[num_extra_dims + (i - 1)];
}

/* Now figure out start/count for record dimension. */
if (fndims > 1 && ndims < fndims && count[1] > 0)
{
/* Set count for record dimension (start cannot be determined so far). */
if (count[1] > 0)
count[0] = 1;
start[0] = frame[0];
}
else if (fndims == ndims)
{
/* ??? */
start[0] += vdesc->record;
}
}
else
{
/* This is a non record variable. */
for (int i = 0; i < ndims; i++)
for (int i = 0; i < fndims; i++)
{
start[i] = region->start[i];
count[i] = region->count[i];
start[i] = region->start[num_extra_dims + i];
count[i] = region->count[num_extra_dims + i];
}
}

#if PIO_ENABLE_LOGGING
/* Log arrays for debug purposes. */
for (int i = 0; i < ndims; i++)
for (int i = 0; i < fndims; i++)
LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i]));
#endif /* PIO_ENABLE_LOGGING */
}
Expand Down Expand Up @@ -214,7 +215,7 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
for (int regioncnt = 0; regioncnt < num_regions; regioncnt++)
{
/* Fill the start/count arrays. */
if ((ierr = find_start_count(iodesc->ndims, fndims, vdesc, region, frame, start, count)))
if ((ierr = find_start_count(iodesc->ndims, iodesc->dimlen, fndims, vdesc, region, start, count)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);

/* IO tasks will run the netCDF/pnetcdf functions to write the data. */
Expand All @@ -225,9 +226,8 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
/* For each variable to be written. */
for (int nv = 0; nv < nvars; nv++)
{
/* Set the start of the record dimension. (Hasn't
* this already been set above ???) */
if (vdesc->record >= 0 && ndims < fndims)
/* Set the start of the record dimension. */
if (vdesc->record >= 0 && fndims > 1)
start[0] = frame[nv];

/* If there is data for this region, get a pointer to it. */
Expand Down Expand Up @@ -329,9 +329,9 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
/* Get the var info. */
vdesc = file->varlist + varids[nv];

/* If this is a record var, set the start for
/* If this is a record (or quasi-record) var, set the start for
* the record dimension. */
if (vdesc->record >= 0 && ndims < fndims)
if (vdesc->record >= 0 && fndims > 1)
for (int rc = 0; rc < rrcnt; rc++)
startlist[rc][0] = frame[nv];

Expand Down Expand Up @@ -402,6 +402,7 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
* @param maxregions the number of regions in the list.
* @param fndims the number of dimensions in the file.
* @param iodesc_ndims the number of dimensions in the decomposition.
* @param dimlen the lengths of dimensions in the decomposition.
* @param vdesc pointer to an array of var_desc_t for the vars being
* written.
* @param tmp_start pointer to an already allocaed array of length
Expand All @@ -415,8 +416,8 @@ int write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *
* @author Jim Edwards, Ed Hartnett
**/
int find_all_start_count(io_region *region, int maxregions, int fndims,
int iodesc_ndims, var_desc_t *vdesc, size_t *tmp_start,
size_t *tmp_count)
int iodesc_ndims, const int *dimlen, var_desc_t *vdesc,
size_t *tmp_start, size_t *tmp_count)
{
/* Check inputs. */
pioassert(maxregions >= 0 && fndims > 0 && iodesc_ndims >= 0 && vdesc &&
Expand All @@ -434,15 +435,25 @@ int find_all_start_count(io_region *region, int maxregions, int fndims,

if (region)
{
if (vdesc->record >= 0)
/* Allow extra outermost dimensions in the decomposition */
int num_extra_dims = (vdesc->record >= 0 && fndims > 1)? (iodesc_ndims - (fndims - 1)) : (iodesc_ndims - fndims);
pioassert(num_extra_dims >= 0, "Unexpected num_extra_dims", __FILE__, __LINE__);
if (num_extra_dims > 0)
{
for (int d = 0; d < num_extra_dims; d++)
pioassert(dimlen[d] == 1, "Extra outermost dimensions must have lengths of 1",
__FILE__, __LINE__);
}

if (vdesc->record >= 0 && fndims > 1)
{
/* This is a record based multidimensional
* array. Copy start/count for non-record
* dimensions. */
for (int i = fndims - iodesc_ndims; i < fndims; i++)
for (int i = 1; i < fndims; i++)
{
tmp_start[i + r * fndims] = region->start[i - (fndims - iodesc_ndims)];
tmp_count[i + r * fndims] = region->count[i - (fndims - iodesc_ndims)];
tmp_start[i + r * fndims] = region->start[num_extra_dims + (i - 1)];
tmp_count[i + r * fndims] = region->count[num_extra_dims + (i - 1)];
LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims,
tmp_start[i + r * fndims], i + r * fndims,
tmp_count[i + r * fndims]));
Expand All @@ -451,10 +462,10 @@ int find_all_start_count(io_region *region, int maxregions, int fndims,
else
{
/* This is not a record based multidimensional array. */
for (int i = 0; i < iodesc_ndims; i++)
for (int i = 0; i < fndims; i++)
{
tmp_start[i + r * fndims] = region->start[i];
tmp_count[i + r * fndims] = region->count[i];
tmp_start[i + r * fndims] = region->start[num_extra_dims + i];
tmp_count[i + r * fndims] = region->count[num_extra_dims + i];
LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims,
tmp_start[i + r * fndims], i + r * fndims,
tmp_count[i + r * fndims]));
Expand Down Expand Up @@ -650,25 +661,21 @@ int recv_and_write_data(file_desc_t *file, const int *varids, const int *frame,
for (int nv = 0; nv < nvars; nv++)
{
LOG((3, "writing buffer var %d", nv));
vdesc = file->varlist + varids[0];
vdesc = file->varlist + varids[nv];

/* Get a pointer to the correct part of the buffer. */
bufptr = (void *)((char *)iobuf + iodesc->mpitype_size * (nv * rlen + loffset));

/* If this var has an unlimited dim, set
/* If this var has a record dim, set
* the start on that dim to the frame
* value for this variable. */
if (vdesc->record >= 0)
if (vdesc->record >= 0 && fndims > 1)
{
if (fndims > 1 && iodesc->ndims < fndims && count[1] > 0)
if (count[1] > 0)
{
count[0] = 1;
start[0] = frame[nv];
}
else if (fndims == iodesc->ndims)
{
start[0] += vdesc->record;
}
}

/* Call the netCDF functions to write the data. */
Expand Down Expand Up @@ -804,7 +811,7 @@ int write_darray_multi_serial(file_desc_t *file, int nvars, int fndims, const in

/* Fill the tmp_start and tmp_count arrays, which contain the
* start and count arrays for all regions. */
if ((ierr = find_all_start_count(region, num_regions, fndims, iodesc->ndims, vdesc,
if ((ierr = find_all_start_count(region, num_regions, fndims, iodesc->ndims, iodesc->dimlen, vdesc,
tmp_start, tmp_count)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);

Expand Down Expand Up @@ -879,10 +886,6 @@ int pio_read_darray_nc(file_desc_t *file, int fndims, io_desc_t *iodesc, int vid
/* Get the number of dimensions in the decomposition. */
ndims = iodesc->ndims;

/* Is this a non-record var? */
if (fndims == ndims)
vdesc->record = -1;

/* IO procs will actially read the data. */
if (ios->ioproc)
{
Expand All @@ -900,10 +903,10 @@ int pio_read_darray_nc(file_desc_t *file, int fndims, io_desc_t *iodesc, int vid
the mpitype. */
region = iodesc->firstregion;

/* ??? */
/* This is a record (or quasi-record) var. If the record
number has not been set yet, set it to 0 by default */
if (fndims > ndims)
{
ndims++;
if (vdesc->record < 0)
vdesc->record = 0;
}
Expand Down Expand Up @@ -932,16 +935,26 @@ int pio_read_darray_nc(file_desc_t *file, int fndims, io_desc_t *iodesc, int vid

LOG((2, "%d %d %d", iodesc->llen - region->loffset, iodesc->llen, region->loffset));

/* Allow extra outermost dimensions in the decomposition */
int num_extra_dims = (vdesc->record >= 0 && fndims > 1)? (ndims - (fndims - 1)) : (ndims - fndims);
pioassert(num_extra_dims >= 0, "Unexpected num_extra_dims", __FILE__, __LINE__);
if (num_extra_dims > 0)
{
for (int d = 0; d < num_extra_dims; d++)
pioassert(iodesc->dimlen[d] == 1, "Extra outermost dimensions must have lengths of 1",
__FILE__, __LINE__);
}

/* Get the start/count arrays. */
if (vdesc->record >= 0 && fndims > 1)
{
/* This is a record var. The unlimited dimension
/* This is a record (or quasi-record) var. The record dimension
* (0) is handled specially. */
start[0] = vdesc->record;
for (int i = 1; i < ndims; i++)
for (int i = 1; i < fndims; i++)
{
start[i] = region->start[i-1];
count[i] = region->count[i-1];
start[i] = region->start[num_extra_dims + (i - 1)];
count[i] = region->count[num_extra_dims + (i - 1)];
}

/* Read one record. */
Expand All @@ -951,10 +964,10 @@ int pio_read_darray_nc(file_desc_t *file, int fndims, io_desc_t *iodesc, int vid
else
{
/* Non-time dependent array */
for (int i = 0; i < ndims; i++)
for (int i = 0; i < fndims; i++)
{
start[i] = region->start[i];
count[i] = region->count[i];
start[i] = region->start[num_extra_dims + i];
count[i] = region->count[num_extra_dims + i];
}
}
}
Expand Down Expand Up @@ -1114,10 +1127,6 @@ int pio_read_darray_nc_serial(file_desc_t *file, int fndims, io_desc_t *iodesc,
/* Get the number of dims in our decomposition. */
ndims = iodesc->ndims;

/* Is this a non-record var? */
if (fndims == ndims)
vdesc->record = -1;

if (ios->ioproc)
{
io_region *region;
Expand All @@ -1133,6 +1142,8 @@ int pio_read_darray_nc_serial(file_desc_t *file, int fndims, io_desc_t *iodesc,
the mpitype. */
region = iodesc->firstregion;

/* This is a record (or quasi-record) var. If the record
number has not been set yet, set it to 0 by default */
if (fndims > ndims)
{
if (vdesc->record < 0)
Expand All @@ -1154,16 +1165,26 @@ int pio_read_darray_nc_serial(file_desc_t *file, int fndims, io_desc_t *iodesc,
}
else
{
/* Allow extra outermost dimensions in the decomposition */
int num_extra_dims = (vdesc->record >= 0 && fndims > 1)? (ndims - (fndims - 1)) : (ndims - fndims);
pioassert(num_extra_dims >= 0, "Unexpected num_extra_dims", __FILE__, __LINE__);
if (num_extra_dims > 0)
{
for (int d = 0; d < num_extra_dims; d++)
pioassert(iodesc->dimlen[d] == 1, "Extra outermost dimensions must have lengths of 1",
__FILE__, __LINE__);
}

if (vdesc->record >= 0 && fndims > 1)
{
/* This is a record var. Find start for record dims. */
/* This is a record (or quasi-record) var. Find start for record dims. */
tmp_start[regioncnt * fndims] = vdesc->record;

/* Find start/count for all non-record dims. */
for (int i = 1; i < fndims; i++)
{
tmp_start[i + regioncnt * fndims] = region->start[i - 1];
tmp_count[i + regioncnt * fndims] = region->count[i - 1];
tmp_start[i + regioncnt * fndims] = region->start[num_extra_dims + (i - 1)];
tmp_count[i + regioncnt * fndims] = region->count[num_extra_dims + (i - 1)];
}

/* Set count for record dimension. */
Expand All @@ -1175,8 +1196,8 @@ int pio_read_darray_nc_serial(file_desc_t *file, int fndims, io_desc_t *iodesc,
/* Non-time dependent array */
for (int i = 0; i < fndims; i++)
{
tmp_start[i + regioncnt * fndims] = region->start[i];
tmp_count[i + regioncnt * fndims] = region->count[i];
tmp_start[i + regioncnt * fndims] = region->start[num_extra_dims + i];
tmp_count[i + regioncnt * fndims] = region->count[num_extra_dims + i];
}
}
}
Expand Down
20 changes: 20 additions & 0 deletions tests/general/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ SET(GENERATED_SRCS pio_file_simple_tests.F90
pio_decomp_frame_tests.F90
pio_decomp_fillval.F90
pio_decomp_fillval2.F90
pio_decomp_extra_dims.F90
pio_iosystem_tests.F90
pio_iosystem_tests2.F90
pio_iosystem_tests3.F90)
Expand Down Expand Up @@ -832,6 +833,25 @@ else ()
TIMEOUT ${DEFAULT_TEST_TIMEOUT})
endif ()

#===== pio_decomp_extra_dims =====
add_executable (pio_decomp_extra_dims EXCLUDE_FROM_ALL
pio_decomp_extra_dims.F90
${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
target_link_libraries (pio_decomp_extra_dims piof)
add_dependencies (tests pio_decomp_extra_dims)

if (PIO_USE_MPISERIAL)
add_test(NAME pio_decomp_extra_dims
COMMAND pio_decomp_extra_dims)
set_tests_properties(pio_decomp_extra_dims
PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
else ()
add_mpi_test(pio_decomp_extra_dims
EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_extra_dims
NUMPROCS 4
TIMEOUT ${DEFAULT_TEST_TIMEOUT})
endif ()

#===== pio_iosystems_test =====
add_executable (pio_iosystem_tests EXCLUDE_FROM_ALL
pio_iosystem_tests.F90
Expand Down
Loading