Skip to content

Commit

Permalink
more documentation
Browse files Browse the repository at this point in the history
  • Loading branch information
@edhartnett
edhartnett committed May 18, 2019
1 parent d80d0d6 commit 977095f
Show file tree
Hide file tree
Showing 4 changed files with 517 additions and 431 deletions.
120 changes: 65 additions & 55 deletions src/clib/pio_darray.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,22 @@
#include <pio.h>
#include <pio_internal.h>
#include <uthash.h>
/* 10MB default limit. */

/** 10MB default limit. */
PIO_Offset pio_buffer_size_limit = PIO_BUFFER_SIZE;

/* Global buffer pool pointer. */
/** Global buffer pool pointer. */
void *CN_bpool = NULL;

/* Maximum buffer usage. */
/** Maximum buffer usage. */
PIO_Offset maxusage = 0;

/* For write_darray_multi_serial() and write_darray_multi_par() to
* indicate whether fill or data are being written. */
/** For write_darray_multi_serial() and write_darray_multi_par() to
* indicate that fill is being written. */
#define DARRAY_FILL 1

/** For write_darray_multi_serial() and write_darray_multi_par() to
* indicate that data are being written. */
#define DARRAY_DATA 0

/**
Expand All @@ -36,7 +40,8 @@ PIO_Offset maxusage = 0;
* @return The previous limit setting.
* @author Jim Edwards
*/
PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit)
PIO_Offset
PIOc_set_buffer_size_limit(PIO_Offset limit)
{
PIO_Offset oldsize = pio_buffer_size_limit;

Expand Down Expand Up @@ -101,9 +106,10 @@ PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit)
* @ingroup PIO_write_darray
* @author Jim Edwards, Ed Hartnett
*/
int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
PIO_Offset arraylen, void *array, const int *frame,
void **fillvalue, bool flushtodisk)
int
PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
PIO_Offset arraylen, void *array, const int *frame,
void **fillvalue, bool flushtodisk)
{
iosystem_desc_t *ios; /* Pointer to io system information. */
file_desc_t *file; /* Pointer to file information. */
Expand Down Expand Up @@ -145,8 +151,8 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
var_desc_t *vdesc;
if ((ierr = get_var_desc(varids[v], &file->varlist, &vdesc)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);
// if (vdesc->pio_type != iodesc->piotype)
// return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
/* if (vdesc->pio_type != iodesc->piotype)
return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);*/
}

/* Get a pointer to the variable info for the first variable. */
Expand All @@ -162,16 +168,17 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[0], &fndims)))
return check_netcdf(file, ierr, __FILE__, __LINE__);
LOG((3, "called PIOc_inq_varndims varids[0] = %d fndims = %d", varids[0], fndims));
for (int v=1; v < nvars; v++){
if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[v], &fndims2)))
return check_netcdf(file, ierr, __FILE__, __LINE__);
if(fndims != fndims2)
return pio_err(ios, file, PIO_EVARDIMMISMATCH, __FILE__, __LINE__);
}
for (int v=1; v < nvars; v++){
if ((ierr = PIOc_inq_varndims(file->pio_ncid, varids[v], &fndims2)))
return check_netcdf(file, ierr, __FILE__, __LINE__);
if(fndims != fndims2)
return pio_err(ios, file, PIO_EVARDIMMISMATCH, __FILE__, __LINE__);
}

}

/* If async is in use, and this is not an IO task, bcast the parameters. */
/* If async is in use, and this is not an IO task, bcast the
* parameters. */
if (ios->async)
{
if (!ios->ioproc)
Expand Down Expand Up @@ -265,7 +272,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
}
else if (file->iotype == PIO_IOTYPE_PNETCDF && ios->ioproc)
{
/* this assures that iobuf is allocated on all iotasks thus
/* this assures that iobuf is allocated on all iotasks thus
assuring that the flush_output_buffer call above is called
collectively (from all iotasks) */
if (!(file->iobuf = bget(1)))
Expand All @@ -274,13 +281,13 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
}
if (iodesc->needssort)
{
if (!(tmparray = malloc(arraylen*nvars*iodesc->piotype_size)))
return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
pio_sorted_copy(array, tmparray, iodesc, nvars, 0);
if (!(tmparray = malloc(arraylen*nvars*iodesc->piotype_size)))
return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
pio_sorted_copy(array, tmparray, iodesc, nvars, 0);
}
else
{
tmparray = array;
tmparray = array;
}

/* Move data from compute to IO tasks. */
Expand Down Expand Up @@ -314,7 +321,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
/* Release resources. */
if (file->iobuf)
{
LOG((3,"freeing variable buffer in pio_darray"));
LOG((3,"freeing variable buffer in pio_darray"));
brel(file->iobuf);
file->iobuf = NULL;
}
Expand All @@ -335,22 +342,22 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
LOG((2, "nvars = %d holegridsize = %ld iodesc->needsfill = %d\n", nvars,
iodesc->holegridsize, iodesc->needsfill));

pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__);
pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__);

/* Get a buffer. */
if (ios->io_rank == 0)
vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->mpitype_size * nvars);
else if (iodesc->holegridsize > 0)
vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->mpitype_size * nvars);
if (ios->io_rank == 0)
vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->mpitype_size * nvars);
else if (iodesc->holegridsize > 0)
vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->mpitype_size * nvars);

/* copying the fill value into the data buffer for the box
* rearranger. This will be overwritten with data where
* provided. */
if(fillvalue)
for (int nv = 0; nv < nvars; nv++)
for (int i = 0; i < iodesc->holegridsize; i++)
memcpy(&((char *)vdesc0->fillbuf)[iodesc->mpitype_size * (i + nv * iodesc->holegridsize)],
&((char *)fillvalue)[iodesc->mpitype_size * nv], iodesc->mpitype_size);
if(fillvalue)
for (int nv = 0; nv < nvars; nv++)
for (int i = 0; i < iodesc->holegridsize; i++)
memcpy(&((char *)vdesc0->fillbuf)[iodesc->mpitype_size * (i + nv * iodesc->holegridsize)],
&((char *)fillvalue)[iodesc->mpitype_size * nv], iodesc->mpitype_size);

/* Write the darray based on the iotype. */
switch (file->iotype)
Expand Down Expand Up @@ -384,7 +391,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
}

if(iodesc->needssort && tmparray != NULL)
free(tmparray);
free(tmparray);

/* Flush data to disk for pnetcdf. */
if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF)
Expand Down Expand Up @@ -510,7 +517,8 @@ pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_siz
* @ingroup PIO_write_darray
* @author Ed Hartnett
*/
int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
int
find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
{
iosystem_desc_t *ios; /* Pointer to io system information. */
int pio_type;
Expand Down Expand Up @@ -603,8 +611,9 @@ int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
* @ingroup PIO_write_darray
* @author Jim Edwards, Ed Hartnett
*/
int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
void *fillvalue)
int
PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
void *fillvalue)
{
iosystem_desc_t *ios; /* Pointer to io system information. */
file_desc_t *file; /* Info about file we are writing to. */
Expand Down Expand Up @@ -656,10 +665,10 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *

/* If the type of the var doesn't match the type of the
* decomposition, return an error. */
// if (iodesc->piotype != vdesc->pio_type)
// return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
// pioassert(iodesc->mpitype_size == vdesc->mpi_type_size, "wrong mpi info",
// __FILE__, __LINE__);
/* if (iodesc->piotype != vdesc->pio_type) */
/* return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__); */
/* pioassert(iodesc->mpitype_size == vdesc->mpi_type_size, "wrong mpi info", */
/* __FILE__, __LINE__); */

/* If we don't know the fill value for this var, get it. */
if (!vdesc->fillvalue)
Expand All @@ -677,7 +686,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
hashid = ioid*10 + vdesc->rec_var;
HASH_FIND_INT( file->buffer, &hashid, wmb);
if (wmb)
LOG((3, "wmb->ioid = %d wmb->recordvar = %d", wmb->ioid, wmb->recordvar));
LOG((3, "wmb->ioid = %d wmb->recordvar = %d", wmb->ioid, wmb->recordvar));

/* If we did not find an existing wmb entry, create a new wmb. */
if (!wmb)
Expand All @@ -695,8 +704,8 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
wmb->data = NULL;
wmb->frame = NULL;
wmb->fillvalue = NULL;
wmb->htid = hashid;
HASH_ADD_INT( file->buffer, htid, wmb );
wmb->htid = hashid;
HASH_ADD_INT( file->buffer, htid, wmb );
}
LOG((2, "wmb->num_arrays = %d arraylen = %d iodesc->mpitype_size = %d\n",
wmb->num_arrays, arraylen, iodesc->mpitype_size));
Expand Down Expand Up @@ -732,7 +741,7 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
https://github.com/pmodels/mpich/pull/2888 */
io_data_size = (1 + wmb->num_arrays) * iodesc->maxiobuflen * iodesc->mpitype_size;
if(io_data_size > INT_MAX)
needsflush = 2;
needsflush = 2;

/* Tell all tasks on the computation communicator whether we need
* to flush data. */
Expand Down Expand Up @@ -848,8 +857,9 @@ int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *
* @ingroup PIO_read_darray
* @author Jim Edwards, Ed Hartnett
*/
int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
void *array)
int
PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
void *array)
{
iosystem_desc_t *ios; /* Pointer to io system information. */
file_desc_t *file; /* Pointer to file information. */
Expand Down Expand Up @@ -899,22 +909,22 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,

if (iodesc->needssort)
{
if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen)))
return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
for(int m=0; m<iodesc->maplen;m++)
((int *) array)[m] = -1;
if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen)))
return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
for(int m=0; m<iodesc->maplen;m++)
((int *) array)[m] = -1;
}
else
tmparray = array;
tmparray = array;

/* Rearrange the data. */
if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, tmparray)))
return pio_err(ios, file, ierr, __FILE__, __LINE__);

if (iodesc->needssort)
{
pio_sorted_copy(tmparray, array, iodesc, 1, 1);
free(tmparray);
pio_sorted_copy(tmparray, array, iodesc, 1, 1);
free(tmparray);
}

/* Free the buffer. */
Expand Down
Loading

0 comments on commit 977095f

Please sign in to comment.