Skip to content

Commit

Permalink
Merge pull request #186 from NCAR/ejh_darray4
Browse files Browse the repository at this point in the history 
Comments for PIOc_write_darray()
  • Loading branch information
@edhartnett
edhartnett authored Nov 10, 2016
2 parents 6eb27a9 + 87dfc72 commit 410f34d
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 68 deletions.
105 changes: 66 additions & 39 deletions src/clib/pio_darray.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,21 +246,22 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid,
const PIO_Offset arraylen, void *array, void *fillvalue)
{
iosystem_desc_t *ios; /* Pointer to io system information. */
file_desc_t *file;
io_desc_t *iodesc;
var_desc_t *vdesc;
void *bufptr;
size_t rlen;
MPI_Datatype vtype;
wmulti_buffer *wmb;
int tsize; /* Total size. */
int *tptr;
void *bptr;
void *fptr;
bool recordvar; /* True if this is a record variable. */
file_desc_t *file; /* Info about file we are writing to. */
io_desc_t *iodesc; /* The IO description. */
var_desc_t *vdesc; /* Info about the var being written. */
void *bufptr; /* A data buffer. */
MPI_Datatype vtype; /* The MPI type of the variable. */
wmulti_buffer *wmb; /* A data buffer. */
int tsize; /* Size of MPI type. */
bool recordvar; /* True if this is a record variable. */
int needsflush = 0; /* True if we need to flush buffer. */
bufsize totfree, maxfree;
bufsize totfree; /* Amount of free space in the buffer. */
bufsize maxfree; /* Max amount of free space in buffer. */
int ierr = PIO_NOERR; /* Return code. */
int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */

LOG((1, "PIOc_write_darray ncid = %d vid = %d ioid = %d arraylen = %d",
ncid, vid, ioid, arraylen));

/* Get the file info. */
if ((ierr = pio_get_file(ncid, &file)))
Expand All @@ -282,10 +283,16 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid,
/* Is this a record variable? */
recordvar = vdesc->record >= 0 ? true : false;

/* Check that the local size of the variable passed in matches the
* size expected by the io descriptor. */
if (iodesc->ndof != arraylen)
piodie("ndof != arraylen",__FILE__,__LINE__);

/* Get a pointer to the buffer space for this file. */
wmb = &file->buffer;

/* If the ioid is not initialized, set it. For non record vars,
* use the negative?? */
if (wmb->ioid == -1)
{
if (recordvar)
Expand All @@ -298,6 +305,10 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid,
/* separate record and non-record variables */
if (recordvar)
{
/* wmb is write multi buffer and is a pointer to one or
* more variables that fit the same description. We are
* moving to the end of the wmb linked list to add the
* current variable. */
while(wmb->next && wmb->ioid != ioid)
if (wmb->next)
wmb = wmb->next;
Expand All @@ -317,12 +328,14 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid,
}
}

/* ?? */
if ((recordvar && wmb->ioid != ioid) || (!recordvar && wmb->ioid != -(ioid)))
{
wmb->next = (wmulti_buffer *)bget((bufsize) sizeof(wmulti_buffer));
if (!wmb->next)
/* Allocate a buffer. */
if (!(wmb->next = (wmulti_buffer *)bget((bufsize)sizeof(wmulti_buffer))))
piomemerror(*ios,sizeof(wmulti_buffer), __FILE__,__LINE__);

/* Set pointer to newly allocated buffer and initialize.*/
wmb = wmb->next;
wmb->next = NULL;
if (recordvar)
Expand All @@ -337,22 +350,33 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid,
wmb->fillvalue = NULL;
}

MPI_Type_size(iodesc->basetype, &tsize);
/* Get the size of the MPI type. */
if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize)))
return check_mpi(file, mpierr, __FILE__, __LINE__);

LOG((2, "%d %d %d\n", wmb->data, wmb->validvars, arraylen,tsize));
LOG((2, "wmb->validvars = %d arraylen = %d tsize = %d\n", wmb->validvars,
arraylen, tsize));

/* At this point wmb should be pointing to a new or existing buffer
so we can add the data */
so we can add the data. */
bfreespace(&totfree, &maxfree);

/* ??? */
if (needsflush == 0)
needsflush = (maxfree <= 1.1 * (1 + wmb->validvars) * arraylen * tsize);
MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, ios->comp_comm);

/* Tell all tests on the computation communicator whether we need
* to flush data. */
if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1, MPI_INT, MPI_MAX, ios->comp_comm)))
return check_mpi(file, mpierr, __FILE__, __LINE__);

/* Flush data if needed. */
if (needsflush > 0)
{
/* need to flush first */
LOG((2, "%ld %d %ld %ld\n", maxfree, wmb->validvars,
(1 + wmb->validvars) * arraylen * tsize, totfree));
LOG((2, "maxfree = %ld wmb->validvars = %d (1 + wmb->validvars) * arraylen * tsize = %ld totfree = %ld\n",
maxfree, wmb->validvars, (1 + wmb->validvars) * arraylen * tsize, totfree));

/* Collect a debug report about buffer. (Shouldn't we be able to turn this off??) */
cn_buffer_report(*ios, true);

/* If needsflush == 2 flush to disk otherwise just flush to io node. */
Expand All @@ -361,34 +385,41 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid,

/* Get memory for data. */
if (arraylen > 0)
if (!(wmb->data = bgetr(wmb->data, (1+wmb->validvars)*arraylen*tsize)))
if (!(wmb->data = bgetr(wmb->data, (1 + wmb->validvars) * arraylen * tsize)))
piomemerror(*ios, (1 + wmb->validvars) * arraylen * tsize, __FILE__, __LINE__);

/* Get memory for ??? */
/* vid is an array of variable ids in the wmb list, grow the list
* and add the new entry. */
if (!(wmb->vid = (int *)bgetr(wmb->vid, sizeof(int) * (1 + wmb->validvars))))
piomemerror(*ios, (1 + wmb->validvars) * sizeof(int), __FILE__, __LINE__);

/* wmb->frame is the record number, we assume that the variables
* in the wmb list may not all have the same unlimited dimension
* value although they usually do. */
if (vdesc->record >= 0)
if (!(wmb->frame = (int *)bgetr(wmb->frame, sizeof(int) * (1 + wmb->validvars))))
piomemerror(*ios, (1 + wmb->validvars) * sizeof(int), __FILE__, __LINE__);

/* Get memory to hold fill value. */
if (iodesc->needsfill)
if (!(wmb->fillvalue = bgetr(wmb->fillvalue, tsize * (1 + wmb->validvars))))
piomemerror(*ios, (1 + wmb->validvars) * tsize, __FILE__, __LINE__);

/* If we need a fill value, get it. */
if (iodesc->needsfill)
{
/* If the user passed a fill value, use that, otherwise use
* the default fill value of the netCDF type. Copy the fill
* value to the buffer. */
if (fillvalue)
{
memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, fillvalue, tsize);
}
else
{
vtype = (MPI_Datatype)iodesc->basetype;
if (vtype == MPI_INTEGER)
{
int fill = PIO_FILL_INT;
memcpy((char *)wmb->fillvalue+tsize*wmb->validvars, &fill, tsize);
memcpy((char *)wmb->fillvalue + tsize * wmb->validvars, &fill, tsize);
}
else if (vtype == MPI_FLOAT || vtype == MPI_REAL4)
{
Expand All @@ -407,32 +438,28 @@ int PIOc_write_darray(const int ncid, const int vid, const int ioid,
}
else
{
fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",vtype);
return PIO_EBADTYPE;
}
}
}

/* Tell the buffer about the data it is getting. */
wmb->arraylen = arraylen;
wmb->vid[wmb->validvars] = vid;

/* Copy the user-provided data to the buffer. */
bufptr = (void *)((char *)wmb->data + arraylen * tsize * wmb->validvars);
if (arraylen > 0)
memcpy(bufptr, array, arraylen * tsize);
/*
if (tsize==8){
double asum=0.0;
printf("%s %d %d %d %d\n",__FILE__,__LINE__,vid,arraylen,iodesc->ndof);
for (int k=0;k<arraylen;k++){
asum += ((double *) array)[k];
}
printf("%s %d %d %g\n",__FILE__,__LINE__,vid,asum);
}
*/

/* Add the unlimited dimension value of this variable to the frame
* array in wmb. */
if (wmb->frame)
wmb->frame[wmb->validvars] = vdesc->record;
wmb->validvars++;

LOG((2, "%d %d %d %d", wmb->validvars, iodesc->maxbytes / tsize, iodesc->ndof, iodesc->llen));
LOG((2, "wmb->validvars = %d iodesc->maxbytes / tsize = %d iodesc->ndof = %d iodesc->llen = %d",
wmb->validvars, iodesc->maxbytes / tsize, iodesc->ndof, iodesc->llen));

/* Call the sync when ??? */
if (wmb->validvars >= iodesc->maxbytes / tsize)
Expand Down
45 changes: 16 additions & 29 deletions tests/unit/test_darray.c
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
/**
* @file
/*
* Tests for darray functions.
*
*/
#include <pio.h>
#include <pio_tests.h>
Expand All @@ -12,24 +10,18 @@
/* The name of this test. */
#define TEST_NAME "test_darray"

#define NDIM 3
#define X_DIM_LEN 400
#define Y_DIM_LEN 400
#define NDIM 1
#define DIM_LEN 4
#define VAR_NAME "foo"

/** The dimension names. */
char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"};
/* The dimension names. */
char dim_name[NC_MAX_NAME + 1] = "dim";

/** Length of the dimensions in the sample data. */
int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
/* Length of the dimensions in the sample data. */
int dim_len[NDIM] = {DIM_LEN};

/** Run Tests for darray Functions.
*
* @param argc argument count
* @param argv array of arguments
*/
int
main(int argc, char **argv)
/* Run Tests for darray Functions. */
int main(int argc, char **argv)
{
int my_rank; /* Zero-based rank of processor. */
int ntasks; /* Number of processors involved in current execution. */
Expand All @@ -48,7 +40,7 @@ main(int argc, char **argv)
float *buffer; /* A buffer for sample data. */
int *read_buffer; /* A buffer for reading data back from the file. */
PIO_Offset *compdof; /* The decomposition mapping. */
int fmt, d, d1, i; /* Index for loops. */
int fmt, d, d1; /* Index for loops. */
MPI_Comm test_comm; /* Contains all tasks running test. */
int num_flavors; /* Number of PIO netCDF flavors in this build. */
int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
Expand All @@ -73,16 +65,15 @@ main(int argc, char **argv)
ERR(ret);

/* Describe the decomposition. This is a 1-based array, so add 1! */
elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks;
elements_per_pe = DIM_LEN / ntasks;
if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
return PIO_ENOMEM;
for (i = 0; i < elements_per_pe; i++) {
for (int i = 0; i < elements_per_pe; i++)
compdof[i] = my_rank * elements_per_pe + i + 1;
}

/* Create the PIO decomposition for this test. */
printf("rank: %d Creating decomposition...\n", my_rank);
if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe,
if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM, &dim_len, (PIO_Offset)elements_per_pe,
compdof, &ioid, NULL, NULL, NULL)))
ERR(ret);
free(compdof);
Expand All @@ -102,12 +93,8 @@ main(int argc, char **argv)

/* Define netCDF dimensions and variable. */
printf("rank: %d Defining netCDF metadata...\n", my_rank);
for (d = 0; d < NDIM; d++) {
printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank,
dim_name[d], dim_len[d]);
if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
ERR(ret);
}
if ((ret = PIOc_def_dim(ncid, dim_name, (PIO_Offset)dim_len[0], &dimids[0])))
ERR(ret);

/* Define a variable. */
if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid)))
Expand All @@ -119,7 +106,7 @@ main(int argc, char **argv)

/* Write some data. */
float fillvalue = 0.0;
PIO_Offset arraylen = 4;
PIO_Offset arraylen = 1;
float test_data[arraylen];
for (int f = 0; f < arraylen; f++)
test_data[f] = my_rank * 10 + f;
Expand Down

0 comments on commit 410f34d

Please sign in to comment.