From 3e333c9b0a60116927a69fe89b72378cbb256bcd Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 May 2019 11:51:56 -0600 Subject: [PATCH 1/9] adding async perf test --- tests/cunit/Makefile.am | 3 +- tests/cunit/test_async_perf.c | 434 ++++++++++++++++++++++++++++++++++ 2 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 tests/cunit/test_async_perf.c diff --git a/tests/cunit/Makefile.am b/tests/cunit/Makefile.am index ff25651eeb7..d79c9f7d163 100644 --- a/tests/cunit/Makefile.am +++ b/tests/cunit/Makefile.am @@ -19,7 +19,7 @@ test_darray_multivar3 test_darray_1d test_darray_3d \ test_decomp_uneven test_decomps test_rearr test_darray_async_simple \ test_darray_async test_darray_async_many test_darray_2sync \ test_async_multicomp test_async_multi2 test_async_manyproc \ -test_darray_fill test_decomp_frame test_perf2 +test_darray_fill test_decomp_frame test_perf2 test_async_perf if RUN_TESTS # Tests will run from a bash script. @@ -62,6 +62,7 @@ test_async_manyproc_SOURCES = test_async_manyproc.c test_common.c pio_tests.h test_darray_fill_SOURCES = test_darray_fill.c test_common.c pio_tests.h test_decomp_frame_SOURCES = test_decomp_frame.c test_common.c pio_tests.h test_perf2_SOURCES = test_perf2.c test_common.c pio_tests.h +test_async_perf_SOURCES = test_async_perf.c test_common.c pio_tests.h # Distribute the test script. EXTRA_DIST = run_tests.sh diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c new file mode 100644 index 00000000000..3eb3905194f --- /dev/null +++ b/tests/cunit/test_async_perf.c @@ -0,0 +1,434 @@ +/* + * Tests for performance of PIO async mode. + * + * @author Ed Hartnett + * @date 2/21/17 + */ +#include +#include +#include +#include +#include + +/* The number of tasks this test should run on. */ +#define TARGET_NTASKS 16 + +/* The minimum number of tasks this test should run on. */ +#define MIN_NTASKS TARGET_NTASKS + +/* The name of this test. */ +#define TEST_NAME "test_async_perf" + +/* Number of computational components to create. */ +#define COMPONENT_COUNT 1 + +/* The number of dimensions in the example data. In this test, we + * are using three-dimensional data. */ +#define NDIM 4 + +/* But sometimes we need arrays of the non-record dimensions. */ +#define NDIM3 3 + +/* The length of our sample data along each dimension. */ +#define X_DIM_LEN 128 +#define Y_DIM_LEN 128 +#define Z_DIM_LEN 16 + +/* The number of timesteps of data to write. */ +#define NUM_TIMESTEPS 10 + +/* The name of the variable in the netCDF output files. */ +#define VAR_NAME "foo" + +/* Test with and without specifying a fill value to + * PIOc_write_darray(). */ +#define NUM_TEST_CASES_FILLVALUE 2 + +/* How many different number of IO tasks to check? */ +#define MAX_IO_TESTS 5 + +/* The dimension names. */ +char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y", "z"}; + +/* Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN}; + +#define DIM_NAME "dim" +#define NDIM1 1 + +/* Run test for each of the rearrangers. */ +#define NUM_REARRANGERS_TO_TEST 2 + +/* Create the decomposition to divide the 4-dimensional sample data + * between the 4 tasks. For the purposes of decomposition we are only + * concerned with 3 dimensions - we ignore the unlimited dimension. + * + * @param ntasks the number of available tasks + * @param my_rank rank of this task. + * @param iosysid the IO system ID. + * @param dim_len an array of length 3 with the dimension sizes. + * @param ioid a pointer that gets the ID of this decomposition. + * @returns 0 for success, error code otherwise. + **/ +int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid) +{ + PIO_Offset elements_per_pe; /* Array elements per processing unit. */ + PIO_Offset *compdof; /* The decomposition mapping. */ + int dim_len_3d[NDIM3] = {X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN}; + int ret; + + /* How many data elements per task? */ + elements_per_pe = X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks; + + /* Allocate space for the decomposition array. */ + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + + /* Describe the decomposition. */ + for (int i = 0; i < elements_per_pe; i++) + compdof[i] = my_rank * elements_per_pe + i; + + /* Create the PIO decomposition for this test. */ + if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, elements_per_pe, + compdof, ioid, 0, NULL, NULL))) + ERR(ret); + + /* Free the mapping. */ + free(compdof); + + return 0; +} + +/** + * Test the darray functionality. Create a netCDF file with 4 + * dimensions and 1 PIO_INT variable, and use darray to write some + * data. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param ntasks number of tasks in test_comm. + * @param num_io_procs number of IO processors. + * @param provide_fill 1 if fillvalue should be provided to PIOc_write_darray(). + * @param rearranger the rearranger in use. + * @returns 0 for success, error code otherwise. + */ +int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, + int my_rank, int ntasks, int num_io_procs, int provide_fill, + int rearranger) +{ + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + int dimids[NDIM]; /* The dimension IDs. */ + int ncid; /* The ncid of the netCDF file. */ + int varid; /* The ID of the netCDF varable. */ + PIO_Offset arraylen = (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks); + int int_fillvalue = NC_FILL_INT; + void *fillvalue = NULL; + int *test_data; + int ret; /* Return code. */ + + if (!(test_data = malloc(sizeof(int) * arraylen))) + ERR(PIO_ENOMEM); + + /* Are we providing a fill value? */ + if (provide_fill) + fillvalue = &int_fillvalue; + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (int fmt = 0; fmt < num_flavors; fmt++) + { + struct timeval starttime, endtime; + long long startt, endt; + long long delta; + float num_megabytes = 0; + float delta_in_sec; + float mb_per_sec; + + /* Create the filename. Use the same filename for all, so we + * don't waste disk space. */ + /* sprintf(filename, "data_%s_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[fmt], */ + /* rearranger); */ + sprintf(filename, "data_%s.nc", TEST_NAME); + + /* Create the netCDF output file. */ + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) + ERR(ret); + + /* Turn on fill mode. */ + if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + for (int d = 0; d < NDIM; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + + /* Define a variable. */ + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, &varid))) + ERR(ret); + + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Start the clock. */ + gettimeofday(&starttime, NULL); + + for (int t = 0; t < NUM_TIMESTEPS; t++) + { + /* Initialize some data. */ + for (int f = 0; f < arraylen; f++) + test_data[f] = (my_rank * 10 + f) + t * 1000; + + /* Set the value of the record dimension. */ + if ((ret = PIOc_setframe(ncid, varid, t))) + ERR(ret); + + /* Write the data. */ + if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) + ERR(ret); + + num_megabytes += (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * sizeof(int))/(1024*1024); + } + + /* Stop the clock. */ + gettimeofday(&endtime, NULL); + + /* Close the netCDF file. */ + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + + /* Compute the time delta */ + startt = (1000000 * starttime.tv_sec) + starttime.tv_usec; + endt = (1000000 * endtime.tv_sec) + endtime.tv_usec; + delta = (endt - startt)/NUM_TIMESTEPS; + delta_in_sec = (float)delta / 1000000; + mb_per_sec = num_megabytes / delta_in_sec; + if (!my_rank) + printf("%d\t%d\t%d\t%d\t%d\t%8.3f\t%8.1f\t%8.3f\n", ntasks, num_io_procs, + rearranger, provide_fill, fmt, delta_in_sec, num_megabytes, mb_per_sec); + } + + free(test_data); + + return PIO_NOERR; +} + +/** + * Test the decomp read/write functionality. + * + * @param iosysid the IO system ID. + * @param ioid the ID of the decomposition. + * @param num_flavors the number of IOTYPES available in this build. + * @param flavor array of available iotypes. + * @param my_rank rank of this task. + * @param ntasks number of tasks in test_comm. + * @param rearranger the rearranger to use (PIO_REARR_BOX or + * PIO_REARR_SUBSET). + * @param test_comm the MPI communicator for this test. + * @returns 0 for success, error code otherwise. + */ +int +test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, + int my_rank, int ntasks, int rearranger, + MPI_Comm test_comm) +{ + + /* for (int fmt = 0; fmt < num_flavors; fmt++) */ + for (int fmt = 0; fmt < 1; fmt++) + { + int ioid2; /* ID for decomposition we will create from file. */ + char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ + char title_in[PIO_MAX_NAME + 1]; /* Optional title. */ + char history_in[PIO_MAX_NAME + 1]; /* Optional history. */ + int fortran_order_in; /* Indicates fortran vs. c order. */ + int ret; /* Return code. */ + + /* Create the filename. */ + snprintf(filename, PIO_MAX_NAME, "decomp_%s_iotype_%d.nc", TEST_NAME, + flavor[fmt]); + + if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0))) + return ret; + + /* Read the data. */ + if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, PIO_INT, + title_in, history_in, &fortran_order_in))) + return ret; + + /* Check the results. */ + { + iosystem_desc_t *ios; + io_desc_t *iodesc; + int expected_maplen = (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks); + + /* Get the IO system info. */ + if (!(ios = pio_get_iosystem_from_id(iosysid))) + return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); + + /* Get the IO desc, which describes the decomposition. */ + if (!(iodesc = pio_get_iodesc_from_id(ioid2))) + return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__); + if (iodesc->ioid != ioid2 || iodesc->maplen != expected_maplen || iodesc->ndims != NDIM3 || + iodesc->ndof != expected_maplen) + return ERR_WRONG; + if (iodesc->rearranger != rearranger || iodesc->maxregions != 1 || + iodesc->needsfill || iodesc->mpitype != MPI_INT) + return ERR_WRONG; + for (int e = 0; e < iodesc->maplen; e++) + if (iodesc->map[e] != my_rank * iodesc->maplen + e + 1) + return ERR_WRONG; + if (iodesc->dimlen[0] != X_DIM_LEN || iodesc->dimlen[1] != Y_DIM_LEN || + iodesc->dimlen[2] != Z_DIM_LEN) + return ERR_WRONG; + if (rearranger == PIO_REARR_SUBSET) + { + if (iodesc->nrecvs != 1 || iodesc->num_aiotasks != ntasks) + return ERR_WRONG; + } + else + { + /* I haven't figured out yet what these should be for + * the box rearranger. */ + /* printf("iodesc->nrecv = %d iodesc->num_aiotasks = %d\n", iodesc->nrecvs, */ + /* iodesc->num_aiotasks); */ + } + } + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid2))) + ERR(ret); + } + return PIO_NOERR; +} + +/** + * Run all the tests. + * + * @param iosysid the IO system ID. + * @param num_flavors number of available iotypes in the build. + * @param flavor pointer to array of the available iotypes. + * @param my_rank rank of this task. + * @param ntasks number of tasks in test_comm. + * @param num_io_procs number of IO procs used. + * @param rearranger the rearranger to use (PIO_REARR_BOX or + * PIO_REARR_SUBSET). + * @param test_comm the communicator the test is running on. + * @returns 0 for success, error code otherwise. + */ +int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, + int ntasks, int num_io_procs, int rearranger, + MPI_Comm test_comm) +{ + int ioid; + int my_test_size; + int ret; /* Return code. */ + + if ((ret = MPI_Comm_size(test_comm, &my_test_size))) + MPIERR(ret); + + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_3d(ntasks, my_rank, iosysid, &ioid))) + return ret; + + /* Test decomposition read/write. */ + if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank, + ntasks, rearranger, test_comm))) + return ret; + + /* Test with/without providing a fill value to PIOc_write_darray(). */ + for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) + { + /* Run a simple darray test. */ + if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, + ntasks, num_io_procs, provide_fill, rearranger))) + return ret; + } + + /* Free the PIO decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + return PIO_NOERR; +} + +/* Run tests for darray functions. */ +int main(int argc, char **argv) +{ + int my_rank; + int ntasks; + MPI_Comm test_comm; /* A communicator for this test. */ + int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; + int iosysid; /* The ID for the parallel I/O system. */ + int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ + int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int num_flavors; /* Number of PIO netCDF flavors in this build. */ + int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ + int num_io_procs[MAX_IO_TESTS] = {1, 4, 16, 64, 128}; /* Number of processors that will do IO. */ + int num_io_tests; /* How many different num IO procs to try? */ + int r, i; + int ret; /* Return code. */ + + /* Initialize test. */ + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, 1, + 0, -1, &test_comm))) + ERR(ERR_INIT); + + if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) + return ret; + + /* Figure out iotypes. */ + if ((ret = get_iotypes(&num_flavors, flavor))) + ERR(ret); + + if (!my_rank) + printf("ntasks\tnio\trearr\tfill\tformat\ttime(s)\tdata size (MB)\t" + "performance(MB/s)\n"); + + /* How many processors for IO? */ + num_io_tests = 1; + if (ntasks >= 32) + num_io_tests = 2; + if (ntasks >= 64) + num_io_tests = 3; + if (ntasks >= 128) + num_io_tests = 4; + if (ntasks >= 512) + num_io_tests = 5; + + for (i = 0; i < num_io_tests; i++) + { + /* for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++) */ + for (r = 0; r < 1; r++) + { + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(test_comm, num_io_procs[i], ioproc_stride, + ioproc_start, rearranger[r], &iosysid))) + return ret; + + /* Run tests. */ + if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, + ntasks, num_io_procs[i], rearranger[r], test_comm))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + + } /* next rearranger */ + } /* next num io procs */ + + if (!my_rank) + printf("finalizing io_test!\n"); + + /* Finalize the MPI library. */ + if ((ret = pio_test_finalize(&test_comm))) + return ret; + + printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; +} From ffad703e0cea5a38f5a56186c19bdfa2ac0217d0 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 May 2019 12:40:33 -0600 Subject: [PATCH 2/9] adding async perf test --- tests/cunit/test_async_perf.c | 817 +++++++++++++++++++++------------- 1 file changed, 503 insertions(+), 314 deletions(-) diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c index 3eb3905194f..335f68813ef 100644 --- a/tests/cunit/test_async_perf.c +++ b/tests/cunit/test_async_perf.c @@ -1,34 +1,39 @@ /* - * Tests for performance of PIO async mode. + * This program tests darrays with async. * * @author Ed Hartnett - * @date 2/21/17 + * @date 5/4/17 */ #include #include -#include #include -#include +#include /* The number of tasks this test should run on. */ -#define TARGET_NTASKS 16 +#define TARGET_NTASKS 4 /* The minimum number of tasks this test should run on. */ -#define MIN_NTASKS TARGET_NTASKS +#define MIN_NTASKS 1 /* The name of this test. */ #define TEST_NAME "test_async_perf" -/* Number of computational components to create. */ -#define COMPONENT_COUNT 1 +/* For 1-D use. */ +#define NDIM1 1 -/* The number of dimensions in the example data. In this test, we - * are using three-dimensional data. */ -#define NDIM 4 +/* For 2-D use. */ +#define NDIM2 2 -/* But sometimes we need arrays of the non-record dimensions. */ +/* For 3-D use. */ #define NDIM3 3 +/* For maplens of 2. */ +#define MAPLEN2 2 + +/* Lengths of non-unlimited dimensions. */ +#define LAT_LEN 2 +#define LON_LEN 3 + /* The length of our sample data along each dimension. */ #define X_DIM_LEN 128 #define Y_DIM_LEN 128 @@ -37,27 +42,196 @@ /* The number of timesteps of data to write. */ #define NUM_TIMESTEPS 10 -/* The name of the variable in the netCDF output files. */ -#define VAR_NAME "foo" +/* Number of vars in test file. */ +#define NVAR 4 -/* Test with and without specifying a fill value to - * PIOc_write_darray(). */ -#define NUM_TEST_CASES_FILLVALUE 2 +/* Number of records written for record var. */ +#define NREC 4 -/* How many different number of IO tasks to check? */ -#define MAX_IO_TESTS 5 +/* Name of record test var. */ +#define REC_VAR_NAME "surface_temperature" +#define REC_VAR_NAME2 "surface_temperature2" -/* The dimension names. */ -char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y", "z"}; +/* Name of non-record test var. */ +#define NOREC_VAR_NAME "surface_height" +#define NOREC_VAR_NAME2 "surface_height2" -/* Length of the dimensions in the sample data. */ -int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN}; +char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlim", "lat", "lon"}; -#define DIM_NAME "dim" -#define NDIM1 1 +/* Length of the dimension. */ +#define LEN3 3 -/* Run test for each of the rearrangers. */ -#define NUM_REARRANGERS_TO_TEST 2 +#define NUM_VAR_SETS 2 + +/* Check the file that was created in this test. */ +int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, + int piotype) +{ + int ncid; + int varid[NVAR] = {0, 1, 2, 3}; + void *data_in = NULL; + void *data_in_norec = NULL; + PIO_Offset type_size; + int ret; + + /* Reopen the file. */ + if ((ret = PIOc_openfile(iosysid, &ncid, &iotype, data_filename, NC_NOWRITE))) + BAIL(ret); + + /* Get the size of the type. */ + if ((ret = PIOc_inq_type(ncid, piotype, NULL, &type_size))) + BAIL(ret); + + /* Allocate memory to read data. */ + if (!(data_in = malloc(LAT_LEN * LON_LEN * type_size * NREC))) + BAIL(PIO_ENOMEM); + if (!(data_in_norec = malloc(LAT_LEN * LON_LEN * type_size))) + BAIL(PIO_ENOMEM); + + /* We have two sets of variables, those with unlimted, and those + * without unlimited dimension. */ + for (int vs = 0; vs < NUM_VAR_SETS; vs++) + { + int rec_varid = vs ? varid[0] : varid[1]; + int norec_varid = vs ? varid[2] : varid[3]; + + /* Read the record data. The values we expect are: 10, 11, 20, 21, 30, + * 31, in each of three records. */ + if ((ret = PIOc_get_var(ncid, rec_varid, data_in))) + BAIL(ret); + + /* Read the non-record data. The values we expect are: 10, 11, 20, 21, 30, + * 31. */ + if ((ret = PIOc_get_var(ncid, norec_varid, data_in_norec))) + BAIL(ret); + + /* Check the results. */ + for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++) + { + int tmp_r = r % (LAT_LEN * LON_LEN); + switch (piotype) + { + case PIO_BYTE: + if (((signed char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) + BAIL(ret); + break; + case PIO_CHAR: + if (((char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) + BAIL(ret); + break; + case PIO_SHORT: + if (((short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) + BAIL(ret); + break; + case PIO_INT: + if (((int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) + BAIL(ret); + break; + case PIO_FLOAT: + if (((float *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + BAIL(ret); + break; + case PIO_DOUBLE: + if (((double *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) + BAIL(ret); + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (((unsigned char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) + BAIL(ret); + break; + case PIO_USHORT: + if (((unsigned short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) + BAIL(ret); + break; + case PIO_UINT: + if (((unsigned int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) + BAIL(ret); + break; + case PIO_INT64: + if (((long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) + BAIL(ret); + break; + case PIO_UINT64: + if (((unsigned long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) + BAIL(ret); + break; +#endif /* _NETCDF4 */ + default: + BAIL(ERR_WRONG); + } + } + + /* Check the results. */ + for (int r = 0; r < LAT_LEN * LON_LEN; r++) + { + switch (piotype) + { + case PIO_BYTE: + if (((signed char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; + case PIO_CHAR: + if (((char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; + case PIO_SHORT: + if (((short *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; + case PIO_INT: + if (((int *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; + case PIO_FLOAT: + if (((float *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; + case PIO_DOUBLE: + if (((double *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + if (((unsigned char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; + case PIO_USHORT: + if (((unsigned short *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; + case PIO_UINT: + if (((unsigned int *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; + case PIO_INT64: + if (((long long *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; + case PIO_UINT64: + if (((unsigned long long *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) + BAIL(ret); + break; +#endif /* _NETCDF4 */ + default: + BAIL(ERR_WRONG); + } + } + } /* next var set */ + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + BAIL(ret); + +exit: + /* Free resources. */ + if (data_in) + free(data_in); + if (data_in_norec) + free(data_in_norec); + + return ret; +} /* Create the decomposition to divide the 4-dimensional sample data * between the 4 tasks. For the purposes of decomposition we are only @@ -99,284 +273,292 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid) return 0; } -/** - * Test the darray functionality. Create a netCDF file with 4 - * dimensions and 1 PIO_INT variable, and use darray to write some - * data. - * - * @param iosysid the IO system ID. - * @param ioid the ID of the decomposition. - * @param num_flavors the number of IOTYPES available in this build. - * @param flavor array of available iotypes. - * @param my_rank rank of this task. - * @param ntasks number of tasks in test_comm. - * @param num_io_procs number of IO processors. - * @param provide_fill 1 if fillvalue should be provided to PIOc_write_darray(). - * @param rearranger the rearranger in use. - * @returns 0 for success, error code otherwise. - */ -int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, - int my_rank, int ntasks, int num_io_procs, int provide_fill, - int rearranger) +/* Run a simple test using darrays with async. */ +int +run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, + MPI_Comm comp_comm, int num_flavors, int *flavor, int piotype) { - char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ - int dimids[NDIM]; /* The dimension IDs. */ - int ncid; /* The ncid of the netCDF file. */ - int varid; /* The ID of the netCDF varable. */ - PIO_Offset arraylen = (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks); - int int_fillvalue = NC_FILL_INT; - void *fillvalue = NULL; - int *test_data; - int ret; /* Return code. */ - - if (!(test_data = malloc(sizeof(int) * arraylen))) - ERR(PIO_ENOMEM); - - /* Are we providing a fill value? */ - if (provide_fill) - fillvalue = &int_fillvalue; - - /* Use PIO to create the example file in each of the four - * available ways. */ - for (int fmt = 0; fmt < num_flavors; fmt++) - { - struct timeval starttime, endtime; - long long startt, endt; - long long delta; - float num_megabytes = 0; - float delta_in_sec; - float mb_per_sec; - - /* Create the filename. Use the same filename for all, so we - * don't waste disk space. */ - /* sprintf(filename, "data_%s_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[fmt], */ - /* rearranger); */ - sprintf(filename, "data_%s.nc", TEST_NAME); - - /* Create the netCDF output file. */ - if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) - ERR(ret); - - /* Turn on fill mode. */ - if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL))) - ERR(ret); - - /* Define netCDF dimensions and variable. */ - for (int d = 0; d < NDIM; d++) - if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) - ERR(ret); - - /* Define a variable. */ - if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, &varid))) - ERR(ret); - - /* End define mode. */ - if ((ret = PIOc_enddef(ncid))) - ERR(ret); - - /* Start the clock. */ - gettimeofday(&starttime, NULL); - - for (int t = 0; t < NUM_TIMESTEPS; t++) - { - /* Initialize some data. */ - for (int f = 0; f < arraylen; f++) - test_data[f] = (my_rank * 10 + f) + t * 1000; + int ioid; + int ioid3; + int dim_len[NDIM3] = {NC_UNLIMITED, 2, 3}; + PIO_Offset elements_per_pe = LAT_LEN; + PIO_Offset compdof[LAT_LEN] = {my_rank * 2 - 2, my_rank * 2 - 1}; + char decomp_filename[PIO_MAX_NAME + 1]; + void *my_data_multi; + int ret; - /* Set the value of the record dimension. */ - if ((ret = PIOc_setframe(ncid, varid, t))) - ERR(ret); + sprintf(decomp_filename, "decomp_rdat_%s_.nc", TEST_NAME); - /* Write the data. */ - if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue))) - ERR(ret); + /* Create the PIO decomposition for this test. */ + if ((ret = PIOc_init_decomp(iosysid, piotype, NDIM2, &dim_len[1], elements_per_pe, + compdof, &ioid, PIO_REARR_BOX, NULL, NULL))) + BAIL(ret); - num_megabytes += (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * sizeof(int))/(1024*1024); - } + /* Decompose the data over the tasks. */ + if ((ret = create_decomposition_3d(ntasks, my_rank, iosysid, &ioid3))) + return ret; - /* Stop the clock. */ - gettimeofday(&endtime, NULL); + /* Write the decomp file (on appropriate tasks). */ + if ((ret = PIOc_write_nc_decomp(iosysid, decomp_filename, 0, ioid, NULL, NULL, 0))) + return ret; - /* Close the netCDF file. */ - if ((ret = PIOc_closefile(ncid))) - ERR(ret); - - /* Compute the time delta */ - startt = (1000000 * starttime.tv_sec) + starttime.tv_usec; - endt = (1000000 * endtime.tv_sec) + endtime.tv_usec; - delta = (endt - startt)/NUM_TIMESTEPS; - delta_in_sec = (float)delta / 1000000; - mb_per_sec = num_megabytes / delta_in_sec; - if (!my_rank) - printf("%d\t%d\t%d\t%d\t%d\t%8.3f\t%8.1f\t%8.3f\n", ntasks, num_io_procs, - rearranger, provide_fill, fmt, delta_in_sec, num_megabytes, mb_per_sec); - } - - free(test_data); - - return PIO_NOERR; -} + int fortran_order; + int ioid2; + if ((ret = PIOc_read_nc_decomp(iosysid, decomp_filename, &ioid2, comp_comm, + PIO_INT, NULL, NULL, &fortran_order))) + return ret; -/** - * Test the decomp read/write functionality. - * - * @param iosysid the IO system ID. - * @param ioid the ID of the decomposition. - * @param num_flavors the number of IOTYPES available in this build. - * @param flavor array of available iotypes. - * @param my_rank rank of this task. - * @param ntasks number of tasks in test_comm. - * @param rearranger the rearranger to use (PIO_REARR_BOX or - * PIO_REARR_SUBSET). - * @param test_comm the MPI communicator for this test. - * @returns 0 for success, error code otherwise. - */ -int -test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, - int my_rank, int ntasks, int rearranger, - MPI_Comm test_comm) -{ + /* Free the decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid2))) + BAIL(ret); - /* for (int fmt = 0; fmt < num_flavors; fmt++) */ - for (int fmt = 0; fmt < 1; fmt++) + /* Test each available iotype. */ + for (int fmt = 0; fmt < num_flavors; fmt++) { - int ioid2; /* ID for decomposition we will create from file. */ - char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ - char title_in[PIO_MAX_NAME + 1]; /* Optional title. */ - char history_in[PIO_MAX_NAME + 1]; /* Optional history. */ - int fortran_order_in; /* Indicates fortran vs. c order. */ - int ret; /* Return code. */ - - /* Create the filename. */ - snprintf(filename, PIO_MAX_NAME, "decomp_%s_iotype_%d.nc", TEST_NAME, - flavor[fmt]); - - if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0))) - return ret; - - /* Read the data. */ - if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, PIO_INT, - title_in, history_in, &fortran_order_in))) - return ret; - - /* Check the results. */ + int ncid; + PIO_Offset type_size; + int dimid[NDIM3]; + int varid[NVAR]; + char data_filename[PIO_MAX_NAME + 1]; + void *my_data; + void *my_data_norec; + signed char my_data_byte[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + char my_data_char[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + short my_data_short[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + int my_data_int[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + float my_data_float[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + double my_data_double[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; +#ifdef _NETCDF4 + unsigned char my_data_ubyte[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + unsigned short my_data_ushort[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + unsigned int my_data_uint[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + long long my_data_int64[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + unsigned long long my_data_uint64[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; +#endif /* _NETCDF4 */ + signed char my_data_byte_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; + char my_data_char_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; + short my_data_short_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; + int my_data_int_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; + float my_data_float_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; + double my_data_double_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; +#ifdef _NETCDF4 + unsigned char my_data_ubyte_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; + unsigned short my_data_ushort_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; + unsigned int my_data_uint_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; + long long my_data_int64_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; + unsigned long long my_data_uint64_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; +#endif /* _NETCDF4 */ + + /* Only netCDF-4 can handle extended types. */ + if (piotype > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && flavor[fmt] != PIO_IOTYPE_NETCDF4P) + continue; + + /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */ + if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (piotype == PIO_BYTE || piotype == PIO_CHAR)) + continue; + + /* Select the correct data to write, depending on type. */ + switch (piotype) { - iosystem_desc_t *ios; - io_desc_t *iodesc; - int expected_maplen = (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks); - - /* Get the IO system info. */ - if (!(ios = pio_get_iosystem_from_id(iosysid))) - return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); - - /* Get the IO desc, which describes the decomposition. */ - if (!(iodesc = pio_get_iodesc_from_id(ioid2))) - return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__); - if (iodesc->ioid != ioid2 || iodesc->maplen != expected_maplen || iodesc->ndims != NDIM3 || - iodesc->ndof != expected_maplen) - return ERR_WRONG; - if (iodesc->rearranger != rearranger || iodesc->maxregions != 1 || - iodesc->needsfill || iodesc->mpitype != MPI_INT) - return ERR_WRONG; - for (int e = 0; e < iodesc->maplen; e++) - if (iodesc->map[e] != my_rank * iodesc->maplen + e + 1) - return ERR_WRONG; - if (iodesc->dimlen[0] != X_DIM_LEN || iodesc->dimlen[1] != Y_DIM_LEN || - iodesc->dimlen[2] != Z_DIM_LEN) - return ERR_WRONG; - if (rearranger == PIO_REARR_SUBSET) - { - if (iodesc->nrecvs != 1 || iodesc->num_aiotasks != ntasks) - return ERR_WRONG; - } - else - { - /* I haven't figured out yet what these should be for - * the box rearranger. */ - /* printf("iodesc->nrecv = %d iodesc->num_aiotasks = %d\n", iodesc->nrecvs, */ - /* iodesc->num_aiotasks); */ - } + case PIO_BYTE: + my_data = my_data_byte; + my_data_norec = my_data_byte_norec; + break; + case PIO_CHAR: + my_data = my_data_char; + my_data_norec = my_data_char_norec; + break; + case PIO_SHORT: + my_data = my_data_short; + my_data_norec = my_data_short_norec; + break; + case PIO_INT: + my_data = my_data_int; + my_data_norec = my_data_int_norec; + break; + case PIO_FLOAT: + my_data = my_data_float; + my_data_norec = my_data_float_norec; + break; + case PIO_DOUBLE: + my_data = my_data_double; + my_data_norec = my_data_double_norec; + break; +#ifdef _NETCDF4 + case PIO_UBYTE: + my_data = my_data_ubyte; + my_data_norec = my_data_ubyte_norec; + break; + case PIO_USHORT: + my_data = my_data_ushort; + my_data_norec = my_data_ushort_norec; + break; + case PIO_UINT: + my_data = my_data_uint; + my_data_norec = my_data_uint_norec; + break; + case PIO_INT64: + my_data = my_data_int64; + my_data_norec = my_data_int64_norec; + break; + case PIO_UINT64: + my_data = my_data_uint64; + my_data_norec = my_data_uint64_norec; + break; +#endif /* _NETCDF4 */ + default: + BAIL(ERR_WRONG); } - /* Free the PIO decomposition. */ - if ((ret = PIOc_freedecomp(iosysid, ioid2))) - ERR(ret); - } - return PIO_NOERR; -} + /* Create sample output file. */ + sprintf(data_filename, "data_%s_iotype_%d_piotype_%d.nc", TEST_NAME, flavor[fmt], + piotype); + if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], data_filename, + NC_CLOBBER))) + BAIL(ret); + + /* Find the size of the type. */ + if ((ret = PIOc_inq_type(ncid, piotype, NULL, &type_size))) + BAIL(ret); + + /* Create the data for the darray_multi call by making two + * copies of the data. */ + if (!(my_data_multi = malloc(2 * type_size * elements_per_pe))) + BAIL(PIO_ENOMEM); + memcpy(my_data_multi, my_data, type_size * elements_per_pe); + memcpy((char *)my_data_multi + type_size * elements_per_pe, my_data, type_size * elements_per_pe); + + /* Define dimensions. */ + for (int d = 0; d < NDIM3; d++) + if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) + BAIL(ret); + + /* Define variables. */ + if ((ret = PIOc_def_var(ncid, REC_VAR_NAME, piotype, NDIM3, dimid, &varid[0]))) + BAIL(ret); + if ((ret = PIOc_def_var(ncid, REC_VAR_NAME2, piotype, NDIM3, dimid, &varid[1]))) + BAIL(ret); + if ((ret = PIOc_def_var(ncid, NOREC_VAR_NAME, piotype, NDIM2, &dimid[1], + &varid[2]))) + BAIL(ret); + if ((ret = PIOc_def_var(ncid, NOREC_VAR_NAME2, piotype, NDIM2, &dimid[1], + &varid[3]))) + BAIL(ret); -/** - * Run all the tests. - * - * @param iosysid the IO system ID. - * @param num_flavors number of available iotypes in the build. - * @param flavor pointer to array of the available iotypes. - * @param my_rank rank of this task. - * @param ntasks number of tasks in test_comm. - * @param num_io_procs number of IO procs used. - * @param rearranger the rearranger to use (PIO_REARR_BOX or - * PIO_REARR_SUBSET). - * @param test_comm the communicator the test is running on. - * @returns 0 for success, error code otherwise. - */ -int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank, - int ntasks, int num_io_procs, int rearranger, - MPI_Comm test_comm) -{ - int ioid; - int my_test_size; - int ret; /* Return code. */ + /* End define mode. */ + if ((ret = PIOc_enddef(ncid))) + BAIL(ret); + + /* Set the record number for the record vars. */ + if ((ret = PIOc_setframe(ncid, varid[0], 0))) + BAIL(ret); + if ((ret = PIOc_setframe(ncid, varid[1], 0))) + BAIL(ret); + + /* Write some data to the record vars. */ + if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL))) + BAIL(ret); + if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL))) + BAIL(ret); + + /* Write some data to the non-record vars. */ + if ((ret = PIOc_write_darray(ncid, varid[2], ioid, elements_per_pe, my_data_norec, NULL))) + BAIL(ret); + if ((ret = PIOc_write_darray(ncid, varid[3], ioid, elements_per_pe, my_data_norec, NULL))) + BAIL(ret); + + /* Sync the file. */ + if ((ret = PIOc_sync(ncid))) + BAIL(ret); + + /* Increment the record number for the record vars. */ + if ((ret = PIOc_advanceframe(ncid, varid[0]))) + BAIL(ret); + if ((ret = PIOc_advanceframe(ncid, varid[1]))) + BAIL(ret); + + /* Write another record. */ + if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL))) + BAIL(ret); + if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL))) + BAIL(ret); + + /* Sync the file. */ + if ((ret = PIOc_sync(ncid))) + BAIL(ret); + + /* Increment the record number for the record var. */ + if ((ret = PIOc_advanceframe(ncid, varid[0]))) + BAIL(ret); + if ((ret = PIOc_advanceframe(ncid, varid[1]))) + BAIL(ret); + + /* Write a third record. */ + if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL))) + BAIL(ret); + if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL))) + BAIL(ret); + + /* Increment the record number for the record var. */ + if ((ret = PIOc_advanceframe(ncid, varid[0]))) + BAIL(ret); + if ((ret = PIOc_advanceframe(ncid, varid[1]))) + BAIL(ret); + + /* Write a forth record, using darray_multi(). */ + int frame[2] = {3, 3}; + if ((ret = PIOc_write_darray_multi(ncid, varid, ioid, 2, elements_per_pe, my_data_multi, frame, NULL, 0))) + BAIL(ret); + + /* Close the file. */ + if ((ret = PIOc_closefile(ncid))) + BAIL(ret); - if ((ret = MPI_Comm_size(test_comm, &my_test_size))) - MPIERR(ret); + /* Free resources. */ + free(my_data_multi); + my_data_multi = NULL; - /* Decompose the data over the tasks. */ - if ((ret = create_decomposition_3d(ntasks, my_rank, iosysid, &ioid))) - return ret; - - /* Test decomposition read/write. */ - if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank, - ntasks, rearranger, test_comm))) - return ret; + /* Check the file for correctness. */ + if ((ret = check_darray_file(iosysid, data_filename, PIO_IOTYPE_NETCDF, my_rank, piotype))) + BAIL(ret); - /* Test with/without providing a fill value to PIOc_write_darray(). */ - for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++) - { - /* Run a simple darray test. */ - if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, - ntasks, num_io_procs, provide_fill, rearranger))) - return ret; - } + } /* next iotype */ - /* Free the PIO decomposition. */ + /* Free the decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid))) - ERR(ret); - - return PIO_NOERR; + BAIL(ret); + + /* Free the decomposition. */ + if ((ret = PIOc_freedecomp(iosysid, ioid3))) + BAIL(ret); +exit: + if (my_data_multi) + free(my_data_multi); + return ret; } -/* Run tests for darray functions. */ +/* Run Tests for pio_spmd.c functions. */ int main(int argc, char **argv) { - int my_rank; - int ntasks; - MPI_Comm test_comm; /* A communicator for this test. */ - int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; - int iosysid; /* The ID for the parallel I/O system. */ - int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ - int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ + int my_rank; /* Zero-based rank of processor. */ + int ntasks; /* Number of processors involved in current execution. */ int num_flavors; /* Number of PIO netCDF flavors in this build. */ int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ - int num_io_procs[MAX_IO_TESTS] = {1, 4, 16, 64, 128}; /* Number of processors that will do IO. */ - int num_io_tests; /* How many different num IO procs to try? */ - int r, i; - int ret; /* Return code. */ + MPI_Comm test_comm; /* A communicator for this test. */ +#ifdef _NETCDF4 +#define NUM_TYPES_TO_TEST 11 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, + PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; +#else +#define NUM_TYPES_TO_TEST 6 + int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; +#endif /* _NETCDF4 */ + int ret; /* Return code. */ /* Initialize test. */ - if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, 1, - 0, -1, &test_comm))) + if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, + TARGET_NTASKS, -1, &test_comm))) ERR(ERR_INIT); - if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) return ret; @@ -384,51 +566,58 @@ int main(int argc, char **argv) if ((ret = get_iotypes(&num_flavors, flavor))) ERR(ret); - if (!my_rank) - printf("ntasks\tnio\trearr\tfill\tformat\ttime(s)\tdata size (MB)\t" - "performance(MB/s)\n"); - - /* How many processors for IO? */ - num_io_tests = 1; - if (ntasks >= 32) - num_io_tests = 2; - if (ntasks >= 64) - num_io_tests = 3; - if (ntasks >= 128) - num_io_tests = 4; - if (ntasks >= 512) - num_io_tests = 5; - - for (i = 0; i < num_io_tests; i++) { - /* for (r = 0; r < NUM_REARRANGERS_TO_TEST; r++) */ - for (r = 0; r < 1; r++) - { - /* Initialize the PIO IO system. This specifies how - * many and which processors are involved in I/O. */ - if ((ret = PIOc_Init_Intracomm(test_comm, num_io_procs[i], ioproc_stride, - ioproc_start, rearranger[r], &iosysid))) - return ret; + int iosysid; - /* Run tests. */ - if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, - ntasks, num_io_procs[i], rearranger[r], test_comm))) - return ret; - - /* Finalize PIO system. */ - if ((ret = PIOc_finalize(iosysid))) - return ret; + /* Initialize with task 0 as IO task, tasks 1-3 as a + * computation component. */ +#define NUM_IO_PROCS 1 +#define NUM_COMPUTATION_PROCS 3 +#define COMPONENT_COUNT 1 + int num_computation_procs = NUM_COMPUTATION_PROCS; + MPI_Comm io_comm; /* Will get a duplicate of IO communicator. */ + MPI_Comm comp_comm[COMPONENT_COUNT]; /* Will get duplicates of computation communicators. */ + int mpierr; - } /* next rearranger */ - } /* next num io procs */ + /* Run the test for each data type. */ + for (int t = 0; t < NUM_TYPES_TO_TEST; t++) + { + if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, + &num_computation_procs, NULL, &io_comm, comp_comm, + PIO_REARR_BOX, &iosysid))) + ERR(ERR_INIT); - if (!my_rank) - printf("finalizing io_test!\n"); + /* This code runs only on computation components. */ + if (my_rank) + { + /* Run the simple darray async test. */ + if ((ret = run_darray_async_test(iosysid, my_rank, ntasks, test_comm, + comp_comm[0], num_flavors, flavor, + test_type[t]))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + + /* Free the computation conomponent communicator. */ + if ((mpierr = MPI_Comm_free(comp_comm))) + MPIERR(mpierr); + } + else + { + /* Free the IO communicator. */ + if ((mpierr = MPI_Comm_free(&io_comm))) + MPIERR(mpierr); + } + } /* next type */ + } /* endif my_rank < TARGET_NTASKS */ /* Finalize the MPI library. */ if ((ret = pio_test_finalize(&test_comm))) return ret; printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); + return 0; } From f265ff83368e0ddd3d2d2abc5f03446296b3c362 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 May 2019 12:46:11 -0600 Subject: [PATCH 3/9] adding async perf test --- tests/cunit/test_async_perf.c | 268 +--------------------------------- 1 file changed, 3 insertions(+), 265 deletions(-) diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c index 335f68813ef..bdaf59a59a5 100644 --- a/tests/cunit/test_async_perf.c +++ b/tests/cunit/test_async_perf.c @@ -63,176 +63,6 @@ char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlim", "lat", "lon"}; #define NUM_VAR_SETS 2 -/* Check the file that was created in this test. */ -int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, - int piotype) -{ - int ncid; - int varid[NVAR] = {0, 1, 2, 3}; - void *data_in = NULL; - void *data_in_norec = NULL; - PIO_Offset type_size; - int ret; - - /* Reopen the file. */ - if ((ret = PIOc_openfile(iosysid, &ncid, &iotype, data_filename, NC_NOWRITE))) - BAIL(ret); - - /* Get the size of the type. */ - if ((ret = PIOc_inq_type(ncid, piotype, NULL, &type_size))) - BAIL(ret); - - /* Allocate memory to read data. */ - if (!(data_in = malloc(LAT_LEN * LON_LEN * type_size * NREC))) - BAIL(PIO_ENOMEM); - if (!(data_in_norec = malloc(LAT_LEN * LON_LEN * type_size))) - BAIL(PIO_ENOMEM); - - /* We have two sets of variables, those with unlimted, and those - * without unlimited dimension. */ - for (int vs = 0; vs < NUM_VAR_SETS; vs++) - { - int rec_varid = vs ? varid[0] : varid[1]; - int norec_varid = vs ? varid[2] : varid[3]; - - /* Read the record data. The values we expect are: 10, 11, 20, 21, 30, - * 31, in each of three records. */ - if ((ret = PIOc_get_var(ncid, rec_varid, data_in))) - BAIL(ret); - - /* Read the non-record data. The values we expect are: 10, 11, 20, 21, 30, - * 31. */ - if ((ret = PIOc_get_var(ncid, norec_varid, data_in_norec))) - BAIL(ret); - - /* Check the results. */ - for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++) - { - int tmp_r = r % (LAT_LEN * LON_LEN); - switch (piotype) - { - case PIO_BYTE: - if (((signed char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - BAIL(ret); - break; - case PIO_CHAR: - if (((char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - BAIL(ret); - break; - case PIO_SHORT: - if (((short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - BAIL(ret); - break; - case PIO_INT: - if (((int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - BAIL(ret); - break; - case PIO_FLOAT: - if (((float *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) - BAIL(ret); - break; - case PIO_DOUBLE: - if (((double *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) - BAIL(ret); - break; -#ifdef _NETCDF4 - case PIO_UBYTE: - if (((unsigned char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - BAIL(ret); - break; - case PIO_USHORT: - if (((unsigned short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - BAIL(ret); - break; - case PIO_UINT: - if (((unsigned int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - BAIL(ret); - break; - case PIO_INT64: - if (((long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - BAIL(ret); - break; - case PIO_UINT64: - if (((unsigned long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - BAIL(ret); - break; -#endif /* _NETCDF4 */ - default: - BAIL(ERR_WRONG); - } - } - - /* Check the results. */ - for (int r = 0; r < LAT_LEN * LON_LEN; r++) - { - switch (piotype) - { - case PIO_BYTE: - if (((signed char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; - case PIO_CHAR: - if (((char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; - case PIO_SHORT: - if (((short *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; - case PIO_INT: - if (((int *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; - case PIO_FLOAT: - if (((float *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; - case PIO_DOUBLE: - if (((double *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; -#ifdef _NETCDF4 - case PIO_UBYTE: - if (((unsigned char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; - case PIO_USHORT: - if (((unsigned short *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; - case PIO_UINT: - if (((unsigned int *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; - case PIO_INT64: - if (((long long *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; - case PIO_UINT64: - if (((unsigned long long *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - BAIL(ret); - break; -#endif /* _NETCDF4 */ - default: - BAIL(ERR_WRONG); - } - } - } /* next var set */ - - /* Close the file. */ - if ((ret = PIOc_closefile(ncid))) - BAIL(ret); - -exit: - /* Free resources. */ - if (data_in) - free(data_in); - if (data_in_norec) - free(data_in_norec); - - return ret; -} - /* Create the decomposition to divide the 4-dimensional sample data * between the 4 tasks. For the purposes of decomposition we are only * concerned with 3 dimensions - we ignore the unlimited dimension. @@ -322,32 +152,8 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, char data_filename[PIO_MAX_NAME + 1]; void *my_data; void *my_data_norec; - signed char my_data_byte[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - char my_data_char[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - short my_data_short[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; int my_data_int[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - float my_data_float[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - double my_data_double[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; -#ifdef _NETCDF4 - unsigned char my_data_ubyte[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - unsigned short my_data_ushort[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - unsigned int my_data_uint[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - long long my_data_int64[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - unsigned long long my_data_uint64[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; -#endif /* _NETCDF4 */ - signed char my_data_byte_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; - char my_data_char_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; - short my_data_short_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; int my_data_int_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; - float my_data_float_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; - double my_data_double_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; -#ifdef _NETCDF4 - unsigned char my_data_ubyte_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; - unsigned short my_data_ushort_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; - unsigned int my_data_uint_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; - long long my_data_int64_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; - unsigned long long my_data_uint64_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; -#endif /* _NETCDF4 */ /* Only netCDF-4 can handle extended types. */ if (piotype > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && flavor[fmt] != PIO_IOTYPE_NETCDF4P) @@ -357,58 +163,8 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (piotype == PIO_BYTE || piotype == PIO_CHAR)) continue; - /* Select the correct data to write, depending on type. */ - switch (piotype) - { - case PIO_BYTE: - my_data = my_data_byte; - my_data_norec = my_data_byte_norec; - break; - case PIO_CHAR: - my_data = my_data_char; - my_data_norec = my_data_char_norec; - break; - case PIO_SHORT: - my_data = my_data_short; - my_data_norec = my_data_short_norec; - break; - case PIO_INT: - my_data = my_data_int; - my_data_norec = my_data_int_norec; - break; - case PIO_FLOAT: - my_data = my_data_float; - my_data_norec = my_data_float_norec; - break; - case PIO_DOUBLE: - my_data = my_data_double; - my_data_norec = my_data_double_norec; - break; -#ifdef _NETCDF4 - case PIO_UBYTE: - my_data = my_data_ubyte; - my_data_norec = my_data_ubyte_norec; - break; - case PIO_USHORT: - my_data = my_data_ushort; - my_data_norec = my_data_ushort_norec; - break; - case PIO_UINT: - my_data = my_data_uint; - my_data_norec = my_data_uint_norec; - break; - case PIO_INT64: - my_data = my_data_int64; - my_data_norec = my_data_int64_norec; - break; - case PIO_UINT64: - my_data = my_data_uint64; - my_data_norec = my_data_uint64_norec; - break; -#endif /* _NETCDF4 */ - default: - BAIL(ERR_WRONG); - } + my_data = my_data_int; + my_data_norec = my_data_int_norec; /* Create sample output file. */ sprintf(data_filename, "data_%s_iotype_%d_piotype_%d.nc", TEST_NAME, flavor[fmt], @@ -499,17 +255,6 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL))) BAIL(ret); - /* Increment the record number for the record var. */ - if ((ret = PIOc_advanceframe(ncid, varid[0]))) - BAIL(ret); - if ((ret = PIOc_advanceframe(ncid, varid[1]))) - BAIL(ret); - - /* Write a forth record, using darray_multi(). */ - int frame[2] = {3, 3}; - if ((ret = PIOc_write_darray_multi(ncid, varid, ioid, 2, elements_per_pe, my_data_multi, frame, NULL, 0))) - BAIL(ret); - /* Close the file. */ if ((ret = PIOc_closefile(ncid))) BAIL(ret); @@ -517,11 +262,6 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, /* Free resources. */ free(my_data_multi); my_data_multi = NULL; - - /* Check the file for correctness. */ - if ((ret = check_darray_file(iosysid, data_filename, PIO_IOTYPE_NETCDF, my_rank, piotype))) - BAIL(ret); - } /* next iotype */ /* Free the decomposition. */ @@ -579,8 +319,6 @@ int main(int argc, char **argv) MPI_Comm comp_comm[COMPONENT_COUNT]; /* Will get duplicates of computation communicators. */ int mpierr; - /* Run the test for each data type. */ - for (int t = 0; t < NUM_TYPES_TO_TEST; t++) { if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, &num_computation_procs, NULL, &io_comm, comp_comm, @@ -593,7 +331,7 @@ int main(int argc, char **argv) /* Run the simple darray async test. */ if ((ret = run_darray_async_test(iosysid, my_rank, ntasks, test_comm, comp_comm[0], num_flavors, flavor, - test_type[t]))) + PIO_INT))) return ret; /* Finalize PIO system. */ From 36b0e33c4563b16c184b03f50ac5bc177a7c2ec0 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 May 2019 12:48:50 -0600 Subject: [PATCH 4/9] adding async perf test --- tests/cunit/test_async_perf.c | 51 ----------------------------------- 1 file changed, 51 deletions(-) diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c index bdaf59a59a5..1315695691f 100644 --- a/tests/cunit/test_async_perf.c +++ b/tests/cunit/test_async_perf.c @@ -114,7 +114,6 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, PIO_Offset elements_per_pe = LAT_LEN; PIO_Offset compdof[LAT_LEN] = {my_rank * 2 - 2, my_rank * 2 - 1}; char decomp_filename[PIO_MAX_NAME + 1]; - void *my_data_multi; int ret; sprintf(decomp_filename, "decomp_rdat_%s_.nc", TEST_NAME); @@ -151,9 +150,7 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, int varid[NVAR]; char data_filename[PIO_MAX_NAME + 1]; void *my_data; - void *my_data_norec; int my_data_int[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - int my_data_int_norec[LAT_LEN] = {my_rank * 20, my_rank * 20 + 1}; /* Only netCDF-4 can handle extended types. */ if (piotype > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && flavor[fmt] != PIO_IOTYPE_NETCDF4P) @@ -164,7 +161,6 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, continue; my_data = my_data_int; - my_data_norec = my_data_int_norec; /* Create sample output file. */ sprintf(data_filename, "data_%s_iotype_%d_piotype_%d.nc", TEST_NAME, flavor[fmt], @@ -177,13 +173,6 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, if ((ret = PIOc_inq_type(ncid, piotype, NULL, &type_size))) BAIL(ret); - /* Create the data for the darray_multi call by making two - * copies of the data. */ - if (!(my_data_multi = malloc(2 * type_size * elements_per_pe))) - BAIL(PIO_ENOMEM); - memcpy(my_data_multi, my_data, type_size * elements_per_pe); - memcpy((char *)my_data_multi + type_size * elements_per_pe, my_data, type_size * elements_per_pe); - /* Define dimensions. */ for (int d = 0; d < NDIM3; d++) if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) @@ -192,14 +181,6 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, /* Define variables. */ if ((ret = PIOc_def_var(ncid, REC_VAR_NAME, piotype, NDIM3, dimid, &varid[0]))) BAIL(ret); - if ((ret = PIOc_def_var(ncid, REC_VAR_NAME2, piotype, NDIM3, dimid, &varid[1]))) - BAIL(ret); - if ((ret = PIOc_def_var(ncid, NOREC_VAR_NAME, piotype, NDIM2, &dimid[1], - &varid[2]))) - BAIL(ret); - if ((ret = PIOc_def_var(ncid, NOREC_VAR_NAME2, piotype, NDIM2, &dimid[1], - &varid[3]))) - BAIL(ret); /* End define mode. */ if ((ret = PIOc_enddef(ncid))) @@ -208,20 +189,10 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, /* Set the record number for the record vars. */ if ((ret = PIOc_setframe(ncid, varid[0], 0))) BAIL(ret); - if ((ret = PIOc_setframe(ncid, varid[1], 0))) - BAIL(ret); /* Write some data to the record vars. */ if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL))) BAIL(ret); - if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL))) - BAIL(ret); - - /* Write some data to the non-record vars. */ - if ((ret = PIOc_write_darray(ncid, varid[2], ioid, elements_per_pe, my_data_norec, NULL))) - BAIL(ret); - if ((ret = PIOc_write_darray(ncid, varid[3], ioid, elements_per_pe, my_data_norec, NULL))) - BAIL(ret); /* Sync the file. */ if ((ret = PIOc_sync(ncid))) @@ -230,14 +201,10 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, /* Increment the record number for the record vars. */ if ((ret = PIOc_advanceframe(ncid, varid[0]))) BAIL(ret); - if ((ret = PIOc_advanceframe(ncid, varid[1]))) - BAIL(ret); /* Write another record. */ if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL))) BAIL(ret); - if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL))) - BAIL(ret); /* Sync the file. */ if ((ret = PIOc_sync(ncid))) @@ -246,22 +213,14 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, /* Increment the record number for the record var. */ if ((ret = PIOc_advanceframe(ncid, varid[0]))) BAIL(ret); - if ((ret = PIOc_advanceframe(ncid, varid[1]))) - BAIL(ret); /* Write a third record. */ if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL))) BAIL(ret); - if ((ret = PIOc_write_darray(ncid, varid[1], ioid, elements_per_pe, my_data, NULL))) - BAIL(ret); /* Close the file. */ if ((ret = PIOc_closefile(ncid))) BAIL(ret); - - /* Free resources. */ - free(my_data_multi); - my_data_multi = NULL; } /* next iotype */ /* Free the decomposition. */ @@ -272,8 +231,6 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, if ((ret = PIOc_freedecomp(iosysid, ioid3))) BAIL(ret); exit: - if (my_data_multi) - free(my_data_multi); return ret; } @@ -285,14 +242,6 @@ int main(int argc, char **argv) int num_flavors; /* Number of PIO netCDF flavors in this build. */ int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ MPI_Comm test_comm; /* A communicator for this test. */ -#ifdef _NETCDF4 -#define NUM_TYPES_TO_TEST 11 - int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE, - PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64}; -#else -#define NUM_TYPES_TO_TEST 6 - int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE}; -#endif /* _NETCDF4 */ int ret; /* Return code. */ /* Initialize test. */ From 84634cd741d7b2e36a73358191a47a35c19fd6cf Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 May 2019 12:51:38 -0600 Subject: [PATCH 5/9] adding async perf test --- tests/cunit/test_async_perf.c | 46 ++++++++++++----------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c index 1315695691f..0ff5262d904 100644 --- a/tests/cunit/test_async_perf.c +++ b/tests/cunit/test_async_perf.c @@ -40,7 +40,7 @@ #define Z_DIM_LEN 16 /* The number of timesteps of data to write. */ -#define NUM_TIMESTEPS 10 +#define NUM_TIMESTEPS 3 /* Number of vars in test file. */ #define NVAR 4 @@ -147,10 +147,11 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, int ncid; PIO_Offset type_size; int dimid[NDIM3]; - int varid[NVAR]; + int varid; char data_filename[PIO_MAX_NAME + 1]; void *my_data; int my_data_int[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; + int t; /* Only netCDF-4 can handle extended types. */ if (piotype > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && flavor[fmt] != PIO_IOTYPE_NETCDF4P) @@ -179,44 +180,29 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, BAIL(ret); /* Define variables. */ - if ((ret = PIOc_def_var(ncid, REC_VAR_NAME, piotype, NDIM3, dimid, &varid[0]))) + if ((ret = PIOc_def_var(ncid, REC_VAR_NAME, piotype, NDIM3, dimid, &varid))) BAIL(ret); /* End define mode. */ if ((ret = PIOc_enddef(ncid))) BAIL(ret); - /* Set the record number for the record vars. */ - if ((ret = PIOc_setframe(ncid, varid[0], 0))) - BAIL(ret); - - /* Write some data to the record vars. */ - if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL))) - BAIL(ret); - - /* Sync the file. */ - if ((ret = PIOc_sync(ncid))) - BAIL(ret); - - /* Increment the record number for the record vars. */ - if ((ret = PIOc_advanceframe(ncid, varid[0]))) - BAIL(ret); + for (t = 0; t < NUM_TIMESTEPS; t++) + { + /* Set the record number for the record vars. */ + if ((ret = PIOc_setframe(ncid, varid, t))) + BAIL(ret); - /* Write another record. */ - if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL))) - BAIL(ret); + /* Write some data to the record vars. */ + if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, my_data, NULL))) + BAIL(ret); - /* Sync the file. */ - if ((ret = PIOc_sync(ncid))) - BAIL(ret); + /* Sync the file. */ + if ((ret = PIOc_sync(ncid))) + BAIL(ret); - /* Increment the record number for the record var. */ - if ((ret = PIOc_advanceframe(ncid, varid[0]))) - BAIL(ret); + } - /* Write a third record. */ - if ((ret = PIOc_write_darray(ncid, varid[0], ioid, elements_per_pe, my_data, NULL))) - BAIL(ret); /* Close the file. */ if ((ret = PIOc_closefile(ncid))) From 229d05de73fd8a6b44354dd0f821f2cbad3939bf Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 May 2019 12:55:02 -0600 Subject: [PATCH 6/9] adding async perf test --- tests/cunit/test_async_perf.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c index 0ff5262d904..44a0798283e 100644 --- a/tests/cunit/test_async_perf.c +++ b/tests/cunit/test_async_perf.c @@ -18,15 +18,15 @@ /* The name of this test. */ #define TEST_NAME "test_async_perf" -/* For 1-D use. */ -#define NDIM1 1 - /* For 2-D use. */ #define NDIM2 2 /* For 3-D use. */ #define NDIM3 3 +/* For 4-D use. */ +#define NDIM3 4 + /* For maplens of 2. */ #define MAPLEN2 2 @@ -56,7 +56,7 @@ #define NOREC_VAR_NAME "surface_height" #define NOREC_VAR_NAME2 "surface_height2" -char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlim", "lat", "lon"}; +char dim_name[NDIM4][PIO_MAX_NAME + 1] = {"unlim", "x", "y", "z"}; /* Length of the dimension. */ #define LEN3 3 @@ -110,7 +110,7 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, { int ioid; int ioid3; - int dim_len[NDIM3] = {NC_UNLIMITED, 2, 3}; + int dim_len[NDIM4] = {NC_UNLIMITED, 2, 3}; PIO_Offset elements_per_pe = LAT_LEN; PIO_Offset compdof[LAT_LEN] = {my_rank * 2 - 2, my_rank * 2 - 1}; char decomp_filename[PIO_MAX_NAME + 1]; From 624ca79dc6ed59bb2e54e82983512f9e48fec3db Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 May 2019 13:01:11 -0600 Subject: [PATCH 7/9] adding async perf test --- tests/cunit/test_async_perf.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c index 44a0798283e..a561c4857cb 100644 --- a/tests/cunit/test_async_perf.c +++ b/tests/cunit/test_async_perf.c @@ -25,7 +25,7 @@ #define NDIM3 3 /* For 4-D use. */ -#define NDIM3 4 +#define NDIM4 4 /* For maplens of 2. */ #define MAPLEN2 2 @@ -42,19 +42,8 @@ /* The number of timesteps of data to write. */ #define NUM_TIMESTEPS 3 -/* Number of vars in test file. */ -#define NVAR 4 - -/* Number of records written for record var. */ -#define NREC 4 - /* Name of record test var. */ #define REC_VAR_NAME "surface_temperature" -#define REC_VAR_NAME2 "surface_temperature2" - -/* Name of non-record test var. */ -#define NOREC_VAR_NAME "surface_height" -#define NOREC_VAR_NAME2 "surface_height2" char dim_name[NDIM4][PIO_MAX_NAME + 1] = {"unlim", "x", "y", "z"}; @@ -110,7 +99,7 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, { int ioid; int ioid3; - int dim_len[NDIM4] = {NC_UNLIMITED, 2, 3}; + int dim_len[NDIM4] = {NC_UNLIMITED, 2, 3, Z_DIM_LEN}; PIO_Offset elements_per_pe = LAT_LEN; PIO_Offset compdof[LAT_LEN] = {my_rank * 2 - 2, my_rank * 2 - 1}; char decomp_filename[PIO_MAX_NAME + 1]; @@ -146,7 +135,7 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, { int ncid; PIO_Offset type_size; - int dimid[NDIM3]; + int dimid[NDIM4]; int varid; char data_filename[PIO_MAX_NAME + 1]; void *my_data; @@ -175,7 +164,7 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, BAIL(ret); /* Define dimensions. */ - for (int d = 0; d < NDIM3; d++) + for (int d = 0; d < NDIM4; d++) if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) BAIL(ret); From 618833c14c017fff566a366687433cc8d914b0d2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 May 2019 13:36:52 -0600 Subject: [PATCH 8/9] adding async perf test --- tests/cunit/test_async_perf.c | 63 ++++++++++------------------------- 1 file changed, 17 insertions(+), 46 deletions(-) diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c index a561c4857cb..6e77c2f0cae 100644 --- a/tests/cunit/test_async_perf.c +++ b/tests/cunit/test_async_perf.c @@ -35,8 +35,8 @@ #define LON_LEN 3 /* The length of our sample data along each dimension. */ -#define X_DIM_LEN 128 -#define Y_DIM_LEN 128 +#define X_DIM_LEN 2 +#define Y_DIM_LEN 3 #define Z_DIM_LEN 16 /* The number of timesteps of data to write. */ @@ -56,7 +56,8 @@ char dim_name[NDIM4][PIO_MAX_NAME + 1] = {"unlim", "x", "y", "z"}; * between the 4 tasks. For the purposes of decomposition we are only * concerned with 3 dimensions - we ignore the unlimited dimension. * - * @param ntasks the number of available tasks + * @param ntasks the number of available tasks (tasks doing + * computation). * @param my_rank rank of this task. * @param iosysid the IO system ID. * @param dim_len an array of length 3 with the dimension sizes. @@ -68,6 +69,7 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid) PIO_Offset elements_per_pe; /* Array elements per processing unit. */ PIO_Offset *compdof; /* The decomposition mapping. */ int dim_len_3d[NDIM3] = {X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN}; + int my_proc_rank = my_rank - 1; int ret; /* How many data elements per task? */ @@ -79,7 +81,7 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid) /* Describe the decomposition. */ for (int i = 0; i < elements_per_pe; i++) - compdof[i] = my_rank * elements_per_pe + i; + compdof[i] = my_proc_rank * elements_per_pe + i; /* Create the PIO decomposition for this test. */ if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, elements_per_pe, @@ -97,39 +99,19 @@ int run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, MPI_Comm comp_comm, int num_flavors, int *flavor, int piotype) { - int ioid; int ioid3; - int dim_len[NDIM4] = {NC_UNLIMITED, 2, 3, Z_DIM_LEN}; - PIO_Offset elements_per_pe = LAT_LEN; - PIO_Offset compdof[LAT_LEN] = {my_rank * 2 - 2, my_rank * 2 - 1}; + int dim_len[NDIM4] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN}; + PIO_Offset elements_per_pe2 = X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / 3; char decomp_filename[PIO_MAX_NAME + 1]; + int niotasks = 1; int ret; sprintf(decomp_filename, "decomp_rdat_%s_.nc", TEST_NAME); - /* Create the PIO decomposition for this test. */ - if ((ret = PIOc_init_decomp(iosysid, piotype, NDIM2, &dim_len[1], elements_per_pe, - compdof, &ioid, PIO_REARR_BOX, NULL, NULL))) - BAIL(ret); - /* Decompose the data over the tasks. */ - if ((ret = create_decomposition_3d(ntasks, my_rank, iosysid, &ioid3))) - return ret; - - /* Write the decomp file (on appropriate tasks). */ - if ((ret = PIOc_write_nc_decomp(iosysid, decomp_filename, 0, ioid, NULL, NULL, 0))) - return ret; - - int fortran_order; - int ioid2; - if ((ret = PIOc_read_nc_decomp(iosysid, decomp_filename, &ioid2, comp_comm, - PIO_INT, NULL, NULL, &fortran_order))) + if ((ret = create_decomposition_3d(ntasks - niotasks, my_rank, iosysid, &ioid3))) return ret; - /* Free the decomposition. */ - if ((ret = PIOc_freedecomp(iosysid, ioid2))) - BAIL(ret); - /* Test each available iotype. */ for (int fmt = 0; fmt < num_flavors; fmt++) { @@ -138,19 +120,11 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, int dimid[NDIM4]; int varid; char data_filename[PIO_MAX_NAME + 1]; - void *my_data; - int my_data_int[LAT_LEN] = {my_rank * 10, my_rank * 10 + 1}; - int t; - - /* Only netCDF-4 can handle extended types. */ - if (piotype > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C && flavor[fmt] != PIO_IOTYPE_NETCDF4P) - continue; - - /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */ - if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (piotype == PIO_BYTE || piotype == PIO_CHAR)) - continue; + int my_data_int[elements_per_pe2]; + int d, t; - my_data = my_data_int; + for (d = 0; d < elements_per_pe2; d++) + my_data_int[d] = my_rank; /* Create sample output file. */ sprintf(data_filename, "data_%s_iotype_%d_piotype_%d.nc", TEST_NAME, flavor[fmt], @@ -169,7 +143,7 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, BAIL(ret); /* Define variables. */ - if ((ret = PIOc_def_var(ncid, REC_VAR_NAME, piotype, NDIM3, dimid, &varid))) + if ((ret = PIOc_def_var(ncid, REC_VAR_NAME, piotype, NDIM4, dimid, &varid))) BAIL(ret); /* End define mode. */ @@ -183,7 +157,8 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, BAIL(ret); /* Write some data to the record vars. */ - if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, my_data, NULL))) + if ((ret = PIOc_write_darray(ncid, varid, ioid3, elements_per_pe2, + my_data_int, NULL))) BAIL(ret); /* Sync the file. */ @@ -198,10 +173,6 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, BAIL(ret); } /* next iotype */ - /* Free the decomposition. */ - if ((ret = PIOc_freedecomp(iosysid, ioid))) - BAIL(ret); - /* Free the decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid3))) BAIL(ret); From 98ee55adc069f00fdb105b10919f7f3b4ac61808 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 13 May 2019 14:23:54 -0600 Subject: [PATCH 9/9] more perf test development --- src/clib/pio_msg.c | 6 +- tests/cunit/test_async_perf.c | 140 +++++++++++++++++++++------------- tests/cunit/test_perf2.c | 6 +- 3 files changed, 96 insertions(+), 56 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index f595619d498..798cbce942f 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -2060,7 +2060,9 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) if ((mpierr = MPI_Bcast(&maplen, 1, MPI_INT, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); - PIO_Offset compmap[maplen]; + PIO_Offset *compmap; + if (!(compmap = malloc(maplen * sizeof(PIO_Offset)))) + return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(compmap, maplen, MPI_OFFSET, 0, ios->intercomm))) return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__); @@ -2102,6 +2104,8 @@ int initdecomp_dof_handler(iosystem_desc_t *ios) iostartp, iocountp); LOG((1, "PIOc_InitDecomp returned")); + + free(compmap); return PIO_NOERR; } diff --git a/tests/cunit/test_async_perf.c b/tests/cunit/test_async_perf.c index 6e77c2f0cae..74b21e0ddd8 100644 --- a/tests/cunit/test_async_perf.c +++ b/tests/cunit/test_async_perf.c @@ -8,6 +8,7 @@ #include #include #include +#include /* The number of tasks this test should run on. */ #define TARGET_NTASKS 4 @@ -35,15 +36,15 @@ #define LON_LEN 3 /* The length of our sample data along each dimension. */ -#define X_DIM_LEN 2 -#define Y_DIM_LEN 3 -#define Z_DIM_LEN 16 +#define X_DIM_LEN 1024 +#define Y_DIM_LEN 1024 +#define Z_DIM_LEN 124 /* The number of timesteps of data to write. */ #define NUM_TIMESTEPS 3 /* Name of record test var. */ -#define REC_VAR_NAME "surface_temperature" +#define REC_VAR_NAME "Duncan_McCloud_of_the_clan_McCloud" char dim_name[NDIM4][PIO_MAX_NAME + 1] = {"unlim", "x", "y", "z"}; @@ -96,8 +97,8 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid) /* Run a simple test using darrays with async. */ int -run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, - MPI_Comm comp_comm, int num_flavors, int *flavor, int piotype) +run_darray_async_test(int iosysid, int fmt, int my_rank, int ntasks, MPI_Comm test_comm, + MPI_Comm comp_comm, int *flavor, int piotype) { int ioid3; int dim_len[NDIM4] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN}; @@ -112,23 +113,25 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, if ((ret = create_decomposition_3d(ntasks - niotasks, my_rank, iosysid, &ioid3))) return ret; - /* Test each available iotype. */ - for (int fmt = 0; fmt < num_flavors; fmt++) { int ncid; PIO_Offset type_size; int dimid[NDIM4]; int varid; char data_filename[PIO_MAX_NAME + 1]; - int my_data_int[elements_per_pe2]; + int *my_data_int; int d, t; + if (!(my_data_int = malloc(elements_per_pe2 * sizeof(int)))) + BAIL(PIO_ENOMEM); + for (d = 0; d < elements_per_pe2; d++) my_data_int[d] = my_rank; /* Create sample output file. */ - sprintf(data_filename, "data_%s_iotype_%d_piotype_%d.nc", TEST_NAME, flavor[fmt], - piotype); + /* sprintf(data_filename, "data_%s_iotype_%d_piotype_%d.nc", TEST_NAME, flavor[fmt], */ + /* piotype); */ + sprintf(data_filename, "data_%s.nc", TEST_NAME); if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], data_filename, NC_CLOBBER))) BAIL(ret); @@ -171,7 +174,9 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, /* Close the file. */ if ((ret = PIOc_closefile(ncid))) BAIL(ret); - } /* next iotype */ + + free(my_data_int); + } /* Free the decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid3))) @@ -180,6 +185,12 @@ run_darray_async_test(int iosysid, int my_rank, int ntasks, MPI_Comm test_comm, return ret; } +/* Initialize with task 0 as IO task, tasks 1-3 as a + * computation component. */ +#define NUM_IO_PROCS 1 +#define NUM_COMPUTATION_PROCS 3 +#define COMPONENT_COUNT 1 + /* Run Tests for pio_spmd.c functions. */ int main(int argc, char **argv) { @@ -188,6 +199,13 @@ int main(int argc, char **argv) int num_flavors; /* Number of PIO netCDF flavors in this build. */ int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ MPI_Comm test_comm; /* A communicator for this test. */ + int iosysid; + + int num_computation_procs = NUM_COMPUTATION_PROCS; + MPI_Comm io_comm; /* Will get a duplicate of IO communicator. */ + MPI_Comm comp_comm[COMPONENT_COUNT]; /* Will get duplicates of computation communicators. */ + int niotasks = NUM_IO_PROCS; + int mpierr; int ret; /* Return code. */ /* Initialize test. */ @@ -201,50 +219,68 @@ int main(int argc, char **argv) if ((ret = get_iotypes(&num_flavors, flavor))) ERR(ret); + + for (int fmt = 0; fmt < num_flavors; fmt++) { - int iosysid; + struct timeval starttime, endtime; + long long startt, endt; + long long delta; + float num_megabytes; + float delta_in_sec; + float mb_per_sec; + + /* Start the clock. */ + if (!my_rank) + { + gettimeofday(&starttime, NULL); + startt = (1000000 * starttime.tv_sec) + starttime.tv_usec; + } - /* Initialize with task 0 as IO task, tasks 1-3 as a - * computation component. */ -#define NUM_IO_PROCS 1 -#define NUM_COMPUTATION_PROCS 3 -#define COMPONENT_COUNT 1 - int num_computation_procs = NUM_COMPUTATION_PROCS; - MPI_Comm io_comm; /* Will get a duplicate of IO communicator. */ - MPI_Comm comp_comm[COMPONENT_COUNT]; /* Will get duplicates of computation communicators. */ - int mpierr; + if ((ret = PIOc_init_async(test_comm, niotasks, NULL, COMPONENT_COUNT, + &num_computation_procs, NULL, &io_comm, comp_comm, + PIO_REARR_BOX, &iosysid))) + ERR(ERR_INIT); + /* This code runs only on computation components. */ + if (my_rank) + { + /* Run the simple darray async test. */ + if ((ret = run_darray_async_test(iosysid, fmt, my_rank, ntasks, test_comm, + comp_comm[0], flavor, PIO_INT))) + return ret; + + /* Finalize PIO system. */ + if ((ret = PIOc_finalize(iosysid))) + return ret; + + /* Free the computation conomponent communicator. */ + if ((mpierr = MPI_Comm_free(comp_comm))) + MPIERR(mpierr); + } + else { - if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT, - &num_computation_procs, NULL, &io_comm, comp_comm, - PIO_REARR_BOX, &iosysid))) - ERR(ERR_INIT); - - /* This code runs only on computation components. */ - if (my_rank) - { - /* Run the simple darray async test. */ - if ((ret = run_darray_async_test(iosysid, my_rank, ntasks, test_comm, - comp_comm[0], num_flavors, flavor, - PIO_INT))) - return ret; - - /* Finalize PIO system. */ - if ((ret = PIOc_finalize(iosysid))) - return ret; - - /* Free the computation conomponent communicator. */ - if ((mpierr = MPI_Comm_free(comp_comm))) - MPIERR(mpierr); - } - else - { - /* Free the IO communicator. */ - if ((mpierr = MPI_Comm_free(&io_comm))) - MPIERR(mpierr); - } - } /* next type */ - } /* endif my_rank < TARGET_NTASKS */ + /* Free the IO communicator. */ + if ((mpierr = MPI_Comm_free(&io_comm))) + MPIERR(mpierr); + } + + if (!my_rank) + { + /* Stop the clock. */ + gettimeofday(&endtime, NULL); + + /* Compute the time delta */ + endt = (1000000 * endtime.tv_sec) + endtime.tv_usec; + delta = (endt - startt)/NUM_TIMESTEPS; + delta_in_sec = (float)delta / 1000000; + num_megabytes = (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * NUM_TIMESTEPS * + sizeof(int))/(1024*1024); + mb_per_sec = num_megabytes / delta_in_sec; + printf("%d\t%d\t%d\t%d\t%d\t%8.3f\t%8.1f\t%8.3f\n", ntasks, niotasks, + 1, 0, fmt, delta_in_sec, num_megabytes, mb_per_sec); + } + + } /* next fmt */ /* Finalize the MPI library. */ if ((ret = pio_test_finalize(&test_comm))) diff --git a/tests/cunit/test_perf2.c b/tests/cunit/test_perf2.c index e00193df43b..4335a167c5f 100644 --- a/tests/cunit/test_perf2.c +++ b/tests/cunit/test_perf2.c @@ -194,13 +194,13 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, num_megabytes += (X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * sizeof(int))/(1024*1024); } - /* Stop the clock. */ - gettimeofday(&endtime, NULL); - /* Close the netCDF file. */ if ((ret = PIOc_closefile(ncid))) ERR(ret); + /* Stop the clock. */ + gettimeofday(&endtime, NULL); + /* Compute the time delta */ startt = (1000000 * starttime.tv_sec) + starttime.tv_usec; endt = (1000000 * endtime.tv_sec) + endtime.tv_usec;