From 63abea02f0109fa38b355c9cb98c99b113484353 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 21 Mar 2019 09:17:52 -0600 Subject: [PATCH 1/9] change to trigger CI --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c998e354dc4..c94f5d8c71d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ A high-level Parallel I/O Library for structured grid applications ## Website -For complete documentation, see our website at [http://ncar.github.io/ParallelIO/](http://ncar.github.io/ParallelIO/). +For complete documentation, see our website at +[http://ncar.github.io/ParallelIO/](http://ncar.github.io/ParallelIO/). ## Mailing List From 27a86387e86863b67ae248e5cfd195995078f7c5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 21 Mar 2019 09:26:59 -0600 Subject: [PATCH 2/9] clean up of test_perf2.c --- tests/cunit/test_perf2.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/cunit/test_perf2.c b/tests/cunit/test_perf2.c index 08fd304be23..36668776108 100644 --- a/tests/cunit/test_perf2.c +++ b/tests/cunit/test_perf2.c @@ -125,9 +125,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank int varid; /* The ID of the netCDF varable. */ int ret; /* Return code. */ PIO_Offset arraylen = EXPECTED_MAPLEN; - struct timeval starttime, endtime; - long long startt, endt; - long long delta; int int_fillvalue = NC_FILL_INT; void *fillvalue = NULL; int *test_data; @@ -163,6 +160,10 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { + struct timeval starttime, endtime; + long long startt, endt; + long long delta; + /* Create the filename. */ /* sprintf(filename, "data_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); */ sprintf(filename, "data__iotype_.nc"); From a240cab3dbeae6830eef3ffda6f9e58a86ed6a06 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 21 Mar 2019 09:31:00 -0600 Subject: [PATCH 3/9] fixing codacy warnings in test --- tests/cunit/test_darray_async_many.c | 109 ++++++++++++++------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/tests/cunit/test_darray_async_many.c b/tests/cunit/test_darray_async_many.c index d993bfaef29..a32cd0136b5 100644 --- a/tests/cunit/test_darray_async_many.c +++ b/tests/cunit/test_darray_async_many.c @@ -90,40 +90,41 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, #endif /* _NETCDF4 */ int expected_int_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1, 2, 1, 3, 2, 3, 2, 4, 3}; float expected_float_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1.5, 2, 1, 3, 2.5, 3, 2, 4, 3.5}; + void *data_in = NULL; + void *data_in2 = NULL; + void *norec_data_in = NULL; /* Reopen the file. */ if ((ret = PIOc_openfile(iosysid, &ncid, &iotype, data_filename, NC_NOWRITE))) - ERR(ret); + BAIL(ret); /* Check metadata. */ int ndims_in, nvars_in, ngatts_in, unlimdimid_in; if ((ret = PIOc_inq(ncid, &ndims_in, &nvars_in, &ngatts_in, &unlimdimid_in))) - ERR(ret); + BAIL(ret); if (ndims_in != NDIM4 || nvars_in != num_types * 2 + NUM_4D_VARS || ngatts_in != 0 || unlimdimid_in != 0) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); /* Check the vars. */ for (int t = 0; t < num_types; t++) { - void *data_in; - void *norec_data_in; PIO_Offset type_size; /* Find size of type. */ if ((ret = PIOc_inq_type(ncid, my_type[t], NULL, &type_size))) - ERR(ret); + BAIL(ret); /* Allocate buffers to hold data. */ if (!(data_in = malloc(LAT_LEN * LON_LEN * NREC * type_size))) - ERR(PIO_ENOMEM); + BAIL(PIO_ENOMEM); if (!(norec_data_in = malloc(LAT_LEN * LON_LEN * type_size))) - ERR(PIO_ENOMEM); + BAIL(PIO_ENOMEM); /* Read record and non-record vars for this type. */ if ((ret = PIOc_get_var(ncid, rec_varid[t], data_in))) - ERR(ret); + BAIL(ret); if ((ret = PIOc_get_var(ncid, norec_varid[t], norec_data_in))) - ERR(ret); + BAIL(ret); /* Check each value of non-record data. */ for (int r = 0; r < LAT_LEN * LON_LEN; r++) @@ -132,52 +133,52 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, { case PIO_BYTE: if (((signed char *)norec_data_in)[r] != expected_byte[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_CHAR: if (((char *)norec_data_in)[r] != expected_char[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_SHORT: if (((short *)norec_data_in)[r] != expected_short[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_INT: if (((int *)norec_data_in)[r] != expected_int[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_FLOAT: if (((float *)norec_data_in)[r] != expected_float[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_DOUBLE: if (((double *)norec_data_in)[r] != expected_double[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; #ifdef _NETCDF4 case PIO_UBYTE: if (((unsigned char *)norec_data_in)[r] != expected_ubyte[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_USHORT: if (((unsigned short *)norec_data_in)[r] != expected_ushort[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_UINT: if (((unsigned int *)norec_data_in)[r] != expected_uint[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_INT64: if (((long long *)norec_data_in)[r] != expected_int64[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_UINT64: if (((unsigned long long *)norec_data_in)[r] != expected_uint64[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; #endif /* _NETCDF4 */ default: - ERR(ERR_WRONG); + BAIL(ERR_WRONG); } } @@ -188,77 +189,76 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, { case PIO_BYTE: if (((signed char *)data_in)[r] != expected_byte[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_CHAR: if (((char *)data_in)[r] != expected_char[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_SHORT: if (((short *)data_in)[r] != expected_short[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_INT: if (((int *)data_in)[r] != expected_int[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_FLOAT: if (((float *)data_in)[r] != expected_float[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_DOUBLE: if (((double *)data_in)[r] != expected_double[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; #ifdef _NETCDF4 case PIO_UBYTE: if (((unsigned char *)data_in)[r] != expected_ubyte[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_USHORT: if (((unsigned short *)data_in)[r] != expected_ushort[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_UINT: if (((unsigned int *)data_in)[r] != expected_uint[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_INT64: if (((long long *)data_in)[r] != expected_int64[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_UINT64: if (((unsigned long long *)data_in)[r] != expected_uint64[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; #endif /* _NETCDF4 */ default: - ERR(ERR_WRONG); + BAIL(ERR_WRONG); } } /* Check the 4D vars. */ for (int v = 0; v < NUM_4D_VARS; v++) { - void *data_in; int xtype; PIO_Offset size; /* Get the type of the 4d var. */ if ((ret = PIOc_inq_vartype(ncid, varid_4d[v], &xtype))) - ERR(ret); + BAIL(ret); /* Get the size of this type. */ if ((ret = PIOc_inq_type(ncid, xtype, NULL, &size))) - ERR(ret); + BAIL(ret); /* Allocate memory for data. */ - if (!(data_in = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC))) - ERR(PIO_ENOMEM); + if (!(data_in2 = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC))) + BAIL(PIO_ENOMEM); /* Read the data. */ - if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in))) - ERR(ret); + if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in2))) + BAIL(ret); /* Check each element of data. */ for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++) @@ -266,31 +266,32 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, switch (xtype) { case PIO_INT: - if (((int *)data_in)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + if (((int *)data_in2)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) + BAIL(ERR_WRONG); break; case PIO_FLOAT: - if (((float *)data_in)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + if (((float *)data_in2)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) + BAIL(ERR_WRONG); break; default: - ERR(ERR_WRONG); + BAIL(ERR_WRONG); } } - - /* Release memory. */ - free(data_in); } - - free(data_in); - free(norec_data_in); } /* Close the file. */ if ((ret = PIOc_closefile(ncid))) - ERR(ret); + BAIL(ret); - return 0; +exit: + if (data_in) + free(data_in); + if (data_in2) + free(data_in2); + if (norec_data_in) + free(norec_data_in); + return ret; } /* Run a simple test using darrays with async. */ From cd1d9cf82fdcbcd581b7bcecc009c04bc4d595f8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 21 Mar 2019 09:36:16 -0600 Subject: [PATCH 4/9] fixing leak on error --- tests/cunit/test_decomps.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/cunit/test_decomps.c b/tests/cunit/test_decomps.c index 473f70eef65..f739544b53d 100644 --- a/tests/cunit/test_decomps.c +++ b/tests/cunit/test_decomps.c @@ -102,7 +102,10 @@ int test_decomp1(int iosysid, int use_io, int my_rank, MPI_Comm test_comm) if (!(iostart = calloc(NDIM2, sizeof(PIO_Offset)))) return ERR_AWFUL; if (!(iocount = calloc(NDIM2, sizeof(PIO_Offset)))) + { + free(iostart); return ERR_AWFUL; + } if (my_rank == 0) for (int i = 0; i < NDIM2; i++) iocount[i] = 4; @@ -111,15 +114,20 @@ int test_decomp1(int iosysid, int use_io, int my_rank, MPI_Comm test_comm) /* Create the PIO decomposition for this test. */ if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, iostart, iocount))) + { + if (iostart) + free(iostart); + if (iocount) + free(iocount); return ret; + } /* Free resources. */ free(compdof); - if (use_io) - { + if (iostart) free(iostart); + if (iocount) free(iocount); - } /* These should not work. */ if (PIOc_write_decomp(DECOMP_FILE, iosysid + TEST_VAL_42, ioid, test_comm) != PIO_EBADID) From 154dc956e0d3279aadec42e5cd85524be3b8e5c2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 22 Mar 2019 08:47:55 -0600 Subject: [PATCH 5/9] fixed memory leaks --- tests/cunit/test_darray_async_many.c | 111 ++++++++++++++++----------- 1 file changed, 66 insertions(+), 45 deletions(-) diff --git a/tests/cunit/test_darray_async_many.c b/tests/cunit/test_darray_async_many.c index a32cd0136b5..9638c64854e 100644 --- a/tests/cunit/test_darray_async_many.c +++ b/tests/cunit/test_darray_async_many.c @@ -62,6 +62,63 @@ int my_type[NTYPE] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, /* Names of the dimensions. */ char dim_name[NDIM4][PIO_MAX_NAME + 1] = {"time", "vert_level", "lat", "lon"}; +int +check_4d_vars(int my_rank, int ncid, int *varid_4d) +{ + void *data_in2 = NULL; + int expected_int_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1, 2, 1, 3, 2, 3, 2, 4, 3}; + float expected_float_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1.5, 2, 1, 3, 2.5, 3, 2, 4, 3.5}; + int ret; + + for (int v = 0; v < NUM_4D_VARS; v++) + { + int xtype; + PIO_Offset size; + + /* Get the type of the 4d var. */ + if ((ret = PIOc_inq_vartype(ncid, varid_4d[v], &xtype))) + BAIL(ret); + + /* Get the size of this type. */ + if ((ret = PIOc_inq_type(ncid, xtype, NULL, &size))) + BAIL(ret); + + /* Allocate memory for data. */ + if (!(data_in2 = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC))) + BAIL(PIO_ENOMEM); + + /* Read the data. */ + if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in2))) + BAIL(ret); + + /* Check each element of data. */ + for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++) + { + switch (xtype) + { + case PIO_INT: + if (((int *)data_in2)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) + BAIL(ERR_WRONG); + break; + case PIO_FLOAT: + if (((float *)data_in2)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) + BAIL(ERR_WRONG); + break; + default: + BAIL(ERR_WRONG); + } + } + free(data_in2); + data_in2 = NULL; + } + +exit: + if (data_in2) + free(data_in2); + + return ret; +} + /* Check the file that was created in this test. */ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, int *rec_varid, int *norec_varid, int num_types, int *varid_4d) @@ -88,10 +145,7 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, 9223372036854775827ULL, 9223372036854775828ULL, 9223372036854775837ULL, 9223372036854775838ULL}; #endif /* _NETCDF4 */ - int expected_int_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1, 2, 1, 3, 2, 3, 2, 4, 3}; - float expected_float_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1.5, 2, 1, 3, 2.5, 3, 2, 4, 3.5}; void *data_in = NULL; - void *data_in2 = NULL; void *norec_data_in = NULL; /* Reopen the file. */ @@ -238,57 +292,24 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, } } - /* Check the 4D vars. */ - for (int v = 0; v < NUM_4D_VARS; v++) - { - int xtype; - PIO_Offset size; - - /* Get the type of the 4d var. */ - if ((ret = PIOc_inq_vartype(ncid, varid_4d[v], &xtype))) - BAIL(ret); - - /* Get the size of this type. */ - if ((ret = PIOc_inq_type(ncid, xtype, NULL, &size))) - BAIL(ret); - - /* Allocate memory for data. */ - if (!(data_in2 = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC))) - BAIL(PIO_ENOMEM); - - /* Read the data. */ - if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in2))) - BAIL(ret); + /* Free memory. */ + free(data_in); + data_in = NULL; + free(norec_data_in); + norec_data_in = NULL; - /* Check each element of data. */ - for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++) - { - switch (xtype) - { - case PIO_INT: - if (((int *)data_in2)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) - BAIL(ERR_WRONG); - break; - case PIO_FLOAT: - if (((float *)data_in2)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) - BAIL(ERR_WRONG); - break; - default: - BAIL(ERR_WRONG); - } - } - } + /* Check the 4D vars. */ + if ((ret = check_4d_vars(my_rank, ncid, varid_4d))) + BAIL(ret); } /* Close the file. */ if ((ret = PIOc_closefile(ncid))) - BAIL(ret); + ERR(ret); exit: if (data_in) free(data_in); - if (data_in2) - free(data_in2); if (norec_data_in) free(norec_data_in); return ret; From 7e799c7979480208531a1ad29958d740dc7560b5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 22 Mar 2019 08:48:46 -0600 Subject: [PATCH 6/9] turned on address sanitizer in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 433a1910e11..c50b7249990 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ env: - CC=mpicc - FC=mpif90 - CPPFLAGS='-I/usr/include' - - CFLAGS='-std=c99' + - CFLAGS='-std=c99 -fsanitize=address -fno-omit-frame-pointer' - LDFLAGS='-L/usr/lib' script: From 26ae9dbb7ff9104582684b648bfc384dd03ecccd Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 22 Mar 2019 09:04:19 -0600 Subject: [PATCH 7/9] travis with address sanitizer --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c50b7249990..3261464ec09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,11 +36,12 @@ env: - CC=mpicc - FC=mpif90 - CPPFLAGS='-I/usr/include' - - CFLAGS='-std=c99 -fsanitize=address -fno-omit-frame-pointer' + - CFLAGS='-std=c99' - LDFLAGS='-L/usr/lib' script: - autoreconf -i + - export CFLAGS='-std=c99 -fsanitize=address -fno-omit-frame-pointer' - ./configure - make -j distcheck - rm -rf build From 0d3556e6b7941632b57002727593b544fa962176 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 22 Mar 2019 09:23:22 -0600 Subject: [PATCH 8/9] for travis, only address sanitize autotools build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3261464ec09..a85b170aab1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,7 @@ script: - export CFLAGS='-std=c99 -fsanitize=address -fno-omit-frame-pointer' - ./configure - make -j distcheck + - export CFLAGS='-std=c99' - rm -rf build - mkdir build - cd build From 64496c8da77a519207d50468b0bab4a89b829805 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 22 Mar 2019 10:03:32 -0600 Subject: [PATCH 9/9] fixed warnings --- src/clib/pio_lists.c | 13 +++---------- src/clib/pio_rearrange.c | 1 - 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/clib/pio_lists.c b/src/clib/pio_lists.c index 090e457e85a..3ac5baea25a 100644 --- a/src/clib/pio_lists.c +++ b/src/clib/pio_lists.c @@ -23,18 +23,12 @@ static file_desc_t *current_file = NULL; */ void pio_add_to_file_list(file_desc_t *file) { - file_desc_t *cfile; - assert(file); - /* Get a pointer to the global list of files. */ - cfile = pio_file_list; - /* Keep a global pointer to the current file. */ current_file = file; - /* If there is nothing in the list, then file will be the first - * entry. Otherwise, move to end of the list. */ + /* Add file to list. */ HASH_ADD_INT(pio_file_list, pio_ncid, file); } @@ -279,10 +273,10 @@ int pio_delete_iodesc_from_list(int ioid) io_desc_t *ciodesc; ciodesc = pio_get_iodesc_from_id(ioid); - if(ciodesc) + if (ciodesc) { HASH_DEL(pio_iodesc_list, ciodesc); - if (current_iodesc = ciodesc) + if (current_iodesc == ciodesc) current_iodesc = pio_iodesc_list; free(ciodesc); return PIO_NOERR; @@ -367,7 +361,6 @@ int get_var_desc(int varid, var_desc_t **varlist, var_desc_t **var_desc) int delete_var_desc(int varid, var_desc_t **varlist) { var_desc_t *v; - var_desc_t *prev = NULL; int ret; /* Check inputs. */ diff --git a/src/clib/pio_rearrange.c b/src/clib/pio_rearrange.c index 8aa1c596cb7..e5ebec55de0 100644 --- a/src/clib/pio_rearrange.c +++ b/src/clib/pio_rearrange.c @@ -991,7 +991,6 @@ int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, int niotasks; int mpierr; /* Return code from MPI calls. */ int ret; - void *tmparray; /* Check inputs. */ pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__);