From 0be3522f88330d7ae95dfd6ce569bcf7dccb4ab1 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 13 Mar 2019 13:42:00 -0600 Subject: [PATCH 01/47] trying cmake build in travis --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 25dcaaf9a27..3a05a21c877 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ before_script: env: global: - CC=mpicc + - FC=mpifort - CPPFLAGS='-I/usr/include' - CFLAGS='-std=c99' - LDFLAGS='-L/usr/lib' @@ -30,4 +31,10 @@ script: - autoreconf -i - ./configure - make -j distcheck + - make -j distclean + - rm -rf build + - mkdir build + - cd build + - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DWITH_PNETCDF=Off .. + - make From ebdfa770bad8cfb69d0df90c7b1136d89a9171b5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 13 Mar 2019 13:48:09 -0600 Subject: [PATCH 02/47] trying cmake build in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3a05a21c877..111639746c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,6 @@ script: - rm -rf build - mkdir build - cd build - - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DWITH_PNETCDF=Off .. + - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DWITH_PNETCDF=Off -DPIO_ENABLE_TIMING=Off .. - make From 051ffbf06cf38a1240218173e89224348d8abfd4 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 13 Mar 2019 13:53:33 -0600 Subject: [PATCH 03/47] trying cmake build in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 111639746c9..8e9825157c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_script: env: global: - CC=mpicc - - FC=mpifort + - FC=mpifc - CPPFLAGS='-I/usr/include' - CFLAGS='-std=c99' - LDFLAGS='-L/usr/lib' From c1efe46acb153df35ee4e044850d7d5e396824c6 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 13 Mar 2019 13:57:31 -0600 Subject: [PATCH 04/47] trying cmake build in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8e9825157c3..b83ceeac4af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ before_script: env: global: - CC=mpicc - - FC=mpifc + - FC=mpif90 - CPPFLAGS='-I/usr/include' - CFLAGS='-std=c99' - LDFLAGS='-L/usr/lib' From 733a15af8fdaa655a69f44a9d18a374ad3a5ce2f Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 13 Mar 2019 14:01:04 -0600 Subject: [PATCH 05/47] trying cmake build in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b83ceeac4af..26e555c101c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,5 +36,5 @@ script: - mkdir build - cd build - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DWITH_PNETCDF=Off -DPIO_ENABLE_TIMING=Off .. - - make + - make VERBOSE=1 From 30ce0458f2d04fa038985026fe6d8da07a9ad757 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 13 Mar 2019 15:31:03 -0600 Subject: [PATCH 06/47] trying cmake build in travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 26e555c101c..44bd9f5995d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ before_install: before_script: - export CC=mpicc + - export FC=mpifort - export CPPFLAGS='-I/usr/include' env: From a1f439a82130b9563f79cedb92dba4bae0e75f18 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 13 Mar 2019 15:38:09 -0600 Subject: [PATCH 07/47] trying cmake build in travis --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 44bd9f5995d..f11f55e09c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,16 +23,16 @@ before_script: env: global: - CC=mpicc - - FC=mpif90 + - FC=mpifort - CPPFLAGS='-I/usr/include' - CFLAGS='-std=c99' - LDFLAGS='-L/usr/lib' script: - - autoreconf -i - - ./configure - - make -j distcheck - - make -j distclean +# - autoreconf -i +# - ./configure +# - make -j distcheck +# - make -j distclean - rm -rf build - mkdir build - cd build From 8e5dc567840df8e05a88c2879890675e2d510468 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 13 Mar 2019 15:43:55 -0600 Subject: [PATCH 08/47] trying cmake build in travis --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f11f55e09c9..7ed17c542e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,13 +29,11 @@ env: - LDFLAGS='-L/usr/lib' script: -# - autoreconf -i -# - ./configure -# - make -j distcheck -# - make -j distclean + - pwd - rm -rf build - mkdir build - cd build - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DWITH_PNETCDF=Off -DPIO_ENABLE_TIMING=Off .. - make VERBOSE=1 + From c9d014e8ff6632fb71f03dc4005b360dfe9ec768 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 13 Mar 2019 16:08:52 -0600 Subject: [PATCH 09/47] trying cmake build in travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7ed17c542e5..1992b7e7387 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,13 +17,13 @@ before_install: before_script: - export CC=mpicc - - export FC=mpifort + - export FC=mpif90 - export CPPFLAGS='-I/usr/include' env: global: - CC=mpicc - - FC=mpifort + - FC=mpif90 - CPPFLAGS='-I/usr/include' - CFLAGS='-std=c99' - LDFLAGS='-L/usr/lib' From 4a6b779bf0bbce3ae6903cf09dcad442fce8ca18 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 06:24:59 -0600 Subject: [PATCH 10/47] more travis work --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1992b7e7387..a4c4323de21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,5 +35,7 @@ script: - cd build - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DWITH_PNETCDF=Off -DPIO_ENABLE_TIMING=Off .. - make VERBOSE=1 + - make test VERBOSE=1 + - make clean VERBOSE=1 From 0923e2172c6140bf5a77b116bc7f32f03801cf92 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 06:35:05 -0600 Subject: [PATCH 11/47] more work on travis build --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4c4323de21..9d907dc9ef4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,7 +35,9 @@ script: - cd build - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DWITH_PNETCDF=Off -DPIO_ENABLE_TIMING=Off .. - make VERBOSE=1 - - make test VERBOSE=1 - - make clean VERBOSE=1 + - make tests VERBOSE=1 + - cd tests/cunit + - mpiexec -n 4 ./test_darray + From 10fa1ffec63398bf82d858aa3b2af315b6b5ed71 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 06:51:17 -0600 Subject: [PATCH 12/47] more work on travis build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9d907dc9ef4..a56acc2fe9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,6 @@ script: - make tests VERBOSE=1 - cd tests/cunit - mpiexec -n 4 ./test_darray - + - make test From 2cf6a296fa8ad866ed8bb6d0c419add4c4fa40ba Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 06:56:59 -0600 Subject: [PATCH 13/47] more work on travis build --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a56acc2fe9a..0eae0508d67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,8 +36,6 @@ script: - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DWITH_PNETCDF=Off -DPIO_ENABLE_TIMING=Off .. - make VERBOSE=1 - make tests VERBOSE=1 - - cd tests/cunit - - mpiexec -n 4 ./test_darray - make test From 0c8d71d6d7abcd609b7060e69233fd8f5bd1dc89 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 07:03:45 -0600 Subject: [PATCH 14/47] more work on travis build --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0eae0508d67..ee97cd4211d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,9 @@ env: - LDFLAGS='-L/usr/lib' script: - - pwd + - autoreconf -i + - ./configure + - make -j distcheck - rm -rf build - mkdir build - cd build From 2c73319b364f90c1764bec0ac30abe30a5e91815 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 07:18:22 -0600 Subject: [PATCH 15/47] trying to limit travis to pr build only --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index ee97cd4211d..1da1f0c06e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ language: c dist: trusty sudo: false +branches: + only: + - master + addons: apt: sources: From 02b13465a4b3ca314535e9cfbe75a82b3d1778d1 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 07:26:36 -0600 Subject: [PATCH 16/47] trying to install pnetcdf --- .travis.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1da1f0c06e3..f07396e1103 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,14 @@ before_script: - export CC=mpicc - export FC=mpif90 - export CPPFLAGS='-I/usr/include' - + - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.11.0.tar.gz + - tar -xzvf pnetcdf-1.11.0.tar.gz + - ls -l + - pushd pnetcdf-1.11.0 + - ./configure --prefix=/usr + - make + - sudo make install + - popd env: global: - CC=mpicc From 52915d4ff12f62bb45b3a6d59a4a8b4cfef5dc98 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 07:33:44 -0600 Subject: [PATCH 17/47] trying to install shared library pnetcdf --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f07396e1103..a27814aee87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,7 @@ before_script: - tar -xzvf pnetcdf-1.11.0.tar.gz - ls -l - pushd pnetcdf-1.11.0 - - ./configure --prefix=/usr + - ./configure --prefix=/usr --enable-shared - make - sudo make install - popd @@ -46,7 +46,7 @@ script: - rm -rf build - mkdir build - cd build - - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DWITH_PNETCDF=Off -DPIO_ENABLE_TIMING=Off .. + - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. - make VERBOSE=1 - make tests VERBOSE=1 - make test From f99b500831ea804f96a8139b02d339e6d69f7cd3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 08:39:19 -0600 Subject: [PATCH 18/47] trying to install shared library pnetcdf --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a27814aee87..433a2e1c556 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,6 +49,8 @@ script: - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. - make VERBOSE=1 - make tests VERBOSE=1 + - cd tests/cunit + - mpiexec -n 4 ./test_pioc - make test From fa5a046e7f95b289ef4db941fe9df5c914c89cca Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 08:50:51 -0600 Subject: [PATCH 19/47] fixing over-sensitive error string test_pioc.c --- tests/cunit/test_pioc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cunit/test_pioc.c b/tests/cunit/test_pioc.c index 61292849bc8..34a19d0fb69 100644 --- a/tests/cunit/test_pioc.c +++ b/tests/cunit/test_pioc.c @@ -540,7 +540,7 @@ int test_iotypes(int my_rank) int check_strerror_netcdf(int my_rank) { #define NUM_NETCDF_TRIES 4 - int errcode[NUM_NETCDF_TRIES] = {PIO_EBADID, NC4_LAST_ERROR - 1, 0, 1}; + int errcode[NUM_NETCDF_TRIES] = {PIO_EBADID, NC4_LAST_ERROR - 10, 0, 1}; const char *expected[NUM_NETCDF_TRIES] = {"NetCDF: Not a valid ID", "Unknown Error: Unrecognized error code", "No error", nc_strerror(1)}; From 01f1cb9d6542463fffc6fb18fb03133f5d8ad3a2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 09:00:27 -0600 Subject: [PATCH 20/47] fixing over-sensitive error string test_pioc.c --- tests/cunit/test_pioc.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/cunit/test_pioc.c b/tests/cunit/test_pioc.c index 34a19d0fb69..2b5f1cfdb77 100644 --- a/tests/cunit/test_pioc.c +++ b/tests/cunit/test_pioc.c @@ -539,11 +539,10 @@ int test_iotypes(int my_rank) */ int check_strerror_netcdf(int my_rank) { -#define NUM_NETCDF_TRIES 4 - int errcode[NUM_NETCDF_TRIES] = {PIO_EBADID, NC4_LAST_ERROR - 10, 0, 1}; +#define NUM_NETCDF_TRIES 3 + int errcode[NUM_NETCDF_TRIES] = {PIO_EBADID, 0, 1}; const char *expected[NUM_NETCDF_TRIES] = {"NetCDF: Not a valid ID", - "Unknown Error: Unrecognized error code", "No error", - nc_strerror(1)}; + "No error", nc_strerror(1)}; int ret; if ((ret = check_error_strings(my_rank, NUM_NETCDF_TRIES, errcode, expected))) From 0b499b94e2f434549972df4a83c572ceba967843 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 09:16:00 -0600 Subject: [PATCH 21/47] fixing over-sensitive error string test_pioc.c --- tests/cunit/test_pioc.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/cunit/test_pioc.c b/tests/cunit/test_pioc.c index 2b5f1cfdb77..7509db9f54d 100644 --- a/tests/cunit/test_pioc.c +++ b/tests/cunit/test_pioc.c @@ -626,14 +626,13 @@ int check_strerror_pnetcdf(int my_rank) */ int check_strerror_pio(int my_rank) { -#define NUM_PIO_TRIES 6 +#define NUM_PIO_TRIES 5 int errcode[NUM_PIO_TRIES] = {PIO_EBADID, - NC_ENOTNC3, NC4_LAST_ERROR - 1, 0, 1, + NC_ENOTNC3, 0, 1, PIO_EBADIOTYPE}; const char *expected[NUM_PIO_TRIES] = {"NetCDF: Not a valid ID", "NetCDF: Attempting netcdf-3 operation on netcdf-4 file", - "Unknown Error: Unrecognized error code", "No error", - nc_strerror(1), "Bad IO type"}; + "No error", nc_strerror(1), "Bad IO type"}; int ret; if ((ret = check_error_strings(my_rank, NUM_PIO_TRIES, errcode, expected))) From 11c1e9243e3f94b69c14309d93fe3f80722d55b2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 10:10:30 -0600 Subject: [PATCH 22/47] cleaning up travis script --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 433a2e1c556..1af34efc718 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,8 +49,6 @@ script: - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. - make VERBOSE=1 - make tests VERBOSE=1 - - cd tests/cunit - - mpiexec -n 4 ./test_pioc - - make test + - make test VERBOSE=1 From 67f44d989373d270feb0f91396f46f05ec4c49ea Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 14 Mar 2019 10:54:06 -0600 Subject: [PATCH 23/47] now build netcdf-4 from source in travis --- .travis.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1af34efc718..6689dea82c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ addons: sources: - ubuntu-toolchain-r-test packages: - - pkg-config netcdf-bin libnetcdf-dev openmpi-bin libopenmpi-dev gfortran + - pkg-config openmpi-bin libopenmpi-dev gfortran zlib1g-dev before_install: - test -n $CC && unset CC @@ -23,12 +23,25 @@ before_script: - export CC=mpicc - export FC=mpif90 - export CPPFLAGS='-I/usr/include' + - wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.5/src/hdf5-1.10.5.tar.gz + - tar -xzvf hdf5-1.10.5.tar.gz + - pushd hdf5-1.10.5 + - ./configure --prefix=/usr --enable-parallel + - make + - sudo make install + - popd + - wget https://github.com/Unidata/netcdf-c/archive/v4.6.3.tar.gz + - tar -xzvf v4.6.3.tar.gz + - pushd netcdf-c-4.6.3 + - ./configure --prefix=/usr + - make + - sudo make install + - popd - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.11.0.tar.gz - tar -xzvf pnetcdf-1.11.0.tar.gz - - ls -l - pushd pnetcdf-1.11.0 - ./configure --prefix=/usr --enable-shared - - make + - make - sudo make install - popd env: From 9b182c5892d6b16bb2531a1a6faa8d5680564a2c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 19 Mar 2019 06:26:10 -0600 Subject: [PATCH 24/47] switched to HDF5 1.10.4 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6689dea82c2..80d1819e2cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,9 +23,9 @@ before_script: - export CC=mpicc - export FC=mpif90 - export CPPFLAGS='-I/usr/include' - - wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.5/src/hdf5-1.10.5.tar.gz - - tar -xzvf hdf5-1.10.5.tar.gz - - pushd hdf5-1.10.5 + - wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.gz + - tar -xzvf hdf5-1.10.4.tar.gz + - pushd hdf5-1.10.4 - ./configure --prefix=/usr --enable-parallel - make - sudo make install From 5ec13ea35aaa9692528d3dff95721c3df82c2a95 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 19 Mar 2019 06:48:43 -0600 Subject: [PATCH 25/47] switched to pnetcdf 1.10 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 80d1819e2cb..9b8b8c23ba7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,9 +37,9 @@ before_script: - make - sudo make install - popd - - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.11.0.tar.gz - - tar -xzvf pnetcdf-1.11.0.tar.gz - - pushd pnetcdf-1.11.0 + - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.10.0.tar.gz + - tar -xzvf pnetcdf-1.10.0.tar.gz + - pushd pnetcdf-1.10.0 - ./configure --prefix=/usr --enable-shared - make - sudo make install From b4e8d5fce22cbcd96d2bdda27cb5952a376a78b8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 19 Mar 2019 07:26:36 -0600 Subject: [PATCH 26/47] switched to pnetcdf 1.10 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9b8b8c23ba7..b203ce62897 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,8 +27,8 @@ before_script: - tar -xzvf hdf5-1.10.4.tar.gz - pushd hdf5-1.10.4 - ./configure --prefix=/usr --enable-parallel - - make - - sudo make install + - make &> /dev/null + - sudo make install - popd - wget https://github.com/Unidata/netcdf-c/archive/v4.6.3.tar.gz - tar -xzvf v4.6.3.tar.gz From 2739c712ba330e2fc207895b899cbb816b2cf8de Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 19 Mar 2019 07:46:49 -0600 Subject: [PATCH 27/47] dorking around with cmake build in travis and cursing cmake --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b203ce62897..a9693370e59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,9 +37,9 @@ before_script: - make - sudo make install - popd - - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.10.0.tar.gz - - tar -xzvf pnetcdf-1.10.0.tar.gz - - pushd pnetcdf-1.10.0 + - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.9.0.tar.gz + - tar -xzvf pnetcdf-1.9.0.tar.gz + - pushd pnetcdf-1.9.0 - ./configure --prefix=/usr --enable-shared - make - sudo make install From 37eb88a8c4369117d94da1f75e1d7b3e1b4fb718 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 19 Mar 2019 07:49:25 -0600 Subject: [PATCH 28/47] removed cmake build in travis --- .travis.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index a9693370e59..2cc24cfd76e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,12 +56,5 @@ script: - autoreconf -i - ./configure - make -j distcheck - - rm -rf build - - mkdir build - - cd build - - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. - - make VERBOSE=1 - - make tests VERBOSE=1 - - make test VERBOSE=1 From bcbcaae07919b2fcfc8b876517a948269eff6f5e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 19 Mar 2019 10:33:31 -0600 Subject: [PATCH 29/47] switched back to pnetcdf 1.11.0 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2cc24cfd76e..36b65ee7e27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,9 +37,9 @@ before_script: - make - sudo make install - popd - - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.9.0.tar.gz - - tar -xzvf pnetcdf-1.9.0.tar.gz - - pushd pnetcdf-1.9.0 + - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.11.0.tar.gz + - tar -xzvf pnetcdf-1.11.0.tar.gz + - pushd pnetcdf-1.11.0 - ./configure --prefix=/usr --enable-shared - make - sudo make install From ef653a64dd40808fc13fcebb5bd268ab714ecb2b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 19 Mar 2019 13:45:31 -0600 Subject: [PATCH 30/47] rolling back to most recent working travis build for reasons of mental health --- .travis.yml | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36b65ee7e27..433a1910e11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ addons: sources: - ubuntu-toolchain-r-test packages: - - pkg-config openmpi-bin libopenmpi-dev gfortran zlib1g-dev + - pkg-config netcdf-bin libnetcdf-dev openmpi-bin libopenmpi-dev gfortran before_install: - test -n $CC && unset CC @@ -23,25 +23,12 @@ before_script: - export CC=mpicc - export FC=mpif90 - export CPPFLAGS='-I/usr/include' - - wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.gz - - tar -xzvf hdf5-1.10.4.tar.gz - - pushd hdf5-1.10.4 - - ./configure --prefix=/usr --enable-parallel - - make &> /dev/null - - sudo make install - - popd - - wget https://github.com/Unidata/netcdf-c/archive/v4.6.3.tar.gz - - tar -xzvf v4.6.3.tar.gz - - pushd netcdf-c-4.6.3 - - ./configure --prefix=/usr - - make - - sudo make install - - popd - wget https://parallel-netcdf.github.io/Release/pnetcdf-1.11.0.tar.gz - tar -xzvf pnetcdf-1.11.0.tar.gz + - ls -l - pushd pnetcdf-1.11.0 - ./configure --prefix=/usr --enable-shared - - make + - make - sudo make install - popd env: @@ -56,5 +43,10 @@ script: - autoreconf -i - ./configure - make -j distcheck - - + - rm -rf build + - mkdir build + - cd build + - cmake -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off .. + - make VERBOSE=1 + - make tests VERBOSE=1 + - make test VERBOSE=1 \ No newline at end of file From 8b2952f4810e9bee95a5e47d6af2192a1e3eedc4 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 20 Mar 2019 05:52:27 -0600 Subject: [PATCH 31/47] fixed shell script warnings --- examples/c/run_tests.sh | 3 ++- tests/cunit/run_tests.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/c/run_tests.sh b/examples/c/run_tests.sh index c4eb19bb6ad..1cfb586d292 100755 --- a/examples/c/run_tests.sh +++ b/examples/c/run_tests.sh @@ -1,3 +1,4 @@ +#!/bin/sh # This is a test script for PIO examples. # Ed Hartnett 5/7/18 @@ -5,7 +6,7 @@ set -e # Stop loop if ctrl-c is pressed. -trap exit SIGINT SIGTERM +trap exit INT TERM printf 'running PIO examples...\n' diff --git a/tests/cunit/run_tests.sh b/tests/cunit/run_tests.sh index 5261b41aee5..105bfd1b90c 100755 --- a/tests/cunit/run_tests.sh +++ b/tests/cunit/run_tests.sh @@ -1,3 +1,4 @@ +#!/bin/sh # This is a test script for PIO. # Ed Hartnett @@ -5,7 +6,7 @@ set -e # Stop loop if ctrl-c is pressed. -trap exit SIGINT SIGTERM +trap exit INT TERM printf 'running PIO tests...\n' From 88c420110e0198138970ab40ffdf1e3fd27e5be8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 20 Mar 2019 05:54:11 -0600 Subject: [PATCH 32/47] fixed missing initializations in pio_msg.c --- src/clib/pio_msg.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c index ec8003a8a47..38dd3f25fbc 100644 --- a/src/clib/pio_msg.c +++ b/src/clib/pio_msg.c @@ -1225,9 +1225,9 @@ int inq_var_deflate_handler(iosystem_desc_t *ios) char shuffle_present; char deflate_present; char deflate_level_present; - int shuffle, *shufflep; - int deflate, *deflatep; - int deflate_level, *deflate_levelp; + int shuffle, *shufflep = NULL; + int deflate, *deflatep = NULL; + int deflate_level, *deflate_levelp = NULL; int mpierr; assert(ios); @@ -2331,9 +2331,9 @@ int get_chunk_cache_handler(iosystem_desc_t *ios) int iosysid; int iotype; char size_present, nelems_present, preemption_present; - PIO_Offset size, *sizep; - PIO_Offset nelems, *nelemsp; - float preemption, *preemptionp; + PIO_Offset size, *sizep = NULL; + PIO_Offset nelems, *nelemsp = NULL; + float preemption, *preemptionp = NULL; int mpierr = MPI_SUCCESS; /* Return code from MPI function codes. */ LOG((1, "get_chunk_cache_handler called")); @@ -2384,9 +2384,9 @@ int get_var_chunk_cache_handler(iosystem_desc_t *ios) int ncid; int varid; char size_present, nelems_present, preemption_present; - PIO_Offset size, *sizep; - PIO_Offset nelems, *nelemsp; - float preemption, *preemptionp; + PIO_Offset size, *sizep = NULL; + PIO_Offset nelems, *nelemsp = NULL; + float preemption, *preemptionp = NULL; int mpierr = MPI_SUCCESS; /* Return code from MPI function codes. */ LOG((1, "get_var_chunk_cache_handler called")); From b58ac3f9bb4e9b7c7530f19e6a993bca823bea09 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 20 Mar 2019 05:55:44 -0600 Subject: [PATCH 33/47] removed unused var in test --- tests/general/test_memleak.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/general/test_memleak.c b/tests/general/test_memleak.c index 3c81f477df1..f9ecbd720df 100644 --- a/tests/general/test_memleak.c +++ b/tests/general/test_memleak.c @@ -122,9 +122,6 @@ main(int argc, char **argv) * example. */ int ioproc_stride = 1; - /** Number of the aggregator? Always 0 in this example. */ - int numAggregator = 0; - /** Zero based rank of first processor to be used for I/O. */ int ioproc_start = 0; From cffcd5510bb41946764f1b41774681cac69e9244 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 20 Mar 2019 05:57:17 -0600 Subject: [PATCH 34/47] fixed memory leak on error --- src/clib/pio_getput_int.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/clib/pio_getput_int.c b/src/clib/pio_getput_int.c index 01f2a4102a2..63541a679bf 100644 --- a/src/clib/pio_getput_int.c +++ b/src/clib/pio_getput_int.c @@ -866,8 +866,14 @@ int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf) /* Find the dimension lengths. */ for (int d = 0; d < ndims; d++) + { if ((ierr = PIOc_inq_dimlen(ncid, dimids[d], &countp[d]))) + { + free(startp); + free(countp); return pio_err(ios, file, ierr, __FILE__, __LINE__); + } + } /* Set up start array. */ for (int d = 0; d < ndims; d++) From 0869129cb696c802ce35241a2ce1498b911f9a8d Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 20 Mar 2019 05:58:20 -0600 Subject: [PATCH 35/47] removed unused var --- src/clib/pio_rearrange.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/clib/pio_rearrange.c b/src/clib/pio_rearrange.c index 688b30c85eb..8aa1c596cb7 100644 --- a/src/clib/pio_rearrange.c +++ b/src/clib/pio_rearrange.c @@ -2076,7 +2076,6 @@ void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc) if (!(wall = bget(2 * 4 * log2 * sizeof(double)))) return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__); double mintime; - int k = 0; if ((mpierr = MPI_Barrier(mycomm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); From b996ca4059632f17e55196b3a1680ed30955bda3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 20 Mar 2019 12:40:41 -0600 Subject: [PATCH 36/47] add --disable-pnetcdf configure option and require it if pnetcdf is not found --- configure.ac | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/configure.ac b/configure.ac index 0779dfde6a2..4246aa0e991 100644 --- a/configure.ac +++ b/configure.ac @@ -38,6 +38,17 @@ if test "x$enable_logging" = xyes; then AC_DEFINE([PIO_ENABLE_LOGGING], 1, [If true, turn on logging.]) fi +# Does the user want to disable pnetcdf? +AC_MSG_CHECKING([whether pnetcdf is to be used]) +AC_ARG_ENABLE([pnetcdf], + [AS_HELP_STRING([--disable-pnetcdf], + [Disable pnetcdf use.])]) +test "x$enable_pnetcdf" = xno || enable_pnetcdf=yes +AC_MSG_RESULT([$enable_pnetcdf]) +if test "x$enable_logging" = xyes; then + AC_DEFINE([PIO_ENABLE_LOGGING], 1, [If true, turn on logging.]) +fi + # NetCDF (at least classic) is required for PIO to build. AC_DEFINE([_NETCDF], [1], [netCDF classic library available]) @@ -66,6 +77,9 @@ AC_CHECK_LIB([netcdf], [nc_create], [], [AC_MSG_ERROR([Can't find or link to the # Check for pnetcdf library. AC_CHECK_LIB([pnetcdf], [ncmpi_create], [], []) +if test "x$ac_cv_lib_pnetcdf_ncmpi_create" = xno -a $enable_pnetcdf = yes; then + AC_MSG_ERROR([Pnetcdf not found. Set CPPFLAGS/LDFLAGS or use --disable-pnetcdf.]) +fi # If we have parallel-netcdf, then set these as well. if test x$ac_cv_lib_pnetcdf_ncmpi_create = xyes; then From fe1019e81ea9668567423e528deadd878bb4bee5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 20 Mar 2019 14:23:34 -0600 Subject: [PATCH 37/47] added some timing to test_perf2.c --- tests/cunit/pio_tests.h | 8 ++++ tests/cunit/test_darray_async.c | 79 +++++++++++++++++---------------- tests/cunit/test_perf2.c | 18 ++++++++ 3 files changed, 67 insertions(+), 38 deletions(-) diff --git a/tests/cunit/pio_tests.h b/tests/cunit/pio_tests.h index d1d0cae9894..bf6bd931646 100644 --- a/tests/cunit/pio_tests.h +++ b/tests/cunit/pio_tests.h @@ -77,6 +77,14 @@ return e; \ } while (0) +/** Handle non-MPI errors by finalizing the MPI library and goto + * exit. */ +#define BAIL(e) do { \ + fprintf(stderr, "%d Error %d in %s, line %d\n", my_rank, e, __FILE__, __LINE__); \ + MPI_Finalize(); \ + goto exit; \ + } while (0) + /** Global err buffer for MPI. When there is an MPI error, this buffer * is used to store the error message that is associated with the MPI * error. */ diff --git a/tests/cunit/test_darray_async.c b/tests/cunit/test_darray_async.c index 199ccad6e91..39c062ab8f5 100644 --- a/tests/cunit/test_darray_async.c +++ b/tests/cunit/test_darray_async.c @@ -61,24 +61,24 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, { int ncid; int varid[NVAR] = {0, 1, 2, 3}; - void *data_in; - void *data_in_norec; + void *data_in = NULL; + void *data_in_norec = NULL; PIO_Offset type_size; int ret; /* Reopen the file. */ if ((ret = PIOc_openfile(iosysid, &ncid, &iotype, data_filename, NC_NOWRITE))) - ERR(ret); + BAIL(ret); /* Get the size of the type. */ if ((ret = PIOc_inq_type(ncid, piotype, NULL, &type_size))) - ERR(ret); + BAIL(ret); /* Allocate memory to read data. */ if (!(data_in = malloc(LAT_LEN * LON_LEN * type_size * NREC))) - ERR(PIO_ENOMEM); + BAIL(PIO_ENOMEM); if (!(data_in_norec = malloc(LAT_LEN * LON_LEN * type_size))) - ERR(PIO_ENOMEM); + BAIL(PIO_ENOMEM); /* We have two sets of variables, those with unlimted, and those * without unlimited dimension. */ @@ -90,12 +90,12 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, /* Read the record data. The values we expect are: 10, 11, 20, 21, 30, * 31, in each of three records. */ if ((ret = PIOc_get_var(ncid, rec_varid, data_in))) - ERR(ret); + BAIL(ret); /* Read the non-record data. The values we expect are: 10, 11, 20, 21, 30, * 31. */ if ((ret = PIOc_get_var(ncid, norec_varid, data_in_norec))) - ERR(ret); + BAIL(ret); /* Check the results. */ for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++) @@ -105,52 +105,52 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, { case PIO_BYTE: if (((signed char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; case PIO_CHAR: if (((char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; case PIO_SHORT: if (((short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; case PIO_INT: if (((int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; case PIO_FLOAT: if (((float *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; case PIO_DOUBLE: if (((double *)data_in)[r] != (tmp_r/2 + 1) * 10.0 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; #ifdef _NETCDF4 case PIO_UBYTE: if (((unsigned char *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; case PIO_USHORT: if (((unsigned short *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; case PIO_UINT: if (((unsigned int *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; case PIO_INT64: if (((long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; case PIO_UINT64: if (((unsigned long long *)data_in)[r] != (tmp_r/2 + 1) * 10 + tmp_r % 2) - ERR(ret); + BAIL(ret); break; #endif /* _NETCDF4 */ default: - ERR(ERR_WRONG); + BAIL(ERR_WRONG); } } @@ -161,65 +161,68 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, { case PIO_BYTE: if (((signed char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; case PIO_CHAR: if (((char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; case PIO_SHORT: if (((short *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; case PIO_INT: if (((int *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; case PIO_FLOAT: if (((float *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; case PIO_DOUBLE: if (((double *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; #ifdef _NETCDF4 case PIO_UBYTE: if (((unsigned char *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; case PIO_USHORT: if (((unsigned short *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; case PIO_UINT: if (((unsigned int *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; case PIO_INT64: if (((long long *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; case PIO_UINT64: if (((unsigned long long *)data_in_norec)[r] != (r/2 + 1) * 20.0 + r%2) - ERR(ret); + BAIL(ret); break; #endif /* _NETCDF4 */ default: - ERR(ERR_WRONG); + BAIL(ERR_WRONG); } } } /* next var set */ - /* Free resources. */ - free(data_in); - free(data_in_norec); - /* Close the file. */ if ((ret = PIOc_closefile(ncid))) - ERR(ret); + BAIL(ret); - return 0; +exit: + /* Free resources. */ + if (data_in) + free(data_in); + if (data_in_norec) + free(data_in_norec); + + return ret; } /* Run a simple test using darrays with async. */ diff --git a/tests/cunit/test_perf2.c b/tests/cunit/test_perf2.c index c3bf3735389..08fd304be23 100644 --- a/tests/cunit/test_perf2.c +++ b/tests/cunit/test_perf2.c @@ -8,6 +8,7 @@ #include #include #include +#include /* The number of tasks this test should run on. */ #define TARGET_NTASKS 4 @@ -124,6 +125,9 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank int varid; /* The ID of the netCDF varable. */ int ret; /* Return code. */ PIO_Offset arraylen = EXPECTED_MAPLEN; + struct timeval starttime, endtime; + long long startt, endt; + long long delta; int int_fillvalue = NC_FILL_INT; void *fillvalue = NULL; int *test_data; @@ -163,6 +167,9 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank /* sprintf(filename, "data_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); */ sprintf(filename, "data__iotype_.nc"); + /* Start the clock. */ + gettimeofday(&starttime, NULL); + /* Create the netCDF output file. */ if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) ERR(ret); @@ -204,6 +211,17 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank if ((ret = PIOc_closefile(ncid))) ERR(ret); + /* Stop the clock. */ + gettimeofday(&endtime, NULL); + + /* Compute the time delta */ + startt = (1000000 * starttime.tv_sec) + starttime.tv_usec; + endt = (1000000 * endtime.tv_sec) + endtime.tv_usec; + delta = (endt - startt)/NUM_TIMESTEPS; + /* if (!my_rank) */ + /* printf("%d\t%d\t%d\t%d\t%lld\n", rearranger, fmt, pio_type, test_multi, */ + /* delta); */ + /* Reopen the file. */ if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE))) ERR(ret); From 79a0e5895f13e75e9ae6aaa3d278198b7334aeb7 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 20 Mar 2019 14:24:12 -0600 Subject: [PATCH 38/47] removed redundant test_perf1.c --- tests/cunit/Makefile.am | 3 +- tests/cunit/test_perf1.c | 382 --------------------------------------- 2 files changed, 1 insertion(+), 384 deletions(-) delete mode 100644 tests/cunit/test_perf1.c diff --git a/tests/cunit/Makefile.am b/tests/cunit/Makefile.am index fab5a0707ed..2811a01be3c 100644 --- a/tests/cunit/Makefile.am +++ b/tests/cunit/Makefile.am @@ -19,7 +19,7 @@ test_darray_multivar3 test_darray_1d test_darray_3d \ test_decomp_uneven test_decomps test_rearr test_darray_async_simple \ test_darray_async test_darray_async_many test_darray_2sync \ test_async_multicomp test_async_multi2 test_async_manyproc \ -test_darray_fill test_decomp_frame test_perf1 test_perf2 +test_darray_fill test_decomp_frame test_perf2 # Tests will run from a bash script. TESTS = run_tests.sh @@ -59,7 +59,6 @@ test_async_multi2_SOURCES = test_async_multi2.c test_common.c pio_tests.h test_async_manyproc_SOURCES = test_async_manyproc.c test_common.c pio_tests.h test_darray_fill_SOURCES = test_darray_fill.c test_common.c pio_tests.h test_decomp_frame_SOURCES = test_decomp_frame.c test_common.c pio_tests.h -test_perf1_SOURCES = test_perf1.c test_common.c pio_tests.h test_perf2_SOURCES = test_perf2.c test_common.c pio_tests.h # Distribute the test script. diff --git a/tests/cunit/test_perf1.c b/tests/cunit/test_perf1.c deleted file mode 100644 index c38e5b27f3a..00000000000 --- a/tests/cunit/test_perf1.c +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Tests for PIO distributed arrays. - * - * @author Ed Hartnett - * @date 2/16/17 - */ -#include -#include -#include -#include -#include - -/* The number of tasks this test should run on. */ -#define TARGET_NTASKS 4 - -/* The minimum number of tasks this test should run on. */ -#define MIN_NTASKS 4 - -/* The name of this test. */ -#define TEST_NAME "test_perf1" - -/* Number of processors that will do IO. */ -#define NUM_IO_PROCS 1 - -/* Number of computational components to create. */ -#define COMPONENT_COUNT 1 - -/* Ranks of different arrays. */ -#define NDIM2 2 -#define NDIM3 3 -#define NDIM4 4 - -/* The length of our sample data along each dimension. */ -#define X_DIM_LEN 16 -#define Y_DIM_LEN 16 -#define Z_DIM_LEN 4 - -/* The number of timesteps of data to write. */ -#define NUM_TIMESTEPS 1 - -/* The number of 4D vars. */ -#define NUM_VARS 1 - -/* The names of variables in the netCDF output files. */ -#define VAR_NAME "Billy-Bob" -#define VAR_NAME2 "Sally-Sue" - -/* Test cases relating to PIOc_write_darray_multi(). */ -#define NUM_TEST_CASES_WRT_MULTI 3 - -/* The dimension names. */ -char dim_name[NDIM4][PIO_MAX_NAME + 1] = {"timestep", "x", "y", "z"}; - -/* Length of the dimensions in the sample data. */ -int dim_len[NDIM4] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN}; - -/* Test with two rearrangers. */ -#define NUM_REARRANGERS_TO_TEST 2 - -/* Test with several types. */ -#define NUM_TYPES_TO_TEST 3 - -/* Create the decomposition to divide the 4-dimensional sample data - * between tasks. For the purposes of decomposition we are only - * concerned with 3 dimensions - we ignore the unlimited dimension. - * - * @param ntasks the number of available tasks - * @param my_rank rank of this task. - * @param iosysid the IO system ID. - * @param dim_len_3d an array of length 3 with the dim lengths. - * @param ioid a pointer that gets the ID of this decomposition. - * @param pio_type the data type to use for the decomposition. - * @returns 0 for success, error code otherwise. - **/ -int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *dim_len_3d, - int *ioid, int pio_type) -{ - PIO_Offset elements_per_pe; /* Array elements per processing unit. */ - PIO_Offset *compdof; /* The decomposition mapping. */ - int ret; - - /* How many data elements per task? */ - elements_per_pe = dim_len_3d[0] * dim_len_3d[1] * dim_len_3d[2]/ ntasks; - - /* Allocate space for the decomposition array. */ - if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) - return PIO_ENOMEM; - - /* Describe the decomposition. This is a 1-based array, so add 1! */ - for (int i = 0; i < elements_per_pe; i++) - compdof[i] = my_rank * elements_per_pe + i + 1; - - /* Create the PIO decomposition for this test. */ - if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM3, dim_len_3d, elements_per_pe, - compdof, ioid, NULL, NULL, NULL))) - ERR(ret); - - - /* Free the mapping. */ - free(compdof); - - return 0; -} - - -/** - * Do some fake computation. - * - * @return 0 always. - */ -int -do_some_computation(long long int max_i) -{ - for (int i = 0; i < max_i; i++) - { - float a, b, c; - a = b * c; - b = a * c; - c = a * b; - } - return 0; -} - -/** - * Test the darray functionality. Create a netCDF file with 4 - * dimensions and some variables, and use darray to write some data. - * - * @param iosysid the IO system ID. - * @param ioid the ID of the decomposition. - * @param num_flavors the number of IOTYPES available in this build. - * @param flavor array of available iotypes. - * @param my_rank rank of this task. - * @param pio_type the type of the data. - * @param fmt indicates the IOtype. - * @param test_multi true for testing muti-var darray write. - * @param rearranger the rearranger - * @returns 0 for success, error code otherwise. - */ -int test_perf1(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, - int ntasks, int pio_type, int fmt, int test_multi, int rearranger) -{ - char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ - int dimids[NDIM4]; /* The dimension IDs. */ - int ncid; /* The ncid of the netCDF file. */ - int varid[NUM_VARS]; /* The ID of the netCDF varable. */ - int varid2; /* The ID of a varable of different type. */ - int wrong_varid = TEST_VAL_42; /* A wrong ID. */ - int ret; /* Return code. */ - PIO_Offset arraylen = X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks; - struct timeval starttime, endtime; - long long startt, endt; - long long delta; - void *test_data; - int test_data_int[arraylen]; - float test_data_float[arraylen]; - double test_data_double[arraylen]; - - /* Initialize some data. */ - for (int f = 0; f < arraylen; f++) - { - test_data_int[f] = my_rank * 10 + f; - test_data_float[f] = my_rank * 10 + f + 0.5; - test_data_double[f] = my_rank * 100000 + f + 0.5; - } - - /* Create the filename. */ - sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d.nc", - TEST_NAME, flavor[fmt], pio_type, test_multi); - - /* Select the fill value and data. */ - switch (pio_type) - { - case PIO_INT: - test_data = test_data_int; - break; - case PIO_FLOAT: - test_data = test_data_float; - break; - case PIO_DOUBLE: - test_data = test_data_double; - break; - default: - ERR(ERR_WRONG); - } - - /* Create the netCDF output file. */ - { - if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER))) - ERR(ret); - - /* Define netCDF dimensions. */ - for (int d = 0; d < NDIM4; d++) - if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) - ERR(ret); - - /* Define the variables. */ - for (int v = 0; v < NUM_VARS; v++) - { - char var_name[NC_MAX_NAME + 1]; - sprintf(var_name, "var_%d", v); - if ((ret = PIOc_def_var(ncid, var_name, pio_type, NDIM4, dimids, &varid[v]))) - ERR(ret); - } - - /* End define mode. */ - if ((ret = PIOc_enddef(ncid))) - ERR(ret); - } - - /* Start the clock. */ - gettimeofday(&starttime, NULL); - - for (int t = 0; t < NUM_TIMESTEPS; t++) - { - - /* Do some fake computation. */ - if ((ret = do_some_computation(100000))) - ERR(ret); - - /* Write a timestep of data in each var. */ - for (int v = 0; v < NUM_VARS; v++) - { - /* Set the value of the record dimension. */ - if ((ret = PIOc_setframe(ncid, varid[v], t))) - ERR(ret); - - if (!test_multi) - { - /* Write the data. */ - if ((ret = PIOc_write_darray(ncid, varid[v], ioid, arraylen, test_data, NULL))) - ERR(ret); - } - } - } - - /* Close the netCDF file. */ - if ((ret = PIOc_closefile(ncid))) - ERR(ret); - - /* Stop the clock. */ - gettimeofday(&endtime, NULL); - - /* Compute the time delta */ - startt = (1000000 * starttime.tv_sec) + starttime.tv_usec; - endt = (1000000 * endtime.tv_sec) + endtime.tv_usec; - delta = (endt - startt)/NUM_TIMESTEPS; - if (!my_rank) - printf("%d\t%d\t%d\t%d\t%lld\n", rearranger, fmt, pio_type, test_multi, - delta); - - return PIO_NOERR; -} - -/** - * Run a performance benchmark. - * - * @param iosysid the IO system ID. - * @param num_flavors number of available iotypes in the build. - * @param flavor pointer to array of the available iotypes. - * @param my_rank rank of this task. - * @param test_comm the communicator the test is running on. - * @param rearranger the rearranger - * @returns 0 for success, error code otherwise. - */ -int run_benchmark(int iosysid, int num_flavors, int *flavor, int my_rank, int ntasks, - MPI_Comm test_comm, int rearranger, int num_types, int *pio_type) -{ - int ioid, ioid3; - char filename[NC_MAX_NAME + 1]; - int dim_len_3d[NDIM3] = {Z_DIM_LEN, X_DIM_LEN, Y_DIM_LEN}; - long long delta; - int ret; /* Return code. */ - - /* for (int t = 0; t < num_types; t++) */ - for (int t = 0; t < 1; t++) - { - float time; - - /* This will be our file name for writing out decompositions. */ - sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank, - *flavor, pio_type[t]); - - /* Decompose the data over the tasks. */ - if ((ret = create_decomposition_3d(TARGET_NTASKS, my_rank, iosysid, dim_len_3d, - &ioid3, pio_type[t]))) - return ret; - - /* Run a simple performance test. */ - /* for (int fmt = 0; fmt < num_flavors; fmt++) */ - for (int fmt = 0; fmt < 2; fmt++) - { - /* for (int test_multi = 0; test_multi < NUM_TEST_CASES_WRT_MULTI; test_multi++) */ - for (int test_multi = 0; test_multi < 1; test_multi++) - { - if ((ret = test_perf1(iosysid, ioid3, num_flavors, flavor, my_rank, ntasks, - pio_type[t], fmt, test_multi, rearranger))) - return ret; - } - } - - /* Free the PIO decomposition. */ - if ((ret = PIOc_freedecomp(iosysid, ioid3))) - ERR(ret); - } - - return PIO_NOERR; -} - -int -run_some_benchmarks(MPI_Comm test_comm, int my_rank, int ntasks, int num_flavors, int *flavor, - int num_rearr, int *rearranger, int num_types, int *pio_type) -{ - /* Only do something on max_ntasks tasks. */ - if (my_rank < TARGET_NTASKS) - { - int iosysid; /* The ID for the parallel I/O system. */ - int ioproc_stride = 1; /* Stride in the mpi rank between io tasks. */ - int ioproc_start = 0; /* Zero based rank of first processor to be used for I/O. */ - - if (!my_rank) - printf("rearr\tfmt\tpio_type\ttest_multi\ttime\n"); - /* for (int r = 0; r < num_rearr; r++) */ - for (int r = 0; r < 1; r++) - { - int ret; /* Return code. */ - - /* Initialize the PIO IO system. This specifies how - * many and which processors are involved in I/O. */ - if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride, - ioproc_start, rearranger[r], &iosysid))) - return ret; - - /* Run tests. */ - if ((ret = run_benchmark(iosysid, num_flavors, flavor, my_rank, ntasks, - test_comm, rearranger[r], num_types, pio_type))) - return ret; - - /* Finalize PIO system. */ - if ((ret = PIOc_finalize(iosysid))) - return ret; - } /* next rearranger */ - } /* endif my_rank < TARGET_NTASKS */ - - return PIO_NOERR; -} - -/* Run benchmarks. */ -int main(int argc, char **argv) -{ - int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET}; - int my_rank; - int ntasks; - int num_flavors; /* Number of PIO netCDF flavors in this build. */ - int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */ - int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; - MPI_Comm test_comm; /* A communicator for this test. */ - int ret; /* Return code. */ - - /* Initialize test. */ - if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, - MIN_NTASKS, 3, &test_comm))) - ERR(ERR_INIT); - - if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL))) - return ret; - - /* Figure out iotypes. */ - if ((ret = get_iotypes(&num_flavors, flavor))) - ERR(ret); - - /* Run a benchmark. */ - if ((ret = run_some_benchmarks(test_comm, my_rank, ntasks, num_flavors, flavor, NUM_REARRANGERS_TO_TEST, - rearranger, NUM_TYPES_TO_TEST, pio_type))) - ERR(ret); - - /* Finalize the MPI library. */ - if ((ret = pio_test_finalize(&test_comm))) - return ret; - - printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME); - return 0; -} From 48a492385751cd19f2cab93f5acd3bea767ffa9a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 20 Mar 2019 15:01:03 -0600 Subject: [PATCH 39/47] fixed fscan warnings --- src/clib/pioc_support.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c index ae50248417a..646f3867dc2 100644 --- a/src/clib/pioc_support.c +++ b/src/clib/pioc_support.c @@ -880,8 +880,8 @@ int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen, if (!fp) pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); - fscanf(fp,"version %d npes %d ndims %d\n",&rversno, &rnpes, ndims); - + if (fscanf(fp,"version %d npes %d ndims %d\n", &rversno, &rnpes, ndims) != 3) + pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); if (rversno != VERSNO) return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); @@ -895,20 +895,23 @@ int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen, if (!(tdims = calloc(*ndims, sizeof(int)))) return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); for (int i = 0; i < *ndims; i++) - fscanf(fp,"%d ", tdims + i); + if (fscanf(fp,"%d ", tdims + i) != 1) + pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); if ((mpierr = MPI_Bcast(tdims, *ndims, MPI_INT, 0, comm))) return check_mpi(NULL, mpierr, __FILE__, __LINE__); for (int i = 0; i < rnpes; i++) { - fscanf(fp, "%d %lld", &j, &maplen); + if (fscanf(fp, "%d %lld", &j, &maplen) != 2) + pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); if (j != i) // Not sure how this could be possible return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); if (!(tmap = malloc(maplen * sizeof(PIO_Offset)))) return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__); for (j = 0; j < maplen; j++) - fscanf(fp, "%lld ", tmap+j); + if (fscanf(fp, "%lld ", tmap + j) != 1) + pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__); if (i > 0) { From 63abea02f0109fa38b355c9cb98c99b113484353 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 21 Mar 2019 09:17:52 -0600 Subject: [PATCH 40/47] change to trigger CI --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c998e354dc4..c94f5d8c71d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ A high-level Parallel I/O Library for structured grid applications ## Website -For complete documentation, see our website at [http://ncar.github.io/ParallelIO/](http://ncar.github.io/ParallelIO/). +For complete documentation, see our website at +[http://ncar.github.io/ParallelIO/](http://ncar.github.io/ParallelIO/). ## Mailing List From 27a86387e86863b67ae248e5cfd195995078f7c5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 21 Mar 2019 09:26:59 -0600 Subject: [PATCH 41/47] clean up of test_perf2.c --- tests/cunit/test_perf2.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/cunit/test_perf2.c b/tests/cunit/test_perf2.c index 08fd304be23..36668776108 100644 --- a/tests/cunit/test_perf2.c +++ b/tests/cunit/test_perf2.c @@ -125,9 +125,6 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank int varid; /* The ID of the netCDF varable. */ int ret; /* Return code. */ PIO_Offset arraylen = EXPECTED_MAPLEN; - struct timeval starttime, endtime; - long long startt, endt; - long long delta; int int_fillvalue = NC_FILL_INT; void *fillvalue = NULL; int *test_data; @@ -163,6 +160,10 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { + struct timeval starttime, endtime; + long long startt, endt; + long long delta; + /* Create the filename. */ /* sprintf(filename, "data_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); */ sprintf(filename, "data__iotype_.nc"); From a240cab3dbeae6830eef3ffda6f9e58a86ed6a06 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 21 Mar 2019 09:31:00 -0600 Subject: [PATCH 42/47] fixing codacy warnings in test --- tests/cunit/test_darray_async_many.c | 109 ++++++++++++++------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/tests/cunit/test_darray_async_many.c b/tests/cunit/test_darray_async_many.c index d993bfaef29..a32cd0136b5 100644 --- a/tests/cunit/test_darray_async_many.c +++ b/tests/cunit/test_darray_async_many.c @@ -90,40 +90,41 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, #endif /* _NETCDF4 */ int expected_int_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1, 2, 1, 3, 2, 3, 2, 4, 3}; float expected_float_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1.5, 2, 1, 3, 2.5, 3, 2, 4, 3.5}; + void *data_in = NULL; + void *data_in2 = NULL; + void *norec_data_in = NULL; /* Reopen the file. */ if ((ret = PIOc_openfile(iosysid, &ncid, &iotype, data_filename, NC_NOWRITE))) - ERR(ret); + BAIL(ret); /* Check metadata. */ int ndims_in, nvars_in, ngatts_in, unlimdimid_in; if ((ret = PIOc_inq(ncid, &ndims_in, &nvars_in, &ngatts_in, &unlimdimid_in))) - ERR(ret); + BAIL(ret); if (ndims_in != NDIM4 || nvars_in != num_types * 2 + NUM_4D_VARS || ngatts_in != 0 || unlimdimid_in != 0) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); /* Check the vars. */ for (int t = 0; t < num_types; t++) { - void *data_in; - void *norec_data_in; PIO_Offset type_size; /* Find size of type. */ if ((ret = PIOc_inq_type(ncid, my_type[t], NULL, &type_size))) - ERR(ret); + BAIL(ret); /* Allocate buffers to hold data. */ if (!(data_in = malloc(LAT_LEN * LON_LEN * NREC * type_size))) - ERR(PIO_ENOMEM); + BAIL(PIO_ENOMEM); if (!(norec_data_in = malloc(LAT_LEN * LON_LEN * type_size))) - ERR(PIO_ENOMEM); + BAIL(PIO_ENOMEM); /* Read record and non-record vars for this type. */ if ((ret = PIOc_get_var(ncid, rec_varid[t], data_in))) - ERR(ret); + BAIL(ret); if ((ret = PIOc_get_var(ncid, norec_varid[t], norec_data_in))) - ERR(ret); + BAIL(ret); /* Check each value of non-record data. */ for (int r = 0; r < LAT_LEN * LON_LEN; r++) @@ -132,52 +133,52 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, { case PIO_BYTE: if (((signed char *)norec_data_in)[r] != expected_byte[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_CHAR: if (((char *)norec_data_in)[r] != expected_char[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_SHORT: if (((short *)norec_data_in)[r] != expected_short[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_INT: if (((int *)norec_data_in)[r] != expected_int[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_FLOAT: if (((float *)norec_data_in)[r] != expected_float[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_DOUBLE: if (((double *)norec_data_in)[r] != expected_double[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; #ifdef _NETCDF4 case PIO_UBYTE: if (((unsigned char *)norec_data_in)[r] != expected_ubyte[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_USHORT: if (((unsigned short *)norec_data_in)[r] != expected_ushort[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_UINT: if (((unsigned int *)norec_data_in)[r] != expected_uint[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_INT64: if (((long long *)norec_data_in)[r] != expected_int64[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_UINT64: if (((unsigned long long *)norec_data_in)[r] != expected_uint64[r]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; #endif /* _NETCDF4 */ default: - ERR(ERR_WRONG); + BAIL(ERR_WRONG); } } @@ -188,77 +189,76 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, { case PIO_BYTE: if (((signed char *)data_in)[r] != expected_byte[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_CHAR: if (((char *)data_in)[r] != expected_char[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_SHORT: if (((short *)data_in)[r] != expected_short[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_INT: if (((int *)data_in)[r] != expected_int[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_FLOAT: if (((float *)data_in)[r] != expected_float[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_DOUBLE: if (((double *)data_in)[r] != expected_double[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; #ifdef _NETCDF4 case PIO_UBYTE: if (((unsigned char *)data_in)[r] != expected_ubyte[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_USHORT: if (((unsigned short *)data_in)[r] != expected_ushort[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_UINT: if (((unsigned int *)data_in)[r] != expected_uint[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_INT64: if (((long long *)data_in)[r] != expected_int64[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; case PIO_UINT64: if (((unsigned long long *)data_in)[r] != expected_uint64[r % (LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + BAIL(ERR_WRONG); break; #endif /* _NETCDF4 */ default: - ERR(ERR_WRONG); + BAIL(ERR_WRONG); } } /* Check the 4D vars. */ for (int v = 0; v < NUM_4D_VARS; v++) { - void *data_in; int xtype; PIO_Offset size; /* Get the type of the 4d var. */ if ((ret = PIOc_inq_vartype(ncid, varid_4d[v], &xtype))) - ERR(ret); + BAIL(ret); /* Get the size of this type. */ if ((ret = PIOc_inq_type(ncid, xtype, NULL, &size))) - ERR(ret); + BAIL(ret); /* Allocate memory for data. */ - if (!(data_in = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC))) - ERR(PIO_ENOMEM); + if (!(data_in2 = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC))) + BAIL(PIO_ENOMEM); /* Read the data. */ - if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in))) - ERR(ret); + if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in2))) + BAIL(ret); /* Check each element of data. */ for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++) @@ -266,31 +266,32 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, switch (xtype) { case PIO_INT: - if (((int *)data_in)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + if (((int *)data_in2)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) + BAIL(ERR_WRONG); break; case PIO_FLOAT: - if (((float *)data_in)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) - ERR(ERR_WRONG); + if (((float *)data_in2)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) + BAIL(ERR_WRONG); break; default: - ERR(ERR_WRONG); + BAIL(ERR_WRONG); } } - - /* Release memory. */ - free(data_in); } - - free(data_in); - free(norec_data_in); } /* Close the file. */ if ((ret = PIOc_closefile(ncid))) - ERR(ret); + BAIL(ret); - return 0; +exit: + if (data_in) + free(data_in); + if (data_in2) + free(data_in2); + if (norec_data_in) + free(norec_data_in); + return ret; } /* Run a simple test using darrays with async. */ From cd1d9cf82fdcbcd581b7bcecc009c04bc4d595f8 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 21 Mar 2019 09:36:16 -0600 Subject: [PATCH 43/47] fixing leak on error --- tests/cunit/test_decomps.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/cunit/test_decomps.c b/tests/cunit/test_decomps.c index 473f70eef65..f739544b53d 100644 --- a/tests/cunit/test_decomps.c +++ b/tests/cunit/test_decomps.c @@ -102,7 +102,10 @@ int test_decomp1(int iosysid, int use_io, int my_rank, MPI_Comm test_comm) if (!(iostart = calloc(NDIM2, sizeof(PIO_Offset)))) return ERR_AWFUL; if (!(iocount = calloc(NDIM2, sizeof(PIO_Offset)))) + { + free(iostart); return ERR_AWFUL; + } if (my_rank == 0) for (int i = 0; i < NDIM2; i++) iocount[i] = 4; @@ -111,15 +114,20 @@ int test_decomp1(int iosysid, int use_io, int my_rank, MPI_Comm test_comm) /* Create the PIO decomposition for this test. */ if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe, compdof, &ioid, NULL, iostart, iocount))) + { + if (iostart) + free(iostart); + if (iocount) + free(iocount); return ret; + } /* Free resources. */ free(compdof); - if (use_io) - { + if (iostart) free(iostart); + if (iocount) free(iocount); - } /* These should not work. */ if (PIOc_write_decomp(DECOMP_FILE, iosysid + TEST_VAL_42, ioid, test_comm) != PIO_EBADID) From 154dc956e0d3279aadec42e5cd85524be3b8e5c2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 22 Mar 2019 08:47:55 -0600 Subject: [PATCH 44/47] fixed memory leaks --- tests/cunit/test_darray_async_many.c | 111 ++++++++++++++++----------- 1 file changed, 66 insertions(+), 45 deletions(-) diff --git a/tests/cunit/test_darray_async_many.c b/tests/cunit/test_darray_async_many.c index a32cd0136b5..9638c64854e 100644 --- a/tests/cunit/test_darray_async_many.c +++ b/tests/cunit/test_darray_async_many.c @@ -62,6 +62,63 @@ int my_type[NTYPE] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, /* Names of the dimensions. */ char dim_name[NDIM4][PIO_MAX_NAME + 1] = {"time", "vert_level", "lat", "lon"}; +int +check_4d_vars(int my_rank, int ncid, int *varid_4d) +{ + void *data_in2 = NULL; + int expected_int_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1, 2, 1, 3, 2, 3, 2, 4, 3}; + float expected_float_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1.5, 2, 1, 3, 2.5, 3, 2, 4, 3.5}; + int ret; + + for (int v = 0; v < NUM_4D_VARS; v++) + { + int xtype; + PIO_Offset size; + + /* Get the type of the 4d var. */ + if ((ret = PIOc_inq_vartype(ncid, varid_4d[v], &xtype))) + BAIL(ret); + + /* Get the size of this type. */ + if ((ret = PIOc_inq_type(ncid, xtype, NULL, &size))) + BAIL(ret); + + /* Allocate memory for data. */ + if (!(data_in2 = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC))) + BAIL(PIO_ENOMEM); + + /* Read the data. */ + if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in2))) + BAIL(ret); + + /* Check each element of data. */ + for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++) + { + switch (xtype) + { + case PIO_INT: + if (((int *)data_in2)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) + BAIL(ERR_WRONG); + break; + case PIO_FLOAT: + if (((float *)data_in2)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) + BAIL(ERR_WRONG); + break; + default: + BAIL(ERR_WRONG); + } + } + free(data_in2); + data_in2 = NULL; + } + +exit: + if (data_in2) + free(data_in2); + + return ret; +} + /* Check the file that was created in this test. */ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, int *rec_varid, int *norec_varid, int num_types, int *varid_4d) @@ -88,10 +145,7 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, 9223372036854775827ULL, 9223372036854775828ULL, 9223372036854775837ULL, 9223372036854775838ULL}; #endif /* _NETCDF4 */ - int expected_int_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1, 2, 1, 3, 2, 3, 2, 4, 3}; - float expected_float_4d[VERT_LEN * LAT_LEN * LON_LEN] = {1, 0, 2, 1.5, 2, 1, 3, 2.5, 3, 2, 4, 3.5}; void *data_in = NULL; - void *data_in2 = NULL; void *norec_data_in = NULL; /* Reopen the file. */ @@ -238,57 +292,24 @@ int check_darray_file(int iosysid, char *data_filename, int iotype, int my_rank, } } - /* Check the 4D vars. */ - for (int v = 0; v < NUM_4D_VARS; v++) - { - int xtype; - PIO_Offset size; - - /* Get the type of the 4d var. */ - if ((ret = PIOc_inq_vartype(ncid, varid_4d[v], &xtype))) - BAIL(ret); - - /* Get the size of this type. */ - if ((ret = PIOc_inq_type(ncid, xtype, NULL, &size))) - BAIL(ret); - - /* Allocate memory for data. */ - if (!(data_in2 = malloc(size * VERT_LEN * LAT_LEN * LON_LEN * NREC))) - BAIL(PIO_ENOMEM); - - /* Read the data. */ - if ((ret = PIOc_get_var(ncid, varid_4d[v], data_in2))) - BAIL(ret); + /* Free memory. */ + free(data_in); + data_in = NULL; + free(norec_data_in); + norec_data_in = NULL; - /* Check each element of data. */ - for (int r = 0; r < LAT_LEN * LON_LEN * NREC; r++) - { - switch (xtype) - { - case PIO_INT: - if (((int *)data_in2)[r] != expected_int_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) - BAIL(ERR_WRONG); - break; - case PIO_FLOAT: - if (((float *)data_in2)[r] != expected_float_4d[r % (VERT_LEN * LAT_LEN * LON_LEN)]) - BAIL(ERR_WRONG); - break; - default: - BAIL(ERR_WRONG); - } - } - } + /* Check the 4D vars. */ + if ((ret = check_4d_vars(my_rank, ncid, varid_4d))) + BAIL(ret); } /* Close the file. */ if ((ret = PIOc_closefile(ncid))) - BAIL(ret); + ERR(ret); exit: if (data_in) free(data_in); - if (data_in2) - free(data_in2); if (norec_data_in) free(norec_data_in); return ret; From 7e799c7979480208531a1ad29958d740dc7560b5 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 22 Mar 2019 08:48:46 -0600 Subject: [PATCH 45/47] turned on address sanitizer in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 433a1910e11..c50b7249990 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ env: - CC=mpicc - FC=mpif90 - CPPFLAGS='-I/usr/include' - - CFLAGS='-std=c99' + - CFLAGS='-std=c99 -fsanitize=address -fno-omit-frame-pointer' - LDFLAGS='-L/usr/lib' script: From 26ae9dbb7ff9104582684b648bfc384dd03ecccd Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 22 Mar 2019 09:04:19 -0600 Subject: [PATCH 46/47] travis with address sanitizer --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c50b7249990..3261464ec09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,11 +36,12 @@ env: - CC=mpicc - FC=mpif90 - CPPFLAGS='-I/usr/include' - - CFLAGS='-std=c99 -fsanitize=address -fno-omit-frame-pointer' + - CFLAGS='-std=c99' - LDFLAGS='-L/usr/lib' script: - autoreconf -i + - export CFLAGS='-std=c99 -fsanitize=address -fno-omit-frame-pointer' - ./configure - make -j distcheck - rm -rf build From 0d3556e6b7941632b57002727593b544fa962176 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 22 Mar 2019 09:23:22 -0600 Subject: [PATCH 47/47] for travis, only address sanitize autotools build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3261464ec09..a85b170aab1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,7 @@ script: - export CFLAGS='-std=c99 -fsanitize=address -fno-omit-frame-pointer' - ./configure - make -j distcheck + - export CFLAGS='-std=c99' - rm -rf build - mkdir build - cd build