diff --git a/.github/actions/buildhdf5/action.yml b/.github/actions/buildhdf5/action.yml
index ee4b97b6d3..4ba086823b 100644
--- a/.github/actions/buildhdf5/action.yml
+++ b/.github/actions/buildhdf5/action.yml
@@ -20,7 +20,7 @@ inputs:
     default: False
     required: False
     type: boolean
-    
+
   install_prefix:
     description: 'Install path of hdf5'
     default: ${GITHUB_WORKSPACE}/hdf5
diff --git a/.github/actions/buildmpich/action.yml b/.github/actions/buildmpich/action.yml
index 4b97d09778..3437a6065f 100644
--- a/.github/actions/buildmpich/action.yml
+++ b/.github/actions/buildmpich/action.yml
@@ -24,7 +24,7 @@ runs:
       run: |
         pushd mpich-src
         ./autogen.sh
-        ./configure --prefix=${{ inputs.install_prefix }} 
+        ./configure --prefix=${{ inputs.install_prefix }}
         make -j4
         make install
         popd
diff --git a/.github/actions/buildnetcdf/action.yml b/.github/actions/buildnetcdf/action.yml
index 2804f3197c..a7d168f657 100644
--- a/.github/actions/buildnetcdf/action.yml
+++ b/.github/actions/buildnetcdf/action.yml
@@ -35,7 +35,7 @@ inputs:
     default: False
     required: False
     type: boolean
-    
+
   install_prefix:
     description: 'Install path of netcdf'
     default: $HOME/netcdf-c
diff --git a/.github/actions/buildopenmpi/action.yml b/.github/actions/buildopenmpi/action.yml
index b9fe93af54..f5f95fcf75 100644
--- a/.github/actions/buildopenmpi/action.yml
+++ b/.github/actions/buildopenmpi/action.yml
@@ -24,7 +24,7 @@ runs:
       run: |
         pushd openmpi-src
         perl ./autogen.pl
-        ./configure --prefix=${{ inputs.install_prefix }} 
+        ./configure --prefix=${{ inputs.install_prefix }}
         make -j4
         make install
         popd
diff --git a/.github/actions/buildpnetcdf/action.yml b/.github/actions/buildpnetcdf/action.yml
index 3095e7b919..43f2ad8c05 100644
--- a/.github/actions/buildpnetcdf/action.yml
+++ b/.github/actions/buildpnetcdf/action.yml
@@ -41,7 +41,7 @@ runs:
         then
           config_opts="${config_opts} --enable-shared=yes"
         fi
-        
+
         config_opts="${config_opts}"
         echo "config_opts=$config_opts"
         autoreconf -i
diff --git a/.github/actions/parallelio_autotools/action.yml b/.github/actions/parallelio_autotools/action.yml
index 448de704b2..fd85ebe8af 100644
--- a/.github/actions/parallelio_autotools/action.yml
+++ b/.github/actions/parallelio_autotools/action.yml
@@ -42,43 +42,43 @@ inputs:
     type: boolean
   with_mpi-serial:
     description: 'Build with mpi-serial library'
-    default: 
+    default:
     required: False
     type: string
   with_pnetcdf:
     description: 'Build with pnetcdf library'
-    default: 
+    default:
     required: False
-    type: string  
+    type: string
   with_netcdf:
     description: 'Build with netcdf library'
     default: /usr
     required: False
-    type: string  
+    type: string
   with_netcdff:
     description: 'Build with netcdff library'
-    default: 
+    default:
     required: False
-    type: string  
+    type: string
   with_valgrind:
     description: 'Build the parallelio valgrind leak check '
     default: False
     required: False
-    type: boolean    
+    type: boolean
   enable_fortran:
     description: 'Build the parallelio Fortran Library '
     default: False
     required: False
-    type: boolean    
+    type: boolean
   extra_cflags:
     description: 'Additional cflags to use'
-    default: 
+    default:
     required: False
     type: string
-    
+
   extra_fflags:
     description: 'Additional fflags to use'
-    default: 
+    default:
     required: False
     type: string
   with_mpiexec:
@@ -91,27 +91,41 @@ inputs:
     default: ${GITHUB_WORKSPACE}/parallelio
     required: False
     type: string
-
+  src_path:
+    description: 'Path to parallelio source'
+    default: ${GITHUB_WORKSPACE}/parallelio-src
+    required: False
+    type: string
+    
 runs:
   using: composite
   steps:
-    - uses: actions/checkout@v3
+    - name: Check if already present
+      uses: andstor/file-existence-action@v2
+      with:
+       files: ${{ inputs.src_path }}
+    - name: get parallelio
+      if: ${{ steps.check_files.outputs.files_exists != 'true' }}
+      uses: actions/checkout@v3
       with:
         repository: NCAR/ParallelIO
-        path: parallelio-src
+        path: ${{ inputs.src_path }}
         ref: ${{ inputs.parallelio_version }}
     - id: parallelio-build
       shell: bash
       run: |
-        cd $GITHUB_WORKSPACE/parallelio-src
+        cd ${{ inputs.src_path }}
         autoreconf -i
-        config_opts='--prefix=${{ inputs.install_prefix }} --enable-fortran=${{ inputs.enable_fortran }} '
+        config_opts='--prefix=${{ inputs.install_prefix }} '
         if [[ -z "${{ inputs.with_pnetcdf }}" ]];
         then
           config_opts="${config_opts} --disable-pnetcdf "
         fi
+        if [ "${{ inputs.enable_fortran }}" = "true" ];
+        then
+          config_opts="${config_opts} --enable-fortran=yes "
+        fi
         echo "config_opts = ${config_opts} --with-mpiexec=${{ inputs.with_mpiexec }}"
         ./configure ${config_opts} --with-mpiexec="${{ inputs.with_mpiexec }}"
         make -j 4 VERBOSE=1
         make install
-
diff --git a/.github/actions/parallelio_cmake/action.yml b/.github/actions/parallelio_cmake/action.yml
index 4cd292c9a0..c4708ef445 100644
--- a/.github/actions/parallelio_cmake/action.yml
+++ b/.github/actions/parallelio_cmake/action.yml
@@ -42,39 +42,39 @@ inputs:
     type: boolean
   with_mpi-serial:
     description: 'Build with mpi-serial library'
-    default: 
+    default:
     required: False
     type: string
   with_valgrind:
     description: 'Build the parallelio valgrind leak check '
     default: False
     required: False
-    type: boolean    
+    type: boolean
   enable_fortran:
     description: 'Build the parallelio Fortran Library '
     default: False
     required: False
-    type: boolean    
+    type: boolean
   extra_cflags:
     description: 'Additional cflags to use'
-    default: 
+    default:
     required: False
     type: string
-    
+
   extra_ldflags:
     description: 'Additional ldflags to use'
-    default: 
+    default:
     required: False
     type: string
-    
+
   extra_fflags:
     description: 'Additional fflags to use'
-    default: 
+    default:
     required: False
     type: string
   mpiexec_flags:
     description: 'extra mpiexec flags'
-    default: 
+    default:
     required: False
     type: string
   netcdf_c_include_dir:
diff --git a/.github/workflows/autotools.yml b/.github/workflows/autotools.yml
index 232ac6e418..eba2d60bfe 100644
--- a/.github/workflows/autotools.yml
+++ b/.github/workflows/autotools.yml
@@ -17,14 +17,14 @@ jobs:
       CFLAGS: "-std=c99 -Wall -Werror"
       CPPFLAGS: "-I/usr/include -I/usr/lib/x86_64-linux-gnu/netcdf/mpi/include/ "
       LDFLAGS: "-L/usr/lib/x86_64-linux-gnu -lnetcdf_mpi -lpnetcdf"
-      FCFLAGS: "-Wall -Werror -fallow-argument-mismatch"
+      FCFLAGS: "-Wall -Werror -fallow-argument-mismatch -Wno-conversion"
     steps:
     - uses: actions/checkout@v3
     - name: Installs
       run: |
         set -x
-        sudo apt-get update 
-        sudo apt-get install netcdf-bin 
+        sudo apt-get update
+        sudo apt-get install netcdf-bin
         sudo apt-get install libnetcdf-mpi-19
         sudo apt-get install libnetcdf-mpi-dev
         sudo apt-get install pnetcdf-bin
@@ -46,10 +46,11 @@ jobs:
         with_pnetcdf: /usr
         with_mpiexec: 'mpiexec --oversubscribe'
         parallelio_version: ${{ env.GITHUB_SHA }}
+        src_path: ${GITHUB_WORKSPACE}
     - name: make check
       run: |
-        cd $GITHUB_WORKSPACE/parallelio-src   
-        make  check
+        cd $GITHUB_WORKSPACE
+        make check
 
 #    - name: Setup tmate session
 #      if: ${{ failure() }}
diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index eb1e41e88a..9f4cc1586f 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -26,7 +26,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install netcdf-bin libnetcdf-dev doxygen graphviz wget gfortran \
         libjpeg-dev libz-dev openmpi-bin libopenmpi-dev cmake pnetcdf-bin libpnetcdf-dev libnetcdff-dev
-        
+
     - name: cmake build
       uses: ./.github/actions/parallelio_cmake
       with:
diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml
index 64d009fcf9..6c617a8924 100644
--- a/.github/workflows/intel.yml
+++ b/.github/workflows/intel.yml
@@ -16,7 +16,7 @@ jobs:
       FC: mpiifort
       # Versions should match the github tag names
       PNETCDF_VERSION: checkpoint.1.12.3
-      NETCDF_C_VERSION: v4.9.0
+      NETCDF_C_VERSION: v4.9.1
       NETCDF_FORTRAN_VERSION: v4.6.0
       HDF5_VERSION: hdf5_1_12_2
     steps:
@@ -24,9 +24,8 @@ jobs:
     - name: Installs
       run: |
         set -x
-        sudo apt-get update 
-        sudo apt-get install wget
-        sudo apt-get install libjpeg-dev
+        sudo apt-get update
+        sudo apt-get install libcurl4-gnutls-dev
         sudo apt-get install libz-dev
     - name: cache intel compiler
       id: cache-intel
@@ -73,9 +72,9 @@ jobs:
     - name: prep netcdf-c
       run: |
         export PATH=$GITHUB_WORKSPACE/hdf5/bin:$GITHUB_WORKSPACE/netcdf/bin:$PATH
-        export LDFLAGS="$LDFLAGS -L$GITHUB_WORKSPACE/hdf5/lib  "
+        export LDFLAGS="$LDFLAGS -L$GITHUB_WORKSPACE/hdf5/lib  -L/usr/lib/x86_64-linux-gnu/ -lcurl"
         export CPPFLAGS="$CPPFLAGS -I$GITHUB_WORKSPACE/hdf5/include -I$GITHUB_WORKSPACE/netcdf/include"
-        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/hdf5/lib"
+        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/hdf5/lib:/usr/lib/x86_64-linux-gnu/"
         printenv >> $GITHUB_ENV
     - name: build-netcdf-c
       if: steps.cache-netcdf-c.outputs.cache-hit != 'true'
diff --git a/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml b/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml
index 812f11710d..8c5a98b731 100644
--- a/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml
+++ b/.github/workflows/netcdf_hdf5_no_pnetcdf_ncint_mpich.yml
@@ -12,9 +12,9 @@ jobs:
       CPPFLAGS: "-I${GITHUB_WORKSPACE}/mpich/include"
       LDFLAGS: "-L${GITHUB_WORKSPACE}/mpich/lib "
       # Note issue https://github.com/NCAR/ParallelIO/issues/1889 netcdf integration currently only works with netcdf 4.7.4
-      NETCDF_C_VERSION: v4.7.4
-      NETCDF_FORTRAN_VERSION: v4.5.4
-      MPICH_VERSION: v4.0.3
+      NETCDF_C_VERSION: v4.9.2
+      NETCDF_FORTRAN_VERSION: v4.6.0
+      MPICH_VERSION: v4.1
       HDF5_VERSION: hdf5_1_12_2
       FFLAGS: "-fallow-argument-mismatch"
       FCFLAGS: "-fallow-argument-mismatch"
@@ -22,7 +22,7 @@ jobs:
     - uses: actions/checkout@v3
     - name: Installs
       run: |
-        sudo apt-get install doxygen graphviz wget gfortran libjpeg-dev libz-dev
+        sudo apt-get install doxygen graphviz wget gfortran libjpeg-dev libz-dev libcurl4-gnutls-dev
     - name: cache-mpich
       id: cache-mpich
       uses: actions/cache@v3
@@ -65,11 +65,11 @@ jobs:
         export FC=mpifort
         export PATH="${GITHUB_WORKSPACE}/hdf5/bin:${GITHUB_WORKSPACE}/mpich/bin:${GITHUB_WORKSPACE}/netcdf/bin:$PATH"
         export CPPFLAGS="$CPPFLAGS -I${GITHUB_WORKSPACE}/hdf5/include -I${GITHUB_WORKSPACE}/netcdf/include"
-        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${GITHUB_WORKSPACE}/hdf5/lib"
-        export LDFLAGS="$LDFLAGS -L${GITHUB_WORKSPACE}/hdf5/lib  -L${GITHUB_WORKSPACE}/netcdf/lib "
+        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:${GITHUB_WORKSPACE}/hdf5/lib:/usr/lib/x86_64-linux-gnu/"
+        export LDFLAGS="$LDFLAGS -L${GITHUB_WORKSPACE}/hdf5/lib  -L${GITHUB_WORKSPACE}/netcdf/lib -L/usr/lib/x86_64-linux-gnu/ -lcurl"
         printenv >> $GITHUB_ENV
-      
-        
+
+
     - name: Build NetCDF C
       if: steps.cache-netcdf.outputs.cache-hit != 'true'
       uses: ./.github/actions/buildnetcdf
@@ -77,12 +77,12 @@ jobs:
         netcdf_version: ${{ env.NETCDF_C_VERSION }}
         install_prefix: ${GITHUB_WORKSPACE}/netcdf
 
-    # - name: cache-netcdf-fortran
-    #   id: cache-netcdf-fortran
-    #   uses: actions/cache@v3
-    #   with:
-    #     path: ~/netcdf-fortran
-    #     key: netcdf-fortran-${{ runner.os }}-${{ env.NETCDF_FORTRAN_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION }}
+    - name: cache-netcdf-fortran
+      id: cache-netcdf-fortran
+      uses: actions/cache@v3
+      with:
+        path: ~/netcdf-fortran
+        key: netcdf-fortran-${{ runner.os }}-${{ env.NETCDF_FORTRAN_VERSION }}-mpich-${{ env.MPICH_VERSION }}-hdf5-${{ env.HDF5_VERSION }}
 
     - name: Build NetCDF Fortran
       if: steps.cache-netcdf.outputs.cache-hit != 'true'
@@ -97,13 +97,12 @@ jobs:
       with:
         enable_fortran: True
         enable_netcdf_integration: True
+        src_path: ${GITHUB_WORKSPACE}
         parallelio_version: ${{ env.GITHUB_SHA }}
     - name: make check
       run: |
-        cd $GITHUB_WORKSPACE/parallelio-src   
+        cd $GITHUB_WORKSPACE
         make -j check
 #    - name: Setup tmate session
 #      if: ${{ failure() }}
 #      uses: mxschmitt/action-tmate@v3
-        
-        
diff --git a/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml b/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml
index ce727f6d09..5b055f0466 100644
--- a/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml
+++ b/.github/workflows/netcdf_hdf5_pnetcdf_ncint_mpich_asan.yml
@@ -37,7 +37,7 @@ jobs:
       with:
         install_prefix: $HOME/mpich
         mpich_version: ${{ env.MPICH_VERSION }}
-        
+
     - name: cache-hdf5
       id: cache-hdf5
       uses: actions/cache@v3
@@ -120,23 +120,11 @@ jobs:
       run: |
         cd $GITHUB_WORKSPACE/parallelio-src
         make -j check
-        
+
     - name: autotools build
       run: |
         set -x
         gcc --version
-#        echo 'export PATH=/home/runner/mpich/bin:$PATH' > .bashrc
-#        source .bashrc
-#        export CPPFLAGS: "-I/home/runner/mpich/include -I/home/runner/hdf5/include -I/home/runner/netcdf-c/include -I/home/runner/netcdf-fortran/include -I/home/runner/pnetcdf/include"
-#        export LDFLAGS: "-L/home/runner/mpich/lib -L/home/runner/hdf5/lib -L/home/runner/netcdf-c/lib -L/home/runner/netcdf-fortran/lib -L/home/runner/pnetcdf/lib"
-#        export CC=/home/runner/mpich/bin/mpicc
-#        export FC=/home/runner/mpich/bin/mpifort
-#        export CFLAGS="-g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan"
-#        export FCFLAGS="$FCFLAGS -g -O0 -fsanitize=address -fno-omit-frame-pointer -static-libasan"
-#        export LDFLAGS="$LDFLAGS -static-libasan"
-#        export ASAN_OPTIONS="detect_odr_violation=0"
         autoreconf -i
         ./configure --enable-fortran --enable-netcdf-integration
         make -j check
-        
-        
diff --git a/.github/workflows/netcdf_pnetcdf_openmpi.yml b/.github/workflows/netcdf_pnetcdf_openmpi.yml
index dfedfe33c2..696e1c75c8 100644
--- a/.github/workflows/netcdf_pnetcdf_openmpi.yml
+++ b/.github/workflows/netcdf_pnetcdf_openmpi.yml
@@ -71,9 +71,9 @@ jobs:
         netcdf_version: ${{ env.NETCDF_C_VERSION }}
         install_prefix: ${GITHUB_WORKSPACE}/netcdf
 
-    - name: Setup tmate session
-      if: ${{ failure() }}
-      uses: mxschmitt/action-tmate@v3
+#    - name: Setup tmate session
+#      if: ${{ failure() }}
+#      uses: mxschmitt/action-tmate@v3
 
     - name: Build NetCDF Fortran
       if: steps.cache-netcdf.outputs.cache-hit != 'true'
@@ -105,7 +105,7 @@ jobs:
     #     parallelio_version: ${{ env.GITHUB_SHA }}
     # - name: make check
     #   run: |
-    #     cd ${GITHUB_WORKSPACE}/parallelio-src   
+    #     cd ${GITHUB_WORKSPACE}/parallelio-src
     #     make -j check
     #     make distclean
     # - name: cmake build
diff --git a/.github/workflows/withspack.yml b/.github/workflows/withspack.yml
index d97c9a6fe9..a9ce6302ad 100644
--- a/.github/workflows/withspack.yml
+++ b/.github/workflows/withspack.yml
@@ -16,7 +16,7 @@ jobs:
     - name: Installs
       run: |
         set -x
-        sudo apt-get update 
+        sudo apt-get update
         sudo apt-get install wget
         sudo apt-get install libjpeg-dev
         sudo apt-get install libz-dev
@@ -47,16 +47,18 @@ jobs:
         source $GITHUB_WORKSPACE/spack/share/spack/setup-env.sh
         spack compiler find
         # Remove the patch for gfortran, we don't want it
-        # 
+        #
         sed -i 's/patch(.*$//' $GITHUB_WORKSPACE/spack/var/spack/repos/builtin/packages/parallelio/package.py
     - name: Build with spack
       run: |
         source $GITHUB_WORKSPACE/spack/share/spack/setup-env.sh
         mkdir genf90
-        pushd genf90    
+        pushd genf90
         ln -fs $GITHUB_WORKSPACE/scripts/genf90.pl .
         popd
         # the || true prevents a fail if parallelio is not installed
+        #        spack uninstall -y parallelio@=develop+pnetcdf+fortran ^mpich || true
+        #        spack dev-build -d $GITHUB_WORKSPACE parallelio@=develop+pnetcdf+fortran ^mpich
         spack uninstall -y parallelio@develop+pnetcdf+fortran ^mpich || true
         spack dev-build -d $GITHUB_WORKSPACE parallelio@develop+pnetcdf+fortran ^mpich
 
@@ -66,10 +68,10 @@ jobs:
         pushd $pioblddir
         make tests
         # Exclude two tests that are timing out.
-        ctest -LE skipforspack
+        ctest -VV -LE skipforspack
         popd
-        
 
-#    - name: Setup tmate session
-#      if: ${{ failure() }}
-#      uses: mxschmitt/action-tmate@v3
+
+    - name: Setup tmate session
+      if: ${{ failure() }}
+      uses: mxschmitt/action-tmate@v3
diff --git a/.gitignore b/.gitignore
index 5055b0c6a8..7df6a3a7c8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,4 @@ m4/
 *.nc
 *.log
 *.gz
-!/decomps/*/*.nc
\ No newline at end of file
+!/decomps/*/*.nc
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2b23bd6693..3b235e9ecf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -24,6 +24,9 @@ SET(PACKAGE_VERSION ${PIO_VERSION_MAJOR}.${PIO_VERSION_MINOR}.${PIO_VERSION_PATC
 # This provides cmake_print_variables() function for debugging.
 include(CMakePrintHelpers)
 
+# This provides check_symbol_exists
+include(CheckSymbolExists)
+
 # Determine the configure date.
 IF(DEFINED ENV{SOURCE_DATE_EPOCH})
 	EXECUTE_PROCESS(
@@ -123,12 +126,6 @@ else()
   set(NETCDF_INTEGRATION 0)
 endif()
 
-if(PIO_USE_MPISERIAL)
-  set(USE_MPI_SERIAL 1)
-else()
-  set(USE_MPI_SERIAL 0)
-endif()
-
 #==============================================================================
 #  PREPEND TO CMAKE MODULE PATH
 #==============================================================================
@@ -168,14 +165,16 @@ if (PIO_USE_MPISERIAL)
   if (MPISERIAL_C_FOUND)
     set (CMAKE_REQUIRED_INCLUDES ${MPISERIAL_C_INCLUDE_DIRS})
   endif ()
+  set(USE_MPI_SERIAL 1)
 else ()
   find_package (MPI REQUIRED)
   set (CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH})
+  set(USE_MPI_SERIAL 0)
 endif ()
 
-#SET(CMAKE_EXTRA_INCLUDE_FILES "mpi.h")
-#check_type_size("MPI_Offset" SIZEOF_MPI_OFFSET)
-#SET(CMAKE_EXTRA_INCLUDE_FILES)
+SET(CMAKE_EXTRA_INCLUDE_FILES "mpi.h")
+CHECK_TYPE_SIZE("MPI_Offset" SIZEOF_MPI_OFFSET)
+SET(CMAKE_EXTRA_INCLUDE_FILES)
 
 #===== Library Variables =====
 set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)")
@@ -358,17 +357,7 @@ int main() {return 0;}" HAVE_SZIP_WRITE)
 ###
 # Check to see if parallel filters are supported by HDF5/netcdf-c.
 ###
-CHECK_C_SOURCE_COMPILES("
-#include <netcdf_meta.h>
-#if !NC_HAS_PAR_FILTERS
-      choke me
-#endif
-int main() {return 0;}" HDF5_HAS_PAR_FILTERS)
-if(HDF5_HAS_PAR_FILTERS)
-  set(HAVE_PAR_FILTERS 1)
-else()
-  set(HAVE_PAR_FILTERS 0)
-endif()
+check_function_exists(nc_inq_filter_avail HAVE_PAR_FILTERS)
 
 ###
 # Check to see if this is netcdf-c-4.7.2, which won't work.
@@ -385,18 +374,21 @@ if (HAVE_472)
 endif ()
 
 ###
-# Check to see if dispatch table version 2 is supported for netcdf integration.
+# Check to see if dispatch table is supported for netcdf integration.
 ###
 CHECK_C_SOURCE_COMPILES("
 #include <netcdf_meta.h>
-#if NC_DISPATCH_VERSION != 2
+#if NC_DISPATCH_VERSION < 2
       choke me
 #endif
-int main() {return 0;}" HAVE_DISPATCH2)
+#if NC_DISPATCH_VERSION > 5
+    choke me
+#endif
+int main() {return 0;}" HAVE_DISPATCH)
 
 if (NETCDF_INTEGRATION)
-  if (NOT HAVE_DISPATCH2)
-    message (FATAL_ERROR "Need newer version of netcdf-c for netcdf integration feature, please upgrade your netCDF library")
+  if (NOT HAVE_DISPATCH)
+    message (FATAL_ERROR "The netcdf-c netcdf integration layer is incompatible with the one in this ParallelIO version")
   endif ()
   set(HAVE_NETCDF_INTEGRATION 1)
 else ()
@@ -451,16 +443,19 @@ SET(LDFLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${CMAKE_BUILD_TY
 
 is_disabled(BUILD_SHARED_LIBS enable_static)
 is_enabled(BUILD_SHARED_LIBS enable_shared)
-
+is_enabled(HAVE_PAR_FILTERS have_par_filters)
 is_enabled(USE_SZIP HAS_SZIP_WRITE)
 is_enabled(STATUS_PNETCDF HAS_PNETCDF)
 is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)
-is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
 is_enabled(HAVE_NETCDF4 HAS_NETCDF4)
 is_enabled(HAVE_NETCDF_PAR HAS_NETCDF4_PAR)
 is_enabled(HAVE_NETCDF_INTEGRATION HAS_NETCDF_INTEGRATION)
 is_enabled(PIO_ENABLE_FORTRAN HAS_PIO_FORTRAN)
 
+if(HAVE_PAR_FILTERS)
+  SET(PIO_HAS_PAR_FILTERS 1)
+endif()
+
 # Generate file from template.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libpio.settings.in"
   "${CMAKE_CURRENT_BINARY_DIR}/libpio.settings"
diff --git a/cmake/FindGPTL.cmake b/cmake/FindGPTL.cmake
index c223c1b346..c4a8b54488 100644
--- a/cmake/FindGPTL.cmake
+++ b/cmake/FindGPTL.cmake
@@ -1,6 +1,6 @@
 # - Try to find GPTL
 #
-# This can be controlled by setting the GPTL_DIR (or, equivalently, the 
+# This can be controlled by setting the GPTL_DIR (or, equivalently, the
 # GPTL environment variable), or GPTL_<lang>_DIR CMake variables, where
 # <lang> is the COMPONENT language one needs.
 #
@@ -55,7 +55,7 @@ foreach (GPTL_comp IN LISTS GPTL_FIND_VALID_COMPONENTS)
             set (mpilibs ${MPI_Fortran_LIBRARIES})
             set (mpifound ${MPI_Fortran_FOUND})
         endif ()
-        
+
         # Search for the package component
         if (mpifound)
             initialize_paths (GPTL_${GPTL_comp}_PATHS
@@ -68,5 +68,5 @@ foreach (GPTL_comp IN LISTS GPTL_FIND_VALID_COMPONENTS)
         endif ()
 
     endif ()
-    
+
 endforeach ()
diff --git a/cmake/FindHDF5.cmake b/cmake/FindHDF5.cmake
index e918277b1a..d19c423ed7 100644
--- a/cmake/FindHDF5.cmake
+++ b/cmake/FindHDF5.cmake
@@ -1,6 +1,6 @@
 # - Try to find HDF5
 #
-# This can be controlled by setting the HDF5_DIR (or, equivalently, the 
+# This can be controlled by setting the HDF5_DIR (or, equivalently, the
 # HDF5 environment variable), or HDF5_<lang>_DIR CMake variables, where
 # <lang> is the COMPONENT language one needs.
 #
@@ -78,14 +78,14 @@ foreach (HDF5_comp IN LISTS HDF5_FIND_VALID_COMPONENTS)
 
             # Dependencies
             if (HDF5_comp STREQUAL C AND NOT HDF5_C_IS_SHARED)
-            
+
                 # DEPENDENCY: LIBZ
                 find_package (LIBZ)
                 if (LIBZ_FOUND)
                     list (APPEND HDF5_C_INCLUDE_DIRS ${LIBZ_INCLUDE_DIRS})
                     list (APPEND HDF5_C_LIBRARIES ${LIBZ_LIBRARIES})
                 endif ()
-                
+
                 # DEPENDENCY: SZIP (Optional)
                 check_macro (HDF5_C_HAS_SZIP
                                 NAME TryHDF5_HAS_SZIP.c
@@ -99,20 +99,20 @@ foreach (HDF5_comp IN LISTS HDF5_FIND_VALID_COMPONENTS)
                         list (APPEND HDF5_C_LIBRARIES ${SZIP_LIBRARIES})
                     endif ()
                 endif ()
-                
+
             elseif (NOT HDF5_${HDF5_comp}_IS_SHARED)
-    
+
                 # DEPENDENCY: HDF5
                 find_package (HDF5 COMPONENTS C)
                 if (HDF5_C_FOUND)
                     list (APPEND HDF5_${HDF5_comp}_INCLUDE_DIRS ${HDF5_C_INCLUDE_DIRS})
                     list (APPEND HDF5_${HDF5_comp}_LIBRARIES ${HDF5_C_LIBRARIES})
                 endif ()
-    
+
             endif ()
 
         endif ()
-             
+
     endif ()
-    
+
 endforeach ()
diff --git a/cmake/FindLIBRT.cmake b/cmake/FindLIBRT.cmake
index 1f55f9f3f1..211984fcb8 100644
--- a/cmake/FindLIBRT.cmake
+++ b/cmake/FindLIBRT.cmake
@@ -1,6 +1,6 @@
 # - Try to find LIBRT
 #
-# This can be controlled by setting the LIBRT_DIR (or, equivalently, the 
+# This can be controlled by setting the LIBRT_DIR (or, equivalently, the
 # LIBRT environment variable).
 #
 # Once done, this will define:
@@ -21,7 +21,7 @@ define_package_component (LIBRT
 
 # SEARCH FOR PACKAGE
 if (NOT LIBRT_FOUND)
-    
+
     # Search for the package
     find_package_component(LIBRT)
 
diff --git a/cmake/FindLIBZ.cmake b/cmake/FindLIBZ.cmake
index 8ebbaefeed..c003eb94ca 100644
--- a/cmake/FindLIBZ.cmake
+++ b/cmake/FindLIBZ.cmake
@@ -1,6 +1,6 @@
 # - Try to find LIBZ
 #
-# This can be controlled by setting the LIBZ_DIR (or, equivalently, the 
+# This can be controlled by setting the LIBZ_DIR (or, equivalently, the
 # LIBZ environment variable).
 #
 # Once done, this will define:
@@ -22,7 +22,7 @@ define_package_component (LIBZ
 # SEARCH FOR PACKAGE
 if (NOT LIBZ_FOUND)
 
-    # Manually add the MPI include and library dirs to search paths 
+    # Manually add the MPI include and library dirs to search paths
     # and search for the package component
     if (MPI_C_FOUND)
         initialize_paths (LIBZ_PATHS
diff --git a/cmake/FindMPE.cmake b/cmake/FindMPE.cmake
index 5a964172da..cdfc360c23 100644
--- a/cmake/FindMPE.cmake
+++ b/cmake/FindMPE.cmake
@@ -13,7 +13,7 @@
 #   MPE_<lang>_LIBRARY      (FILE) - Path to the C library file
 #   MPE_<lang>_LIBRARIES    (LIST) - link these to use MPE
 #
-# The available COMPONENTS are: C 
+# The available COMPONENTS are: C
 include (LibFind)
 include (LibCheck)
 
@@ -46,5 +46,5 @@ foreach (NCDFcomp IN LISTS MPE_FIND_VALID_COMPONENTS)
     endif ()
 
   endif ()
-  
+
 endforeach ()
diff --git a/cmake/FindMPISERIAL.cmake b/cmake/FindMPISERIAL.cmake
index 09906eb7a2..47d30077b1 100644
--- a/cmake/FindMPISERIAL.cmake
+++ b/cmake/FindMPISERIAL.cmake
@@ -1,6 +1,6 @@
 # - Try to find MPISERIAL
 #
-# This can be controlled by setting the MPISERIAL_PATH (or, equivalently, the 
+# This can be controlled by setting the MPISERIAL_PATH (or, equivalently, the
 # MPISERIAL environment variable).
 #
 # Once done, this will define:
@@ -38,7 +38,7 @@ foreach (MPISERIAL_comp IN LISTS MPISERIAL_FIND_VALID_COMPONENTS)
 
         # Search for the package
         find_package_component(MPISERIAL COMPONENT ${MPISERIAL_comp})
-        
+
     endif ()
 
 endforeach ()
diff --git a/cmake/FindPAPI.cmake b/cmake/FindPAPI.cmake
index dcf1445bc7..369abb7f8c 100644
--- a/cmake/FindPAPI.cmake
+++ b/cmake/FindPAPI.cmake
@@ -1,6 +1,6 @@
 # - Try to find PAPI
 #
-# This can be controlled by setting the PAPI_DIR (or, equivalently, the 
+# This can be controlled by setting the PAPI_DIR (or, equivalently, the
 # PAPI environment variable).
 #
 # Once done, this will define:
diff --git a/cmake/FindPnetCDF.cmake b/cmake/FindPnetCDF.cmake
index b87d245cd1..7d644a6380 100644
--- a/cmake/FindPnetCDF.cmake
+++ b/cmake/FindPnetCDF.cmake
@@ -1,6 +1,6 @@
 # - Try to find PnetCDF
 #
-# This can be controlled by setting the PnetCDF_PATH (or, equivalently, the 
+# This can be controlled by setting the PnetCDF_PATH (or, equivalently, the
 # PNETCDF environment variable), or PnetCDF_<lang>_PATH CMake variables, where
 # <lang> is the COMPONENT language one needs.
 #
@@ -54,7 +54,7 @@ foreach (PNCDFcomp IN LISTS PnetCDF_FIND_VALID_COMPONENTS)
 
         # Continue only if component found
         if (PnetCDF_${PNCDFcomp}_FOUND)
-        
+
             # Check version
             check_version (PnetCDF
                            NAME "pnetcdf.h"
@@ -62,7 +62,7 @@ foreach (PNCDFcomp IN LISTS PnetCDF_FIND_VALID_COMPONENTS)
                            MACRO_REGEX "PNETCDF_VERSION_")
 
         endif ()
-            
+
     endif ()
-    
+
 endforeach ()
diff --git a/cmake/FindSZIP.cmake b/cmake/FindSZIP.cmake
index e65cfe5fd6..ab8c497f59 100644
--- a/cmake/FindSZIP.cmake
+++ b/cmake/FindSZIP.cmake
@@ -1,6 +1,6 @@
 # - Try to find SZIP
 #
-# This can be controlled by setting the SZIP_DIR (or, equivalently, the 
+# This can be controlled by setting the SZIP_DIR (or, equivalently, the
 # SZIP environment variable).
 #
 # Once done, this will define:
@@ -22,7 +22,7 @@ define_package_component (SZIP
 # SEARCH FOR PACKAGE
 if (NOT SZIP_FOUND)
 
-    # Manually add the MPI include and library dirs to search paths 
+    # Manually add the MPI include and library dirs to search paths
     # and search for the package component
     if (MPI_C_FOUND)
         initialize_paths (SZIP_PATHS
diff --git a/cmake/LibCheck.cmake b/cmake/LibCheck.cmake
index 3f12bdf796..2e8cadcbdd 100644
--- a/cmake/LibCheck.cmake
+++ b/cmake/LibCheck.cmake
@@ -101,4 +101,4 @@ function (check_version PKG)
 
     endif ()
 
-endfunction ()
\ No newline at end of file
+endfunction ()
diff --git a/cmake/LibFind.cmake b/cmake/LibFind.cmake
index 61cd93aa37..ac13748091 100644
--- a/cmake/LibFind.cmake
+++ b/cmake/LibFind.cmake
@@ -77,7 +77,7 @@ function (define_package_component PKG)
     else ()
         set (PKGCOMP ${PKG})
     endif ()
-    
+
     # Set return values
     if (${PKG}_COMPONENT)
         if (${PKG}_DEFAULT)
@@ -96,7 +96,7 @@ endfunction ()
 #______________________________________________________________________________
 # - Function to find valid package components
 #
-#   Assumes pre-defined variables: 
+#   Assumes pre-defined variables:
 #     ${PKG}_FIND_COMPONENTS        (LIST)
 #     ${PKG}_DEFAULT_COMPONENT      (STRING)
 #     ${PKG}_VALID_COMPONENTS       (LIST)
@@ -109,7 +109,7 @@ function (find_valid_components PKG)
     if (NOT ${PKG}_FIND_COMPONENTS)
         set (${PKG}_FIND_COMPONENTS ${${PKG}_DEFAULT_COMPONENT})
     endif ()
-    
+
     set (FIND_VALID_COMPONENTS)
     foreach (comp IN LISTS ${PKG}_FIND_COMPONENTS)
         if (";${${PKG}_VALID_COMPONENTS};" MATCHES ";${comp};")
@@ -118,7 +118,7 @@ function (find_valid_components PKG)
     endforeach ()
 
     set (${PKG}_FIND_VALID_COMPONENTS ${FIND_VALID_COMPONENTS} PARENT_SCOPE)
-    
+
 endfunction ()
 
 
@@ -137,7 +137,7 @@ function (initialize_paths PATHLIST)
     # Parse the input arguments
     set (multiValueArgs INCLUDE_DIRECTORIES LIBRARIES)
     cmake_parse_arguments (INIT "" "" "${multiValueArgs}" ${ARGN})
-    
+
     set (paths)
     foreach (inc IN LISTS INIT_INCLUDE_DIRECTORIES)
         list (APPEND paths ${inc})
@@ -156,7 +156,7 @@ function (initialize_paths PATHLIST)
             list (APPEND paths ${prefx})
         endif ()
     endforeach ()
-    
+
     set (${PATHLIST} ${paths} PARENT_SCOPE)
 
 endfunction ()
@@ -180,7 +180,7 @@ function (find_package_component PKG)
     set (options)
     set (oneValueArgs COMPONENT)
     set (multiValueArgs HINTS PATHS)
-    cmake_parse_arguments (${PKG} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})    
+    cmake_parse_arguments (${PKG} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
     set (COMP ${${PKG}_COMPONENT})
     if (COMP)
         set (PKGCOMP ${PKG}_${COMP})
@@ -189,10 +189,10 @@ function (find_package_component PKG)
     endif ()
     string (TOUPPER ${PKG} PKGUP)
     string (TOUPPER ${PKGCOMP} PKGCOMPUP)
-    
+
     # Only continue if package not found already
     if (NOT ${PKGCOMP}_FOUND)
-    
+
         # Handle QUIET and REQUIRED arguments
         if (${${PKG}_FIND_QUIETLY})
             set (${PKGCOMP}_FIND_QUIETLY TRUE)
@@ -200,7 +200,7 @@ function (find_package_component PKG)
         if (${${PKG}_FIND_REQUIRED})
             set (${PKGCOMP}_FIND_REQUIRED TRUE)
         endif ()
-        
+
         # Determine search order
         set (SEARCH_DIRS)
         if (${PKG}_HINTS)
@@ -234,7 +234,7 @@ function (find_package_component PKG)
         endforeach ()
 
         foreach (dir IN LISTS SEARCH_DIRS)
-        
+
             # Search for include file names in current dirrectory
             foreach (iname IN LISTS ${PKGCOMP}_INCLUDE_NAMES)
                 if (EXISTS ${dir}/${iname})
@@ -248,7 +248,7 @@ function (find_package_component PKG)
                     break ()
                 endif ()
             endforeach ()
-    
+
             # Search for library file names in the found prefix only!
             if (${PKGCOMP}_PREFIX)
                 find_library (${PKGCOMP}_LIBRARY
@@ -256,12 +256,12 @@ function (find_package_component PKG)
                               PATHS ${${PKGCOMP}_PREFIX}
                               PATH_SUFFIXES lib
                               NO_DEFAULT_PATH)
-                              
-                # If found, check if library is static or dynamic 
+
+                # If found, check if library is static or dynamic
                 if (${PKGCOMP}_LIBRARY)
                     is_shared_library (${PKGCOMP}_IS_SHARED ${${PKGCOMP}_LIBRARY})
-    
-                    # If we want only shared libraries, and it isn't shared...                
+
+                    # If we want only shared libraries, and it isn't shared...
                     if (PREFER_SHARED AND NOT ${PKGCOMP}_IS_SHARED)
                         find_shared_library (${PKGCOMP}_SHARED_LIBRARY
                                              NAMES ${${PKGCOMP}_LIBRARY_NAMES}
@@ -272,7 +272,7 @@ function (find_package_component PKG)
                             set (${PKGCOMP}_LIBRARY ${${PKGCOMP}_SHARED_LIBRARY})
                             set (${PKGCOMP}_IS_SHARED TRUE)
                         endif ()
-    
+
                     # If we want only static libraries, and it is shared...
                     elseif (PREFER_STATIC AND ${PKGCOMP}_IS_SHARED)
                         find_static_library (${PKGCOMP}_STATIC_LIBRARY
@@ -286,11 +286,11 @@ function (find_package_component PKG)
                         endif ()
                     endif ()
                 endif ()
-                              
+
                 # If include dir and library both found, then we're done
                 if (${PKGCOMP}_INCLUDE_DIR AND ${PKGCOMP}_LIBRARY)
                     break ()
-                    
+
                 # Otherwise, reset the search variables and continue
                 else ()
                     set (${PKGCOMP}_PREFIX ${PKGCOMP}_PREFIX-NOTFOUND)
@@ -298,19 +298,19 @@ function (find_package_component PKG)
                     set (${PKGCOMP}_LIBRARY ${PKGCOMP}_LIBRARY-NOTFOUND)
                 endif ()
             endif ()
-            
+
         endforeach ()
-        
-        # handle the QUIETLY and REQUIRED arguments and 
+
+        # handle the QUIETLY and REQUIRED arguments and
         # set NetCDF_C_FOUND to TRUE if all listed variables are TRUE
         find_package_handle_standard_args (${PKGCOMP} DEFAULT_MSG
-                                           ${PKGCOMP}_LIBRARY 
+                                           ${PKGCOMP}_LIBRARY
                                            ${PKGCOMP}_INCLUDE_DIR)
         mark_as_advanced (${PKGCOMP}_INCLUDE_DIR ${PKGCOMP}_LIBRARY)
-        
+
         # HACK For bug in CMake v3.0:
         set (${PKGCOMP}_FOUND ${${PKGCOMPUP}_FOUND})
-    
+
         # Set return variables
         if (${PKGCOMP}_FOUND)
             set (${PKGCOMP}_INCLUDE_DIRS ${${PKGCOMP}_INCLUDE_DIR})
@@ -324,10 +324,7 @@ function (find_package_component PKG)
         set (${PKGCOMP}_LIBRARY      ${${PKGCOMP}_LIBRARY}       PARENT_SCOPE)
         set (${PKGCOMP}_LIBRARIES    ${${PKGCOMP}_LIBRARIES}     PARENT_SCOPE)
         set (${PKGCOMP}_IS_SHARED    ${${PKGCOMP}_IS_SHARED}     PARENT_SCOPE)
-        
+
     endif ()
 
 endfunction ()
-
-
-
diff --git a/cmake/mpiexec.alcf b/cmake/mpiexec.alcf
index 48765fd022..62780b898d 100755
--- a/cmake/mpiexec.alcf
+++ b/cmake/mpiexec.alcf
@@ -10,7 +10,7 @@ NP=$1
 shift
 
 ${BGQ_RUNJOB:-runjob} --np $NP --block $COBALT_PARTNAME \
-    --envs GPFSMPIO_NAGG_PSET=16 GPFSMPIO_ONESIDED_ALWAYS_RMW=1 \ 
+    --envs GPFSMPIO_NAGG_PSET=16 GPFSMPIO_ONESIDED_ALWAYS_RMW=1 \
            GPFSMPIO_BALANCECONTIG=1 GPFSMPIO_WRITE_AGGMETHOD=2 \
            GPFSMPIO_READ_AGGMETHOD=2 PAMID_TYPED_ONESIDED=1 \
            PAMID_RMA_PENDING=1M GPFSMPIO_BRIDGERINGAGG=1 : $@
diff --git a/cmake_config.h.in b/cmake_config.h.in
index 586030c202..68b9e8dc4b 100644
--- a/cmake_config.h.in
+++ b/cmake_config.h.in
@@ -26,6 +26,7 @@
 
 #define USE_VARD @USE_VARD@
 
+#cmakedefine PIO_HAS_PAR_FILTERS 
 /* Does netCDF support netCDF/HDF5 files? */
 #cmakedefine HAVE_NETCDF4
 
diff --git a/configure.ac b/configure.ac
index baee0e4c62..7995c1dadd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -38,13 +38,11 @@ fi
 LD=ld # Required for MPE to work.
 LT_INIT
 
-# Find and learn about install and ranlib.
+# Find and learn about install 
 AC_PROG_INSTALL
-AC_PROG_RANLIB
 
 # Find and learn about the C compiler.
 AC_PROG_CC
-AC_PROG_CC_C99
 
 # Compiler with version information. This consists of the full path
 # name of the compiler and the reported version number.
@@ -292,12 +290,8 @@ AC_MSG_RESULT([${have_szip_write}])
 
 # Do we have parallel filter support? Parallel filters are required
 # for iotype NETCDF4P to use compression.
-AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include "netcdf_meta.h"],
-[[#if !NC_HAS_PAR_FILTERS
-# error
-#endif]
-])], [have_par_filters=yes], [have_par_filters=no])
 AC_MSG_CHECKING([whether netCDF provides parallel filter support])
+AC_CHECK_LIB([netcdf], [nc_inq_filter_avail], [have_par_filters=yes], [have_par_filters=no])
 AC_MSG_RESULT([${have_par_filters}])
 if test "x$have_par_filters" = xyes ; then
    AC_DEFINE([HAVE_PAR_FILTERS], [1], [if true, netcdf-c supports filters with parallel I/O])
@@ -319,14 +313,17 @@ fi
 # Do we have the correct dispatch table version in netcdf-c for netcdf
 # integration?
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include "netcdf_meta.h"],
-[[#if NC_DISPATCH_VERSION != 2
+[[#if NC_DISPATCH_VERSION < 2
+# error
+#endif
+#if NC_DISPATCH_VERSION > 5
 # error
 #endif]
-])], [have_dispatch2=yes], [have_dispatch2=no])
-AC_MSG_CHECKING([whether netcdf-c supports version 2 of dispatch table for netcdf integration])
-AC_MSG_RESULT([${have_dispatch2}])
-if test "x$enable_netcdf_integration" = xyes -a "x$have_dispatch2" = xno; then
-   AC_MSG_ERROR([NetCDF integration cannot be used with this version of netcdf-c, please upgrade your netCDF version.])
+])], [have_dispatch=yes], [have_dispatch=no])
+AC_MSG_CHECKING([whether netcdf-c supports compatible dispatch table for netcdf integration])
+AC_MSG_RESULT([${have_dispatch}])
+if test "x$enable_netcdf_integration" = xyes -a "x$have_dispatch" = xno; then
+   AC_MSG_ERROR([NetCDF integration version is not compatible with that in ParallelIO])
 fi
 
 # Set some build settings for when netcdf-4 is supported.
@@ -336,12 +333,12 @@ fi
 AM_CONDITIONAL(BUILD_NETCDF4, [test "x$have_netcdf_par" = xyes])
 
 # Not working for some reason, so I will just set it...
-#AC_CHECK_TYPE([MPI_Offset], [], [], [#include <mpi.h>])
-#if test "x${ac_cv_type_MPI_Offset}" = xyes; then
-#   AC_CHECK_SIZEOF([MPI_Offset], [], [#include <mpi.h>])
-#else
-#   AC_MSG_ERROR([Unable to find type MPI_Offset in mpi.h])
-#fi
+AC_CHECK_TYPE([MPI_Offset], [], [], [#include <mpi.h>])
+if test "x${ac_cv_type_MPI_Offset}" = xyes; then
+   AC_CHECK_SIZEOF([MPI_Offset], [], [#include <mpi.h>])
+else
+   AC_MSG_ERROR([Unable to find type MPI_Offset in mpi.h])
+fi
 
 # If we want the timing library, we must find it.
 if test "x$enable_timing" = xyes; then
@@ -446,6 +443,7 @@ AX_SET_META([PIO_HAS_NETCDF_INTEGRATION], [$enable_netcdf_integration],[yes])
 
 # Create output variables from various shell variables, for use in
 # generating libpio.settings.
+
 AC_SUBST([enable_shared])
 AC_SUBST([enable_static])
 AC_SUBST([CFLAGS])
@@ -457,13 +455,13 @@ AC_SUBST([FPPFLAGS]) # ignored by autotools
 AC_SUBST(HAS_PNETCDF,[$enable_pnetcdf])
 AC_SUBST(HAS_LOGGING, [$enable_logging])
 AC_SUBST(HAS_SZIP_WRITE, [$have_szip_write])
-AC_SUBST([HAS_PAR_FILTERS], [$have_par_filters])
+AC_SUBST([have_par_filters])
 AC_SUBST([HAS_NETCDF4], [$have_netcdf4])
 AC_SUBST([HAS_NETCDF4_PAR], [$have_netcdf_par])
 AC_SUBST([HAS_NETCDF_INTEGRATION], [$enable_netcdf_integration])
 AC_SUBST([HAS_PIO_FORTRAN], [$enable_fortran])
 
-# Create the build summary file. 
+# Create the build summary file.
 AC_CONFIG_FILES([libpio.settings
                  src/clib/pio_meta.h
                  ])
@@ -473,30 +471,30 @@ AC_CONFIG_LINKS([tests/unit/input.nl:tests/unit/input.nl])
 AC_CONFIG_HEADERS([config.h])
 
 # Create the makefiles.
-AC_OUTPUT(Makefile
-          src/Makefile
-          src/clib/Makefile
-          src/ncint/Makefile
-          src/flib/Makefile
-          src/gptl/Makefile
-          tests/Makefile
-          tests/cunit/Makefile
-          tests/ncint/Makefile
-          tests/fncint/Makefile
-          tests/unit/Makefile
-          tests/general/Makefile
-          tests/general/util/Makefile
-          tests/performance/Makefile
-          doc/Makefile
-          doc/source/Makefile
-          doc/images/Makefile
-          examples/Makefile
-          examples/c/Makefile
-          examples/f03/Makefile
-          cmake/Makefile
-          scripts/Makefile)
+AC_CONFIG_FILES(Makefile
+	        src/Makefile
+        	src/clib/Makefile
+                src/ncint/Makefile
+                src/flib/Makefile
+                src/gptl/Makefile
+                tests/Makefile
+                tests/cunit/Makefile
+                tests/ncint/Makefile
+                tests/fncint/Makefile
+                tests/unit/Makefile
+                tests/general/Makefile
+                tests/general/util/Makefile
+                tests/performance/Makefile
+                doc/Makefile
+                doc/source/Makefile
+                doc/images/Makefile
+                examples/Makefile
+                examples/c/Makefile
+                examples/f03/Makefile
+                cmake/Makefile
+                scripts/Makefile)
+
+AC_OUTPUT()
 
 # Show the build summary.
 cat libpio.settings
-
-
diff --git a/ctest/CTestEnvironment-alcf.cmake b/ctest/CTestEnvironment-alcf.cmake
index 607076479d..5c1edff260 100644
--- a/ctest/CTestEnvironment-alcf.cmake
+++ b/ctest/CTestEnvironment-alcf.cmake
@@ -1,7 +1,7 @@
 #==============================================================================
 #
 #  This file sets the environment variables needed to configure and build
-#  on the Argonne Leadership Computing Facility systems 
+#  on the Argonne Leadership Computing Facility systems
 #  (mira/cetus/vesta/cooley).
 #
 #==============================================================================
diff --git a/ctest/CTestEnvironment-anlworkstation.cmake b/ctest/CTestEnvironment-anlworkstation.cmake
index 38e5b4b0a9..07ba92a2a7 100644
--- a/ctest/CTestEnvironment-anlworkstation.cmake
+++ b/ctest/CTestEnvironment-anlworkstation.cmake
@@ -23,4 +23,3 @@ endif ()
 if (DEFINED ENV{VALGRIND_CHECK})
     set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_VALGRIND_CHECK=ON")
 endif ()
-
diff --git a/ctest/CTestEnvironment-ncsa.cmake b/ctest/CTestEnvironment-ncsa.cmake
index 706946ec2b..c09bdf0245 100644
--- a/ctest/CTestEnvironment-ncsa.cmake
+++ b/ctest/CTestEnvironment-ncsa.cmake
@@ -1,7 +1,7 @@
 #==============================================================================
 #
 #  This file sets the environment variables needed to configure and build
-#  on the NCSA systems 
+#  on the NCSA systems
 #  (Blue Waters).
 #
 #==============================================================================
diff --git a/ctest/CTestEnvironment-nersc.cmake b/ctest/CTestEnvironment-nersc.cmake
index 6b1ac8fa79..ac20364297 100644
--- a/ctest/CTestEnvironment-nersc.cmake
+++ b/ctest/CTestEnvironment-nersc.cmake
@@ -1,7 +1,7 @@
 #==============================================================================
 #
 #  This file sets the environment variables needed to configure and build
-#  on the NERSC systems 
+#  on the NERSC systems
 #  (edison/ corip1).
 #
 #==============================================================================
diff --git a/ctest/runctest-ncsa.sh b/ctest/runctest-ncsa.sh
index c3cd75e300..353066174d 100755
--- a/ctest/runctest-ncsa.sh
+++ b/ctest/runctest-ncsa.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 #==============================================================================
 #
-#  This script defines how to run CTest on the National Center for 
+#  This script defines how to run CTest on the National Center for
 #  Supercomputing Applications system (blue waters).
 #
 #  This assumes the CTest model name (e.g., "Nightly") is passed to it when
diff --git a/ctest/runctest-nersc.sh b/ctest/runctest-nersc.sh
index a84d26bbeb..3bfd3c8a59 100755
--- a/ctest/runctest-nersc.sh
+++ b/ctest/runctest-nersc.sh
@@ -45,7 +45,7 @@ case "$NERSC_HOST" in
     cori)
 	salloc -N 1 -C knl ./runctest.slurm
 	;;
-esac 
+esac
 # Wait for the job to complete before exiting
 #while true; do
 #	status=`squeue -j $jobid`
diff --git a/doc/CMakeFiles/CMakeOutput.log b/doc/CMakeFiles/CMakeOutput.log
index c254632f6b..36520873ae 100644
--- a/doc/CMakeFiles/CMakeOutput.log
+++ b/doc/CMakeFiles/CMakeOutput.log
@@ -1,8 +1,8 @@
 The system is: Linux - 3.10.0-123.el7.x86_64 - x86_64
 Compiling the C compiler identification source file "CMakeCCompilerId.c" succeeded.
-Compiler: /usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc 
-Build flags: 
-Id flags: 
+Compiler: /usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc
+Build flags:
+Id flags:
 
 The output was:
 0
@@ -14,9 +14,9 @@ Compilation of the C compiler identification source "CMakeCCompilerId.c" produce
 The C compiler identification is Intel, found in "/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/3.2.3/CompilerIdC/a.out"
 
 Compiling the CXX compiler identification source file "CMakeCXXCompilerId.cpp" succeeded.
-Compiler: /usr/bin/c++ 
-Build flags: 
-Id flags: 
+Compiler: /usr/bin/c++
+Build flags:
+Id flags:
 
 The output was:
 0
@@ -38,7 +38,7 @@ Building C object CMakeFiles/cmTryCompileExec1905307408.dir/testCCompiler.c.o
 icc: command line remark #10148: option '-i-dynamic' not supported
 Linking C executable cmTryCompileExec1905307408
 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec1905307408.dir/link.txt --verbose=1
-/usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc       CMakeFiles/cmTryCompileExec1905307408.dir/testCCompiler.c.o  -o cmTryCompileExec1905307408 -rdynamic 
+/usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc       CMakeFiles/cmTryCompileExec1905307408.dir/testCCompiler.c.o  -o cmTryCompileExec1905307408 -rdynamic
 icc: command line remark #10148: option '-i-dynamic' not supported
 gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp'
 
@@ -55,7 +55,7 @@ Building C object CMakeFiles/cmTryCompileExec3327212404.dir/CMakeCCompilerABI.c.
 icc: command line remark #10148: option '-i-dynamic' not supported
 Linking C executable cmTryCompileExec3327212404
 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec3327212404.dir/link.txt --verbose=1
-/usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc      -v CMakeFiles/cmTryCompileExec3327212404.dir/CMakeCCompilerABI.c.o  -o cmTryCompileExec3327212404 -rdynamic  
+/usr/mpi/intel/mvapich2-1.8.1-qlc/bin/mpicc      -v CMakeFiles/cmTryCompileExec3327212404.dir/CMakeCCompilerABI.c.o  -o cmTryCompileExec3327212404 -rdynamic
 mpicc for MVAPICH2 version 1.8.1
 icc: command line remark #10148: option '-i-dynamic' not supported
 icc version 15.0.2 (gcc version 4.8.3 compatibility)
@@ -190,7 +190,7 @@ Building CXX object CMakeFiles/cmTryCompileExec2556829595.dir/testCXXCompiler.cx
 /usr/bin/c++     -o CMakeFiles/cmTryCompileExec2556829595.dir/testCXXCompiler.cxx.o -c /scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp/testCXXCompiler.cxx
 Linking CXX executable cmTryCompileExec2556829595
 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec2556829595.dir/link.txt --verbose=1
-/usr/bin/c++        CMakeFiles/cmTryCompileExec2556829595.dir/testCXXCompiler.cxx.o  -o cmTryCompileExec2556829595 -rdynamic 
+/usr/bin/c++        CMakeFiles/cmTryCompileExec2556829595.dir/testCXXCompiler.cxx.o  -o cmTryCompileExec2556829595 -rdynamic
 gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp'
 
 
@@ -205,14 +205,14 @@ Building CXX object CMakeFiles/cmTryCompileExec1080422183.dir/CMakeCXXCompilerAB
 /usr/bin/c++     -o CMakeFiles/cmTryCompileExec1080422183.dir/CMakeCXXCompilerABI.cpp.o -c /home/katec/cmake/cmake-3.2.3/Modules/CMakeCXXCompilerABI.cpp
 Linking CXX executable cmTryCompileExec1080422183
 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec1080422183.dir/link.txt --verbose=1
-/usr/bin/c++       -v CMakeFiles/cmTryCompileExec1080422183.dir/CMakeCXXCompilerABI.cpp.o  -o cmTryCompileExec1080422183 -rdynamic  
+/usr/bin/c++       -v CMakeFiles/cmTryCompileExec1080422183.dir/CMakeCXXCompilerABI.cpp.o  -o cmTryCompileExec1080422183 -rdynamic
 Using built-in specs.
 COLLECT_GCC=/usr/bin/c++
 COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/4.8.3/lto-wrapper
 Target: x86_64-redhat-linux
 Configured with: ../configure --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-bootstrap --enable-shared --enable-threads=posix --enable-checking=release --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-linker-hash-style=gnu --enable-languages=c,c++,objc,obj-c++,java,fortran,ada,go,lto --enable-plugin --enable-initfini-array --disable-libgcj --with-isl=/builddir/build/BUILD/gcc-4.8.3-20140911/obj-x86_64-redhat-linux/isl-install --with-cloog=/builddir/build/BUILD/gcc-4.8.3-20140911/obj-x86_64-redhat-linux/cloog-install --enable-gnu-indirect-function --with-tune=generic --with-arch_32=x86-64 --build=x86_64-redhat-linux
 Thread model: posix
-gcc version 4.8.3 20140911 (Red Hat 4.8.3-9) (GCC) 
+gcc version 4.8.3 20140911 (Red Hat 4.8.3-9) (GCC)
 COMPILER_PATH=/usr/local/intel-cluster-15.0.2.164/:/usr/libexec/gcc/x86_64-redhat-linux/4.8.3/:/usr/libexec/gcc/x86_64-redhat-linux/4.8.3/:/usr/libexec/gcc/x86_64-redhat-linux/:/usr/lib/gcc/x86_64-redhat-linux/4.8.3/:/usr/lib/gcc/x86_64-redhat-linux/
 LIBRARY_PATH=/usr/lib/gcc/x86_64-redhat-linux/4.8.3/:/usr/lib/gcc/x86_64-redhat-linux/4.8.3/../../../../lib64/:/lib/../lib64/:/usr/lib/../lib64/:/usr/lib/gcc/x86_64-redhat-linux/4.8.3/../../../:/lib/:/usr/lib/
 COLLECT_GCC_OPTIONS='-v' '-o' 'cmTryCompileExec1080422183' '-rdynamic' '-shared-libgcc' '-mtune=generic' '-march=x86-64'
@@ -301,7 +301,7 @@ Building CXX object CMakeFiles/cmTryCompileExec2444100226.dir/feature_tests.cxx.
 /usr/bin/c++    -std=c++1y -o CMakeFiles/cmTryCompileExec2444100226.dir/feature_tests.cxx.o -c /scratch/cluster/katec/ParallelIO/doc/CMakeFiles/feature_tests.cxx
 Linking CXX executable cmTryCompileExec2444100226
 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec2444100226.dir/link.txt --verbose=1
-/usr/bin/c++        CMakeFiles/cmTryCompileExec2444100226.dir/feature_tests.cxx.o  -o cmTryCompileExec2444100226 -rdynamic 
+/usr/bin/c++        CMakeFiles/cmTryCompileExec2444100226.dir/feature_tests.cxx.o  -o cmTryCompileExec2444100226 -rdynamic
 gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp'
 
 
@@ -375,7 +375,7 @@ Building CXX object CMakeFiles/cmTryCompileExec295155124.dir/feature_tests.cxx.o
 /usr/bin/c++    -std=c++11 -o CMakeFiles/cmTryCompileExec295155124.dir/feature_tests.cxx.o -c /scratch/cluster/katec/ParallelIO/doc/CMakeFiles/feature_tests.cxx
 Linking CXX executable cmTryCompileExec295155124
 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec295155124.dir/link.txt --verbose=1
-/usr/bin/c++        CMakeFiles/cmTryCompileExec295155124.dir/feature_tests.cxx.o  -o cmTryCompileExec295155124 -rdynamic 
+/usr/bin/c++        CMakeFiles/cmTryCompileExec295155124.dir/feature_tests.cxx.o  -o cmTryCompileExec295155124 -rdynamic
 gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp'
 
 
@@ -449,7 +449,7 @@ Building CXX object CMakeFiles/cmTryCompileExec3307289814.dir/feature_tests.cxx.
 /usr/bin/c++    -std=c++98 -o CMakeFiles/cmTryCompileExec3307289814.dir/feature_tests.cxx.o -c /scratch/cluster/katec/ParallelIO/doc/CMakeFiles/feature_tests.cxx
 Linking CXX executable cmTryCompileExec3307289814
 /home/katec/cmake/cmake-3.2.3/bin/cmake -E cmake_link_script CMakeFiles/cmTryCompileExec3307289814.dir/link.txt --verbose=1
-/usr/bin/c++        CMakeFiles/cmTryCompileExec3307289814.dir/feature_tests.cxx.o  -o cmTryCompileExec3307289814 -rdynamic 
+/usr/bin/c++        CMakeFiles/cmTryCompileExec3307289814.dir/feature_tests.cxx.o  -o cmTryCompileExec3307289814 -rdynamic
 gmake[1]: Leaving directory `/scratch/cluster/katec/ParallelIO/doc/CMakeFiles/CMakeTmp'
 
 
diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in
index 713d3b56aa..3047a79e14 100644
--- a/doc/Doxyfile.in
+++ b/doc/Doxyfile.in
@@ -291,7 +291,7 @@ OPTIMIZE_OUTPUT_VHDL   = NO
 # Note that for custom extensions you also need to set FILE_PATTERNS otherwise
 # the files are not read by doxygen.
 
-EXTENSION_MAPPING      = f90=Fortran 
+EXTENSION_MAPPING      = f90=Fortran
 
 # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
 # according to the Markdown format, which allows for more readable
@@ -772,7 +772,7 @@ INPUT                  = @CMAKE_CURRENT_SOURCE_DIR@/../doc/source \
                          @CMAKE_CURRENT_SOURCE_DIR@/../examples/c \
                          @CMAKE_CURRENT_SOURCE_DIR@/../examples/f03 \
 			 @FORTRAN_SRC_FILES@ \
-			 @C_SRC_FILES@ 
+			 @C_SRC_FILES@
 
 
 # This tag can be used to specify the character encoding of the source files
diff --git a/doc/DoxygenLayout.xml b/doc/DoxygenLayout.xml
index 563e9b1df9..c0f0b1cb89 100644
--- a/doc/DoxygenLayout.xml
+++ b/doc/DoxygenLayout.xml
@@ -6,7 +6,7 @@
       <tab type="namespacemembers" visible="yes" title="Symbol Index"/>
       <tab type="modules" visible="yes" title="Modules"/>
       <tab type="classes" visible="yes" title="Data Types"/>
-      <tab type="examples" visible="yes" title="Examples"/>  
+      <tab type="examples" visible="yes" title="Examples"/>
       <tab type="pages" visible="yes" title="Page Index"/>
       <tab type="namespaces" visible="no" title="Namespaces (JMD)">
         <tab type="namespaces" visible="yes" title=""/>
diff --git a/doc/customdoxygen.css b/doc/customdoxygen.css
index 5f14f59d3f..f870af6320 100644
--- a/doc/customdoxygen.css
+++ b/doc/customdoxygen.css
@@ -141,11 +141,11 @@ a.elRef {
 }
 
 a.code, a.code:visited {
-	color: #4665A2; 
+	color: #4665A2;
 }
 
 a.codeRef, a.codeRef:visited {
-	color: #4665A2; 
+	color: #4665A2;
 }
 
 /* @end */
@@ -289,7 +289,7 @@ p.formulaDsp {
 }
 
 img.formulaDsp {
-	
+
 }
 
 img.formulaInl {
@@ -347,20 +347,20 @@ span.charliteral {
 	color: #008080
 }
 
-span.vhdldigit { 
-	color: #ff00ff 
+span.vhdldigit {
+	color: #ff00ff
 }
 
-span.vhdlchar { 
-	color: #000000 
+span.vhdlchar {
+	color: #000000
 }
 
-span.vhdlkeyword { 
-	color: #700070 
+span.vhdlkeyword {
+	color: #700070
 }
 
-span.vhdllogic { 
-	color: #ff0000 
+span.vhdllogic {
+	color: #ff0000
 }
 
 blockquote {
@@ -555,9 +555,9 @@ table.memberdecls {
 }
 
 .memdoc, dl.reflist dd {
-        border-bottom: 1px solid #A8B8D9;      
-        border-left: 1px solid #A8B8D9;      
-        border-right: 1px solid #A8B8D9; 
+        border-bottom: 1px solid #A8B8D9;
+        border-left: 1px solid #A8B8D9;
+        border-right: 1px solid #A8B8D9;
         padding: 6px 10px 2px 10px;
         background-color: #FBFCFD;
         border-top-width: 0;
@@ -609,18 +609,18 @@ dl.reflist dd {
 .params, .retval, .exception, .tparams {
         margin-left: 0px;
         padding-left: 0px;
-}       
+}
 
 .params .paramname, .retval .paramname {
         font-weight: bold;
         vertical-align: top;
 }
-        
+
 .params .paramtype {
         font-style: italic;
         vertical-align: top;
-}       
-        
+}
+
 .params .paramdir {
         font-family: "courier new",courier,monospace;
         vertical-align: top;
@@ -797,8 +797,8 @@ table.fieldtable {
 
 .fieldtable td.fielddoc p:first-child {
         margin-top: 2px;
-}       
-        
+}
+
 .fieldtable td.fielddoc p:last-child {
         margin-bottom: 2px;
 }
@@ -872,7 +872,7 @@ table.fieldtable {
 	color: #283A5D;
 	font-family: 'Lucida Grande',Geneva,Helvetica,Arial,sans-serif;
 	text-shadow: 0px 1px 1px rgba(255, 255, 255, 0.9);
-	text-decoration: none;        
+	text-decoration: none;
 }
 
 .navpath li.navelem a:hover
@@ -901,7 +901,7 @@ div.summary
 	padding-right: 5px;
 	width: 50%;
 	text-align: right;
-}       
+}
 
 div.summary a
 {
@@ -1013,19 +1013,19 @@ dl.section dd {
 	vertical-align: bottom;
 	border-collapse: separate;
 }
- 
+
 #projectlogo img
-{ 
+{
 	border: 0px none;
 }
- 
+
 #projectname
 {
 	font: 300% Tahoma, Arial,sans-serif;
 	margin: 0px;
 	padding: 2px 0px;
 }
-    
+
 #projectbrief
 {
 	font: 120% Tahoma, Arial,sans-serif;
@@ -1120,7 +1120,7 @@ div.toc ul {
         list-style: none outside none;
         border: medium none;
         padding: 0px;
-}       
+}
 
 div.toc li.level1 {
         margin-left: 0px;
@@ -1181,4 +1181,3 @@ tr.heading h2 {
     display:inline;
   }
 }
-
diff --git a/doc/source/CAMexample.txt b/doc/source/CAMexample.txt
index 77bd3df9fb..d6a3c20f3c 100644
--- a/doc/source/CAMexample.txt
+++ b/doc/source/CAMexample.txt
@@ -13,7 +13,7 @@
  * Documents produced by Doxygen are derivative works derived from the
  * input used in their production; they are not affected by this license.
  *
- */ 
+ */
 /*! \page CAMexample Community Atmosphere Model (CAM)
 
 \section cam Implementation of PIO in CAM
@@ -37,7 +37,7 @@ initializes PIO in CAM. Init_pio_subsystem calls read_namelist_pio, which calls
 set_pio_parameters. The main parameters set includes the IO mode (netcdf vs pnetcdf),
 number of IO tasks, and IO stride.
 
-Cam_pio_createfile and cam_pio_openfile create and open a PIO file, respectively. 
+Cam_pio_createfile and cam_pio_openfile create and open a PIO file, respectively.
 Cam_pio_createfile is called from cam_write_restart, h_define (called from wshist,
 which is called from write_restart_history), and atm_write_srfrest_mct.
 
@@ -72,7 +72,7 @@ decomposition already exists. The routine performing that search is find_iodesc
 (called from get_phys_decomp and get_dyn_decomp). This capability is supported only
 when the range of history output is the whole domain.
 
-Get_phys_decomp calls get_phys_ldof (or get_column_ldof), and get_dyn_decomp calls 
+Get_phys_decomp calls get_phys_ldof (or get_column_ldof), and get_dyn_decomp calls
 get_dyn_ldof (or get_column_ldof). These routines do the bulk of the work in
 constructing the IO decompositions. Get_column_ldof is called when the history output
 is restricted to a subset of the domain.
diff --git a/doc/source/Decomp.txt b/doc/source/Decomp.txt
index f47c8bf18e..27dc1f1249 100644
--- a/doc/source/Decomp.txt
+++ b/doc/source/Decomp.txt
@@ -38,21 +38,21 @@ As an example suppose we have a global two dimensional grid of size
 terms of offset from the initial element ie
 
 <pre>
-     0  1  2  3 
-     4  5  6  7 
+     0  1  2  3
+     4  5  6  7
      8  9 10 11
     12 13 14 15
-    16 17 18 19 
+    16 17 18 19
 </pre>
 
 Now suppose this data is distributed over the compute tasks as follows:
 
 <pre>
-0: {   0  4 8 12  } 
-1: {  16 1 5 9  } 
-2: {  13 17 2 6  } 
-3: {  10 14 18 3  } 
-4: {   7 11 15 19  } 
+0: {   0  4 8 12  }
+1: {  16 1 5 9  }
+2: {  13 17 2 6  }
+3: {  10 14 18 3  }
+4: {   7 11 15 19  }
 </pre>
 
 If we have 2 io tasks the Box rearranger would give:
@@ -83,4 +83,3 @@ IO task provides a measure of what you might expect the performance of
 the underlying IO library to be if it were used without PIO.
 
 */
-
diff --git a/doc/source/Error.txt b/doc/source/Error.txt
index 1234c9f7c1..2911446de3 100644
--- a/doc/source/Error.txt
+++ b/doc/source/Error.txt
@@ -1,4 +1,4 @@
-/*! 
+/*!
 \page error Error Handling
 
 By default, PIO handles errors internally by printing a string
diff --git a/doc/source/Examples.txt b/doc/source/Examples.txt
index 932d278957..db7a8c4e8b 100644
--- a/doc/source/Examples.txt
+++ b/doc/source/Examples.txt
@@ -28,11 +28,11 @@ The Fortran examples are in the examples/f03 subdirectory.
 
 ### Other Examples
 
-  PIO has been implemented in several geophysical component models, including the 
-Community Atmosphere Model (CAM), the Community Land Model (CLM), the Parallel Ocean Program 
-(POP), the Community Ice CodE (CICE), and coupler for used by CCSM4.0 (CPL7).  We also provide 
-several simpler example code as well as a test code that is suitable for regression testing and 
-benchmarking.    
+  PIO has been implemented in several geophysical component models, including the
+Community Atmosphere Model (CAM), the Community Land Model (CLM), the Parallel Ocean Program
+(POP), the Community Ice CodE (CICE), and coupler for used by CCSM4.0 (CPL7).  We also provide
+several simpler example code as well as a test code that is suitable for regression testing and
+benchmarking.
 
  - \subpage CAMexample
  - \subpage testpio_example
diff --git a/doc/source/Installing.txt b/doc/source/Installing.txt
index d0423abc1f..f2e32f622b 100644
--- a/doc/source/Installing.txt
+++ b/doc/source/Installing.txt
@@ -17,7 +17,7 @@ pio-2.4.3.tar.gz.
 
 PIO can use NetCDF (version 4.6.1+) and/or PnetCDF (version 1.9.0+) for I/O.
 Ideally, the NetCDF version should be built with MPI, which requires that it
-be linked with an MPI-enabled version of HDF5.  Optionally, NetCDF can be 
+be linked with an MPI-enabled version of HDF5.  Optionally, NetCDF can be
 built with DAP support, which introduces a dependency on CURL.  Additionally,
 HDF5, itself, introduces dependencies on LIBZ and (optionally) SZIP.
 
@@ -91,10 +91,10 @@ where `mpicc` and `mpif90` are the appropriate MPI-enabled compiler wrappers
 for your system.
 
 The `OPTIONS` section typically should consist of pointers to the install
-locations for various dependencies, assuming these dependencies are not 
-located in *canonical* search locations.  
+locations for various dependencies, assuming these dependencies are not
+located in *canonical* search locations.
 
-For each dependency `XXX`, one can specify the location of its 
+For each dependency `XXX`, one can specify the location of its
 installation path with the CMake variable `XXX_PATH`.  If the `C` and
 `Fortran` libraries for the dependency are installed in different locations
 (such as can be done with NetCDF), then you can specify individually
@@ -111,7 +111,7 @@ CMake configuration line:
 
 This works for the dependencies: `NetCDF`, `PnetCDF`, `HDF5`, `LIBZ`, `SZIP`.
 
-For specific instructions to install on various commonly used super computers, please read the [walk-through guide to PIO Installation](@ref mach_walkthrough). 
+For specific instructions to install on various commonly used super computers, please read the [walk-through guide to PIO Installation](@ref mach_walkthrough).
 
 ### Additional CMake Options ###
 
@@ -125,14 +125,14 @@ libraries are already installed on the system, the user can point PIO to the
 location of these libraries with the `GPTL_PATH` variable (or, individually,
 `GPTL_C_PATH` and `GPTL_Fortran_Perf_PATH` variables).  However, if these
 GPTL libraries are not installed on the system, and GPTL cannot be found,
-then PIO will build its own internal version of GPTL.  
+then PIO will build its own internal version of GPTL.
 
 If PnetCDF is not installed on the system, the user can disable its use by
 setting `-DWITH_PNETCDF=OFF`.  This will disable the search for PnetCDF on the
 system and disable the use of PnetCDF from within PIO.
 
 If the user wishes to disable the PIO tests, then the user can set the
-variable `-DPIO_ENABLE_TESTS=OFF`.  This will entirely disable the CTest 
+variable `-DPIO_ENABLE_TESTS=OFF`.  This will entirely disable the CTest
 testing suite, as well as remove all of the test build targets.
 
 If you wish to install PIO in a safe location for use later with other
@@ -168,7 +168,7 @@ Once the tests have been built, you may run tests with:
 _Note: If you have not run `make tests` before you run `ctest`, then you will see
 all of the tests fail._
 
-Alternatively, you may build the test executables and then run tests 
+Alternatively, you may build the test executables and then run tests
 immediately with:
 
 
diff --git a/doc/source/Testing.txt b/doc/source/Testing.txt
index 0ef7c48a31..058df9ba78 100644
--- a/doc/source/Testing.txt
+++ b/doc/source/Testing.txt
@@ -112,17 +112,17 @@ the PIO build directory).
 The contents of the namelist file should look like:
 
      &pioperf
-     
+
      decompfile = "/u/home/user/piodecomp30tasks01dims06.dat"
-     
+
      pio_typenames = 'pnetcdf'
-     
+
      niotasks = 30
-     
+
      rearrangers = 1
-     
+
      nvars = 2
-     
+
      /
 
 Here, the second line ("decompfile") points to the path for your
diff --git a/doc/source/api.txt b/doc/source/api.txt
index 0e36baa82a..435f59f972 100644
--- a/doc/source/api.txt
+++ b/doc/source/api.txt
@@ -19,7 +19,7 @@
     - \ref PIO_inquire_variable
     - \ref PIO_inq_att
   \section metadata Defining Metadata
-    - \ref PIO_def_dim 
+    - \ref PIO_def_dim
     - \ref PIO_def_var
     - \ref PIO_get_att
     - \ref PIO_put_att
diff --git a/doc/source/base.txt b/doc/source/base.txt
index d53491e1d8..d81d27f4e4 100644
--- a/doc/source/base.txt
+++ b/doc/source/base.txt
@@ -1,4 +1,4 @@
-/*! 
+/*!
 
 @mainpage Parallel I/O Libraries (PIO)
 
diff --git a/doc/source/contributing_code.txt b/doc/source/contributing_code.txt
index 3472c425ee..4b070b3f35 100644
--- a/doc/source/contributing_code.txt
+++ b/doc/source/contributing_code.txt
@@ -121,7 +121,7 @@ all issues have been resolved.
 main.
 
 <li>When a branch is ready, it is submitted to code review via pull
-request. A number of Github tests will be run automatically. 
+request. A number of Github tests will be run automatically.
 
 <li>When code review is complete, and the changes are approved, the PR
 is merged into the main branch.
diff --git a/doc/source/example/simple-bc b/doc/source/example/simple-bc
index ee58c4bbeb..6a8d49ee28 100644
--- a/doc/source/example/simple-bc
+++ b/doc/source/example/simple-bc
@@ -1,6 +1,6 @@
 
  type (iosystem_desc_t)     :: iosystem
- integer (i4)               :: dims(1) 
+ integer (i4)               :: dims(1)
  integer (kind=PIO_OFFSET)  :: start(1), count(1)
  type (io_desc_t)           :: iodesc
  ...
@@ -11,4 +11,3 @@
  start(1) = 3
  count(1) = 3
  call PIO_initdecomp(iosystem,PIO_double,dims,start,count,iodesc)
-
diff --git a/doc/source/example/simple-bc-rearr b/doc/source/example/simple-bc-rearr
index c919f29cc5..ea1cd843ad 100644
--- a/doc/source/example/simple-bc-rearr
+++ b/doc/source/example/simple-bc-rearr
@@ -1,6 +1,6 @@
 
  type (iosystem_desc_t)     :: iosystem
- integer (i4)               :: dims(1) 
+ integer (i4)               :: dims(1)
  integer (kind=PIO_OFFSET)  :: compstart(1), compcount(1)
  integer (kind=PIO_OFFSET)  :: iostart(1), iocount(1)
  type (io_desc_t)           :: iodesc
@@ -14,4 +14,3 @@
  iostart(1) = 1
  iocount(1) = 4
  call PIO_initdecomp(iosystem,PIO_double,dims,compstart,compcount,iodesc,iostart=iostart,iocount=iocount)
-
diff --git a/doc/source/example/simple-bc-rearr-pe1 b/doc/source/example/simple-bc-rearr-pe1
index 5399ad8377..9f98f09f34 100644
--- a/doc/source/example/simple-bc-rearr-pe1
+++ b/doc/source/example/simple-bc-rearr-pe1
@@ -1,6 +1,6 @@
 
  type (iosystem_desc_t)     :: iosystem
- integer (i4)               :: dims(1) 
+ integer (i4)               :: dims(1)
  integer (kind=PIO_OFFSET)  :: compstart(1), compcount(1)
  type (io_desc_t)           :: iodesc
  ...
@@ -11,4 +11,3 @@
  compstart(1) = 1
  compcount(1) = 2
  call PIO_initdecomp(iosystem,PIO_double,dims,compstart,compcount,iodesc)
-
diff --git a/doc/source/example/simple-bc-rearr-pe2 b/doc/source/example/simple-bc-rearr-pe2
index 31880f4f87..917e09ba63 100644
--- a/doc/source/example/simple-bc-rearr-pe2
+++ b/doc/source/example/simple-bc-rearr-pe2
@@ -1,6 +1,6 @@
 
  type (iosystem_desc_t)     :: iosystem
- integer (i4)               :: dims(1) 
+ integer (i4)               :: dims(1)
  integer (kind=PIO_OFFSET)  :: compstart(1), compcount(1)
  integer (kind=PIO_OFFSET)  :: iostart(1), iocount(1)
  type (io_desc_t)           :: iodesc
@@ -14,4 +14,3 @@
  iostart(1) = 5
  iocount(1) = 4
  call PIO_initdecomp(iosystem,PIO_double,dims,compstart,compcount,iodesc,iostart=iostart,iocount=iocount)
-
diff --git a/doc/source/example/simple-dof b/doc/source/example/simple-dof
index b53a82d799..e967bf22ef 100644
--- a/doc/source/example/simple-dof
+++ b/doc/source/example/simple-dof
@@ -1,6 +1,6 @@
 
  type (iosystem_desc_t)     :: iosystem
- integer (i4)               :: dims(1) 
+ integer (i4)               :: dims(1)
  integer (i4)               :: compdof
  type (io_desc_t)           :: iodesc
  ...
@@ -10,4 +10,3 @@
  dims(1) = 8
  compdof = (/2,4,5/)
  call PIO_initdecomp(iosystem,PIO_double,dims,compdof,iodesc)
-
diff --git a/doc/source/example/simple-dof-rearr b/doc/source/example/simple-dof-rearr
index af8b9910c2..dd95125153 100644
--- a/doc/source/example/simple-dof-rearr
+++ b/doc/source/example/simple-dof-rearr
@@ -1,6 +1,6 @@
 
  type (iosystem_desc_t)     :: iosystem
- integer (i4)               :: dims(1) 
+ integer (i4)               :: dims(1)
  integer (i4)               :: compdof
  type (io_desc_t)           :: iodesc
  integer (kind=PIO_OFFSET)  :: iostart(:),iocount(:)
@@ -14,4 +14,3 @@
  iostart(1) = 1
  iocount(1) = 4
  call PIO_initdecomp(iosystem,PIO_double,dims,compdof,iodesc,iostart=iostart,iocount=iocount)
-
diff --git a/doc/source/faq.txt b/doc/source/faq.txt
index c3c4bca336..713174004c 100644
--- a/doc/source/faq.txt
+++ b/doc/source/faq.txt
@@ -1,14 +1,14 @@
-/*! \page faq Frequently Asked Questions 
+/*! \page faq Frequently Asked Questions
 
 Here is a list of frequently asked questions and their answers.
-   
+
 <dl>
 
 <dt>How do I specify which tasks perform IO?</dt>
 
 <dd>This is done in the call to \ref PIO_init which has two
 interfaces: init_intracom and init_intercom.
-    
+
   <ul>
 
     <li> In the init_intracom interface, use the num_iotasks and
@@ -32,7 +32,7 @@ the tunable blocksize parameter, \ref PIO_set_blocksize, determines
 the actual number of tasks used for a given IO operation.
 </dd>
 
-<dt>How do I test if PIO is installed and working correctly?</dt> 
+<dt>How do I test if PIO is installed and working correctly?</dt>
 
 <dd>The PIO Library distribution contains tests for PIO. They are run
 my 'make check'. The tests use mpiexec to run tests on 4, 8, or 16
diff --git a/doc/source/iosystem.txt b/doc/source/iosystem.txt
index 5acce7fc92..b4616c84fa 100644
--- a/doc/source/iosystem.txt
+++ b/doc/source/iosystem.txt
@@ -21,4 +21,3 @@ for each open IOSystem.
 @image html PIO_Async.png "PIO Async Mode"
 
 */
-
diff --git a/doc/source/mach_walkthrough.txt b/doc/source/mach_walkthrough.txt
index 357b888213..d741cd28e7 100644
--- a/doc/source/mach_walkthrough.txt
+++ b/doc/source/mach_walkthrough.txt
@@ -77,13 +77,13 @@ Building PIO requires running the CMake configure and then make. In the PIO_buil
 <ol>
 <li>Directory setup
 
-Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. 
+Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it.
 
 <li>Modules
 
-Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Edison. 
+Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Edison.
 
-+ Intel 
++ Intel
 
   %> module purge<br>
   %> module load PrgEnv-intel<br>
@@ -156,7 +156,7 @@ Building PIO requires running the CMake configure and then make. In the PIO_buil
 <ol>
 <li>Directory setup
 
-Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. 
+Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it.
 
 <li> Softenv packages and environment variables
 
@@ -187,11 +187,11 @@ Building PIO requires running the CMake configure and then make. In the PIO_buil
 <ol>
 <li>Directory setup
 
-Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. 
+Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it.
 
 <li>Modules
 
-Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Hobart. 
+Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Hobart.
 
 + Intel
 
@@ -244,11 +244,11 @@ Building PIO requires running the CMake configure and then make. In the PIO_buil
 <ol>
 <li>Directory setup
 
-Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it. 
+Download a copy of the PIO source into a sub-directory of your working directory (refered to here as the PIO_source directory). Create another sub-directory for the build (refered to here as the PIO_build directory) and 'cd' into it.
 
 <li>Modules
 
-Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Hobart. 
+Modules required for installation depend on your prefered compiler. Issue the commands below to set the module environment for building PIO on Hobart.
 
 + Intel
 
@@ -261,7 +261,7 @@ Modules required for installation depend on your prefered compiler. Issue the co
   %> module purge<br>
   %> module load compiler/nag/6.0<br>
   %> module load tool/parallel-netcdf/1.6.1/nag/openmpi<br>
-  
+
 + PGI
 
   %> module purge<br>
diff --git a/doc/source/testpio_example.txt b/doc/source/testpio_example.txt
index c5d4ef7e5c..770ced7528 100644
--- a/doc/source/testpio_example.txt
+++ b/doc/source/testpio_example.txt
@@ -1,8 +1,8 @@
 /*! \page testpio_example testpio: a regression and benchmarking code
 
-The testpio directory, included with the release package, tests both the accuracy 
-and performance of reading and writing data 
-using the pio library. 
+The testpio directory, included with the release package, tests both the accuracy
+and performance of reading and writing data
+using the pio library.
 
 The testpio directory contains 3 perl scripts that you can use to build and run the testpio.F90 code.
 <ul>
@@ -49,7 +49,7 @@ block, io_nml, contains some general settings:
                      ("bin","pnc","snc"), binary, pnetcdf, or serial netcdf</td>
   </tr>
   <tr>
-    <td>rearr</td>          <td>string, type of rearranging to be done 
+    <td>rearr</td>          <td>string, type of rearranging to be done
                      ("none","mct","box","boxauto")</td>
   </tr>
   <tr>
@@ -61,15 +61,15 @@ block, io_nml, contains some general settings:
     <td>base</td>           <td>integer, base pe associated with nprocIO striding</td>
   </tr>
   <tr>
-    <td>stride</td>         <td>integer, the stride of io pes across the global pe set. A stride=-1 
+    <td>stride</td>         <td>integer, the stride of io pes across the global pe set. A stride=-1
     	 	     directs PIO to calculate the stride automatically.</td>
   </tr>
   <tr>
-    <td>num_aggregator</td> <td>integer, mpi-io number of aggregators, only used if no 
+    <td>num_aggregator</td> <td>integer, mpi-io number of aggregators, only used if no
                      pio rearranging is done</td>
   </tr>
   <tr>
-    <td>dir</td>            <td>string, directory to write output data, this must exist 
+    <td>dir</td>            <td>string, directory to write output data, this must exist
                      before the model starts up</td>
   </tr>
   <tr>
@@ -85,19 +85,19 @@ block, io_nml, contains some general settings:
     <td>compdof_input</td>  <td>string, setting of the compDOF ('namelist' or a filename)</td>
   </tr>
   <tr>
-    <td>compdof_output</td> <td>string, whether the compDOF is saved to disk 
+    <td>compdof_output</td> <td>string, whether the compDOF is saved to disk
                      ('none' or a filename)</td>
   </tr>
 </table>
 
 Notes:
   - the "mct" rearr option is not currently available
-  - if rearr is set to "none", then the computational decomposition is also 
+  - if rearr is set to "none", then the computational decomposition is also
     going to be used as the IO decomposition.  The computation decomposition
     must therefore be suited to the underlying I/O methods.
   - if rearr is set to "box", then pio is going to generate an internal
     IO decomposition automatically and pio will rearrange to that decomp.
-  - num_aggregator is used with mpi-io and no pio rearranging.  mpi-io is only 
+  - num_aggregator is used with mpi-io and no pio rearranging.  mpi-io is only
     used with binary data.
   - nprocsIO, base, and stride implementation has some special options
     - if nprocsIO >  0 and stride >  0, then use input values
@@ -123,7 +123,7 @@ blocks are identical in use.
                  ("xyz","xzy","yxz","yzx","zxy","zyx")</td>
   </tr>
   <tr>
-    <td>grddecomp</td>  <td>string, sets up the block size with gdx, gdy, and gdz, see 
+    <td>grddecomp</td>  <td>string, sets up the block size with gdx, gdy, and gdz, see
                  below, ("x","y","z","xy","xye","xz","xze","yz","yze",
                  "xyz","xyze","setblk")</td>
   </tr>
@@ -167,7 +167,7 @@ are provided below.
 
 Testpio writes out several files including summary information to
 stdout, data files to the namelists directory, and a netcdf
-file summarizing the decompositions.  The key output information 
+file summarizing the decompositions.  The key output information
 is written to stdout and contains the timing information.  In addition,
 a netcdf file called gdecomp.nc is written that provides both the
 block and task ids for each gridcell as computed by the decompositions.
@@ -221,24 +221,24 @@ combinations of these cpp flags.
 
 The decomposition implementation supports the decomposition of
 a general 3 dimensional "nx * ny * nz" grid into multiple blocks
-of gridcells which are then ordered and assigned to processors.  
-In general, blocks in the decomposition are rectangular, 
-"gdx * gdy * gdz" and the same size, although some blocks around 
-the edges of the domain may be smaller if the decomposition is uneven.  
-Both gridcells within the block and blocks within the domain can be 
+of gridcells which are then ordered and assigned to processors.
+In general, blocks in the decomposition are rectangular,
+"gdx * gdy * gdz" and the same size, although some blocks around
+the edges of the domain may be smaller if the decomposition is uneven.
+Both gridcells within the block and blocks within the domain can be
 ordered in any of the possible dimension hierarchies, such as "xyz"
-where the first dimension is the fastest.  
+where the first dimension is the fastest.
 
-The gdx, gdy, and gdz inputs allow the user to specify the size in 
-any dimension and the grddecomp input specifies which dimensions are 
-to be further optimized.  In general, automatic decomposition generation 
-of 3 dimensional grids can be done in any of possible combination of 
+The gdx, gdy, and gdz inputs allow the user to specify the size in
+any dimension and the grddecomp input specifies which dimensions are
+to be further optimized.  In general, automatic decomposition generation
+of 3 dimensional grids can be done in any of possible combination of
 dimensions, (x, y, z, xy, xz, yz, or xyz), with the other dimensions having a
 fixed block size.  The automatic generation of the decomposition is
 based upon an internal algorithm that tries to determine the most
 "square" blocks with an additional constraint on minimizing the maximum
 number of gridcells across processors.  If evenly divided grids are
-desired, use of the "e" addition to grddecomp specifies that the grid 
+desired, use of the "e" addition to grddecomp specifies that the grid
 decomposition must be evenly divided.  The setblk option uses the
 prescibed gdx, gdy, and gdz inputs without further automation.
 
@@ -247,7 +247,7 @@ in mapping blocks to processors, but has a few additional options.
 "cont1d" (contiguous 1d) basically unwraps the blocks in the order specified
 by the blkorder input and then decomposes that "1d" list of blocks
 onto processors by contiguously grouping blocks together and allocating
-them to a processor.  The number of contiguous blocks that are 
+them to a processor.  The number of contiguous blocks that are
 allocated to a processor is the maximum of the values of bdx, bdy, and
 bdz inputs.  Contiguous blocks are allocated to each processor in turn
 in a round robin fashion until all blocks are allocated.  The
@@ -256,13 +256,13 @@ contiguous blocks are set automatically such that each processor
 recieves only 1 set of contiguous blocks.  The ysym2 and ysym4
 blkdecomp2 options modify the original block layout such that
 the tasks assigned to the blocks are 2-way or 4-way symetric
-in the y axis. 
+in the y axis.
 
 The decomposition tool is extremely flexible, but arbitrary
 inputs will not always yield valid decompositions.  If a valid
 decomposition cannot be computed based on the global grid size,
-number of pes, number of blocks desired, and decomposition options, 
-the model will stop.  
+number of pes, number of blocks desired, and decomposition options,
+the model will stop.
 
 As indicated above, the IO decomposition must be suited to the
 IO methods, so decompositions are even further limited by those
@@ -290,7 +290,7 @@ Some decomposition examples:
 Standard xyz ordering, 2d decomp:
 note: blkdecomp plays no role since there is 1 block per pe
 <pre>
- nx_global  6       
+ nx_global  6
  ny_global  4
  nz_global  1           ______________________________
  npes       4          |B3  P3        |B4  P4         |
@@ -311,7 +311,7 @@ note: blkdecomp plays no role since there is 1 block per pe
 Same as above but yxz ordering, 2d decomp
 note: blkdecomp plays no role since there is 1 block per pe
 <pre>
- nx_global  6       
+ nx_global  6
  ny_global  4
  nz_global  1           _____________________________
  npes       4          |B2  P2        |B4  P4        |
@@ -329,11 +329,11 @@ note: blkdecomp plays no role since there is 1 block per pe
  bdz        0
 </pre>
 
-xyz grid ordering, 1d x decomp 
+xyz grid ordering, 1d x decomp
                note: blkdecomp plays no role since there is 1 block per pe
                note: blkorder plays no role since it's a 1d decomp
 <pre>
- nx_global  8       
+ nx_global  8
  ny_global  4
  nz_global  1           _____________________________________
  npes       4          |B1  P1  |B2  P2   |B3  P3  |B4  P4   |
@@ -353,7 +353,7 @@ xyz grid ordering, 1d x decomp
 
 yxz block ordering, 2d grid decomp, 2d block decomp, 4 block per pe
 <pre>
- nx_global  8       
+ nx_global  8
  ny_global  4
  nz_global  1           _____________________________________
  npes       4          |B4  P2  |B8  P2   |B12  P4 |B16  P4  |
diff --git a/doc/source/users_guide.txt b/doc/source/users_guide.txt
index 539aa0cda3..a03198d9d7 100644
--- a/doc/source/users_guide.txt
+++ b/doc/source/users_guide.txt
@@ -1,4 +1,4 @@
-/*! 
+/*!
 
 @page users_guide PIO User's Guide
 
diff --git a/examples/basic/CMakeLists.txt b/examples/basic/CMakeLists.txt
index efa784e01e..01238266a6 100644
--- a/examples/basic/CMakeLists.txt
+++ b/examples/basic/CMakeLists.txt
@@ -7,7 +7,7 @@ ADD_CUSTOM_COMMAND(
 )
 ENDFOREACH()
 
-SET(SRC check_mod.F90  gdecomp_mod.F90  kinds_mod.F90  namelist_mod.F90  
+SET(SRC check_mod.F90  gdecomp_mod.F90  kinds_mod.F90  namelist_mod.F90
             testpio.F90  utils_mod.F90 ${TEMPSRCF90})
 SET(WSSRC wstest.c)
 
@@ -15,7 +15,7 @@ INCLUDE_DIRECTORIES(${PIO_INCLUDE_DIRS})
 LINK_DIRECTORIES(${PIO_LIB_DIR})
 ADD_EXECUTABLE(testpio ${SRC})
 ADD_EXECUTABLE(wstest ${WSSRC})
-if(${PIO_BUILD_TIMING} MATCHES "ON") 
+if(${PIO_BUILD_TIMING} MATCHES "ON")
   SET(TIMING_LINK_LIB timing)
 endif()
 
@@ -25,8 +25,3 @@ else()
   TARGET_LINK_LIBRARIES(testpio piof pioc ${TIMING_LINK_LIB})
 endif()
 TARGET_LINK_LIBRARIES(wstest pioc ${TIMING_LINK_LIB})
-
-
-
-
-
diff --git a/examples/basic/MPASA30km.csh b/examples/basic/MPASA30km.csh
index a141354fa9..37e7cf7b48 100755
--- a/examples/basic/MPASA30km.csh
+++ b/examples/basic/MPASA30km.csh
@@ -1,10 +1,9 @@
 #!/usr/bin/csh
 set id = `date "+%m%d%y-%H%M"`
 set host = 'kraken'
-#./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 120 --bench MPASA30km -numIO 20 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close 
+#./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 120 --bench MPASA30km -numIO 20 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 240 --bench MPASA30km -numIO 40 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 480 --bench MPASA30km -numIO 80 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 960 --bench MPASA30km -numIO 160 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 1920 --bench MPASA30km -numIO 320 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close
 ./testpio_bench.pl --maxiter 10 --iofmt pnc --numvars 10 --pecount 3840 --bench MPASA30km -numIO 320 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close
-
diff --git a/examples/basic/MPASA60km.csh b/examples/basic/MPASA60km.csh
index 0c3cd5a2c1..b909a3295b 100755
--- a/examples/basic/MPASA60km.csh
+++ b/examples/basic/MPASA60km.csh
@@ -7,4 +7,3 @@ set host = 'kraken'
 ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 480 --bench MPASA60km -numIO 80 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close
 ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 960 --bench MPASA60km -numIO 160 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close
 ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 1020 --bench MPASA60km -numIO 170 --partdir /lustre/scratch/jdennis/MPAS --logfile-suffix trunk_close
-
diff --git a/examples/basic/POPB.csh b/examples/basic/POPB.csh
index 0f9cf55ec8..1103966b15 100755
--- a/examples/basic/POPB.csh
+++ b/examples/basic/POPB.csh
@@ -7,4 +7,3 @@ set host = 'kraken'
 ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 160 --bench POPB --numIO 24 --logfile-suffix trunk_close
 ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 320 --bench POPB --numIO 48 --logfile-suffix trunk_close
 ./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 640 --bench POPB --numIO 96 --logfile-suffix trunk_close
-
diff --git a/examples/basic/POPD.csh b/examples/basic/POPD.csh
index 62d0f1da3e..d36be4c5b2 100644
--- a/examples/basic/POPD.csh
+++ b/examples/basic/POPD.csh
@@ -11,4 +11,3 @@ set host = 'kraken'
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 2000 --bench POPD --numIO 320 --log ${host}.2000.pnc.iotask_320.log.${id}
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 4000 --bench POPD --numIO 640 --log ${host}.4000.pnc.iotask_640.log.${id}
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id}
-
diff --git a/examples/basic/POPDv0.csh b/examples/basic/POPDv0.csh
index d51d7bb31a..0bb7f7951a 100755
--- a/examples/basic/POPDv0.csh
+++ b/examples/basic/POPDv0.csh
@@ -17,4 +17,3 @@ set host = 'kraken'
 ./testpio_bench.pl --maxiter 10 --iofmt snc --pecount 4000 --bench POPD --numIO 1 --log ${host}.4000.snc.box.iotask_1.log.${id}
 
 #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id}
-
diff --git a/examples/basic/POPDv1.csh b/examples/basic/POPDv1.csh
index f10ab6f302..a02ab1fff0 100755
--- a/examples/basic/POPDv1.csh
+++ b/examples/basic/POPDv1.csh
@@ -17,4 +17,3 @@ set id = 012211-1449
 ./testpio_bench.pl --maxiter 10 --iofmt snc --pecount 4000 --bench POPD --numIO 640 --log ${host}.4000.snc.box.iotask_640.log.${id}
 
 #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id}
-
diff --git a/examples/basic/POPDv2.csh b/examples/basic/POPDv2.csh
index 57d3d58e47..981e6d288f 100755
--- a/examples/basic/POPDv2.csh
+++ b/examples/basic/POPDv2.csh
@@ -35,5 +35,3 @@ set host = 'hopper'
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 1600 --bench POPD --numIO 320 --mpi-cb-buffer-size=8388608 --logfile-suffix DnR1
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 2000 --bench POPD --numIO 640 --mpi-cb-buffer-size=8388608 --logfile-suffix DnR1
 #./testpio_bench.pl --maxiter 10 --iofmt pnc --pecount 4000 --bench POPD --numIO 640 --mpi-cb-buffer-size=8388608 --logfile-suffix DnR1
-
-
diff --git a/examples/basic/POPDv3.csh b/examples/basic/POPDv3.csh
index 4483057f12..65b32fcf6d 100755
--- a/examples/basic/POPDv3.csh
+++ b/examples/basic/POPDv3.csh
@@ -13,4 +13,3 @@ set host = 'kraken'
 ./testpio_bench.pl --maxiter 10 --rearr none --iofmt pnc --pecount 4000 --bench POPD --numIO 4000 --log ${host}.4000.pnc.none.iotask_4000.log.${id}
 
 #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id}
-
diff --git a/examples/basic/POPDv4.csh b/examples/basic/POPDv4.csh
index 918ea7e87b..c7b80fdd79 100755
--- a/examples/basic/POPDv4.csh
+++ b/examples/basic/POPDv4.csh
@@ -14,4 +14,3 @@ set host = 'kraken'
 ./testpio_bench.pl --maxiter 10 --iofmt bin --pecount 4000 --bench POPD --numIO 640 --log ${host}.4000.bin.box.iotask_640.log.012211-2233
 
 #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id}
-
diff --git a/examples/basic/POPDv5.csh b/examples/basic/POPDv5.csh
index c1267dd790..994152107e 100755
--- a/examples/basic/POPDv5.csh
+++ b/examples/basic/POPDv5.csh
@@ -14,4 +14,3 @@ set host = 'kraken'
 #./testpio_bench.pl --maxiter 10 --rearr none --iofmt bin --pecount 4000 --bench POPD --numIO 4000 --log ${host}.4000.bin.none.iotask_4000.log.${id}
 
 #./testpio_bench.pl --maxiter 2 --iofmt pnc --pecount 8000 --bench POPD --numIO 640 --log ${host}.8000.pnc.iotask_640.log.${id}
-
diff --git a/examples/basic/README.testpio b/examples/basic/README.testpio
index db914e3c77..1e2f0cc907 100644
--- a/examples/basic/README.testpio
+++ b/examples/basic/README.testpio
@@ -1,7 +1,7 @@
 
 TESTPIO README
 
-Testpio tests both the accuracy and performance of reading and writing data 
+Testpio tests both the accuracy and performance of reading and writing data
 using the pio library.  The tests are controlled via namelist.  There are a
 set of general namelist and then namelist to setup a computational
 decomposition and an IO decomposition.  The computational decomposition
@@ -21,34 +21,34 @@ block, io_nml, contains some general settings:
     nx_global      - integer, global size of "x" dimension
     ny_global      - integer, global size of "y" dimension
     nz_global      - integer, glboal size of "z" dimension
-    ioFMT          - string, type and i/o method of data file 
+    ioFMT          - string, type and i/o method of data file
                      ("bin","pnc","snc"), binary, pnetcdf, or serial netcdf
-    rearr          - string, type of rearranging to be done 
+    rearr          - string, type of rearranging to be done
                      ("none","mct","box","boxauto")
     nprocsIO       - integer, number of IO processors used only when rearr is
                      not "none", if rearr is "none", then the IO decomposition
                      will be the computational decomposition
     base           - integer, base pe associated with nprocIO striding
     stride         - integer, the stride of io pes across the global pe set
-    num_aggregator - integer, mpi-io number of aggregators, only used if no 
+    num_aggregator - integer, mpi-io number of aggregators, only used if no
                      pio rearranging is done
-    dir            - string, directory to write output data, this must exist 
+    dir            - string, directory to write output data, this must exist
                      before the model starts up
     num_iodofs     - tests either 1dof or 2dof init decomp interfaces (1,2)
     maxiter        - integer, the number of trials for the test
     DebugLevel     - integer, sets the debug level (0,1,2,3)
     compdof_input  - string, setting of the compDOF ('namelist' or a filename)
-    compdof_output - string, whether the compDOF is saved to disk 
+    compdof_output - string, whether the compDOF is saved to disk
                      ('none' or a filename)
 
 Notes:
   - the "mct" rearr option is not currently available
-  - if rearr is set to "none", then the computational decomposition is also 
+  - if rearr is set to "none", then the computational decomposition is also
     going to be used as the IO decomposition.  The computation decomposition
     must therefore be suited to the underlying I/O methods.
   - if rearr is set to "box", then pio is going to generate an internal
     IO decomposition automatically and pio will rearrange to that decomp.
-  - num_aggregator is used with mpi-io and no pio rearranging.  mpi-io is only 
+  - num_aggregator is used with mpi-io and no pio rearranging.  mpi-io is only
     used with binary data.
   - nprocsIO, base, and stride implementation has some special options
     if nprocsIO >  0 and stride >  0, then use input values
@@ -66,7 +66,7 @@ blocks are identical in use.
                  increasing this increases the flexibility of decompositions.
     grdorder   - string, sets the gridcell ordering within the block
                  ("xyz","xzy","yxz","yzx","zxy","zyx")
-    grddecomp  - string, sets up the block size with gdx, gdy, and gdz, see 
+    grddecomp  - string, sets up the block size with gdx, gdy, and gdz, see
                  below, ("x","y","z","xy","xye","xz","xze","yz","yze",
                  "xyz","xyze","setblk")
     gdx        - integer, "x" size of block
@@ -89,7 +89,7 @@ are provided below.
 
 Testpio writes out several files including summary information to
 stdout, data files to the namelist dir directory, and a netcdf
-file summarizing the decompositions.  The key output information 
+file summarizing the decompositions.  The key output information
 is stdout, which contains the timing information.  In addition,
 a netcdf file called gdecomp.nc is written that provides both the
 block and task ids for each gridcell as computed by the decompositions.
@@ -110,7 +110,7 @@ option to testpio_run.pl
 
 There are several testpio_in files for the pio test suite.  The ones that
 come with pio test specific things.  In general, there are tests for
- sn = serial netcdf and no rearrangement 
+ sn = serial netcdf and no rearrangement
  sb = serial netcdf and box rearrangement
  pn = parallel netcdf and no rearrangement
  pb = parallel netcdf and box rearrangement
@@ -121,7 +121,7 @@ and the test number (01, etc) is consistent across I/O methods with
  02 = simple 2d xy decomp across all pes with all pes active in I/O
  03 = all data on root pe, all pes active in I/O
  04 = simple 2d xy decomp with yxz ordering and stride=4 pes active in I/O
- 05 = 2d xy decomp with 4 blocks/pe, yxz ordering, xy block decomp, and 
+ 05 = 2d xy decomp with 4 blocks/pe, yxz ordering, xy block decomp, and
       stride=4 pes active in I/O
  06 = 3d xy decomp with 4 blocks/pe, yxz ordering, xy block decomp, and
       stride=4 pes active in I/O
@@ -146,24 +146,24 @@ DECOMPOSITION:
 
 The decomposition implementation supports the decomposition of
 a general 3 dimensional "nx * ny * nz" grid into multiple blocks
-of gridcells which are then ordered and assigned to processors.  
-In general, blocks in the decomposition are rectangular, 
-"gdx * gdy * gdz" and the same size, although some blocks around 
-the edges of the domain may be smaller if the decomposition is uneven.  
-Both gridcells within the block and blocks within the domain can be 
+of gridcells which are then ordered and assigned to processors.
+In general, blocks in the decomposition are rectangular,
+"gdx * gdy * gdz" and the same size, although some blocks around
+the edges of the domain may be smaller if the decomposition is uneven.
+Both gridcells within the block and blocks within the domain can be
 ordered in any of the possible dimension hierarchies, such as "xyz"
-where the first dimension is the fastest.  
+where the first dimension is the fastest.
 
-The gdx, gdy, and gdz inputs allow the user to specify the size in 
-any dimension and the grddecomp input specifies which dimensions are 
-to be further optimized.  In general, automatic decomposition generation 
-of 3 dimensional grids can be done in any of possible combination of 
+The gdx, gdy, and gdz inputs allow the user to specify the size in
+any dimension and the grddecomp input specifies which dimensions are
+to be further optimized.  In general, automatic decomposition generation
+of 3 dimensional grids can be done in any of possible combination of
 dimensions, (x, y, z, xy, xz, yz, or xyz), with the other dimensions having a
 fixed block size.  The automatic generation of the decomposition is
 based upon an internal algorithm that tries to determine the most
 "square" blocks with an additional constraint on minimizing the maximum
 number of gridcells across processors.  If evenly divided grids are
-desired, use of the "e" addition to grddecomp specifies that the grid 
+desired, use of the "e" addition to grddecomp specifies that the grid
 decomposition must be evenly divided.  the setblk option uses the
 prescibed gdx, gdy, and gdz inputs withtout further automation.
 
@@ -172,7 +172,7 @@ in mapping blocks to processors, but has a few additional options.
 "cont1d" (contiguous 1d) basically unwraps the blocks in the order specified
 by the blkorder input and then decomposes that "1d" list of blocks
 onto processors by contiguously grouping blocks together and allocating
-them to a processor.  The number of contiguous blocks that are 
+them to a processor.  The number of contiguous blocks that are
 allocated to a processor is the maximum of the values of bdx, bdy, and
 bdz inputs.  Contiguous blocks are allocated to each processor in turn
 in a round robin fashion until all blocks are allocated.  The
@@ -181,13 +181,13 @@ contiguous blocks are set automatically such that each processor
 recieves only 1 set of contiguous blocks.  The ysym2 and ysym4
 blkdecomp2 options modify the original block layout such that
 the tasks assigned to the blocks are 2-way or 4-way symetric
-in the y axis. 
+in the y axis.
 
 The decomposition tool is extremely flexible, but arbitrary
 inputs will not always yield valid decompositions.  If a valid
 decomposition cannot be computed based on the global grid size,
-number of pes, number of blocks desired, and decomposition options, 
-the model will stop.  
+number of pes, number of blocks desired, and decomposition options,
+the model will stop.
 
 As indicated above, the IO decomposition must be suited to the
 IO methods, so decompositions are even further limited by those
@@ -212,7 +212,7 @@ Some decomposition examples:
 
 Standard xyz ordering, 2d decomp:
                note: blkdecomp plays no role since there is 1 block per pe
- nx_global  6       
+ nx_global  6
  ny_global  4
  nz_global  1           ______________________________
  npes       4          |B3  P3        |B4  P4         |
@@ -231,7 +231,7 @@ Standard xyz ordering, 2d decomp:
 
 Same as above but yxz ordering, 2d decomp
                note: blkdecomp plays no role since there is 1 block per pe
- nx_global  6       
+ nx_global  6
  ny_global  4
  nz_global  1           _____________________________
  npes       4          |B2  P2        |B4  P4        |
@@ -248,10 +248,10 @@ Same as above but yxz ordering, 2d decomp
  bdy        0          |______________|______________|
  bdz        0
 
-xyz grid ordering, 1d x decomp 
+xyz grid ordering, 1d x decomp
                note: blkdecomp plays no role since there is 1 block per pe
                note: blkorder plays no role since it's a 1d decomp
- nx_global  8       
+ nx_global  8
  ny_global  4
  nz_global  1           _____________________________________
  npes       4          |B1  P1  |B2  P2   |B3  P3  |B4  P4   |
@@ -269,7 +269,7 @@ xyz grid ordering, 1d x decomp
  bdz        0
 
 yxz block ordering, 2d grid decomp, 2d block decomp, 4 block per pe
- nx_global  8       
+ nx_global  8
  ny_global  4
  nz_global  1           _____________________________________
  npes       4          |B4  P2  |B8  P2   |B12  P4 |B16  P4  |
@@ -285,4 +285,3 @@ yxz block ordering, 2d grid decomp, 2d block decomp, 4 block per pe
  bdx        0          | 1   2  | 1   2   |        |         |
  bdy        0          |________|_________|________|_________|
  bdz        0
-
diff --git a/examples/basic/alloc_mod.F90.in b/examples/basic/alloc_mod.F90.in
index f70259d6ef..7843c7f69f 100644
--- a/examples/basic/alloc_mod.F90.in
+++ b/examples/basic/alloc_mod.F90.in
@@ -1,6 +1,6 @@
 #define __PIO_FILE__ "alloc_mod.F90.in"
 !>
-!! @file 
+!! @file
 !! $Revision$
 !! $LastChangedDate$
 !! @brief Internal allocation routines for PIO
@@ -15,18 +15,18 @@ module alloc_mod
 
 !>
 !! @private
-!! PIO internal memory allocation check routines.  
+!! PIO internal memory allocation check routines.
 !<
   public:: alloc_check
 !>
 !! @private
-!! PIO internal memory allocation check routines.  
+!! PIO internal memory allocation check routines.
 !<
-  public:: dealloc_check 
+  public:: dealloc_check
 
   interface alloc_check
      ! TYPE long,int,real,double ! DIMS 1,2
-     module procedure alloc_check_{DIMS}d_{TYPE} 
+     module procedure alloc_check_{DIMS}d_{TYPE}
      ! TYPE double,long,int,real
      module procedure alloc_check_0d_{TYPE}
  end interface
@@ -42,19 +42,19 @@ module alloc_mod
 
 !>
 !! @private
-!! PIO internal memory allocation check routines.  
+!! PIO internal memory allocation check routines.
 !<
   public :: alloc_print_usage
 
 !>
 !! @private
-!! PIO internal memory allocation check routines.  
+!! PIO internal memory allocation check routines.
 !<
   public :: alloc_trace_on
 
 !>
 !! @private
-!! PIO internal memory allocation check routines.  
+!! PIO internal memory allocation check routines.
 !<
   public :: alloc_trace_off
 
@@ -66,7 +66,7 @@ contains
   ! Instantiate all the variations of alloc_check_ and dealloc_check_
   !
 
-  ! TYPE long,int,real,double 
+  ! TYPE long,int,real,double
   subroutine alloc_check_1d_{TYPE} (data,varlen,msg)
 
     {VTYPE}, pointer :: data(:)
@@ -102,7 +102,7 @@ contains
 
   end subroutine alloc_check_1d_{TYPE}
 
-  ! TYPE long,int,real,double 
+  ! TYPE long,int,real,double
   subroutine alloc_check_2d_{TYPE} (data,size1, size2,msg)
 
     {VTYPE}, pointer :: data(:,:)
@@ -214,7 +214,7 @@ end subroutine dealloc_check_0d_{TYPE}
 !>
 !! @private
 !! @fn alloc_print_usage
-!! PIO internal memory allocation check routines.  
+!! PIO internal memory allocation check routines.
 !<
   subroutine alloc_print_usage(rank,msg)
 #ifndef NO_MPIMOD
diff --git a/examples/basic/build_defaults.xml b/examples/basic/build_defaults.xml
index 0bea13fd8e..0bf3766e18 100644
--- a/examples/basic/build_defaults.xml
+++ b/examples/basic/build_defaults.xml
@@ -31,7 +31,7 @@
 #BSUB -J testpio_suite
 #BSUB -W 3:00
 "
- />  
+ />
 
 <!--  PNETCDF_PATH="/glade/u/home/jedwards/pnetcdf/svn1163/intel/" -->
 
@@ -65,7 +65,7 @@
 #BSUB -J testpio_suite
 #BSUB -W 1:00
 '
- />  
+ />
 
 <yellowstone_pgi
   PNETCDF_PATH="${PNETCDF}"
@@ -96,7 +96,7 @@
 #BSUB -J testpio_suite
 #BSUB -W 1:00
 '
- />  
+ />
 <yellowstone_gnu
   PNETCDF_PATH="${PNETCDF}"
   NETCDF_PATH="${NETCDF}"
@@ -126,7 +126,7 @@
 #BSUB -J testpio_suite
 #BSUB -W 1:00
 '
- />  
+ />
 
 
 <janus
@@ -155,7 +155,7 @@
 #PBS -q janus-short
 #PBS -V
 "
- />  
+ />
 
 <carver
   PNETCDF_PATH="/global/homes/j/jedwards/pnetcdf/carver/intel/"
@@ -182,15 +182,15 @@
 #PBS -q debug
 #PBS -V
 "
- />  
+ />
 
 
 
 <intrepid
   ENV_IBMCMP_INCLUDE="/soft/apps/ibmcmp-jan2013/vac/bg/9.0/include:/soft/apps/ibmcmp-jan2010/vacpp/bg/9.0/include:/soft/apps/ibmcmp-jan2010/xlf/bg/11.1/include:/soft/apps/ibmcmp-jan2010/xlmass/bg/4.4/include:/soft/apps/ibmcmp-jan2010/xlsmp/bg/1.7/include"
   ADDENV_PATH="/soft/apps/darshan/bin/:/soft/apps/ibmcmp-jan2013/xlf/bg/11.1/bin/:/soft/apps/ibmcmp-jan2010/vac/bg/9.0/bin/"
-  NETCDF_PATH="/soft/apps/current/netcdf-4.1.3-disable_netcdf_4" 
-  PNETCDF_PATH="/home/robl/soft/pnetcdf-1.3.0pre1-xl" 
+  NETCDF_PATH="/soft/apps/current/netcdf-4.1.3-disable_netcdf_4"
+  PNETCDF_PATH="/home/robl/soft/pnetcdf-1.3.0pre1-xl"
   MPIFC = "/software/common/apps/misc-scripts/tmpixlf90"
   MPICC = "/software/common/apps/misc-scripts/tmpixlc"
   FC = "/software/common/apps/misc-scripts/tmpixlf90"
@@ -216,7 +216,7 @@
   MPICC = "mpicc"
   FC = "gfortran"
   CC = "gcc"
-  NETCDF_PATH="" 
+  NETCDF_PATH=""
   FFLAGS = "-g --free-line-length-none "
   submit="qsub"
   run="mpirun"
@@ -230,7 +230,7 @@
 #PBS -V
 "
   testsuites = "mpiio"
-/>  
+/>
 
 <cyberstar
   ADDENV_PATH = ""
@@ -240,7 +240,7 @@
   MPICC = "mpicc"
   FC = "pgf90"
   CC = "gcc"
-  NETCDF_PATH="" 
+  NETCDF_PATH=""
   FFLAGS = "-g"
   submit="qsub"
   run="mpirun"
@@ -254,7 +254,7 @@
 #PBS -V
 "
   testsuites = "mpiio"
-/>  
+/>
 
 
 <mirage1
@@ -263,7 +263,7 @@
   MPICC = "icc"
   FC = "ifort"
   CC = "icc"
-  NETCDF_PATH="/fs/local/apps/netcdf-3.6.2/" 
+  NETCDF_PATH="/fs/local/apps/netcdf-3.6.2/"
   FFLAGS = "-g "
   submit=""
   run=""
@@ -272,14 +272,14 @@
   corespernode="1"
   preamble = ""
   testsuites = "snet "
-/>  
+/>
 
 <titan
   MPIFC = "ftn "
   MPICC = "cc "
   FC = "pgf90"
   CC = "pgcc"
-  CPPDEFS = " -DFORTRANUNDERSCORE"  
+  CPPDEFS = " -DFORTRANUNDERSCORE"
   ENV_NETCDF_PATH="$NETCDF_DIR"
   ENV_PNETCDF_PATH="$PARALLEL_NETCDF_DIR"
   workdir = "/tmp/work/${USER}/testpio"
@@ -322,8 +322,8 @@
 <kraken
   MPIFC = "ftn -L$LIBSCI_BASE_DIR/pgi/lib"
   MPICC  = "cc -L$LIBSCI_BASE_DIR/pgi/lib"
-  FC    = "pgf90" 
-  CC     = "pgcc" 
+  FC    = "pgf90"
+  CC     = "pgcc"
   FFLAGS = "-D_USE_FLOW_CONTROL"
   NETCDF_PATH = "$NETCDF_PATH"
   PNETCDF_PATH = "$PNETCDF_DIR"
@@ -343,8 +343,8 @@
 <columbia
   MPIFC = "mpif90"
   MPICC  = "mpicc"
-  FC    = "ifort" 
-  CC     = "icc" 
+  FC    = "ifort"
+  CC     = "icc"
   FFLAGS = "-D_USE_FLOW_CONTROL"
   NETCDF_PATH = "$NETCDF_DIR"
   PNETCDF_PATH = "$PNETCDF_DIR"
@@ -412,7 +412,7 @@
   NETCDF_PATH = "${NETCDF}"
   LDLIBS="`$NETCDF/bin/nc-config --flibs` -lssl -lcrypto -ldl -lz -L/contrib/libidn/1.19/gnu/lib -lidn -L/opt/cray/pmi/1.0-1.0000.8160.39.1.ss/lib64 -lpmi"
   workdir = "/ptmp/${USER}/testpio.intel"
-  run = "aprun" 
+  run = "aprun"
   corespernode = "12"
   submit = "qsub "
   conopts = "   --enable-filesystem-hints=lustre "
diff --git a/examples/basic/check_mod.F90 b/examples/basic/check_mod.F90
index fe0d7d4c4d..b9291bb555 100644
--- a/examples/basic/check_mod.F90
+++ b/examples/basic/check_mod.F90
@@ -1,5 +1,5 @@
 module check_mod
-  
+
   use kinds_mod
   use pio_types, only : PIO_NOERR  ! _EXTERNAL
   use alloc_mod  ! _EXTERNAL
@@ -8,11 +8,11 @@ module check_mod
   use mpi !_EXTERNAL
 #endif
   implicit none
-  private 
+  private
 #ifdef NO_MPIMOD
   include 'mpif.h'    ! _EXTERNAL
 #endif
-  public :: checkpattern 
+  public :: checkpattern
 
   interface checkpattern
       module procedure check_1D_r8, &
@@ -23,7 +23,7 @@ module check_mod
 		       check_3D_i4
   end interface
 
-contains 
+contains
 
 subroutine check_1D_r8(my_comm, fname,wr_array,rd_array,len,iostat)
     integer, intent(in) :: my_comm
@@ -38,7 +38,7 @@ subroutine check_1D_r8(my_comm, fname,wr_array,rd_array,len,iostat)
     integer(i4) :: ierr,cbad,rank, maxbadloc(1)
 
     if(present(iostat)) iostat = PIO_noerr
-   
+
     call alloc_check(diff,len,' check_1D_r8:diff ')
 
     if(len>0) then
@@ -62,7 +62,7 @@ subroutine check_1D_r8(my_comm, fname,wr_array,rd_array,len,iostat)
             wr_array(maxbadloc), rd_array(maxbadloc)
        if(present(iostat)) iostat = -20
     endif
-    call dealloc_check(diff)	
+    call dealloc_check(diff)
 end subroutine check_1D_r8
 
 subroutine check_3D_r8(my_comm, fname,wr_array,rd_array)
@@ -76,17 +76,17 @@ subroutine check_3D_r8(my_comm, fname,wr_array,rd_array)
     real(r8) :: lsum,gsum
     integer(i4) :: ierr,cbad,rank
     integer(i4) :: len1,len2,len3
-   
+
     len1 = SIZE(wr_array,dim=1)
     len2 = SIZE(wr_array,dim=2)
     len3 = SIZE(wr_array,dim=3)
-   
+
     allocate(diff(len1,len2,len3))
-  
+
     diff = wr_array - rd_array
     cbad = COUNT(diff .ne. 0.0)
     lsum = SUM(diff)
-   
+
     call MPI_Allreduce(lsum,gsum,1,MPI_REAL8,MPI_SUM,MY_COMM,ierr)
     call CheckMPIReturn('Call to MPI_Allreduce()',ierr,__FILE__,__LINE__)
 
@@ -96,7 +96,7 @@ subroutine check_3D_r8(my_comm, fname,wr_array,rd_array)
        if(lsum .ne. 0.0) print *,'IAM: ', rank, 'File: ',TRIM(fname),&
             ' Error detected for correctness test(3D,R8): ',lsum,' # bad: ',cbad
     endif
-    deallocate(diff)	
+    deallocate(diff)
 
 end subroutine check_3D_r8
 
@@ -111,17 +111,17 @@ subroutine check_3D_r4(my_comm, fname,wr_array,rd_array)
     real(r4) :: lsum,gsum
     integer(i4) :: ierr,cbad,rank
     integer(i4) :: len1,len2,len3
-   
+
     len1 = SIZE(wr_array,dim=1)
     len2 = SIZE(wr_array,dim=2)
     len3 = SIZE(wr_array,dim=3)
-   
+
     allocate(diff(len1,len2,len3))
-  
+
     diff = wr_array - rd_array
     cbad = COUNT(diff .ne. 0.0)
     lsum = SUM(diff)
-   
+
     call MPI_Allreduce(lsum,gsum,1,MPI_REAL,MPI_SUM,MY_COMM,ierr)
     call CheckMPIReturn('Call to MPI_Allreduce()',ierr,__FILE__,__LINE__)
 
@@ -131,7 +131,7 @@ subroutine check_3D_r4(my_comm, fname,wr_array,rd_array)
        if(lsum .ne. 0) print *,'IAM: ', rank, 'File: ',TRIM(fname),&
             ' Error detected for correctness test(3D,R4): ',lsum,' # bad: ',cbad
     endif
-    deallocate(diff)	
+    deallocate(diff)
 
 end subroutine check_3D_r4
 
@@ -146,17 +146,17 @@ subroutine check_3D_i4(my_comm, fname,wr_array,rd_array)
     integer(i4) :: lsum,gsum
     integer(i4) :: ierr,cbad,rank
     integer(i4) :: len1,len2,len3
-   
+
     len1 = SIZE(wr_array,dim=1)
     len2 = SIZE(wr_array,dim=2)
     len3 = SIZE(wr_array,dim=3)
-   
+
     allocate(diff(len1,len2,len3))
-  
+
     diff = wr_array - rd_array
     cbad = COUNT(diff .ne. 0.0)
     lsum = SUM(diff)
-   
+
     call MPI_Allreduce(lsum,gsum,1,MPI_INTEGER,MPI_SUM,MY_COMM,ierr)
     call CheckMPIReturn('Call to MPI_Allreduce()',ierr,__FILE__,__LINE__)
     if(gsum .ne. 0.0) then
@@ -165,13 +165,13 @@ subroutine check_3D_i4(my_comm, fname,wr_array,rd_array)
        if(lsum .ne. 0) print *,'IAM: ', rank, 'File: ',TRIM(fname),&
             ' Error detected for correctness test(3D,I4): ',lsum,' # bad: ',cbad
     endif
-    deallocate(diff)	
+    deallocate(diff)
 
 end subroutine check_3D_i4
 
 subroutine check_1D_r4(my_comm,fname,wr_array,rd_array,len,iostat)
     integer, intent(in) :: my_comm
- 
+
     character(len=*) :: fname
     real(r4) :: wr_array(:)
     real(r4) :: rd_array(:)
@@ -181,11 +181,11 @@ subroutine check_1D_r4(my_comm,fname,wr_array,rd_array,len,iostat)
     real(r4) :: lsum,gsum
     integer(i4) :: ierr,len,cbad,rank
 
-    
+
 
 ! Set default (no error) value for iostat if present)
     if(present(iostat)) iostat = PIO_noerr
-   
+
     call alloc_check(diff,len,' check_1D_r4:diff ')
 
     if(len>0) then
@@ -195,7 +195,7 @@ subroutine check_1D_r4(my_comm,fname,wr_array,rd_array,len,iostat)
     else
        lsum = 0
     end if
-    
+
     call MPI_Allreduce(lsum,gsum,1,MPI_REAL,MPI_SUM,MY_COMM,ierr)
     call CheckMPIReturn('Call to MPI_Allreduce()',ierr,__FILE__,__LINE__)
     if(abs(gsum) > tiny(gsum)) then
@@ -205,7 +205,7 @@ subroutine check_1D_r4(my_comm,fname,wr_array,rd_array,len,iostat)
             ' Error detected for correctness test(1D,R4): ',lsum,' # bad: ',cbad
        if(present(iostat)) iostat = -20
     endif
-    deallocate(diff)	
+    deallocate(diff)
 
 end subroutine check_1D_r4
 
@@ -221,11 +221,11 @@ subroutine check_1D_i4(my_comm, fname,wr_array,rd_array,len,iostat)
     integer(i4) :: lsum,gsum
     integer(i4) :: ierr,cbad,rank, lloc(1)
 
-    
+
 
 ! Set default (no error) value for iostat if present)
     if(present(iostat)) iostat = PIO_noerr
-   
+
     call alloc_check(diff,len,' check_1D_r4:diff ')
     if(len>0) then
        diff = wr_array - rd_array
@@ -245,7 +245,7 @@ subroutine check_1D_i4(my_comm, fname,wr_array,rd_array,len,iostat)
             lloc, wr_array(lloc(1)), rd_array(lloc(1))
        if(present(iostat)) iostat = -20
     endif
-    deallocate(diff)	
+    deallocate(diff)
 
 end subroutine check_1D_i4
 
diff --git a/examples/basic/config_bench.xml b/examples/basic/config_bench.xml
index 0f8beaf2ea..78e0beb10f 100644
--- a/examples/basic/config_bench.xml
+++ b/examples/basic/config_bench.xml
@@ -75,39 +75,39 @@
 
 <!--              CUBE4K   -->
 <CompConfig nprocs="512" bench_name="CUBE2K"
-   <ldx>256</ldx> <ldy>256</ldy> <ldz>256</ldz> 
+   <ldx>256</ldx> <ldy>256</ldy> <ldz>256</ldz>
 </CompConfig>
 
 <!--   POPB benchmark -->
 <CompConfig nprocs="24" bench_name="POPB"
-   <ldx>80</ldx> <ldy>48</ldy> <ldz>60</ldz> 
+   <ldx>80</ldx> <ldy>48</ldy> <ldz>60</ldz>
 </CompConfig>
 
 <CompConfig nprocs="80" bench_name="POPB"
-   <ldx>32</ldx> <ldy>48</ldy> <ldz>60</ldz> 
+   <ldx>32</ldx> <ldy>48</ldy> <ldz>60</ldz>
 </CompConfig>
 
 <CompConfig nprocs="160" bench_name="POPB"
-   <ldx>32</ldx> <ldy>24</ldy> <ldz>60</ldz> 
+   <ldx>32</ldx> <ldy>24</ldy> <ldz>60</ldz>
 </CompConfig>
 
 <CompConfig nprocs="320" bench_name="POPB"
-   <ldx>16</ldx> <ldy>24</ldy> <ldz>60</ldz> 
+   <ldx>16</ldx> <ldy>24</ldy> <ldz>60</ldz>
 </CompConfig>
 
 <CompConfig nprocs="640" bench_name="POPB"
-   <ldx>16</ldx> <ldy>12</ldy> <ldz>60</ldz> 
+   <ldx>16</ldx> <ldy>12</ldy> <ldz>60</ldz>
 </CompConfig>
 
 
-<!--               WRFB benchmark                                  --> 
+<!--               WRFB benchmark                                  -->
 
 <CompConfig nprocs="125" bench_name="WRFB"
-   <ldx>90</ldx> <ldy>72</ldy> <ldz>100</ldz> 
+   <ldx>90</ldx> <ldy>72</ldy> <ldz>100</ldz>
 </CompConfig>
 
 <CompConfig nprocs="250" bench_name="WRFB"
-   <ldx>90</ldx> <ldy>36</ldy> <ldz>100</ldz> 
+   <ldx>90</ldx> <ldy>36</ldy> <ldz>100</ldz>
 </CompConfig>
 
 <CompConfig nprocs="450" bench_name="WRFB"
@@ -115,23 +115,23 @@
 </CompConfig>
 
 <CompConfig nprocs="500" bench_name="WRFB"
-   <ldx>45</ldx> <ldy>36</ldy> <ldz>100</ldz> 
+   <ldx>45</ldx> <ldy>36</ldy> <ldz>100</ldz>
 </CompConfig>
 
 <CompConfig nprocs="900" bench_name="WRFB"
-   <ldx>30</ldx> <ldy>30</ldy> <ldz>100</ldz> 
+   <ldx>30</ldx> <ldy>30</ldy> <ldz>100</ldz>
 </CompConfig>
 
 <CompConfig nprocs="1000" bench_name="WRFB"
-   <ldx>45</ldx> <ldy>18</ldy> <ldz>100</ldz> 
+   <ldx>45</ldx> <ldy>18</ldy> <ldz>100</ldz>
 </CompConfig>
 
 <CompConfig nprocs="2025" bench_name="WRFB"
-   <ldx>20</ldx> <ldy>40</ldy> <ldz>100</ldz> 
+   <ldx>20</ldx> <ldy>40</ldy> <ldz>100</ldz>
 </CompConfig>
 
 <CompConfig nprocs="4050" bench_name="WRFB"
-   <ldx>20</ldx> <ldy>20</ldy> <ldz>100</ldz> 
+   <ldx>20</ldx> <ldy>20</ldy> <ldz>100</ldz>
 </CompConfig>
 
 <CompConfig nprocs="8100" bench_name="WRFB"
@@ -142,7 +142,7 @@
    <ldx>10</ldx> <ldy>10</ldy> <ldz>100</ldz>
 </CompConfig>
 
-<!--               CAM05 benchmark                                  --> 
+<!--               CAM05 benchmark                                  -->
 <CompConfig nprocs="64" bench_name="CAM05"
    <ldx>576</ldx> <ldy>6</ldy> <ldz>26</ldz>
 </CompConfig>
@@ -312,4 +312,3 @@
 
 
 </Config>
-
diff --git a/examples/basic/fdepends.awk b/examples/basic/fdepends.awk
index 03bab9769d..2980920cf2 100644
--- a/examples/basic/fdepends.awk
+++ b/examples/basic/fdepends.awk
@@ -19,9 +19,9 @@ BEGIN { IGNORECASE=1
 
 
 #
-# awk reads each line of the filename argument $2 until it finds 
+# awk reads each line of the filename argument $2 until it finds
 # a "use" or "#include"
-# 
+#
 
 
 /^[ \t]*use[ \t]+/ {
@@ -30,7 +30,7 @@ BEGIN { IGNORECASE=1
         if ( $0 ~ /_EXTERNAL/ ) next
 
         # Assume the second field is the F90 module name,
-        # remove any comma at the end of the second field (due to 
+        # remove any comma at the end of the second field (due to
         # ONLY or rename), and print it in a dependency line.
 
         sub(/,$/,"",$2)
@@ -49,8 +49,8 @@ BEGIN { IGNORECASE=1
          if ( $0 ~ /_EXTERNAL/ ) next
 
          # Remove starting or ending quote or angle bracket
-         sub(/^["<']/,"",$2) 
-         sub(/[">']$/,"",$2) 
+         sub(/^["<']/,"",$2)
+         sub(/[">']$/,"",$2)
          print PRLINE $2
- 
+
       }
diff --git a/examples/basic/gdecomp_mod.F90 b/examples/basic/gdecomp_mod.F90
index 7480188138..d848931d4a 100644
--- a/examples/basic/gdecomp_mod.F90
+++ b/examples/basic/gdecomp_mod.F90
@@ -32,7 +32,7 @@ module gdecomp_mod
       character(len=128):: nml_file   ! namelist filename if used
       character(len=16) :: nml_var    ! namelist variable if used
    end type
-      
+
    character(len=*),parameter :: modname = 'gdecomp_mod'
    integer(i4),parameter :: main_task = 0
 
@@ -51,7 +51,7 @@ subroutine gdecomp_set(gdecomp,nxg,nyg,nzg,gdx,gdy,gdz,bdx,bdy,bdz, &
    type(gdecomp_type), intent(inout) :: gdecomp
 
 !  NOTE: not all of these are optional, but optional allows
-!        them to be called in arbitrary order 
+!        them to be called in arbitrary order
 
    integer(i4),optional :: nxg,nyg,nzg      ! global grid size
    integer(i4),optional :: gdx,gdy,gdz      ! block size
@@ -175,7 +175,7 @@ subroutine gdecomp_set(gdecomp,nxg,nyg,nzg,gdx,gdy,gdz,bdx,bdy,bdz, &
    endif
 
    end subroutine gdecomp_set
-   
+
 !==================================================================
    subroutine gdecomp_read_nml(gdecomp,nml_file,nml_var,my_task,ntasks,gdims)
 
@@ -264,7 +264,7 @@ subroutine gdecomp_read_nml(gdecomp,nml_file,nml_var,my_task,ntasks,gdims)
    endif
 
    end subroutine gdecomp_read_nml
-   
+
 !==================================================================
 
    subroutine gdecomp_print(gdecomp)
@@ -384,7 +384,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test)
 !DBG   print *,'IAM: ',my_task,'gdecomp_DOF: point #3 gsiz:',gsiz
 !DBG   print *,'IAM: ',my_task,'gdecomp_DOF: point #3 bsiz:',bsiz
 
-   if(wdecomp) then 
+   if(wdecomp) then
      allocate(blkid(gsiz(1),gsiz(2),gsiz(3)))
      allocate(tskid(gsiz(1),gsiz(2),gsiz(3)))
      blkid = -1
@@ -562,7 +562,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test)
 !     ii = (n3-1)*gsiz(2)*gsiz(1) + (n2-1)*gsiz(1) + n1
       nbxyz = ((n3-1)/bsiz(3))*nblk(2)*nblk(1) + ((n2-1)/bsiz(2))*nblk(1) + &
               ((n1-1)/bsiz(1)) + 1
-      if(wdecomp) then 
+      if(wdecomp) then
           blkid(n1,n2,n3) = bxyzbord(nbxyz)
           tskid(n1,n2,n3) = bxyzpord(nbxyz)
       endif
@@ -579,7 +579,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test)
    cntmax = maxval(cnta)
 
    ! --- map gridcells to dof ---
-   
+
    if (testonly) then
       allocate(testdof(cntmax,0:gnpes-1))
       testdof = 0
@@ -667,7 +667,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test)
          write(6,*) trim(subname),' start and count could NOT be computed '
    endif
 
-!------- main TASK WRITE ------------------------------------- 
+!------- main TASK WRITE -------------------------------------
 
    if (my_task == main_task) then
 
@@ -717,7 +717,7 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test)
    if (first_call) then
       rcode = nf90_create(ncname,nf90_clobber,ncid)
    else
-      rcode = nf90_open(ncname,nf90_write,ncid)      
+      rcode = nf90_open(ncname,nf90_write,ncid)
    endif
    rcode = nf90_redef(ncid)
    dname = trim(gdecomp%nml_var)//'_nx'
@@ -738,9 +738,9 @@ subroutine gdecomp_DOF(gdecomp,my_task,DOF,start,count,write_decomp,test)
 
    endif   ! testonly
 
-!------- END main TASK WRITE --------------------------------- 
+!------- END main TASK WRITE ---------------------------------
 
-   if(wdecomp) then 
+   if(wdecomp) then
      deallocate(blkid,tskid)
    endif
    deallocate(cnta,cntb,bxyzbord,bxyzpord,bordpord)
@@ -944,7 +944,7 @@ subroutine calcbsiz(npes,gsiz,bsiz,option,ierr)
             npes2 = npes2/m
             bs = bs - 1
          else
-            write(6,*) trim(subname),' ERROR: bsiz not allowed ',n,gsiz(n),bsiz(n),m,npes,npes2 
+            write(6,*) trim(subname),' ERROR: bsiz not allowed ',n,gsiz(n),bsiz(n),m,npes,npes2
             call piodie(__FILE__,__LINE__)
          endif
       endif
@@ -1112,7 +1112,7 @@ end subroutine piodie
   subroutine mpas_decomp_generator(dim1,dim2,dim3,my_task,fname,dof)
     integer :: dim1, dim2, dim3
     integer, intent(in)          :: my_task         ! my MPI rank
-    character(len=*),intent(in)  :: fname           ! name of MPAS partition file 
+    character(len=*),intent(in)  :: fname           ! name of MPAS partition file
     integer(kind=pio_offset_kind), pointer             :: dof(:)
 
 !  Local variables
@@ -1132,7 +1132,7 @@ subroutine mpas_decomp_generator(dim1,dim2,dim3,my_task,fname,dof)
     !   1st dimension:	vertical
     !   2nd dimension:  horizontal
 
-    gnz = dim1     
+    gnz = dim1
     nCellsGlobal = dim2*dim3
     call get_global_id_list(my_task,fname,nCellsSolve,nCellsGlobal,globalIDList)
 
@@ -1197,7 +1197,7 @@ end subroutine get_global_id_list
 
   subroutine camlike_decomp_generator(gnx, gny, gnz, myid, ntasks, npr_yz, dof)
     integer, intent(in) :: gnx, gny, gnz, myid, ntasks, npr_yz(4)
-    integer(kind=pio_offset_kind), pointer :: dof(:), tdof(:), tchk(:) 
+    integer(kind=pio_offset_kind), pointer :: dof(:), tdof(:), tchk(:)
     real, pointer :: rdof(:)
     integer(kind=pio_offset_kind) :: dofsize,tdofsize
 
@@ -1280,7 +1280,7 @@ subroutine camlike_decomp_generator(gnx, gny, gnz, myid, ntasks, npr_yz, dof)
        end do
     end do
 
-    CALL qsRecursive(1_PIO_OFFSET_KIND, dofsize, dof) !kicks off the recursive 
+    CALL qsRecursive(1_PIO_OFFSET_KIND, dofsize, dof) !kicks off the recursive
 
     deallocate(tdof)
 
@@ -1323,7 +1323,7 @@ integer(kind=pio_offset_kind) FUNCTION qsPartition (loin, hiin, list)
           hi = hi - 1
        END DO
        IF (hi /= lo) then !move the entry indexed by hi to left side of partition
-          list(lo) = list(hi) 
+          list(lo) = list(hi)
           lo = lo + 1
        END IF
        DO !move in from the left
@@ -1331,7 +1331,7 @@ integer(kind=pio_offset_kind) FUNCTION qsPartition (loin, hiin, list)
           lo = lo + 1
        END DO
        IF (hi /= lo) then !move the entry indexed by hi to left side of partition
-          list(hi) = list(lo) 
+          list(hi) = list(lo)
           hi = hi - 1
        END IF
     END DO
diff --git a/examples/basic/namelist_mod.F90 b/examples/basic/namelist_mod.F90
index fa237a7198..1dbd343cb2 100644
--- a/examples/basic/namelist_mod.F90
+++ b/examples/basic/namelist_mod.F90
@@ -12,7 +12,7 @@ module namelist_mod
 
     use pio_support, only : piodie, CheckMPIReturn ! _EXTERNAL
     use pio, only : pio_offset_kind
-    implicit none    
+    implicit none
     private
 
     public :: broadcast_namelist
@@ -21,7 +21,7 @@ module namelist_mod
     integer(kind=i4), public, parameter :: buffer_size_str_len = 20
     integer(kind=i4), public, parameter :: true_false_str_len = 6
     integer(kind=i4), public, parameter :: romio_str_len = 10
-    
+
     logical, public, save :: async
     integer(i4), public, save :: nx_global,ny_global,nz_global
     integer(i4), public, save :: rearr_type
@@ -49,9 +49,9 @@ module namelist_mod
 
     integer(kind=i4), public, save :: set_lustre_values = 0 !! Set to one for true
     integer(kind=i4), public, save :: lfs_ost_count = 1
-    
+
     character(len=80), save, public :: compdof_input
-    character(len=80), save, public :: iodof_input 
+    character(len=80), save, public :: iodof_input
     character(len=80), save, public :: compdof_output
     character(len=256), save, public :: part_input
     character(len=256), save, public :: casename
@@ -125,7 +125,7 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror)
     character(len=*), parameter :: myname_=myname//'ReadPIO_Namelist'
 
     !-------------------------------------------------
-    ! set default values for namelist io_nml variables 
+    ! set default values for namelist io_nml variables
     !-------------------------------------------------
 
     async = .false.
@@ -175,14 +175,14 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror)
 
     open (device, file=filename,status='old',iostat=ierror)
 
-    if(ierror /= 0) then 
+    if(ierror /= 0) then
        write(*,*) caller,'->',myname_,':: Error opening file ',filename, &
             ' on device ',device,' with iostat=',ierror
        ierror = -1
     else
        ierror =  1
     endif
-    
+
     do while (ierror > 0)
        read(device, nml=io_nml, iostat=ierror)
     enddo
@@ -318,7 +318,7 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror)
           stride = (nprocs-base)/num_iotasks
        endif
     elseif (nprocsIO <= 0) then
-#ifdef BGx 
+#ifdef BGx
        ! A negative value for num_iotasks has a special meaning on Blue Gene
        num_iotasks = nprocsIO
 #else
@@ -333,7 +333,7 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror)
     endif
 
     !------------------------------------------------
-    ! reset stride if there are not enough processors 
+    ! reset stride if there are not enough processors
     !------------------------------------------------
     if (base + num_iotasks * (stride-1) > nprocs-1) then
        stride = FLOOR(real((nprocs - 1 - base),kind=r8)/real(num_iotasks,kind=r8))
@@ -342,9 +342,9 @@ subroutine ReadTestPIO_Namelist(device, nprocs, filename, caller, ierror)
     !-------------------------------------------------------
     ! If rearrangement is 'none' reset to the proper values
     !-------------------------------------------------------
-    if(trim(rearr) == 'none') then  
+    if(trim(rearr) == 'none') then
         stride = 1
-        num_iotasks = nprocs	
+        num_iotasks = nprocs
     endif
 
     write(*,*) trim(string),' n_iotasks  = ',num_iotasks,'  (updated)'
@@ -381,7 +381,7 @@ subroutine Broadcast_Namelist(caller, myID, root, comm, ierror)
   integer(i4) :: itmp
 
   !------------------------------------------
-  ! broadcast namelist info to all processors 
+  ! broadcast namelist info to all processors
   !------------------------------------------
 
   if(async) then
diff --git a/examples/basic/perl5lib/ChangeLog b/examples/basic/perl5lib/ChangeLog
index d5bfab8368..a5bbed9f87 100644
--- a/examples/basic/perl5lib/ChangeLog
+++ b/examples/basic/perl5lib/ChangeLog
@@ -6,7 +6,7 @@ Originator(s):  erik
 Date: Sat Jun 13, 2009
 One-line Summary: Add %ymd indicator for streams so can do year-month-days
 
-M      Streams/Template.pm ---- Add ability to write out %ymd year-month-day 
+M      Streams/Template.pm ---- Add ability to write out %ymd year-month-day
                                 for filenames in streams. It assumes a noleap
                                 calendar -- could easily be extended to make
                                 Gregorian optional.
@@ -14,7 +14,7 @@ M      t/01.t ---- Change formatting of successful test
 M      t/02.t ---- Add more tests for %ymd, and offset
 M      t/03.t ---- Change formatting of successful test
 M      t/04.t ---- Change formatting of successful test
-M      t/datm.streams.txt ---------- Add another year and the last-month 
+M      t/datm.streams.txt ---------- Add another year and the last-month
                                      to start for testing
 A      t/datm.ymd.streams.txt ------ Add streams test file with %ymd
 M      t/datm.template.streams.xml - Add CPLHIST test section with %ymd
@@ -27,7 +27,7 @@ Date: Tue Jun 9, 2009
 One-line Summary: add offset support for streams template
 
 M      Streams/Template.pm
-	
+
 ==============================================================
 Tag name:  perl5lib_090424
 Originator(s):  erik
@@ -79,7 +79,7 @@ Build/Namelist.pm
 . Change validate_variable_value() from an object method to a class method,
   and remove the unused argument.
 . add fix to _split_namelist_value method to replace embedded newlines by
-  spaces. 
+  spaces.
 
 Build/NamelistDefaults.pm
 . make the method interfaces case insensitive by converting all variable
@@ -146,7 +146,7 @@ Originator(s):  erik (KLUZEK ERIK 1326 CGD)
 Date: Mon Aug 11 10:44:52 MDT 2008
 One-line Summary: Turn off printing of file existance if NOT -verbose
 
-M      Streams/Template.pm ----------- Turn off printing of file 
+M      Streams/Template.pm ----------- Turn off printing of file
                 checking if NOT $printing;
 
 ==============================================================
@@ -190,8 +190,8 @@ about needing to do validation as is done now. Change the validate methods a bit
 and make them more robust.
 
 M  Build/Config.pm --------------- Add get_valid_values method and use it internally.
-M  Build/NamelistDefinition.pm --- Add namelist validate_variable_value to validate 
-                                   method. Add option to return without quotes to 
+M  Build/NamelistDefinition.pm --- Add namelist validate_variable_value to validate
+                                   method. Add option to return without quotes to
                                    get_valid_values method.
 M  Build/Namelist.pm ------------- Make validate_variable_value more robust.
 
diff --git a/examples/basic/perl5lib/XML/Changes b/examples/basic/perl5lib/XML/Changes
index d0be5104f7..43f2860475 100644
--- a/examples/basic/perl5lib/XML/Changes
+++ b/examples/basic/perl5lib/XML/Changes
@@ -24,4 +24,3 @@ Revision history for Perl extension XML::Lite.
 0.01  Sat Aug 25 13:31:48 2001
 	- original version; created by h2xs 1.20 with options
 		-XA -n XML::Lite
-
diff --git a/examples/basic/perl5lib/XML/Lite.pm b/examples/basic/perl5lib/XML/Lite.pm
index d6aa32e978..5ace82b52e 100644
--- a/examples/basic/perl5lib/XML/Lite.pm
+++ b/examples/basic/perl5lib/XML/Lite.pm
@@ -35,12 +35,12 @@ my $xml = new XML::Lite( xml => 'a_file.xml' );
 
 =head1 DESCRIPTION
 
-XML::Lite is a lightweight XML parser, with basic element traversing 
-methods. It is entirely self-contained, pure Perl (i.e. I<not> based on 
-expat). It provides useful methods for reading most XML files, including 
-traversing and finding elements, reading attributes and such. It is 
-designed to take advantage of Perl-isms (Attribute lists are returned as 
-hashes, rather than, say, lists of objects). It provides only methods 
+XML::Lite is a lightweight XML parser, with basic element traversing
+methods. It is entirely self-contained, pure Perl (i.e. I<not> based on
+expat). It provides useful methods for reading most XML files, including
+traversing and finding elements, reading attributes and such. It is
+designed to take advantage of Perl-isms (Attribute lists are returned as
+hashes, rather than, say, lists of objects). It provides only methods
 for reading a file, currently.
 
 =head1 METHODS
@@ -50,7 +50,7 @@ The following methods are available:
 =over 4
 
 =cut
- 
+
 use XML::Lite::Element;
 BEGIN {
 	use vars       qw( $VERSION @ISA );
@@ -75,17 +75,17 @@ use vars      qw( %ERRORS );
 =item my $xml = new XML::Lite( xml => $source[, ...] );
 
 Creates a new XML::Lite object. The XML::Lite object acts as the document
-object for the $source that is sent to it to parse. This means that you 
-create a new object for each document (or document sub-section). As the 
+object for the $source that is sent to it to parse. This means that you
+create a new object for each document (or document sub-section). As the
 objects are lightweight this should not be a performance consideration.
 
 The object constructor can take several named parameters. Parameter names
-may begin with a '-' (as in the example above) but are not required to. The 
+may begin with a '-' (as in the example above) but are not required to. The
 following parameters are recognized.
 
-  xml      The source XML to parse. This can be a filename, a scalar that 
+  xml      The source XML to parse. This can be a filename, a scalar that
            contains the document (or document fragment), or an IO handle.
-  
+
 
 As a convenince, if only on parameter is given, it is assumed to be the source.
 So you can use this, if you wish:
@@ -99,7 +99,7 @@ sub new {
 	my $proto = shift;
 	my %parms;
 	my $class = ref($proto) || $proto;
-	
+
 	# Parse parameters
 	$self->{settings} = {};
 	if( @_ > 1 ) {
@@ -109,7 +109,7 @@ sub new {
 		while( ($k, $v) = each %parms ) {
 			$k =~ s/^-//;		# Removed leading '-' if it exists. (Why do Perl programmers use this?)
 			$self->{settings}{$k} = $v;
-		} # end while 
+		} # end while
 	} else {
 		$self->{settings}{xml} = $_[0];
 	} # end if;
@@ -121,10 +121,10 @@ sub new {
 	$self->{doc} = '';
 	$self->{_CDATA} = [];
 	$self->{handlers} = {};
-	
+
 	# Refer to global error messages
 	$self->{ERRORS} = $self->{settings}{error_messages} || \%ERRORS;
-	
+
 	# Now parse the XML document and build look-up tables
 	return undef unless $self->_parse_it();
 
@@ -181,8 +181,8 @@ sub root_element {
 Returns a list of all elements that match C<$name>.
 C<@list> is a list of L<XML::Lite::Element> objects
 If called in a scalar context, this will return the
-first element found that matches (it's more efficient 
-to call in a scalar context than assign the results 
+first element found that matches (it's more efficient
+to call in a scalar context than assign the results
 to a list of one scalar).
 
 If no matching elements are found then returns C<undef>
@@ -201,7 +201,7 @@ sub element_by_name;
 sub elements_by_name {
 	my $self = shift;
 	my( $name ) = @_;
-	
+
 	if( wantarray ) {
 		my @list = ();
 		foreach( @{$self->{elements}{$name}} ) {
@@ -241,7 +241,7 @@ sub elements_by_name {
 # ----------------------------------------------------------
 sub _parse_it {
 	my $self = shift;
-	
+
 	# Get the xml content
 	if( $self->{settings}{xml} =~ /^\s*</ ) {
 		$self->{doc} = $self->{settings}{xml};
@@ -268,26 +268,26 @@ sub _parse_it {
 		$self->{doc_offset} = length $1;		# Store the number of removed chars for messages
 	} # end if
 	$self->{doc} =~ s/\s+$//;
-	
-	
+
+
 	# Build lookup tables
 	$self->{elements} = {};
 	$self->{tree} = [];
 	# - These are used in the building process
 	my $element_list = [];
 	my $current_element = $self->{tree};
-	
+
 	# Call init handler if defined
 	&{$self->{handlers}{init}}($self) if defined $self->{handlers}{init};
-	
+
 	# Make a table of offsets to each element start and end point
 	# Table is a hash of element names to lists of offsets:
 	# [start_tag_start, start_tag_end, end_tag_start, end_tag_end]
 	# where tags include the '<' and '>'
-	
-	# Also make a tree of linked lists. List contains root element 
+
+	# Also make a tree of linked lists. List contains root element
 	# and other nodes. Each node consits of a list ref (the position list)
-	# and a following list containing the child element. Text nodes are 
+	# and a following list containing the child element. Text nodes are
 	# a list ref (with just two positions).
 
 	# Find the opening and closing of the XML, giving errors if not well-formed
@@ -297,22 +297,22 @@ sub _parse_it {
 	$self->_error( 'ROOT_NOT_CLOSED', $start_pos + $self->{doc_offset} ) if $end_pos == -1;
 	my $doc_end = rindex( $self->{doc}, '>' );
 	$self->_error( 'ROOT_NOT_CLOSED' ) if $doc_end == -1;
-	
+
 	# Now walk through the document, one tag at a time, building up our
 	# lookup tables
 	while( $end_pos <= $doc_end ) {
-		
+
 		# Get a tag
 		my $tag = substr( $self->{doc}, $start_pos, $end_pos - $start_pos + 1 );
 
 		# Get the tag name and see if it's an end tag (starts with </)
 		my( $end, $name ) = $tag =~ m{^<\s*(/?)\s*([^/>\s]+)};
-		
+
 		if( $end ) {
 			# If there is no start tag for this end tag then throw an error
 			$self->_error( 'NO_START', $start_pos + $self->{doc_offset}, $tag ) unless defined $self->{elements}{$name};
-			
-			# Otherwise, add the end point to the array for the last element in 
+
+			# Otherwise, add the end point to the array for the last element in
 			# the by-name lookup hash
 			my( $x, $found ) = (@{$self->{elements}{$name}} - 1, 0);
 			while( $x >= 0 ) {
@@ -329,24 +329,24 @@ sub _parse_it {
 
 			# If we didn't find an open element then throw an error
 			$self->_error( 'NO_START', $start_pos + $self->{doc_offset}, $tag ) unless $found;
-			
+
 			# Call an end-tag handler if defined (not yet exposed)
 			&{$self->{handlers}{end}}($self, $name) if defined $self->{handlers}{end};
-			
+
 			# Close element in linked list (tree)
 			$current_element = pop @$element_list;
-			
+
 		} else {
-			# Make a new list in the by-name lookup hash if none found by this name yet 
+			# Make a new list in the by-name lookup hash if none found by this name yet
 			$self->{elements}{$name} = [] unless defined $self->{elements}{$name};
-			
+
 			# Add start points to the array of positions and push it on the hash
 			my $pos_list = [$start_pos, $end_pos];
 			push @{$self->{elements}{$name}}, $pos_list;
-			
+
 			# Call start-tag handler if defined (not yet exposed)
 			&{$self->{handlers}{start}}($self, $name) if defined $self->{handlers}{start};
-			
+
 			# If this is a single-tag element (e.g. <.../>) then close it immediately
 			if( $tag =~ m{/\s*>$} ) {
 				push @$current_element, $pos_list;
@@ -364,7 +364,7 @@ sub _parse_it {
 			} # end if
 
 		} # end if
-		
+
 		# Move the start pointer to beginning of next element
 		$start_pos = index( $self->{doc}, '<', $start_pos + 1 );
 		last if $start_pos == -1 || $end_pos == $doc_end;
@@ -372,16 +372,16 @@ sub _parse_it {
 		# Now $end_pos is end of old tag and $start_pos is start of new
 		# So do things on the data between the tags as needed
 		if( $start_pos - $end_pos > 1 ) {
-			# Call any character data handler 
+			# Call any character data handler
 			&{$self->{handlers}{char}}($self, substr($self->{doc}, $end_pos + 1, $start_pos - $end_pos - 1)) if defined $self->{handlers}{char};
 
 			# Inserting the text into the linked list as well
 #			push @$current_element, [$end_pos + 1, $start_pos - 1];
 		} # end if
-		
+
 		# Now finish by incrementing the parser to the next element
 		$end_pos = index( $self->{doc}, '>', $start_pos + 1 );
-		
+
 		# If there is no next element, and we're not at the end of the document,
 		# then throw an error
 		$self->_error( 'ELM_NOT_CLOSED', $start_pos + $self->{doc_offset} ) if $end_pos == -1;
@@ -401,7 +401,7 @@ sub _parse_it {
 #
 # Returns: Scalar content of $file, undef on error
 #
-# Description: Reads from $file and returns the content. 
+# Description: Reads from $file and returns the content.
 # $file may be either a filename or an IO handle
 # ----------------------------------------------------------
 # Date      Modification                              Author
@@ -412,7 +412,7 @@ sub _get_a_file {
 	my $self = shift;
 	my $file = shift;
 	my $content = undef;
-	
+
 	# If it's a ref and a handle, then read that
 	if( ref($file) ) {
 		$content = join '', <$file>;
@@ -422,12 +422,12 @@ sub _get_a_file {
 		open( XML, $file ) || return undef;
 		$content = join '', <XML>;
 		close XML || return undef;
-	} 
+	}
 	# Don't know how to handle this type of parameter
 	else {
 		return undef;
 	} # end if
-	
+
 	return $content;
 } # end _get_a_file
 
@@ -448,10 +448,10 @@ sub _error {
 	my $self = shift;
 	my( $code, @args ) = @_;
 	my $msg = $self->{ERRORS}{$code};
-	
+
 	# Handle replacement codes
 	$msg =~ s/\%(\d+)/$args[$1]/g;
-	
+
 	# Throw exception
 	die ref($self) . ":$msg\n";
 } # end _error
@@ -462,7 +462,7 @@ sub _error {
 #
 # Args: $content
 #
-# Returns: A reference to the CDATA element, padded to 
+# Returns: A reference to the CDATA element, padded to
 # original size.
 #
 # Description: Stores the CDATA element in the internal
@@ -498,13 +498,13 @@ sub _store_cdata {
 sub _dump_tree {
 	my $self = shift;
 	my $node = shift || $self->{tree};
-	
+
 	my $tree = '';
 	for( my $i = 0; $i < scalar(@$node) && defined $node->[$i]; $i++ ) {
 		if( (scalar(@{$node->[$i]}) == 4) && (defined $node->[$i][2]) ) {
 			$tree .= '[' . join( ',', @{$node->[$i]} ) . "] "
-					. substr($self->{doc}, $node->[$i][0], $node->[$i][1] - $node->[$i][0] + 1) 
-					. "..." 
+					. substr($self->{doc}, $node->[$i][0], $node->[$i][1] - $node->[$i][0] + 1)
+					. "..."
 					. substr($self->{doc}, $node->[$i][2], $node->[$i][3] - $node->[$i][2] + 1) . " (child $i)\n";
 			# Do child list
 			$i++;
@@ -530,7 +530,7 @@ END { }
 =head1 BUGS
 
 Lots. This 'parser' (Matt Sergeant takes umbrance to my us of that word) will handle some XML
-documents, but not all. 
+documents, but not all.
 
 =head1 VERSION
 
@@ -547,4 +547,3 @@ This library is free software; you can redistribute it and/or
 modify it under the same terms as Perl itself.
 
 =cut
-
diff --git a/examples/basic/perl5lib/XML/Lite/Element.pm b/examples/basic/perl5lib/XML/Lite/Element.pm
index 388511d89a..05debd50e2 100644
--- a/examples/basic/perl5lib/XML/Lite/Element.pm
+++ b/examples/basic/perl5lib/XML/Lite/Element.pm
@@ -33,18 +33,18 @@ print $elm->get_attribute( 'attribute_name' );
 
 =head1 DESCRIPTION
 
-C<XML::Lite::Element> objects contain rudimentary methods for querying XML 
-elements in an XML document as parsed by XML::Lite. Usually these objects 
+C<XML::Lite::Element> objects contain rudimentary methods for querying XML
+elements in an XML document as parsed by XML::Lite. Usually these objects
 are returned by method calls in XML::Lite.
 
 =head1 METHODS
 
-The following methods are available. All methods like 'get_name' can be 
+The following methods are available. All methods like 'get_name' can be
 abbeviated as 'name.'
 
 =over 4
 
-=cut 
+=cut
 
 use strict;
 BEGIN {
@@ -63,8 +63,8 @@ use vars      qw();
 
 Creates a new XML::Lite::Element object from the XML::Lite object, C<$owner_document>.
 
-Currently, you must not call this manually. You can create an object with one of 
-the 'factory' methods in XML::Lite, such as C<element_by_name> or C<root_element> 
+Currently, you must not call this manually. You can create an object with one of
+the 'factory' methods in XML::Lite, such as C<element_by_name> or C<root_element>
 or with one of the XML::Lite::Element 'factory' methods below, like C<get_children>.
 
 =cut
@@ -77,15 +77,15 @@ sub new {
 	# The arguments are as follows:
 	#   $owner_document   is an XML::Lite object within which this element lives
 	#   \@pointers        is a two or four element array ref containing the offsets
-	#                     into the original document of the start and end points of 
+	#                     into the original document of the start and end points of
 	#                     the opening and closing (when it exists) tags for the element
-	
+
 	# Validate arguments
 	return undef unless @_ >= 2;
 	return undef unless ref($_[0]) && (ref($_[1]) eq 'ARRAY');
-	
+
 	# Load 'em up
-	
+
 	# The data structure for the ::Element object has these properties
 	#   doc               A reference to the containing XML::Lite object
 	#   node              A reference to an array of pointers to our element in the document
@@ -94,11 +94,11 @@ sub new {
 	#   name              The name on our tag
 	#   _attrs            A string of the attibutes in our tag (unparsed)
 	#  attrs              A hash ref of attributes in our tag
-	
+
 	$self->{doc} = $_[0];
 	$self->{node} = $_[1];
-	
-	# Using the pointers, find out tag name, and attribute list from the 
+
+	# Using the pointers, find out tag name, and attribute list from the
 	# opening tag (if there are any attributes).
 	my $tag = substr( $self->{doc}{doc}, $self->{node}[0], $self->{node}[1] - $self->{node}[0] + 1 );
 	if( $tag =~ m{^<\s*([^/>\s]+)\s+([^>]+)\s*/?\s*>$} ) {
@@ -111,7 +111,7 @@ sub new {
 		# Should have been caught in the parsing! maybe an assert?
 		$self->{doc}->_error( 'ELM_NOT_CLOSED', $self->{node}[0] + $self->{doc}->{doc_offset} );
 	} # end if
-	
+
 	# Good. Now returns it.
 	bless ($self, $class);
 	return $self;
@@ -142,16 +142,16 @@ sub content;
 sub get_content {
 	my $self = shift;
 
-	# If we don't have any content, then we should return 
+	# If we don't have any content, then we should return
 	# '' right away.
 	return '' unless defined $self->{node}[2];
-	
+
 	# Using our pointers, find everything between our tags
 	my $content = substr( $self->{doc}{doc}, $self->{node}[1] + 1, $self->{node}[2] - $self->{node}[1] - 1 );
-	
+
 	# Now, restore any CDATA chunks that may have been pulled out
 	$content =~ s/<!\[CDATA\[(\S+)\s*\]\]\/>/<![CDATA[$self->{doc}{_CDATA}[$1]]]>/g;
-	
+
 	# And return the content
 	return $content;
 } # end get_content
@@ -173,11 +173,11 @@ sub attributes;
 *attributes = \&get_attributes;
 sub get_attributes {
 	my $self = shift;
-	
+
 	# Parse the attribute string into a hash of name-value pairs
 	# unless we've already done that.
 	$self->_parse_attrs() unless defined $self->{attrs};
-	
+
 	# Just return a *copy* of the hash (this is read-only after all!)
        if ( defined($self->{attrs}) ) {
                 return %{$self->{attrs}};
@@ -202,10 +202,10 @@ sub attribute;
 sub get_attribute {
 	my $self = shift;
 	my( $name ) = @_;
-	
+
 	# If we haven't parsed the attribute string into a hash, then do that.
 	$self->_parse_attrs() unless defined $self->{attrs};
-	
+
 	# Now return the requested attribute. If it's not there
 	# then 'undef' is returned
 	return $self->{attrs}{$name};
@@ -233,9 +233,9 @@ sub get_name {
 
 =item my @children = $element->get_children()
 
-Returns a list of XML::Lite::Element objects for each element contained 
-within the current element. This does not return any text or CDATA in 
-the content of this element. You can parse that through the L<get_content> 
+Returns a list of XML::Lite::Element objects for each element contained
+within the current element. This does not return any text or CDATA in
+the content of this element. You can parse that through the L<get_content>
 method.
 
 If no child elements exist then an empty list is returned.
@@ -256,7 +256,7 @@ sub get_children {
 	my $self = shift;
 	my @children = ();
 
-	# If we don't have any content, then we should return an emtpty 
+	# If we don't have any content, then we should return an emtpty
 	# list right away -- we have no children.
 	return @children unless defined $self->{node}[2];
 
@@ -264,8 +264,8 @@ sub get_children {
 	# This will also load {children} and {parent} as well
 	$self->_find_self() unless defined $self->{self};
 
-	# Now that we know who we are (if this didn't fail) we can 
-	# iterate through the sub nodes (our child list) and make 
+	# Now that we know who we are (if this didn't fail) we can
+	# iterate through the sub nodes (our child list) and make
 	# XML::Lite::Elements objects for each child
 	if( defined $self->{children} ) {
 		my $i = 0;
@@ -276,7 +276,7 @@ sub get_children {
 			$node = $self->{children}[++$i];
 		} # end while
 	} # end if
-	
+
 	return @children;
 } # end get_children
 
@@ -304,14 +304,14 @@ sub get_text {
 	my $self = shift;
 	my $content = '';
 
-	# If we don't have any content, then we should return  
+	# If we don't have any content, then we should return
 	# $content right away -- we have no text
 	return $content unless defined $self->{node}[2];
 
 	# Otherwise get out content and children
 	my @children = $self->get_children;
 	my $orig_content = $self->get_content;
-	
+
 	# Then remove the child elements from our content
 	my $start = 0;
 	foreach( @children ) {
@@ -320,10 +320,10 @@ sub get_text {
 		$start = ($_->{node}[3] || $_->{node}[1]) - $self->{node}[1];
 	} # end foreach
 	$content .= substr( $orig_content, $start ) if $start < length($orig_content);
-	
+
 	# Remove the CDATA wrapper, preserving the content
 	$content =~ s/<!\[CDATA\[(.+?)]\]>/$1/g;
-	
+
 	# Return the left-over text
 	return $content;
 } # end get_text
@@ -352,7 +352,7 @@ sub get_text {
 # ----------------------------------------------------------
 sub _parse_attrs {
 	my $self = shift;
-	
+
 	my $attrs = $self->{_attrs};
 	if ( defined($attrs) ) {
 		$attrs =~ s/^\s+//;
@@ -364,7 +364,7 @@ sub _parse_attrs {
 			$attrs =~ s/^\s+//;
 		} # end while
 	}
-	
+
 	return 1;
 } # end _parse_atttrs
 
@@ -376,7 +376,7 @@ sub _parse_attrs {
 # Returns: A reference to our node or undef on error
 #
 # Description: Traverses the owner document's tree to find
-# the node that references the current element. Sets 
+# the node that references the current element. Sets
 # $self-{self} as a side-effect. Even if this is already set,
 # _find_self will traverse again, so don't call unless needed.
 # ----------------------------------------------------------
@@ -387,8 +387,8 @@ sub _parse_attrs {
 # ----------------------------------------------------------
 sub _find_self {
 	my $self = shift;
-	
-	# We actually just call this recusively, so the first 
+
+	# We actually just call this recusively, so the first
 	# argument can be a starting point to descend from
 	# but we don't doc that above
 	my $node = shift || $self->{doc}{tree};
@@ -405,10 +405,10 @@ sub _find_self {
 		# If this is our self, then we're done!
 		# 	NOTE: Since the list references are the same in the by-name hash
 		# 	and tree objects, we can just do a reference compare here.
-		# 	If objects are ever created with non-factory methods then we need to 
+		# 	If objects are ever created with non-factory methods then we need to
 		# 	use a _compare_lists call.
-# 		if( _compare_lists( $node->[$i], $self->{node} ) ) { 
- 		if( $node->[$i] eq $self->{node} ) { 
+# 		if( _compare_lists( $node->[$i], $self->{node} ) ) {
+ 		if( $node->[$i] eq $self->{node} ) {
 			$self->{parent} = $node;
 			$self->{self} = $node->[$i];
 			# If this list has children, then add a pointer to that list
@@ -453,16 +453,16 @@ sub _find_self {
 # ----------------------------------------------------------
 sub _compare_lists {
 	my( $rA, $rB ) = @_;
-	
+
 	# Lists are not equal unless same size
 	return 0 unless scalar(@$rA) == scalar(@$rB);
-	
+
 	# Now compare item by item.
 	my $i;
 	for( $i = 0; $i < scalar(@$rA); $i++ ) {
 		return 0 unless $rA->[$i] eq $rB->[$i];
 	} # end for
-	
+
 	return 1;
 } # end _compare_lists
 
@@ -488,4 +488,3 @@ This library is free software; you can redistribute it and/or
 modify it under the same terms as Perl itself.
 
 =cut
-
diff --git a/examples/basic/perl5lib/XML/README b/examples/basic/perl5lib/XML/README
index 6234a760ce..6f9ebf8456 100644
--- a/examples/basic/perl5lib/XML/README
+++ b/examples/basic/perl5lib/XML/README
@@ -7,7 +7,7 @@ for most things you need to do with XML files.
 
 It is not dependent on any other modules or external programs for installation.
 
-NOTE that this parser will do many things that you want with XML but 
+NOTE that this parser will do many things that you want with XML but
 not everything. It is not a validating parser! It will not handle
 international characters (unless run on those systems). Use
 at your own risk.
@@ -16,5 +16,3 @@ Copyright 2001-2003 Wadsack-Allen. All rights reserved.
 
 This library is free software; you can redistribute it and/or
 modify it under the same terms as Perl itself.
-
-
diff --git a/examples/basic/perl5lib/XML/man3/XML_Lite.3 b/examples/basic/perl5lib/XML/man3/XML_Lite.3
index f2b3912ed7..f0adb0e00a 100644
--- a/examples/basic/perl5lib/XML/man3/XML_Lite.3
+++ b/examples/basic/perl5lib/XML/man3/XML_Lite.3
@@ -148,12 +148,12 @@ use \s-1XML:\s0:Lite;
 my \f(CW$xml\fR = new \s-1XML:\s0:Lite( xml => 'a_file.xml' );
 .SH "DESCRIPTION"
 .IX Header "DESCRIPTION"
-\&\s-1XML:\s0:Lite is a lightweight \s-1XML\s0 parser, with basic element traversing 
-methods. It is entirely self-contained, pure Perl (i.e. \fInot\fR based on 
-expat). It provides useful methods for reading most \s-1XML\s0 files, including 
-traversing and finding elements, reading attributes and such. It is 
-designed to take advantage of Perl-isms (Attribute lists are returned as 
-hashes, rather than, say, lists of objects). It provides only methods 
+\&\s-1XML:\s0:Lite is a lightweight \s-1XML\s0 parser, with basic element traversing
+methods. It is entirely self-contained, pure Perl (i.e. \fInot\fR based on
+expat). It provides useful methods for reading most \s-1XML\s0 files, including
+traversing and finding elements, reading attributes and such. It is
+designed to take advantage of Perl-isms (Attribute lists are returned as
+hashes, rather than, say, lists of objects). It provides only methods
 for reading a file, currently.
 .SH "METHODS"
 .IX Header "METHODS"
@@ -161,16 +161,16 @@ The following methods are available:
 .Ip "my \f(CW$xml\fR = new \s-1XML:\s0:Lite( xml => \f(CW$source\fR[, ...] );" 4
 .IX Item "my $xml = new XML::Lite( xml => $source[, ...] );"
 Creates a new \s-1XML:\s0:Lite object. The \s-1XML:\s0:Lite object acts as the document
-object for the \f(CW$source\fR that is sent to it to parse. This means that you 
-create a new object for each document (or document sub-section). As the 
+object for the \f(CW$source\fR that is sent to it to parse. This means that you
+create a new object for each document (or document sub-section). As the
 objects are lightweight this should not be a performance consideration.
 .Sp
 The object constructor can take several named parameters. Parameter names
-may begin with a '\-' (as in the example above) but are not required to. The 
+may begin with a '\-' (as in the example above) but are not required to. The
 following parameters are recognized.
 .Sp
 .Vb 2
-\&  xml      The source XML to parse. This can be a filename, a scalar that 
+\&  xml      The source XML to parse. This can be a filename, a scalar that
 \&           contains the document (or document fragment), or an IO handle.
 .Ve
 As a convenince, if only on parameter is given, it is assumed to be the source.
@@ -190,8 +190,8 @@ Returns \f(CW\*(C`undef\*(C'\fR on errors.
 Returns a list of all elements that match \f(CW\*(C`$name\*(C'\fR.
 \&\f(CW\*(C`@list\*(C'\fR is a list of the XML::Lite::Element manpage objects
 If called in a scalar context, this will return the
-first element found that matches (it's more efficient 
-to call in a scalar context than assign the results 
+first element found that matches (it's more efficient
+to call in a scalar context than assign the results
 to a list of one scalar).
 .Sp
 If no matching elements are found then returns \f(CW\*(C`undef\*(C'\fR
@@ -199,7 +199,7 @@ in scalar context or an empty list in array context.
 .SH "BUGS"
 .IX Header "BUGS"
 Lots. This 'parser' (Matt Sergeant takes umbrance to my us of that word) will handle some \s-1XML\s0
-documents, but not all. 
+documents, but not all.
 .SH "VERSION"
 .IX Header "VERSION"
 0.14
diff --git a/examples/basic/perl5lib/XML/man3/XML_Lite_Element.3 b/examples/basic/perl5lib/XML/man3/XML_Lite_Element.3
index 5eaf684214..f31d1336e4 100644
--- a/examples/basic/perl5lib/XML/man3/XML_Lite_Element.3
+++ b/examples/basic/perl5lib/XML/man3/XML_Lite_Element.3
@@ -151,19 +151,19 @@ my \f(CW$elm\fR = \f(CW$xml\fR->elements_by_name( 'element_name' );
 print \f(CW$elm\fR->get_attribute( 'attribute_name' );
 .SH "DESCRIPTION"
 .IX Header "DESCRIPTION"
-\&\f(CW\*(C`XML::Lite::Element\*(C'\fR objects contain rudimentary methods for querying \s-1XML\s0 
-elements in an \s-1XML\s0 document as parsed by \s-1XML:\s0:Lite. Usually these objects 
+\&\f(CW\*(C`XML::Lite::Element\*(C'\fR objects contain rudimentary methods for querying \s-1XML\s0
+elements in an \s-1XML\s0 document as parsed by \s-1XML:\s0:Lite. Usually these objects
 are returned by method calls in \s-1XML:\s0:Lite.
 .SH "METHODS"
 .IX Header "METHODS"
-The following methods are available. All methods like 'get_name' can be 
+The following methods are available. All methods like 'get_name' can be
 abbeviated as 'name.'
 .Ip "my \f(CW$element\fR = new \s-1XML:\s0:Lite::Element( \f(CW$owner_document\fR, \e@pointers );" 4
 .IX Item "my $element = new XML::Lite::Element( $owner_document, @pointers );"
 Creates a new \s-1XML:\s0:Lite::Element object from the \s-1XML:\s0:Lite object, \f(CW\*(C`$owner_document\*(C'\fR.
 .Sp
-Currently, you must not call this manually. You can create an object with one of 
-the 'factory' methods in \s-1XML:\s0:Lite, such as \f(CW\*(C`element_by_name\*(C'\fR or \f(CW\*(C`root_element\*(C'\fR 
+Currently, you must not call this manually. You can create an object with one of
+the 'factory' methods in \s-1XML:\s0:Lite, such as \f(CW\*(C`element_by_name\*(C'\fR or \f(CW\*(C`root_element\*(C'\fR
 or with one of the \s-1XML:\s0:Lite::Element 'factory' methods below, like \f(CW\*(C`get_children\*(C'\fR.
 .Ip "my \f(CW$content\fR = \f(CW$element\fR->\fIget_content()\fR" 4
 .IX Item "my $content = $element->get_content()"
@@ -180,9 +180,9 @@ Returns the value of the named attribute for this element.
 Returns the name of the element tag
 .Ip "my \f(CW@children\fR = \f(CW$element\fR->\fIget_children()\fR" 4
 .IX Item "my @children = $element->get_children()"
-Returns a list of \s-1XML:\s0:Lite::Element objects for each element contained 
-within the current element. This does not return any text or \s-1CDATA\s0 in 
-the content of this element. You can parse that through the the get_content manpage 
+Returns a list of \s-1XML:\s0:Lite::Element objects for each element contained
+within the current element. This does not return any text or \s-1CDATA\s0 in
+the content of this element. You can parse that through the the get_content manpage
 method.
 .Sp
 If no child elements exist then an empty list is returned.
diff --git a/examples/basic/testdecomp.F90 b/examples/basic/testdecomp.F90
index 9684e14a19..6ea015fd56 100644
--- a/examples/basic/testdecomp.F90
+++ b/examples/basic/testdecomp.F90
@@ -7,7 +7,7 @@ program testdecomp
    use gdecomp_mod
 
    implicit none
-   
+
    integer, pointer  :: compDOF(:), ioDOF(:)
    integer :: startcomp(3),cntcomp(3)
    integer :: startio(3),cntio(3),gdims(3)
@@ -23,7 +23,7 @@ program testdecomp
    num_tasks = 192
    gdims(1) = 3600
    gdims(2) = 2400
-   gdims(3) = 40 
+   gdims(3) = 40
 
 !   call gdecomp_read_nml(gdecomp,fin,'comp',my_task)
 !   print  *,'after gdecomp_read_nml'
diff --git a/examples/basic/testdecomp.bluefire.run b/examples/basic/testdecomp.bluefire.run
index d3f1ddbf79..371d6fc128 100644
--- a/examples/basic/testdecomp.bluefire.run
+++ b/examples/basic/testdecomp.bluefire.run
@@ -27,6 +27,3 @@ cp -f $srcdir/testdecomp_in ./testdecomp_in
 mpirun.lsf ./testdecomp >& testdecomp.out.$LID
 
 cp testdecomp.out.$LID $srcdir/
-
-
-
diff --git a/examples/basic/testpio.F90 b/examples/basic/testpio.F90
index 1fb4f1b4bb..5b4201f0b0 100644
--- a/examples/basic/testpio.F90
+++ b/examples/basic/testpio.F90
@@ -46,7 +46,7 @@ program testpio
   integer(i4) :: indx
   integer(i4) :: mode
 
-  integer(i4) :: ip,numPhases 
+  integer(i4) :: ip,numPhases
   character(len=*), parameter :: TestR8CaseName = 'r8_test'
   character(len=*), parameter :: TestR4CaseName = 'r4_test'
   character(len=*), parameter :: TestI4CaseName  = 'i4_test'
@@ -101,11 +101,11 @@ program testpio
   real(r8) :: dt_write_r8, dt_write_r4, dt_write_i4 ! individual write times
   real(r8) :: dt_read_r8, dt_read_r4, dt_read_i4 ! individual read times
   ! Arrays to hold globally reduced read/write times--one element per time trial
-  real(r8), dimension(:), pointer :: gdt_write_r8, gdt_write_r4, gdt_write_i4 
+  real(r8), dimension(:), pointer :: gdt_write_r8, gdt_write_r4, gdt_write_i4
   real(r8), dimension(:), pointer :: gdt_read_r8, gdt_read_r4, gdt_read_i4
 
   integer(i4) :: nprocs
-  integer(i4) :: lLength    ! local number of words in the computational decomposition 
+  integer(i4) :: lLength    ! local number of words in the computational decomposition
 
   integer(i4), parameter ::  nml_in = 10
   character(len=*), parameter :: nml_filename = 'testpio_in'
@@ -149,7 +149,7 @@ program testpio
 
   call MPI_INIT(ierr)
   call CheckMPIReturn('Call to MPI_INIT()',ierr,__FILE__,__LINE__)
-  
+
 
   !    call enable_abort_on_exit
 
@@ -200,7 +200,7 @@ program testpio
   endif
 #endif
 
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -212,7 +212,7 @@ program testpio
   !----------------------------------------------------------------
 
   if(Debug)    print *,'testpio: before call to readTestPIO_Namelist'
-  if(my_task == main_task) then 
+  if(my_task == main_task) then
      call ReadTestPIO_Namelist(nml_in, nprocs, nml_filename, myname, nml_error)
   endif
   if(Debug) print *,'testpio: before call to broadcast_namelist'
@@ -224,7 +224,7 @@ program testpio
   ! Checks (num_iotasks can be negative on BGx)
   !-------------------------------------
 
-#if !defined(BGx) 
+#if !defined(BGx)
   if (num_iotasks <= 0) then
      write(*,*) trim(myname),' ERROR: ioprocs invalid num_iotasks=',num_iotasks
      call piodie(__FILE__,__LINE__)
@@ -234,7 +234,7 @@ program testpio
   ! ----------------------------------------------------------------
   ! if stride is and num_iotasks is incompatible than reset stride (ignore stride on BGx)
   ! ----------------------------------------------------------------
-#if !defined(BGx) 
+#if !defined(BGx)
   if (base + num_iotasks * (stride-1) > nprocs-1) then
      write(*,*) trim(myname),' ERROR: num_iotasks, base and stride too large', &
           ' base=',base,' num_iotasks=',num_iotasks,' stride=',stride,' nprocs=',nprocs
@@ -243,7 +243,7 @@ program testpio
 #endif
 
   !--------------------------------------
-  ! Initalizes the parallel IO subsystem 
+  ! Initalizes the parallel IO subsystem
   !--------------------------------------
   call PIO_setDebugLevel(DebugLevel)
 
@@ -266,7 +266,7 @@ program testpio
      call MPI_COMM_SIZE(MPI_COMM_COMPUTE,nprocs,ierr)
 
   else
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -279,7 +279,7 @@ program testpio
      call PIO_init(my_task, MPI_COMM_COMPUTE, num_iotasks, num_aggregator, stride, &
           rearr_type, PIOSYS, base)
 
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -289,7 +289,7 @@ program testpio
 
   end if
 
-  
+
 
 
   if(Debug)    print *,'testpio: after call to PIO_init', nprocs,mpi_comm_io
@@ -339,7 +339,7 @@ program testpio
                           trim(ibm_io_sparse_access))
      end if
   end if
-!  if(set_lustre_values /= 0) then 
+!  if(set_lustre_values /= 0) then
 !        call PIO_setnum_OST(PIOSYS,lfs_ost_count)
 !  endif
 
@@ -355,11 +355,11 @@ program testpio
 
   if(index(casename,'CAM')==1) then
      call camlike_decomp_generator(gdims3d(1),gdims3d(2),gdims3d(3),my_task,nprocs,npr_yz,compDOF)
-  elseif(index(casename,'MPAS')==1) then 
+  elseif(index(casename,'MPAS')==1) then
 !     print *,'testpio: before call to mpas_decomp_generator: (',TRIM(part_input),') gdims3d: ',gdims3d
      call mpas_decomp_generator(gdims3d(1),gdims3d(2),gdims3d(3),my_task,part_input,compDOF)
   else  if (trim(compdof_input) == 'namelist') then
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -369,7 +369,7 @@ program testpio
      if(Debug)       print *,'iam: ',My_task,'testpio: point #1'
      call gdecomp_read_nml(gdecomp,nml_filename,'comp',my_task,nprocs,gDims3D(1:3))
 
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -380,7 +380,7 @@ program testpio
      if(Debug)       print *,'iam: ',My_task,'testpio: point #2'
      call gdecomp_DOF(gdecomp,My_task,compDOF,start,count)
      if(Debug)       print *,'iam: ',My_task,'testpio: point #3', minval(compdof),maxval(compdof)
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -413,7 +413,7 @@ program testpio
      endif
   endif
 
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -427,7 +427,7 @@ program testpio
      call pio_writedof(trim(compdof_output),gdims3d, compDOF,MPI_COMM_COMPUTE,75)
   endif
 
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -482,7 +482,7 @@ program testpio
   else
      call piodie(__FILE__,__LINE__,' rearr '//trim(rearr)//' not supported')
   endif
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -505,19 +505,19 @@ program testpio
   lLength = size(compDOF)
 
   !----------------------
-  ! allocate and set test arrays 
+  ! allocate and set test arrays
   !----------------------
 
-  if(TestR8 .or. TestCombo) then 
+  if(TestR8 .or. TestCombo) then
      call alloc_check(test_r8wr,lLength,'testpio:test_r8wr')
   endif
-  if(TestR4 .or. TestCombo) then 
+  if(TestR4 .or. TestCombo) then
      call alloc_check(test_r4wr,lLength,'testpio:test_r4wr' )
   endif
-  if(TestInt .or. TestCombo) then 
+  if(TestInt .or. TestCombo) then
      call alloc_check(test_i4wr,lLength,'testpio:test_i4wr')
   endif
-  if(TestInt) then 
+  if(TestInt) then
     call alloc_check(test_i4i ,lLength,'testpio:test_i4i ')
     call alloc_check(test_i4j ,lLength,'testpio:test_i4j ')
     call alloc_check(test_i4k ,lLength,'testpio:test_i4k ')
@@ -527,27 +527,27 @@ program testpio
 
   do n = 1,lLength
      call c1dto3d(compdof(n),gDims3D(1),gDims3D(2),gDims3D(3),i1,j1,k1)
-     if(TestInt) then 
+     if(TestInt) then
 	test_i4dof(n) = compdof(n)
         test_i4i(n) = i1
         test_i4j(n) = j1
         test_i4k(n) = k1
         test_i4m(n) = my_task
      endif
-     if(TestR8 .or. TestCombo) then 
+     if(TestR8 .or. TestCombo) then
 !         test_r8wr(n) = 10.0_r8*cos(20.*real(i1,kind=r8)/real(gDims3D(1),kind=r8))* &
 !          cos(10.*real(j1,kind=r8)/real(gDims3D(2),kind=r8))* &
 !          (1.0+1.0*real(j1,kind=r8)/real(gDims3D(2),kind=r8))* &
 !          cos(25.*real(k1,kind=r8)/real(gDims3D(3),kind=r8))
         test_r8wr = compdof
      endif
-     if(TestR4 .or. TestCombo) then 
+     if(TestR4 .or. TestCombo) then
          test_r4wr(n) = 10.0_r4*cos(20.*real(i1,kind=r4)/real(gDims3D(1),kind=r4))* &
           cos(10.*real(j1,kind=r4)/real(gDims3D(2),kind=r4))* &
           (1.0+1.0*real(j1,kind=r4)/real(gDims3D(2),kind=r4))* &
           cos(25.*real(k1,kind=r4)/real(gDims3D(3),kind=r4))
      endif
-     if(TestInt .or. TestCombo) then 
+     if(TestInt .or. TestCombo) then
         test_i4wr(n) = compdof(n)
 !         test_i4wr(n) = nint(10.0_r8*cos(20.*real(i1,kind=r8)/real(gDims3D(1),kind=r8))* &
 !          cos(10.*real(j1,kind=r8)/real(gDims3D(2),kind=r8))* &
@@ -558,7 +558,7 @@ program testpio
 
   if(Debug)       print *,'iam: ',My_task,'testpio: point #10'
 
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -581,7 +581,7 @@ program testpio
   !--------------------------------
   ! allocate arrays for holding globally-reduced timing information
   !--------------------------------
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -595,25 +595,25 @@ program testpio
   call alloc_check(gdt_write_i4, maxiter, ' testpio:gdt_write_i4 ')
   call alloc_check(gdt_read_i4, maxiter, ' testpio:gdt_read_i4 ')
   if(Debug)       print *,'iam: ',My_task,'testpio: point #11'
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
        print *,__FILE__,__LINE__,'mem=',rss,' it=',it
     end if
 #endif
-  if(splitPhase) then 
+  if(splitPhase) then
     numPhases = 2
   else
     numPhases = 1
   endif
 
   do ip=1,numPhases
-     if(numPhases == 1) then 
+     if(numPhases == 1) then
         readPhase = .true.
         writePhase = .true.
      else
-        if(ip == 1) then 
+        if(ip == 1) then
 	   writePhase = .true.
 	   readPhase = .false.
 	else
@@ -623,9 +623,9 @@ program testpio
      endif
      if(log_main_task) print *,'{write,read}Phase:  ',writePhase,readPhase
 
-     
+
      do it=1,maxiter
-#ifdef MEMCHK	
+#ifdef MEMCHK
     call GPTLget_memusage(msize, rss, mshare, mtext, mstack)
     if(rss>lastrss) then
        lastrss=rss
@@ -644,7 +644,7 @@ program testpio
                    IOdesc_r8,iostart=startpio,iocount=countpio)
               glenr8 = product(gdims3d)
               if(Debug)       print *,'iam: ',My_task,'testpio: point #7.1'
-              
+
               if(TestR4 .or. TestCombo) then
                  call PIO_initDecomp(PIOSYS,PIO_real,    gDims3D,compDOF,&
                       IOdesc_r4,iostart=startpio,iocount=countpio)
@@ -675,7 +675,7 @@ program testpio
               if(TestInt .or. TestCombo) then
 !                 print *,__FILE__,__LINE__,gdims3d
 !                 print *,__FILE__,__LINE__,compdof
-              
+
                  call PIO_initDecomp(PIOSYS,PIO_int,     gDims3D,compDOF,&
                       IOdesc_i4)
                  if(Debug)       print *,'iam: ',My_task,'testpio: point #8.4'
@@ -750,7 +750,7 @@ program testpio
            endif
         endif
         if(Debug)       print *,'iam: ',My_task,'testpio: point #9'
-        
+
         if(Debug) then
            write(*,'(a,2(a,i8))') myname,':: After call to initDecomp.  comp_rank=',my_task, &
                 ' io_rank=',iorank
@@ -758,10 +758,10 @@ program testpio
 
         call PIO_getnumiotasks(PIOSYS,num_iotasks)
         !------------
-        ! Open file{s} 
+        ! Open file{s}
         !------------
         write(citer,'(i3.3)') it
-        
+
         fname    = TRIM(dir)//'foo.'//citer//'.'//TRIM(Iofmtd)
         fname_r8 = TRIM(dir)//'foo.r8.'//citer//'.'//TRIM(Iofmtd)
         fname_r4 = TRIM(dir)//'foo.r4.'//citer//'.'//TRIM(Iofmtd)
@@ -772,7 +772,7 @@ program testpio
         !   print *, __FILE__,__LINE__,'>',fname_r4,'<'
         mode = pio_64bit_offset
 
-        if(writePhase) then 
+        if(writePhase) then
            if(TestCombo) then
               if(Debug) write(*,'(2a,i8)') myname,':: Combination Test:  Creating File...it=',it
               ierr = PIO_CreateFile(PIOSYS,File,iotype,trim(fname), mode)
@@ -797,21 +797,21 @@ program testpio
               call check_pioerr(ierr,__FILE__,__LINE__,' i4 createfile')
            endif
 
-           
+
            allocate(vard_r8(nvars), vard_r4(nvars))
-           
+
 
            !---------------------------
-           ! Code specifically for netCDF files 
+           ! Code specifically for netCDF files
            !---------------------------
-           if(iotype == iotype_pnetcdf .or. & 
+           if(iotype == iotype_pnetcdf .or. &
                 iotype == iotype_netcdf .or. &
                 iotype == PIO_iotype_netcdf4p .or. &
                 iotype == PIO_iotype_netcdf4c) then
 
-              if(TestR8) then 
+              if(TestR8) then
                  !-----------------------------------
-                 ! for the single record real*8 file 
+                 ! for the single record real*8 file
                  !-----------------------------------
                  call WriteHeader(File_r8,nx_global,ny_global,nz_global,dimid_x,dimid_y,dimid_z)
 
@@ -829,9 +829,9 @@ program testpio
                  call check_pioerr(iostat,__FILE__,__LINE__,' r8 enddef')
               endif
 
-              if(TestR4) then 
+              if(TestR4) then
                  !-----------------------------------
-                 ! for the single record real*4 file 
+                 ! for the single record real*4 file
                  !-----------------------------------
                  call WriteHeader(File_r4,nx_global,ny_global,nz_global,dimid_x,dimid_y,dimid_z)
                  iostat = PIO_def_dim(File_r4,'charlen',strlen,charlen)
@@ -846,14 +846,14 @@ program testpio
                  call check_pioerr(iostat,__FILE__,__LINE__,' i4 enddef')
               endif
 
-              if(TestInt) then 
+              if(TestInt) then
                  !-----------------------------------
-                 ! for the single record integer file 
+                 ! for the single record integer file
                  !-----------------------------------
                  call WriteHeader(File_i4,nx_global,ny_global,nz_global,dimid_x,dimid_y,dimid_z)
-                 iostat = PIO_def_var(File_i4,'fdof',PIO_int,(/dimid_x,dimid_y,dimid_z/),vard_i4dof)                
+                 iostat = PIO_def_var(File_i4,'fdof',PIO_int,(/dimid_x,dimid_y,dimid_z/),vard_i4dof)
                  call check_pioerr(iostat,__FILE__,__LINE__,' i4dof defvar')
-                 
+
                  iostat = PIO_def_var(File_i4,'field',PIO_int,(/dimid_x,dimid_y,dimid_z/),vard_i4)
                  call check_pioerr(iostat,__FILE__,__LINE__,' i4 defvar')
                  iostat = PIO_def_var(File_i4,'fi',PIO_int,(/dimid_x,dimid_y,dimid_z/),vard_i4i)
@@ -869,10 +869,10 @@ program testpio
                  iostat = PIO_enddef(File_i4)
                  call check_pioerr(iostat,__FILE__,__LINE__,' i4 enddef')
               endif
-        
-              if(TestCombo) then 
+
+              if(TestCombo) then
                  !-----------------------------------
-                 ! for the multi record file 
+                 ! for the multi record file
                  !-----------------------------------
                  call WriteHeader(File,nx_global,ny_global,nz_global,dimid_x,dimid_y,dimid_z)
                  iostat = PIO_def_var(File,'field_r8',PIO_double,(/dimid_x,dimid_y,dimid_z/),vard_r8c)
@@ -902,12 +902,12 @@ program testpio
            endif
 
            !-------------------------
-           ! Time the parallel write 
+           ! Time the parallel write
            !-------------------------
 
-	   
+
            dt_write_r8 = 0.
-	   
+
            if(TestR8) then
               if(iofmtd .ne. 'bin') then
                  iostat = pio_put_var(file_r8,varfn_r8,fname_r8)
@@ -962,7 +962,7 @@ program testpio
            endif
               if(Debug)       print *,'iam: ',My_task,'testpio: point #13'
 
-           if(TestInt) then 
+           if(TestInt) then
               dt_write_i4 = 0.
               call MPI_Barrier(MPI_COMM_COMPUTE,ierr)
               call CheckMPIReturn('Call to MPI_BARRIER()',ierr,__FILE__,__LINE__)
@@ -1001,10 +1001,10 @@ program testpio
 
               call PIO_write_darray(File_i4,vard_i4m,iodesc_i4,test_i4m,iostat)
               call check_pioerr(iostat,__FILE__,__LINE__,' i4m write_darray')
-              call PIO_CloseFile(File_i4) 
+              call PIO_CloseFile(File_i4)
            endif
 
-           if(TestCombo) then 
+           if(TestCombo) then
               if(iofmtd .ne. 'bin') then
                  iostat = pio_put_var(file,varfn,fname)
                  iostat = pio_put_var(file,varfruit,fruits)
@@ -1022,9 +1022,9 @@ program testpio
            endif
 
            if(Debug) then
-              write(*,'(a,2(a,i8),i8)') myname,':: After calls to PIO_write_darray.  comp_rank=',my_task, & 
+              write(*,'(a,2(a,i8),i8)') myname,':: After calls to PIO_write_darray.  comp_rank=',my_task, &
                    ' io_rank=',iorank,mpi_comm_io
-              
+
            endif
 
         endif
@@ -1033,17 +1033,17 @@ program testpio
               if(Debug)       print *,'iam: ',My_task,'testpio: point #14'
 
 
-        if (readPhase) then 
+        if (readPhase) then
            !-------------------------------------
            ! Open the file back up and check data
            !-------------------------------------
-           
-           if(TestR8) then 
+
+           if(TestR8) then
               ierr = PIO_OpenFile(PIOSYS, File_r8, iotype, fname_r8)
               call check_pioerr(ierr,__FILE__,__LINE__,' r8 openfile')
            endif
               if(Debug)       print *,'iam: ',My_task,'testpio: point #15'
-           
+
            if(TestR4) then
               ierr = PIO_OpenFile(PIOSYS,File_r4,iotype, fname_r4)
               call check_pioerr(ierr,__FILE__,__LINE__,' r4 openfile')
@@ -1059,13 +1059,13 @@ program testpio
            if(Debug) then
               write(*,'(2a,i8)') myname,':: After calls to PIO_OpenFile.  my_task=',my_task
            endif
-           
+
            if(Debug) print *,__FILE__,__LINE__
 
            if(iotype == iotype_pnetcdf .or. &
                 iotype == iotype_netcdf) then
               do ivar=1,nvars
-                 if(TestR8) then 
+                 if(TestR8) then
                     iostat = PIO_inq_varid(file_r8,'filename',varfn_r8)
 
 
@@ -1073,15 +1073,15 @@ program testpio
                     call check_pioerr(iostat,__FILE__,__LINE__,' r8 inq_varid')
                  endif
 
-                 if(TestR4) then 
+                 if(TestR4) then
                     if(iofmtd(2:3) .eq. 'nc') then
                        iostat = PIO_inq_varid(file_r4,'filename',varfn_r4)
                     end if
-	            iostat = PIO_inq_varid(File_r4,'field00001',vard_r4(ivar))  
+	            iostat = PIO_inq_varid(File_r4,'field00001',vard_r4(ivar))
                    call check_pioerr(iostat,__FILE__,__LINE__,' r4 inq_varid')
                  endif
               end do
-              if(TestInt) then 
+              if(TestInt) then
                  iostat = PIO_inq_varid(File_i4,'field',vard_i4)
                  call check_pioerr(iostat,__FILE__,__LINE__,' i4 inq_varid')
               endif
@@ -1093,7 +1093,7 @@ program testpio
            ! Time the parallel  read
            !-------------------------
            dt_read_r8 = 0.
-           if(TestR8) then 
+           if(TestR8) then
               if(iofmtd(2:3) .eq. 'nc') then
                  iostat = pio_get_var(file_r8,varfn_r8, fnamechk)
                  if(fnamechk /= fname_r8) then
@@ -1116,7 +1116,7 @@ program testpio
               call t_stopf('testpio_read')
 #endif
               et = MPI_Wtime()
-              dt_read_r8 = dt_read_r8 + (et - st)/nvars        
+              dt_read_r8 = dt_read_r8 + (et - st)/nvars
               call check_pioerr(iostat,__FILE__,__LINE__,' r8 read_darray')
            endif
 
@@ -1166,12 +1166,12 @@ program testpio
            endif
 
            !-------------------------------
-           ! Print the maximum memory usage 
+           ! Print the maximum memory usage
            !-------------------------------
 !           call alloc_print_usage(0,'testpio: after calls to PIO_read_darray')
 
 #ifdef TESTMEM
-!           stop 
+!           stop
 #endif
 
            if(Debug) then
@@ -1180,7 +1180,7 @@ program testpio
            endif
 
      !-------------------
-     ! close the file up 
+     ! close the file up
      !-------------------
            if(TestR8) call PIO_CloseFile(File_r8)
            if(TestR4) call PIO_CloseFile(File_r4)
@@ -1196,13 +1196,13 @@ program testpio
 !           endif
 
      !-----------------------------
-     ! Perform correctness testing 
+     ! Perform correctness testing
      !-----------------------------
-           if(TestR8 .and. CheckArrays) then 
+           if(TestR8 .and. CheckArrays) then
              call checkpattern(mpi_comm_compute, fname_r8,test_r8wr,test_r8rd,lLength,iostat)
               call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern r8 test')
            endif
-     
+
            if( TestR4 .and. CheckArrays) then
               call checkpattern(mpi_comm_compute, fname_r4,test_r4wr,test_r4rd,lLength,iostat)
               call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern r4 test')
@@ -1214,15 +1214,15 @@ program testpio
            endif
               if(Debug)       print *,'iam: ',My_task,'testpio: point #21'
 
-           if(TestCombo .and. CheckArrays) then 
+           if(TestCombo .and. CheckArrays) then
 
               !-------------------------------------
-              !  Open up and read the combined file 
+              !  Open up and read the combined file
               !-------------------------------------
-              
+
               ierr = PIO_OpenFile(PIOSYS,File,iotype,fname)
               call check_pioerr(ierr,__FILE__,__LINE__,' combo test read openfile')
-              
+
               if(iofmtd(1:2).eq.'nc') then
                  iostat = PIO_inq_varid(File,'field_r8',vard_r8c)
                  call check_pioerr(iostat,__FILE__,__LINE__,' combo test r8 inq_varid')
@@ -1256,22 +1256,22 @@ program testpio
               call PIO_CloseFile(File)
               if(Debug)       print *,'iam: ',My_task,'testpio: point #22a'
               et = MPI_Wtime()
-              dt_read_r8 = dt_read_r8 + (et - st)/nvars           
+              dt_read_r8 = dt_read_r8 + (et - st)/nvars
               !-----------------------------
-              ! Check the combined file 
+              ! Check the combined file
               !-----------------------------
               call checkpattern(mpi_comm_compute, fname,test_r8wr,test_r8rd,lLength,iostat)
               call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern test_r8 ')
-              
+
               call checkpattern(mpi_comm_compute, fname,test_r4wr,test_r4rd,lLength,iostat)
               call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern test_r4 ')
-        
+
               call checkpattern(mpi_comm_compute, fname,test_i4wr,test_i4rd,lLength,iostat)
               call check_pioerr(iostat,__FILE__,__LINE__,' checkpattern test_i4 ')
-              
+
            endif
      !---------------------------------------
-     ! Print out the performance measurements 
+     ! Print out the performance measurements
      !---------------------------------------
            call MPI_Barrier(MPI_COMM_COMPUTE,ierr)
         endif
@@ -1289,7 +1289,7 @@ program testpio
            if(writePhase) call GetMaxTime(dt_write_r4, gdt_write_r4(it), MPI_COMM_COMPUTE, ierr)
         endif
               if(Debug)       print *,'iam: ',My_task,'testpio: point #24'
-        
+
         if(TestInt) then
            ! Maximum read/write times
            if(readPhase)  call GetMaxTime(dt_read_i4, gdt_read_i4(it), MPI_COMM_COMPUTE, ierr)
@@ -1307,30 +1307,30 @@ program testpio
 
 
   !--------------------------------
-  ! Clean up initialization memory 
+  ! Clean up initialization memory
   !   note: make sure DOFs are not used later
   !--------------------------------
   if (My_task >= 0) call dealloc_check(compDOF)
 
   !----------------------------------
-  ! Print summary bandwidth statistics 
+  ! Print summary bandwidth statistics
   !----------------------------------
 
   if(Debug)       print *,'iam: ',My_task,'testpio: point #26'
   if(TestR8 .or. TestCombo .and. (iorank == 0) ) then
-     call WriteTimeTrialsStats(casename,TestR8CaseName, fname_r8, glenr8, gdt_read_r8, gdt_write_r8, maxiter) 
+     call WriteTimeTrialsStats(casename,TestR8CaseName, fname_r8, glenr8, gdt_read_r8, gdt_write_r8, maxiter)
   endif
 
   if(TestR4 .and. (iorank == 0) ) then
-     call WriteTimeTrialsStats(casename,TestR4CaseName, fname_r4, glenr4, gdt_read_r4, gdt_write_r4, maxiter) 
+     call WriteTimeTrialsStats(casename,TestR4CaseName, fname_r4, glenr4, gdt_read_r4, gdt_write_r4, maxiter)
   endif
 
   if(TestInt .and. (iorank == 0) ) then
-     call WriteTimeTrialsStats(casename,TestI4CaseName, fname_i4, gleni4, gdt_read_i4, gdt_write_i4, maxiter) 
+     call WriteTimeTrialsStats(casename,TestI4CaseName, fname_i4, gleni4, gdt_read_i4, gdt_write_i4, maxiter)
   endif
 
   !-------------------------------
-  ! Print timers and memory usage 
+  ! Print timers and memory usage
   !-------------------------------
 
 #ifdef TIMING
@@ -1451,7 +1451,7 @@ end subroutine WriteStats
 
   !=============================================================================
 
-  subroutine WriteTimeTrialsStats(casename,TestName, FileName, glen, ReadTimes, WriteTimes, nTrials) 
+  subroutine WriteTimeTrialsStats(casename,TestName, FileName, glen, ReadTimes, WriteTimes, nTrials)
 
     implicit none
 
@@ -1660,4 +1660,3 @@ end subroutine check_pioerr
   !=============================================================================
 
 end program testpio
-
diff --git a/examples/basic/testpio_bench.pl b/examples/basic/testpio_bench.pl
index fe3a223964..a47eada67f 100755
--- a/examples/basic/testpio_bench.pl
+++ b/examples/basic/testpio_bench.pl
@@ -275,7 +275,7 @@ sub usage{
 	if($attributes{NETCDF_PATH} =~ /netcdf-4/){
 	    $enablenetcdf4="--enable-netcdf4";
 	}
-    }    
+    }
 }
 
 if(defined $suites){
@@ -314,7 +314,7 @@ sub usage{
                       ldz => 0,
                       partfile => 'null',
                       partdir  => 'foo',
-                      iofmt => 'pnc', 
+                      iofmt => 'pnc',
                       rearr => 'box',
                       numprocsIO => 10,
    		      stride => -1,
@@ -435,7 +435,7 @@ sub usage{
 	   $configuration{$name}=$value;
      }
      $found = 1;
-  } 
+  }
 }
 #my $suffix = $bname . "-" . $pecount;
 my $suffix = $bname . "_PE-" . $pecount . "_IO-" . $iofmt . "-" . $numIO;
@@ -587,8 +587,8 @@ sub usage{
     }elsif(/ENV_(.*)/){
         print "set $1 $attributes{$_}\n";
 	print F "\$ENV{$1}=\"$attributes{$_}\"\;\n";
-    }	
-    
+    }
+
 }
 
 
@@ -659,7 +659,7 @@ sub usage{
     my \@testlist = \"$suffix";
 #    unlink("../pio/Makefile.conf");
 #    copy("testpio_in","$tstdir"); # copy the namelist file into test directory
-    
+
     chdir ("$tstdir");
     my \$test;
     my \$run = "$attributes{run}";
@@ -709,7 +709,7 @@ sub usage{
 	    open(LOG,\$log);
 	    my \@logout = <LOG>;
 	    close(LOG);
-	    
+
 	    my \$cnt = grep /testpio completed successfully/ , \@logout;
             open(T,">TestStatus");
 	    if(\$cnt>0){
@@ -724,7 +724,7 @@ sub usage{
 	    close(T);
 	}
     }else{
-	print "suite \$suite FAILED to configure or build\\n";	
+	print "suite \$suite FAILED to configure or build\\n";
     }
 }
 print "test complete on $host \$passcnt tests PASS, \$failcnt tests FAIL\\n";
diff --git a/examples/basic/testpio_build.pl b/examples/basic/testpio_build.pl
index 56faccd69f..a447f381e6 100644
--- a/examples/basic/testpio_build.pl
+++ b/examples/basic/testpio_build.pl
@@ -47,8 +47,8 @@
     }elsif(/ENV_(.*)/){
         print "set $1 $attributes{$_}\n";
 	$ENV{$1}="$attributes{$_}";
-    }	
-    
+    }
+
 }
 
 
@@ -70,4 +70,3 @@
     system('gmake clean') if($clean eq 'yes');
     system('gmake');
 }
-
diff --git a/examples/basic/testpio_run.pl b/examples/basic/testpio_run.pl
index d0f8900bf8..6c7b2ab606 100755
--- a/examples/basic/testpio_run.pl
+++ b/examples/basic/testpio_run.pl
@@ -79,8 +79,8 @@ sub usage{
 #    }elsif(/ENV_(.*)/){
 #        print "set $1 $attributes{$_}\n";
 #	print F "\$ENV{$1}=\"$attributes{$_}\n\"";
-#    }	
-    
+#    }
+
 }
 
 if(defined $suites){
@@ -233,7 +233,7 @@ sub usage{
          # \$ENV{MP_PROCS} = 1;
           #system("hostname > $tstdir/hostfile");
           #\$ENV{MP_HOSTFILE}="$tstdir/hostfile";
-	  
+
        # }
 	if("$host" eq "yellowstone_pgi") {
 	    \$ENV{LD_PRELOAD}="/opt/ibmhpc/pe1304/ppe.pami/gnu/lib64/pami64/libpami.so";
@@ -242,7 +242,7 @@ sub usage{
         if("$host" eq "erebus" or "$host" =~ /^yellowstone/){
         #  \$ENV{MP_PROCS}=\$saveprocs;
          # delete \$ENV{MP_HOSTFILE};
-        } 
+        }
     }
     my \$test;
 
@@ -276,9 +276,9 @@ sub usage{
 
 	    unlink("testpio") if(-e "testpio");
             if($twopass){
-		copy("$tstdir/testpio.\$suite","testpio");  
+		copy("$tstdir/testpio.\$suite","testpio");
             }else{
-		copy("$tstdir/testpio","testpio");  
+		copy("$tstdir/testpio","testpio");
 	    }
 	    chmod 0755,"testpio";
 
@@ -316,7 +316,7 @@ sub usage{
 	    open(LOG,\$log);
 	    my \@logout = <LOG>;
 	    close(LOG);
-	    
+
 	    my \$cnt = grep /testpio completed successfully/ , \@logout;
             open(T,">TestStatus");
 	    if(\$cnt>0){
@@ -331,7 +331,7 @@ sub usage{
 	    close(T);
 	}
     }else{
-	print "suite \$suite FAILED to configure or build\\n";	
+	print "suite \$suite FAILED to configure or build\\n";
     }
 }
 if($twopass && \$thispass==1){
@@ -341,7 +341,7 @@ sub usage{
 	print "Run ($script) second pass with \$subsys\n";
     }else{
 	exec(\$subsys);
-    }   
+    }
 }
 
 print "test complete on $host \$passcnt tests PASS, \$failcnt tests FAIL\\n";
diff --git a/examples/basic/utils_mod.F90 b/examples/basic/utils_mod.F90
index 203064da41..6919390a8e 100644
--- a/examples/basic/utils_mod.F90
+++ b/examples/basic/utils_mod.F90
@@ -11,7 +11,7 @@ module utils_mod
 
 !>
 !! @private
-!! @brief Writes netcdf header information for testpio. 
+!! @brief Writes netcdf header information for testpio.
 !! @param File @copydoc file_desc_t
 !! @param nx
 !! @param ny
diff --git a/examples/basic/wstest.c b/examples/basic/wstest.c
index de4f05dcd5..c2fa962855 100644
--- a/examples/basic/wstest.c
+++ b/examples/basic/wstest.c
@@ -27,7 +27,7 @@ int main(int argc, char *argv[])
 
  PIOc_Init_Intracomm(MPI_COMM_WORLD, npe, 1, 0, PIO_REARR_SUBSET,&iosysid);
 
- // Create a weak scaling test - 
+ // Create a weak scaling test -
  nx=6;
  ny=6;
  nz=2;
@@ -52,15 +52,15 @@ int main(int argc, char *argv[])
 
  PIOc_createfile(iosysid, &ncid, &iotype, "wstest.nc", PIO_CLOBBER);
  // Order of dims in c is slowest first
- PIOc_def_dim(ncid, "nx", (PIO_Offset) gdim[2], dimids+2); 
- PIOc_def_dim(ncid, "ny", (PIO_Offset) gdim[1], dimids+1); 
+ PIOc_def_dim(ncid, "nx", (PIO_Offset) gdim[2], dimids+2);
+ PIOc_def_dim(ncid, "ny", (PIO_Offset) gdim[1], dimids+1);
  PIOc_def_dim(ncid, "nz", (PIO_Offset) gdim[0], dimids);
 
  PIOc_def_var(ncid, "idof", PIO_INT, 3, dimids, &vid);
- 
+
  PIOc_enddef(ncid);
 
- 
+
 
  PIOc_write_darray(ncid, vid, iodesc,(PIO_Offset) (nx*ny*nz), iarray, NULL);
 
diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt
index 4e5576ea66..215f887164 100644
--- a/examples/c/CMakeLists.txt
+++ b/examples/c/CMakeLists.txt
@@ -29,6 +29,10 @@ ADD_EXECUTABLE(examplePio EXCLUDE_FROM_ALL examplePio.c)
 TARGET_LINK_LIBRARIES(examplePio pioc)
 add_dependencies(tests examplePio)
 
+ADD_EXECUTABLE(piorcw piorcw.c)
+TARGET_LINK_LIBRARIES(piorcw pioc)
+add_dependencies(tests piorcw)
+
 ADD_EXECUTABLE(example1 example1.c)
 TARGET_LINK_LIBRARIES(example1 pioc)
 add_dependencies(tests example1)
@@ -47,6 +51,7 @@ if (PIO_USE_MPISERIAL)
 else ()
   add_mpi_test(examplePio EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/examplePio NUMPROCS 4 TIMEOUT 60)
   add_mpi_test(example1 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/example1 NUMPROCS 4 TIMEOUT 60)
+  add_mpi_test(piorcw EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/piorcw  NUMPROCS 16 TIMEOUT 600)
   #add_mpi_test(darray_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_async NUMPROCS 5 TIMEOUT 60)
   add_mpi_test(darray_no_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_no_async NUMPROCS 4 TIMEOUT 60)
 endif ()
diff --git a/examples/c/darray_async.c b/examples/c/darray_async.c
index ea5f62ac2d..1186f998c6 100644
--- a/examples/c/darray_async.c
+++ b/examples/c/darray_async.c
@@ -64,7 +64,7 @@
 
 /* Number of computation components. */
 #define COMPONENT_COUNT 1
-        
+
 /* Lengths of dimensions. */
 int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
 
@@ -114,7 +114,7 @@ char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"};
 /*     { */
 /*         char my_dim_name[PIO_MAX_NAME]; */
 /*         PIO_Offset dimlen;  */
-        
+
 /*         if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen))) */
 /*             return ret; */
 /*         if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d])) */
@@ -136,7 +136,7 @@ char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"};
 /*     for (int t = 0; t < NUM_TIMESTEPS; t++) */
 /*     { */
 /*         int varid = 0; /\* There's only one var in sample file. *\/ */
-        
+
 /*         /\* This is the data we expect for this timestep. *\/ */
 /*         for (int i = 0; i < elements_per_pe; i++) */
 /*             buffer[i] = 100 * t + START_DATA_VAL + my_rank; */
@@ -207,7 +207,7 @@ int main(int argc, char* argv[])
     if ((ret = GPTLinitialize ()))
         return ret;
 #endif
-        
+
     /* Initialize MPI. */
     if ((ret = MPI_Init(&argc, &argv)))
         MPIERR(ret);
@@ -239,7 +239,7 @@ int main(int argc, char* argv[])
 
     /* Num procs for computation. */
     int num_procs2[COMPONENT_COUNT] = {4};
-        
+
     /* Is the current process a computation task? */
     int comp_task = my_rank < NUM_IO_TASKS ? 0 : 1;
 
@@ -258,7 +258,7 @@ int main(int argc, char* argv[])
     {
         /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */
         /* int ioid;     /\* The I/O description ID. *\/ */
-            
+
         /* /\* How many elements on each computation task? *\/ */
         /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / NUM_COMP_TASKS; */
 
@@ -318,7 +318,7 @@ int main(int argc, char* argv[])
 /*                 /\* Create some data for this timestep. *\/ */
 /*                 for (int i = 0; i < elements_per_pe; i++) */
 /*                     buffer[i] = 100 * t + START_DATA_VAL + my_rank; */
-                
+
 /*                 /\* Write data to the file. *\/ */
 /*                 printf("rank: %d Writing sample data...\n", my_rank); */
 /*                 if ((ret = PIOc_setframe(ncid, varid, t))) */
diff --git a/examples/c/examplePio.c b/examples/c/examplePio.c
index 4f678815ba..3ec01748ab 100644
--- a/examples/c/examplePio.c
+++ b/examples/c/examplePio.c
@@ -1,10 +1,10 @@
 /**
- * @file 
+ * @file
  * @brief A simple C example for the ParallelIO Library.
  *
  * This example creates a netCDF output file with one dimension and
  * one variable. It first writes, then reads the sample file using the
- * ParallelIO library. 
+ * ParallelIO library.
  *
  * This example can be run in parallel for 1, 2, 4, 8, or 16
  * processors.
@@ -65,7 +65,7 @@ typedef struct examplePioClass
 
     /** Pointer to function that cleans up example memory, and library resources. */
     struct examplePioClass* (*cleanUp) (struct examplePioClass*);
-    
+
     /** Pointer to function that handles errors. */
     struct examplePioClass* (*errorHandler) (struct examplePioClass*, const char*, const int);
 
@@ -136,12 +136,12 @@ typedef struct examplePioClass
     PIO_Offset *compdof;
 
     /** The example file name. */
-    char *fileName;     
-    
+    char *fileName;
+
 } examplePioClass;
 
-/** @brief Initialize libraries, create sample data. 
-    
+/** @brief Initialize libraries, create sample data.
+
     This function is called as part of the creation of a sample data
     file for this example.
 
@@ -175,7 +175,7 @@ struct examplePioClass* epc_init( struct examplePioClass* this )
     /*
     ** initialize MPI
     */
-    
+
     MPI_Init(NULL, NULL);
     MPI_Comm_rank(MPI_COMM_WORLD, &this->myRank);
     MPI_Comm_size(MPI_COMM_WORLD, &this->ntasks);
@@ -198,35 +198,35 @@ struct examplePioClass* epc_init( struct examplePioClass* this )
     /*
     ** set up PIO for rest of example
     */
-        
+
     this->stride        = 1;
     this->numAggregator = 0;
     this->optBase       = 0;
     this->iotype        = PIO_IOTYPE_NETCDF;
     this->fileName      = "examplePio_c.nc";
     this->dimLen[0]     = LEN;
-    
+
     this->niotasks = this->ntasks; /* keep things simple - 1 iotask per MPI process */
-    
+
     if (this->myRank == 0){
         printf("Running with %d MPI processes and %d PIO processes. \n",this->ntasks,this->niotasks);
     }
-    
+
     PIOc_Init_Intracomm(MPI_COMM_WORLD, this->niotasks, this->stride, this->optBase, PIO_REARR_SUBSET, &this->pioIoSystem);
-    
+
     /*
     ** set up some data that we will write to a netcdf file
     */
-    
+
     this->arrIdxPerPe = LEN / this->ntasks;
-    
+
     if (this->arrIdxPerPe < 1) {
         this->errorHandler(this, "Not enough work to distribute among pes",ERR_CODE);
     }
-        
+
     this->ista = this->myRank * this->arrIdxPerPe;
     this->isto = this->ista + (this->arrIdxPerPe - 1);
-    
+
     this->dataBuffer = (int *)malloc(this->arrIdxPerPe * sizeof (int));
     this->readBuffer = (int *)malloc(this->arrIdxPerPe * sizeof (int));
     this->compdof = (PIO_Offset *)malloc(this->arrIdxPerPe * sizeof(PIO_Offset));
@@ -234,14 +234,14 @@ struct examplePioClass* epc_init( struct examplePioClass* this )
     /*
     ** assign values to various arrays
     */
-    
+
     localVal = this->ista;
     for (i = 0; i < this->arrIdxPerPe; i++ ){
-        
+
         this->dataBuffer[i] = this->myRank + VAL;
         this->compdof[i] = localVal + 1;
         this->readBuffer[i] = 99;
-        
+
         if (localVal > this->isto) {
             printf("error, should ABORT \n");
         }
@@ -331,11 +331,11 @@ struct examplePioClass* epc_createDecomp( struct examplePioClass* this )
     Uses the function PIOc_createfile() to create the netCDF output
     file. The format of the file is created in accordance with the
     iotype member variable, which specifies one of the following
-    values: 
+    values:
 
-    - PIO_IOTYPE_PNETCDF=1 Parallel Netcdf (parallel) 
-    - PIO_IOTYPE_NETCDF=2 Netcdf3 Classic format (serial) 
-    - PIO_IOTYPE_NETCDF4C=3 NetCDF4 (HDF5) compressed format (serial) 
+    - PIO_IOTYPE_PNETCDF=1 Parallel Netcdf (parallel)
+    - PIO_IOTYPE_NETCDF=2 Netcdf3 Classic format (serial)
+    - PIO_IOTYPE_NETCDF4C=3 NetCDF4 (HDF5) compressed format (serial)
     - PIO_IOTYPE_NETCDF4P=4 NetCDF4 (HDF5) parallel
 
     The PIOc_createfile() function has the following parameters:
@@ -346,7 +346,7 @@ struct examplePioClass* epc_createDecomp( struct examplePioClass* this )
     - the name of the sample file.
     - the NetCDF file creating mode, PIO_CLOBBER means overwrite any
       existing file with this name.
-    
+
     @param [in] this  Pointer to self.
     @retval examplePioClass* Pointer to self.
  */
@@ -360,7 +360,7 @@ struct examplePioClass* epc_createFile( struct examplePioClass* this )
 }
 
 /** @brief Define netCDF metadata.
-    
+
     This function is called as part of the creation of a sample data
     file for this example.
 
@@ -370,7 +370,7 @@ struct examplePioClass* epc_createFile( struct examplePioClass* this )
 
     All of the functions take the pioFileDesc returned by
     PIOc_createfile(). This is the ncid of the netCDF file.
-    
+
     @param [in] this  Pointer to self.
     @retval examplePioClass* Pointer to self.
  */
@@ -393,7 +393,7 @@ struct examplePioClass* epc_defineVar( struct examplePioClass* this )
     The data are written with the PIOc_write_darray() function. After
     the write is complete, ensure the file is synced for all processes
     after the write.
-    
+
     @param [in] this  Pointer to self.
     @retval examplePioClass* Pointer to self.
  */
@@ -404,7 +404,7 @@ struct examplePioClass* epc_writeVar( struct examplePioClass* this )
     PIOc_write_darray(this->pioFileDesc, this->pioVar, this->iodescNCells,
                       (PIO_Offset)this->arrIdxPerPe, this->dataBuffer, NULL);
     PIOc_sync(this->pioFileDesc);
-    
+
     return this;
 }
 
@@ -415,14 +415,14 @@ struct examplePioClass* epc_writeVar( struct examplePioClass* this )
 
     This function reads the data that has been written to the sample
     data file. The data are read with the PIOc_read_darray() function.
-    
+
     @param [in] this  Pointer to self.
     @retval examplePioClass* Pointer to self.
  */
 struct examplePioClass* epc_readVar( struct examplePioClass* this )
 {
     int i;
-    
+
     PIOc_read_darray(this->pioFileDesc, this->pioVar, this->iodescNCells,
                      (PIO_Offset)this->arrIdxPerPe, this->readBuffer);
 
@@ -432,7 +432,7 @@ struct examplePioClass* epc_readVar( struct examplePioClass* this )
 	    this->errorHandler(this, "The data was not what was expected!", ERR_CODE);
     if (this->verbose)
 	printf("rank: %d Data read matches expected data.\n", this->myRank);
-    
+
     return this;
 }
 
@@ -449,7 +449,7 @@ struct examplePioClass* epc_closeFile( struct examplePioClass* this )
     if (this->verbose)
 	printf("rank: %d Closing the sample data file...\n", this->myRank);
     PIOc_closefile(this->pioFileDesc);
-    
+
     return this;
 }
 
@@ -459,7 +459,7 @@ struct examplePioClass* epc_closeFile( struct examplePioClass* this )
     ParallelIO library function PIOc_freedecomp() to free
     decomposition resources. Then calles PIOc_free_iosystem() and
     MPI_finalize() to free library resources.
-    
+
     @param [in] this  Pointer to self.
     @retval examplePioClass* Pointer to self.
  */
@@ -470,12 +470,12 @@ struct examplePioClass* epc_cleanUp( struct examplePioClass* this )
     free(this->dataBuffer);
     free(this->readBuffer);
     free(this->compdof);
-    
+
     PIOc_freedecomp(this->pioIoSystem, this->iodescNCells);
     PIOc_free_iosystem(this->pioIoSystem);
 
     MPI_Finalize();
-    
+
     return this;
 }
 
@@ -484,7 +484,7 @@ struct examplePioClass* epc_cleanUp( struct examplePioClass* this )
     On error, process with rank zero will print error message, the
     netCDF file will be closed with PIOc_closefile(), and MPI_Abort is
     called to end the example execution on all processes.
-    
+
     @param [in] this  Pointer to self.
     @param [in] errMsg  an error message
     @param [in] retVal  the non-zero return value that indicated an error
@@ -495,17 +495,17 @@ struct examplePioClass* epc_errorHandler(struct examplePioClass* this, const cha
    /* class(pioExampleClass), intent(inout) :: this
     character(len=*),       intent(in)    :: errMsg
     integer,                intent(in)    :: retVal*/
-    
+
     if (retVal != PIO_NOERR){
-        
+
         if (this->myRank == 0){
             printf("%d %s\n",retVal,errMsg);
         }
-    
+
         PIOc_closefile(this->pioFileDesc);
         MPI_Abort(MPI_COMM_WORLD, retVal);
     }
-    
+
     return this;
 }
 
@@ -514,16 +514,16 @@ struct examplePioClass* epc_errorHandler(struct examplePioClass* this, const cha
     This function allocates memory for the struct that contains the
     code and data for this example. Then pointers are to the functions
     used in the example.
-    
+
     @param [in] verbose  Non-zero for output to stdout.
     @retval examplePioClass* Pointer to self.
  */
 struct examplePioClass* epc_new(int verbose)
 {
     struct examplePioClass* this = malloc((sizeof(struct examplePioClass)));
-    
+
     /* assign function pointers to impls */
-    
+
     this->init = epc_init;
     this->createDecomp = epc_createDecomp;
     this->createFile = epc_createFile;
@@ -534,7 +534,7 @@ struct examplePioClass* epc_new(int verbose)
     this->cleanUp = epc_cleanUp;
     this->errorHandler = epc_errorHandler;
     this->verbose = verbose;
-    
+
     return this;
 }
 
@@ -571,7 +571,7 @@ struct examplePioClass* epc_new(int verbose)
         foo = 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 45, 45, 45, 45 ;
     }
     </pre>
-    
+
     @param [in] argc argument count (should be zero)
     @param [in] argv argument array (should be NULL)
     @retval examplePioClass* Pointer to self.
@@ -597,12 +597,12 @@ int main(int argc, char* argv[])
         return ret;
 
     struct examplePioClass* pioExInst = epc_new(verbose);
-    
-#ifdef TIMING    
+
+#ifdef TIMING
     /* Initialize the GPTL timing library. */
     if ((ret = GPTLinitialize ()))
       return ret;
-#endif    
+#endif
 
     pioExInst->init(pioExInst);
     pioExInst->createDecomp(pioExInst);
@@ -618,12 +618,12 @@ int main(int argc, char* argv[])
 /* #endif /\* USE_MPE *\/ */
 
     pioExInst->cleanUp(pioExInst);
-    
-#ifdef TIMING    
+
+#ifdef TIMING
 	/* Finalize the GPTL timing library. */
 	if ((ret = GPTLfinalize ()))
 	    return ret;
-#endif    
+#endif
 
     free(pioExInst);
     return 0;
diff --git a/examples/c/piorcw.c b/examples/c/piorcw.c
new file mode 100644
index 0000000000..2e09801529
--- /dev/null
+++ b/examples/c/piorcw.c
@@ -0,0 +1,249 @@
+#include <config.h>
+#include <mpi.h>
+#include <pio.h>
+#include <pio_internal.h>
+
+static int debug = 0;
+
+double *dvarw, *dvarr;
+float *fvarw, *fvarr;
+int *ivarw, *ivarr;
+char *cvarw, *cvarr;
+
+typedef struct dimlist
+{
+    char name[16];
+    int value;
+} dimlist;
+
+
+
+int rcw_write_darray(int iosys, int rank)
+{
+    int ierr;
+    int comm_size;
+    int ncid;
+    int iotype = PIO_IOTYPE_NETCDF4P;
+    int ndims;
+    int *global_dimlen;
+    int num_tasks;
+    int *maplen;
+    int maxmaplen;
+    int *full_map;
+    int *dimid;
+    int varid;
+    int globalsize;
+    int ioid;
+    char dimname[PIO_MAX_NAME];
+    char varname[PIO_MAX_NAME];
+
+
+    ierr = MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
+    ierr = PIOc_createfile(iosys, &ncid, &iotype, "testfile.nc4", PIO_CLOBBER);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    dimid = calloc(ndims,sizeof(int));
+    for(int i=0; i<ndims; i++)
+    {
+        sprintf(dimname,"dim%4.4d",i);
+        ierr = PIOc_def_dim(ncid, dimname, (PIO_Offset) global_dimlen[i], &(dimid[i]));
+        if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+    }
+    ierr = PIOc_def_var(ncid, varname, PIO_DOUBLE, ndims, dimid, &varid);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    free(dimid);
+    ierr = PIOc_enddef(ncid);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    PIO_Offset *dofmap;
+
+    if (!(dofmap = malloc(sizeof(PIO_Offset) * maplen[rank])))
+        return PIO_ENOMEM;
+
+    /* Copy array into PIO_Offset array. */
+    dvarw = malloc(sizeof(double)*maplen[rank]);
+    for (int e = 0; e < maplen[rank]; e++)
+    {
+        dofmap[e] = full_map[rank * maxmaplen + e]+1;
+        dvarw[e] = dofmap[e];
+    }
+    /* allocated in pioc_read_nc_decomp_int */
+    free(full_map);
+    ierr = PIOc_InitDecomp(iosys, PIO_DOUBLE, ndims, global_dimlen, maplen[rank],
+                           dofmap, &ioid, NULL, NULL, NULL);
+
+    free(global_dimlen);
+    double dsum=0;
+    for(int i=0; i < maplen[rank]; i++)
+        dsum += dvarw[i];
+    if(dsum != rank)
+        printf("%d: dvarwsum = %d\n",rank, dsum);
+
+    ierr = PIOc_write_darray(ncid, varid, ioid, maplen[rank], dvarw, NULL);
+    free(maplen);
+    ierr = PIOc_closefile(ncid);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    return ierr;
+}
+
+
+int rcw_read_darray(int iosys,int rank)
+{
+    int ierr;
+    int comm_size;
+    int ncid;
+    int iotype = PIO_IOTYPE_PNETCDF;
+    int ndims;
+    int *global_dimlen;
+    int num_tasks;
+    int *maplen;
+    int maxmaplen;
+    int *full_map;
+    int *dimid;
+    int varid;
+    int globalsize;
+    int ioid;
+    int pio_type;
+    char dimname[PIO_MAX_NAME];
+    char varname[PIO_MAX_NAME];
+
+    int ndims, nvars, natts, unlimdim;
+    dimlist *dim;
+
+    ierr = MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
+
+    ierr = PIOc_openfile(iosys, &ncid, &iotype, "testfile.nc", PIO_NOWRITE);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    ierr = PIOc_inq(ncid, &ndims, &nvars, &natts, &unlimdim);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+
+    dim = (dimlist *) malloc(ndims*sizeof(dimlist));
+    for(i=0; i<ndims; i++){
+        ierr = PIOc_inq_dim(ncid, i, dim[i].name, dim[i].value);
+        if(ierr || debug) printf("%d %d i=%d\n",__LINE__,ierr, i);
+    }
+
+
+
+
+    /* TODO: support multiple variables and types*/
+    if(myvarname != NULL)
+        sprintf(varname,"%s",myvarname);
+    else
+        sprintf(varname,"var%4.4d",0);
+    ierr = PIOc_inq_varid(ncid, varname, &varid);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    ierr = PIOc_inq_vartype(ncid, varid, &pio_type);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    dimid = calloc(ndims,sizeof(int));
+    ierr = PIOc_inq_vardimid(ncid, varid, dimid);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    for(int i=0; i<ndims; i++)
+    {
+        PIO_Offset gdimlen;
+        ierr = PIOc_inq_dimlen(ncid, dimid[i], &gdimlen);
+
+        pioassert(gdimlen == global_dimlen[i], "testfile.nc does not match decomposition file",__FILE__,__LINE__);
+    }
+    free(dimid);
+    PIO_Offset *dofmap;
+
+    if (!(dofmap = malloc(sizeof(PIO_Offset) * maplen[rank])))
+        return PIO_ENOMEM;
+
+    for (int e = 0; e < maplen[rank]; e++)
+    {
+        dofmap[e] = full_map[rank * maxmaplen + e] + 1;
+    }
+    free(full_map);
+//    PIOc_set_log_level(3);
+    ierr = PIOc_InitDecomp(iosys, pio_type, ndims, global_dimlen, maplen[rank],
+                           dofmap, &ioid, NULL, NULL, NULL);
+    free(dofmap);
+    free(global_dimlen);
+    switch(pio_type)
+    {
+    case PIO_DOUBLE:
+        dvarr = malloc(sizeof(double)*maplen[rank]);
+        ierr = PIOc_read_darray(ncid, varid, ioid, maplen[rank], dvarr);
+        double dsum=0;
+        for(int i=0; i < maplen[rank]; i++)
+            dsum += dvarr[i];
+        if(dsum != rank)
+            printf("%d: dsum = %d\n",rank, dsum);
+        break;
+    case PIO_INT:
+        ivarr = malloc(sizeof(int)*maplen[rank]);
+        ierr = PIOc_read_darray(ncid, varid, ioid, maplen[rank], ivarr);
+        int isum=0;
+        for(int i=0; i < maplen[rank]; i++)
+            isum += ivarr[i];
+        printf("%d: isum = %d\n",rank, isum);
+        break;
+    case PIO_FLOAT:
+        fvarr = malloc(sizeof(float)*maplen[rank]);
+        ierr = PIOc_read_darray(ncid, varid, ioid, maplen[rank], fvarr);
+        float fsum=0;
+        for(int i=0; i < maplen[rank]; i++)
+            fsum += fvarr[i];
+        printf("%d: fsum = %f\n",rank, fsum);
+        break;
+    case PIO_BYTE:
+        cvarr = malloc(sizeof(char)*maplen[rank]);
+        ierr = PIOc_read_darray(ncid, varid, ioid, maplen[rank], cvarr);
+        int csum=0;
+        for(int i=0; i < maplen[rank]; i++)
+            csum += (int) cvarr[i];
+        printf("%d: csum = %d\n",rank, csum);
+        break;
+    }
+
+    ierr = PIOc_closefile(ncid);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+
+    free(maplen);
+    return ierr;
+
+}
+
+
+
+int main(int argc, char *argv[])
+{
+    struct arguments arguments;
+    int ierr;
+    int rank;
+    int comm_size;
+    int iosys;
+    int iotasks;
+    MPI_Init(&argc, &argv);
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+    MPI_Comm_size(MPI_COMM_WORLD, &comm_size);
+
+    iotasks = comm_size/36;
+
+    ierr = PIOc_Init_Intracomm(MPI_COMM_WORLD, iotasks, 36, 0, PIO_REARR_SUBSET, &iosys);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    ierr = rcw_read_darray(iosys, rank);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    ierr = rcw_write_darray(iosys, rank);
+    if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
+
+    MPI_Finalize();
+
+}
diff --git a/examples/c/valsupp_example1.supp b/examples/c/valsupp_example1.supp
index 63f3e07383..ace46e4738 100644
--- a/examples/c/valsupp_example1.supp
+++ b/examples/c/valsupp_example1.supp
@@ -12,4 +12,4 @@
    fun:flush_buffer
    fun:PIOc_sync
    fun:main
-}
\ No newline at end of file
+}
diff --git a/examples/cxx/examplePio.cxx b/examples/cxx/examplePio.cxx
index b4f03cc13e..73cdd0cab8 100644
--- a/examples/cxx/examplePio.cxx
+++ b/examples/cxx/examplePio.cxx
@@ -13,26 +13,26 @@ class pioExampleClass {
 
 pioExampleClass::pioExampleClass(){
     // user defined ctor with no arguments
-    
+
     cout << " pioExampleClass::pioExampleClass() "<< endl;
-    
+
 }
 
 void pioExampleClass::init () {
-    
+
     cout << " pioExampleClass::init() " << endl;
-    
+
 }
 
 int main () {
-    
+
     pioExampleClass *pioExInst;
-    
+
     pioExInst = new pioExampleClass();
-    
+
     pioExInst->init();
 
     delete(pioExInst);
-    
+
     return 0;
-}
\ No newline at end of file
+}
diff --git a/examples/f03/exampleAsyncPio.F90 b/examples/f03/exampleAsyncPio.F90
index 180e3cd794..e8a6f629e6 100644
--- a/examples/f03/exampleAsyncPio.F90
+++ b/examples/f03/exampleAsyncPio.F90
@@ -235,7 +235,6 @@ subroutine createDecomp(this)
         implicit none
 
         class(pioExampleClass), intent(inout) :: this
-        integer :: ierr
 
         call PIO_initdecomp(this%pioIoSystem(1), PIO_int, this%dimLen, this%compdof(this%ista:this%isto), &
             this%iodescNCells)
diff --git a/libpio.settings.in b/libpio.settings.in
index c3e0904585..444a9e1cbc 100644
--- a/libpio.settings.in
+++ b/libpio.settings.in
@@ -28,11 +28,10 @@ More Fortran Flags:	@FPPFLAGS@
 --------
 PnetCDF Support:	@HAS_PNETCDF@
 SZIP Write Support:	@HAS_SZIP_WRITE@
-Parallel Filters:	@HAS_PAR_FILTERS@
+Parallel Filters:	@have_par_filters@
 NetCDF/HDF5 Support:	@HAS_NETCDF4@
 NetCDF/HDF5 Par I/O:	@HAS_NETCDF4_PAR@
 NetCDF Integration:	@HAS_NETCDF_INTEGRATION@
 PIO Logging:		@HAS_LOGGING@
 MPIEXEC:		@WITH_MPIEXEC@
 Fortran:		@HAS_PIO_FORTRAN@
-
diff --git a/scripts/prune_decomps.pl b/scripts/prune_decomps.pl
index 58e4f66f0f..52f2781acb 100644
--- a/scripts/prune_decomps.pl
+++ b/scripts/prune_decomps.pl
@@ -18,7 +18,7 @@ sub rem_dup_decomp_files
 {
     my($dirname) = @_;
     # Find files in current directory that are
-    # named *piodecomp* - these are the pio 
+    # named *piodecomp* - these are the pio
     # decomposition files
     opendir(F,$dirname);
     #my @decompfiles = grep(/^piodecomp/,readdir(F));
@@ -30,7 +30,7 @@ sub rem_dup_decomp_files
     #for(my $i=0; $i<$ndecompfile_info; $i++){
     #  print "File : $decompfile_info[$i]->{FNAME} , size = $decompfile_info[$i]->{SIZE}\n";
     #}
-    
+
     my $rmfile=0;
     # Compare the decomposition files to find
     # duplicates - and delete the dups
@@ -56,7 +56,7 @@ sub rem_dup_decomp_files
                     my $nline = shift (@file2);
                     # Ignore stack traces when comparing files
                     # The stack traces start with a line containing
-                    # "Obtained" 
+                    # "Obtained"
                     # Also, stack trace is the last line being
                     # compared
                     if(($line =~ /${BEGIN_STACK_TRACE}/)
@@ -67,7 +67,7 @@ sub rem_dup_decomp_files
                         last;
                     }
                     next if($line eq $nline);
-                    # Files are different, don't remove    
+                    # Files are different, don't remove
                     $rmfile = 0;
                     last;
                 }
@@ -158,5 +158,3 @@ ()
     if($verbose){ print "Decoding stack traces for decomposition files from : \"", $rundir, "\"\n"; }
     &decode_stack_traces($rundir, $exe);
 }
-
-    
diff --git a/src/clib/CMakeLists.txt b/src/clib/CMakeLists.txt
index e1f6a13bd7..609de6f575 100644
--- a/src/clib/CMakeLists.txt
+++ b/src/clib/CMakeLists.txt
@@ -9,8 +9,8 @@ project (PIOC C)
 if (CMAKE_BUILD_TYPE)
     define_property(
         SOURCE
-        PROPERTY COMPILE_FLAGS 
-        INHERITED 
+        PROPERTY COMPILE_FLAGS
+        INHERITED
         BRIEF_DOCS "brief-doc"
         FULL_DOCS  "full-doc"
     )
@@ -34,10 +34,10 @@ add_library (pioc ${src})
 # Always use -fPIC
 set_property(TARGET pioc PROPERTY POSITION_INDEPENDENT_CODE ON)
 
-set_source_files_properties( 
-    pioc_async.c 
+set_source_files_properties(
+    pioc_async.c
     PROPERTIES
-    COMPILE_FLAGS -O0 
+    COMPILE_FLAGS -O0
 )
 
 
@@ -63,7 +63,7 @@ target_compile_definitions (pioc
 
 # Compiler-specific compiler options
 if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
-  string(APPEND CMAKE_C_FLAGS  " -std=c99 " )
+  string(APPEND CMAKE_C_FLAGS  " -std=gnu99 " )
 elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
   string(APPEND CMAKE_C_FLAGS  " -c99 ")
 elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
diff --git a/src/clib/parallel_sort.c b/src/clib/parallel_sort.c
index 4cc05846c6..7757106a4f 100644
--- a/src/clib/parallel_sort.c
+++ b/src/clib/parallel_sort.c
@@ -96,8 +96,8 @@ bool is_unique(CVector v) {
  * @param comm the MPI communicator over which v is distributed
  * @param v A CVector distributed over comm
  * @param ierr indicates an error was encountered
- * @return A CVector sorted over comm, the size of the new vector may be different 
- *         than v with a worst case of the entire result on one task. 
+ * @return A CVector sorted over comm, the size of the new vector may be different
+ *         than v with a worst case of the entire result on one task.
  */
 
 CVector parallel_sort(MPI_Comm comm, CVector v, int *ierr) {
@@ -130,7 +130,7 @@ CVector parallel_sort(MPI_Comm comm, CVector v, int *ierr) {
 
   if(!(pivot_pos = malloc((size + 1) * sizeof(*pivot_pos))))
       *ierr = pio_err(NULL, NULL, PIO_ENOMEM, __FILE__,__LINE__);
-  
+
   pivot_pos[0] = v.data;
   for (size_t i = 0; i < size - 1; ++i)
     pivot_pos[i + 1] = partition(pivot_pos[i], v.data + v.N, local_pivots[i]);
@@ -190,14 +190,14 @@ CVector parallel_sort(MPI_Comm comm, CVector v, int *ierr) {
 int run_unique_check(MPI_Comm comm, size_t N,datatype *v, bool *has_dups)
 {
   int rank, size;
-  int mpierr=MPI_SUCCESS, mpierr2=MPI_SUCCESS;
+  int mpierr=MPI_SUCCESS;
   int ierr;
   if ((mpierr = MPI_Comm_rank(comm, &rank)))
-    check_mpi(NULL, NULL, mpierr2, __FILE__, __LINE__);
+    check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
 
   if ((mpierr = MPI_Comm_size(comm, &size)))
-    check_mpi(NULL, NULL, mpierr2, __FILE__, __LINE__);
-    
+    check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
   srand(time(NULL) * rank);
 
   CVector sorted = parallel_sort(comm, (CVector){v, N}, &ierr);
@@ -205,13 +205,13 @@ int run_unique_check(MPI_Comm comm, size_t N,datatype *v, bool *has_dups)
   int i_have_dups = is_unique(sorted) ? 0:1;
   int global_dups;
   if ((mpierr = MPI_Allreduce(&i_have_dups, &global_dups, 1, MPI_INT, MPI_MAX, comm)))
-    check_mpi(NULL, NULL, mpierr2, __FILE__, __LINE__);
-    
+    check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+
   if(global_dups > 0)
     *has_dups = true;
   else
     *has_dups = false;
-  
+
 #ifdef DEBUG_PARALLEL_SORT
   for (r=0; r<size; r++)
     {
diff --git a/src/clib/pio.h b/src/clib/pio.h
index cab66db096..b2d96e000d 100644
--- a/src/clib/pio.h
+++ b/src/clib/pio.h
@@ -17,6 +17,16 @@
 #include <uthash.h>
 
 #include <netcdf.h>
+#include <netcdf_meta.h>
+
+#define NETCDF_VERSION_LE(Maj, Min, Pat) \
+    (((NC_VERSION_MAJOR == Maj) && (NC_VERSION_MINOR == Min) && (NC_VERSION_PATCH <= Pat)) || \
+     ((NC_VERSION_MAJOR == Maj) && (NC_VERSION_MINOR < Min)) || (NC_VERSION_MAJOR < Maj))
+
+#define NETCDF_VERSION_GE(Maj, Min, Pat) \
+    (((NC_VERSION_MAJOR == Maj) && (NC_VERSION_MINOR == Min) && (NC_VERSION_PATCH >= Pat)) || \
+     ((NC_VERSION_MAJOR == Maj) && (NC_VERSION_MINOR > Min)) || (NC_VERSION_MAJOR > Maj))
+
 
 /** PIO_OFFSET is an integer type of size sufficient to represent the
  * size (in bytes) of the largest file supported by MPI. This is not
@@ -306,7 +316,7 @@ typedef struct io_desc_t
      * sort it. */
     bool needssort;
 
-    /** If the decomp has repeated values it can only be used for reading 
+    /** If the decomp has repeated values it can only be used for reading
         since it doesn't make sense to write a single value from more than one location. */
     bool readonly;
 
@@ -677,6 +687,13 @@ enum PIO_ERROR_HANDLERS
 #define PIO_64BIT_OFFSET NC_64BIT_OFFSET /**< Use large (64-bit) file offsets. Mode flag for nc_create(). */
 #define PIO_64BIT_DATA NC_64BIT_DATA /**< CDF5 format. */
 
+#ifdef NC_HAS_QUANTIZE
+#define PIO_NOQUANTIZE NC_NOQUANTIZE
+#define PIO_QUANTIZE_BITGROOM NC_QUANTIZE_BITGROOM
+#define PIO_QUANTIZE_GRANULARBR NC_QUANTIZE_GRANULARBR
+#define PIO_QUANTIZE_BITROUND  NC_QUANTIZE_BITROUND  /**< Use BitRound quantization. */
+#endif
+
 /** Define the netCDF-based error codes. */
 #define PIO_NOERR  NC_NOERR           /**< No Error */
 #define PIO_EBADID NC_EBADID          /**< Bad ncid */
@@ -946,6 +963,13 @@ extern "C" {
     int PIOc_set_fill(int ncid, int fillmode, int *old_modep);
     int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value);
     int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep);
+#ifdef NC_HAS_BZ2
+  int PIOc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp);
+#endif
+#ifdef NC_HAS_ZSTD
+  int PIOc_def_var_zstandard(int ncid, int varid, int level);
+  int PIOc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp);
+#endif
     int PIOc_rename_var(int ncid, int varid, const char *name);
 
     /* These variable settings only apply to netCDF-4 files. */
@@ -1243,6 +1267,18 @@ extern "C" {
                                const long long *op);
     int PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum,
                                 const unsigned long long *op);
+/* use this variable in the NETCDF library (introduced in v4.9.0) to determine if the following
+   functions are available */
+#ifdef NC_HAS_MULTIFILTERS
+  int PIOc_def_var_filter(int ncid, int varid,unsigned int id, size_t nparams, unsigned int *params);
+  int PIOc_inq_var_filter_ids(int ncid, int varid, size_t *nfiltersp, unsigned int *ids);
+  int PIOc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t *nparamsp, unsigned int *params );
+  int PIOc_inq_filter_avail(int ncid, unsigned int id );
+#endif
+#ifdef NC_HAS_QUANTIZE
+  int PIOc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd );
+  int PIOc_inq_var_quantize(int ncid, int varid, int *quantize_mode, int *nsdp );
+#endif
 
     /* These functions are for the netCDF integration layer. */
     int nc_def_iosystem(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr,
diff --git a/src/clib/pio_darray_int.c b/src/clib/pio_darray_int.c
index 6ef60594a6..30359ecb85 100644
--- a/src/clib/pio_darray_int.c
+++ b/src/clib/pio_darray_int.c
@@ -455,11 +455,12 @@ write_darray_multi_par(file_desc_t *file, int nvars, int fndims, const int *vari
                         bufptr = (void *)((char *)iobuf + iodesc->mpitype_size * (nv * llen + region->loffset));
 
                     /* Ensure collective access. */
-                    ierr = nc_var_par_access(file->fh, varids[nv], NC_COLLECTIVE);
+                    if((ierr = nc_var_par_access(file->fh, varids[nv], NC_COLLECTIVE)))
+                        return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
                     /* Write the data for this variable. */
-                    if (!ierr)
-                        ierr = nc_put_vara(file->fh, varids[nv], (size_t *)start, (size_t *)count, bufptr);
+                    if((ierr = nc_put_vara(file->fh, varids[nv], (size_t *)start, (size_t *)count, bufptr)))
+                        return pio_err(ios, file, ierr, __FILE__, __LINE__);
                 }
                 break;
 #endif
@@ -1681,11 +1682,11 @@ pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
                  * ios->num_iotasks is the number of iotasks actually
                  * used in this decomposition. */
                 if (rtask < ios->num_iotasks && tmp_bufsize > 0){
-		  
+
                     if ((mpierr = MPI_Send(iobuf, tmp_bufsize, iodesc->mpitype, rtask,
                                            4 * ios->num_iotasks + rtask, ios->io_comm)))
                         return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
-		}
+                }
             }
         }
     }
diff --git a/src/clib/pio_internal.h b/src/clib/pio_internal.h
index efd4f8182e..c360ae4e48 100644
--- a/src/clib/pio_internal.h
+++ b/src/clib/pio_internal.h
@@ -13,11 +13,14 @@
 #include <config.h>
 #include <pio.h>
 #include <pio_error.h>
+#include <stdint.h>
 #include <limits.h>
 #include <math.h>
 #include <netcdf.h>
 #ifdef _NETCDF4
 #include <netcdf_par.h>
+#include <netcdf_filter.h>
+#include <netcdf_meta.h>
 #endif
 #ifdef _PNETCDF
 #include <pnetcdf.h>
@@ -30,6 +33,22 @@
 #include <mpe.h>
 #endif /* USE_MPE */
 
+/* define an MPI type equivalent to size_t */
+#if SIZE_MAX == UCHAR_MAX
+   #define PIO_MPI_SIZE_T MPI_UNSIGNED_CHAR
+#elif SIZE_MAX == USHRT_MAX
+   #define PIO_MPI_SIZE_T MPI_UNSIGNED_SHORT
+#elif SIZE_MAX == UINT_MAX
+   #define PIO_MPI_SIZE_T MPI_UNSIGNED
+#elif SIZE_MAX == ULONG_MAX
+   #define PIO_MPI_SIZE_T MPI_UNSIGNED_LONG
+#elif SIZE_MAX == ULLONG_MAX
+   #define PIO_MPI_SIZE_T MPI_UNSIGNED_LONG_LONG
+#else
+   #error "what is happening here?"
+#endif
+
+
 //#ifndef MPI_OFFSET
 /** MPI_OFFSET is an integer type of size sufficient to represent the
  * size (in bytes) of the largest file supported by MPI. In some MPI
@@ -65,7 +84,7 @@
 /** Some MPI implementations do not allow passing MPI_DATATYPE_NULL to
  * comm functions even though the send or recv length is 0, in these
  * cases we use MPI_CHAR, after this issue raised its ugly head again in mpich
- * 4.0.0 we decided to use this workaround in all cases.  
+ * 4.0.0 we decided to use this workaround in all cases.
  * See https://github.com/NCAR/ParallelIO/issues/1945 */
 
 #define PIO_DATATYPE_NULL MPI_CHAR
@@ -613,6 +632,7 @@ enum PIO_MSG
     PIO_MSG_DEF_VAR_DEFLATE,
     PIO_MSG_INQ_VAR_DEFLATE,
     PIO_MSG_INQ_VAR_SZIP,
+    PIO_MSG_DEF_VAR_SZIP,
     PIO_MSG_DEF_VAR_FLETCHER32,
     PIO_MSG_INQ_VAR_FLETCHER32,
     PIO_MSG_DEF_VAR_CHUNKING,
@@ -638,7 +658,24 @@ enum PIO_MSG
     PIO_MSG_GET_ATT,
     PIO_MSG_PUT_ATT,
     PIO_MSG_INQ_TYPE,
-    PIO_MSG_INQ_UNLIMDIMS
+    PIO_MSG_INQ_UNLIMDIMS,
+#ifdef NC_HAS_BZ2
+    PIO_MSG_INQ_VAR_BZIP2,
+    PIO_MSG_DEF_VAR_BZIP2,
+#endif
+#ifdef NC_HAS_ZSTD
+    PIO_MSG_INQ_VAR_ZSTANDARD,
+    PIO_MSG_DEF_VAR_ZSTANDARD,
+#endif
+    PIO_MSG_DEF_VAR_FILTER,
+    PIO_MSG_INQ_VAR_FILTER_IDS,
+    PIO_MSG_INQ_VAR_FILTER_INFO,
+    PIO_MSG_INQ_FILTER_AVAIL,
+
+#ifdef NC_HAS_QUANTIZE
+    PIO_MSG_DEF_VAR_QUANTIZE,
+    PIO_MSG_INQ_VAR_QUANTIZE,
+#endif
 };
 
 #endif /* __PIO_INTERNAL__ */
diff --git a/src/clib/pio_msg.c b/src/clib/pio_msg.c
index ffb3e88aa6..36473decf9 100644
--- a/src/clib/pio_msg.c
+++ b/src/clib/pio_msg.c
@@ -1142,6 +1142,349 @@ int inq_var_chunking_handler(iosystem_desc_t *ios)
 
     return PIO_NOERR;
 }
+#ifdef PIO_HAS_PAR_FILTERS
+/**
+ * Do an inq_var_filter_ids on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_filter_ids_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    size_t *nfiltersp=NULL;
+    unsigned int *ids=NULL;
+    size_t nfilters;
+    char nfilters_present;
+    char ids_present;
+    size_t idsize=0;
+    int mpierr;
+
+    assert(ios);
+    PLOG((1, "inq_var_filter_ids_handler"));
+
+    /* Get the parameters for this function that the the comp main
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nfilters_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ids_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if(ids_present){
+        if ((mpierr = MPI_Bcast(&idsize, 1, PIO_MPI_SIZE_T, 0, ios->intercomm)))
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+        if (!(ids = malloc(idsize *sizeof(size_t))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    }
+
+    PLOG((2,"inq_var_filter_ids_handler ncid = %d varid = %d nfilters_present = %d ids_present = %d idsize = %d",
+          ncid, varid, nfilters_present, ids_present, idsize));
+
+    /* Set the non-NULL pointers. */
+    if(nfilters_present)
+        nfiltersp = &nfilters;
+
+    /* Call the inq function to get the values. */
+    PIOc_inq_var_filter_ids(ncid, varid, nfiltersp, ids);
+
+    if(ids_present)
+        free(ids);
+
+    return PIO_NOERR;
+}
+#ifdef NC_HAS_BZ2
+/**
+ * Do an inq_var_bzip2 on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_bzip2_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int *hasfilterp=NULL;
+    int *levelp=NULL;
+    char hasfilterp_present;
+    char levelp_present;
+    int hasfilter;
+    int level;
+    int mpierr;
+
+    assert(ios);
+    PLOG((1, "inq_var_bzip2_handler"));
+
+    /* Get the parameters for this function that the the comp main
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&hasfilterp_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&levelp_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    PLOG((2,"inq_var_bzip2_handler ncid = %d varid = %d hasfilter_present = %d  ",
+          ncid, varid, hasfilterp_present, levelp_present));
+
+    /* Set the non-NULL pointers. */
+    if(hasfilterp_present)
+        hasfilterp = &hasfilter;
+    if(levelp_present)
+        levelp = &level;
+
+    /* Call the inq function to get the values. */
+    PIOc_inq_var_bzip2(ncid, varid, hasfilterp, levelp);
+
+    return PIO_NOERR;
+}
+#endif
+
+#ifdef PIO_HAS_PAR_FILTERS
+/**
+ * Do an inq_var_filter_info on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_filter_info_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    unsigned int id;
+    size_t *nparamsp = NULL;
+    size_t nparams;
+    unsigned int *params = NULL;
+    char nparams_present;
+    char params_present;
+    size_t paramssize;
+    int mpierr;
+
+    assert(ios);
+    PLOG((1, "inq_var_filter_info_handler"));
+
+    /* Get the parameters for this function that the the comp main
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&id, 1, MPI_UNSIGNED, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nparams_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&params_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if(params_present){
+        if ((mpierr = MPI_Bcast(&paramssize, 1, PIO_MPI_SIZE_T, 0, ios->intercomm)))
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+        if (!(params = malloc(paramssize *sizeof(unsigned int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    }
+    PLOG((2,"inq_var_filter_info_handler ncid = %d varid = %d nparams_present = %d params_present = %d",
+          ncid, varid, nparams_present, params_present));
+
+    /* Set the non-NULL pointers. */
+    if (nparams_present)
+        nparamsp = &nparams;
+
+    /* Call the inq function to get the values. */
+    PIOc_inq_var_filter_info(ncid, varid, id, nparamsp, params);
+
+    if(params_present)
+        free(params);
+
+    return PIO_NOERR;
+}
+#endif
+#endif
+#ifdef NC_HAS_QUANTIZE
+/**
+ * Do an inq_var_quantize on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_quantize_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int *quantize_modep = NULL;
+    int *nsdp = NULL;
+    int qmode;
+    int nsd;
+    int mpierr;
+    char qmode_present;
+    char nsd_present;
+
+    assert(ios);
+    PLOG((1, "inq_var_chunking_handler"));
+
+    /* Get the parameters for this function that the the comp main
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&qmode_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nsd_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+
+
+    PLOG((2,"inq_var_handler ncid = %d varid = %d",
+          ncid, varid));
+
+    if (qmode_present)
+        quantize_modep = &qmode;
+    if(nsd_present)
+        nsdp = &nsd;
+    /* Call the inq function to get the values. */
+    PIOc_inq_var_quantize(ncid, varid, quantize_modep, nsdp);
+
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define a netCDF
+ *  variable quantize level.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ * @author Jim Edwards, Ed Hartnett
+ */
+int def_var_quantize_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mode;
+    int nsd;
+    int mpierr;
+
+    PLOG((1, "def_var_quantize_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp main
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nsd, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    PLOG((1, "def_var_quantize_handler got parameters ncid = %d "
+          "varid = %d mode = %d nsd = %d ", ncid, varid, mode, nsd));
+
+    /* Call the function. */
+    PIOc_def_var_quantize(ncid, varid, mode, nsd);
+
+
+    PLOG((1, "def_var_quantize_handler succeeded!"));
+    return PIO_NOERR;
+}
+#endif
+#ifdef NC_HAS_ZSTD
+/**
+ * Do an inq_var_bzip2 on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_zstandard_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int *hasfilterp=NULL;
+    int *levelp=NULL;
+    char hasfilterp_present;
+    char levelp_present;
+    int hasfilter;
+    int level;
+    int mpierr;
+
+    assert(ios);
+    PLOG((1, "inq_var_zstandard_handler"));
+
+    /* Get the parameters for this function that the the comp main
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&hasfilterp_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&levelp_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    PLOG((2,"inq_var_zstandard_handler ncid = %d varid = %d hasfilter_present = %d  ",
+          ncid, varid, hasfilterp_present, levelp_present));
+
+    /* Set the non-NULL pointers. */
+    if(hasfilterp_present)
+        hasfilterp = &hasfilter;
+    if(levelp_present)
+        levelp = &level;
+
+    /* Call the inq function to get the values. */
+    PIOc_inq_var_zstandard(ncid, varid, hasfilterp, levelp);
+
+    return PIO_NOERR;
+}
+/**
+ * This function is run on the IO tasks to define a netCDF
+ *  variable quantize level.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ * @author Jim Edwards, Ed Hartnett
+ */
+int def_var_zstandard_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int level;
+    int mpierr;
+
+    PLOG((1, "def_var_zstandard_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp main
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&level, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    PLOG((1, "def_var_zstandard_handler got parameters ncid = %d "
+          "varid = %d level = %d ", ncid, varid, level));
+
+    /* Call the function. */
+    PIOc_def_var_zstandard(ncid, varid, level);
+    
+    PLOG((1, "def_var_zstandard_handler succeeded!"));
+    return PIO_NOERR;
+}
+#endif
 
 /**
  * Do an inq_var_fill on a netCDF variable. This function is only
@@ -2032,8 +2375,10 @@ int open_file_handler(iosystem_desc_t *ios)
     }
     else
     {
+//    PIOc_set_log_level(3);
         PIOc_openfile_retry(ios->iosysid, &ncid, &iotype, filename, mode, 0,
                             use_ext_ncid);
+//    PIOc_set_log_level(0);
     }
 
     return PIO_NOERR;
@@ -2592,12 +2937,12 @@ int finalize_handler(iosystem_desc_t *ios, int index)
  */
 int set_loglevel_handler(iosystem_desc_t *ios)
 {
-#if PIO_ENABLE_LOGGING    
+#if PIO_ENABLE_LOGGING
     int iosysid;
     int level;
     int mpierr;
 #endif
-    
+
     PLOG((0, "set_loglevel_handler called"));
     assert(ios);
 #if PIO_ENABLE_LOGGING
@@ -2609,8 +2954,89 @@ int set_loglevel_handler(iosystem_desc_t *ios)
 #endif
     return PIO_NOERR;
 }
+#ifdef PIO_HAS_PAR_FILTERS
+/**
+ * Do an inq_var_filter_avail on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_filter_avail_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    unsigned int id;
+    int mpierr;
+
+    assert(ios);
+    PLOG((1, "inq_filter_avail_handler"));
+
+    /* Get the parameters for this function that the the comp main
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&id, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    PLOG((2,"inq_filter_avail_handler ncid = %d id = %d",
+          ncid, id));
+
+    /* Call the inq function to get the values. */
+    PIOc_inq_filter_avail(ncid, id);
+
+    return PIO_NOERR;
+}
+/**
+ * This function is run on the IO tasks to define a netCDF
+ *  variable filter.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ * @author Jim Edwards, Ed Hartnett
+ */
+int def_var_filter_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int id;
+    size_t nparams;
+    unsigned int *params;
+    int mpierr;
+
+    PLOG((1, "def_var_filter_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp main
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&id, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nparams, 1, PIO_MPI_SIZE_T, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (!(params = malloc(nparams * sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Bcast(params, nparams, MPI_UNSIGNED, 0, ios->intercomm)))
+        return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
 
+    PLOG((1, "def_var_filter_handler got parameters ncid = %d "
+          "varid = %d id = %d nparams = %d ", ncid, varid, id, nparams));
+
+    /* Call the function. */
+    PIOc_def_var_filter(ncid, varid, id, nparams, params);
+
+    /* Free resources. */
+    free(params);
 
+    PLOG((1, "def_var_filter_handler succeeded!"));
+    return PIO_NOERR;
+}
+#endif
 /**
  * This function is called by the IO tasks.  This function will not
  * return, unless there is an error.
@@ -2682,7 +3108,7 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
         if ((mpierr = MPI_Bcast(&outcount, 1, MPI_INT, 0, io_comm)))
             return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
         PLOG((3, "outcount MPI_Bcast complete outcount = %d", outcount));
-	
+
 	for(int creq=0; creq < outcount; creq++)
 	{
 	  int idx = index[creq];
@@ -2750,6 +3176,14 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
 	    case PIO_MSG_DEF_VAR:
 	      ret = def_var_handler(my_iosys);
 	      break;
+#ifdef NC_HAS_ZSTD
+	    case PIO_MSG_INQ_VAR_ZSTANDARD:
+	      ret = inq_var_zstandard_handler(my_iosys);
+	      break;
+	    case PIO_MSG_DEF_VAR_ZSTANDARD:
+	      ret = def_var_zstandard_handler(my_iosys);
+	      break;
+#endif
 	    case PIO_MSG_DEF_VAR_CHUNKING:
 	      ret = def_var_chunking_handler(my_iosys);
 	      break;
@@ -2852,6 +3286,28 @@ int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
 	    case PIO_MSG_SETLOGLEVEL:
 	      ret = set_loglevel_handler(my_iosys);
 	      break;
+#ifdef NC_HAS_QUANTIZE
+            case PIO_MSG_DEF_VAR_QUANTIZE:
+              ret = def_var_quantize_handler(my_iosys);
+              break;
+            case PIO_MSG_INQ_VAR_QUANTIZE:
+              ret = inq_var_quantize_handler(my_iosys);
+              break;
+#endif
+#ifdef PIO_HAS_PAR_FILTERS
+            case PIO_MSG_DEF_VAR_FILTER:
+              ret = def_var_filter_handler(my_iosys);
+              break;
+            case PIO_MSG_INQ_FILTER_AVAIL:
+              ret = inq_filter_avail_handler(my_iosys);
+              break;
+            case PIO_MSG_INQ_VAR_FILTER_IDS:
+              ret = inq_var_filter_ids_handler(my_iosys);
+              break;
+            case PIO_MSG_INQ_VAR_FILTER_INFO:
+              ret = inq_var_filter_info_handler(my_iosys);
+              break;
+#endif
 	    case PIO_MSG_EXIT:
 	      finalize++;
 	      ret = finalize_handler(my_iosys, idx);
diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index ca85c4acef..747a3d3f26 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -49,7 +49,7 @@ PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
         return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
 
     PLOG((1, "PIOc_def_var_deflate ncid = %d varid = %d shuffle = %d deflate = %d deflate_level = %d",
-	  ncid, varid, shuffle, deflate, deflate_level));
+          ncid, varid, shuffle, deflate, deflate_level));
 
     /* If async is in use, and this is not an IO task, bcast the parameters. */
     if (ios->async)
@@ -83,8 +83,261 @@ PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
     if (ios->ioproc)
     {
 #ifdef _NETCDF4
-	if (file->do_io)
-	    ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
+        if (file->do_io)
+            ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set szip settings for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param option_mask The options mask. Can be PIO_SZIP_EC or PIO_SZIP_NN.
+ * @param pixels_per_block Pixels per block. Must be even and not greater than 32, with typical
+ * values being 8, 10, 16, or 32. This parameter affects compression
+ * ratio; the more pixel values vary, the smaller this number should be
+ * to achieve better performance. If pixels_per_block is bigger than the
+ * total number of elements in a dataset chunk, NC_EINVAL will be
+ * returned.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var_c
+ * @author Jim Edwards, Ed Hartnett
+ */
+int
+PIOc_def_var_szip(int ncid, int varid, int options_mask, int pixels_per_block)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    PLOG((1, "PIOc_def_var_szip ncid = %d varid = %d mask = %d ppb = %d",
+          ncid, varid, options_mask, pixels_per_block));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_SZIP;
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&options_mask, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&pixels_per_block, 1, MPI_INT, ios->compmain, ios->intercomm);
+        }
+
+        /* Handle MPI errors from computation tasks. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_def_var_szip(file->fh, varid, options_mask, pixels_per_block);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+#ifdef  NC_HAS_BZ2
+/**
+ * Set bzip2 settings for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param bzip2_level 1 to 9, with 1 being faster and 9 being more
+ * compressed.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var_c
+ * @author Jim Edwards, Ed Hartnett
+ */
+int
+PIOc_def_var_bzip2(int ncid, int varid, int level)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    PLOG((1, "PIOc_def_var_bzip2 ncid = %d varid = %d level = %d",
+          ncid, varid, level));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_BZIP2;
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&level, 1, MPI_INT, ios->compmain, ios->intercomm);
+        }
+
+        /* Handle MPI errors from computation tasks. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_def_var_bzip2(file->fh, varid, level);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+#endif
+
+#ifdef NC_HAS_ZSTD
+/**
+ * Set zstandard settings for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param zstandard_level 1 to 9, with 1 being faster and 9 being more
+ * compressed.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var_c
+ * @author Jim Edwards, Ed Hartnett
+ */
+int
+PIOc_def_var_zstandard(int ncid, int varid, int level)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    PLOG((1, "PIOc_def_var_zstandard ncid = %d varid = %d level = %d",
+          ncid, varid, level));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_ZSTANDARD;
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&level, 1, MPI_INT, ios->compmain, ios->intercomm);
+        }
+
+        /* Handle MPI errors from computation tasks. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(NULL, file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_def_var_zstandard(file->fh, varid, level);
 #endif
     }
 
@@ -96,6 +349,7 @@ PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
 
     return PIO_NOERR;
 }
+#endif
 
 /**
  * This function only applies to netCDF-4 files. When used with netCDF
@@ -977,7 +1231,7 @@ PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nel
     int ierr;              /* Return code from function calls. */
     int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
 
-    PLOG((1, "PIOc_get_var_chunk_cache ncid = %d varid = %d"));
+    PLOG((1, "PIOc_get_var_chunk_cache ncid = %d varid = %d", ncid, varid));
 
     /* Get the file info. */
     if ((ierr = pio_get_file(ncid, &file)))
@@ -1052,3 +1306,755 @@ PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nel
 
     return PIO_NOERR;
 }
+/* use this variable in the NETCDF library (introduced in v4.9.0) to determine if the following
+   functions are available */
+#ifdef NC_HAS_MULTIFILTERS
+/**
+ * Set the variable filter ids
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ *  See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param id set the filter id.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_filters
+ * @author Jim Edwards/Ed Hartnett
+ */
+int
+PIOc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, unsigned int* params)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    PLOG((1, "PIOc_def_var_filter ncid = %d varid = %d id = %d nparams = %d", ncid, varid, id, nparams));
+#ifdef DEBUG
+    for(i=0; i<nparams; i++)
+        PLOG(1, "  param %d %d\n",i, params[i]);
+#endif
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_FILTER; /* Message for async notification. */
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&id, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nparams, 1, PIO_MPI_SIZE_T, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(params, nparams, MPI_UNSIGNED, ios->compmain, ios->intercomm);
+
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        if (file->do_io)
+            ierr = nc_def_var_filter(file->fh, varid, id, nparams, params);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the variable filter ids if any
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Note that these settings are not part of the data file - they apply
+ * only to the open file as long as it is open.
+ *
+ *  See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param nfiltersp Pointer to the number of filters; may be 0.
+ * @param ids return the filter ids.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_filters
+ * @author Jim Edwards/Ed Hartnett
+ */
+int
+PIOc_inq_var_filter_ids(int ncid, int varid, size_t *nfiltersp, unsigned int *ids)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    PLOG((1, "PIOc_inq_var_filter_ids ncid = %d varid = %d", ncid, varid));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_FILTER_IDS; /* Message for async notification. */
+            char cnt_present = nfiltersp ? true : false;
+            char ids_present = ids ? true : false;
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&cnt_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ids_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+            if(!mpierr && ids_present){
+                size_t idcnt;
+                idcnt = sizeof(ids);
+                mpierr = MPI_Bcast(&idcnt, 1, PIO_MPI_SIZE_T, ios->compmain, ios->intercomm);
+            }
+
+            PLOG((2, "PIOc_inq_var_filter_ids cnt_present = %d ids_present = %d",
+                  cnt_present, ids_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        if (file->do_io)
+          ierr = nc_inq_var_filter_ids(file->fh, varid, nfiltersp, ids);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (nfiltersp && !ierr)
+        if ((mpierr = MPI_Bcast(nfiltersp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if ((*nfiltersp)> 0 && ids && !ierr)
+        if ((mpierr = MPI_Bcast(ids, *nfiltersp, MPI_UNSIGNED, ios->ioroot, ios->my_comm)))
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+#ifdef PIO_HAS_PAR_FILTERS
+/**
+ * Get the variable filter info if any
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Note that these settings are not part of the data file - they apply
+ * only to the open file as long as it is open.
+ *
+ *  See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param id The filter id of interest
+ * @param nparamsp (OUT) Storage which will get the number of parameters to the filter
+ * @param params   (OUT) Storage which will get the associated parameters.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_filters
+ * @author Jim Edwards/Ed Hartnett
+ */
+int
+PIOc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t *nparamsp, unsigned int *params )
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    PLOG((1, "PIOc_inq_var_filter_info ncid = %d varid = %d id=%d", ncid, varid, id));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_FILTER_INFO; /* Message for async notification. */
+            char nparamsp_present = nparamsp ? true : false;
+            char params_present = params ? true : false;
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&id, 1, MPI_UNSIGNED, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nparamsp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&params_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+            if(!mpierr && params_present){
+                size_t paramsize;
+                paramsize = sizeof(params);
+                mpierr = MPI_Bcast(&paramsize, 1, PIO_MPI_SIZE_T, ios->compmain, ios->intercomm);
+            }
+            PLOG((2, "PIOc_inq_var_filter_info nparamsp_present = %d params_present = %d ",
+                  nparamsp_present, params_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        if (file->do_io)
+          ierr = nc_inq_var_filter_info(file->fh, varid, id, nparamsp, params);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (nparamsp && !ierr)
+        if ((mpierr = MPI_Bcast(nparamsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if ((*nparamsp)> 0 && params && !ierr)
+      if ((mpierr = MPI_Bcast(params, *(nparamsp), MPI_UNSIGNED, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+#endif
+#ifdef NC_HAS_BZ2
+/**
+ * Get the variable bzip2 filter info if any
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ *
+ *  See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param hasfilterp (OUT) Pointer that gets a 0 if bzip2 is not in use for this var and a 1 if it is.  Ignored if NULL
+ * @param levelp (OUT) Pointer that gets the level setting (1 - 9) Ignored if NULL
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_filters
+ * @author Jim Edwards/Ed Hartnett
+ */
+int
+PIOc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    PLOG((1, "PIOc_inq_var_bzip2 ncid = %d varid = %d", ncid, varid));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_BZIP2; /* Message for async notification. */
+            char hasfilterp_present = hasfilterp ? true : false;
+            char levelp_present = levelp ? true : false;
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&hasfilterp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&levelp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+            PLOG((2, "PIOc_inq_var_bzip2 hasfilterp_present = %d levelp_present = %d ",
+                  hasfilterp_present, levelp_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        if (file->do_io)
+          ierr = nc_inq_var_bzip2(file->fh, varid, hasfilterp, levelp);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (hasfilterp && !ierr)
+        if ((mpierr = MPI_Bcast(hasfilterp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+
+    if (levelp && !ierr)
+        if ((mpierr = MPI_Bcast(levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+#endif
+#ifdef NC_HAS_ZSTD
+/**
+ * Get the variable zstandard filter info if any
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ *
+ *  See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param hasfilterp (OUT) Pointer that gets a 0 if zstandard is not in use for this var and a 1 if it is.  Ignored if NULL
+ * @param levelp (OUT) Pointer that gets the level setting (1 - 9) Ignored if NULL
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_filters
+ * @author Jim Edwards/Ed Hartnett
+ */
+int
+PIOc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    PLOG((1, "PIOc_inq_var_zstandard ncid = %d varid = %d", ncid, varid));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_ZSTANDARD; /* Message for async notification. */
+            char hasfilterp_present = hasfilterp ? true : false;
+            char levelp_present = levelp ? true : false;
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&hasfilterp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&levelp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+            PLOG((2, "PIOc_inq_var_zstandard hasfilterp_present = %d levelp_present = %d ",
+                  hasfilterp_present, levelp_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        if (file->do_io)
+          ierr = nc_inq_var_zstandard(file->fh, varid, hasfilterp, levelp);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (hasfilterp && !ierr)
+        if ((mpierr = MPI_Bcast(hasfilterp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+
+    if (levelp && !ierr)
+        if ((mpierr = MPI_Bcast(levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+#endif
+#ifdef PIO_HAS_PAR_FILTERS
+/**
+ *
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Note that these settings are not part of the data file - they apply
+ * only to the open file as long as it is open.
+ *
+ *  See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param id   the filter of interest
+ * @return PIO_NOERR if the filter is available, PIO_ENOFILTER if unavailable
+ * @ingroup PIO_filters
+ * @author Jim Edwards/Ed Hartnett
+ */
+int
+PIOc_inq_filter_avail(int ncid, unsigned int id )
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    PLOG((1, "PIOc_inq_filter_avail ncid = %d id = %d ", ncid, id));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_FILTER_AVAIL; /* Message for async notification. */
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&id, 1, MPI_INT, ios->compmain, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        if (file->do_io)
+          ierr = nc_inq_filter_avail(file->fh, id);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr && ierr !=NC_ENOFILTER)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+
+    return ierr;
+}
+#endif
+#endif
+#ifdef NC_HAS_QUANTIZE
+/**
+ * Turn on quantization for a variable
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Note that these settings are not part of the data file - they apply
+ * only to the open file as long as it is open.
+ *
+ *  See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param quantize_mode
+ * @param nsd Number of significant digits.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var_c
+ * @author Jim Edwards/Ed Hartnett
+ */
+int
+PIOc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd )
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    PLOG((1, "PIOc_def_var_quantize ncid = %d varid = %d quantize_mode=%d nsd=%d", ncid, varid, quantize_mode, nsd));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_QUANTIZE; /* Message for async notification. */
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&quantize_mode, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nsd, 1, MPI_INT, ios->compmain, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        if (file->do_io)
+          ierr = nc_def_var_quantize(file->fh, varid, quantize_mode, nsd);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Learn whether quantization is on for a variable, and, if so, the NSD setting.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Note that these settings are not part of the data file - they apply
+ * only to the open file as long as it is open.
+ *
+ *  See the <a
+ * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF
+ * variable documentation</a> for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param quantize_modep Pointer that gets a 0 if quantization is not in use for this var, and a 1 if it is. Ignored if NULL.
+ * @param nsdp Pointer that gets the NSD setting (from 1 to 15), if quantization is in use. Ignored if NULL.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var_c
+ * @author Jim Edwards/Ed Hartnett
+ */
+int
+PIOc_inq_var_quantize(int ncid, int varid, int *quantize_mode, int *nsdp )
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    PLOG((1, "PIOc_inq_var_quantize ncid = %d varid = %d ", ncid, varid));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_QUANTIZE; /* Message for async notification. */
+            char qmode_present = quantize_mode ? true : false;
+            char nsdp_present = nsdp ? true : false;
+
+            if (ios->compmain == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&qmode_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nsdp_present, 1, MPI_CHAR, ios->compmain, ios->intercomm);
+
+            PLOG((2, "PIOc_inq_var_quantize qmode_present = %d nsdp_present = %d ",
+                  qmode_present, nsdp_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        if (file->do_io)
+          ierr = nc_inq_var_quantize(file->fh, varid, quantize_mode, nsdp);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (quantize_mode && !ierr)
+        if ((mpierr = MPI_Bcast(quantize_mode, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+    if (nsdp && !ierr)
+      if ((mpierr = MPI_Bcast(nsdp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+#endif
diff --git a/src/clib/pio_rearrange.c b/src/clib/pio_rearrange.c
index 6a7a677bf5..2aabacb5bd 100644
--- a/src/clib/pio_rearrange.c
+++ b/src/clib/pio_rearrange.c
@@ -11,7 +11,7 @@
 #if PIO_USE_MPISERIAL
 #define MPI_Type_create_hvector MPI_Type_hvector
 #endif
-#define SIZEOF_MPI_OFFSET sizeof(MPI_Offset)
+
 /**
  * Convert a 1-D index into a coordinate value in an arbitrary
  * dimension space. E.g., for index 4 into a array defined as a[3][2],
@@ -35,7 +35,7 @@ idx_to_dim_list(int ndims, const int *gdimlen, PIO_Offset idx,
     /* Check inputs. */
     pioassert(ndims >= 0 && gdimlen && idx >= -1 && dim_list, "invalid input",
               __FILE__, __LINE__);
-    PLOG((2, "idx_to_dim_list ndims = %d idx = %d", ndims, idx));
+    PLOG((3, "idx_to_dim_list ndims = %d idx = %d", ndims, idx));
 
     /* Easiest to start from the right and move left. */
     for (int i = ndims - 1; i >= 0; --i)
@@ -314,7 +314,7 @@ create_mpi_datatypes(MPI_Datatype mpitype, int msgcnt,
     if (mindex)
     {
       for(int j=0; j<numinds; j++)
-        PLOG((3,"mindex[%d] = %d",j,mindex[j]));
+        PLOG((5,"mindex[%d] = %d",j,mindex[j]));
       if (!(lindex = malloc(numinds * sizeof(PIO_Offset))))
             return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
         memcpy(lindex, mindex, (size_t)(numinds * sizeof(PIO_Offset)));
@@ -476,7 +476,7 @@ define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc)
 
                 /* The different rearrangers get different values for mfrom. */
                 int *mfrom = iodesc->rearranger == PIO_REARR_SUBSET ? iodesc->rfrom : NULL;
-                
+
                 /* Create the MPI datatypes. */
                 PLOG((2, "Calling create_mpi_datatypes at line %d ",__LINE__));
                 if ((ret = create_mpi_datatypes(iodesc->mpitype, iodesc->nrecvs, iodesc->rindex,
@@ -510,7 +510,7 @@ define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc)
 
             /* Remember how many types we created for the send side. */
             iodesc->num_stypes = ntypes;
-            
+
             /* Create the MPI data types. */
             PLOG((2, "Calling create_mpi_datatypes at line %d",__LINE__));
             if ((ret = create_mpi_datatypes(iodesc->mpitype, ntypes, iodesc->sindex,
@@ -1113,7 +1113,7 @@ rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
 
     /* Data in sbuf on the ionodes is sent to rbuf on the compute
      * nodes. */
-    
+
     if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts,
                          rdispls, recvtypes, mycomm, &iodesc->rearr_opts.io2comp)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
diff --git a/src/clib/pio_spmd.c b/src/clib/pio_spmd.c
index 090e08ff82..68ee05fd1d 100644
--- a/src/clib/pio_spmd.c
+++ b/src/clib/pio_spmd.c
@@ -364,12 +364,13 @@ int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendty
      * them here. */
     if (steps > 0)
     {
-        PLOG((2, "Waiting for outstanding msgs"));
-        if ((mpierr = MPI_Waitall(steps, rcvids, MPI_STATUSES_IGNORE)))
-            return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
-        if (fc->isend)
-            if ((mpierr = MPI_Waitall(steps, sndids, MPI_STATUSES_IGNORE)))
-                return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+      MPI_Status statuses[steps];
+      PLOG((2, "Waiting for outstanding msgs"));
+      if ((mpierr = MPI_Waitall(steps, rcvids, statuses)))
+	return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
+      if (fc->isend)
+	if ((mpierr = MPI_Waitall(steps, sndids, statuses)))
+	  return check_mpi(NULL, NULL, mpierr, __FILE__, __LINE__);
     }
 
     return PIO_NOERR;
diff --git a/src/clib/pioc.c b/src/clib/pioc.c
index 3b944baae1..1be5923457 100644
--- a/src/clib/pioc.c
+++ b/src/clib/pioc.c
@@ -352,14 +352,8 @@ PIOc_get_local_array_size(int ioid)
 int
 PIOc_Set_IOSystem_Error_Handling(int iosysid, int method)
 {
-    iosystem_desc_t *ios;
     int oldmethod;
 
-    /* Get the iosystem info. */
-    if (iosysid != PIO_DEFAULT)
-        if (!(ios = pio_get_iosystem_from_id(iosysid)))
-            piodie("Could not find IO system.", __FILE__, __LINE__);
-
     /* Set the error handler. */
     if (PIOc_set_iosystem_error_handling(iosysid, method, &oldmethod))
         piodie("Could not set the IOSystem error hanlder", __FILE__, __LINE__);
@@ -1139,7 +1133,7 @@ PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int m
     /* Add 1 to all elements in compmap. */
     for (int e = 0; e < maplen; e++)
     {
-        PLOG((3, "zero-based compmap[%d] = %d", e, compmap[e]));
+        PLOG((5, "zero-based compmap[%d] = %d", e, compmap[e]));
         compmap_1_based[e] = compmap[e] + 1;
     }
 
@@ -1690,7 +1684,6 @@ PIOc_iotype_available(int iotype)
 #ifdef _PNETCDF
     case PIO_IOTYPE_PNETCDF:
         return 1;
-        break;
 #endif
     default:
         return 0;
diff --git a/src/clib/pioc_support.c b/src/clib/pioc_support.c
index 43a36febdd..0cb8ddade3 100644
--- a/src/clib/pioc_support.c
+++ b/src/clib/pioc_support.c
@@ -1685,8 +1685,8 @@ pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **glo
         return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
     PLOG((1, "pioc_read_nc_decomp_int iosysid = %d filename = %s", iosysid, filename));
-    
-    
+
+
     /* Open the netCDF decomp file. */
     if ((ret = PIOc_open(iosysid, filename, NC_WRITE, &ncid)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
@@ -2148,8 +2148,10 @@ PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
         case PIO_IOTYPE_NETCDF:
             if (!ios->io_rank)
             {
+//                PIOc_set_log_level(3);
                 PLOG((2, "Calling nc_create mode = %d", mode));
                 ierr = nc_create(filename, mode, &file->fh);
+                PLOG((2, "Called nc_create mode = %d %d %s", mode, ierr, filename));
             }
             break;
 #ifdef _PNETCDF
@@ -2330,7 +2332,7 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars,
     /* Check inputs. */
     pioassert(rec_var && pio_type && pio_type_size && mpi_type && mpi_type_size,
               "pointers must be provided", __FILE__, __LINE__);
-
+    ret = PIO_NOERR;
     /* How many vars in the file? */
     if (iotype == PIO_IOTYPE_PNETCDF)
     {
@@ -2384,7 +2386,6 @@ inq_file_metadata(file_desc_t *file, int ncid, int iotype, int *nvars,
             return pio_err(NULL, file, ret, __FILE__, __LINE__);
 #endif /* _NETCDF4 */
     }
-
     /* Learn the unlimited dimension ID(s), if there are any. */
     if (nunlimdims)
     {
@@ -2542,7 +2543,6 @@ find_iotype_from_omode(int mode, int *iotype)
         else
             *iotype = PIO_IOTYPE_NETCDF;
     }
-
     return PIO_NOERR;
 }
 
@@ -2623,12 +2623,12 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
     int *mpi_type_size = NULL;
     int *ndims = NULL;
     int mpierr = MPI_SUCCESS, mpierr2;  /** Return code from MPI function codes. */
-    int ierr = PIO_NOERR;      /* Return code from function calls. */
+    int ierr;      /* Return code from function calls. */
 
 #ifdef USE_MPE
     pio_start_mpe_log(OPEN);
 #endif /* USE_MPE */
-
+    ierr = PIO_NOERR;      /* Return code from function calls. */
     /* Get the IO system info from the iosysid. */
     if (!(ios = pio_get_iosystem_from_id(iosysid)))
         return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
@@ -2639,8 +2639,8 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
     if (*iotype < PIO_IOTYPE_PNETCDF || *iotype > PIO_IOTYPE_NETCDF4P)
         return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
 
-    PLOG((2, "PIOc_openfile_retry iosysid = %d iotype = %d filename = %s mode = %d retry = %d",
-          iosysid, *iotype, filename, mode, retry));
+    PLOG((2, "PIOc_openfile_retry iosysid = %d iotype = %d filename = %s mode = %d retry = %d ierr=%d",
+          iosysid, *iotype, filename, mode, retry, ierr));
 
     /* Allocate space for the file info. */
     if (!(file = calloc(sizeof(*file), 1)))
@@ -2693,6 +2693,8 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
         if (mpierr)
             return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     }
+    PLOG((2, "%d: PIOc_openfile_retry ierr=%d",__LINE__,ierr));
+
 
     /* If this is an IO task, then call the netCDF function. */
     if (ios->ioproc)
@@ -2707,9 +2709,11 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
 #else
             imode = mode |  NC_MPIIO;
             if ((ierr = nc_open_par(filename, imode, ios->io_comm, ios->info,
-                                    &file->fh)))
-                break;
+                                    &file->fh))){
+	      PLOG((2, "%d: PIOc_openfile_retry nc_open_par ierr=%d",__LINE__,ierr));
 
+	      break;
+	    }
             /* Check the vars for valid use of unlim dims. */
             if ((ierr = check_unlim_use(file->fh)))
                 break;
@@ -2727,11 +2731,15 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
         case PIO_IOTYPE_NETCDF4C:
             if (ios->io_rank == 0)
             {
-                if ((ierr = nc_open(filename, mode, &file->fh)))
+                if ((ierr = nc_open(filename, mode, &file->fh))){
+                    PLOG((2, "%d: PIOc_openfile_retry ierr=%d filename=%s mode=%d",__LINE__,ierr, filename, mode));
                     break;
+                }
                 /* Check the vars for valid use of unlim dims. */
-                if ((ierr = check_unlim_use(file->fh)))
+                if ((ierr = check_unlim_use(file->fh))){
+                    PLOG((2, "%d: PIOc_openfile_retry ierr=%d",__LINE__,ierr));
                     break;
+                }
                 ierr = inq_file_metadata(file, file->fh, PIO_IOTYPE_NETCDF4C,
                                          &nvars, &rec_var, &pio_type,
                                          &pio_type_size, &mpi_type,
@@ -2739,6 +2747,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
                 PLOG((2, "PIOc_openfile_retry:nc_open for 4C filename = %s mode = %d "
                       "ierr = %d", filename, mode, ierr));
             }
+        PLOG((2, "%d: PIOc_openfile_retry ierr=%d",__LINE__,ierr));
             break;
 #endif /* _NETCDF4 */
 
@@ -2782,13 +2791,15 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
             return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
         }
 
+        PLOG((2, "%d: PIOc_openfile_retry ierr=%d",__LINE__,ierr));
+
         /* If the caller requested a retry, and we failed to open a
            file due to an incompatible type of NetCDF, try it once
            with just plain old basic NetCDF. */
         if (retry)
         {
             PLOG((2, "retry error code ierr = %d io_rank %d", ierr, ios->io_rank));
-            if ((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF))
+            if ((ierr == NC_ENOTNC || ierr == NC_EINVAL || ierr == NC_ENOTBUILT) && (file->iotype != PIO_IOTYPE_NETCDF))
             {
                 if (ios->iomain == MPI_ROOT)
                     printf("PIO2 pio_file.c retry NETCDF\n");
@@ -2821,6 +2832,7 @@ PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
     if (ios->ioroot == ios->union_rank)
         PLOG((2, "Bcasting error code ierr %d ios->ioroot %d ios->my_comm %d",
               ierr, ios->ioroot, ios->my_comm));
+
     if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
         return check_mpi(NULL, file, mpierr, __FILE__, __LINE__);
     PLOG((2, "Bcast openfile_retry error code ierr = %d", ierr));
@@ -3250,7 +3262,7 @@ determine_procs(int num_io_procs, int component_count, int *num_procs_per_comp,
 
 /**
  * Used in check_compmap to sort the compmap in accending order.
- * 
+ *
  * @param a pointer to an offset
  * @param b pointer to another offset
  * @returns 0 if offsets are the same or if either pointer is NULL
@@ -3264,7 +3276,7 @@ int offsetsort(const void *a,const void *b)
 
 /**
  * check_compmap gathers the entire compmap to comp task 0, sorts it into accending order
- * then looks for repeated values > 0.  If any repeated values are found the iodesc is marked 
+ * then looks for repeated values > 0.  If any repeated values are found the iodesc is marked
  * read only.
  *
  * @param ios pointer to the iosystem_desc_t struct.
@@ -3282,7 +3294,7 @@ bool check_compmap(iosystem_desc_t *ios, io_desc_t *iodesc,const PIO_Offset *com
     int ierr;
     bool readonly = 0;
 
-    if(ios->compproc) 
+    if(ios->compproc)
     {
 #ifdef OLDWAY
         int *gmaplen;
@@ -3332,7 +3344,7 @@ bool check_compmap(iosystem_desc_t *ios, io_desc_t *iodesc,const PIO_Offset *com
                 displs[i] = displs[i-1] + gmaplen[i-1];
             }
             gcompmaplen = displs[ios->num_comptasks-1] + gmaplen[ios->num_comptasks-1];
-            gcompmaps = malloc(gcompmaplen * sizeof(PIO_Offset)); 
+            gcompmaps = malloc(gcompmaplen * sizeof(PIO_Offset));
 //            printf("gcompmaplen %d\n",gcompmaplen);
 //            for(int i=0;i<ios->num_comptasks; i++)
 //                printf("gmaplen=%d displs[%d]=%d\n",gmaplen[i], i,displs[i]);
@@ -3359,7 +3371,7 @@ bool check_compmap(iosystem_desc_t *ios, io_desc_t *iodesc,const PIO_Offset *com
             free(displs);
             free(gcompmaps);
         }
-        
+
     }
     MPI_Bcast(&readonly, 1, MPI_CHAR, ios->comproot, ios->my_comm);
 #else
diff --git a/src/flib/pio.F90 b/src/flib/pio.F90
index fabe61a7e9..3a916dd7ce 100644
--- a/src/flib/pio.F90
+++ b/src/flib/pio.F90
@@ -8,6 +8,7 @@
 !! @defgroup PIO_set_blocksize Box Rearranger Settings
 !! Set the box rearranger blocksize in Fortran.
 #include "config.h"
+#include <netcdf_meta.h>
 
 module pio
 ! Package all exposed variables and functions under one roof
@@ -16,6 +17,14 @@ module pio
 
   use pio_kinds, only :  pio_offset_kind
 
+  use pionfatt_mod, only : PIO_put_att   => put_att,        &
+       PIO_get_att   => get_att, &
+       PIO_inq_var_fill => inq_var_fill
+  use pionfput_mod, only : PIO_put_var   => put_var
+  use pionfget_mod, only : PIO_get_var   => get_var
+  use pio_support, only: pio_writedof, pio_readdof, pio_write_nc_dof, pio_read_nc_dof
+  use iso_c_binding
+
   use piolib_mod, only : pio_initdecomp, &
        pio_openfile, pio_closefile, pio_createfile, pio_setdebuglevel, &
        pio_seterrorhandling, pio_setframe, pio_init, pio_get_local_array_size, &
@@ -41,8 +50,11 @@ module pio
        pio_max_name, pio_max_var_dims, pio_rearr_subset, pio_rearr_box, &
        pio_nofill, pio_unlimited, pio_fill_int, pio_fill_double, pio_fill_float, &
        pio_64bit_offset, pio_64bit_data, pio_fill, &
+#ifdef NC_HAS_QUANTIZE
+       PIO_NOQUANTIZE, PIO_QUANTIZE_BITGROOM, PIO_QUANTIZE_GRANULARBR, PIO_QUANTIZE_BITROUND, &
+#endif
+  ! last line of use clause needs to be outside of macro
        pio_internal_error, pio_bcast_error, pio_return_error, pio_default
-
   use piodarray, only : pio_read_darray, pio_write_darray, pio_set_buffer_size_limit
 
   use pio_nf, only:        &
@@ -64,7 +76,16 @@ module pio
        PIO_inq_unlimdim, &
        PIO_def_dim   ,        &
        PIO_def_var   ,        &
-       PIO_def_var_deflate   ,        &
+#ifdef NC_HAS_BZ
+       PIO_def_var_bzip2,     &
+#endif
+#ifdef NC_HAS_ZSTD
+       PIO_def_var_zstandard, &
+#endif
+       PIO_def_var_szip, &
+       PIO_def_var_deflate   ,&
+       PIO_def_var_chunking, &
+       PIO_inq_var_chunking, &
        PIO_redef     ,          &
        PIO_set_log_level,          &
        PIO_inquire_variable , &
@@ -74,15 +95,17 @@ module pio
        PIO_set_var_chunk_cache, &
        PIO_get_var_chunk_cache, &
        PIO_set_fill, &
+#ifdef NC_HAS_QUANTIZE
+       PIO_def_var_quantize     , &
+       PIO_inq_var_quantize     , &
+#endif
+#ifdef PIO_HAS_PAR_FILTERS
+       PIO_inq_var_filter_ids   , &
+       PIO_inq_var_filter_info  , &
+       PIO_inq_filter_avail     , &
+#endif
        PIO_strerror
 
-  use pionfatt_mod, only : PIO_put_att   => put_att,        &
-       PIO_get_att   => get_att, &
-       PIO_inq_var_fill => inq_var_fill
-  use pionfput_mod, only : PIO_put_var   => put_var
-  use pionfget_mod, only : PIO_get_var   => get_var
-  use pio_support, only: pio_writedof, pio_readdof, pio_write_nc_dof, pio_read_nc_dof
-  use iso_c_binding
 
   implicit none
   public
diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index f08ff754cc..909fb668c5 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -1,4 +1,5 @@
 #include "config.h"
+#include <netcdf_meta.h>
 !>
 !! @file
 !! Code to implement the classic netCDF Fortran API in PIO.
@@ -134,6 +135,26 @@ module pio_nf
        pio_redef                                            , &
        pio_set_log_level                                    , &
        pio_strerror                                         , &
+#ifdef NC_HAS_QUANTIZE
+       pio_def_var_quantize                                 , &
+       pio_inq_var_quantize                                 , &
+#endif
+#ifdef NC_HAS_MULTIFILTERS
+#ifdef NC_HAS_BZ
+       pio_inq_var_bzip2                                    , &
+       pio_def_var_bzip2                                    , &
+#endif
+#ifdef NC_HAS_ZSTD
+       pio_inq_var_zstandard                                , &
+       pio_def_var_zstandard                                , &
+#endif
+       pio_def_var_szip                                     , &
+#ifdef PIO_HAS_PAR_FILTERS
+       pio_inq_var_filter_ids                               , &
+       pio_inq_var_filter_info                              , &
+       pio_inq_filter_avail                                 , &
+#endif
+#endif
        pio_set_fill
   !       pio_copy_att    to be done
 
@@ -151,8 +172,44 @@ module pio_nf
   end interface pio_def_var_deflate
   interface pio_def_var_chunking
      module procedure &
-          def_var_chunking_desc
+          def_var_chunking_desc, &
+          def_var_chunking_int, &
+          def_var_chunking_vid
   end interface pio_def_var_chunking
+#ifdef NC_HAS_BZ
+  interface pio_def_var_bzip2
+     module procedure &
+          def_var_bzip2_desc, &
+          def_var_bzip2_int, &
+          def_var_bzip2_vid
+  end interface pio_def_var_bzip2
+  interface pio_inq_var_bzip2
+     module procedure &
+          inq_var_bzip2_desc                                 , &
+          inq_var_bzip2_vid                                  , &
+          inq_var_bzip2_id
+  end interface pio_inq_var_bzip2
+#endif
+  interface pio_def_var_szip
+     module procedure &
+          def_var_szip_desc, &
+          def_var_szip_int, &
+          def_var_szip_vid
+  end interface pio_def_var_szip
+#ifdef NC_HAS_ZSTD
+  interface pio_def_var_zstandard
+     module procedure &
+          def_var_zstandard_desc, &
+          def_var_zstandard_int, &
+          def_var_zstandard_vid
+  end interface pio_def_var_zstandard
+  interface pio_inq_var_zstandard
+     module procedure &
+          inq_var_zstandard_desc                                 , &
+          inq_var_zstandard_vid                                  , &
+          inq_var_zstandard_id
+  end interface pio_inq_var_zstandard
+#endif
   interface pio_inq_attname
      module procedure &
           inq_attname_desc                                  , &
@@ -215,6 +272,7 @@ module pio_nf
           inq_var_deflate_vid                                  , &
           inq_var_deflate_id
   end interface pio_inq_var_deflate
+
   interface pio_inq_var_chunking
      module procedure &
           inq_var_chunking_desc                                 , &
@@ -337,7 +395,35 @@ module pio_nf
           get_var_chunk_cache_desc                          , &
           get_var_chunk_cache_id
   end interface pio_get_var_chunk_cache
-
+#ifdef NC_HAS_QUANTIZE
+  interface pio_def_var_quantize
+     module procedure &
+          def_var_quantize_desc                          , &
+          def_var_quantize_id
+  end interface pio_def_var_quantize
+  interface pio_inq_var_quantize
+     module procedure &
+          inq_var_quantize_desc                          , &
+          inq_var_quantize_id
+  end interface pio_inq_var_quantize
+#endif
+#ifdef PIO_HAS_PAR_FILTERS
+  interface pio_inq_var_filter_ids
+     module procedure &
+          inq_var_filter_ids_desc                          , &
+          inq_var_filter_ids_id
+  end interface pio_inq_var_filter_ids
+  interface pio_inq_var_filter_info
+     module procedure &
+          inq_var_filter_info_desc                          , &
+          inq_var_filter_info_id
+  end interface pio_inq_var_filter_info
+  interface pio_inq_filter_avail
+     module procedure &
+          inq_filter_avail_desc                          , &
+          inq_filter_avail_id
+  end interface pio_inq_filter_avail
+#endif
 contains
 
   !>
@@ -1461,6 +1547,144 @@ end function PIOc_inq_var_chunking
 
   end function inq_var_chunking_id
 
+#ifdef NC_HAS_BZ
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param storage 0 for chunked, 1 for contiguous
+  !! @param chunksizes Array of chunk sizes.
+  !! @retval ierr @copydoc error_return
+  !! @author Ed Hartnett
+  !<
+  integer function inq_var_bzip2_desc(File, vardesc, hasfilter, level) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    type (Var_desc_t), intent(in) :: vardesc
+    logical, intent(out) :: hasfilter
+    integer, intent(out) :: level
+
+    ierr = pio_inq_var_bzip2(File%fh, vardesc%varid, hasfilter, level)
+  end function inq_var_bzip2_desc
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Ed Hartnett
+  !<
+  integer function inq_var_bzip2_vid(File, varid, hasfilter, level) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    integer, intent(in) :: varid
+    logical, intent(out) :: hasfilter
+    integer, intent(out) :: level
+
+    ierr = pio_inq_var_bzip2(File%fh, varid, hasfilter, level)
+  end function inq_var_bzip2_vid
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Ed Hartnett
+  !<
+  integer function inq_var_bzip2_id(ncid, varid, hasfilter, level) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    logical, intent(out) :: hasfilter
+    integer, intent(out) :: level
+
+    integer :: hasfilterp
+    interface
+       integer(C_INT) function PIOc_inq_var_bzip2(ncid, varid, hasfilterp, levelp) &
+            bind(C, name="PIOc_inq_var_bzip2")
+         use iso_c_binding
+         integer(C_INT), value :: ncid
+         integer(C_INT), value :: varid
+         integer(C_INT) :: hasfilterp
+         integer(C_INT) :: levelp
+       end function PIOc_inq_var_bzip2
+    end interface
+
+    ierr = PIOc_inq_var_bzip2(ncid, varid-1, hasfilterp, level)
+    hasfilter = .false.
+    if(hasfilterp .ne. 0) hasfilter = .true.
+
+  end function inq_var_bzip2_id
+#endif
+#ifdef NC_HAS_ZSTD
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !!
+  !! @param File @copydoc file_desc_t
+  !! @param vardesc @copydoc var_desc_t
+  !! @param storage 0 for chunked, 1 for contiguous
+  !! @param chunksizes Array of chunk sizes.
+  !! @retval ierr @copydoc error_return
+  !! @author Ed Hartnett
+  !<
+  integer function inq_var_zstandard_desc(File, vardesc, hasfilter, level) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    type (Var_desc_t), intent(in) :: vardesc
+    logical, intent(out) :: hasfilter
+    integer, intent(out) :: level
+
+    ierr = pio_inq_var_zstandard(File%fh, vardesc%varid, hasfilter, level)
+  end function inq_var_zstandard_desc
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Ed Hartnett
+  !<
+  integer function inq_var_zstandard_vid(File, varid, hasfilter, level) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    integer, intent(in) :: varid
+    logical, intent(out) :: hasfilter
+    integer, intent(out) :: level
+
+    ierr = pio_inq_var_zstandard(File%fh, varid, hasfilter, level)
+  end function inq_var_zstandard_vid
+
+  !>
+  !! @public
+  !! @ingroup PIO_inquire_variable
+  !! Gets metadata information for netcdf file.
+  !! @author Ed Hartnett
+  !<
+  integer function inq_var_zstandard_id(ncid, varid, hasfilter, level) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    logical, intent(out) :: hasfilter
+    integer, intent(out) :: level
+
+    integer :: hasfilterp
+    interface
+       integer(C_INT) function PIOc_inq_var_zstandard(ncid, varid, hasfilterp, levelp) &
+            bind(C, name="PIOc_inq_var_zstandard")
+         use iso_c_binding
+         integer(C_INT), value :: ncid
+         integer(C_INT), value :: varid
+         integer(C_INT) :: hasfilterp
+         integer(C_INT) :: levelp
+       end function PIOc_inq_var_zstandard
+    end interface
+
+    ierr = PIOc_inq_var_zstandard(ncid, varid-1, hasfilterp, level)
+    hasfilter = .false.
+    if(hasfilterp .ne. 0) hasfilter = .true.
+
+  end function inq_var_zstandard_id
+#endif
   !>
   !! @public
   !! @ingroup PIO_inquire_variable
@@ -1931,6 +2155,32 @@ integer function def_var_chunking_desc(file, vardesc, storage, chunksizes) resul
     type (var_desc_t), intent(in) :: vardesc
     integer, intent(in) :: storage
     integer, intent(in) :: chunksizes(:)
+
+    ierr = pio_def_var_chunking(file%fh, vardesc%varid, storage, chunksizes)
+  end function def_var_chunking_desc
+  !>
+  !! @ingroup PIO_def_var_chunking
+  !! Changes chunking settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_chunking_vid(file, varid, storage, chunksizes) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    integer, intent(in) :: varid
+    integer, intent(in) :: storage
+    integer, intent(in) :: chunksizes(:)
+
+    ierr = pio_def_var_chunking(file%fh, varid, storage, chunksizes)
+  end function def_var_chunking_vid
+  !>
+  !! @ingroup PIO_def_var_chunking
+  !! Changes chunking settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_chunking_int(ncid, varid, storage, chunksizes) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(in) :: storage
+    integer, intent(in) :: chunksizes(:)
     integer(kind=PIO_OFFSET_KIND) :: cchunksizes(PIO_MAX_VAR_DIMS)
     integer :: ndims, i
 
@@ -1949,8 +2199,154 @@ end function PIOc_def_var_chunking
        cchunksizes(i) = chunksizes(ndims-i+1)
     enddo
 
-    ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, cchunksizes)
-  end function def_var_chunking_desc
+    ierr = PIOc_def_var_chunking(ncid, varid-1, storage, cchunksizes)
+  end function def_var_chunking_int
+#ifdef NC_HAS_BZ
+  !>
+  !! @ingroup PIO_def_var_bzip2
+  !! Changes bzip2 settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_bzip2_desc(file, vardesc, level) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in) :: vardesc
+    integer, intent(in) :: level
+
+    ierr = pio_def_var_bzip2(file%fh, vardesc%varid, level)
+  end function def_var_bzip2_desc
+  !>
+  !! @ingroup PIO_def_var_bzip2
+  !! Changes chunking settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_bzip2_vid(file, varid, level) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    integer, intent(in) :: varid
+    integer, intent(in) :: level
+
+    ierr = pio_def_var_bzip2(file%fh, varid, level)
+  end function def_var_bzip2_vid
+  !>
+  !! @ingroup PIO_def_var_bzip2
+  !! Changes chunking settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_bzip2_int(ncid, varid, level) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(in) :: level
+
+    interface
+       integer (C_INT) function PIOc_def_var_bzip2(ncid, varid, level) &
+            bind(c,name="PIOc_def_var_bzip2")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int), value :: level
+       end function PIOc_def_var_bzip2
+    end interface
+
+    ierr = PIOc_def_var_bzip2(ncid, varid-1, level)
+  end function def_var_bzip2_int
+#endif
+#ifdef NC_HAS_ZSTD
+  !>
+  !! @ingroup PIO_def_var_zstandard
+  !! Changes chunking settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_zstandard_desc(file, vardesc, level) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in) :: vardesc
+    integer, intent(in) :: level
+
+    ierr = pio_def_var_zstandard(file%fh, vardesc%varid, level)
+  end function def_var_zstandard_desc
+  !>
+  !! @ingroup PIO_def_var_zstandard
+  !! Changes zstandard settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_zstandard_vid(file, varid, level) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    integer, intent(in) :: varid
+    integer, intent(in) :: level
+
+    ierr = pio_def_var_zstandard(file%fh, varid, level)
+  end function def_var_zstandard_vid
+  !>
+  !! @ingroup PIO_def_var_zstandard
+  !! Changes chunking settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_zstandard_int(ncid, varid, level) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(in) :: level
+
+    interface
+       integer (C_INT) function PIOc_def_var_zstandard(ncid, varid, level) &
+            bind(c,name="PIOc_def_var_zstandard")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int), value :: level
+       end function PIOc_def_var_zstandard
+    end interface
+
+    ierr = PIOc_def_var_zstandard(ncid, varid-1, level)
+  end function def_var_zstandard_int
+#endif
+  !>
+  !! @ingroup PIO_def_var_szip
+  !! Changes chunking settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_szip_desc(file, vardesc, mask, ppb) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in) :: vardesc
+    integer, intent(in) :: mask
+    integer, intent(in) :: ppb
+
+    ierr = pio_def_var_szip(file%fh, vardesc%varid, mask, ppb)
+  end function def_var_szip_desc
+  !>
+  !! @ingroup PIO_def_var_szip
+  !! Changes szip settings for a netCDF-4/HDF5 variable.
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function def_var_szip_vid(file, varid, mask, ppb) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    integer, intent(in) :: varid
+    integer, intent(in) :: mask
+    integer, intent(in) :: ppb
+
+    ierr = pio_def_var_szip(file%fh, varid, mask, ppb)
+  end function def_var_szip_vid
+  !>
+  !! @ingroup PIO_def_var_szip
+  !! Changes chunking settings for a netCDF-4/HDF5 variable.
+  !! @author Ed Hartnett
+  !<
+  integer function def_var_szip_int(ncid, varid, mask, ppb) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(in) :: mask
+    integer, intent(in) :: ppb
+
+    interface
+       integer (C_INT) function PIOc_def_var_szip(ncid, varid, options_mask, pixels_per_block) &
+            bind(c,name="PIOc_def_var_szip")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int), value :: options_mask
+         integer(c_int), value :: pixels_per_block
+       end function PIOc_def_var_szip
+    end interface
+
+    ierr = PIOc_def_var_szip(ncid, varid-1, mask, ppb)
+  end function def_var_szip_int
 
   !>
   !! @ingroup PIO_set_chunk_cache
@@ -2105,5 +2501,189 @@ end function PIOc_get_var_chunk_cache
     ierr = PIOc_get_var_chunk_cache(file%fh, varid-1, chunk_cache_size, &
          chunk_cache_nelems, chunk_cache_preemption)
   end function get_var_chunk_cache_id
+#ifdef NC_HAS_QUANTIZE
+  !>
+  !! @ingroup PIO_def_var_quantize
+  !! Set quantize level for a netCDF-4/HDF5 variable
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function def_var_quantize_desc(file, vardesc, quantize_mode, nsd)  result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in)   :: vardesc
+    integer, intent(in)             :: quantize_mode
+    integer, intent(in)             :: nsd
+
+    ierr = def_var_quantize_id(file%fh, vardesc%varid, quantize_mode, nsd)
+  end function def_var_quantize_desc
+  !>
+  !! @ingroup PIO_def_var_quantize
+  !! Set quantize level for a netCDF-4/HDF5 variable.
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function def_var_quantize_id(ncid, varid, quantize_mode , nsd) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(in) :: quantize_mode
+    integer, intent(in) :: nsd
+
+    interface
+       integer (C_INT) function PIOc_def_var_quantize(ncid, varid, quantize_mode, nsd) &
+            bind(c,name="PIOc_def_var_quantize")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int), value :: quantize_mode
+         integer(c_int), value :: nsd
+       end function PIOc_def_var_quantize
+    end interface
+
+    ierr = PIOc_def_var_quantize(ncid, varid-1, quantize_mode, nsd)
+  end function def_var_quantize_id
+  !>
+  !! @ingroup PIO_inq_var_quantize
+  !! Set quantize level for a netCDF-4/HDF5 variable
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function inq_var_quantize_desc(file, vardesc, quantize_mode, nsd)  result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in)   :: vardesc
+    integer, intent(out)             :: quantize_mode
+    integer, intent(out)             :: nsd
+
+    ierr = inq_var_quantize_id(file%fh, vardesc%varid, quantize_mode, nsd)
+  end function inq_var_quantize_desc
+  !>
+  !! @ingroup PIO_inq_var_quantize
+  !! Set quantize level for a netCDF-4/HDF5 variable.
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function inq_var_quantize_id(ncid, varid, quantize_mode , nsd) result(ierr)
+    integer, intent(in)  :: ncid
+    integer, intent(in)  :: varid
+    integer, intent(out) :: quantize_mode
+    integer, intent(out) :: nsd
+
+    interface
+       integer (C_INT) function PIOc_inq_var_quantize(ncid, varid, quantize_mode, nsd) &
+            bind(c,name="PIOc_inq_var_quantize")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int)        :: quantize_mode
+         integer(c_int)        :: nsd
+       end function PIOc_inq_var_quantize
+    end interface
+
+    ierr = PIOc_inq_var_quantize(ncid, varid-1, quantize_mode, nsd)
+  end function inq_var_quantize_id
+#endif
+#ifdef PIO_HAS_PAR_FILTERS
+  !>
+  !! @ingroup PIO_inq_var_filter_ids
+  !! Inquire filter ids for a netCDF-4/HDF5 variable.
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function inq_var_filter_ids_id(ncid, varid, nfilters, filterids) result(ierr)
+    integer, intent(in)  :: ncid
+    integer, intent(in)  :: varid
+    integer, intent(out) :: nfilters
+    integer, intent(out) :: filterids(:)
+
+    interface
+       integer (C_INT) function PIOc_inq_var_filter_ids(ncid, varid, nfiltersp, filterids) &
+            bind(c,name="PIOc_inq_var_filter_ids")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int)        :: nfiltersp
+         integer(c_int)        :: filterids(:)
+       end function PIOc_inq_var_filter_ids
+    end interface
+
+    ierr = PIOc_inq_var_filter_ids(ncid, varid-1, nfilters, filterids)
+  end function inq_var_filter_ids_id
+  !>
+  !! @ingroup PIO_inq_var_filter_ids
+  !! Inquire filter ids for a netCDF-4/HDF5 variable.
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function inq_var_filter_ids_desc(file, vardesc, nfilters, filterids) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in)   :: vardesc
+    integer, intent(out) :: nfilters
+    integer, intent(out) :: filterids(:)
+
+    ierr = inq_var_filter_ids_id(file%fh, vardesc%varid, nfilters, filterids)
+  end function inq_var_filter_ids_desc
+  !>
+  !! @ingroup PIO_inq_var_filter_info
+  !! Inquire filter ids for a netCDF-4/HDF5 variable.
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function inq_var_filter_info_id(ncid, varid, id, params) result(ierr)
+    integer, intent(in)  :: ncid
+    integer, intent(in)  :: varid
+    integer, intent(in)  :: id
+    integer, intent(out) :: params(:)
 
+    interface
+       integer (C_INT) function PIOc_inq_var_filter_info(ncid, varid, id, params) &
+            bind(c,name="PIOc_inq_var_filter_info")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int), value :: id
+         integer(c_int)        :: params(:)
+       end function PIOc_inq_var_filter_info
+    end interface
+
+    ierr = PIOc_inq_var_filter_info(ncid, varid-1, id, params)
+  end function inq_var_filter_info_id
+  !>
+  !! @ingroup PIO_inq_var_filter_info
+  !! Inquire filter ids for a netCDF-4/HDF5 variable.
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function inq_var_filter_info_desc(file, vardesc, id, params) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in)   :: vardesc
+    integer, intent(in) :: id
+    integer, intent(out) :: params(:)
+
+    ierr = inq_var_filter_info_id(file%fh, vardesc%varid, id, params)
+  end function inq_var_filter_info_desc
+#ifdef PIO_HAS_PAR_FILTERS
+  !>
+  !! @ingroup PIO_inq_filter_avail_id
+  !! Inquire filter available for a netCDF-4/HDF5 file.
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function inq_filter_avail_id(ncid, id) result(ierr)
+    integer, intent(in)  :: ncid
+    integer, intent(in)  :: id
+
+    interface
+       integer (C_INT) function PIOc_inq_filter_avail(ncid, id) &
+            bind(c,name="PIOc_inq_filter_avail")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: id
+       end function PIOc_inq_filter_avail
+    end interface
+
+    ierr = PIOc_inq_filter_avail(ncid, id)
+  end function inq_filter_avail_id
+  !>
+  !! @ingroup PIO_inq_var_filter_info
+  !! Inquire filter ids for a netCDF-4/HDF5 variable.
+  !! @author Jim Edwards, Ed Hartnett
+  !<
+  integer function inq_filter_avail_desc(file, id) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    integer, intent(in) :: id
+
+    ierr = inq_filter_avail_id(file%fh, id)
+  end function inq_filter_avail_desc
+#endif
+#endif
 end module pio_nf
diff --git a/src/flib/pio_support.F90 b/src/flib/pio_support.F90
index 95405b83c4..773aa18364 100644
--- a/src/flib/pio_support.F90
+++ b/src/flib/pio_support.F90
@@ -93,9 +93,9 @@ subroutine piodie (file,line, msg, ival1, msg2, ival2, msg3, ival3, mpirank)
     call xl__trbk()
 #endif
 
-    ! passing an argument of 1 to mpi_abort will lead to a STOPALL output 
+    ! passing an argument of 1 to mpi_abort will lead to a STOPALL output
     ! error code of 257
-    call mpi_abort (MPI_COMM_WORLD, 1, ierr)  
+    call mpi_abort (MPI_COMM_WORLD, 1, ierr)
 
 #ifdef CPRNAG
     stop
@@ -162,7 +162,7 @@ integer(c_int) function PIOc_writemap_from_f90(file, ndims, gdims, maplen, map,
          character(C_CHAR), intent(in) :: file
          integer(C_INT), value, intent(in) :: ndims
          integer(C_INT), intent(in) :: gdims(*)
-         integer(C_SIZE_T), value, intent(in) :: maplen 
+         integer(C_SIZE_T), value, intent(in) :: maplen
          integer(C_SIZE_T), intent(in) :: map(*)
          integer(C_INT), value, intent(in) :: f90_comm
        end function PIOc_writemap_from_f90
@@ -195,11 +195,11 @@ subroutine pio_write_nc_dof(ios, filename, cmode, iodesc, ret, title, history, f
     character(len=*), optional :: title
     character(len=*), optional :: history
     logical, optional :: fortran_order
-    
+
     interface
        integer(c_int) function PIOc_write_nc_decomp(iosysid, filename, cmode, &
             ioid, title, history, fortran_order) &
-            bind(C,name="PIOc_write_nc_decomp") 
+            bind(C,name="PIOc_write_nc_decomp")
          use iso_c_binding
          integer(C_INT), value :: iosysid
          character(kind=c_char) :: filename
@@ -213,19 +213,17 @@ end function PIOc_write_nc_decomp
     character(len=:), allocatable :: ctitle, chistory
     integer :: nl
     integer :: forder
-    integer :: i
 
-    
     if(present(title)) then
-       ctitle = trim(title)//C_NULL_CHAR
+       ctitle(1:len_trim(title)+1) = trim(title)//C_NULL_CHAR
     else
-       ctitle = C_NULL_CHAR
+       ctitle(1:1) = C_NULL_CHAR
     endif
 
     if(present(history)) then
-       chistory = trim(history)//C_NULL_CHAR
+       chistory(1:len_trim(history)+1) = trim(history)//C_NULL_CHAR
     else
-       chistory = C_NULL_CHAR
+       chistory(1:1) = C_NULL_CHAR
     endif
 
     if(present(fortran_order)) then
@@ -237,7 +235,6 @@ end function PIOc_write_nc_decomp
     endif
     nl = len_trim(filename)
     ret = PIOc_write_nc_decomp(ios%iosysid, filename(:nl)//C_NULL_CHAR, cmode, iodesc%ioid, ctitle, chistory, forder)
-    
   end subroutine pio_write_nc_dof
 
 
@@ -266,7 +263,7 @@ subroutine pio_readdof (file, ndims, gdims, DOF, comm)
     type(C_PTR) :: tgdims, tmap
     interface
        integer(C_INT) function PIOc_readmap_from_f90(file, ndims, gdims, maplen, map, f90_comm) &
-            bind(C,name="PIOc_readmap_from_f90") 
+            bind(C,name="PIOc_readmap_from_f90")
          use iso_c_binding
          character(C_CHAR), intent(in) :: file
          integer(C_INT), intent(out) :: ndims
@@ -303,11 +300,11 @@ subroutine pio_read_nc_dof(ios, filename, iodesc, ret, title, history, fortran_o
     character(len=*), optional :: title
     character(len=*), optional :: history
     logical, optional :: fortran_order
-    
+
     interface
        integer(c_int) function PIOc_read_nc_decomp(iosysid, filename, ioid, &
             title, history, fortran_order) &
-            bind(C,name="PIOc_read_nc_decomp") 
+            bind(C,name="PIOc_read_nc_decomp")
          use iso_c_binding
          integer(C_INT), value :: iosysid
          character(kind=c_char) :: filename
@@ -317,17 +314,17 @@ integer(c_int) function PIOc_read_nc_decomp(iosysid, filename, ioid, &
          integer(c_int), value :: fortran_order
        end function PIOc_read_nc_decomp
     end interface
-    character(len=:), allocatable :: ctitle, chistory
     integer :: nl
     integer :: forder
 
     nl = len_trim(filename)
+    forder = 0
     ret = PIOc_read_nc_decomp(ios%iosysid, filename(:nl)//C_NULL_CHAR, iodesc%ioid, title, history, forder)
     if(present(fortran_order)) then
        if(forder /= 0) then
           fortran_order = .true.
        else
-          fortran_order = .true.
+          fortran_order = .false.
        endif
     endif
   end subroutine pio_read_nc_dof
diff --git a/src/flib/pio_types.F90 b/src/flib/pio_types.F90
index dad1b49944..7dc88ebec1 100644
--- a/src/flib/pio_types.F90
+++ b/src/flib/pio_types.F90
@@ -1,4 +1,5 @@
 #include "config.h"
+#include <netcdf_meta.h>
 !>
 !! @file
 !! Derived datatypes and constants for PIO Fortran API.
@@ -153,6 +154,26 @@ module pio_types
      enumerator :: PIO_rearr_comm_p2p = 0 !< do point-to-point communications using mpi send and recv calls.
      enumerator :: PIO_rearr_comm_coll    !< use the MPI_ALLTOALLW function of the mpi library
   end enum
+#ifdef NC_HAS_QUANTIZE
+  enum, bind(c)
+     enumerator :: PIO_NOQUANTIZE = 0
+     enumerator :: PIO_QUANTIZE_BITGROOM
+     enumerator :: PIO_QUANTIZE_GRANULARBR
+     enumerator :: PIO_QUANTIZE_BITROUND
+  end enum
+#endif
+#ifdef NC_HAS_MULTIFILTERS
+  enum, bind(c)
+     enumerator :: PIO_FILTER_NONE = 0
+     enumerator :: PIO_FILTER_DEFLATE
+     enumerator :: PIO_FILTER_SHUFFLE
+     enumerator :: PIO_FILTER_FLETCHER32
+     enumerator :: PIO_FILTER_SZIP
+     enumerator :: PIO_FILTER_NBIT
+     enumerator :: PIO_FILTER_SCALEOFFSET
+  end ENUM
+#endif
+
 
   !>
   !! @defgroup PIO_rearr_comm_t Rearranger Communication
@@ -206,6 +227,9 @@ module pio_types
   end type PIO_rearr_opt_t
 
   public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,&
+#ifdef NC_HAS_QUANTIZE
+       PIO_NOQUANTIZE, PIO_QUANTIZE_BITGROOM, PIO_QUANTIZE_GRANULARBR, PIO_QUANTIZE_BITROUND, &
+#endif
        PIO_rearr_comm_fc_2d_enable, PIO_rearr_comm_fc_1d_comp2io,&
        PIO_rearr_comm_fc_1d_io2comp, PIO_rearr_comm_fc_2d_disable
 
diff --git a/src/gptl/ChangeLog b/src/gptl/ChangeLog
index 3d11911e6b..8bbbbcfe4f 100644
--- a/src/gptl/ChangeLog
+++ b/src/gptl/ChangeLog
@@ -2,15 +2,15 @@ timing_120921: Add code for cmake build, should not have any affect otherwise
 timing_120803: Bug fix in setting timing_detail_limit default.
                [Patrick Worley]
 timing_120731: Correction in Makefile for serial build [Jim Edwards]
-timing_120728: Replace process subset optional parameter in t_prf with 
-	       outpe_thispe optional parameter. Change def_perf_outpe_num to 0. 
+timing_120728: Replace process subset optional parameter in t_prf with
+	       outpe_thispe optional parameter. Change def_perf_outpe_num to 0.
 	       [Patrick Worley]
 timing_120717: Retain timestamp on cp in Makefile   [Jim Edwards]
 timing_120710: Correct issue in Makefile [Jim Edwards]
 timing_120709: Change for BGP to measure on compute nodes rather than IO nodes only,
                minor Change in Makefile so that gptl can build seperate from csm_share
                in cesm [Jim Edwards]
-timing_120512: Bug fix in global statistics logic for when a thread has no events 
+timing_120512: Bug fix in global statistics logic for when a thread has no events
 	       to contribute to the merge (mods to gptl.c)
 	       [Patrick Worley]
 timing_120419: Minor changes for mpi-serial compile (jedwards)
@@ -18,7 +18,7 @@ timing_120408: Make HAVE_COMM_F2C default to true. (jedwards)
 timing_120110: Update to GPTL 4.1 source (mods to gptl.c and GPTLprint_memusage)
                [Jim Rosinski (GPTL 4.1), Patrick Worley]
 timing_120109: Bug fix (adding shr_kind_i8 to shr_kind_mod list)
-timing_111205: Update to gptl 4.0 (introducing CESM customizations); 
+timing_111205: Update to gptl 4.0 (introducing CESM customizations);
                support for handles in t_startf/t_stopf;
                support for restricting output to explicitly named process subsets
                [Jim Rosinski (gptl 4.0), Patrick Worley]
@@ -29,7 +29,7 @@ timing_101210: Fix interface to cesm build system, add workaround for xlf bug
 timing_101202: updated get_memusage and print_memusage from GPTL version 3.7; adds
 	       improved support for MacOS and SLASHPROC
 	       [Jim Rosinski, Chuck Bardeen (integrated by P. Worley)]
-timing_091021: update to GPTL version 3.5; rewrite of GPTLpr_summary: much faster, merging 
+timing_091021: update to GPTL version 3.5; rewrite of GPTLpr_summary: much faster, merging
 	       events from all processes and all threads (not just process 0/thread 0);
 	       miscellaneous fixes
 	       [Jim Rosinski (gptl 3.5), Joseph Singh, Patrick Worley]
@@ -39,7 +39,7 @@ timing_090929: added explicit support for the GPTL-native token HAVE_MPI (indica
 timing_081221: restore default assumption that gettimeofday available
 timing_081028: bug fix in include order in gptl_papi.c
 timing_081026: change in output format to make postprocessing simpler
-timing_081024: support for up to one million processes and writing timing files to 
+timing_081024: support for up to one million processes and writing timing files to
 	       subdirectories
 timing_081017: updated to gptl version 3_4_2. Changed some defaults.
                [Jim Rosinski, Patrick Worley]
@@ -57,8 +57,8 @@ timing_071023: updated to gptl version 2.16, added support for output of global
                statistics; removed dependencies on shr and CAM routines; renamed
                gptlutil.c to GPTLutil.c
                [Patrick Worley, Jim Rosinski]
-timing_071019: modified namelist logic to abort if try to set unknown namelist parameters; 
-	       changed default number of reporting processes to 1; 
+timing_071019: modified namelist logic to abort if try to set unknown namelist parameters;
+	       changed default number of reporting processes to 1;
                reversed meaning and changed names of CPP tokens to NO_C99_INLINE and NO_VPRINTF
                [Patrick Worley]
 timing_071010: modified gptl.c to remove the 'inline' specification unless the
@@ -67,75 +67,75 @@ timing_071010: modified gptl.c to remove the 'inline' specification unless the
 timing_070810: added ChangeLog
                updated to latest version of GPTL (from Jim Rosinski)
                modified perf_mod.F90:
-               - added perf_outpe_num and perf_outpe_stride to perf_inparm 
+               - added perf_outpe_num and perf_outpe_stride to perf_inparm
 	         namelist to control which processes output timing data
 	       - added perf_papi_enable to perf_inparm namelist to enable
-	         PAPI counters 
+	         PAPI counters
                - added papi_inparm namelist and papi_ctr1,2,3,4 namelist
 	         parameters to specify PAPI counters
                [Patrick Worley, Jim Rosinski]
-timing_070525: bug fix in gptl.c 
+timing_070525: bug fix in gptl.c
                - unitialized pointer, testing for null pter
                  before traversing
                [Patrick Worley]
 timing_070328: modified perf_mod.F90
 	       - deleted HIDE_MPI cpp token
 	       [Erik Kluzek]
-timing_070327: bug fixes in gptl.c 
-	       - testing for null pters before traversing 
+timing_070327: bug fixes in gptl.c
+	       - testing for null pters before traversing
                  links; added missing type declaration to GPTLallocate for sum
-	       bug fixes in perf_mod.F90 
-	       - fixed OMP-related logic, modified settings reporting, 
+	       bug fixes in perf_mod.F90
+	       - fixed OMP-related logic, modified settings reporting,
 	         modified to work when namelist input is
                  missing; moved timer depth logic back into gptl.c
 	       [Patrick Worley]
-timing_070308: added perf_mod.F90 
-	       - defines all t_xxx entry points - calling gptlxxx directly 
+timing_070308: added perf_mod.F90
+	       - defines all t_xxx entry points - calling gptlxxx directly
 	         and removing all external gptlxxx dependencies,
                  added detail option as an alternative way to disable
 	         event timing, added runtime selection of timing_disable,
 	         perf_timer, timer_depth_limit, timing_detail_limit,
 	         timing_barrier, perf_single_file via namelist parameters
-	       modified f_wrappers.c 
-	       - replaced all t_xxx entry points with gptlxxx entry points, 
+	       modified f_wrappers.c
+	       - replaced all t_xxx entry points with gptlxxx entry points,
 	         added new gptlxxx entry points, deleted _fcd support
-	       modified gptl.c 
+	       modified gptl.c
 	       - deleted DISABLE_TIMERS cpp token, modified GPTLpr call
 	         and logic to move some of support for concatenating timing
 	         output into a single file to perf_mod.F90
-	       modified gptl.h 
-	       - exposed gptlxxx entry points and to add support for choice 
+	       modified gptl.h
+	       - exposed gptlxxx entry points and to add support for choice
 	         of GPTL timer
 	       modified gptl.inc
 	       - removed t_xxx entry points and expose gptlxxx entry points
 	       [Patrick Worley]
-timing_061207: modified gptl.c 
-	       - improved event output ordering 
+timing_061207: modified gptl.c
+	       - improved event output ordering
 	       [Jim Edwards]
-timing_061124: modified gptl.c 
+timing_061124: modified gptl.c
 	       - modified GPTLpr to add option to concatenate
 	         all timing data in a single output file, added GPTL_enable
-	         and GPTL_disable as runtime control of event timing, 
+	         and GPTL_disable as runtime control of event timing,
 	         process 0-only reporting of timing options - unless DEBUG
 	         cpp token defined
-	       modified gptl.h 
+	       modified gptl.h
 	       - redefined GPTLpr parameters
-	       modified f_wrappers.c 
-	       - added t_enablef and t_disablef to call GPTL_enable and 
+	       modified f_wrappers.c
+	       - added t_enablef and t_disablef to call GPTL_enable and
 	         GPTL_disable, added t_pr_onef, added string.h include
-	       bug fix in f_wrappers.c 
+	       bug fix in f_wrappers.c
 	       - changed character string size declaration from int to size_t
-	       bug fix in gptl_papi.c 
+	       bug fix in gptl_papi.c
 	       - modified error message - from Jim Edwards
 	       modified private.h
 	       - increased maximum event name length
 	       [Patrick Worley]
-timing_061028: modified f_wrappers.c 
+timing_061028: modified f_wrappers.c
 	       - deleted dependency on cfort.h
 	       [Patrick Worley]
-timing_060524: modified f_wrappers.c 
-	       - added support for CRAY cpp token and fixed routine 
+timing_060524: modified f_wrappers.c
+	       - added support for CRAY cpp token and fixed routine
 	         type declarations
 	       [Patrick Worley]
-timing_051212: original subversion version 
+timing_051212: original subversion version
 	       - see CAM ChangeLog for earlier history
diff --git a/src/gptl/GPTLget_memusage.c b/src/gptl/GPTLget_memusage.c
index 4ccdef8b2a..4b0d138b2b 100644
--- a/src/gptl/GPTLget_memusage.c
+++ b/src/gptl/GPTLget_memusage.c
@@ -4,7 +4,7 @@
 ** Author: Jim Rosinski
 **   Credit to Chuck Bardeen for MACOS section (__APPLE__ ifdef)
 **
-** get_memusage: 
+** get_memusage:
 **
 **   Designed to be called from Fortran, returns information about memory
 **   usage in each of 5 input int* args.  On Linux read from the /proc
@@ -63,7 +63,7 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
 
   long long total;
   int node_config;
- 
+
  /* memory available */
   Kernel_GetPersonality(&pers, sizeof(pers));
   total = BGP_Personality_DDRSizeMB(&pers);
@@ -116,7 +116,7 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
   ** arguments, close the file and return.
   */
 
-  ret = fscanf (fd, "%d %d %d %d %d %d %d", 
+  ret = fscanf (fd, "%d %d %d %d %d %d %d",
 		size, rss, share, text, datastack, &dum, &dum);
   ret = fclose (fd);
   return 0;
@@ -124,9 +124,9 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
 #elif (defined __APPLE__)
 
   FILE *fd;
-  char cmd[60];  
+  char cmd[60];
   int pid = (int) getpid ();
-  
+
   sprintf (cmd, "ps -o vsz -o rss -o tsiz -p %d | grep -v RSS", pid);
   fd = popen (cmd, "r");
 
@@ -145,7 +145,7 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
 
   if (getrusage (RUSAGE_SELF, &usage) < 0)
     return -1;
-  
+
   *size      = -1;
   *rss       = usage.ru_maxrss;
   *share     = -1;
diff --git a/src/gptl/GPTLprint_memusage.c b/src/gptl/GPTLprint_memusage.c
index 5ab873dccb..a185d61100 100644
--- a/src/gptl/GPTLprint_memusage.c
+++ b/src/gptl/GPTLprint_memusage.c
@@ -30,13 +30,13 @@ int GPTLprint_memusage (const char *str)
   static const int nbytes = 1024*1024*10; /* allocate 10 MB */
   static double blockstomb;               /* convert blocks to MB */
   void *space;                            /* allocated space */
-  
+
   if (GPTLget_memusage (&size, &rss, &share, &text, &datastack) < 0)
     return -1;
 
 #if (defined HAVE_SLASHPROC || defined __APPLE__)
   /*
-  ** Determine size in bytes of memory usage info presented by the OS. Method: allocate a 
+  ** Determine size in bytes of memory usage info presented by the OS. Method: allocate a
   ** known amount of memory and see how much bigger the process becomes.
   */
 
@@ -47,7 +47,7 @@ int GPTLprint_memusage (const char *str)
 	/*
 	** Estimate bytes per block, then refine to nearest power of 2.
 	** The assumption is that the OS presents memory usage info in
-	** units that are a power of 2. 
+	** units that are a power of 2.
 	*/
 	bytesperblock = (int) ((nbytes / (double) (size2 - size)) + 0.5);
 	bytesperblock = nearest_powerof2 (bytesperblock);
@@ -57,19 +57,19 @@ int GPTLprint_memusage (const char *str)
     }
     free (space);
   }
-  
+
   if (bytesperblock > 0)
-    printf ("%s size=%.1f MB rss=%.1f MB share=%.1f MB text=%.1f MB datastack=%.1f MB\n", 
-	    str, size*blockstomb, rss*blockstomb, share*blockstomb, 
+    printf ("%s size=%.1f MB rss=%.1f MB share=%.1f MB text=%.1f MB datastack=%.1f MB\n",
+	    str, size*blockstomb, rss*blockstomb, share*blockstomb,
 	    text*blockstomb, datastack*blockstomb);
   else
-    printf ("%s size=%d rss=%d share=%d text=%d datastack=%d\n", 
+    printf ("%s size=%d rss=%d share=%d text=%d datastack=%d\n",
 	    str, size, rss, share, text, datastack);
 
 #else
 
   /*
-  ** Use max rss as returned by getrusage. If someone knows how to 
+  ** Use max rss as returned by getrusage. If someone knows how to
   ** get the process size under AIX please tell me.
   */
 
@@ -85,7 +85,7 @@ int GPTLprint_memusage (const char *str)
 }
 
 /*
-** nearest_powerof2: 
+** nearest_powerof2:
 **   Determine nearest integer which is a power of 2.
 **   Note: algorithm can't use anything that requires -lm because this is a library,
 **   and we don't want to burden the user with having to add extra libraries to the
@@ -112,7 +112,7 @@ static int nearest_powerof2 (const int val)
 
   delta1 = val - lower;
   delta2 = higher - val;
-  
+
   if (delta1 < delta2)
     return lower;
   else
diff --git a/src/gptl/GPTLutil.c b/src/gptl/GPTLutil.c
index b1c7cf80df..d9a1a93866 100644
--- a/src/gptl/GPTLutil.c
+++ b/src/gptl/GPTLutil.c
@@ -25,10 +25,10 @@ static int max_error = 500;         /* max number of error print msgs */
 int GPTLerror (const char *fmt, ...)
 {
   va_list args;
-  
+
   va_start (args, fmt);
   static int num_error = 0;
-  
+
   if (fmt != NULL && num_error < max_error) {
 #ifndef NO_VPRINTF
     (void) vfprintf (stderr, fmt, args);
@@ -39,10 +39,10 @@ int GPTLerror (const char *fmt, ...)
       (void) fprintf (stderr, "Truncating further error print now after %d msgs",
 		      num_error);
     ++num_error;
-  }    
-  
+  }
+
   va_end (args);
-  
+
   if (abort_on_error)
     exit (-1);
 
@@ -79,4 +79,3 @@ void *GPTLallocate (const int nbytes)
 
   return ptr;
 }
-
diff --git a/src/gptl/README b/src/gptl/README
index 2f0991da21..f8f3f7f7a0 100644
--- a/src/gptl/README
+++ b/src/gptl/README
@@ -18,7 +18,7 @@ Of course these events can only be enabled if the PAPI counters they require
 are available on the target architecture.
 
 
-Using GPTL 
+Using GPTL
 ----------
 
 C codes making GPTL library calls should #include <gptl.h>. Fortran codes
@@ -63,7 +63,7 @@ GPTLfinalize() can be called to clean up the GPTL environment.  All space
 malloc'ed by the GPTL library will be freed by this call.
 
 
-Example 
+Example
 -------
 
 From "man GPTLstart", a simple example calling sequence to time a couple of
@@ -86,7 +86,7 @@ do_work();                              /* do some work */
 (void) GPTLpr (mympitaskid);            /* print the results to timing.<mympitaskid> */
 
 
-Auto-instrumentation 
+Auto-instrumentation
 --------------------
 
 If the regions to be timed are defined by function entry and exit points, and
@@ -128,7 +128,7 @@ Running hex2name.pl converts the function addresses back to human-readable
 function names. It uses the UNIX "nm" utility to do this.
 
 
-Multi-processor instrumented codes 
+Multi-processor instrumented codes
 ----------------------------------
 
 For instrumented codes which make use of threading and/or MPI, a
diff --git a/src/gptl/f_wrappers.c b/src/gptl/f_wrappers.c
index 02f4b75678..b1da29ec4e 100644
--- a/src/gptl/f_wrappers.c
+++ b/src/gptl/f_wrappers.c
@@ -2,7 +2,7 @@
 ** $Id: f_wrappers.c,v 1.56 2010-12-29 18:46:42 rosinski Exp $
 **
 ** Author: Jim Rosinski
-** 
+**
 ** Fortran wrappers for timing library routines
 */
 
@@ -175,12 +175,12 @@ int gptlsetoption (int *option, int *val);
 int gptlenable (void);
 int gptldisable (void);
 int gptlsetutr (int *option);
-int gptlquery (const char *name, int *t, int *count, int *onflg, double *wallclock, 
-		      double *usr, double *sys, long long *papicounters_out, int *maxcounters, 
+int gptlquery (const char *name, int *t, int *count, int *onflg, double *wallclock,
+		      double *usr, double *sys, long long *papicounters_out, int *maxcounters,
 		      int nc);
 int gptlquerycounters (const char *name, int *t, long long *papicounters_out, int nc);
 int gptlget_wallclock (const char *name, int *t, double *value, int nc);
-int gptlget_eventvalue (const char *timername, const char *eventname, int *t, double *value, 
+int gptlget_eventvalue (const char *timername, const char *eventname, int *t, double *value,
 			int nc1, int nc2);
 int gptlget_nregions (int *t, int *nregions);
 int gptlget_regionname (int *t, int *region, char *name, int nc);
@@ -258,7 +258,7 @@ int gptlpr_summary (int *fcomm)
 #endif
 #else
   int ccomm = 0;
-#endif 
+#endif
 
   return GPTLpr_summary (ccomm);
 }
@@ -278,7 +278,7 @@ int gptlpr_summary_file (int *fcomm, char *file, int nc1)
 #endif
 #else
   int ccomm = 0;
-#endif 
+#endif
 
   if ( ! (locfile = (char *) malloc (nc1+1)))
     return GPTLerror ("gptlpr_summary_file: malloc error\n");
@@ -304,7 +304,7 @@ int gptlbarrier (int *fcomm, char *name, int nc1)
 #endif
 #else
   int ccomm = 0;
-#endif 
+#endif
 
   numchars = MIN (nc1, MAX_CHARS);
   strncpy (cname, name, numchars);
@@ -394,8 +394,8 @@ int gptlsetutr (int *option)
   return GPTLsetutr (*option);
 }
 
-int gptlquery (const char *name, int *t, int *count, int *onflg, double *wallclock, 
-	       double *usr, double *sys, long long *papicounters_out, int *maxcounters, 
+int gptlquery (const char *name, int *t, int *count, int *onflg, double *wallclock,
+	       double *usr, double *sys, long long *papicounters_out, int *maxcounters,
 	       int nc)
 {
   char cname[MAX_CHARS+1];
@@ -430,7 +430,7 @@ int gptlget_wallclock (const char *name, int *t, double *value, int nc)
   return GPTLget_wallclock (cname, *t, value);
 }
 
-int gptlget_eventvalue (const char *timername, const char *eventname, int *t, double *value, 
+int gptlget_eventvalue (const char *timername, const char *eventname, int *t, double *value,
 			int nc1, int nc2)
 {
   char ctimername[MAX_CHARS+1];
diff --git a/src/gptl/gptl.c b/src/gptl/gptl.c
index d06d33f088..a038422fc9 100644
--- a/src/gptl/gptl.c
+++ b/src/gptl/gptl.c
@@ -17,7 +17,7 @@
 #include <assert.h>
 
 #ifndef HAVE_C99_INLINE
-#define inline 
+#define inline
 #endif
 
 #ifdef HAVE_PAPI
@@ -134,7 +134,7 @@ static char **timerlist;         /* list of all timers */
 typedef struct {
   int val;                       /* depth in calling tree */
   int padding[31];               /* padding is to mitigate false cache sharing */
-} Nofalse; 
+} Nofalse;
 static Timer ***callstack;       /* call stack */
 static Nofalse *stackidx;        /* index into callstack: */
 
@@ -260,7 +260,7 @@ int GPTLsetoption (const int option,  /* option */
   switch (option) {
   case GPTLcpu:
 #ifdef HAVE_TIMES
-    cpustats.enabled = (bool) val; 
+    cpustats.enabled = (bool) val;
     if (verbose)
       printf ("%s: cpustats = %d\n", thisfunc, val);
 #else
@@ -268,56 +268,56 @@ int GPTLsetoption (const int option,  /* option */
       return GPTLerror ("%s: times() not available\n", thisfunc);
 #endif
     return 0;
-  case GPTLwall:     
-    wallstats.enabled = (bool) val; 
+  case GPTLwall:
+    wallstats.enabled = (bool) val;
     if (verbose)
       printf ("%s: boolean wallstats = %d\n", thisfunc, val);
     return 0;
-  case GPTLoverhead: 
-    overheadstats.enabled = (bool) val; 
+  case GPTLoverhead:
+    overheadstats.enabled = (bool) val;
     if (verbose)
       printf ("%s: boolean overheadstats = %d\n", thisfunc, val);
     return 0;
-  case GPTLdepthlimit: 
-    depthlimit = val; 
+  case GPTLdepthlimit:
+    depthlimit = val;
     if (verbose)
       printf ("%s: depthlimit = %d\n", thisfunc, val);
     return 0;
-  case GPTLverbose: 
-    verbose = (bool) val; 
+  case GPTLverbose:
+    verbose = (bool) val;
 #ifdef HAVE_PAPI
     (void) GPTL_PAPIsetoption (GPTLverbose, val);
 #endif
     if (verbose)
       printf ("%s: boolean verbose = %d\n", thisfunc, val);
     return 0;
-  case GPTLpercent: 
-    percent = (bool) val; 
+  case GPTLpercent:
+    percent = (bool) val;
     if (verbose)
       printf ("%s: boolean percent = %d\n", thisfunc, val);
     return 0;
-  case GPTLdopr_preamble: 
-    dopr_preamble = (bool) val; 
+  case GPTLdopr_preamble:
+    dopr_preamble = (bool) val;
     if (verbose)
       printf ("%s: boolean dopr_preamble = %d\n", thisfunc, val);
     return 0;
-  case GPTLdopr_threadsort: 
-    dopr_threadsort = (bool) val; 
+  case GPTLdopr_threadsort:
+    dopr_threadsort = (bool) val;
     if (verbose)
       printf ("%s: boolean dopr_threadsort = %d\n", thisfunc, val);
     return 0;
-  case GPTLdopr_multparent: 
-    dopr_multparent = (bool) val; 
+  case GPTLdopr_multparent:
+    dopr_multparent = (bool) val;
     if (verbose)
       printf ("%s: boolean dopr_multparent = %d\n", thisfunc, val);
     return 0;
-  case GPTLdopr_collision: 
-    dopr_collision = (bool) val; 
+  case GPTLdopr_collision:
+    dopr_collision = (bool) val;
     if (verbose)
       printf ("%s: boolean dopr_collision = %d\n", thisfunc, val);
     return 0;
   case GPTLprint_method:
-    method = (Method) val; 
+    method = (Method) val;
     if (verbose)
       printf ("%s: print_method = %s\n", thisfunc, methodstr (method));
     return 0;
@@ -338,8 +338,8 @@ int GPTLsetoption (const int option,  /* option */
       printf ("%s: boolean sync_mpi = %d\n", thisfunc, val);
     return 0;
 
-  /* 
-  ** Allow GPTLmultiplex to fall through because it will be handled by 
+  /*
+  ** Allow GPTLmultiplex to fall through because it will be handled by
   ** GPTL_PAPIsetoption()
   */
 
@@ -405,7 +405,7 @@ int GPTLsetutr (const int option)
 ** GPTLinitialize (): Initialization routine must be called from single-threaded
 **   region before any other timing routines may be called.  The need for this
 **   routine could be eliminated if not targetting timing library for threaded
-**   capability. 
+**   capability.
 **
 ** return value: 0 (success) or GPTLerror (failure)
 */
@@ -469,12 +469,12 @@ int GPTLinitialize (void)
     return GPTLerror ("%s: Failure from GPTL_PAPIinitialize\n", thisfunc);
 #endif
 
-  /* 
+  /*
   ** Call init routine for underlying timing routine.
   */
 
   if ((*funclist[funcidx].funcinit)() < 0) {
-    fprintf (stderr, "%s: Failure initializing %s. Reverting underlying timer to %s\n", 
+    fprintf (stderr, "%s: Failure initializing %s. Reverting underlying timer to %s\n",
 	     thisfunc, funclist[funcidx].name, funclist[0].name);
     funcidx = 0;
   }
@@ -620,12 +620,12 @@ int GPTLstart_instr (void *self)
 
   ptr = getentry_instr (hashtable[t], self, &indx);
 
-  /* 
-  ** Recursion => increment depth in recursion and return.  We need to return 
+  /*
+  ** Recursion => increment depth in recursion and return.  We need to return
   ** because we don't want to restart the timer.  We want the reported time for
   ** the timer to reflect the outermost layer of recursion.
   */
-  
+
   if (ptr && ptr->onflg) {
     ++ptr->recurselvl;
     return 0;
@@ -662,7 +662,7 @@ int GPTLstart_instr (void *self)
     return GPTLerror ("%s: update_ptr error\n", thisfunc);
 
   return (0);
-}  
+}
 
 /*
 ** GPTLstart: start a timer
@@ -700,15 +700,15 @@ int GPTLstart (const char *name)               /* timer name */
     return 0;
   }
 
-  /* 
+  /*
   ** ptr will point to the requested timer in the current list,
-  ** or NULL if this is a new entry 
+  ** or NULL if this is a new entry
   */
 
   ptr = getentry (hashtable[t], name, &indx);
 
-  /* 
-  ** Recursion => increment depth in recursion and return.  We need to return 
+  /*
+  ** Recursion => increment depth in recursion and return.  We need to return
   ** because we don't want to restart the timer.  We want the reported time for
   ** the timer to reflect the outermost layer of recursion.
   */
@@ -786,7 +786,7 @@ int GPTLstart_handle (const char *name,  /* timer name */
   }
 
   /*
-  ** If on input, handle references a non-zero value, assume it's a previously returned Timer* 
+  ** If on input, handle references a non-zero value, assume it's a previously returned Timer*
   ** passed in by the user. If zero, generate the hash entry and return it to the user.
   */
 
@@ -795,9 +795,9 @@ int GPTLstart_handle (const char *name,  /* timer name */
   } else {
     ptr = getentry (hashtable[t], name, &indx);
   }
-    
-  /* 
-  ** Recursion => increment depth in recursion and return.  We need to return 
+
+  /*
+  ** Recursion => increment depth in recursion and return.  We need to return
   ** because we don't want to restart the timer.  We want the reported time for
   ** the timer to reflect the outermost layer of recursion.
   */
@@ -869,7 +869,7 @@ static int update_ll_hash (Timer *ptr, const int t, const unsigned int indx)
   last[t] = ptr;
   ++hashtable[t][indx].nument;
   nument = hashtable[t][indx].nument;
-  
+
   eptr = (Timer **) realloc (hashtable[t][indx].entries, nument * sizeof (Timer *));
   if ( ! eptr)
     return GPTLerror ("update_ll_hash: realloc error\n");
@@ -898,7 +898,7 @@ static inline int update_ptr (Timer *ptr, const int t)
 
   if (cpustats.enabled && get_cpustamp (&ptr->cpu.last_utime, &ptr->cpu.last_stime) < 0)
     return GPTLerror ("update_ptr: get_cpustamp error");
-  
+
   if (wallstats.enabled) {
     tp2 = (*ptr2wtimefunc) ();
     ptr->wall.last = tp2;
@@ -922,9 +922,9 @@ static inline int update_ptr (Timer *ptr, const int t)
 ** Return value: 0 (success) or GPTLerror (failure)
 */
 
-static inline int update_parent_info (Timer *ptr, 
-				      Timer **callstackt, 
-				      int stackidxt) 
+static inline int update_parent_info (Timer *ptr,
+				      Timer **callstackt,
+				      int stackidxt)
 {
   int n;             /* loop index through known parents */
   Timer *pptr;       /* pointer to parent in callstack */
@@ -941,7 +941,7 @@ static inline int update_parent_info (Timer *ptr,
 
   callstackt[stackidxt] = ptr;
 
-  /* 
+  /*
   ** If the region has no parent, bump its orphan count
   ** (should never happen since "GPTL_ROOT" added).
   */
@@ -1010,7 +1010,7 @@ int GPTLstop_instr (void *self)
     return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc);
 
   /* Get the timestamp */
-    
+
   if (wallstats.enabled) {
     tp1 = (*ptr2wtimefunc) ();
   }
@@ -1033,7 +1033,7 @@ int GPTLstop_instr (void *self)
 
   ptr = getentry_instr (hashtable[t], self, &indx);
 
-  if ( ! ptr) 
+  if ( ! ptr)
     return GPTLerror ("%s: timer for %p had not been started.\n", thisfunc, self);
 
   if ( ! ptr->onflg )
@@ -1041,7 +1041,7 @@ int GPTLstop_instr (void *self)
 
   ++ptr->count;
 
-  /* 
+  /*
   ** Recursion => decrement depth in recursion and return.  We need to return
   ** because we don't want to stop the timer.  We want the reported time for
   ** the timer to reflect the outermost layer of recursion.
@@ -1085,7 +1085,7 @@ int GPTLstop (const char *name)               /* timer name */
     return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc);
 
   /* Get the timestamp */
-    
+
   if (wallstats.enabled) {
     tp1 = (*ptr2wtimefunc) ();
   }
@@ -1114,7 +1114,7 @@ int GPTLstop (const char *name)               /* timer name */
 
   ++ptr->count;
 
-  /* 
+  /*
   ** Recursion => decrement depth in recursion and return.  We need to return
   ** because we don't want to stop the timer.  We want the reported time for
   ** the timer to reflect the outermost layer of recursion.
@@ -1160,7 +1160,7 @@ int GPTLstop_handle (const char *name,     /* timer name */
     return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc);
 
   /* Get the timestamp */
-    
+
   if (wallstats.enabled) {
     tp1 = (*ptr2wtimefunc) ();
   }
@@ -1182,7 +1182,7 @@ int GPTLstop_handle (const char *name,     /* timer name */
   }
 
   /*
-  ** If on input, handle references a non-zero value, assume it's a previously returned Timer* 
+  ** If on input, handle references a non-zero value, assume it's a previously returned Timer*
   ** passed in by the user. If zero, generate the hash entry and return it to the user.
   */
 
@@ -1198,7 +1198,7 @@ int GPTLstop_handle (const char *name,     /* timer name */
 
   ++ptr->count;
 
-  /* 
+  /*
   ** Recursion => decrement depth in recursion and return.  We need to return
   ** because we don't want to stop the timer.  We want the reported time for
   ** the timer to reflect the outermost layer of recursion.
@@ -1224,7 +1224,7 @@ int GPTLstop_handle (const char *name,     /* timer name */
 }
 
 /*
-** update_stats: update stats inside ptr. Called by GPTLstop, GPTLstop_instr, 
+** update_stats: update stats inside ptr. Called by GPTLstop, GPTLstop_instr,
 **               GPTLstop_handle
 **
 ** Input arguments:
@@ -1237,9 +1237,9 @@ int GPTLstop_handle (const char *name,     /* timer name */
 ** Return value: 0 (success) or GPTLerror (failure)
 */
 
-static inline int update_stats (Timer *ptr, 
-				const double tp1, 
-				const long usr, 
+static inline int update_stats (Timer *ptr,
+				const double tp1,
+				const long usr,
 				const long sys,
 				const int t)
 {
@@ -1375,7 +1375,7 @@ int GPTLreset (void)
   return 0;
 }
 
-/* 
+/*
 ** GPTLpr_set_append: set GPTLpr_file and GPTLpr_summary_file
 ** to use append mode
 */
@@ -1386,20 +1386,20 @@ int GPTLpr_set_append (void)
   return 0;
 }
 
-/* 
+/*
 ** GPTLpr_query_append: query whether GPTLpr_file and GPTLpr_summary_file
 ** use append mode
 */
 
 int GPTLpr_query_append (void)
 {
-  if (pr_append) 
+  if (pr_append)
     return 1;
-  else 
+  else
     return 0;
 }
 
-/* 
+/*
 ** GPTLpr_set_write: set GPTLpr_file and GPTLpr_summary_file
 ** to use write mode
 */
@@ -1410,20 +1410,20 @@ int GPTLpr_set_write (void)
   return 0;
 }
 
-/* 
+/*
 ** GPTLpr_query_write: query whether GPTLpr_file and GPTLpr_summary_file
 ** use write mode
 */
 
 int GPTLpr_query_write (void)
 {
-  if (pr_append) 
+  if (pr_append)
     return 0;
-  else 
+  else
     return 1;
 }
 
-/* 
+/*
 ** GPTLpr: Print values of all timers
 **
 ** Input arguments:
@@ -1448,7 +1448,7 @@ int GPTLpr (const int id)   /* output file will be named "timing.<id>" */
   return 0;
 }
 
-/* 
+/*
 ** GPTLpr_file: Print values of all timers
 **
 ** Input arguments:
@@ -1500,9 +1500,9 @@ int GPTLpr_file (const char *outfile) /* output file to write */
 
   /* 2 is for "/" plus null */
   if (outdir)
-    totlen = strlen (outdir) + strlen (outfile) + 2; 
+    totlen = strlen (outdir) + strlen (outfile) + 2;
   else
-    totlen = strlen (outfile) + 2; 
+    totlen = strlen (outfile) + 2;
 
   outpath = (char *) GPTLallocate (totlen);
 
@@ -1619,11 +1619,11 @@ int GPTLpr_file (const char *outfile) /* output file to write */
   }
 
   sum = (float *) GPTLallocate (nthreads * sizeof (float));
-  
+
   for (t = 0; t < nthreads; ++t) {
 
     /*
-    ** Construct tree for printing timers in parent/child form. get_max_depth() must be called 
+    ** Construct tree for printing timers in parent/child form. get_max_depth() must be called
     ** AFTER construct_tree() because it relies on the per-parent children arrays being complete.
     */
 
@@ -1671,7 +1671,7 @@ int GPTLpr_file (const char *outfile) /* output file to write */
 
     printself_andchildren (timers[t], fp, t, -1, tot_overhead);
 
-    /* 
+    /*
     ** Sum of overhead across timers is meaningful.
     ** Factor of 2 is because there are 2 utr calls per start/stop pair.
     */
@@ -1721,8 +1721,8 @@ int GPTLpr_file (const char *outfile) /* output file to write */
     /* Start at next to skip dummy */
 
     for (ptr = timers[0]->next; ptr; ptr = ptr->next) {
-      
-      /* 
+
+      /*
       ** To print sum stats, first create a new timer then copy thread 0
       ** stats into it. then sum using "add", and finally print.
       */
@@ -1874,7 +1874,7 @@ int GPTLpr_file (const char *outfile) /* output file to write */
     totmem += gptlmem;
     fprintf (fp, "\n");
     fprintf (fp, "Thread %d total memory usage = %g KB\n", t, gptlmem*.001);
-    fprintf (fp, "  Hashmem                   = %g KB\n" 
+    fprintf (fp, "  Hashmem                   = %g KB\n"
 	         "  Regionmem                 = %g KB (papimem portion = %g KB)\n"
 	         "  Parent/child arrays       = %g KB\n",
 	     hashmem*.001, regionmem*.001, papimem*.001, pchmem*.001);
@@ -1892,7 +1892,7 @@ int GPTLpr_file (const char *outfile) /* output file to write */
   return 0;
 }
 
-/* 
+/*
 ** construct_tree: Build the parent->children tree starting with knowledge of
 **                 parent list for each child.
 **
@@ -1944,7 +1944,7 @@ int construct_tree (Timer *timerst, Method method)
       }
       break;
     case GPTLfull_tree:
-      /* 
+      /*
       ** Careful: this one can create *lots* of output!
       */
       for (n = 0; n < ptr->nparent; ++n) {
@@ -1959,7 +1959,7 @@ int construct_tree (Timer *timerst, Method method)
   return 0;
 }
 
-/* 
+/*
 ** methodstr: Return a pointer to a string which represents the method
 **
 ** Input arguments:
@@ -1980,9 +1980,9 @@ static char *methodstr (Method method)
     return "Unknown";
 }
 
-/* 
+/*
 ** newchild: Add an entry to the children list of parent. Use function
-**   is_descendant() to prevent infinite loops. 
+**   is_descendant() to prevent infinite loops.
 **
 ** Input arguments:
 **   parent: parent node
@@ -2017,7 +2017,7 @@ static int newchild (Timer *parent, Timer *child)
   }
 
   /*
-  ** To guarantee no loops, ensure that proposed parent isn't already a descendant of 
+  ** To guarantee no loops, ensure that proposed parent isn't already a descendant of
   ** proposed child
   */
 
@@ -2040,13 +2040,13 @@ static int newchild (Timer *parent, Timer *child)
   return 0;
 }
 
-/* 
+/*
 ** get_max_depth: Determine the maximum call tree depth by traversing the
 **   tree recursively
 **
 ** Input arguments:
 **   ptr:        Starting timer
-**   startdepth: current depth when function invoked 
+**   startdepth: current depth when function invoked
 **
 ** Return value: maximum depth
 */
@@ -2064,7 +2064,7 @@ static int get_max_depth (const Timer *ptr, const int startdepth)
   return maxdepth;
 }
 
-/* 
+/*
 ** num_descendants: Determine the number of descendants of a timer by traversing
 **   the tree recursively. This function is not currently used. It could be
 **   useful in a pruning algorithm
@@ -2086,7 +2086,7 @@ static int num_descendants (Timer *ptr)
   return ptr->num_desc;
 }
 
-/* 
+/*
 ** is_descendant: Determine whether node2 is in the descendant list for
 **   node1
 **
@@ -2114,7 +2114,7 @@ static int is_descendant (const Timer *node1, const Timer *node2)
   return 0;
 }
 
-/* 
+/*
 ** printstats: print a single timer
 **
 ** Input arguments:
@@ -2224,7 +2224,7 @@ static void printstats (const Timer *timer,
   else
     fprintf (fp, "%13.3e ", timer->nbytes / timer->count);
 #endif
-  
+
 #ifdef HAVE_PAPI
   GPTL_PAPIpr (fp, &timer->aux, t, timer->count, timer->wall.accum);
 #endif
@@ -2232,13 +2232,13 @@ static void printstats (const Timer *timer,
   fprintf (fp, "\n");
 }
 
-/* 
-** print_multparentinfo: 
+/*
+** print_multparentinfo:
 **
 ** Input arguments:
 ** Input/output arguments:
 */
-void print_multparentinfo (FILE *fp, 
+void print_multparentinfo (FILE *fp,
 			   Timer *ptr)
 {
   int n;
@@ -2263,7 +2263,7 @@ void print_multparentinfo (FILE *fp,
     fprintf (fp, "%8.1e   %-32s\n\n", (float) ptr->count, ptr->name);
 }
 
-/* 
+/*
 ** add: add the contents of tin to tout
 **
 ** Input arguments:
@@ -2272,14 +2272,14 @@ void print_multparentinfo (FILE *fp,
 **   tout: output timer summed into
 */
 
-static void add (Timer *tout,   
+static void add (Timer *tout,
 		 const Timer *tin)
 {
   tout->count += tin->count;
 
   if (wallstats.enabled) {
     tout->wall.accum += tin->wall.accum;
-    
+
     tout->wall.max = MAX (tout->wall.max, tin->wall.max);
     tout->wall.min = MIN (tout->wall.min, tin->wall.min);
   }
@@ -2293,8 +2293,8 @@ static void add (Timer *tout,
 #endif
 }
 
-/* 
-** GPTLpr_summary: Gather and print summary stats across 
+/*
+** GPTLpr_summary: Gather and print summary stats across
 **                 threads and MPI tasks
 **
 ** Input arguments:
@@ -2315,10 +2315,10 @@ int GPTLpr_summary (int comm)
 }
 
 #ifdef HAVE_MPI
-int GPTLpr_summary_file (MPI_Comm comm, 
+int GPTLpr_summary_file (MPI_Comm comm,
                          const char *outfile)
 #else
-int GPTLpr_summary_file (int comm, 
+int GPTLpr_summary_file (int comm,
                          const char *outfile)
 #endif
 {
@@ -2362,7 +2362,7 @@ int GPTLpr_summary_file (int comm,
     return GPTLerror ("%s: GPTLinitialize() has not been called\n", thisfunc);
 
   /*
-  ** Each process gathers stats for its threads. 
+  ** Each process gathers stats for its threads.
   ** Binary tree used combine results.
   ** Master prints results.
   */
@@ -2411,7 +2411,7 @@ int GPTLpr_summary_file (int comm,
     /* allocate storage for data for all timers */
     if( !( storage = malloc( sizeof(Summarystats) * count ) ) && count )
       return GPTLerror ("%s: memory allocation failed\n", thisfunc);
-  
+
     if ( (ret = collect_data( iam, comm, &count, &storage) ) != 0 )
       return GPTLerror ("%s: master collect_data failed\n", thisfunc);
 
@@ -2526,7 +2526,7 @@ static int merge_thread_data()
 
     /* count timers for thread 0 */
     count_r = 0;
-    for (ptr = timers[0]->next; ptr; ptr = ptr->next) count_r++; 
+    for (ptr = timers[0]->next; ptr; ptr = ptr->next) count_r++;
 
     timerlist    = (char **) GPTLallocate( sizeof (char *));
     if( !( timerlist[0] = (char *)malloc( count_r * length * sizeof (char)) ) && count_r)
@@ -2551,7 +2551,7 @@ static int merge_thread_data()
 
     /* count timers for thread */
     count[t] = 0;
-    for (ptr = timers[t]->next; ptr; ptr = ptr->next) count[t]++; 
+    for (ptr = timers[t]->next; ptr; ptr = ptr->next) count[t]++;
 
     if( count[t] > max_count || max_count == 0 ) max_count = count[t];
 
@@ -2587,24 +2587,24 @@ static int merge_thread_data()
     k = 0;
     n = 0;
     num_newtimers = 0;
-    while( k < count[0] && n < count[t] ) { 
+    while( k < count[0] && n < count[t] ) {
       /* linear comparison of timers */
       compare = strcmp( sort[0][k], sort[t][n]  );
 
-      if( compare == 0 ) { 
+      if( compare == 0 ) {
         /* both have, nothing needs to be done */
         k++;
         n++;
         continue;
       }
 
-      if( compare < 0 ) { 
+      if( compare < 0 ) {
         /* event that only master has, nothing needs to be done */
         k++;
         continue;
       }
 
-      if( compare > 0 ) { 
+      if( compare > 0 ) {
         /* event that only slave thread has, need to add */
         newtimers[num_newtimers] = sort[t][n];
         n++;
@@ -2612,8 +2612,8 @@ static int merge_thread_data()
       }
     }
 
-    while( n < count[t] ) { 
-      /* adds any remaining timers, since we know that all the rest 
+    while( n < count[t] ) {
+      /* adds any remaining timers, since we know that all the rest
          are new since have checked all master thread timers */
       newtimers[num_newtimers] = sort[t][n];
       num_newtimers++;
@@ -2622,7 +2622,7 @@ static int merge_thread_data()
 
     if( num_newtimers ) {
       /* sorts by memory address to restore original order */
-      qsort( newtimers, num_newtimers, sizeof(char*), ncmp ); 
+      qsort( newtimers, num_newtimers, sizeof(char*), ncmp );
 
       /* reallocate memory to hold additional timers */
       if( !( sort[0] = realloc( sort[0], (count[0] + num_newtimers) * sizeof (char *)) ) )
@@ -2631,7 +2631,7 @@ static int merge_thread_data()
         return GPTLerror ("%s: memory reallocation failed\n", thisfunc);
 
       k = count[0];
-      for (n = 0; n < num_newtimers; n++) { 
+      for (n = 0; n < num_newtimers; n++) {
         /* add new found timers */
         memcpy( timerlist[0] + (count[0] + n) *  length, newtimers[n], length * sizeof (char) );
       }
@@ -2639,7 +2639,7 @@ static int merge_thread_data()
       count[0] += num_newtimers;
 
       /* reassign pointers in sort since realloc will have broken them if it moved the memory. */
-      x = 0; 
+      x = 0;
       for (k = 0; k < count[0]; k++) {
         sort[0][k] = timerlist[0] + x;
         x += length;
@@ -2649,7 +2649,7 @@ static int merge_thread_data()
     }
   }
 
-  free(sort[0]); 
+  free(sort[0]);
   /* don't free timerlist[0], since needed for subsequent steps in gathering global statistics */
   for (t = 1; t < nthreads; t++) {
     free(sort[t]);
@@ -2679,14 +2679,14 @@ static int merge_thread_data()
 */
 
 #ifdef HAVE_MPI
-static int collect_data(const int iam, 
+static int collect_data(const int iam,
                         MPI_Comm comm,
-                        int *count, 
+                        int *count,
                         Summarystats **summarystats_cumul )
 #else
-static int collect_data(const int iam, 
+static int collect_data(const int iam,
                         int comm,
-                        int *count, 
+                        int *count,
                         Summarystats **summarystats_cumul )
 #endif
 {
@@ -2809,11 +2809,11 @@ static int collect_data(const int iam,
           {
             compare = strcmp(sort_master[k], sort_slave[n]);
 
-            if (compare == 0) { 
+            if (compare == 0) {
               /* matching timers found */
 
               /* find element number of the name in original timerlist so that it can be matched with its summarystats */
-              m_index = get_index( timerlist[0], sort_master[k] ); 
+              m_index = get_index( timerlist[0], sort_master[k] );
 
               s_index = get_index( timers_slave, sort_slave[n] );
               get_summarystats (&summarystats[m_index], &summarystats_slave[s_index]);
@@ -2822,7 +2822,7 @@ static int collect_data(const int iam,
               continue;
             }
 
-            if (compare > 0) { 
+            if (compare > 0) {
               /* s1 >s2 . slave has event; master does not */
               newtimers[num_newtimers] = sort_slave[n];
               num_newtimers++;
@@ -2834,7 +2834,7 @@ static int collect_data(const int iam,
               k++;
           }
 
-          while (n < count_slave) { 
+          while (n < count_slave) {
             /* add all remaining timers which only the slave has */
             newtimers[num_newtimers] = sort_slave[n];
             num_newtimers++;
@@ -2842,7 +2842,7 @@ static int collect_data(const int iam,
           }
 
           /* sort by memory address to get original order */
-          qsort (newtimers, num_newtimers, sizeof(char*), ncmp); 
+          qsort (newtimers, num_newtimers, sizeof(char*), ncmp);
 
           /* reallocate to hold new timer names and summary stats from slave */
           if (!(timerlist[0] = realloc( timerlist[0], length * (*count + num_newtimers) * sizeof (char) ) ))
@@ -2922,7 +2922,7 @@ static int collect_data(const int iam,
 ** Return value: index of element in list
 */
 
-int get_index( const char * list, 
+int get_index( const char * list,
                const char * element )
 {
   return (( element - list ) / ( MAX_CHARS  + 1 ));
@@ -2957,7 +2957,7 @@ static int ncmp( const void *x, const void *y )
     GPTLerror("%s: shared memory address between timers\n", thisfunc);
 }
 
-/* 
+/*
 ** get_threadstats: gather stats for timer "name" over all threads
 **
 ** Input arguments:
@@ -2967,7 +2967,7 @@ static int ncmp( const void *x, const void *y )
 **   summarystats: max/min stats over all threads
 */
 
-void get_threadstats (const int iam, 
+void get_threadstats (const int iam,
                       const char *name,
 		      Summarystats *summarystats)
 {
@@ -3019,7 +3019,7 @@ void get_threadstats (const int iam,
 	  summarystats->papimax[n]   = value;
 	  summarystats->papimax_t[n] = t;
 	}
-	
+
 	if (value < summarystats->papimin[n] || summarystats->papimin[n] == 0.) {
 	  summarystats->papimin[n]   = value;
 	  summarystats->papimin_t[n] = t;
@@ -3032,7 +3032,7 @@ void get_threadstats (const int iam,
   if ( summarystats->count ) summarystats->processes = 1;
 }
 
-/* 
+/*
 ** get_summarystats: write max/min stats into mpistats based on comparison
 **                   with  summarystats_slave
 **
@@ -3042,7 +3042,7 @@ void get_threadstats (const int iam,
 **   summarystats:       stats (starts out as master stats)
 */
 
-void get_summarystats (Summarystats *summarystats, 
+void get_summarystats (Summarystats *summarystats,
 		       const Summarystats *summarystats_slave)
 {
   if (summarystats_slave->count == 0) return;
@@ -3053,7 +3053,7 @@ void get_summarystats (Summarystats *summarystats,
     summarystats->wallmax_t = summarystats_slave->wallmax_t;
   }
 
-  if ((summarystats_slave->wallmin < summarystats->wallmin) || 
+  if ((summarystats_slave->wallmin < summarystats->wallmin) ||
       (summarystats->count == 0)){
     summarystats->wallmin   = summarystats_slave->wallmin;
     summarystats->wallmin_p = summarystats_slave->wallmin_p;
@@ -3070,7 +3070,7 @@ void get_summarystats (Summarystats *summarystats,
 	summarystats->papimax_t[n] = summarystats_slave->papimax_t[n];
       }
 
-      if ((summarystats_slave->papimin[n] < summarystats->papimin[n]) || 
+      if ((summarystats_slave->papimin[n] < summarystats->papimin[n]) ||
           (summarystats->count == 0)){
 	summarystats->papimin[n]   = summarystats_slave->papimin[n];
 	summarystats->papimin_p[n] = summarystats_slave->papimin_p[n];
@@ -3087,7 +3087,7 @@ void get_summarystats (Summarystats *summarystats,
   summarystats->threads   += summarystats_slave->threads;
 }
 
-/* 
+/*
 ** GPTLbarrier: When MPI enabled, set and time an MPI barrier
 **
 ** Input arguments:
@@ -3140,10 +3140,10 @@ static inline int get_cpustamp (long *usr, long *sys)
 }
 
 /*
-** GPTLquery: return current status info about a timer. If certain stats are not 
+** GPTLquery: return current status info about a timer. If certain stats are not
 ** enabled, they should just have zeros in them. If PAPI is not enabled, input
 ** counter info is ignored.
-** 
+**
 ** Input args:
 **   name:        timer name
 **   maxcounters: max number of PAPI counters to get info for
@@ -3158,7 +3158,7 @@ static inline int get_cpustamp (long *usr, long *sys)
 **   papicounters_out: accumulated PAPI counters
 */
 
-int GPTLquery (const char *name, 
+int GPTLquery (const char *name,
 	       int t,
 	       int *count,
 	       int *onflg,
@@ -3171,14 +3171,14 @@ int GPTLquery (const char *name,
   Timer *ptr;                /* linked list pointer */
   unsigned int indx;         /* linked list index returned from getentry (unused) */
   static const char *thisfunc = "GPTLquery";
-  
+
   if ( ! initialized)
     return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc);
-  
+
   /*
   ** If t is < 0, assume the request is for the current thread
   */
-  
+
   if (t < 0) {
     if ((t = get_thread_num ()) < 0)
       return GPTLerror ("%s: get_thread_num failure\n", thisfunc);
@@ -3186,7 +3186,7 @@ int GPTLquery (const char *name,
     if (t >= maxthreads)
       return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t);
   }
-  
+
   ptr = getentry (hashtable[t], name, &indx);
   if ( !ptr)
     return GPTLerror ("%s: requested timer %s does not have a name hash\n", thisfunc, name);
@@ -3205,7 +3205,7 @@ int GPTLquery (const char *name,
 /*
 ** GPTLquerycounters: return current PAPI counters for a timer.
 ** THIS ROUTINE ID DEPRECATED. USE GPTLget_eventvalue() instead
-** 
+**
 ** Input args:
 **   name: timer name
 **   t:    thread number (if < 0, the request is for the current thread)
@@ -3214,21 +3214,21 @@ int GPTLquery (const char *name,
 **   papicounters_out: accumulated PAPI counters
 */
 
-int GPTLquerycounters (const char *name, 
+int GPTLquerycounters (const char *name,
 		       int t,
 		       long long *papicounters_out)
 {
   Timer *ptr;            /* linked list pointer */
   unsigned int indx;     /* hash index returned from getentry */
   static const char *thisfunc = "GPTLquery_counters";
-  
+
   if ( ! initialized)
     return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc);
-  
+
   /*
   ** If t is < 0, assume the request is for the current thread
   */
-  
+
   if (t < 0) {
     if ((t = get_thread_num ()) < 0)
       return GPTLerror ("%s: get_thread_num failure\n", thisfunc);
@@ -3236,7 +3236,7 @@ int GPTLquerycounters (const char *name,
     if (t >= maxthreads)
       return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t);
   }
-  
+
   ptr = getentry (hashtable[t], name, &indx);
   if ( !ptr)
     return GPTLerror ("%s: requested timer %s does not have a name hash\n", thisfunc, name);
@@ -3250,7 +3250,7 @@ int GPTLquerycounters (const char *name,
 
 /*
 ** GPTLget_wallclock: return wallclock accumulation for a timer.
-** 
+**
 ** Input args:
 **   timername: timer name
 **   t:         thread number (if < 0, the request is for the current thread)
@@ -3267,17 +3267,17 @@ int GPTLget_wallclock (const char *timername,
   Timer *ptr;          /* linked list pointer */
   unsigned int indx;   /* hash index returned from getentry (unused) */
   static const char *thisfunc = "GPTLget_wallclock";
-  
+
   if ( ! initialized)
     return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc);
 
   if ( ! wallstats.enabled)
     return GPTLerror ("%s: wallstats not enabled\n", thisfunc);
-  
+
   /*
   ** If t is < 0, assume the request is for the current thread
   */
-  
+
   if (t < 0) {
     if ((t = get_thread_num ()) < 0)
       return GPTLerror ("%s: bad return from get_thread_num\n", thisfunc);
@@ -3285,9 +3285,9 @@ int GPTLget_wallclock (const char *timername,
     if (t >= maxthreads)
       return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t);
   }
-  
-  /* 
-  ** Don't know whether hashtable entry for timername was generated with 
+
+  /*
+  ** Don't know whether hashtable entry for timername was generated with
   ** *_instr() or not, so try both possibilities
   */
 
@@ -3307,7 +3307,7 @@ int GPTLget_wallclock (const char *timername,
 /*
 ** GPTLget_eventvalue: return PAPI-based event value for a timer. All values will be
 **   returned as doubles, even if the event is not derived.
-** 
+**
 ** Input args:
 **   timername: timer name
 **   eventname: event name (must be currently enabled)
@@ -3326,14 +3326,14 @@ int GPTLget_eventvalue (const char *timername,
   Timer *ptr;          /* linked list pointer */
   unsigned int indx;   /* hash index returned from getentry (unused) */
   static const char *thisfunc = "GPTLget_eventvalue";
-  
+
   if ( ! initialized)
     return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc);
-  
+
   /*
   ** If t is < 0, assume the request is for the current thread
   */
-  
+
   if (t < 0) {
     if ((t = get_thread_num ()) < 0)
       return GPTLerror ("%s: get_thread_num failure\n", thisfunc);
@@ -3341,9 +3341,9 @@ int GPTLget_eventvalue (const char *timername,
     if (t >= maxthreads)
       return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t);
   }
-  
-  /* 
-  ** Don't know whether hashtable entry for timername was generated with 
+
+  /*
+  ** Don't know whether hashtable entry for timername was generated with
   ** *_instr() or not, so try both possibilities
   */
 
@@ -3359,13 +3359,13 @@ int GPTLget_eventvalue (const char *timername,
 #ifdef HAVE_PAPI
   return GPTL_PAPIget_eventvalue (eventname, &ptr->aux, value);
 #else
-  return GPTLerror ("%s: PAPI not enabled\n", thisfunc); 
+  return GPTLerror ("%s: PAPI not enabled\n", thisfunc);
 #endif
 }
 
 /*
 ** GPTLget_nregions: return number of regions (i.e. timer names) for this thread
-** 
+**
 ** Input args:
 **   t:    thread number (if < 0, the request is for the current thread)
 **
@@ -3373,7 +3373,7 @@ int GPTLget_eventvalue (const char *timername,
 **   nregions: number of regions
 */
 
-int GPTLget_nregions (int t, 
+int GPTLget_nregions (int t,
 		      int *nregions)
 {
   Timer *ptr;     /* walk through linked list */
@@ -3381,11 +3381,11 @@ int GPTLget_nregions (int t,
 
   if ( ! initialized)
     return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc);
-  
+
   /*
   ** If t is < 0, assume the request is for the current thread
   */
-  
+
   if (t < 0) {
     if ((t = get_thread_num ()) < 0)
       return GPTLerror ("%s: get_thread_num failure\n", thisfunc);
@@ -3393,9 +3393,9 @@ int GPTLget_nregions (int t,
     if (t >= maxthreads)
       return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t);
   }
-  
+
   *nregions = 0;
-  for (ptr = timers[t]->next; ptr; ptr = ptr->next) 
+  for (ptr = timers[t]->next; ptr; ptr = ptr->next)
     ++*nregions;
 
   return 0;
@@ -3403,7 +3403,7 @@ int GPTLget_nregions (int t,
 
 /*
 ** GPTLget_regionname: return region name for this thread
-** 
+**
 ** Input args:
 **   t:      thread number (if < 0, the request is for the current thread)
 **   region: region number
@@ -3425,11 +3425,11 @@ int GPTLget_regionname (int t,      /* thread number */
 
   if ( ! initialized)
     return GPTLerror ("%s: GPTLinitialize has not been called\n", thisfunc);
-  
+
   /*
   ** If t is < 0, assume the request is for the current thread
   */
-  
+
   if (t < 0) {
     if ((t = get_thread_num ()) < 0)
       return GPTLerror ("%s: get_thread_num failure\n", thisfunc);
@@ -3437,7 +3437,7 @@ int GPTLget_regionname (int t,      /* thread number */
     if (t >= maxthreads)
       return GPTLerror ("%s: requested thread %d is too big\n", thisfunc, t);
   }
-  
+
   ptr = timers[t]->next;
   for (i = 0; i < region; i++) {
     if ( ! ptr)
@@ -3448,7 +3448,7 @@ int GPTLget_regionname (int t,      /* thread number */
   if (ptr) {
     ncpy = MIN (nc, strlen (ptr->name));
     strncpy (name, ptr->name, ncpy);
-    
+
     /*
     ** Adding the \0 is only important when called from C
     */
@@ -3525,7 +3525,7 @@ static inline Timer *getentry (const Hashentry *hashtable, /* hash table */
   const unsigned char *c;     /* pointer to elements of "name" */
   Timer *ptr = 0;             /* return value when entry not found */
 
-  /* 
+  /*
   ** Hash value is sum of: chars times their 1-based position index, modulo tablesize
   */
 
@@ -3537,7 +3537,7 @@ static inline Timer *getentry (const Hashentry *hashtable, /* hash table */
 
   *indx %= tablesize;
 
-  /* 
+  /*
   ** If nument exceeds 1 there was a hash collision and we must search
   ** linearly through an array for a match
   */
@@ -3725,7 +3725,7 @@ static int init_papitime ()
   return GPTLerror ("%s: not enabled\n", thisfunc);
 #endif
 }
-  
+
 static inline double utr_papitime ()
 {
 #ifdef HAVE_PAPI
@@ -3737,8 +3737,8 @@ static inline double utr_papitime ()
 #endif
 }
 
-/* 
-** Probably need to link with -lrt for this one to work 
+/*
+** Probably need to link with -lrt for this one to work
 */
 
 static int init_clock_gettime ()
@@ -3833,7 +3833,7 @@ static inline double utr_gettimeofday ()
 #endif
 }
 
-/* 
+/*
 ** Determine underlying timing routine overhead: call it 1000 times.
 */
 
@@ -3854,7 +3854,7 @@ static double utr_getoverhead ()
 */
 
 static void printself_andchildren (const Timer *ptr,
-				   FILE *fp, 
+				   FILE *fp,
 				   const int t,
 				   const int depth,
 				   const double tot_overhead)
@@ -3870,9 +3870,9 @@ static void printself_andchildren (const Timer *ptr,
 
 #ifdef ENABLE_PMPI
 /*
-** GPTLgetentry: called ONLY from pmpi.c (i.e. not a public entry point). Returns a pointer to the 
+** GPTLgetentry: called ONLY from pmpi.c (i.e. not a public entry point). Returns a pointer to the
 **               requested timer name by calling internal function getentry()
-** 
+**
 ** Return value: 0 (NULL) or the return value of getentry()
 */
 
@@ -3896,7 +3896,7 @@ Timer *GPTLgetentry (const char *name)
 }
 
 /*
-** GPTLpr_file_has_been_called: Called ONLY from pmpi.c (i.e. not a public entry point). Return 
+** GPTLpr_file_has_been_called: Called ONLY from pmpi.c (i.e. not a public entry point). Return
 **                              whether GPTLpr_file has been called. MPI_Finalize wrapper needs
 **                              to know whether it needs to call GPTLpr.
 */
@@ -3919,7 +3919,7 @@ int GPTLpr_has_been_called (void)
 ** $Id: gptl.c,v 1.157 2011-03-28 20:55:18 rosinski Exp $
 **
 ** Author: Jim Rosinski
-** 
+**
 ** Utility functions handle thread-based GPTL needs.
 */
 
@@ -3927,7 +3927,7 @@ int GPTLpr_has_been_called (void)
 #define MAX_THREADS 128
 
 /**********************************************************************************/
-/* 
+/*
 ** 3 sets of routines: OMP threading, PTHREADS, unthreaded
 */
 
@@ -3953,13 +3953,13 @@ static int threadinit (void)
   if (omp_get_thread_num () != 0)
     return GPTLerror ("OMP %s: MUST only be called by the master thread\n", thisfunc);
 
-  /* 
-  ** Allocate the threadid array which maps physical thread IDs to logical IDs 
+  /*
+  ** Allocate the threadid array which maps physical thread IDs to logical IDs
   ** For OpenMP this will be just threadid_omp[iam] = iam;
   */
 
-  if (threadid_omp) 
-    return GPTLerror ("OMP %s: has already been called.\nMaybe mistakenly called by multiple threads?", 
+  if (threadid_omp)
+    return GPTLerror ("OMP %s: has already been called.\nMaybe mistakenly called by multiple threads?",
 		      thisfunc);
 
   maxthreads = MAX ((1), (omp_get_max_threads ()));
@@ -3977,7 +3977,7 @@ static int threadinit (void)
 #ifdef VERBOSE
   printf ("OMP %s: Set maxthreads=%d\n", thisfunc, maxthreads);
 #endif
-  
+
   return 0;
 }
 
@@ -4020,7 +4020,7 @@ static inline int get_thread_num (void)
   if (t == threadid_omp[t])
     return t;
 
-  /* 
+  /*
   ** Thread id not found. Modify threadid_omp with our ID, then start PAPI events if required.
   ** Due to the setting of threadid_omp, everything below here will only execute once per thread.
   */
@@ -4051,7 +4051,7 @@ static inline int get_thread_num (void)
   /*
   ** nthreads = maxthreads based on setting in threadinit
   */
-  
+
   nthreads = maxthreads;
 #ifdef VERBOSE
   printf ("OMP %s: nthreads=%d\n", thisfunc, nthreads);
@@ -4071,7 +4071,7 @@ static void print_threadmapping (FILE *fp)
 }
 
 /**********************************************************************************/
-/* 
+/*
 ** PTHREADS
 */
 
@@ -4098,7 +4098,7 @@ static int threadinit (void)
   static const char *thisfunc = "threadinit";
 
   /*
-  ** The following test is not rock-solid, but it's pretty close in terms of guaranteeing that 
+  ** The following test is not rock-solid, but it's pretty close in terms of guaranteeing that
   ** threadinit gets called by only 1 thread. Problem is, mutex hasn't yet been initialized
   ** so we can't use it.
   */
@@ -4114,7 +4114,7 @@ static int threadinit (void)
   ** Previously, t_mutex = PTHREAD_MUTEX_INITIALIZER on the static declaration line was
   ** adequate to initialize the mutex. But this failed in programs that invoked
   ** GPTLfinalize() followed by GPTLinitialize().
-  ** "man pthread_mutex_init" indicates that passing NULL as the second argument to 
+  ** "man pthread_mutex_init" indicates that passing NULL as the second argument to
   ** pthread_mutex_init() should appropriately initialize the mutex, assuming it was
   ** properly destroyed by a previous call to pthread_mutex_destroy();
   */
@@ -4123,16 +4123,16 @@ static int threadinit (void)
   if ((ret = pthread_mutex_init ((pthread_mutex_t *) &t_mutex, NULL)) != 0)
     return GPTLerror ("PTHREADS %s: mutex init failure: ret=%d\n", thisfunc, ret);
 #endif
-  
-  /* 
-  ** Allocate the threadid array which maps physical thread IDs to logical IDs 
+
+  /*
+  ** Allocate the threadid array which maps physical thread IDs to logical IDs
   */
 
-  if (threadid) 
+  if (threadid)
     return GPTLerror ("PTHREADS %s: threadid not null\n", thisfunc);
   else if ( ! (threadid = (pthread_t *) GPTLallocate (MAX_THREADS * sizeof (pthread_t))))
     return GPTLerror ("PTHREADS %s: malloc failure for %d elements of threadid\n", thisfunc, MAX_THREADS);
-  
+
   maxthreads = MAX_THREADS;
 
   /*
@@ -4177,7 +4177,7 @@ static void threadfinalize ()
 **
 ** Output results:
 **   nthreads: Updated number of threads
-**   threadid: Our thread id added to list on 1st call 
+**   threadid: Our thread id added to list on 1st call
 **
 ** Return value: thread number (success) or GPTLerror (failure)
 */
@@ -4212,7 +4212,7 @@ static inline int get_thread_num (void)
       return t;
 #endif
 
-  /* 
+  /*
   ** Thread id not found. Define a critical region, then start PAPI counters if
   ** necessary and modify threadid[] with our id.
   */
@@ -4236,7 +4236,7 @@ static inline int get_thread_num (void)
   threadid[nthreads] = mythreadid;
 
 #ifdef VERBOSE
-  printf ("PTHREADS %s: 1st call threadid=%lu maps to location %d\n", 
+  printf ("PTHREADS %s: 1st call threadid=%lu maps to location %d\n",
 	  thisfunc, (unsigned long) mythreadid, nthreads);
 #endif
 
@@ -4249,14 +4249,14 @@ static inline int get_thread_num (void)
 
   if (GPTLget_npapievents () > 0) {
 #ifdef VERBOSE
-    printf ("PTHREADS get_thread_num: Starting EventSet threadid=%lu location=%d\n", 
+    printf ("PTHREADS get_thread_num: Starting EventSet threadid=%lu location=%d\n",
 	    (unsigned long) mythreadid, nthreads);
 #endif
     if (GPTLcreate_and_start_events (nthreads) < 0) {
       if (unlock_mutex () < 0)
 	fprintf (stderr, "PTHREADS %s: mutex unlock failure\n", thisfunc);
 
-      return GPTLerror ("PTHREADS %s: error from GPTLcreate_and_start_events for thread %d\n", 
+      return GPTLerror ("PTHREADS %s: error from GPTLcreate_and_start_events for thread %d\n",
 			thisfunc, nthreads);
     }
   }
diff --git a/src/gptl/gptl.inc b/src/gptl/gptl.inc
index 4d9d782a79..2ed2ca5c07 100644
--- a/src/gptl/gptl.inc
+++ b/src/gptl/gptl.inc
@@ -97,7 +97,7 @@
       integer gptlstart_handle
       integer gptlstop
       integer gptlstop_handle
-      integer gptlstamp 
+      integer gptlstamp
       integer gptlpr_set_append
       integer gptlpr_query_append
       integer gptlpr_set_write
@@ -107,7 +107,7 @@
       integer gptlpr_summary
       integer gptlpr_summary_file
       integer gptlbarrier
-      integer gptlreset 
+      integer gptlreset
       integer gptlfinalize
       integer gptlget_memusage
       integer gptlprint_memusage
@@ -130,7 +130,7 @@
       external gptlstart_handle
       external gptlstop
       external gptlstop_handle
-      external gptlstamp 
+      external gptlstamp
       external gptlpr_set_append
       external gptlpr_query_append
       external gptlpr_set_write
@@ -140,7 +140,7 @@
       external gptlpr_summary
       external gptlpr_summary_file
       external gptlbarrier
-      external gptlreset 
+      external gptlreset
       external gptlfinalize
       external gptlget_memusage
       external gptlprint_memusage
diff --git a/src/gptl/gptl_papi.c b/src/gptl/gptl_papi.c
index a8e42fd132..1f701cb897 100644
--- a/src/gptl/gptl_papi.c
+++ b/src/gptl/gptl_papi.c
@@ -5,7 +5,7 @@
 **
 ** Contains routines which interface to PAPI library
 */
- 
+
 #include "private.h"
 #include "gptl.h"
 
@@ -149,8 +149,8 @@ static const Entry derivedtable [] = {
 };
 static const int nderivedentries = sizeof (derivedtable) / sizeof (Entry);
 
-static int npapievents = 0;              /* number of PAPI events: initialize to 0 */ 
-static int nevents = 0;                  /* number of events: initialize to 0 */ 
+static int npapievents = 0;              /* number of PAPI events: initialize to 0 */
+static int nevents = 0;                  /* number of events: initialize to 0 */
 static int *EventSet;                    /* list of events to be counted by PAPI */
 static long_long **papicounters;         /* counters returned from PAPI */
 
@@ -171,11 +171,11 @@ static int enable (int);
 static int getderivedidx (int);
 
 /*
-** GPTL_PAPIsetoption: enable or disable PAPI event defined by "counter". Called 
+** GPTL_PAPIsetoption: enable or disable PAPI event defined by "counter". Called
 **   from GPTLsetoption.  Since all events are off by default, val=false degenerates
 **   to a no-op.  Coded this way to be consistent with the rest of GPTL
 **
-** Input args: 
+** Input args:
 **   counter: PAPI counter
 **   val:     true or false for enable or disable
 **
@@ -219,7 +219,7 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
     break;
   }
 
-  /* 
+  /*
   ** If val is false, return an error if the event has already been enabled.
   ** Otherwise just warn that attempting to disable a PAPI-based event
   ** that has already been enabled doesn't work--for now it's just a no-op
@@ -238,10 +238,10 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
   /* If the event has already been enabled for printing, exit */
 
   if (already_enabled (counter))
-    return GPTLerror ("GPTL_PAPIsetoption: counter %d has already been enabled\n", 
+    return GPTLerror ("GPTL_PAPIsetoption: counter %d has already been enabled\n",
 		      counter);
 
-  /* 
+  /*
   ** Initialize PAPI if it hasn't already been done.
   ** From here on down we can assume the intent is to enable (not disable) an option
   */
@@ -267,7 +267,7 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
     pr_event[nevents].numidx   = enable (PAPI_TOT_INS);
     pr_event[nevents].denomidx = enable (PAPI_TOT_CYC);
     if (verbose)
-      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_TOT_INS / PAPI_TOT_CYC\n", 
+      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_TOT_INS / PAPI_TOT_CYC\n",
 	      pr_event[nevents].event.namestr);
     ++nevents;
     return 0;
@@ -278,18 +278,18 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
       pr_event[nevents].numidx   = enable (PAPI_FP_OPS);
       pr_event[nevents].denomidx = enable (PAPI_LST_INS);
       if (verbose)
-	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_LST_INS\n", 
+	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_LST_INS\n",
 		pr_event[nevents].event.namestr);
     } else if (canenable2 (PAPI_FP_OPS, PAPI_L1_DCA)) {
       pr_event[nevents].event    = derivedtable[idx];
       pr_event[nevents].numidx   = enable (PAPI_FP_OPS);
       pr_event[nevents].denomidx = enable (PAPI_L1_DCA);
 #ifdef DEBUG
-      printf ("GPTL_PAPIsetoption: pr_event %d is derived and will be PAPI event %d / %d\n", 
+      printf ("GPTL_PAPIsetoption: pr_event %d is derived and will be PAPI event %d / %d\n",
 	      nevents, pr_event[nevents].numidx, pr_event[nevents].denomidx);
 #endif
       if (verbose)
-	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_L1_DCA\n", 
+	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_L1_DCA\n",
 		pr_event[nevents].event.namestr);
     } else {
       return GPTLerror ("GPTL_PAPIsetoption: GPTL_CI unavailable\n");
@@ -305,7 +305,7 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
     pr_event[nevents].numidx   = enable (PAPI_FP_OPS);
     pr_event[nevents].denomidx = enable (PAPI_TOT_CYC);
     if (verbose)
-      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_TOT_CYC\n", 
+      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_TOT_CYC\n",
 	      pr_event[nevents].event.namestr);
     ++nevents;
     return 0;
@@ -318,7 +318,7 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
     pr_event[nevents].numidx   = enable (PAPI_FP_OPS);
     pr_event[nevents].denomidx = enable (PAPI_TOT_INS);
     if (verbose)
-      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_TOT_INS\n", 
+      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_FP_OPS / PAPI_TOT_INS\n",
 	      pr_event[nevents].event.namestr);
     ++nevents;
     return 0;
@@ -329,14 +329,14 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
       pr_event[nevents].numidx   = enable (PAPI_LST_INS);
       pr_event[nevents].denomidx = enable (PAPI_TOT_INS);
       if (verbose)
-	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_TOT_INS\n", 
+	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_TOT_INS\n",
 		pr_event[nevents].event.namestr);
     } else if (canenable2 (PAPI_L1_DCA, PAPI_TOT_INS)) {
       pr_event[nevents].event    = derivedtable[idx];
       pr_event[nevents].numidx   = enable (PAPI_L1_DCA);
       pr_event[nevents].denomidx = enable (PAPI_TOT_INS);
       if (verbose)
-	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_TOT_INS\n", 
+	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_TOT_INS\n",
 		pr_event[nevents].event.namestr);
     } else {
       return GPTLerror ("GPTL_PAPIsetoption: GPTL_LSTPI unavailable\n");
@@ -352,7 +352,7 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
     pr_event[nevents].numidx   = enable (PAPI_L1_DCM);
     pr_event[nevents].denomidx = enable (PAPI_L1_DCA);
     if (verbose)
-      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCM / PAPI_L1_DCA\n", 
+      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCM / PAPI_L1_DCA\n",
 	      pr_event[nevents].event.namestr);
     ++nevents;
     return 0;
@@ -363,14 +363,14 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
       pr_event[nevents].numidx   = enable (PAPI_LST_INS);
       pr_event[nevents].denomidx = enable (PAPI_L1_DCM);
       if (verbose)
-	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_L1_DCM\n", 
+	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_L1_DCM\n",
 		pr_event[nevents].event.namestr);
     } else if (canenable2 (PAPI_L1_DCA, PAPI_L1_DCM)) {
       pr_event[nevents].event    = derivedtable[idx];
       pr_event[nevents].numidx   = enable (PAPI_L1_DCA);
       pr_event[nevents].denomidx = enable (PAPI_L1_DCM);
       if (verbose)
-	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_L1_DCM\n", 
+	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_L1_DCM\n",
 		pr_event[nevents].event.namestr);
     } else {
       return GPTLerror ("GPTL_PAPIsetoption: GPTL_LSTPDCM unavailable\n");
@@ -389,7 +389,7 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
     pr_event[nevents].numidx   = enable (PAPI_L2_TCM);
     pr_event[nevents].denomidx = enable (PAPI_L2_TCA);
     if (verbose)
-      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L2_TCM / PAPI_L2_TCA\n", 
+      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L2_TCM / PAPI_L2_TCA\n",
 	      pr_event[nevents].event.namestr);
     ++nevents;
     return 0;
@@ -400,14 +400,14 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
       pr_event[nevents].numidx   = enable (PAPI_LST_INS);
       pr_event[nevents].denomidx = enable (PAPI_L2_TCM);
       if (verbose)
-	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_L2_TCM\n", 
+	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_LST_INS / PAPI_L2_TCM\n",
 		pr_event[nevents].event.namestr);
     } else if (canenable2 (PAPI_L1_DCA, PAPI_L2_TCM)) {
       pr_event[nevents].event    = derivedtable[idx];
       pr_event[nevents].numidx   = enable (PAPI_L1_DCA);
       pr_event[nevents].denomidx = enable (PAPI_L2_TCM);
       if (verbose)
-	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_L2_TCM\n", 
+	printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L1_DCA / PAPI_L2_TCM\n",
 		pr_event[nevents].event.namestr);
     } else {
       return GPTLerror ("GPTL_PAPIsetoption: GPTL_LSTPL2M unavailable\n");
@@ -423,7 +423,7 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
     pr_event[nevents].numidx   = enable (PAPI_L3_TCM);
     pr_event[nevents].denomidx = enable (PAPI_L3_TCR);
     if (verbose)
-      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L3_TCM / PAPI_L3_TCR\n", 
+      printf ("GPTL_PAPIsetoption: enabling derived event %s = PAPI_L3_TCM / PAPI_L3_TCR\n",
 	      pr_event[nevents].event.namestr);
     ++nevents;
     return 0;
@@ -444,11 +444,11 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
 	pr_event[nevents].numidx = enable (counter);
 	pr_event[nevents].denomidx = -1;     /* flag says not derived (no denominator) */
       } else {
-	return GPTLerror ("GPTL_PAPIsetoption: Can't enable event \n", 
+	return GPTLerror ("GPTL_PAPIsetoption: Can't enable event \n",
 			  papitable[n].longstr);
       }
       if (verbose)
-	printf ("GPTL_PAPIsetoption: enabling PAPI preset event %s\n", 
+	printf ("GPTL_PAPIsetoption: enabling PAPI preset event %s\n",
 		pr_event[nevents].event.namestr);
       ++nevents;
       return 0;
@@ -458,9 +458,9 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
   /*
   ** Check native events last: If PAPI_event_code_to_name fails, give up
   */
-  
+
   if ((ret = PAPI_event_code_to_name (counter, eventname)) != PAPI_OK)
-    return GPTLerror ("GPTL_PAPIsetoption: name not found for counter %d: PAPI_strerror: %s\n", 
+    return GPTLerror ("GPTL_PAPIsetoption: name not found for counter %d: PAPI_strerror: %s\n",
 		      counter, PAPI_strerror (ret));
 
   /*
@@ -514,12 +514,12 @@ int GPTL_PAPIsetoption (const int counter,  /* PAPI counter (or option) */
 /*
 ** canenable: determine whether a PAPI counter can be enabled
 **
-** Input args: 
+** Input args:
 **   counter: PAPI counter
 **
 ** Return value: 0 (success) or non-zero (failure)
 */
- 
+
 int canenable (int counter)
 {
   char eventname[PAPI_MAX_STR_LEN]; /* returned from PAPI_event_code_to_name */
@@ -539,13 +539,13 @@ int canenable (int counter)
 /*
 ** canenable2: determine whether 2 PAPI counters can be enabled
 **
-** Input args: 
+** Input args:
 **   counter1: PAPI counter
 **   counter2: PAPI counter
 **
 ** Return value: 0 (success) or non-zero (failure)
 */
- 
+
 int canenable2 (int counter1, int counter2)
 {
   char eventname[PAPI_MAX_STR_LEN]; /* returned from PAPI_event_code_to_name */
@@ -573,12 +573,12 @@ int canenable2 (int counter1, int counter2)
 **   well as output directly. E.g. PAPI_FP_OPS is used to compute
 **   computational intensity, and floating point ops per instruction.
 **
-** Input args: 
+** Input args:
 **   counter: PAPI counter
 **
 ** Return value: index into papieventlist (success) or negative (not found)
 */
- 
+
 int papievent_is_enabled (int counter)
 {
   int n;
@@ -591,14 +591,14 @@ int papievent_is_enabled (int counter)
 
 /*
 ** already_enabled: determine whether a PAPI-based event has already been
-**   enabled for printing. 
+**   enabled for printing.
 **
-** Input args: 
+** Input args:
 **   counter: PAPI or derived counter
 **
 ** Return value: 1 (true) or 0 (false)
 */
- 
+
 int already_enabled (int counter)
 {
   int n;
@@ -613,12 +613,12 @@ int already_enabled (int counter)
 ** enable: enable a PAPI event. ASSUMES that canenable() has already determined
 **   that the event can be enabled.
 **
-** Input args: 
+** Input args:
 **   counter: PAPI counter
 **
 ** Return value: index into papieventlist
 */
- 
+
 int enable (int counter)
 {
   int n;
@@ -643,7 +643,7 @@ int enable (int counter)
 /*
 ** getderivedidx: find the table index of a derived counter
 **
-** Input args: 
+** Input args:
 **   counter: derived counter
 **
 ** Return value: index into derivedtable (success) or GPTLerror (failure)
@@ -672,7 +672,7 @@ int GPTL_PAPIlibraryinit ()
 
   if ((ret = PAPI_is_initialized ()) == PAPI_NOT_INITED) {
     if ((ret = PAPI_library_init (PAPI_VER_CURRENT)) != PAPI_VER_CURRENT) {
-      fprintf (stderr, "GPTL_PAPIlibraryinit: ret=%d PAPI_VER_CURRENT=%d\n", 
+      fprintf (stderr, "GPTL_PAPIlibraryinit: ret=%d PAPI_VER_CURRENT=%d\n",
 	       ret, (int) PAPI_VER_CURRENT);
       return GPTLerror ("GPTL_PAPIlibraryinit: PAPI_library_init failure:%s\n",
 			PAPI_strerror (ret));
@@ -683,16 +683,16 @@ int GPTL_PAPIlibraryinit ()
 
 /*
 ** GPTL_PAPIinitialize(): Initialize the PAPI interface. Called from GPTLinitialize.
-**   PAPI_library_init must be called before any other PAPI routines.  
+**   PAPI_library_init must be called before any other PAPI routines.
 **   PAPI_thread_init is called subsequently if threading is enabled.
 **   Finally, allocate space for PAPI counters and start them.
 **
-** Input args: 
+** Input args:
 **   maxthreads: number of threads
 **
 ** Return value: 0 (success) or GPTLerror or -1 (failure)
 */
- 
+
 int GPTL_PAPIinitialize (const int maxthreads,     /* number of threads */
 			 const bool verbose_flag,  /* output verbosity */
 			 int *nevents_out,         /* nevents needed by gptl.c */
@@ -748,8 +748,8 @@ int GPTL_PAPIinitialize (const int maxthreads,     /* number of threads */
 **   Threaded routine to create the "event set" (PAPI terminology) and start
 **   the counters. This is only done once, and is called from get_thread_num
 **   for the first time for the thread.
-** 
-** Input args: 
+**
+** Input args:
 **   t: thread number
 **
 ** Return value: 0 (success) or GPTLerror (failure)
@@ -764,7 +764,7 @@ int GPTLcreate_and_start_events (const int t)  /* thread number */
   /* Create the event set */
 
   if ((ret = PAPI_create_eventset (&EventSet[t])) != PAPI_OK)
-    return GPTLerror ("GPTLcreate_and_start_events: thread %d failure creating eventset: %s\n", 
+    return GPTLerror ("GPTLcreate_and_start_events: thread %d failure creating eventset: %s\n",
 		      t, PAPI_strerror (ret));
 
   if (verbose)
@@ -797,20 +797,20 @@ int GPTLcreate_and_start_events (const int t)  /* thread number */
 
     if ((ret = PAPI_cleanup_eventset (EventSet[t])) != PAPI_OK)
       return GPTLerror ("GPTLcreate_and_start_events: %s\n", PAPI_strerror (ret));
-    
+
     if ((ret = PAPI_destroy_eventset (&EventSet[t])) != PAPI_OK)
       return GPTLerror ("GPTLcreate_and_start_events: %s\n", PAPI_strerror (ret));
 
     if ((ret = PAPI_create_eventset (&EventSet[t])) != PAPI_OK)
-      return GPTLerror ("GPTLcreate_and_start_events: failure creating eventset: %s\n", 
+      return GPTLerror ("GPTLcreate_and_start_events: failure creating eventset: %s\n",
 			PAPI_strerror (ret));
 
     if ((ret = PAPI_multiplex_init ()) != PAPI_OK)
-      return GPTLerror ("GPTLcreate_and_start_events: failure from PAPI_multiplex_init%s\n", 
+      return GPTLerror ("GPTLcreate_and_start_events: failure from PAPI_multiplex_init%s\n",
 			PAPI_strerror (ret));
 
     if ((ret = PAPI_set_multiplex (EventSet[t])) != PAPI_OK)
-      return GPTLerror ("GPTLcreate_and_start_events: failure from PAPI_set_multiplex: %s\n", 
+      return GPTLerror ("GPTLcreate_and_start_events: failure from PAPI_set_multiplex: %s\n",
 			PAPI_strerror (ret));
 
     for (n = 0; n < npapievents; n++) {
@@ -825,20 +825,20 @@ int GPTLcreate_and_start_events (const int t)  /* thread number */
   /* Start the event set.  It will only be read from now on--never stopped */
 
   if ((ret = PAPI_start (EventSet[t])) != PAPI_OK)
-    return GPTLerror ("GPTLcreate_and_start_events: failed to start event set: %s\n", 
+    return GPTLerror ("GPTLcreate_and_start_events: failed to start event set: %s\n",
 		      PAPI_strerror (ret));
 
   return 0;
 }
 
 /*
-** GPTL_PAPIstart: Start the PAPI counters (actually they are just read).  
+** GPTL_PAPIstart: Start the PAPI counters (actually they are just read).
 **   Called from GPTLstart.
 **
-** Input args:  
+** Input args:
 **   t: thread number
 **
-** Output args: 
+** Output args:
 **   aux: struct containing the counters
 **
 ** Return value: 0 (success) or GPTLerror (failure)
@@ -849,7 +849,7 @@ int GPTL_PAPIstart (const int t,          /* thread number */
 {
   int ret;  /* return code from PAPI lib calls */
   int n;    /* loop index */
-  
+
   /* If no events are to be counted just return */
 
   if (npapievents == 0)
@@ -860,25 +860,25 @@ int GPTL_PAPIstart (const int t,          /* thread number */
   if ((ret = PAPI_read (EventSet[t], papicounters[t])) != PAPI_OK)
     return GPTLerror ("GPTL_PAPIstart: %s\n", PAPI_strerror (ret));
 
-  /* 
+  /*
   ** Store the counter values.  When GPTL_PAPIstop is called, the counters
   ** will again be read, and differenced with the values saved here.
   */
 
   for (n = 0; n < npapievents; n++)
     aux->last[n] = papicounters[t][n];
-  
+
   return 0;
 }
 
 /*
-** GPTL_PAPIstop: Stop the PAPI counters (actually they are just read).  
+** GPTL_PAPIstop: Stop the PAPI counters (actually they are just read).
 **   Called from GPTLstop.
 **
 ** Input args:
 **   t: thread number
 **
-** Input/output args: 
+** Input/output args:
 **   aux: struct containing the counters
 **
 ** Return value: 0 (success) or GPTLerror (failure)
@@ -900,8 +900,8 @@ int GPTL_PAPIstop (const int t,         /* thread number */
 
   if ((ret = PAPI_read (EventSet[t], papicounters[t])) != PAPI_OK)
     return GPTLerror ("GPTL_PAPIstop: %s\n", PAPI_strerror (ret));
-  
-  /* 
+
+  /*
   ** Accumulate the difference since timer start in aux.
   ** Negative accumulation can happen when multiplexing is enabled, so don't
   ** set count to BADCOUNT in that case.
@@ -924,14 +924,14 @@ int GPTL_PAPIstop (const int t,         /* thread number */
 ** GPTL_PAPIprstr: Print the descriptive string for all enabled PAPI events.
 **   Called from GPTLpr.
 **
-** Input args: 
+** Input args:
 **   fp: file descriptor
 */
 
 void GPTL_PAPIprstr (FILE *fp)
 {
   int n;
-  
+
   if (narrowprint) {
     for (n = 0; n < nevents; n++) {
       fprintf (fp, "%8.8s ", pr_event[n].event.str8);
@@ -957,7 +957,7 @@ void GPTL_PAPIprstr (FILE *fp)
 ** GPTL_PAPIpr: Print PAPI counter values for all enabled events, including
 **   derived events. Called from GPTLpr.
 **
-** Input args: 
+** Input args:
 **   fp: file descriptor
 **   aux: struct containing the counters
 */
@@ -989,7 +989,7 @@ void GPTL_PAPIpr (FILE *fp,                          /* file descriptor to write
       denomidx = pr_event[n].denomidx;
 
 #ifdef DEBUG
-      printf ("GPTL_PAPIpr: derived event: numidx=%d denomidx=%d values = %ld %ld\n", 
+      printf ("GPTL_PAPIpr: derived event: numidx=%d denomidx=%d values = %ld %ld\n",
 	      numidx, denomidx, (long) aux->accum[numidx], (long) aux->accum[denomidx]);
 #endif
       /* Protect against divide by zero */
@@ -1003,7 +1003,7 @@ void GPTL_PAPIpr (FILE *fp,                          /* file descriptor to write
     } else {                               /* Raw PAPI event */
 
 #ifdef DEBUG
-      printf ("GPTL_PAPIpr: raw event: numidx=%d value = %ld\n", 
+      printf ("GPTL_PAPIpr: raw event: numidx=%d value = %ld\n",
 	      numidx, (long) aux->accum[numidx]);
 #endif
       if (aux->accum[numidx] < PRTHRESH)
@@ -1055,12 +1055,12 @@ void GPTL_PAPIprintenabled (FILE *fp)
 	fprintf (fp, "  %s\n", eventname);
     fprintf (fp, "\n");
   }
-}  
+}
 
 /*
 ** GPTL_PAPIadd: Accumulate PAPI counters. Called from add.
 **
-** Input/Output args: 
+** Input/Output args:
 **   auxout: auxout = auxout + auxin
 **
 ** Input args:
@@ -1071,7 +1071,7 @@ void GPTL_PAPIadd (Papistats *auxout,      /* output struct */
 		   const Papistats *auxin) /* input struct */
 {
   int n;
-  
+
   for (n = 0; n < npapievents; n++)
     if (auxin->accum[n] == BADCOUNT || auxout->accum[n] == BADCOUNT)
       auxout->accum[n] = BADCOUNT;
@@ -1229,7 +1229,7 @@ int GPTLevent_name_to_code (const char *name, int *code)
   int n;     /* loop over derived entries */
 
   /*
-  ** First check derived events 
+  ** First check derived events
   */
 
   for (n = 0; n < nderivedentries; ++n) {
@@ -1272,7 +1272,7 @@ int GPTLevent_code_to_name (const int code, char *name)
   int n;     /* loop over derived entries */
 
   /*
-  ** First check derived events 
+  ** First check derived events
   */
 
   for (n = 0; n < nderivedentries; ++n) {
@@ -1323,4 +1323,3 @@ int GPTLevent_code_to_name (int code, char *name)
 }
 
 #endif  /* HAVE_PAPI */
-
diff --git a/src/gptl/perf_mod.F90 b/src/gptl/perf_mod.F90
index 622d5cf2c1..8490a35d26 100644
--- a/src/gptl/perf_mod.F90
+++ b/src/gptl/perf_mod.F90
@@ -1,14 +1,14 @@
 module perf_mod
 
-!----------------------------------------------------------------------- 
-! 
+!-----------------------------------------------------------------------
+!
 ! Purpose: This module is responsible for controlling the performance
 !          timer logic.
-! 
+!
 ! Author:  P. Worley, January 2007
 !
 ! $Id$
-! 
+!
 !-----------------------------------------------------------------------
 
 !-----------------------------------------------------------------------
@@ -62,7 +62,7 @@ module perf_mod
 !-----------------------------------------------------------------------
 !- include statements --------------------------------------------------
 !-----------------------------------------------------------------------
-#include <mpif.h>  
+#include <mpif.h>
 #include "gptl.inc"
 
 !-----------------------------------------------------------------------
@@ -93,7 +93,7 @@ module perf_mod
    integer, parameter :: def_timer_depth_limit = 99999         ! default
    integer, private   :: timer_depth_limit = def_timer_depth_limit
                          ! integer indicating maximum number of levels of
-                         ! timer nesting 
+                         ! timer nesting
 
    integer, parameter :: def_timing_detail_limit = 1           ! default
    integer, private   :: timing_detail_limit = def_timing_detail_limit
@@ -111,19 +111,19 @@ module perf_mod
    logical, parameter :: def_perf_single_file = .false.         ! default
    logical, private   :: perf_single_file = def_perf_single_file
                          ! flag indicating whether the performance timer
-                         ! output should be written to a single file 
-                         ! (per component communicator) or to a 
+                         ! output should be written to a single file
+                         ! (per component communicator) or to a
                          ! separate file for each process
 
    integer, parameter :: def_perf_outpe_num = 0                ! default
    integer, private   :: perf_outpe_num = def_perf_outpe_num
-                         ! maximum number of processes writing out 
+                         ! maximum number of processes writing out
                          ! timing data (for this component communicator)
 
    integer, parameter :: def_perf_outpe_stride = 1             ! default
    integer, private   :: perf_outpe_stride = def_perf_outpe_stride
                          ! separation between process ids for processes
-                         ! that are writing out timing data 
+                         ! that are writing out timing data
                          ! (for this component communicator)
 
    logical, parameter :: def_perf_global_stats = .true.        ! default
@@ -176,9 +176,9 @@ module perf_mod
 !========================================================================
 !
    subroutine t_getLogUnit(LogUnit)
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose:  Get log unit number.
-! Author:   P. Worley 
+! Author:   P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
 !
@@ -193,9 +193,9 @@ end subroutine t_getLogUnit
 !========================================================================
 !
    subroutine t_setLogUnit(LogUnit)
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose:  Set log unit number.
-! Author:   P. Worley 
+! Author:   P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
 !
@@ -222,9 +222,9 @@ subroutine perf_defaultopts(timing_disable_out, &
                                perf_single_file_out, &
                                perf_global_stats_out, &
                                perf_papi_enable_out )
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Return default runtime options
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
    ! timers disable/enable option
@@ -296,16 +296,16 @@ subroutine perf_setopts(mastertask, &
                            perf_single_file_in, &
                            perf_global_stats_in, &
                            perf_papi_enable_in )
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Set runtime options
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments----------------------------
 !
    ! master process?
    logical, intent(in) :: mastertask
    ! Print out to log file?
-   logical, intent(IN) :: LogPrint        
+   logical, intent(IN) :: LogPrint
    ! timers disable/enable option
    logical, intent(in), optional :: timing_disable_in
    ! performance timer option
@@ -337,7 +337,7 @@ subroutine perf_setopts(mastertask, &
          timing_disable = timing_disable_in
          if (timing_disable) then
             ierr = GPTLdisable()
-         else 
+         else
             ierr = GPTLenable()
          endif
       endif
@@ -392,17 +392,17 @@ subroutine perf_setopts(mastertask, &
       endif
 !
       if (mastertask .and. LogPrint) then
-         write(p_logunit,*) '(t_initf) Using profile_disable=', timing_disable, &             
+         write(p_logunit,*) '(t_initf) Using profile_disable=', timing_disable, &
                             ' profile_timer=', perf_timer
-         write(p_logunit,*) '(t_initf)  profile_depth_limit=', timer_depth_limit, &    
+         write(p_logunit,*) '(t_initf)  profile_depth_limit=', timer_depth_limit, &
                             ' profile_detail_limit=', timing_detail_limit
          write(p_logunit,*) '(t_initf)  profile_barrier=', timing_barrier, &
                             ' profile_outpe_num=', perf_outpe_num
          write(p_logunit,*) '(t_initf)  profile_outpe_stride=', perf_outpe_stride , &
                             ' profile_single_file=', perf_single_file
          write(p_logunit,*) '(t_initf)  profile_global_stats=', perf_global_stats , &
-                            ' profile_papi_enable=', perf_papi_enable 
-      endif                                                                               
+                            ' profile_papi_enable=', perf_papi_enable
+      endif
 !
 #ifdef DEBUG
    else
@@ -420,9 +420,9 @@ subroutine papi_defaultopts(papi_ctr1_out, &
                                papi_ctr2_out, &
                                papi_ctr3_out, &
                                papi_ctr4_out  )
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Return default runtime PAPI counter options
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
    ! PAPI counter option #1
@@ -456,9 +456,9 @@ subroutine papi_setopts(papi_ctr1_in, &
                            papi_ctr2_in, &
                            papi_ctr3_in, &
                            papi_ctr4_in  )
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Set runtime PAPI counter options
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments----------------------------
 !
@@ -518,12 +518,12 @@ end subroutine papi_setopts
 !========================================================================
 !
    logical function t_profile_onf()
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Return flag indicating whether profiling is currently active.
 !          Part of workaround to implement FVbarrierclock before
 !          communicators exposed in Pilgrim. Does not check level of
 !          event nesting.
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 
    if ((.not. timing_initialized) .or. &
@@ -539,10 +539,10 @@ end function t_profile_onf
 !========================================================================
 !
    logical function t_barrier_onf()
-!----------------------------------------------------------------------- 
-! Purpose: Return timing_barrier. Part of workaround to implement 
-!          FVbarrierclock before communicators exposed in Pilgrim. 
-! Author: P. Worley 
+!-----------------------------------------------------------------------
+! Purpose: Return timing_barrier. Part of workaround to implement
+!          FVbarrierclock before communicators exposed in Pilgrim.
+! Author: P. Worley
 !-----------------------------------------------------------------------
 
    t_barrier_onf = timing_barrier
@@ -552,10 +552,10 @@ end function t_barrier_onf
 !========================================================================
 !
    logical function t_single_filef()
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Return perf_single_file. Used to control output of other
 !          performance data, only spmdstats currently.
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 
    t_single_filef = perf_single_file
@@ -565,9 +565,9 @@ end function t_single_filef
 !========================================================================
 !
    subroutine t_stampf(wall, usr, sys)
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Record wallclock, user, and system times (seconds).
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Output arguments-----------------------------
 !
@@ -596,14 +596,14 @@ end subroutine t_stampf
 !========================================================================
 !
    subroutine t_startf(event, handle)
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Start an event timer
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
 !
    ! performance timer event name
-   character(len=*), intent(in) :: event  
+   character(len=*), intent(in) :: event
 !
 !---------------------------Input/Output arguments----------------------
 !
@@ -634,14 +634,14 @@ end subroutine t_startf
 !========================================================================
 !
    subroutine t_stopf(event, handle)
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Stop an event timer
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
 !
    ! performance timer event name
-   character(len=*), intent(in) :: event  
+   character(len=*), intent(in) :: event
 !
 !---------------------------Input/Output arguments----------------------
 !
@@ -672,10 +672,10 @@ end subroutine t_stopf
 !========================================================================
 !
    subroutine t_enablef()
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Enable t_startf, t_stopf, t_stampf, and t_barrierf. Ignored
 !          in threaded regions.
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Local workspace-----------------------------
 !
@@ -709,10 +709,10 @@ end subroutine t_enablef
 !========================================================================
 !
    subroutine t_disablef()
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Disable t_startf, t_stopf, t_stampf, and t_barrierf. Ignored
 !          in threaded regions.
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Local workspace-----------------------------
 !
@@ -744,9 +744,9 @@ end subroutine t_disablef
 !========================================================================
 !
    subroutine t_adj_detailf(detail_adjustment)
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Modify current detail level. Ignored in threaded regions.
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
 !
@@ -776,11 +776,11 @@ end subroutine t_adj_detailf
 !========================================================================
 !
    subroutine t_barrierf(event, mpicom)
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Call (and time) mpi_barrier. Ignored inside OpenMP
 !          threaded regions. Note that barrier executed even if
 !          event not recorded because of level of timer event nesting.
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
    ! mpi communicator id
@@ -835,9 +835,9 @@ end subroutine t_barrierf
 !
    subroutine t_prf(filename, mpicom, num_outpe, stride_outpe, &
                     single_file, global_stats, output_thispe)
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: Write out performance timer data
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
 !
@@ -847,7 +847,7 @@ subroutine t_prf(filename, mpicom, num_outpe, stride_outpe, &
    integer, intent(in), optional :: mpicom
    ! maximum number of processes writing out timing data
    integer, intent(in), optional :: num_outpe
-   ! separation between process ids for processes writing out data 
+   ! separation between process ids for processes writing out data
    integer, intent(in), optional :: stride_outpe
    ! enable/disable the writing of data to a single file
    logical, intent(in), optional :: single_file
@@ -862,7 +862,7 @@ subroutine t_prf(filename, mpicom, num_outpe, stride_outpe, &
                                   !  all data to a single file
    logical  glb_stats             ! flag indicting whether to compute
                                   !  global statistics
-   logical  pr_write              ! flag indicating whether the current 
+   logical  pr_write              ! flag indicating whether the current
                                   !  GPTL output mode is write
    logical  write_data            ! flag indicating whether this process
                                   !  should output its timing data
@@ -1113,8 +1113,8 @@ end subroutine t_prf
 !========================================================================
 !
    subroutine t_initf(NLFilename, LogPrint, LogUnit, mpicom, MasterTask)
-!----------------------------------------------------------------------- 
-! Purpose:  Set default values of runtime timing options 
+!-----------------------------------------------------------------------
+! Purpose:  Set default values of runtime timing options
 !           before namelists prof_inparm and papi_inparm are read,
 !           read namelists (and broadcast, if SPMD),
 !           then initialize timing library.
@@ -1224,12 +1224,12 @@ subroutine t_initf(NLFilename, LogPrint, LogUnit, mpicom, MasterTask)
        open( unitn, file=trim(NLFilename), status='old', iostat=ierr )
        if (ierr .eq. 0) then
 
-          ! Look for prof_inparm group name in the input file.  
+          ! Look for prof_inparm group name in the input file.
           ! If found, leave the file positioned at that namelist group.
           call find_group_name(unitn, 'prof_inparm', status=ierr)
 
           if (ierr == 0) then  ! found prof_inparm
-             read(unitn, nml=prof_inparm, iostat=ierr)  
+             read(unitn, nml=prof_inparm, iostat=ierr)
              if (ierr /= 0) then
                 call shr_sys_abort( subname//':: namelist read returns an'// &
                                     ' error condition for prof_inparm' )
@@ -1291,12 +1291,12 @@ subroutine t_initf(NLFilename, LogPrint, LogUnit, mpicom, MasterTask)
           ierr = 1
           open( unitn, file=trim(NLFilename), status='old', iostat=ierr )
           if (ierr .eq. 0) then
-             ! Look for papi_inparm group name in the input file.  
+             ! Look for papi_inparm group name in the input file.
              ! If found, leave the file positioned at that namelist group.
              call find_group_name(unitn, 'papi_inparm', status=ierr)
 
              if (ierr == 0) then  ! found papi_inparm
-                read(unitn, nml=papi_inparm, iostat=ierr)  
+                read(unitn, nml=papi_inparm, iostat=ierr)
                 if (ierr /= 0) then
                    call shr_sys_abort( subname//':: namelist read returns an'// &
                                       ' error condition for papi_inparm' )
@@ -1355,12 +1355,12 @@ subroutine t_initf(NLFilename, LogPrint, LogUnit, mpicom, MasterTask)
 
 !$OMP MASTER
    !
-   ! Set options and initialize timing library.  
-   ! 
+   ! Set options and initialize timing library.
+   !
    ! Set timer
    if (gptlsetutr (perf_timer) < 0) call shr_sys_abort (subname//':: gptlsetutr')
    !
-   ! For logical settings, 2nd arg 0 
+   ! For logical settings, 2nd arg 0
    ! to gptlsetoption means disable, non-zero means enable
    !
    ! Turn off CPU timing (expensive)
@@ -1404,9 +1404,9 @@ end subroutine t_initf
 !========================================================================
 !
    subroutine t_finalizef()
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose: shut down timing library
-! Author: P. Worley 
+! Author: P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Local workspace-----------------------------
 !
diff --git a/src/gptl/perf_utils.F90 b/src/gptl/perf_utils.F90
index 2ab74ada4f..76c7294136 100644
--- a/src/gptl/perf_utils.F90
+++ b/src/gptl/perf_utils.F90
@@ -1,15 +1,15 @@
 module perf_utils
 
-!----------------------------------------------------------------------- 
-! 
+!-----------------------------------------------------------------------
+!
 ! Purpose: This module supplies the csm_share and CAM utilities
 !          needed by perf_mod.F90 (when the csm_share and CAM utilities
 !          are not available).
-! 
+!
 ! Author:  P. Worley, October 2007
 !
 ! $Id$
-! 
+!
 !-----------------------------------------------------------------------
 #ifndef NO_MPIMOD
   use mpi
@@ -49,7 +49,7 @@ module perf_utils
 !- include statements --------------------------------------------------
 !-----------------------------------------------------------------------
 #ifdef NO_MPIMOD
-#include <mpif.h>  
+#include <mpif.h>
 #endif
 #include "gptl.inc"
 
@@ -82,9 +82,9 @@ module perf_utils
 !========================================================================
 !
    subroutine perfutils_setunit(LogUnit)
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
 ! Purpose:  Set log unit number.
-! Author:   P. Worley 
+! Author:   P. Worley
 !-----------------------------------------------------------------------
 !---------------------------Input arguments-----------------------------
 !
@@ -327,7 +327,7 @@ END SUBROUTINE shr_mpi_bcastl0
 !     2005-Dec-14 - E. Kluzek - creation
 !     2007-Oct-21 - P. Worley - dumbed down for use in perf_mod
 !
-! !INTERFACE: ------------------------------------------------------------------  
+! !INTERFACE: ------------------------------------------------------------------
 
 INTEGER FUNCTION shr_file_getUnit ()
 
@@ -377,7 +377,7 @@ END FUNCTION shr_file_getUnit
 !     2005-Dec-14 - E. Kluzek - creation
 !     2007-Oct-21 - P. Worley - dumbed down for use in perf_mod
 !
-! !INTERFACE: ------------------------------------------------------------------  
+! !INTERFACE: ------------------------------------------------------------------
 
 SUBROUTINE shr_file_freeUnit ( unit)
 
@@ -418,12 +418,12 @@ END SUBROUTINE shr_file_freeUnit
 subroutine find_group_name(unit, group, status)
 
 !---------------------------------------------------------------------------------------
-! Purpose: 
+! Purpose:
 ! Search a file that contains namelist input for the specified namelist group name.
 ! Leave the file positioned so that the current record is the first record of the
 ! input for the specified group.
-! 
-! Method: 
+!
+! Method:
 ! Read the file line by line.  Each line is searched for an '&' which may only
 ! be preceded by blanks, immediately followed by the group name which is case
 ! insensitive.  If found then backspace the file so the current record is the
@@ -489,17 +489,17 @@ end subroutine find_group_name
 
 function to_lower(str)
 
-!----------------------------------------------------------------------- 
-! Purpose: 
+!-----------------------------------------------------------------------
+! Purpose:
 ! Convert character string to lower case.
-! 
-! Method: 
+!
+! Method:
 ! Use achar and iachar intrinsics to ensure use of ascii collating sequence.
 !
 ! Author:  B. Eaton, July 2001
-!     
+!
 ! $Id$
-!----------------------------------------------------------------------- 
+!-----------------------------------------------------------------------
    implicit none
 
    character(len=*), intent(in) :: str      ! String to convert to lower case
@@ -518,7 +518,7 @@ function to_lower(str)
       ctmp = str(i:i)
       aseq = iachar(ctmp)
       if ( aseq >= iachar("A") .and. aseq <= iachar("Z") ) &
-           ctmp = achar(aseq + upper_to_lower)	
+           ctmp = achar(aseq + upper_to_lower)
       to_lower(i:i) = ctmp
    end do
 
diff --git a/src/gptl/private.h b/src/gptl/private.h
index 8d14b1479a..c8a52a9f35 100644
--- a/src/gptl/private.h
+++ b/src/gptl/private.h
@@ -32,7 +32,7 @@
 /* longest timer name allowed (probably safe to just change) */
 #define MAX_CHARS 63
 
-/* 
+/*
 ** max allowable number of PAPI counters, or derived events. For convenience,
 ** set to max (# derived events, # papi counters required) so "avail" lists
 ** all available options.
@@ -61,7 +61,7 @@ typedef struct {
   long long last[MAX_AUX];  /* array of saved counters from "start" */
   long long accum[MAX_AUX]; /* accumulator for counters */
 } Papistats;
-  
+
 typedef struct {
   int counter;      /* PAPI or Derived counter */
   char *namestr;    /* PAPI or Derived counter as string */
@@ -84,7 +84,7 @@ typedef struct TIMER {
 #endif
 #ifdef HAVE_PAPI
   Papistats aux;            /* PAPI stats  */
-#endif 
+#endif
   Wallstats wall;           /* wallclock stats */
   Cpustats cpu;             /* cpu stats */
   unsigned long count;      /* number of start/stop calls */
@@ -127,7 +127,7 @@ extern void __cyg_profile_func_exit (void *, void *);
 };
 #endif
 
-/* 
+/*
 ** These are needed for communication between gptl.c and gptl_papi.c
 */
 
diff --git a/src/gptl/threadutil.c b/src/gptl/threadutil.c
index 85d9c52d15..86e4681c1c 100644
--- a/src/gptl/threadutil.c
+++ b/src/gptl/threadutil.c
@@ -2,7 +2,7 @@
 ** $Id: threadutil.c,v 1.28 2009/12/31 01:51:59 rosinski Exp $
 **
 ** Author: Jim Rosinski
-** 
+**
 ** Utility functions handle thread-based GPTL needs.
 */
 
@@ -21,7 +21,7 @@
 static bool first = true;
 
 /**********************************************************************************/
-/* 
+/*
 ** 3 sets of routines: OMP threading, PTHREADS, unthreaded
 */
 
@@ -62,7 +62,7 @@ int threadinit (int *nthreads, int *maxthreads)
 #ifdef VERBOSE
   printf ("OMP threadinit: Set *maxthreads=%d *nthreads=%d\n", *maxthreads, *nthreads);
 #endif
-  
+
   return 0;
 }
 
@@ -138,7 +138,7 @@ void print_threadmapping (int nthreads, FILE *fp)
 }
 
 /**********************************************************************************/
-/* 
+/*
 ** PTHREADS
 */
 
@@ -179,7 +179,7 @@ int threadinit (int *nthreads, int *maxthreads)
   first = false;
 
   /*
-  ** Initialize nthreads to 0 and define the threadid array now that initialization 
+  ** Initialize nthreads to 0 and define the threadid array now that initialization
   ** is done. The actual value will be determined as get_thread_num is called.
   */
 
@@ -228,8 +228,8 @@ int get_thread_num (int *nthreads, int *maxthreads)
     return GPTLerror ("get_thread_num: mutex lock failure\n");
 
   /*
-  ** Loop over known physical thread IDs.  When my id is found, map it 
-  ** to logical thread id for indexing.  If not found return a negative 
+  ** Loop over known physical thread IDs.  When my id is found, map it
+  ** to logical thread id for indexing.  If not found return a negative
   ** number.
   ** A critical region is necessary because acess to
   ** the array threadid must be by only one thread at a time.
@@ -251,7 +251,7 @@ int get_thread_num (int *nthreads, int *maxthreads)
 
       return GPTLerror ("get_thread_num: nthreads=%d is too big Recompile "
 			"with larger value of MAX_THREADS\n", *nthreads);
-    }    
+    }
 
     threadid[n] = mythreadid;
 
@@ -268,7 +268,7 @@ int get_thread_num (int *nthreads, int *maxthreads)
 
     if (GPTLget_npapievents () > 0) {
 #ifdef VERBOSE
-      printf ("PTHREADS get_thread_num: Starting EventSet threadid=%lu location=%d\n", 
+      printf ("PTHREADS get_thread_num: Starting EventSet threadid=%lu location=%d\n",
 	      (unsigned long) mythreadid, n);
 #endif
       if (GPTLcreate_and_start_events (n) < 0) {
@@ -286,7 +286,7 @@ int get_thread_num (int *nthreads, int *maxthreads)
     printf ("PTHREADS get_thread_num: *nthreads=%d\n", *nthreads);
 #endif
   }
-    
+
   if (unlock_mutex () < 0)
     return GPTLerror ("get_thread_num: mutex unlock failure\n");
 
diff --git a/src/ncint/ncint_pio.c b/src/ncint/ncint_pio.c
index a8c606dd80..e4a59bad28 100644
--- a/src/ncint/ncint_pio.c
+++ b/src/ncint/ncint_pio.c
@@ -25,14 +25,16 @@ nc_def_iosystem(MPI_Comm comp_comm, int num_iotasks, int stride, int base,
     int ret;
 
     /* Make sure PIO was initialized. */
-    if ((ret = PIO_NCINT_initialize()))
+    if ((ret = PIO_NCINT_initialize())){
+        //printf("%s %d ret=%d\n",__FILE__,__LINE__,ret);
         return ret;
-
+    }
     /* Call the PIOc_ function to initialize the intracomm. */
     if ((ret = PIOc_Init_Intracomm(comp_comm, num_iotasks, stride, base, rearr,
-                                   iosysidp)))
+                                   iosysidp))){
+        //printf("%s %d ret=%d\n",__FILE__,__LINE__,ret);
         return ret;
-
+    }
     /* Remember the io system id. */
     diosysid = *iosysidp;
 
diff --git a/src/ncint/ncintdispatch.c b/src/ncint/ncintdispatch.c
index db442a8c7f..a77396bd7b 100644
--- a/src/ncint/ncintdispatch.c
+++ b/src/ncint/ncintdispatch.c
@@ -25,18 +25,18 @@ int diosysid;
 int ncint_initialized = 0;
 
 /** Version of dispatch table. */
-#define DISPATCH_VERSION 2
+#define DISPATCH_VERSION NC_DISPATCH_VERSION
 
 /* Internal filter actions - copied from nc4internal.h */
-#define NCFILTER_DEF		1
-#define NCFILTER_REMOVE  	2
-#define NCFILTER_INQ	    	3
+#define NCFILTER_DEF            1
+#define NCFILTER_REMOVE         2
+#define NCFILTER_INQ            3
 #define NCFILTER_FILTERIDS      4
-#define NCFILTER_INFO		5
-#define NCFILTER_FREESPEC	6
-#define NCFILTER_CLIENT_REG	10
-#define NCFILTER_CLIENT_UNREG	11
-#define NCFILTER_CLIENT_INQ	12
+#define NCFILTER_INFO           5
+#define NCFILTER_FREESPEC       6
+#define NCFILTER_CLIENT_REG     10
+#define NCFILTER_CLIENT_UNREG   11
+#define NCFILTER_CLIENT_INQ     12
 
 /* This is the dispatch object that holds pointers to all the
  * functions that make up the NCINT dispatch interface. */
@@ -127,7 +127,20 @@ NC_Dispatch NCINT_dispatcher = {
     NC_NOTNC4_def_var_filter,
     NC_NOTNC4_set_var_chunk_cache,
     NC_NOTNC4_get_var_chunk_cache,
-    PIO_NCINT_filter_actions
+#if NC_DISPATCH_VERSION == 2
+    PIO_NCINT_filter_actions,
+#endif
+#if NC_DISPATCH_VERSION >= 3
+    PIOc_inq_var_filter_ids,
+    PIOc_inq_var_filter_info,
+#endif
+#if NC_DISPATCH_VERSION >= 4
+    PIOc_def_var_quantize,
+    PIOc_inq_var_quantize,
+#endif
+#if NC_DISPATCH_VERSION >= 5
+    PIOc_inq_filter_avail,
+#endif
 };
 
 /**
@@ -206,9 +219,8 @@ PIO_NCINT_create(const char *path, int cmode, size_t initialsz, int basepe,
     if (!(ios = pio_get_iosystem_from_id(diosysid)))
         return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
 
-    /* Turn of NC_UDF0 in the mode flag. */
-    cmode = cmode & ~NC_UDF0;
-
+    /* Turn off NC_UDF0 in the mode flag. */
+    cmode = ((cmode) & ~(NC_UDF0));
     /* Find the IOTYPE from the mode flag. */
     if ((ret = find_iotype_from_omode(cmode, &iotype)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
@@ -257,8 +269,8 @@ PIO_NCINT_open(const char *path, int mode, int basepe, size_t *chunksizehintp,
     if (!(ios = pio_get_iosystem_from_id(diosysid)))
         return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
 
-    /* Turn of NC_UDF0 in the mode flag. */
-    mode = mode & ~NC_UDF0;
+    /* Turn off NC_UDF0 in the mode flag. */
+    mode = (mode) & ~(NC_UDF0);
 
     /* Find the IOTYPE from the mode flag. */
     if ((ret = find_iotype_from_omode(mode, &iotype)))
@@ -266,11 +278,11 @@ PIO_NCINT_open(const char *path, int mode, int basepe, size_t *chunksizehintp,
 
     /* Add necessary structs to hold netcdf-4 file data. */
     if ((ret = nc4_file_list_add(ncid, path, mode, NULL)))
-        return ret;
+      return ret;
 
     /* Open the file with PIO. Tell openfile_retry to accept the
      * externally assigned ncid. */
-    if ((ret = PIOc_openfile_retry(diosysid, &ncid, &iotype, path, mode, 0, 1)))
+    if ((ret = PIOc_openfile_retry(diosysid, &ncid, &iotype, path, mode, 1, 1)))
         return ret;
 
     return NC_NOERR;
@@ -417,8 +429,8 @@ PIO_NCINT_inq_format_extended(int ncid, int *formatp, int *modep)
 
     PLOG((2, "%s: ncid 0x%x", __func__, ncid));
 
-    if ((retval = nc4_file_list_get(ncid, NULL, &my_mode, NULL)))
-        return NC_EBADID;
+    if ((retval = PIOc_inq_format(ncid, &my_mode)))
+        return retval;
 
     if (modep)
         *modep = my_mode|NC_UDF0;
@@ -883,18 +895,22 @@ PIO_NCINT_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
                       unsigned int *idp, size_t *nparamsp, unsigned int *params)
 {
     int ret;
+    int format;
 
     ret = PIOc_inq_var(ncid, varid, name, xtypep, ndimsp, dimidsp, nattsp);
+#ifdef _NETCDF4
+    ret = PIOc_inq_format(ncid, &format);
+    if (!ret && (format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) ){
+        if (!ret && contiguousp && chunksizesp)
+            ret = PIOc_inq_var_chunking(ncid, varid, contiguousp, (MPI_Offset *)chunksizesp);
 
-    if (!ret)
-	ret = PIOc_inq_var_chunking(ncid, varid, contiguousp, (MPI_Offset *)chunksizesp);
+        if (!ret && shufflep && deflatep && deflate_levelp)
+            ret = PIOc_inq_var_deflate(ncid, varid, shufflep, deflatep, deflate_levelp);
 
-    if (!ret)
-	ret = PIOc_inq_var_deflate(ncid, varid, shufflep, deflatep, deflate_levelp);
-
-    if (!ret)
-	ret = PIOc_inq_var_endian(ncid, varid, endiannessp);
-    
+        if (!ret && endiannessp)
+            ret = PIOc_inq_var_endian(ncid, varid, endiannessp);
+    }
+#endif
     return ret;
 }
 
@@ -993,7 +1009,7 @@ PIO_NCINT_inq_type_equal(int ncid1, nc_type typeid1, int ncid2,
  */
 int
 PIO_NCINT_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
-			  int deflate_level)
+                          int deflate_level)
 {
     return PIOc_def_var_deflate(ncid, varid, shuffle, deflate, deflate_level);
 }
@@ -1028,6 +1044,7 @@ PIO_NCINT_def_var_chunking(int ncid, int varid, int storage, const size_t *chunk
     return PIOc_def_var_chunking(ncid, varid, storage, (const PIO_Offset *)chunksizesp);
 }
 
+#if NC_DISPATCH_VERSION == 2
 /**
  * @internal Carry out one of several filter actions
  *
@@ -1043,7 +1060,8 @@ PIO_NCINT_filter_actions(int ncid, int varid, int action, struct NC_Filterobject
 {
     if (action == NCFILTER_INFO)
     {
-        
+
     }
     return PIO_NOERR;
 }
+#endif
diff --git a/src/ncint/ncintdispatch.h b/src/ncint/ncintdispatch.h
index 8934740f55..26764726dc 100644
--- a/src/ncint/ncintdispatch.h
+++ b/src/ncint/ncintdispatch.h
@@ -147,15 +147,15 @@ extern "C" {
 
     extern int
     PIO_NCINT_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
-			      int deflate_level);
+                              int deflate_level);
 
     extern int
     PIO_NCINT_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp);
 
-
+#if NC_DISPATCH_VERSION == 2
     extern int
     PIO_NCINT_filter_actions(int ncid, int varid, int action, struct NC_Filterobject* spec);
-
+#endif
 #if defined(__cplusplus)
 }
 #endif
diff --git a/tests/cperf/CMakeLists.txt b/tests/cperf/CMakeLists.txt
index fed02b9e33..e82c297357 100644
--- a/tests/cperf/CMakeLists.txt
+++ b/tests/cperf/CMakeLists.txt
@@ -5,9 +5,10 @@ include_directories("${CMAKE_SOURCE_DIR}/src/clib")
 include_directories("${CMAKE_BINARY_DIR}")
 
 # Compiler-specific compiler options
+message("Compiler is ${CMAKE_C_COMPILER_ID}")
 if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu99")
 elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -c99")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -c99")
@@ -15,8 +16,8 @@ elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
 elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu99")
 endif()
 #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0")
 #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
diff --git a/tests/cperf/piodecomptest.c b/tests/cperf/piodecomptest.c
index 63ea7118b1..5efedb0266 100644
--- a/tests/cperf/piodecomptest.c
+++ b/tests/cperf/piodecomptest.c
@@ -7,7 +7,7 @@
 const char *argp_program_version = "pioperformance 0.1";
 const char *argp_program_bug_address = "<https://github.com/NCAR/ParallelIO>";
 
-static char doc[] = 
+static char doc[] =
     "a test of pio for performance and correctness of a given decomposition";
 
 static struct argp_option options[] = {
@@ -17,7 +17,7 @@ static struct argp_option options[] = {
     { 0 }
 };
 
-struct arguments 
+struct arguments
 {
     char *args[2];
     char *wdecomp_file;
@@ -84,12 +84,11 @@ int test_write_darray(int iosys, const char decomp_file[], int rank, const char
     int *full_map;
     int *dimid;
     int varid;
-    int globalsize;
     int ioid;
     char dimname[PIO_MAX_NAME];
     char varname[PIO_MAX_NAME];
 
-    ierr = pioc_read_nc_decomp_int(iosys, decomp_file, &ndims, &global_dimlen, &num_tasks, 
+    ierr = pioc_read_nc_decomp_int(iosys, decomp_file, &ndims, &global_dimlen, &num_tasks,
                                    &maplen, &maxmaplen, &full_map, NULL, NULL, NULL, NULL, NULL);
     if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
 
@@ -103,10 +102,10 @@ int test_write_darray(int iosys, const char decomp_file[], int rank, const char
             ierr = MPI_Abort(MPI_COMM_WORLD, -1);
         }
     }
-       
+
     ierr = PIOc_createfile(iosys, &ncid, &iotype, "testfile.nc", PIO_CLOBBER);
     if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
-    
+
     dimid = calloc(ndims,sizeof(int));
     for(int i=0; i<ndims; i++)
     {
@@ -116,7 +115,7 @@ int test_write_darray(int iosys, const char decomp_file[], int rank, const char
     }
     /* TODO: support multiple variables and types*/
     if(myvarname != NULL)
-        sprintf(varname,"%s",myvarname);        
+        sprintf(varname,"%s",myvarname);
     else
         sprintf(varname,"var%4.4d",0);
 
@@ -149,7 +148,7 @@ int test_write_darray(int iosys, const char decomp_file[], int rank, const char
     for(int i=0; i < maplen[rank]; i++)
         dsum += dvarw[i];
     if(dsum != rank)
-        printf("%d: dvarwsum = %d\n",rank, dsum);
+        printf("%d: dvarwsum = %g\n",rank, dsum);
 
     ierr = PIOc_write_darray(ncid, varid, ioid, maplen[rank], dvarw, NULL);
     free(maplen);
@@ -174,13 +173,13 @@ int test_read_darray(int iosys,const char decomp_file[], int rank, const char my
     int *full_map;
     int *dimid;
     int varid;
-    int globalsize;
+    //int globalsize;
     int ioid;
     int pio_type;
-    char dimname[PIO_MAX_NAME];
+    //char dimname[PIO_MAX_NAME];
     char varname[PIO_MAX_NAME];
 
-    ierr = pioc_read_nc_decomp_int(iosys, decomp_file, &ndims, &global_dimlen, &num_tasks, 
+    ierr = pioc_read_nc_decomp_int(iosys, decomp_file, &ndims, &global_dimlen, &num_tasks,
                                    &maplen, &maxmaplen, &full_map, NULL, NULL, NULL, NULL, NULL);
     if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
 
@@ -194,13 +193,13 @@ int test_read_darray(int iosys,const char decomp_file[], int rank, const char my
             ierr = MPI_Abort(MPI_COMM_WORLD, -1);
         }
     }
-       
+
     ierr = PIOc_openfile(iosys, &ncid, &iotype, "testfile.nc", PIO_NOWRITE);
     if(ierr || debug) printf("%d %d\n",__LINE__,ierr);
-    
+
     /* TODO: support multiple variables and types*/
     if(myvarname != NULL)
-        sprintf(varname,"%s",myvarname);        
+        sprintf(varname,"%s",myvarname);
     else
         sprintf(varname,"var%4.4d",0);
     ierr = PIOc_inq_varid(ncid, varname, &varid);
@@ -220,7 +219,7 @@ int test_read_darray(int iosys,const char decomp_file[], int rank, const char my
     {
         PIO_Offset gdimlen;
         ierr = PIOc_inq_dimlen(ncid, dimid[i], &gdimlen);
-        
+
         pioassert(gdimlen == global_dimlen[i], "testfile.nc does not match decomposition file",__FILE__,__LINE__);
     }
     free(dimid);
@@ -248,7 +247,7 @@ int test_read_darray(int iosys,const char decomp_file[], int rank, const char my
         for(int i=0; i < maplen[rank]; i++)
             dsum += dvarr[i];
         if(dsum != rank)
-            printf("%d: dsum = %d\n",rank, dsum);
+            printf("%d: dsum = %g\n",rank, dsum);
         break;
     case PIO_INT:
         ivarr = malloc(sizeof(int)*maplen[rank]);
@@ -327,4 +326,3 @@ int main(int argc, char *argv[])
     MPI_Finalize();
 
 }
-
diff --git a/tests/cunit/CMakeLists.txt b/tests/cunit/CMakeLists.txt
index 4cd63c46fb..dc9239816e 100644
--- a/tests/cunit/CMakeLists.txt
+++ b/tests/cunit/CMakeLists.txt
@@ -3,6 +3,7 @@ include (LibMPI)
 include_directories("${CMAKE_SOURCE_DIR}/tests/cunit")
 include_directories("${CMAKE_SOURCE_DIR}/src/clib")
 include_directories("${CMAKE_BINARY_DIR}")
+include_directories("${CMAKE_BINARY_DIR}/src/clib")
 
 # Compiler-specific compiler options
 if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
@@ -69,6 +70,8 @@ if (NOT PIO_USE_MPISERIAL)
   target_link_libraries (test_pioc_fill pioc)
   add_executable (test_darray EXCLUDE_FROM_ALL test_darray.c test_common.c)
   target_link_libraries (test_darray pioc)
+  add_executable (test_darray_lossycompress EXCLUDE_FROM_ALL test_darray_lossycompress.c test_common.c)
+  target_link_libraries (test_darray_lossycompress pioc)
   add_executable (test_darray_append EXCLUDE_FROM_ALL test_darray_append.c test_common.c)
   target_link_libraries (test_darray_append pioc)
   add_executable (test_darray_frame EXCLUDE_FROM_ALL test_darray_frame.c test_common.c)
@@ -127,6 +130,7 @@ add_dependencies (tests test_pioc_unlim)
 add_dependencies (tests test_pioc_putget)
 add_dependencies (tests test_pioc_fill)
 add_dependencies (tests test_darray)
+add_dependencies (tests test_darray_lossycompress)
 add_dependencies (tests test_darray_append)
 add_dependencies (tests test_darray_frame)
 add_dependencies (tests test_darray_multi)
@@ -246,6 +250,10 @@ else ()
     EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray
     NUMPROCS ${AT_LEAST_FOUR_TASKS}
     TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_lossycompress
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_lossycompress
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
   add_mpi_test(test_darray_append
     EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_append
     NUMPROCS ${AT_LEAST_FOUR_TASKS}
diff --git a/tests/cunit/Makefile.am b/tests/cunit/Makefile.am
index 7a8edd6194..f7afb15a63 100644
--- a/tests/cunit/Makefile.am
+++ b/tests/cunit/Makefile.am
@@ -19,7 +19,8 @@ test_decomp_uneven test_decomps test_rearr test_darray_async_simple	\
 test_darray_async test_darray_async_many test_darray_2sync		\
 test_async_multicomp test_async_multi2 test_async_manyproc		\
 test_darray_fill test_decomp_frame test_perf2 test_async_perf		\
-test_darray_vard test_async_1d test_darray_append test_simple
+test_darray_vard test_async_1d test_darray_append test_simple           \
+test_darray_lossycompress
 
 if RUN_TESTS
 # Tests will run from a bash script.
@@ -41,6 +42,7 @@ test_pioc_unlim_SOURCES = test_pioc_unlim.c test_common.c test_shared.c pio_test
 test_pioc_putget_SOURCES = test_pioc_putget.c test_common.c test_shared.c pio_tests.h
 test_pioc_fill_SOURCES = test_pioc_fill.c test_common.c test_shared.c pio_tests.h
 test_darray_SOURCES = test_darray.c test_common.c pio_tests.h
+test_darray_lossycompress_SOURCES = test_darray_lossycompress.c test_common.c pio_tests.h
 test_darray_append_SOURCES = test_darray_append.c test_common.c pio_tests.h
 test_darray_multi_SOURCES = test_darray_multi.c test_common.c pio_tests.h
 test_darray_multivar_SOURCES = test_darray_multivar.c test_common.c pio_tests.h
diff --git a/tests/cunit/test_async_mpi.c b/tests/cunit/test_async_mpi.c
index 4d6d4d41d7..698284b640 100644
--- a/tests/cunit/test_async_mpi.c
+++ b/tests/cunit/test_async_mpi.c
@@ -63,7 +63,7 @@ int resultlen;
 int get_test_comm(int my_rank, int ntasks, int min_ntasks, int max_ntasks, MPI_Comm *comm)
 {
     int ret;
-    
+
     /* Check that a valid number of processors was specified. */
     if (ntasks < min_ntasks)
     {
@@ -102,7 +102,7 @@ int get_test_comm(int my_rank, int ntasks, int min_ntasks, int max_ntasks, MPI_C
  * This function is called by the IO task.  This function will not
  * return, unless there is an error.
  *
- * @param verbose non-zero to turn on printf statements. 
+ * @param verbose non-zero to turn on printf statements.
  * @param my_rank rank of this task.
  * @param io_rank rank of the IO processor in union_comm.
  * @param component_count number of computation components
@@ -260,7 +260,7 @@ int main(int argc, char **argv)
         MPI_Group world_group;
         MPI_Comm io_comm;
         MPI_Group io_group;
-        int my_io_proc_list[1] = {0}; /* List of processors in IO component. */        
+        int my_io_proc_list[1] = {0}; /* List of processors in IO component. */
         int num_io_procs = 1;
         int num_procs_per_comp[COMPONENT_COUNT] = {1, 1};
         int in_io = my_rank ? 0 : 1; /* Non-zero if this task is in IO. */
@@ -270,12 +270,12 @@ int main(int argc, char **argv)
         MPI_Group group[COMPONENT_COUNT];       /* Group with comp tasks. */
         MPI_Group union_group[COMPONENT_COUNT]; /* Group with IO and comp tasks. */
         int my_proc_list[COMPONENT_COUNT][1] = {{1}, {2}};   /* Tasks for computation components. */
-        MPI_Comm comp_comm[COMPONENT_COUNT];    
+        MPI_Comm comp_comm[COMPONENT_COUNT];
         MPI_Comm union_comm[COMPONENT_COUNT];
         MPI_Comm intercomm[COMPONENT_COUNT];
         int in_cmp[COMPONENT_COUNT] = {0, 0};    /* Is this process in this computation component? */
 	int verbose = 0;    /* Non-zero to turn on printf statements. */
-        
+
         /* Create group for world. */
         if ((ret = MPI_Comm_group(test_comm, &world_group)))
             MPIERR(ret);
@@ -301,7 +301,7 @@ int main(int argc, char **argv)
         if (verbose)
             printf("my_rank %d in_io %d io_rank %d IO %s\n", my_rank, in_io,
                    io_rank, iomain == MPI_ROOT ? "main" : "SERVANT");
-        
+
         /* For each computation component. */
         for (int cmp = 0; cmp < COMPONENT_COUNT; cmp++)
         {
@@ -317,7 +317,7 @@ int main(int argc, char **argv)
             int union_rank = -1;
             int pidx;
             MPI_Comm io_comm2;
-            
+
             /* Create a group for this component. */
             if ((ret = MPI_Group_incl(world_group, 1, my_proc_list[cmp], &group[cmp])))
                 MPIERR(ret);
@@ -345,7 +345,7 @@ int main(int argc, char **argv)
             if ((ret = MPI_Comm_create(test_comm, group[cmp], &comp_comm[cmp])))
                 MPIERR(ret);
             MPI_Group_free(&group[cmp]);
-            
+
             if (in_cmp[cmp])
             {
                 /* Get the rank in this comp comm. */
@@ -355,7 +355,7 @@ int main(int argc, char **argv)
             if (verbose)
                 printf("my_rank %d intracomm created for cmp = %d comp_comm[cmp] = %lld comp_rank = %d\n",
                        my_rank, cmp, (long long int)comp_comm[cmp], comp_rank);
-            
+
             /* If this is the IO component, make a copy of the IO comm for
              * each computational component. */
             if (in_io)
@@ -386,7 +386,7 @@ int main(int argc, char **argv)
                     MPIERR(ret);
                 if (verbose)
                     printf("my_rank %d union_rank %d\n", my_rank, union_rank);
-                
+
                 if (in_io)
                 {
                     /* Create the intercomm from IO to computation component. */
@@ -414,7 +414,7 @@ int main(int argc, char **argv)
         if (in_io)
 	{
 	    int comproot[COMPONENT_COUNT] = {1, 1};
-	    
+
             if ((ret = msg_handler(verbose, my_rank, 0, COMPONENT_COUNT, union_comm, comp_comm,
                                    comproot, io_comm)))
                 ERR(ret);
@@ -429,7 +429,7 @@ int main(int argc, char **argv)
                 {
 		    int ioroot = 0;
 		    int msg = MSG_EXIT;
-		    
+
                     /* if (verbose) */
                     /*     printf("my_rank %d sending exit message on union_comm %d\n", my_rank, union_comm[cmp]); */
                     if ((mpierr = MPI_Send(&msg, 1, MPI_INT, ioroot, 1, union_comm[cmp])))
@@ -443,7 +443,7 @@ int main(int argc, char **argv)
             printf("my_rank %d freeing resources\n", my_rank);
         for (int cmp = 0; cmp < COMPONENT_COUNT; cmp++)
         {
-            if (comp_comm[cmp] != MPI_COMM_NULL)            
+            if (comp_comm[cmp] != MPI_COMM_NULL)
                 MPI_Comm_free(&comp_comm[cmp]);
             if (union_comm[cmp] != MPI_COMM_NULL)
                 MPI_Comm_free(&union_comm[cmp]);
@@ -451,12 +451,12 @@ int main(int argc, char **argv)
                 MPI_Comm_free(&intercomm[cmp]);
         }
         MPI_Group_free(&world_group);
-        if (io_comm != MPI_COMM_NULL)            
-            MPI_Comm_free(&io_comm);            
+        if (io_comm != MPI_COMM_NULL)
+            MPI_Comm_free(&io_comm);
     }
 
     /* Free the MPI communicator for this test. */
-    MPI_Comm_free(&test_comm);    
+    MPI_Comm_free(&test_comm);
 
     /* Finalize MPI. */
     MPI_Finalize();
diff --git a/tests/cunit/test_common.c b/tests/cunit/test_common.c
index 81a6bb1330..cc9f15c31b 100644
--- a/tests/cunit/test_common.c
+++ b/tests/cunit/test_common.c
@@ -80,7 +80,7 @@ int pio_type[NUM_PIO_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, P
 
 /* Attribute test data. */
 signed char byte_att_data[ATT_LEN] = {NC_MAX_BYTE, NC_MIN_BYTE, NC_MAX_BYTE};
-char char_att_data[ATT_LEN] = {NC_MAX_CHAR, 0, NC_MAX_CHAR};
+char char_att_data[ATT_LEN] = {(char) NC_MAX_CHAR, (char) 0, (char) NC_MAX_CHAR};
 short short_att_data[ATT_LEN] = {NC_MAX_SHORT, NC_MIN_SHORT, NC_MAX_SHORT};
 int int_att_data[ATT_LEN] = {NC_MAX_INT, NC_MIN_INT, NC_MAX_INT};
 float float_att_data[ATT_LEN] = {NC_MAX_FLOAT, NC_MIN_FLOAT, NC_MAX_FLOAT};
@@ -95,7 +95,7 @@ unsigned long long uint64_att_data[ATT_LEN] = {NC_MAX_UINT64, 0, NC_MAX_UINT64};
 
 /* Scalar variable test data. */
 signed char byte_scalar_data = NC_MAX_BYTE;
-char char_scalar_data = NC_MAX_CHAR;
+char char_scalar_data = (char) NC_MAX_CHAR;
 short short_scalar_data = NC_MAX_SHORT;
 int int_scalar_data = NC_MAX_INT;
 float float_scalar_data = NC_MAX_FLOAT;
@@ -441,13 +441,10 @@ create_nc_sample(int sample, int iosysid, int format, char *filename, int my_ran
     {
     case 0:
         return create_nc_sample_0(iosysid, format, filename, my_rank, ncid);
-        break;
     case 1:
         return create_nc_sample_1(iosysid, format, filename, my_rank, ncid);
-        break;
     case 2:
         return create_nc_sample_2(iosysid, format, filename, my_rank, ncid);
-        break;
     }
     return PIO_EINVAL;
 }
@@ -460,13 +457,10 @@ check_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank
     {
     case 0:
         return check_nc_sample_0(iosysid, format, filename, my_rank, ncid);
-        break;
     case 1:
         return check_nc_sample_1(iosysid, format, filename, my_rank, ncid);
-        break;
     case 2:
         return check_nc_sample_2(iosysid, format, filename, my_rank, ncid);
-        break;
     }
     return PIO_EINVAL;
 }
@@ -828,7 +822,7 @@ int
 check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncidp)
 {
     int ncid;
-    int ret;
+    int ret=PIO_NOERR;
     int ndims, nvars, ngatts, unlimdimid;
     int ndims2, nvars2, ngatts2, unlimdimid2;
     int dimid2;
@@ -998,7 +992,7 @@ check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *nci
             return ERR_CHECK;
     }
 
-    return 0;
+    return ret;
 }
 
 /* Create the decomposition to divide the 3-dimensional sample data
@@ -1122,7 +1116,7 @@ int create_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx,
     sprintf(var_name, "%s_%d", THREED_VAR_NAME, my_comp_idx);
     if ((ret = PIOc_def_var(ncid, var_name, PIO_SHORT, NDIM3, dimid, &varid[2])))
         ERR(ret);
-    
+
     /* End define mode. */
     if ((ret = PIOc_enddef(ncid)))
         ERR(ret);
@@ -1138,7 +1132,7 @@ int create_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx,
     /* Write the 2-D variable with put_var(). */
     if ((ret = PIOc_put_var_short(ncid, 1, data_2d)))
         ERR(ret);
-    
+
     /* Write the 3D data. */
     if (use_darray)
     {
@@ -1156,11 +1150,11 @@ int create_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx,
     {
         PIO_Offset start[NDIM3] = {0, 0, 0};
         PIO_Offset count[NDIM3] = {1, DIM_X_LEN, DIM_Y_LEN};
-        
+
         /* Write a record of the 3-D variable with put_vara(). */
         if ((ret = PIOc_put_vara_short(ncid, varid[2], start, count, data_2d)))
             ERR(ret);
-        
+
         /* Write another record of the 3-D variable with put_vara(). */
         start[0] = 1;
         if ((ret = PIOc_put_vara_short(ncid, varid[2], start, count, data_2d)))
@@ -1281,14 +1275,14 @@ int check_nc_sample_3(int iosysid, int iotype, int my_rank, int my_comp_idx,
     {
         PIO_Offset start[NDIM3] = {0, 0, 0};
         PIO_Offset count[NDIM3] = {1, DIM_X_LEN, DIM_Y_LEN};
-        
+
         /* Read a record of the 3-D variable with get_vara(). */
         if ((ret = PIOc_get_vara_short(ncid, 2, start, count, data_2d)))
             ERR(ret);
         for (int i = 0; i < DIM_X_LEN * DIM_Y_LEN; i++)
             if (data_2d[i] != my_comp_idx + i)
                 ERR(ERR_WRONG);
-        
+
         /* Read another record of the 3-D variable with get_vara(). */
         start[0] = 1;
         if ((ret = PIOc_get_vara_short(ncid, 2, start, count, data_2d)))
diff --git a/tests/cunit/test_darray_2sync.c b/tests/cunit/test_darray_2sync.c
index 657bbdd703..c093165100 100644
--- a/tests/cunit/test_darray_2sync.c
+++ b/tests/cunit/test_darray_2sync.c
@@ -312,8 +312,8 @@ int darray_fill_test(int iosysid, int my_rank, int num_iotypes, int *iotype,
                     unsigned char data_in[elements_per_pe * NUM_COMPUTATION_PROCS];
                     if ((ret = PIOc_get_var_uchar(ncid2, 0, data_in)))
                         AERR(ret);
-                    if (my_rank && data_in[0] != 1 && data_in[1] != -1 && data_in[2] != 2 &&
-                        data_in[3] != -2 && data_in[4] != 3 && data_in[5] != -3)
+                    if (my_rank && data_in[0] != 1 && data_in[1] != 1 && data_in[2] != 2 &&
+                        data_in[3] != 2 && data_in[4] != 3 && data_in[5] != 3)
                         AERR(ERR_WRONG);
                 }
                 break;
@@ -322,8 +322,8 @@ int darray_fill_test(int iosysid, int my_rank, int num_iotypes, int *iotype,
                     unsigned short data_in[elements_per_pe * NUM_COMPUTATION_PROCS];
                     if ((ret = PIOc_get_var_ushort(ncid2, 0, data_in)))
                         AERR(ret);
-                    if (my_rank && data_in[0] != 1 && data_in[1] != -1 && data_in[2] != 2 &&
-                        data_in[3] != -2 && data_in[4] != 3 && data_in[5] != -3)
+                    if (my_rank && data_in[0] != 1 && data_in[1] != 1 && data_in[2] != 2 &&
+                        data_in[3] != 2 && data_in[4] != 3 && data_in[5] != 3)
                         AERR(ERR_WRONG);
                 }
                 break;
@@ -332,8 +332,8 @@ int darray_fill_test(int iosysid, int my_rank, int num_iotypes, int *iotype,
                     unsigned int data_in[elements_per_pe * NUM_COMPUTATION_PROCS];
                     if ((ret = PIOc_get_var_uint(ncid2, 0, data_in)))
                         AERR(ret);
-                    if (my_rank && data_in[0] != 1 && data_in[1] != -1 && data_in[2] != 2 &&
-                        data_in[3] != -2 && data_in[4] != 3 && data_in[5] != -3)
+                    if (my_rank && data_in[0] != 1 && data_in[1] != 1 && data_in[2] != 2 &&
+                        data_in[3] != 2 && data_in[4] != 3 && data_in[5] != 3)
                         AERR(ERR_WRONG);
                 }
                 break;
diff --git a/tests/cunit/test_darray_3d.c b/tests/cunit/test_darray_3d.c
index dc77f40ee3..476dffbb48 100644
--- a/tests/cunit/test_darray_3d.c
+++ b/tests/cunit/test_darray_3d.c
@@ -254,7 +254,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor,
     for (int fmt = 0; fmt < num_flavors; fmt++)
     {
 	int ret;              /* Return code. */
-	
+
         /* Create the filename. */
         sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
 
diff --git a/tests/cunit/test_darray_fill.c b/tests/cunit/test_darray_fill.c
index 0dafa22899..f4d6e64a5b 100644
--- a/tests/cunit/test_darray_fill.c
+++ b/tests/cunit/test_darray_fill.c
@@ -164,7 +164,7 @@ int main(int argc, char **argv)
 #define NUM_TYPES 11
                 int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
                                             PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64};
-                
+
 #endif /* _NETCDF4 */
 
                 /* Determine what data to write. Put value of 42 into
diff --git a/tests/cunit/test_darray_lossycompress.c b/tests/cunit/test_darray_lossycompress.c
new file mode 100644
index 0000000000..041aea633f
--- /dev/null
+++ b/tests/cunit/test_darray_lossycompress.c
@@ -0,0 +1,499 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * @author Ed Hartnett
+ * @date 2/16/17
+ */
+#include <config.h>
+#include <pio.h>
+#include <pio_internal.h>
+#include <pio_tests.h>
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_lossycompress"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The names of variables in the netCDF output files. */
+#define VAR_NAME "Billy-Bob"
+#define VAR_NAME2 "Sally-Sue"
+
+/* Test cases relating to PIOc_write_darray_multi(). */
+#define NUM_TEST_CASES_WRT_MULTI 3
+
+/* Test with and without specifying a fill value to
+ * PIOc_write_darray(). */
+#define NUM_TEST_CASES_FILLVALUE 1
+
+/* This struct allows us to treat float as uint32_t
+ * types. */
+union FU {
+    float f;
+    uint32_t u;
+};
+
+/* This struct allows us to treat double points as uint64_t
+ * types. */
+union DU {
+    double d;
+    uint64_t u;
+};
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/**
+ * Test the darray functionality. Create a netCDF file with 3
+ * dimensions and 1 PIO_INT variable, and use darray to write some
+ * data.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type of the data.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+		int pio_type)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int ncid2;     /* The ncid of the re-opened netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    int varid2;     /* The ID of a netCDF varable of different type. */
+    int wrong_varid = TEST_VAL_42;  /* A wrong ID. */
+    int ret;       /* Return code. */
+    MPI_Datatype mpi_type;
+    int type_size; /* size of a variable of type pio_type */
+    int other_type; /* another variable of the same size but different type */
+    PIO_Offset arraylen = 4;
+    void *fillvalue, *ofillvalue;
+    void *test_data;
+    void *test_data_in;
+    int fillvalue_int = NC_FILL_INT;
+    int test_data_int[arraylen];
+    int test_data_int_in[arraylen];
+    float fillvalue_float = NC_FILL_FLOAT;
+    float test_data_float[arraylen];
+    float test_data_float_in[arraylen] ;
+    double fillvalue_double = NC_FILL_DOUBLE;
+    double test_data_double[arraylen];
+    double test_data_double_in[arraylen];
+#ifdef NC_NOQUANTIZE
+    //    union FU fin;
+
+    //xpect[0].u = 0x3f8e3000;
+    //xpect[1].u = 0x3f800fff;
+    //xpect[2].u = 0x41200000;
+    //xpect[3].u = 0x4640efff;
+//    xpect[4].u = 0x3dfcd000;
+    //double_xpect[0].u = 0x3ff1c60000000000;
+    //double_xpect[1].u = 0x3ff001ffffffffff;
+    //double_xpect[2].u = 0x4023fe0000000000;
+    //double_xpect[3].u = 0x41d265ffffffffff;
+//    double_xpect[4].u = 0x42dc120000000000;
+#endif
+    /* Initialize some data. */
+    for (int f = 0; f < arraylen; f++)
+    {
+	test_data_int[f] = my_rank * 10 + f;
+    }
+    test_data_float[0] = 1.11111111;
+    test_data_float[1] = 1.0;
+    test_data_float[2] = 9.99999999;
+    test_data_float[3] = 12345.67;
+//    test_data_float[4] = .1234567;
+
+    test_data_double[0] = 1.1111111;
+    test_data_double[1] = 1.0;
+    test_data_double[2] = 9.999999999;
+    test_data_double[3] = 1234567890.12345;
+//    test_data_double[4] = 123456789012345.0;
+
+
+
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+	if(flavor[fmt] == PIO_IOTYPE_NETCDF4P || flavor[fmt] == PIO_IOTYPE_NETCDF4C)
+	{
+	    /* Add a couple of extra tests for the
+	     * PIOc_write_darray_multi() function. */
+	    for (int test_multi = 0; test_multi < NUM_TEST_CASES_WRT_MULTI; test_multi++)
+	    {
+		/* Test with/without providing a fill value to PIOc_write_darray(). */
+		for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++)
+		{
+		    /* Create the filename. */
+		    sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME,
+			    flavor[fmt], pio_type, test_multi, provide_fill);
+		    /* Select the fill value and data. */
+		    switch (pio_type)
+		    {
+		    case PIO_INT:
+			fillvalue = provide_fill ? &fillvalue_int : NULL;
+			test_data = test_data_int;
+			test_data_in = test_data_int_in;
+			break;
+		    case PIO_FLOAT:
+			fillvalue = provide_fill ? &fillvalue_float : NULL;
+			test_data = test_data_float;
+			test_data_in = test_data_float_in;
+			break;
+		    case PIO_DOUBLE:
+			fillvalue = provide_fill ? &fillvalue_double : NULL;
+			test_data = test_data_double;
+			test_data_in = test_data_double_in;
+			break;
+		    default:
+			ERR(ERR_WRONG);
+		    }
+
+		    /* Create the netCDF output file. */
+		    if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+			ERR(ret);
+
+		    /* Define netCDF dimensions and variable. */
+		    for (int d = 0; d < NDIM; d++)
+			if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+			    ERR(ret);
+
+		    /* Define a variable. */
+		    if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid)))
+			ERR(ret);
+#ifdef NC_NOQUANTIZE
+		    if(pio_type == PIO_REAL || pio_type == PIO_DOUBLE)
+		    {
+		        if ((ret = PIOc_def_var_quantize(ncid, varid, PIO_QUANTIZE_BITGROOM, 3)))
+		            ERR(ret);
+		    }
+		    else
+		    {
+		        /* this should fail */
+		        if ((ret = PIOc_def_var_quantize(ncid, varid, PIO_QUANTIZE_BITROUND, 5) != PIO_EINVAL ))
+		            ERR(ret);
+
+		    }
+#endif
+		    if ((ret = PIOc_def_var_deflate(ncid, varid, 0, 1, 1)))
+			ERR(ret);
+
+
+		    /* Define a variable with a different type but same size. */
+		    if ((ret = find_mpi_type(pio_type, &mpi_type, &type_size)))
+			ERR(ret);
+		    if (type_size == NETCDF_INT_FLOAT_SIZE)
+			other_type = pio_type == PIO_INT ? PIO_FLOAT : PIO_INT;
+//		else if(type_size == NETCDF_DOUBLE_INT64_SIZE)
+//		    other_type = pio_type == PIO_INT64 ? PIO_DOUBLE : PIO_INT64;
+		    else
+			other_type = 0; /* skip the test */
+		    switch (other_type)
+		    {
+		    case PIO_INT:
+			ofillvalue = provide_fill ? &fillvalue_int : NULL;
+			break;
+		    case PIO_FLOAT:
+			ofillvalue = provide_fill ? &fillvalue_float : NULL;
+			break;
+		    default:
+			break;
+		    }
+		    if (other_type && (ret = PIOc_def_var(ncid, VAR_NAME2, other_type, NDIM, dimids, &varid2)))
+			ERR(ret);
+
+		    /* End define mode. */
+		    if ((ret = PIOc_enddef(ncid)))
+			ERR(ret);
+
+		    /* Set the value of the record dimension. */
+		    if ((ret = PIOc_setframe(ncid, varid, 0)))
+			ERR(ret);
+		    if (other_type && (ret = PIOc_setframe(ncid, varid2, 0)))
+			ERR(ret);
+
+		    int frame = 0;
+		    int flushtodisk = test_multi - 1;
+		    if (!test_multi)
+		    {
+			/* These should not work. */
+			if (PIOc_write_darray(ncid + TEST_VAL_42, varid, ioid, arraylen, test_data, fillvalue) != PIO_EBADID)
+			    ERR(ERR_WRONG);
+			if (PIOc_write_darray(ncid, varid, ioid + TEST_VAL_42, arraylen, test_data, fillvalue) != PIO_EBADID)
+			    ERR(ERR_WRONG);
+			if (PIOc_write_darray(ncid, varid, ioid, arraylen - 1, test_data, fillvalue) != PIO_EINVAL)
+			    ERR(ERR_WRONG);
+			if (PIOc_write_darray(ncid, TEST_VAL_42, ioid, arraylen, test_data, fillvalue) != PIO_ENOTVAR)
+			    ERR(ERR_WRONG);
+
+			/* This should work - library type conversion */
+			if (other_type && (ret = PIOc_write_darray(ncid, varid2, ioid, arraylen, test_data, ofillvalue)))
+			    ERR(ret);
+
+			/* Write the data. */
+//                        printf("test_data[0] = %f\n",test_data_float[0]);
+			if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+			    ERR(ret);
+
+		    }
+		    else
+		    {
+			int varid_big = PIO_MAX_VARS + TEST_VAL_42;
+
+			/* These will not work. */
+			if (PIOc_write_darray_multi(ncid + TEST_VAL_42, &varid, ioid, 1, arraylen, test_data, &frame,
+						    fillvalue, flushtodisk) != PIO_EBADID)
+			    ERR(ERR_WRONG);
+			if (PIOc_write_darray_multi(ncid, NULL, ioid, 1, arraylen, test_data, &frame,
+						    fillvalue, flushtodisk) != PIO_EINVAL)
+			    ERR(ERR_WRONG);
+			if (PIOc_write_darray_multi(ncid, &varid, ioid + TEST_VAL_42, 1, arraylen, test_data, &frame,
+						    fillvalue, flushtodisk) != PIO_EBADID)
+			    ERR(ERR_WRONG);
+			if (PIOc_write_darray_multi(ncid, &varid, ioid, -1, arraylen, test_data, &frame,
+						    fillvalue, flushtodisk) != PIO_EINVAL)
+			    ERR(ERR_WRONG);
+			if (PIOc_write_darray_multi(ncid, &varid_big, ioid, 1, arraylen, test_data, &frame,
+						    fillvalue, flushtodisk) != PIO_ENOTVAR)
+			    ERR(ERR_WRONG);
+			if (PIOc_write_darray_multi(ncid, &wrong_varid, ioid, 1, arraylen, test_data, &frame,
+						    fillvalue, flushtodisk) != PIO_ENOTVAR)
+			    ERR(ERR_WRONG);
+//		    pio_setloglevel(0);
+
+		    /* This should work - library type conversion */
+			if (other_type && (ret = PIOc_write_darray_multi(ncid, &varid2, ioid, 1, arraylen, test_data, &frame,
+									 fillvalue, flushtodisk)))
+			    ERR(ret);
+
+			/* Write the data with the _multi function. */
+			if ((ret = PIOc_write_darray_multi(ncid, &varid, ioid, 1, arraylen, test_data, &frame,
+							   fillvalue, flushtodisk)))
+			    ERR(ret);
+		    }
+
+		    /* Close the netCDF file. */
+		    if ((ret = PIOc_closefile(ncid)))
+			ERR(ret);
+
+
+		    /* Reopen the file. */
+		    if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
+			ERR(ret);
+
+		    PIO_Offset dimlen;
+		    /* check the unlimited dim size - it should be 1 */
+		    if ((ret = PIOc_inq_dimlen(ncid2, dimids[0], &dimlen)))
+			ERR(ret);
+		    if (dimlen != 1)
+			ERR(ERR_WRONG);
+
+		    /* These should not work. */
+		    if (PIOc_read_darray(ncid2 + TEST_VAL_42, varid, ioid, arraylen,
+					 test_data_in) != PIO_EBADID)
+			ERR(ERR_WRONG);
+		    if (PIOc_read_darray(ncid2, varid, ioid + TEST_VAL_42, arraylen,
+					 test_data_in) != PIO_EBADID)
+			ERR(ERR_WRONG);
+
+		    /* Set the record number. */
+		    if ((ret = PIOc_setframe(ncid2, varid, 0)))
+			ERR(ret);
+
+		    /* Read the data. */
+		    if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
+			ERR(ret);
+
+		    /* /\* Read the data. *\/ */
+		    /* if ((ret = PIOc_get_vard(ncid2, varid, ioid, 0, (void *)test_data_in))) */
+		    /*     ERR(ret); */
+
+		    /* Check the results. */
+                    /* HOW does one test a lossy compression algorythm? */
+#ifdef NC_NOQUANTIZE
+		    for (int f = 0; f < arraylen; f++)
+		    {
+			switch (pio_type)
+			{
+			case PIO_INT:
+			    if (test_data_int_in[f] != test_data_int[f])
+				return ERR_WRONG;
+			    break;
+			case PIO_FLOAT:
+			    /*
+			      We do not expect an exact match for lossy data - so how do we test?
+			    if (test_data_float_in[f] != test_data_float[f])
+				return ERR_WRONG;
+			    */
+			    break;
+			case PIO_DOUBLE:
+			  /*
+			    if (test_data_double_in[f] != test_data_double[f])
+				return ERR_WRONG;
+			    */
+			  break;
+			default:
+			  ERR(ERR_WRONG);
+			}
+		    }
+#else
+                    printf("Quantize support was not available in the netcdf build and thus is not tested\n");
+#endif
+
+		    /* Try to write, but it won't work, because we opened file read-only. */
+		    if (!test_multi)
+		    {
+			if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, fillvalue) != PIO_EPERM)
+			    ERR(ERR_WRONG);
+		    }
+		    else
+		    {
+			if (PIOc_write_darray_multi(ncid2, &varid, ioid, 1, arraylen, test_data, &frame,
+						    fillvalue, flushtodisk) != PIO_EPERM)
+			    ERR(ERR_WRONG);
+		    }
+		    /* Close the netCDF file. */
+		    if ((ret = PIOc_closefile(ncid2)))
+			ERR(ret);
+		} /* next fillvalue test case */
+	    } /* next test multi */
+	}/* only netcdf4 iotypes allowed */
+    } /* next iotype */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests.
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the communicator the test is running on.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+		    MPI_Comm test_comm)
+{
+#define NUM_TYPES_TO_TEST 2
+    int ioid;
+    char filename[PIO_MAX_NAME + 1];
+    int pio_type[NUM_TYPES_TO_TEST] = {PIO_FLOAT, PIO_DOUBLE};
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int ret; /* Return code. */
+
+    for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+    {
+	/* This will be our file name for writing out decompositions. */
+	sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank,
+		*flavor, pio_type[t]);
+
+	/* Decompose the data over the tasks. */
+	if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d,
+					   &ioid, pio_type[t])))
+	    return ret;
+
+	/* Run a simple darray test. */
+	if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t])))
+	    return ret;
+
+	/* Free the PIO decomposition. */
+	if ((ret = PIOc_freedecomp(iosysid, ioid)))
+	    ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+#define NUM_REARRANGERS_TO_TEST 2
+    int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+			      MIN_NTASKS, -1, &test_comm)))
+	ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+	return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+	int iosysid;  /* The ID for the parallel I/O system. */
+	int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+	int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+	int ret;      /* Return code. */
+
+	/* Figure out iotypes. */
+	if ((ret = get_iotypes(&num_flavors, flavor)))
+	    ERR(ret);
+
+	for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+	{
+	    /* Initialize the PIO IO system. This specifies how
+	     * many and which processors are involved in I/O. */
+	    if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+					   ioproc_start, rearranger[r], &iosysid)))
+		return ret;
+
+	    /* Run tests. */
+	    if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm)))
+		return ret;
+
+	    /* Finalize PIO system. */
+	    if ((ret = PIOc_free_iosystem(iosysid)))
+		return ret;
+	} /* next rearranger */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    if ((ret = pio_test_finalize(&test_comm)))
+	return ret;
+    /* if ((ret = pio_test_finalize2(&test_comm, TEST_NAME))) */
+    /*     return ret; */
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/tests/cunit/test_darray_multivar3.c b/tests/cunit/test_darray_multivar3.c
index 21b2d88724..f44e389765 100644
--- a/tests/cunit/test_darray_multivar3.c
+++ b/tests/cunit/test_darray_multivar3.c
@@ -214,7 +214,7 @@ int test_multivar_darray(int iosysid, int ioid, int ioid_float, int num_flavors,
                     if (file_fv_int != custom_fillvalue_int)
                         return ERR_WRONG;
                 }
-                
+
                 /* Check the float fill value. */
                 if ((ret = PIOc_get_var1_float(ncid2, varid[2], idx, &file_fv_float)))
                     return ret;
diff --git a/tests/cunit/test_decomp_frame.c b/tests/cunit/test_decomp_frame.c
index 54969a8b4c..3aa893d6e7 100644
--- a/tests/cunit/test_decomp_frame.c
+++ b/tests/cunit/test_decomp_frame.c
@@ -165,7 +165,7 @@ int main(int argc, char **argv)
 #define NUM_TYPES 11
                 int test_type[NUM_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
                                             PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64};
-                
+
 #endif /* _NETCDF4 */
 
                 /* Determine what data to write. Put value of 42 into
diff --git a/tests/cunit/test_decomp_uneven.c b/tests/cunit/test_decomp_uneven.c
index 8008c06d2d..c916d3abdf 100644
--- a/tests/cunit/test_decomp_uneven.c
+++ b/tests/cunit/test_decomp_uneven.c
@@ -128,7 +128,7 @@ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor,
     for (int fmt = 0; fmt < 1; fmt++)
     {
         int ret;              /* Return code. */
-      
+
         /* Create the filename. */
         snprintf(filename, PIO_MAX_NAME, "decomp_%s_pio_type_%d_dims_%d_x_%d_x_%d.nc",
 		 TEST_NAME, pio_type, dim_len[0], dim_len[1], dim_len[2]);
@@ -329,7 +329,7 @@ int main(int argc, char **argv)
                 for (int dc = 0; dc < NUM_DIM_COMBOS_TO_TEST; dc++)
                 {
 		    int test_type[NUM_TYPES_TO_TEST] = {PIO_INT};
-		    
+
                     /* What is length of map for this combo? */
                     int full_maplen = 1;
                     for (int d = 0; d < NDIM3; d++)
diff --git a/tests/cunit/test_iosystem2_simple2.c b/tests/cunit/test_iosystem2_simple2.c
index d4db1540f7..26f487682f 100644
--- a/tests/cunit/test_iosystem2_simple2.c
+++ b/tests/cunit/test_iosystem2_simple2.c
@@ -49,7 +49,7 @@ int main(int argc, char **argv)
     {
 	int num_flavors; /* Number of PIO netCDF flavors in this build. */
 	int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
-	
+
         /* Figure out iotypes. */
         if ((ret = get_iotypes(&num_flavors, flavor)))
             ERR(ret);
diff --git a/tests/cunit/test_perf2.c b/tests/cunit/test_perf2.c
index 523b00227c..ea77cf41d7 100644
--- a/tests/cunit/test_perf2.c
+++ b/tests/cunit/test_perf2.c
@@ -160,7 +160,7 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
 
         /* How many megabytes will we write? */
         num_megabytes = (NUM_TIMESTEPS * X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN * sizeof(int))/(MILLION);
-        
+
         sprintf(filename, "data_%s_iotype_%d_rearr_%d.nc", TEST_NAME, flavor[fmt],
                 rearranger);
         /* Create the filename. Use the same filename for all, so we
@@ -268,7 +268,7 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
         /* Now reopen the file and re-read the data. */
         {
             int *test_data_in;
-            
+
             if (!(test_data_in = malloc(sizeof(int) * arraylen)))
                 ERR(PIO_ENOMEM);
 
@@ -319,9 +319,9 @@ test_darray(int iosysid, int ioid, int num_flavors, int *flavor,
 
             /* Free resources. */
             free(test_data_in);
-            
+
         } /* re-reading file */
-        
+
         if (!my_rank)
             printf("%d,\t%d,\t%s,\t%s,\t%s,\t%8.3f,\t%8.3f,\t%8.1f,\t%8.3f,\t%8.3f\n", ntasks, num_io_procs,
                    (rearranger == 1 ? "box" : "subset"), (provide_fill ? "fill" : "nofill"),
diff --git a/tests/cunit/test_pioc.c b/tests/cunit/test_pioc.c
index a9289c0198..d3dda4503a 100644
--- a/tests/cunit/test_pioc.c
+++ b/tests/cunit/test_pioc.c
@@ -7,6 +7,7 @@
 #include <pio.h>
 #include <pio_internal.h>
 #include <pio_tests.h>
+#include <pio_meta.h>
 
 /* The number of tasks this test should run on. */
 #define TARGET_NTASKS 4
@@ -1048,7 +1049,7 @@ int test_empty_files(int iosysid, int num_flavors, int *flavor, int my_rank)
         if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
             ERR(ret);
         sprintf(filename, "%s_empty_%s.nc", TEST_NAME, iotype_name);
-        
+
         if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
             ERR(ret);
 
@@ -1076,10 +1077,10 @@ int test_empty_files(int iosysid, int num_flavors, int *flavor, int my_rank)
 /* Check that the fill values are correctly reported by find_var_fill().
  *
  * @param ncid the ID of the open test file.
- * @param ntypes the number ot types we are testing. 
+ * @param ntypes the number ot types we are testing.
  * @param use_custom_fill true if custom fill values were used.
  * @param my_rank rank of this task.
- * @return 0 on success. 
+ * @return 0 on success.
  */
 int check_fillvalues(int ncid, int num_types, int use_custom_fill, int my_rank)
 {
@@ -1128,7 +1129,7 @@ int check_fillvalues(int ncid, int num_types, int use_custom_fill, int my_rank)
 
     if ((ret = pio_get_file(ncid, &file)))
         ERR(ret);
-            
+
     for (int v = 0; v < num_types; v++)
     {
         var_desc_t *vdesc;
@@ -1136,7 +1137,7 @@ int check_fillvalues(int ncid, int num_types, int use_custom_fill, int my_rank)
         /* Get the var info. */
         if ((ret = get_var_desc(v, &file->varlist, &vdesc)))
             ERR(ret);
-                
+
         /* Check the fill value with this internal function. */
         if ((ret = find_var_fillvalue(file, v, vdesc)))
             ERR(ret);
@@ -1194,7 +1195,7 @@ int check_fillvalues(int ncid, int num_types, int use_custom_fill, int my_rank)
 
     return PIO_NOERR;
 }
-        
+
 /* Test the internal function that determins a var's fillvalue.
  *
  * @param iosysid the iosystem ID that will be used for the test.
@@ -1571,10 +1572,10 @@ int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank)
                 ERR(ret);
 
             /* Setting deflate works with parallel iotype starting
-	     * with netcdf-c-4.7.4. If present, HAVE_PAR_FILTERS will
+	     * with netcdf-c-4.7.4. If present, PIO_HAS_PAR_FILTERS will
 	     * be defined. */
 	    ret = PIOc_def_var_deflate(ncid, 0, 0, 1, 1);
-#ifdef HAVE_PAR_FILTERS
+#ifdef PIO_HAS_PAR_FILTERS
 	    if (ret)
 		ERR(ret);
 #else
@@ -1613,16 +1614,16 @@ int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank)
                     ERR(ERR_AWFUL);
 
             /* For parallel netCDF-4, we turned on deflate above, if
-	     * HAVE_PAR_FILTERS is defined. */
+	     * PIO_HAS_PAR_FILTERS is defined. */
             if (flavor[fmt] == PIO_IOTYPE_NETCDF4P)
 	    {
-#ifdef HAVE_PAR_FILTERS
+#ifdef PIO_HAS_PAR_FILTERS
 		if (shuffle || !deflate || deflate_level != 1)
                     ERR(ERR_AWFUL);
 #else
 		if (shuffle || deflate)
                     ERR(ERR_AWFUL);
-#endif /* HAVE_PAR_FILTERS */
+#endif /* PIO_HAS_PAR_FILTERS */
 	    }
 
             /* Check setting the chunk cache for the variable. */
diff --git a/tests/cunit/test_pioc_putget.c b/tests/cunit/test_pioc_putget.c
index ce2732718f..748ed85c4f 100644
--- a/tests/cunit/test_pioc_putget.c
+++ b/tests/cunit/test_pioc_putget.c
@@ -823,7 +823,7 @@ int test_read_att(int ncid, int *varid, int flavor)
       unsigned int uint_array_in[ATT_LEN];
       long long int64_array_in[ATT_LEN];
       unsigned long long uint64_array_in[ATT_LEN];
-      
+
         if ((ret = PIOc_get_att_uchar(ncid, varid[7], UCHAR_ATT_NAME, ubyte_array_in)))
             return ret;
         if ((ret = PIOc_get_att_ushort(ncid, varid[8], USHORT_ATT_NAME, ushort_array_in)))
@@ -2059,5 +2059,4 @@ int main(int argc, char **argv)
     return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, -1,
                          TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS);
 
-    return 0;
 }
diff --git a/tests/cunit/test_pioc_unlim.c b/tests/cunit/test_pioc_unlim.c
index 88621523e3..be56ee1d84 100644
--- a/tests/cunit/test_pioc_unlim.c
+++ b/tests/cunit/test_pioc_unlim.c
@@ -102,24 +102,24 @@ int create_test_file(int iosysid, int ioid, int iotype, int my_rank, int *ncid,
 
     /* Create the filename. */
     sprintf(filename, "%s_iotype_%d.nc", TEST_NAME, iotype);
-    
+
     /* Create the netCDF output file. */
     if ((ret = PIOc_createfile(iosysid, ncid, &iotype, filename, PIO_CLOBBER)))
         ERR(ret);
-    
+
     /* Define netCDF dimensions and variable. */
     for (int d = 0; d < NDIM; d++)
         if ((ret = PIOc_def_dim(*ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
             ERR(ret);
-    
+
     /* Define a variable. */
     if ((ret = PIOc_def_var(*ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, varid)))
         ERR(ret);
-    
+
     /* End define mode. */
     if ((ret = PIOc_enddef(*ncid)))
         ERR(ret);
-    
+
     return PIO_NOERR;
 }
 
@@ -147,11 +147,11 @@ int run_multiple_unlim_test(int iosysid, int ioid, int iotype, int my_rank,
 
     /* Add unlimited dimension. */
     if ((ret = PIOc_def_dim(ncid, UDIM1_NAME, NC_UNLIMITED, &dimid[0])))
-        ERR(ret);        
+        ERR(ret);
 
     /* Add another unlimited dimension. */
     if ((ret = PIOc_def_dim(ncid, UDIM2_NAME, NC_UNLIMITED, &dimid[1])))
-        ERR(ret);        
+        ERR(ret);
 
     /* Check for correctness. */
     if ((ret = PIOc_inq_unlimdims(ncid, &nunlimdims, unlimdimids)))
@@ -168,7 +168,7 @@ int run_multiple_unlim_test(int iosysid, int ioid, int iotype, int my_rank,
     {
         int nunlimdims;
         int unlimdimids[NUM_UNLIM_DIMS];
-        
+
         /* These should also work. */
         if ((ret = PIOc_inq_unlimdims(ncid, NULL, NULL)))
             ERR(ret);
@@ -191,7 +191,7 @@ int run_multiple_unlim_test(int iosysid, int ioid, int iotype, int my_rank,
     if (PIOc_def_var(ncid, VAR_NAME2, PIO_INT, NUM_UNLIM_DIMS, unlimdimids,
                      &varid) != PIO_EINVAL)
         ERR(ERR_WRONG);
-    
+
     /* Close the file. */
     if ((PIOc_closefile(ncid)))
         return ret;
@@ -203,10 +203,10 @@ int run_multiple_unlim_test(int iosysid, int ioid, int iotype, int my_rank,
         int ncid;
         int dimids[NUM_UNLIM_DIMS];
         int varid;
-        
+
         if ((ret = nc_create(NETCDF4_UNLIM_FILE_NAME, NC_CLOBBER|NC_NETCDF4, &ncid)))
             ERR(ret);
-        
+
         if ((ret = nc_def_dim(ncid, DIM_NAME1, NC_UNLIMITED, &dimids[0])))
             ERR(ret);
         if ((ret = nc_def_dim(ncid, DIM_NAME2, NC_UNLIMITED, &dimids[1])))
@@ -241,7 +241,7 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te
 
     if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
         MPIERR(ret);
-    
+
     /* This will be our file name for writing out decompositions. */
     sprintf(filename, "decomp_%d.txt", my_rank);
 
@@ -273,7 +273,7 @@ int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm te
                 return ret;
             if (vdesc->record != 1)
                 return ERR_WRONG;
-            
+
             if ((PIOc_closefile(ncid)))
                 return ret;
 
diff --git a/tests/doftests/CMakeLists.txt b/tests/doftests/CMakeLists.txt
index 9e0825978b..4940b32d58 100644
--- a/tests/doftests/CMakeLists.txt
+++ b/tests/doftests/CMakeLists.txt
@@ -16,4 +16,3 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
   #    target_compile_options (gptl
   #        PRIVATE -mismatch_all)
 endif ()
-
diff --git a/tests/doftests/dofcopy.F90 b/tests/doftests/dofcopy.F90
index e676ba58da..1f2bbc54e5 100644
--- a/tests/doftests/dofcopy.F90
+++ b/tests/doftests/dofcopy.F90
@@ -6,7 +6,7 @@ program dofcopy
   use mpi
 #endif
   use pio
-  
+
   implicit none
 #ifdef NO_MPIMOD
 #include <mpif.h>
@@ -36,7 +36,7 @@ program dofcopy
   endif
 
   CALL get_command_argument(1, infile)
-  
+
   call pio_readdof(trim(infile), ndims, gdims, compmap, MPI_COMM_WORLD)
 
   if(mype < npe) then
@@ -44,7 +44,7 @@ program dofcopy
 
      call PIO_InitDecomp(iosystem, PIO_INT, gdims, compmap, iodesc, rearr=rearr)
      write(outfile, *) trim(infile)//".nc"
-     call PIO_write_nc_dof(iosystem, outfile, PIO_64BIT_DATA, iodesc, ierr) 
+     call PIO_write_nc_dof(iosystem, outfile, PIO_64BIT_DATA, iodesc, ierr)
      call PIO_finalize(iosystem, ierr)
   endif
 
diff --git a/tests/general/CMakeLists.txt b/tests/general/CMakeLists.txt
index 80fa196593..83b6ec7563 100644
--- a/tests/general/CMakeLists.txt
+++ b/tests/general/CMakeLists.txt
@@ -24,6 +24,7 @@ SET(GENERATED_SRCS pio_file_simple_tests.F90
   pio_decomp_fillval.F90
   pio_iosystem_tests.F90
   pio_iosystem_async_tests.F90
+  pio_filter_tests.F90
   pio_iosystem_tests2.F90
   pio_iosystem_tests3.F90)
 
@@ -38,7 +39,7 @@ foreach (SRC_FILE IN LISTS GENERATED_SRCS)
 endforeach ()
 
 if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
-  add_definitions(-ffree-line-length-none)
+  add_definitions(-ffree-line-length-none -Wno-conversion)
 endif()
 
 if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
@@ -56,7 +57,7 @@ endif()
 #==============================================================================
 
 # Test Timeout (4 min = 240 sec)
-set (DEFAULT_TEST_TIMEOUT 480)
+set (DEFAULT_TEST_TIMEOUT 600)
 
 add_library(pio_tutil util/pio_tutil.F90)
 target_link_libraries(pio_tutil piof pioc)
@@ -100,24 +101,6 @@ else ()
     TIMEOUT ${DEFAULT_TEST_TIMEOUT})
 endif ()
 
-#===== pio_file_simple_tests =====
-add_executable (pio_file_simple_tests EXCLUDE_FROM_ALL
-  pio_file_simple_tests.F90)
-target_link_libraries (pio_file_simple_tests pio_tutil)
-add_dependencies (tests pio_file_simple_tests)
-
-if (PIO_USE_MPISERIAL)
-  add_test(NAME pio_file_simple_tests
-    COMMAND pio_file_simple_tests)
-  set_tests_properties(pio_file_simple_tests
-    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
-else ()
-  add_mpi_test(pio_file_simple_tests
-    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_simple_tests
-    NUMPROCS 2
-    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
-endif ()
-
 #===== pio_file_fail =====
 add_executable (pio_file_fail EXCLUDE_FROM_ALL
   pio_file_fail.F90)
@@ -140,6 +123,24 @@ else ()
     TIMEOUT ${DEFAULT_TEST_TIMEOUT})
 endif ()
 
+#===== pio_file_simple_tests =====
+add_executable (pio_file_simple_tests EXCLUDE_FROM_ALL
+  pio_file_simple_tests.F90)
+target_link_libraries (pio_file_simple_tests pio_tutil)
+add_dependencies (tests pio_file_simple_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_file_simple_tests
+    COMMAND pio_file_simple_tests)
+  set_tests_properties(pio_file_simple_tests
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_file_simple_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_simple_tests
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
 #===== ncdf_simple_tests =====
 add_executable (ncdf_simple_tests EXCLUDE_FROM_ALL
   ncdf_simple_tests.F90)
@@ -255,6 +256,8 @@ else ()
 	NUMPROCS 4
 	TIMEOUT ${DEFAULT_TEST_TIMEOUT})
 endif ()
+# Test times out in github action.
+set_tests_properties(pio_rearr_opts PROPERTIES LABELS "skipforspack")
 
 #===== pio_rearr_opts2 =====
 add_executable (pio_rearr_opts2 EXCLUDE_FROM_ALL
@@ -814,6 +817,18 @@ if (NOT PIO_USE_MPISERIAL)
     TIMEOUT ${DEFAULT_TEST_TIMEOUT})
 endif ()
 
+add_executable (pio_filter_tests EXCLUDE_FROM_ALL
+  pio_filter_tests.F90)
+target_link_libraries (pio_filter_tests pio_tutil)
+add_dependencies (tests pio_filter_tests)
+
+if (NOT PIO_USE_MPISERIAL AND PIO_USE_PARALLEL_FILTERS)
+  add_mpi_test(pio_filter_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_filter_tests
+    NUMPROCS 5
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
 
 
 #===== pio_iosystems_test2 =====
diff --git a/tests/general/README.md b/tests/general/README.md
index 8353978988..0fa10a65c7 100644
--- a/tests/general/README.md
+++ b/tests/general/README.md
@@ -5,7 +5,7 @@
 * Improved testcase code readability : Most of the boiler plate code is generated by the framework
 * Better error checking and reporting : User friendly parallel asserts
 * More testing with less code : Templated tests, auto generation of boiler plate PIO code
-* Easy integration with CTest 
+* Easy integration with CTest
 
 # Writing a test
 
@@ -81,7 +81,7 @@
 
      call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
 
-     ... 
+     ...
 
     PIO_TF_AUTO_tEST_SUB_END templated_hellow
 
@@ -123,7 +123,7 @@
 * Add the testcase stub source into `GENERATED_SRCS`
 * Add a CMake build step : To generate the test executable
 
-        add_executable (pio_init_finalize EXCLUDE_FROM_ALL 
+        add_executable (pio_init_finalize EXCLUDE_FROM_ALL
             pio_init_finalize.F90
             ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
         target_link_libraries (pio_init_finalize piof)
@@ -137,6 +137,6 @@
                 TIMEOUT ${DEFAULT_TEST_TIMEOUT})
 
 # Debugging failed tests
- 
+
 * Run the test suite using ctest with the "--verbose" option. This outputs a lot of information from the testing framework (to stdout), including the various tests run and the reason for failure. All information in the output from the PIO testing framework is prepended with the `PIO_TF: ` tag. To run a single failing test use the "-R" option available with ctest (e.g. To only run "pio_iosystem_tests3", ctest --verbose -R pio_iosystem_tests3).
 * Run the failing test manually (using the MPI job launcher) and debug it. Also consider running the test with PIO log level > 0 (mpiexec -n 4 ./tests/general/pio_iosystem_tests3 --pio-tf-log-level=6)
diff --git a/tests/general/ncdf_fail.F90.in b/tests/general/ncdf_fail.F90.in
index eed98aad6f..74a8afd6ec 100644
--- a/tests/general/ncdf_fail.F90.in
+++ b/tests/general/ncdf_fail.F90.in
@@ -52,21 +52,25 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_redef_twice
   type(file_desc_t) :: pio_file
   integer :: ret
 
+  ret = PIO_NOERR
+  ! as of netcdf 8.4.0 netcdf4p and netcdf4c do not generate an error when redef is called twice.
   ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_write)
   PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(tgv_fname))
 
   ! A simple redef and then enddef
   ret = PIO_redef(pio_file)
   PIO_TF_CHECK_ERR(ret, "Failed to enter redef mode" // trim(tgv_fname))
-
   ret = PIO_redef(pio_file)
-  PIO_TF_PASSERT(ret /= PIO_NOERR, "Entering redef twice did not fail as expected")
-
+  if(tgv_iotype == PIO_IOTYPE_PNETCDF .or. tgv_iotype == PIO_IOTYPE_NETCDF) then
+     PIO_TF_PASSERT(ret /= PIO_NOERR, "Entering redef twice did not fail as expected")
+  else
+     PIO_TF_CHECK_ERR(ret, "Entering redef twice did not pass as expected")
+  endif
   ret = PIO_enddef(pio_file)
   PIO_TF_CHECK_ERR(ret, "Failed to end redef mode" // trim(tgv_fname))
-
   call PIO_closefile(pio_file)
 
+
 PIO_TF_AUTO_TEST_SUB_END test_redef_twice
 
 PIO_TF_TEST_DRIVER_BEGIN
diff --git a/tests/general/ncdf_inq.F90.in b/tests/general/ncdf_inq.F90.in
index 9745bd1b8e..983612a8a1 100644
--- a/tests/general/ncdf_inq.F90.in
+++ b/tests/general/ncdf_inq.F90.in
@@ -65,7 +65,7 @@ SUBROUTINE test_teardown(ret)
   integer, intent(out) :: ret
 
   ret = PIO_NOERR
-  call PIO_deletefile(pio_tf_iosystem_, tgv_fname)  
+  call PIO_deletefile(pio_tf_iosystem_, tgv_fname)
 END SUBROUTINE test_teardown
 
 SUBROUTINE test_inq_var(pio_file, ret)
@@ -118,7 +118,7 @@ SUBROUTINE test_inq_dim(pio_file, ret)
   PIO_TF_CHECK_ERR(ret, "Failed to inq dimid :"//trim(tgv_fname))
 
   ret = pio_inq_dimname(pio_file, dim_id, dim_name)
-  PIO_TF_PASSERT(dim_name .eq. tgv_dim_name, "Dim name is not the expected value") 
+  PIO_TF_PASSERT(dim_name .eq. tgv_dim_name, "Dim name is not the expected value")
 
   ret = pio_inq_dimlen(pio_file, dim_id, dim_len)
   PIO_TF_PASSERT(dim_len == TGV_DIM_LEN, "Dim length is not the expected value")
@@ -157,7 +157,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_inq
   character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
   integer :: num_iotypes
   integer :: i, ret
-  
+
   num_iotypes = 0
   call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
   do i=1,num_iotypes
diff --git a/tests/general/ncdf_simple_tests.F90.in b/tests/general/ncdf_simple_tests.F90.in
index 12368f1300..26650e3e3d 100644
--- a/tests/general/ncdf_simple_tests.F90.in
+++ b/tests/general/ncdf_simple_tests.F90.in
@@ -123,7 +123,10 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_data_conversion
   dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_
   compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps
   wbuf = pio_tf_world_rank_;
-  exp_val = pio_tf_world_rank_;
+
+! Type conversions here to avoid gfortran warnings
+
+  exp_val = pio_tf_world_rank_
 
   ! Set the decomposition for writing data as PIO_int
   call PIO_initdecomp(pio_tf_iosystem_, PIO_int, dims, compdof, wiodesc)
diff --git a/tests/general/pio_decomp_frame_tests.F90.in b/tests/general/pio_decomp_frame_tests.F90.in
index 70edaf6fbb..8b168274a9 100644
--- a/tests/general/pio_decomp_frame_tests.F90.in
+++ b/tests/general/pio_decomp_frame_tests.F90.in
@@ -1,7 +1,7 @@
 #include "config.h"
 ! Get a 3D column decomposition
 ! If force_rearrange is FALSE, the decomposition is such that
-! # All even procs have VEC_HGT_SZ blocks of 
+! # All even procs have VEC_HGT_SZ blocks of
 ! (VEC_COL_SZ rows x VEC_ROW_SZ columns) elements
 ! # All odd procs have  VEC_HGT_SZ blocks of
 ! (VEC_COL_SZ rows x VEC_ROW_SZ + 1 columns) elements
diff --git a/tests/general/pio_decomp_tests.F90.in b/tests/general/pio_decomp_tests.F90.in
index cb6f939869..deeadcab15 100644
--- a/tests/general/pio_decomp_tests.F90.in
+++ b/tests/general/pio_decomp_tests.F90.in
@@ -62,7 +62,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_darray
   filename = "test_pio_decomp_simple_tests.testfile"
   do i=1,num_iotypes
     PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
-    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
     PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
 
     ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
@@ -163,7 +163,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_const_buf_sz
   filename = "test_pio_decomp_simple_tests.testfile"
   do i=1,num_iotypes
     PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
-    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
     PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
 
     ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
@@ -230,11 +230,11 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_reuse_decomp
   do i=1,num_iotypes
     PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
     ierr = PIO_createfile(pio_tf_iosystem_, pio_file1, iotypes(i),&
-                            filename1, PIO_CLOBBER) 
+                            filename1, PIO_CLOBBER)
     PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename1))
 
     ierr = PIO_createfile(pio_tf_iosystem_, pio_file2, iotypes(i),&
-                            filename2, PIO_CLOBBER) 
+                            filename2, PIO_CLOBBER)
     PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename1))
 
     ierr = PIO_def_dim(pio_file1, 'PIO_TF_test_dim', dims(1), pio_dim_file1)
diff --git a/tests/general/pio_decomp_tests_3d.F90.in b/tests/general/pio_decomp_tests_3d.F90.in
index 17a20b2d0e..a8f0b5961f 100644
--- a/tests/general/pio_decomp_tests_3d.F90.in
+++ b/tests/general/pio_decomp_tests_3d.F90.in
@@ -1,7 +1,7 @@
 #include "config.h"
 ! Get a 3D column decomposition
 ! If force_rearrange is FALSE, the decomposition is such that
-! # All even procs have VEC_HGT_SZ blocks of 
+! # All even procs have VEC_HGT_SZ blocks of
 ! (VEC_COL_SZ rows x VEC_ROW_SZ columns) elements
 ! # All odd procs have  VEC_HGT_SZ blocks of
 ! (VEC_COL_SZ rows x VEC_ROW_SZ + 1 columns) elements
@@ -117,7 +117,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp
   nrows = count(1)
   ncols = count(2)
   nhgts = count(3)
-  
+
   allocate(wbuf(nrows, ncols, nhgts))
   allocate(compdof(nrows * ncols * nhgts))
   do k=1,nhgts
@@ -139,7 +139,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp
   nrows = count(1)
   ncols = count(2)
   nhgts = count(3)
-  
+
   allocate(rbuf(nrows, ncols, nhgts))
   allocate(compdof(nrows * ncols * nhgts))
   allocate(exp_val(nrows, ncols, nhgts))
@@ -163,7 +163,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp
   filename = "test_pio_decomp_simple_tests.testfile"
   do i=1,num_iotypes
     PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
-    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
     PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
 
     ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
@@ -193,7 +193,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp
     PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
 
     call PIO_closefile(pio_file)
-    
+
     call PIO_deletefile(pio_tf_iosystem_, filename);
   end do
 
diff --git a/tests/general/pio_file_fail.F90.in b/tests/general/pio_file_fail.F90.in
index 8ef1063180..e718bde1ba 100644
--- a/tests/general/pio_file_fail.F90.in
+++ b/tests/general/pio_file_fail.F90.in
@@ -37,7 +37,7 @@ PIO_TF_TEST_DRIVER_BEGIN
   integer :: num_uiotypes
 
   num_uiotypes = 0
-  call PIO_TF_Get_undef_iotypes(uiotypes, uiotype_descs, num_uiotypes) 
+  call PIO_TF_Get_undef_iotypes(uiotypes, uiotype_descs, num_uiotypes)
   dummy_file = "test_pio_file_fail.testfile"
   do i=1,num_uiotypes
     PIO_TF_TEST_RUN(create_file_always_fail(uiotypes(i), dummy_file), trim(uiotype_descs(i)))
@@ -49,4 +49,3 @@ PIO_TF_TEST_DRIVER_BEGIN
   end if
 
 PIO_TF_TEST_DRIVER_END
-
diff --git a/tests/general/pio_filter_tests.F90.in b/tests/general/pio_filter_tests.F90.in
new file mode 100644
index 0000000000..996dd5de74
--- /dev/null
+++ b/tests/general/pio_filter_tests.F90.in
@@ -0,0 +1,417 @@
+#include "config.h"
+#include <netcdf_meta.h>
+! Get a 2D column decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_COL_SZ rows of VEC_ROW_SZ elements
+! # All odd procs have  VEC_COL_SZ rows of VEC_ROW_SZ + 1 elements
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2 and ranks 0, 1, 2,
+! e.g. 1) |(1,1) (1,2)|  |(1,3) (1,4) (1,5)|  |(1,6) (1,7)|
+!         |(2,1) (2,2)|, |(2,3) (2,4) (2,5)|, |(2,6) (2,7)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2 and ranks 0, 1, 2
+! e.g. 1) |(1,3) (1,4) (1,5)|  |(1,1) (1,2)|   |(1,6) (1,7)|
+!         |(2,3) (2,4) (2,5)|, |(2,1) (2,2)|,  |(2,6) (2,7)|
+SUBROUTINE get_2d_col_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_ROW_SZ = 7
+  integer, parameter :: VEC_COL_SZ = 7
+  integer, parameter :: NDIMS = 2
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = VEC_COL_SZ
+  dims(2) = num_even_procs * VEC_ROW_SZ + num_odd_procs * (VEC_ROW_SZ + 1)
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  start(1) = 1
+  count(1) = VEC_COL_SZ
+
+  ! Columns
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(2) = VEC_ROW_SZ + 1
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + (VEC_ROW_SZ) + 1
+      else
+        count(2) = VEC_ROW_SZ
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(2) = VEC_ROW_SZ
+      start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) - (VEC_ROW_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(2) = VEC_ROW_SZ
+    else
+      count(2) = VEC_ROW_SZ + 1
+    end if
+    start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+  end if
+
+END SUBROUTINE
+
+! Get a 2D row decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_COL_SZ rows of VEC_ROW_SZ elements
+! # All odd procs have  VEC_COL_SZ+1 rows of VEC_ROW_SZ elements
+! e.g. For VEC_ROW_SZ = 6, VEC_COL_SZ = 1 and ranks 0, 1, 2,
+! e.g. 1) |(1,1) (1,2) (1,3) (1,4) (1,5) (1,6)|,
+!         |(2,1) (2,2) (2,3) (2,4) (2,5) (2,6)|
+!         |(3,1) (3,2) (3,3) (3,4) (3,5) (3,6)|,
+!         |(4,1) (4,2) (4,3) (4,4) (4,5) (4,6)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements (rows) with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_ROW_SZ = 6, VEC_COL_SZ = 1 and ranks 0, 1, 2
+! e.g. 1) |(2,1) (2,2) (2,3) (2,4) (2,5) (2,6)|
+!         |(3,1) (3,2) (3,3) (3,4) (3,5) (3,6)|,
+!         |(1,1) (1,2) (1,3) (1,4) (1,5) (1,6)|,
+!         |(4,1) (4,2) (4,3) (4,4) (4,5) (4,6)|
+SUBROUTINE get_2d_row_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_COL_SZ = 7
+  integer, parameter :: VEC_ROW_SZ = 7
+  integer, parameter :: NDIMS = 2
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = num_even_procs * VEC_COL_SZ + num_odd_procs * (VEC_COL_SZ + 1)
+  dims(2) = VEC_ROW_SZ
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(1) = VEC_COL_SZ + 1
+        start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + (VEC_COL_SZ) + 1
+      else
+        count(1) = VEC_COL_SZ
+        start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(1) = VEC_COL_SZ
+      start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) - (VEC_COL_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(1) = VEC_COL_SZ
+    else
+      count(1) = VEC_COL_SZ + 1
+    end if
+    start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + 1
+  end if
+
+  ! Columns
+  start(2) = 1
+  count(2) = VEC_ROW_SZ
+
+END SUBROUTINE
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE>
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_2d_col_decomp
+  implicit none
+  type(var_desc_t)  :: pio_var(6)   ! one for each potential filter type
+  type(file_desc_t) :: pio_file
+  integer, parameter :: NDIMS = 2
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, tmp_idx, ierr, nrows, ncols
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  integer, dimension(:), allocatable :: filtertypes
+
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: filtertype_descs
+  character(len=PIO_TF_MAX_STR_LEN) :: msg
+  integer :: num_iotypes
+  integer :: num_filtertypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_2d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+
+  allocate(wbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  do j=1,ncols
+    do i=1,nrows
+      wbuf(i,j) = (start(2) - 1 + j - 1) * nrows + i
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = int(wbuf(i,j))
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_2d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+
+  allocate(rbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  allocate(exp_val(nrows, ncols))
+  do j=1,ncols
+    do i=1,nrows
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = (start(2) - 1 + j - 1) * nrows + i
+      ! Expected value, after reading, is the same as the compdof
+      exp_val(i,j) = compdof(tmp_idx)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+
+  filename = "test_pio_decomp_filter_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", trim(iotype_descs(i))
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+#ifdef PIO_HAS_PAR_FILTERS
+    if(iotypes(i) == PIO_IOTYPE_NETCDF4P .or. iotypes(i) == PIO_IOTYPE_NETCDF4C) then
+       call PIO_TF_Get_nc4_filtertypes(pio_file, filtertypes, filtertype_descs, num_filtertypes)
+       print *,__FILE__,__LINE__,num_filtertypes
+       do j=1, num_filtertypes
+          print *,__FILE__,__LINE__,trim(filtertype_descs(j))
+       enddo
+    endif
+#else
+    num_filtertypes = 1
+#endif
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define dim1")
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define dim2")
+#ifdef NC_HAS_BZ
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var_bzip2', PIO_TF_DATA_TYPE, pio_dims, pio_var(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define var")
+#endif
+#ifdef NC_HAS_ZSTD
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var_zstandard', PIO_TF_DATA_TYPE, pio_dims, pio_var(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define var")
+#endif
+#if defined(PIO_HAS_PAR_FILTERS)
+    if((iotypes(i) == PIO_IOTYPE_NETCDF4P .or. iotypes(i) == PIO_IOTYPE_NETCDF4C) &
+         .and. (PIO_TF_DATA_TYPE==PIO_double .or. PIO_TF_DATA_TYPE==PIO_real)) then
+       ierr = PIO_def_var_szip(pio_file, pio_var(1), 32, 32)
+       write(msg, *) "Failed to define var szip compression.  ierr=",ierr
+       PIO_TF_CHECK_ERR(ierr, msg)
+
+       ierr = PIO_def_var_zstandard(pio_file, pio_var(2), 4)
+       write(msg, *) "Failed to define var zstandard compression.  ierr=",ierr
+       PIO_TF_CHECK_ERR(ierr, msg)
+
+!       ierr = PIO_def_var_deflate(pio_file, pio_var(2), shuffle, deflate, deflate_level)
+!       write(msg, *) "Failed to define var deflate compression.  ierr=",ierr
+!       PIO_TF_CHECK_ERR(ierr, msg)
+
+
+
+       else
+          PIO_TF_LOG(0,*) 'Do not test compression here ',trim(iotype_descs(i)), PIO_TF_DATA_TYPE
+       endif
+#endif
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+    do j=1,2
+       ! Write the variable out
+       call PIO_write_darray(pio_file, pio_var(j), wr_iodesc, wbuf, ierr)
+       PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+       call PIO_syncfile(pio_file)
+
+       call PIO_read_darray(pio_file, pio_var(j), rd_iodesc, rbuf, ierr)
+       PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+       PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+    enddo
+    call PIO_closefile(pio_file)
+
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_2d_col_decomp
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE<PIO_TF_PREDEF_TYPENAME PIO_TF_DATA_TYPE, PIO_TF_PREDEF_TYPENAME PIO_TF_FC_DATA_TYPE>
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_2d_row_decomp
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  integer, parameter :: NDIMS = 2
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, tmp_idx, ierr, nrows, ncols
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_2d_row_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+
+  allocate(wbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  do j=1,ncols
+    do i=1,nrows
+      wbuf(i,j) = (start(2) - 1 + j - 1) * dims(1) + start(1) + i - 1
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = int(wbuf(i,j))
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_2d_row_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+
+  allocate(rbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  allocate(exp_val(nrows, ncols))
+  do j=1,ncols
+    do i=1,nrows
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = (start(2) - 1 + j - 1) * dims(1) + start(1) + i - 1
+      ! Expected value, after reading, is the same as the compdof
+      exp_val(i,j) = compdof(tmp_idx)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_filter_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", trim(iotype_descs(i))
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define dim1")
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define dim2")
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : PIO_TF_test_var")
+!#if defined(NC_HAS_QUANTIZE) && PIO_TF_DATA_TYPE==PIO_DOUBLE || PIO_TF_DATA_TYPE==PIO_REAL
+!    if(iotypes(i) == PIO_IOTYPE_NETCDF4P .or. iotypes(i) == PIO_IOTYPE_NETCDF4C) then
+!       ierr = PIO_def_var_quantize(pio_file, pio_var, PIO_QUANTIZE_BITROUND, 4)
+!       PIO_TF_CHECK_ERR(ierr, "Failed to quantize a var : " // trim(filename))
+!    else
+!       PIO_TF_LOG(0,*) 'Do not test compression here ',trim(iotype_descs(i)), PIO_TF_DATA_TYPE
+!    endif
+!#endif
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+    call PIO_closefile(pio_file)
+
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_2d_row_decomp
diff --git a/tests/general/pio_init_finalize.F90.in b/tests/general/pio_init_finalize.F90.in
index 27e03f8ea7..571f9587cd 100644
--- a/tests/general/pio_init_finalize.F90.in
+++ b/tests/general/pio_init_finalize.F90.in
@@ -3,5 +3,3 @@ PIO_TF_AUTO_TEST_SUB_BEGIN init_finalize
   ! The default test driver should initialize and finalize PIO
   PRINT *, "Hello world"
 PIO_TF_AUTO_TEST_SUB_END init_finalize
-
-
diff --git a/tests/general/pio_iosystem_async_tests.F90.in b/tests/general/pio_iosystem_async_tests.F90.in
index 117ee1de53..e814878807 100644
--- a/tests/general/pio_iosystem_async_tests.F90.in
+++ b/tests/general/pio_iosystem_async_tests.F90.in
@@ -99,12 +99,11 @@ END SUBROUTINE create_file
 ! Check the contents of file : Check the
 ! global attribute 'filename' (should be equal to the
 ! name of the file, fname)
-SUBROUTINE check_file(comm, iosys, pio_file, fname, attname, dimname, ret)
+SUBROUTINE check_file(comm, pio_file, fname, attname, dimname, ret)
     use pio_tutil
     implicit none
 
     integer, intent(in) :: comm
-    type(iosystem_desc_t), intent(in) :: iosys
     type(file_desc_t), intent(inout) :: pio_file
     character(len=*), intent(in) :: fname
     character(len=*), intent(in) :: attname
@@ -129,7 +128,7 @@ SUBROUTINE check_file(comm, iosys, pio_file, fname, attname, dimname, ret)
     PIO_TF_PASSERT(val .eq. fname, comm, "Attribute value is not the expected value")
 
     call PIO_SetErrorHandling(pio_file, PIO_BCAST_ERROR, old_eh)
-!    ret = pio_set_log_level(iosys, 3)
+!    ret = pio_set_log_level(iosys, 0)
     ret = PIO_get_att(pio_file, pio_var, "wrongname", ival)
     write(errstr, *) "Got wrong error ",ret," on getatt in file:", trim(fname)
     PIO_TF_PASSERT(ret==PIO_ENOTATT, comm, errstr)
@@ -155,11 +154,15 @@ SUBROUTINE open_and_check_file(comm, iosys, iotype, pio_file, fname, &
     character(len=*), intent(in) :: dimname
     logical, intent(in) :: disable_fclose
     integer, intent(inout) :: ret
-
+    logical res
+!    ret = pio_set_log_level(3)
+    inquire(file=trim(fname), exist=res)
+    print *,__FILE__,__LINE__,trim(fname), 'res=',res
     ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write)
     PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname)
+!    ret = pio_set_log_level(0)
 
-    call check_file(comm, iosys, pio_file, fname, attname, dimname, ret)
+    call check_file(comm, pio_file, fname, attname, dimname, ret)
     PIO_TF_CHECK_ERR(ret, comm, "Checking contents of file failed:" // fname)
 
     if(.not. disable_fclose) then
@@ -177,7 +180,6 @@ PIO_TF_AUTO_TEST_SUB_BEGIN two_comps_odd_even_async
   character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_async_test_file2.nc"
   character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
   character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
-  character(len=PIO_TF_MAX_STR_LEN), pointer :: fname
   integer, dimension(:), allocatable :: iotypes
   character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
   integer :: i, num_iotypes = 0
@@ -189,9 +191,11 @@ PIO_TF_AUTO_TEST_SUB_BEGIN two_comps_odd_even_async
   integer :: comp_comm(2), io_comm
   integer :: all_comp_comm  ! comm common to all components
   integer :: ret
+  logical :: res
 
   ! Split world to odd even and io procs
   call split_world_odd_even_io(pio_tf_comm_, all_comp_comm, comp_comm, io_comm, ret)
+  print *,__FILE__,__LINE__,'comp_comm: ',comp_comm
   call PIO_init(iosys, pio_tf_comm_, comp_comm, io_comm, PIO_REARR_BOX)
   if(io_comm == MPI_COMM_NULL) then
      if(comp_comm(1) /= MPI_COMM_NULL) then
@@ -207,19 +211,21 @@ PIO_TF_AUTO_TEST_SUB_BEGIN two_comps_odd_even_async
         ! Create two files to be opened later
         if(comp_comm(1) /= MPI_COMM_NULL) then
            is_even = .false.
-!           print *,__FILE__,__LINE__,'create_file', is_even
            call create_file(comp_comm(1), iosys(1), iotypes(i), &
                 fname1, attname, dimname, ret)
+           if (iotypes(i) == PIO_IOTYPE_NETCDF4C) ret = pio_set_log_level(0)
+           print *,__FILE__,__LINE__,'create_file', is_even, trim(fname1)
+           inquire(file=trim(fname1), exist=res)
+           print *,__FILE__,__LINE__,trim(fname1), res
            PIO_TF_CHECK_ERR(ret, comp_comm(1), "Failed to create file :" // fname1)
         else
            is_even = .true.
-!           print *,__FILE__,__LINE__,'create_file', is_even
+           print *,__FILE__,__LINE__,'create_file', is_even, trim(fname2)
            call create_file(comp_comm(2), iosys(2), iotypes(i), &
                 fname2, attname, dimname, ret)
            PIO_TF_CHECK_ERR(ret, comp_comm(2), "Failed to create file :" // fname2)
         endif
 
-!        print *,__FILE__,__LINE__,'at barrier', is_even
         call mpi_barrier(all_comp_comm, ret)
 
         ! Open file1 from odd processes and file2 from even processes
@@ -234,6 +240,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN two_comps_odd_even_async
            PIO_TF_CHECK_ERR(ret, comp_comm(1), "Checking contents of file failed :" // fname2)
            call pio_deletefile(iosys(1), fname2)
         end if
+        call mpi_barrier(all_comp_comm, ret)
      end do
      if (is_even) then
         call PIO_finalize(iosys(2), ret)
diff --git a/tests/general/pio_rearr.F90.in b/tests/general/pio_rearr.F90.in
index 202ad10958..53ad53e447 100644
--- a/tests/general/pio_rearr.F90.in
+++ b/tests/general/pio_rearr.F90.in
@@ -197,7 +197,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_rearrs_combs
                   pio_rearr_subset, pio_rearr_subset,&
                   pio_rearr_box, pio_rearr_box/),&
                   (/NUM_REARRANGERS,MAX_PERMS/)&
-              ) 
+              )
   character(len=PIO_TF_MAX_STR_LEN) :: rearrs_perms_info(NUM_REARRANGERS,MAX_PERMS) =&
             reshape(&
               (/"PIO_REARR_SUBSET", "PIO_REARR_BOX   ",&
@@ -248,4 +248,3 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_rearrs_combs
     deallocate(iotype_descs)
   end if
 PIO_TF_AUTO_TEST_SUB_END test_rearrs_combs
-
diff --git a/tests/general/pio_rearr_opts.F90.in b/tests/general/pio_rearr_opts.F90.in
index d209860c2f..2a81603e82 100644
--- a/tests/general/pio_rearr_opts.F90.in
+++ b/tests/general/pio_rearr_opts.F90.in
@@ -281,7 +281,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts
       integer, dimension(1), intent(out) :: dims
       integer, intent(out) :: ret
     end subroutine create_decomp_and_init_buf
-  end interface 
+  end interface
 
   integer, parameter :: NUM_REARRANGERS = 2
   integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/)
@@ -333,7 +333,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts
   call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
   do i=1,num_iotypes
     PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
-    ! Create the file and decomp 
+    ! Create the file and decomp
     call create_file_and_var(iotypes(i), ret)
     PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
 
@@ -404,7 +404,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts
                 do cur_enable_hs_i2c=1,num_enable_hs_opts_io2comp
                   pio_rearr_opts%comm_fc_opts_io2comp%enable_hs =&
                     enable_hs_opts(cur_enable_hs_i2c)
-                
+
                   do cur_enable_isend_i2c=1,num_enable_isend_opts_io2comp
                     pio_rearr_opts%comm_fc_opts_io2comp%enable_isend =&
                       enable_isend_opts(cur_enable_isend_i2c)
@@ -459,7 +459,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts
                     end do ! cur_max_pend_req_i2c
                   end do ! cur_enable_isend_i2c
                 end do ! cur_enable_hs_i2c
-              end do ! cur_max_pend_req_c2i    
+              end do ! cur_max_pend_req_c2i
             end do ! cur_enable_isend_c2i
           end do ! cur_enable_hs_c2i
         end do ! cur_fcd_opt
diff --git a/tests/general/pio_rearr_opts2.F90.in b/tests/general/pio_rearr_opts2.F90.in
index 0d6a48c8a8..4491f1b766 100644
--- a/tests/general/pio_rearr_opts2.F90.in
+++ b/tests/general/pio_rearr_opts2.F90.in
@@ -230,7 +230,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write
       integer, dimension(1), intent(out) :: dims
       integer, intent(out) :: ret
     end subroutine create_decomp_and_init_buf
-  end interface 
+  end interface
 
   type(pio_rearr_opt_t) :: pio_rearr_opts
 
@@ -277,7 +277,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write
   call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
   do i=1,num_iotypes
     PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
-    ! Create the file and decomp 
+    ! Create the file and decomp
     call create_file_and_var(iotypes(i), ret)
     PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
 
@@ -347,7 +347,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write
               do cur_enable_hs_i2c=1,num_enable_hs_opts_io2comp
                 pio_rearr_opts%comm_fc_opts_io2comp%enable_hs =&
                   enable_hs_opts(cur_enable_hs_i2c)
-              
+
                 do cur_enable_isend_i2c=1,num_enable_isend_opts_io2comp
                   pio_rearr_opts%comm_fc_opts_io2comp%enable_isend =&
                     enable_isend_opts(cur_enable_isend_i2c)
@@ -400,7 +400,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write
                   end do ! cur_max_pend_req_i2c
                 end do ! cur_enable_isend_i2c
               end do ! cur_enable_hs_i2c
-            end do ! cur_max_pend_req_c2i    
+            end do ! cur_max_pend_req_c2i
           end do ! cur_enable_isend_c2i
         end do ! cur_enable_hs_c2i
       end do ! cur_fcd_opt
diff --git a/tests/general/util/pio_tf_f90gen.pl b/tests/general/util/pio_tf_f90gen.pl
index 696f50aaae..beb0cb267c 100755
--- a/tests/general/util/pio_tf_f90gen.pl
+++ b/tests/general/util/pio_tf_f90gen.pl
@@ -53,13 +53,13 @@ sub init_predef_types
 {
   $template_predef_typename_types{"PIO_TF_DATA_TYPE"} = [];
   $template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"} = [];
-  push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_int");
+  push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_INT");
   push(@{$template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"}}, "integer(kind=fc_int)");
-  push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_short");
+  push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_SHORT");
   push(@{$template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"}}, "integer(kind=fc_short)");
-  push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_real");
+  push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_REAL");
   push(@{$template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"}}, "real(kind=fc_real)");
-  push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_double");
+  push(@{$template_predef_typename_types{"PIO_TF_DATA_TYPE"}}, "PIO_DOUBLE");
   push(@{$template_predef_typename_types{"PIO_TF_FC_DATA_TYPE"}}, "real(kind=fc_double)");
 }
 
@@ -624,7 +624,7 @@ sub get_default_test_main
   $out_line = $out_line . "    CALL MPI_Init(ierr)\n";
   $out_line = $out_line . "    DO i=1,SIZE(rearrs)\n";
   if($test_type eq "async"){
-    $out_line = $out_line . "      CALL PIO_TF_Init_async_(rearrs(i))\n";
+    $out_line = $out_line . "      CALL PIO_TF_Init_async_()\n";
   }else{
     $out_line = $out_line . "      CALL PIO_TF_Init_(rearrs(i))\n";
   }
@@ -698,6 +698,8 @@ sub get_default_test_driver
     $out_line = $out_line . "    END IF\n";
     $cur_test_case_num += 1;
   }
+  # This line just avoids a gfortran warning
+  $out_line = $out_line . "      mpierr = 0\n";
   $out_line = $out_line . "  END SUBROUTINE PIO_TF_Test_driver_\n";
   return $out_line;
 }
diff --git a/tests/general/util/pio_tutil.F90 b/tests/general/util/pio_tutil.F90
index 71cf930a5a..89a9c5b1d9 100644
--- a/tests/general/util/pio_tutil.F90
+++ b/tests/general/util/pio_tutil.F90
@@ -1,3 +1,4 @@
+#include "config.h"
 ! PIO Testing framework utilities module
 MODULE pio_tutil
   USE pio
@@ -66,6 +67,9 @@ MODULE pio_tutil
   PUBLIC  :: PIO_TF_Init_, PIO_TF_Finalize_, PIO_TF_Passert_
   PUBLIC  :: PIO_TF_Is_netcdf
   PUBLIC  :: PIO_TF_Get_nc_iotypes, PIO_TF_Get_undef_nc_iotypes
+#ifdef NC_HAS_MULTIFILTERS
+  public  :: pio_tf_get_nc4_filtertypes
+#endif
   PUBLIC  :: PIO_TF_Get_iotypes, PIO_TF_Get_undef_iotypes
   PUBLIC  :: PIO_TF_Get_data_types
   PUBLIC  :: PIO_TF_Check_val_
@@ -190,7 +194,7 @@ SUBROUTINE  PIO_TF_Init_(rearr)
   END SUBROUTINE PIO_TF_Init_
 
   ! Initialize Testing framework - Internal (Not directly used by unit tests)
-  SUBROUTINE  PIO_TF_Init_async_(rearr)
+  SUBROUTINE  PIO_TF_Init_async_()
 #ifdef TIMING
    use perf_mod
 #endif
@@ -199,7 +203,6 @@ SUBROUTINE  PIO_TF_Init_async_(rearr)
 #else
     include 'mpif.h'
 #endif
-    INTEGER, INTENT(IN) :: rearr
     INTEGER ierr
 
     CALL MPI_COMM_DUP(MPI_COMM_WORLD, pio_tf_comm_, ierr);
@@ -355,7 +358,6 @@ SUBROUTINE PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
       ! pnetcdf
       num_iotypes = num_iotypes + 1
 #endif
-
     ! ALLOCATE with 0 elements ok?
     ALLOCATE(iotypes(num_iotypes))
     ALLOCATE(iotype_descs(num_iotypes))
@@ -385,7 +387,59 @@ SUBROUTINE PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
       i = i + 1
 #endif
   END SUBROUTINE
-
+#ifdef PIO_HAS_PAR_FILTERS
+  ! Returns a list of defined netcdf4 filter types
+  ! pio_file : An open file to check
+  ! filtertypes : After the routine returns contains a list of defined
+  !             netcdf4 filter types
+  ! filtertype_descs : After the routine returns contains description of
+  !                 the netcdf4 filter types returned in filtertypes
+  ! num_filtertypes : After the routine returns contains the number of
+  !                 of defined netcdf4 types, i.e., size of filtertypes and
+  !                 filtertype_descs arrays
+  SUBROUTINE PIO_TF_Get_nc4_filtertypes(pio_file, filtertypes, filtertype_descs, num_filtertypes)
+    use pio, only : pio_inq_filter_avail
+    type(file_desc_t), intent(in) :: pio_file
+    INTEGER, DIMENSION(:), ALLOCATABLE, INTENT(OUT) :: filtertypes
+    CHARACTER(LEN=*), DIMENSION(:), ALLOCATABLE, INTENT(OUT) :: filtertype_descs
+    INTEGER, INTENT(OUT) :: num_filtertypes
+    INTEGER :: i
+    integer :: ierr
+    integer, parameter :: num_possible_filters = 6
+    INTEGER :: tmpfiltertypes(num_possible_filters)
+
+    num_filtertypes = 0
+    ! First find the number of filter types
+
+    do i=1,num_possible_filters
+       ierr = pio_inq_filter_avail(pio_file, i)
+       if(ierr == PIO_NOERR) then
+          num_filtertypes = num_filtertypes + 1
+          tmpfiltertypes(num_filtertypes) = i
+       endif
+    enddo
+    allocate(filtertypes(num_filtertypes))
+    allocate(filtertype_descs(num_filtertypes))
+
+    filtertypes = tmpfiltertypes(1:num_filtertypes)
+    do i=1,num_filtertypes
+       select case(filtertypes(i))
+       case (1)
+          filtertype_descs(i) = "DEFLATE"
+       case (2)
+          filtertype_descs(i) = "SHUFFLE"
+       case (3)
+          filtertype_descs(i) = "FLETCHER32"
+       case (4)
+          filtertype_descs(i) = "SZIP"
+       case (5)
+          filtertype_descs(i) = "NBIT"
+       case (6)
+          filtertype_descs(i) = "SCALEOFFSET"
+       end select
+    enddo
+  END SUBROUTINE PIO_TF_Get_nc4_filtertypes
+#endif
   ! Returns a list of undefined netcdf iotypes
   ! e.g. This list could be used by a test to make sure that PIO
   !       fails gracefully for undefined types
diff --git a/tests/ncint/CMakeLists.txt b/tests/ncint/CMakeLists.txt
index f2aa696682..dfdbb832ff 100644
--- a/tests/ncint/CMakeLists.txt
+++ b/tests/ncint/CMakeLists.txt
@@ -9,7 +9,7 @@ include (LibMPI)
 include_directories("${CMAKE_SOURCE_DIR}/tests/ncint")
 include_directories("${CMAKE_BINARY_DIR}")
 
-set (my_tests tst_async_multi tst_ncint_async_perf
+set (my_tests tst_async_multi tst_ncint_async_perf tst_ncint_open
   tst_ncint_perf tst_pio_async tst_pio_udf tst_var_compress)
 
 # Test Timeout in seconds.
@@ -28,4 +28,3 @@ FOREACH(tst ${my_tests})
     NUMPROCS 4
     TIMEOUT ${DEFAULT_TEST_TIMEOUT})
 ENDFOREACH()
-
diff --git a/tests/ncint/ncint.h b/tests/ncint/ncint.h
new file mode 100644
index 0000000000..e0ead41312
--- /dev/null
+++ b/tests/ncint/ncint.h
@@ -0,0 +1,59 @@
+#include <pio.h>
+
+#if NETCDF_VERSION_LE(4,9,1)
+// only netcdf4 formats supported
+#define NUM_MODES 4
+#else
+#define NUM_MODES 10
+#endif
+
+
+#if NUM_MODES==4
+            int cmode[NUM_MODES] = {NC_PIO|NC_NETCDF4,
+                                    NC_PIO|NC_NETCDF4|NC_MPIIO,
+                                    NC_PIO|NC_NETCDF4|NC_CLASSIC_MODEL,
+                                    NC_PIO|NC_NETCDF4|NC_MPIIO|NC_CLASSIC_MODEL};
+            char mode_name[NUM_MODES][NC_MAX_NAME] = {"netcdf4 serial          ",
+                                                      "netcdf4 parallel        ",
+                                                      "netcdf4 classic serial  ",
+                                                      "netcdf4 classic parallel"};
+            int expected_format[NUM_MODES] = {NC_PIO|NC_FORMAT_NETCDF4,
+                                              NC_PIO|NC_FORMAT_NETCDF4,
+                                              NC_PIO|NC_FORMAT_NETCDF4_CLASSIC,
+                                              NC_PIO|NC_FORMAT_NETCDF4_CLASSIC};
+#endif
+#if NUM_MODES==10
+int cmode[NUM_MODES] = {NC_PIO,
+                        NC_PIO|NC_64BIT_OFFSET,
+                        NC_PIO|NC_64BIT_DATA,
+                        NC_PIO|NC_PNETCDF,
+                        NC_PIO|NC_PNETCDF|NC_64BIT_OFFSET,
+                        NC_PIO|NC_PNETCDF|NC_64BIT_DATA,
+                        NC_PIO|NC_NETCDF4,
+                        NC_PIO|NC_NETCDF4|NC_CLASSIC_MODEL,
+			NC_PIO|NC_NETCDF4|NC_MPIIO,
+			NC_PIO|NC_NETCDF4|NC_MPIIO|NC_CLASSIC_MODEL};
+
+            char mode_name[NUM_MODES][NC_MAX_NAME] = {"classic serial          ",
+                                                      "64bit offset serial     ",
+                                                      "64bit data serial       ",
+                                                      "classic pnetcdf         ",
+                                                      "64bit offset pnetcdf    ",
+                                                      "64bit data pnetcdf      ",
+                                                      "netcdf4 serial          ",
+                                                      "netcdf4 classic serial  ",
+                                                      "netcdf4 parallel        ",
+                                                      "netcdf4 classic parallel"};
+
+
+            int expected_format[NUM_MODES] = {NC_PIO|NC_FORMAT_CLASSIC,
+                                              NC_PIO|NC_FORMAT_64BIT_OFFSET,
+                                              NC_PIO|NC_FORMAT_64BIT_DATA,
+                                              NC_PIO|NC_FORMAT_CLASSIC,
+                                              NC_PIO|NC_FORMAT_64BIT_OFFSET,
+                                              NC_PIO|NC_FORMAT_64BIT_DATA,
+                                              NC_PIO|NC_FORMAT_NETCDF4,
+                                              NC_PIO|NC_FORMAT_NETCDF4_CLASSIC,
+                                              NC_PIO|NC_FORMAT_NETCDF4,
+                                              NC_PIO|NC_FORMAT_NETCDF4_CLASSIC};
+#endif
diff --git a/tests/ncint/pio_err_macros.h b/tests/ncint/pio_err_macros.h
index 40ef85b4cc..af7a221ab1 100644
--- a/tests/ncint/pio_err_macros.h
+++ b/tests/ncint/pio_err_macros.h
@@ -73,7 +73,7 @@ static int total_err = 0, err = 0;
     } while (0)
 
 /* This is also defined in tests/cunit/pio_tests.h. It will reduce
- * confusion to use the same value. */    
+ * confusion to use the same value. */
 #define ERR_WRONG 1112
 
 #endif /* _PIO_ERR_MACROS_H */
diff --git a/tests/ncint/tst_async_multi.c b/tests/ncint/tst_async_multi.c
index 576b05d165..1e5b12dcbf 100644
--- a/tests/ncint/tst_async_multi.c
+++ b/tests/ncint/tst_async_multi.c
@@ -42,7 +42,7 @@ create_file(int file_num, int my_rank, int ntasks, int num_io_procs,
 
     /* Create a file with a 3D record var. */
     sprintf(file_name, "%s_file_%d.nc", TEST_NAME, file_num);
-    if (nc_create(file_name, NC_PIO|NC_NETCDF4, &ncid)) PERR;
+    if (nc_create(file_name, NC_PIO, &ncid)) PERR;
     if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
     if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
     if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
diff --git a/tests/ncint/tst_ncint_open.c b/tests/ncint/tst_ncint_open.c
new file mode 100644
index 0000000000..676b2f3f0a
--- /dev/null
+++ b/tests/ncint/tst_ncint_open.c
@@ -0,0 +1,99 @@
+/* Test openfile function in ncint layer.
+
+   This test simply makes sure that a file created in any mode can be reopened.
+
+
+*/
+#include "config.h"
+#include "pio_err_macros.h"
+#include "ncint.h"
+
+#define FILE_NAME "tst_pio_udf_open_"
+#define VAR_NAME "data_var"
+#define DIM_NAME_UNLIMITED "dim_unlimited"
+#define DIM_NAME_X "dim_x"
+#define DIM_NAME_Y "dim_y"
+#define DIM_LEN_X 4
+#define DIM_LEN_Y 4
+#define NDIM2 2
+#define NDIM3 3
+#define TEST_VAL_42 42
+
+extern NC_Dispatch NCINT_dispatcher;
+
+int
+main(int argc, char **argv)
+{
+    int my_rank;
+    int ntasks;
+    char filename[30];
+    
+    /* Initialize MPI. */
+    if (MPI_Init(&argc, &argv)) PERR;
+
+    /* Learn my rank and the total number of processors. */
+    if (MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)) PERR;
+    if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR;
+
+    if (!my_rank)
+        printf("\n*** Testing netCDF integration layer.\n");
+
+
+    if (!my_rank)
+        printf("*** testing simple use of netCDF integration layer format...\n");
+    {
+        int ncid;
+        int dimid[NDIM3], varid;
+        int dimlen[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
+        int iosysid;
+        NC_Dispatch *disp_in;
+        int n, m;
+
+        /* Turn on logging for PIO library. */
+        /* PIOc_set_log_level(3); */
+
+        /* Initialize the intracomm. */
+        if (nc_def_iosystem(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR;
+
+        for( m=0; m < NUM_MODES; m++){
+	  sprintf(filename, "%s%d.nc",FILE_NAME,m);
+	  
+          /* Create a file with a 3D record var. */
+          if(!my_rank)
+            printf("\ncreate with: cmode = %d name=%s\n", m,mode_name[m]);
+          if (nc_create(filename, cmode[m], &ncid)) PERR;
+          if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
+          if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
+          if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
+          if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
+          if (nc_enddef(ncid)) PERR;
+
+          if (nc_close(ncid)) PERR;
+
+          /* Check that our user-defined format has been added. */
+          if (nc_inq_user_format(NC_PIO, &disp_in, NULL)) PERR;
+          if (disp_in != &NCINT_dispatcher) PERR;
+
+          for(n=0; n < NUM_MODES; n++){
+            /* Open the file. */
+              if(!my_rank)
+		printf("open %s with: %d, %s\n", filename, cmode[n],mode_name[n] );
+	      
+	      if (nc_open(filename, cmode[n], &ncid)) PERR;
+
+            /* Close file. */
+            if (nc_close(ncid)) PERR;
+          }
+          /* Free resources. */
+
+          /* delete file. */
+          PIOc_deletefile(iosysid, filename);
+        }
+        if (nc_free_iosystem(iosysid)) PERR;
+    }
+    PSUMMARIZE_ERR;
+
+    /* Finalize MPI. */
+    MPI_Finalize();
+    PFINAL_RESULTS;
+}
diff --git a/tests/ncint/tst_ncint_perf.c b/tests/ncint/tst_ncint_perf.c
index a5833cc0db..a077f7f913 100644
--- a/tests/ncint/tst_ncint_perf.c
+++ b/tests/ncint/tst_ncint_perf.c
@@ -11,8 +11,9 @@
 #include <pio.h>
 #include <sys/time.h>
 #include "pio_err_macros.h"
+#include "ncint.h"
 
-#define FILE_NAME "tst_ncint_perf.nc"
+#define FILE_PREFIX "tst_ncint_perf"
 #define VAR_NAME "data_var"
 #define DIM_NAME_UNLIMITED "dim_unlimited"
 #define DIM_NAME_X "dim_x"
@@ -24,7 +25,7 @@
 #define NDIM2 2
 #define NDIM3 3
 #define NUM_TIMESTEPS 1
-#define NUM_MODES 4
+
 
 extern NC_Dispatch NCINT_dispatcher;
 
@@ -45,7 +46,7 @@ main(int argc, char **argv)
     if (MPI_Comm_size(MPI_COMM_WORLD, &ntasks)) PERR;
 
     if (!my_rank)
-        printf("\n*** Testing netCDF integration PIO performance.\n");
+      printf("\n*** Testing netCDF integration PIO performance.\n");
     if (!my_rank)
         printf("*** testing simple intercomm use of netCDF integration layer...\n");
     {
@@ -58,11 +59,11 @@ main(int argc, char **argv)
         int *my_data;
         int num_io_procs;
         int i;
-
+        int found_format;
         /* Turn on logging for PIO library. */
-        /* PIOc_set_log_level(4); */
-        /* if (!my_rank) */
-        /*     nc_set_log_level(3); */
+/*         PIOc_set_log_level(4);
+         if (!my_rank)
+         nc_set_log_level(3);  */
         if (ntasks <= 16)
             num_io_procs = 1;
         else if (ntasks <= 64)
@@ -87,13 +88,6 @@ main(int argc, char **argv)
             float num_megabytes = DIM_LEN_X * DIM_LEN_Y * sizeof(int) / (float)1000000 * NUM_TIMESTEPS;
             float delta_in_sec;
             float mb_per_sec;
-            int cmode[NUM_MODES] = {NC_PIO, NC_PIO|NC_NETCDF4,
-                                    NC_PIO|NC_NETCDF4|NC_MPIIO,
-                                    NC_PIO|NC_PNETCDF};
-            char mode_name[NUM_MODES][NC_MAX_NAME + 1] = {"classic sequential   ",
-                                                          "netCDF-4 sequential  ",
-                                                          "netCDF-4 parallel I/O",
-                                                          "pnetcdf              "};
             int t, m;
 
             /* Print header. */
@@ -104,7 +98,13 @@ main(int argc, char **argv)
             for (m = 0; m < NUM_MODES; m++)
             {
                 /* Create a file with a 3D record var. */
-                if (nc_create(FILE_NAME, cmode[m], &ncid)) PERR;
+                char filename[strlen(FILE_PREFIX)+16];
+                sprintf(filename,"%s%d.nc",FILE_PREFIX,cmode[m]);
+                /* Turn on logging for PIO library. */
+//                PIOc_set_log_level(2);
+//                if (!my_rank)
+//                    nc_set_log_level(2);
+                if (nc_create(filename, cmode[m], &ncid)) PERR;
                 if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
                 if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
                 if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
@@ -139,8 +139,19 @@ main(int argc, char **argv)
                 /* Write some data with distributed arrays. */
                 for (t = 0; t < NUM_TIMESTEPS; t++)
                     if (nc_put_vard_int(ncid, varid, ioid, t, my_data)) PERR;
-                if (nc_close(ncid)) PERR;
 
+                /* check the file format */
+                if (nc_inq_format_extended(ncid, NULL, &found_format)) PERR;
+                if (found_format != expected_format[m]) {
+                        printf("expected format 0x%x found format 0x%x\n",expected_format[m],
+                           found_format);
+                    PERR;
+                }
+
+                if (nc_close(ncid)) PERR;
+//                PIOc_set_log_level(0);
+//                if (!my_rank)
+//                    nc_set_log_level(0);
                 /* Stop the clock. */
                 gettimeofday(&endtime, NULL);
 
@@ -154,9 +165,9 @@ main(int argc, char **argv)
                     printf("%s,\t%d,\t%d,\t%d,\t%8.3f,\t%8.1f,\t%8.3f\n", mode_name[m],
                            ntasks, num_io_procs, 1, delta_in_sec, num_megabytes,
                            mb_per_sec);
-		
-		free(my_data);
-		if (nc_free_decomp(ioid)) PERR;
+
+                free(my_data);
+                if (nc_free_decomp(ioid)) PERR;
 
             } /* next mode flag */
         }
diff --git a/tests/ncint/tst_pio_async.c b/tests/ncint/tst_pio_async.c
index 23f09537b6..13d09d9bb5 100644
--- a/tests/ncint/tst_pio_async.c
+++ b/tests/ncint/tst_pio_async.c
@@ -9,6 +9,7 @@
 #include "config.h"
 #include <pio.h>
 #include "pio_err_macros.h"
+#include "ncint.h"
 
 #define FILE_NAME "tst_pio_async.nc"
 #define VAR_NAME "data_var"
@@ -41,7 +42,7 @@ main(int argc, char **argv)
     if (!my_rank)
         printf("\n*** Testing netCDF integration layer.\n");
     if (!my_rank)
-        printf("*** testing simple async use of netCDF integration layer...");
+        printf("*** testing simple async use of netCDF integration layer...\n");
     {
         int ncid, ioid;
         int dimid[NDIM3], varid;
@@ -69,8 +70,12 @@ main(int argc, char **argv)
 
         if (my_rank)
         {
+            int m;
             /* Create a file with a 3D record var. */
-            if (nc_create(FILE_NAME, NC_PIO|NC_NETCDF4, &ncid)) PERR;
+          for( m=0; m<NUM_MODES; m++){
+            if(my_rank==1)
+              printf("     cmode = %d\n", cmode[m]);
+            if (nc_create(FILE_NAME, cmode[m], &ncid)) PERR;
             if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
             if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
             if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
@@ -113,7 +118,7 @@ main(int argc, char **argv)
                 size_t dim_len_in;
 
                 /* Open the file. */
-                if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR;
+                if (nc_open(FILE_NAME, cmode[m], &ncid)) PERR;
 
                 /* Check the file. */
                 if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) PERR;
@@ -148,6 +153,7 @@ main(int argc, char **argv)
 
             free(my_data);
             if (nc_free_decomp(ioid)) PERR;
+          }
             if (nc_free_iosystem(iosysid)) PERR;
         }
     }
diff --git a/tests/ncint/tst_pio_udf.c b/tests/ncint/tst_pio_udf.c
index 0dc68e1cae..f61c9d7a2a 100644
--- a/tests/ncint/tst_pio_udf.c
+++ b/tests/ncint/tst_pio_udf.c
@@ -9,6 +9,7 @@
 #include "config.h"
 #include "pio_err_macros.h"
 #include <pio.h>
+#include "ncint.h"
 
 #define FILE_NAME "tst_pio_udf.nc"
 #define VAR_NAME "data_var"
@@ -50,7 +51,7 @@ main(int argc, char **argv)
     PSUMMARIZE_ERR;
 
     if (!my_rank)
-        printf("*** testing simple use of netCDF integration layer format...");
+        printf("*** testing simple use of netCDF integration layer format...\n");
     {
         int ncid, ioid;
         int dimid[NDIM3], varid;
@@ -61,7 +62,7 @@ main(int argc, char **argv)
         size_t *compdof; /* The decomposition mapping. */
         int *my_data;
         int *data_in;
-        int i;
+        int i, m;
 
         /* Turn on logging for PIO library. */
         /* PIOc_set_log_level(3); */
@@ -69,56 +70,61 @@ main(int argc, char **argv)
         /* Initialize the intracomm. */
         if (nc_def_iosystem(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR;
 
-        /* Create a file with a 3D record var. */
-        if (nc_create(FILE_NAME, NC_PIO, &ncid)) PERR;
-        if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
-        if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
-        if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
-        if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
-
-        /* Calculate a decomposition for distributed arrays. */
-        elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks;
-        if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
+        for( m=0; m < NUM_MODES; m++){
+          /* Create a file with a 3D record var. */
+          if(!my_rank)
+            printf("     cmode = %d\n", cmode[m]);
+          if (nc_create(FILE_NAME, cmode[m], &ncid)) PERR;
+          if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
+          if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
+          if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
+          if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
+          if (nc_enddef(ncid)) PERR;
+
+          /* Calculate a decomposition for distributed arrays. */
+          elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks;
+          if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
             PERR;
-        for (i = 0; i < elements_per_pe; i++)
+          for (i = 0; i < elements_per_pe; i++)
             compdof[i] = my_rank * elements_per_pe + i;
 
-        /* Create the PIO decomposition for this test. */
-        if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
-                          compdof, &ioid, 1, NULL, NULL)) PERR;
-        free(compdof);
+          /* Create the PIO decomposition for this test. */
+          if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
+                            compdof, &ioid, 1, NULL, NULL)) PERR;
+          free(compdof);
 
-        /* Create some data on this processor. */
-        if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR;
-        for (i = 0; i < elements_per_pe; i++)
+          /* Create some data on this processor. */
+          if (!(my_data = malloc(elements_per_pe * sizeof(int)))) PERR;
+          for (i = 0; i < elements_per_pe; i++)
             my_data[i] = my_rank * 10 + i;
 
-        /* Write some data with distributed arrays. */
-        if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR;
-        if (nc_close(ncid)) PERR;
+          /* Write some data with distributed arrays. */
+          if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR;
+          if (nc_close(ncid)) PERR;
 
-        /* Check that our user-defined format has been added. */
-        if (nc_inq_user_format(NC_PIO, &disp_in, NULL)) PERR;
-        if (disp_in != &NCINT_dispatcher) PERR;
+          /* Check that our user-defined format has been added. */
+          if (nc_inq_user_format(NC_PIO, &disp_in, NULL)) PERR;
+          if (disp_in != &NCINT_dispatcher) PERR;
 
-        /* Open the file. */
-        if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR;
+          /* Open the file. */
+          if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR;
 
-        /* Read distributed arrays. */
-        if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR;
-        if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR;
+          /* Read distributed arrays. */
+          if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR;
+          if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR;
 
-        /* Check results. */
-        for (i = 0; i < elements_per_pe; i++)
+          /* Check results. */
+          for (i = 0; i < elements_per_pe; i++)
             if (data_in[i] != my_data[i]) PERR;
 
-        /* Close file. */
-        if (nc_close(ncid)) PERR;
+          /* Close file. */
+          if (nc_close(ncid)) PERR;
 
-        /* Free resources. */
-        free(data_in);
-        free(my_data);
-        if (nc_free_decomp(ioid)) PERR;
+          /* Free resources. */
+          free(data_in);
+          free(my_data);
+          if (nc_free_decomp(ioid)) PERR;
+        }
         if (nc_free_iosystem(iosysid)) PERR;
     }
     PSUMMARIZE_ERR;
diff --git a/tests/ncint/tst_var_compress.c b/tests/ncint/tst_var_compress.c
index 8e3060a630..28bb420858 100644
--- a/tests/ncint/tst_var_compress.c
+++ b/tests/ncint/tst_var_compress.c
@@ -76,10 +76,10 @@ run_var_compress_test(int my_rank, int ntasks, int iosysid)
 	int endian_in;
 	int d;
         /* int ret; */
-	
+
 	/* Open the file. */
 	if (nc_open(FILE_NAME, NC_PIO, &ncid)) PERR;
-	
+
 	/* Check the variable deflate. */
 	/* if ((ret = nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in))) */
         /*     NCPERR(ret); */
@@ -95,15 +95,15 @@ run_var_compress_test(int my_rank, int ntasks, int iosysid)
 	/* Check the endianness. */
 	if (nc_inq_var_endian(ncid, 0, &endian_in)) PERR;
 	if (endian_in != NC_ENDIAN_BIG) PERR;
-	
+
 	/* Read distributed arrays. */
 	if (!(data_in = malloc(elements_per_pe * sizeof(int)))) PERR;
 	if (nc_get_vard_int(ncid, varid, ioid, 0, data_in)) PERR;
-	
+
 	/* Check results. */
 	for (i = 0; i < elements_per_pe; i++)
 	    if (data_in[i] != my_data[i]) PERR;
-	
+
 	/* Close file. */
 	if (nc_close(ncid)) PERR;
 
@@ -147,7 +147,7 @@ main(int argc, char **argv)
 
     /* Free the iosystem. */
     if (nc_free_iosystem(iosysid)) PERR;
-    
+
     PSUMMARIZE_ERR;
 #endif /* _NETCDF4 */
 
diff --git a/tests/performance/Pioperformance.md b/tests/performance/Pioperformance.md
index 186e489e8d..8a70c7ad9a 100644
--- a/tests/performance/Pioperformance.md
+++ b/tests/performance/Pioperformance.md
@@ -27,16 +27,16 @@ decompfile="ROUNDROBIN", or decompfile="BLOCK"
 
 They call init_ideal_dof which internally generates a dof as follows:
 
-    if(doftype .eq. 'ROUNDROBIN') then                                          
-       do i=1,varsize                                                           
-          compmap(i) = (i-1)*npe+mype+1                                         
-       enddo                                                                    
-    else if(doftype .eq. 'BLOCK') then                                          
-       do i=1,varsize                                                           
-          compmap(i) =  (i+varsize*mype)                                        
-       enddo                                                                    
+    if(doftype .eq. 'ROUNDROBIN') then
+       do i=1,varsize
+          compmap(i) = (i-1)*npe+mype+1
+       enddo
+    else if(doftype .eq. 'BLOCK') then
+       do i=1,varsize
+          compmap(i) =  (i+varsize*mype)
+       enddo
     endif
-    
+
 The size of the variable is npes*varsize where varsize can be set in
 the namelist. varsize is the variable array size per task. You can add
 variables by changing nvars in the namelist.
@@ -59,4 +59,4 @@ When this is run, output like the following will appear:
 These are read and write rates in units of MB/s for Box and Subset
 rearrangers - the time measured is from the call to readdof or
 writedof to the completion of the close (since writes are buffered the
-close needs to be included).
\ No newline at end of file
+close needs to be included).
diff --git a/tests/performance/kt.PIO1.perfmakefile b/tests/performance/kt.PIO1.perfmakefile
index 67f0d3be11..fdb18e2c85 100644
--- a/tests/performance/kt.PIO1.perfmakefile
+++ b/tests/performance/kt.PIO1.perfmakefile
@@ -1,7 +1,7 @@
 all: pioperf pioperfp1
 
 pioperf: pioperformance.o
-	mpif90 pioperformance.o  -o pioperf  ../pio_build_int/src/flib/libpiof.a ../pio_build_int/src/clib/libpioc.a ../pio_build_int/src/gptl/libgptl.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdff.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdf.so /glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/lib/libpnetcdf.a -lirng -ldecimal -lcilkrts -lstdc++ 
+	mpif90 pioperformance.o  -o pioperf  ../pio_build_int/src/flib/libpiof.a ../pio_build_int/src/clib/libpioc.a ../pio_build_int/src/gptl/libgptl.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdff.a /glade/apps/opt/netcdf-mpi/4.3.3.1/intel/default/lib/libnetcdf.so /glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/lib/libpnetcdf.a -lirng -ldecimal -lcilkrts -lstdc++
 
 pioperformance.o: pioperformance.F90
 	mpif90  -DCPRINTEL -DHAVE_MPI -DINCLUDE_CMAKE_FCI -DLINUX -DTIMING -DUSEMPIIO -DUSE_PNETCDF_VARN -DUSE_PNETCDF_VARN_ON_READ -D_NETCDF -D_NETCDF4 -D_PNETCDF -I/glade/p/work/katec/pio_work/ncar_pio2/src/flib -I/glade/p/work/katec/pio_work/pio_build_int/src/flib -I/glade/apps/opt/netcdf-mpi/4.3.2/intel/default/include -I/glade/apps/opt/pnetcdf/1.6.1/intel/15.0.3/include -I/glade/p/work/katec/pio_work/ncar_pio2/src/clib -I/glade/p/work/katec/pio_work/ncar_pio2/src/gptl -I/glade/p/work/katec/pio_work/pio_build_int/src/gptl -c pioperformance.F90 -o pioperformance.o
@@ -16,4 +16,4 @@ cleanp1:
 	rm pioperformancep1.o pioperfp1
 
 clean:
-	rm pioperformance.o pioperf
\ No newline at end of file
+	rm pioperformance.o pioperf
diff --git a/tests/unit/Levy_Notes b/tests/unit/Levy_Notes
index 952c7ab0f5..4fc4a627cf 100644
--- a/tests/unit/Levy_Notes
+++ b/tests/unit/Levy_Notes
@@ -27,7 +27,7 @@ Frankfurt
 ---------
 "write_mpiio_int after call to file_set_view:MPI_ERR_ARG: invalid argument of so
  me other kind
- pio_support::pio_die:: myrank=          -1 : ERROR: iompi_mod.F90:         223 
+ pio_support::pio_die:: myrank=          -1 : ERROR: iompi_mod.F90:         223
  : (no message)"
 
 Yellowstone
@@ -35,7 +35,7 @@ Yellowstone
 "write_mpiio_int after call to file_set_view:Invalid argument, error stack:
 MPI_
  FILE_SET_VIEW(57): Invalid displacement argument
- pio_support::pio_die:: myrank=          -1 : ERROR: iompi_mod.F90:         223 
+ pio_support::pio_die:: myrank=          -1 : ERROR: iompi_mod.F90:         223
  : (no message)"
 
 5) For tests where we expect failure (e.g. writing a file that was opened nowrite)
diff --git a/tests/unit/ftst_vars_chunking.F90 b/tests/unit/ftst_vars_chunking.F90
index af04112d5f..37d4840eb8 100644
--- a/tests/unit/ftst_vars_chunking.F90
+++ b/tests/unit/ftst_vars_chunking.F90
@@ -9,13 +9,13 @@ program ftst_vars_chunking
   use mpi
   use pio
   use pio_nf
-  
+
   integer, parameter :: NUM_IOTYPES = 2
   integer, parameter :: NDIM2 = 2
-  
+
   type(iosystem_desc_t) :: pio_iosystem
   type(file_desc_t)     :: pio_file
-  type(var_desc_t)      :: pio_var  
+  type(var_desc_t)      :: pio_var
   integer :: my_rank, ntasks
   integer :: niotasks = 1, stride = 1
   character(len=64) :: filename = 'ftst_vars_chunking.nc'
@@ -28,7 +28,7 @@ program ftst_vars_chunking
   integer (kind=PIO_OFFSET_KIND) :: chunksizes_in(NDIM2)
   integer :: iotype(NUM_IOTYPES) = (/ PIO_iotype_netcdf4c, PIO_iotype_netcdf4p /)
   integer :: iotype_idx, ierr
-  
+
   ! Set up MPI
   call MPI_Init(ierr)
   call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr)
@@ -44,7 +44,7 @@ program ftst_vars_chunking
        PIO_rearr_subset, pio_iosystem, base=1)
 
   ! Set error handling for test.
-  call PIO_seterrorhandling(pio_iosystem, PIO_RETURN_ERROR)  
+  call PIO_seterrorhandling(pio_iosystem, PIO_RETURN_ERROR)
   call PIO_seterrorhandling(PIO_DEFAULT, PIO_RETURN_ERROR)
 
   ! Uncomment (and build with --enable-logging) to turn on logging.
@@ -52,32 +52,32 @@ program ftst_vars_chunking
 
   ! Try this test for NETCDF4C and NETCDF4P.
   do iotype_idx = 1, NUM_IOTYPES
-     
+
      ! Create a file.
      ierr = PIO_createfile(pio_iosystem, pio_file, iotype(iotype_idx), filename)
      if (ierr .ne. PIO_NOERR) stop 3
-     
+
      ! Define dims.
      ret_val = PIO_def_dim(pio_file, dim_name_1, dim_len1, dimid1)
      if (ierr .ne. PIO_NOERR) stop 5
      ret_val = PIO_def_dim(pio_file, dim_name_2, dim_len2, dimid2)
      if (ierr .ne. PIO_NOERR) stop 6
-     
+
      ! Define a var.
      ret_val = PIO_def_var(pio_file, var_name, PIO_int, (/dimid1, dimid2/), pio_var)
      if (ierr .ne. PIO_NOERR) stop 7
-     
+
      ! Define chunking for var.
      ret_val = PIO_def_var_chunking(pio_file, pio_var, 0, (/chunksize1, chunksize2/))
      if (ierr .ne. PIO_NOERR) stop 9
-     
+
      ! Close the file.
      call PIO_closefile(pio_file)
-     
+
      ! Open the file.
-     ret_val = PIO_openfile(pio_iosystem, pio_file, iotype(iotype_idx), filename, PIO_nowrite)  
+     ret_val = PIO_openfile(pio_iosystem, pio_file, iotype(iotype_idx), filename, PIO_nowrite)
      if (ierr .ne. PIO_NOERR) stop 23
-     
+
      ! Find var chunksizes using varid.
      ret_val = PIO_inq_var_chunking(pio_file, 1, storage_in, chunksizes_in)
      if (ierr .ne. PIO_NOERR) stop 25
@@ -88,11 +88,11 @@ program ftst_vars_chunking
      call PIO_closefile(pio_file)
 
   end do ! next IOTYPE
-  
+
   ! Finalize PIO.
   call PIO_finalize(pio_iosystem, ierr)
-  
+
   if (my_rank .eq. 0) print *,'SUCCESS!'
-#endif 
-  call MPI_Finalize(ierr)        
+#endif
+  call MPI_Finalize(ierr)
 end program ftst_vars_chunking