Merge branch 'ParallelIO_branch' (PIO2 master)

This commit brings in the latest version of PIO2 - 1fe2631 - to CIME. Note that this merge from PIO2 master ignores whitespace changes made in * 5f8f997 * 9a01a0b git-subtree-dir: externals/pio2 git-subtree-merge-commit: 1fe2631
ESMCI · Aug 16, 2016 · abfbee4 · abfbee4
2 parents 9331f85 + 1fe2631
commit abfbee4
Show file tree

Hide file tree

Showing 46 changed files with 15,460 additions and 2,037 deletions.
diff --git a/externals/pio2/CMakeLists.txt b/externals/pio2/CMakeLists.txt
@@ -14,18 +14,34 @@ mark_as_advanced(VERSION_MAJOR VERSION_MINOR VERSION_PATCH)
 #===== Library Options =====
 option (PIO_ENABLE_FORTRAN   "Enable the Fortran library builds"            ON)
 option (PIO_ENABLE_TIMING    "Enable the use of the GPTL timing library"    ON)
+option (PIO_ENABLE_ASYNC     "Enable the use of asychronus IO operations"   OFF)
+option (PIO_ENABLE_LOGGING   "Enable debug logging (large output possible)" OFF)
 option (PIO_TEST_BIG_ENDIAN  "Enable test to see if machine is big endian"  ON)
 option (PIO_USE_MPIIO        "Enable support for MPI-IO auto detect"        ON)
 option (PIO_USE_MPISERIAL    "Enable mpi-serial support (instead of MPI)"   OFF)
 option (PIO_USE_MALLOC       "Use native malloc (instead of bget package)"  OFF)
 option (WITH_PNETCDF         "Require the use of PnetCDF"                   ON)
 
+# Set a variable that appears in the config.h.in file.
 if(PIO_USE_MALLOC)
   set(USE_MALLOC 1)
 else()
   set(USE_MALLOC 0)
 endif()
 
+# Set a variable that appears in the config.h.in file.
+if(PIO_ENABLE_LOGGING)
+  set(ENABLE_LOGGING 1)
+else()
+  set(ENABLE_LOGGING 0)
+endif()
+
+if(PIO_USE_MPISERIAL)
+  set(USE_MPI_SERIAL 1)
+else()
+  set(USE_MPI_SERIAL 0)
+endif()
+
 #===== Library Variables =====
 set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)")
 

diff --git a/externals/pio2/ctest/CTestEnvironment-nwsc.cmake b/externals/pio2/ctest/CTestEnvironment-nwsc.cmake
@@ -10,7 +10,7 @@
 # set with existing environment variables: NETCDF, PNETCDF, HDF5, etc.
 
 # Define the extra CMake configure options
-set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE")
+set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_ASYNC=TRUE")
 
 # If MPISERIAL environment variable is set, then enable MPISERIAL
 if (DEFINED ENV{MPISERIAL})

diff --git a/externals/pio2/ctest/runcdash-cgd-nag.sh b/externals/pio2/ctest/runcdash-cgd-nag.sh
@@ -8,16 +8,16 @@ else
 fi
 
 module purge
-module load compiler/nag/6.0
-module load tool/parallel-netcdf/1.6.1/nag/openmpi
+module load compiler/nag/6.1
+module load tool/parallel-netcdf/1.7.0/nag/mvapich2
 
 export CC=mpicc
 export FC=mpif90
 export PIO_DASHBOARD_SITE="cgd"
 export PIO_DASHBOARD_ROOT=/scratch/cluster/katec/dashboard
 export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src
 export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY}
-export PIO_COMPILER_ID=Nag-6.0-gcc-`gcc --version | head -n 1 | cut -d' ' -f3`
+export PIO_COMPILER_ID=Nag-6.1-gcc-`gcc --version | head -n 1 | cut -d' ' -f3`
 
 if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then
   mkdir "$PIO_DASHBOARD_ROOT"

diff --git a/externals/pio2/ctest/runctest-cgd.sh b/externals/pio2/ctest/runctest-cgd.sh
@@ -29,7 +29,7 @@ echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh
 chmod +x runctest.sh
 
 # Submit the job to the queue
-jobid=`/usr/local/bin/qsub -l nodes=1:ppn=4 runctest.sh`
+jobid=`/usr/local/bin/qsub -l nodes=1:ppn=4 runctest.sh -q short`
 
 # Wait for the job to complete before exiting
 while true; do

diff --git a/externals/pio2/doc/CMakeLists.txt b/externals/pio2/doc/CMakeLists.txt
@@ -5,17 +5,30 @@
 #==============================================================================
 
 find_package(Doxygen)
+
+# This supports the build with/witout async code. Once async code is
+# fully merged, remove the definition of C_SRC_FILES and its mention
+# in Doxyfile.in for simplicity.
+if (PIO_ENABLE_ASYNC)
+  SET(C_SRC_FILES "@CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_sc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_darray_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_get_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_internal.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc4.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_put_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_spmd.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_support.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_lists.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc_async.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_varm.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/dtypes.h  @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_file.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_msg.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_rearrange.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/topology.c")
+else ()
+  SET(C_SRC_FILES "@CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_sc.c  @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_internal.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc4.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_spmd.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/bget.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pioc_support.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_darray.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_get_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_lists.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_put_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_varm.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/dtypes.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_file.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio.h @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_msg.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_nc.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/pio_rearrange.c @CMAKE_CURRENT_SOURCE_DIR@/../src/clib/topology.c")
+endif ()
+
 if(DOXYGEN_FOUND)
-    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in
-                   ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
-    add_custom_target(doc
-        COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/customdoxygen.css
-                                         ${CMAKE_CURRENT_BINARY_DIR}
-        COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenLayout.xml
-                                         ${CMAKE_CURRENT_BINARY_DIR}
-        COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/doxygen.sty
-                                         ${CMAKE_CURRENT_BINARY_DIR}
-        COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-        COMMENT "Generating API documentation with Doxygen" VERBATIM)
+  # Process the Doxyfile using options set during configure.
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in 
+    ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
+
+  # Copy necessary files.
+  add_custom_target(doc
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/customdoxygen.css 
+    ${CMAKE_CURRENT_BINARY_DIR}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenLayout.xml 
+    ${CMAKE_CURRENT_BINARY_DIR}
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/doxygen.sty 
+    ${CMAKE_CURRENT_BINARY_DIR}
+    COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Generating API documentation with Doxygen" VERBATIM)
 endif(DOXYGEN_FOUND)
diff --git a/externals/pio2/doc/Doxyfile.in b/externals/pio2/doc/Doxyfile.in
@@ -291,7 +291,7 @@ OPTIMIZE_OUTPUT_VHDL   = NO
 # Note that for custom extensions you also need to set FILE_PATTERNS otherwise
 # the files are not read by doxygen.
 
-EXTENSION_MAPPING      = F90=FortranFree
+EXTENSION_MAPPING      = F90=FortranFree 
 
 # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
 # according to the Markdown format, which allows for more readable
@@ -769,11 +769,15 @@ WARN_LOGFILE           =
 # Note: If this tag is empty the current directory is searched.
 
 INPUT                  = @CMAKE_CURRENT_SOURCE_DIR@/source \
-                         @CMAKE_CURRENT_SOURCE_DIR@/../src/clib \
                          @CMAKE_CURRENT_SOURCE_DIR@/../src/flib \
                          @CMAKE_CURRENT_SOURCE_DIR@/../examples/c \
                          @CMAKE_CURRENT_SOURCE_DIR@/../examples/f03 \
-                         @CMAKE_BINARY_DIR@/src/flib
+                         @CMAKE_BINARY_DIR@/src/clib \
+                         @CMAKE_BINARY_DIR@/src/flib \
+			 @C_SRC_FILES@
+
+# Uncomment this after the async code is fully merged into PIO.
+#                         @CMAKE_CURRENT_SOURCE_DIR@/../src/clib 
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

diff --git a/externals/pio2/doc/source/Error.txt b/externals/pio2/doc/source/Error.txt
@@ -1,29 +1,26 @@
 /******************************************************************************
  *
+ * 
  *
- *
- * Copyright (C) 2009
+ * Copyright (C) 2009 
  *
  * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
+ * documentation under the terms of the GNU General Public License is hereby 
+ * granted. No representations are made about the suitability of this software 
  * for any purpose. It is provided "as is" without express or implied warranty.
  * See the GNU General Public License for more details.
  *
  * Documents produced by Doxygen are derivative works derived from the
  * input used in their production; they are not affected by this license.
  *
- */ /*!
+ */ /*! 
 \page error Error Handling
 
 By default, PIO handles errors internally by printing a string
 describing the error and then calling mpi_abort.  Application
 developers can change this behaivior with a call to
 \ref PIO_seterrorhandling
 
-For example, if a developer wanted
-to see if an input netcdf file contained the variable 'U' they might do the following:
-
 \verbinclude errorhandle
 
 \copydoc PIO_error_method

diff --git a/externals/pio2/doc/source/Examples.txt b/externals/pio2/doc/source/Examples.txt
@@ -1,12 +1,12 @@
 /******************************************************************************
  *
+ * 
  *
- *
- * Copyright (C) 2009
+ * Copyright (C) 2009 
  *
  * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
+ * documentation under the terms of the GNU General Public License is hereby 
+ * granted. No representations are made about the suitability of this software 
  * for any purpose. It is provided "as is" without express or implied warranty.
  * See the GNU General Public License for more details.
  *
@@ -41,11 +41,13 @@ The Fortran examples are in the examples/f03 subdirectory.
 
 - examplePio.f90 A simple example showing a write, then read, of a 1D variable.
 
-  PIO has been implemented in several geophysical component models, including the
-Community Atmosphere Model (CAM), the Community Land Model (CLM), the Parallel Ocean Program
-(POP), the Community Ice CodE (CICE), and coupler for used by CCSM4.0 (CPL7).  We also provide
-several simpler example code as well as a test code that is suitable for regression testing and
-benchmarking.
+### Other Examples
+
+  PIO has been implemented in several geophysical component models, including the 
+Community Atmosphere Model (CAM), the Community Land Model (CLM), the Parallel Ocean Program 
+(POP), the Community Ice CodE (CICE), and coupler for used by CCSM4.0 (CPL7).  We also provide 
+several simpler example code as well as a test code that is suitable for regression testing and 
+benchmarking.    
 
  - \subpage CAMexample
  - \subpage testpio_example

diff --git a/externals/pio2/doc/source/Installing.txt b/externals/pio2/doc/source/Installing.txt
@@ -1,12 +1,12 @@
 /******************************************************************************
  *
- *
+ * 
  *
  * Copyright (C) 2013
  *
  * Permission to use, copy, modify, and distribute this software and its
- * documentation under the terms of the GNU General Public License is hereby
- * granted. No representations are made about the suitability of this software
+ * documentation under the terms of the GNU General Public License is hereby 
+ * granted. No representations are made about the suitability of this software 
  * for any purpose. It is provided "as is" without express or implied warranty.
  * See the GNU General Public License for more details.
  *
@@ -20,9 +20,9 @@ The PIO code is currently stored on github at <https://github.com/PARALLELIO/Par
 
 ### Dependencies ###
 
-PIO can use NetCDF (version 4.3.3+) and/or PnetCDF (version 1.6.0+) for I/O.
+PIO can use NetCDF (version 4.3.3+) and/or PnetCDF (version 1.6.1+) for I/O.
 Ideally, the NetCDF version should be built with MPI, which requires that it
-be linked with an MPI-enabled version of HDF5.  Optionally, NetCDF can be
+be linked with an MPI-enabled version of HDF5.  Optionally, NetCDF can be 
 built with DAP support, which introduces a dependency on CURL.  Additionally,
 HDF5, itself, introduces dependencies on LIBZ and (optionally) SZIP.
 
@@ -39,10 +39,10 @@ where `mpicc` and `mpif90` are the appropriate MPI-enabled compiler wrappers
 for your system.
 
 The `OPTIONS` section typically should consist of pointers to the install
-locations for various dependencies, assuming these dependencies are not
-located in *canonical* search locations.
+locations for various dependencies, assuming these dependencies are not 
+located in *canonical* search locations.  
 
-For each dependency `XXX`, one can specify the location of its
+For each dependency `XXX`, one can specify the location of its 
 installation path with the CMake variable `XXX_PATH`.  If the `C` and
 `Fortran` libraries for the dependency are installed in different locations
 (such as can be done with NetCDF), then you can specify individually
@@ -59,7 +59,7 @@ CMake configuration line:
 
 This works for the dependencies: `NetCDF`, `PnetCDF`, `HDF5`, `LIBZ`, `SZIP`.
 
-For specific instructions to install on various commonly used super computers, please read the [walk-through guide to PIO Installation](@ref mach_walkthrough).
+For specific instructions to install on various commonly used super computers, please read the [walk-through guide to PIO Installation](@ref mach_walkthrough). 
 
 ### Additional CMake Options ###
 
@@ -73,14 +73,14 @@ libraries are already installed on the system, the user can point PIO to the
 location of these libraries with the `GPTL_PATH` variable (or, individually,
 `GPTL_C_PATH` and `GPTL_Fortran_Perf_PATH` variables).  However, if these
 GPTL libraries are not installed on the system, and GPTL cannot be found,
-then PIO will build its own internal version of GPTL.
+then PIO will build its own internal version of GPTL.  
 
 If PnetCDF is not installed on the system, the user can disable its use by
 setting `-DWITH_PNETCDF=OFF`.  This will disable the search for PnetCDF on the
 system and disable the use of PnetCDF from within PIO.
 
 If the user wishes to disable the PIO tests, then the user can set the
-variable `-DPIO_ENABLE_TESTS=OFF`.  This will entirely disable the CTest
+variable `-DPIO_ENABLE_TESTS=OFF`.  This will entirely disable the CTest 
 testing suite, as well as remove all of the test build targets.
 
 If you wish to install PIO in a safe location for use later with other
@@ -116,7 +116,7 @@ Once the tests have been built, you may run tests with:
 _Note: If you have not run `make tests` before you run `ctest`, then you will see
 all of the tests fail._
 
-Alternatively, you may build the test executables and then run tests
+Alternatively, you may build the test executables and then run tests 
 immediately with:
 
 
@@ -126,7 +126,7 @@ immediately with:
 (similar to the typical `make check` Autotools target).
 
 *ANOTHER NOTE:* These tests are designed to run in parallel.
-If you are on one of the supported supercomputing platforms (i.e., NERSC, NWSC, ALCF,
+If you are on one of the supported supercomputing platforms (i.e., NERSC, NWSC, ALCF, 
 etc.), then the `ctest` command will assume that the tests will be run in an appropriately configured and scheduled parallel job.  This can be done by requesting an interactive session from the login nodes and then running `ctest` from within the interactive terminal.  Alternatively, this can be done by running the `ctest` command from a job submission script.  It is important to understand, however, that `ctest` itself will preface all of the test executable commands with the appropriate `mpirun`/`mpiexec`/`runjob`/etc. Hence, you should not further preface the `ctest` command with these MPI launchers.
 
 ### Installing ###