Merge pull request #3447 from jedwards4b/pio2_update

Pio2 update
Updates pio2 to the latest master (hash e41e6a6)
This change only affects nuopc driver where pio2 is default.

Test suite: SMS_D_Vnuopc.f09_g17.I2000Clm50SpGs.cheyenne_intel with pio_rearranger=1,2
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes ESCOMP/CTSM#940

User interface changes?:

Update gh-pages html (Y/N)?:

Code review:

config/cesm/machines/config_pio.xml

@@ -55,15 +55,13 @@
       <value mpilib="mpi-serial">netcdf</value>
     </values>
   </entry>
-
+<!--
   <entry id="PIO_REARRANGER">
     <values>
-      <!-- Note: problem not using box rearranger with nuopc caps -->
-      <!-- <value>$PIO_VERSION</value> -->
       <value>1</value>
     </values>
   </entry>
-
+-->
   <!--- uncomment and fill in relevant sections
   <entry id="PIO_DEBUG_LEVEL">
     <values>

src/externals/pio2/CMakeLists.txt

@@ -4,8 +4,8 @@ project (PIO C Fortran)
 
 # The project version number.
 set(VERSION_MAJOR   2   CACHE STRING "Project major version number.")
-set(VERSION_MINOR   4   CACHE STRING "Project minor version number.")
-set(VERSION_PATCH   4   CACHE STRING "Project patch version number.")
+set(VERSION_MINOR   5   CACHE STRING "Project minor version number.")
+set(VERSION_PATCH   0   CACHE STRING "Project patch version number.")
 mark_as_advanced(VERSION_MAJOR VERSION_MINOR VERSION_PATCH)
 
 # The size of the data buffer for write/read_darray().
@@ -26,7 +26,7 @@ option (PIO_INTERNAL_DOC     "Enable PIO developer documentation"           OFF)
 option (PIO_TEST_BIG_ENDIAN  "Enable test to see if machine is big endian"  ON)
 option (PIO_USE_MPIIO        "Enable support for MPI-IO auto detect"        ON)
 option (PIO_USE_MPISERIAL    "Enable mpi-serial support (instead of MPI)"   OFF)
-option (PIO_USE_MALLOC       "Use native malloc (instead of bget package)"  OFF)
+option (PIO_USE_MALLOC       "Use native malloc (instead of bget package)"  ON)
 option (PIO_USE_PNETCDF_VARD       "Use pnetcdf put_vard "                  OFF)
 option (WITH_PNETCDF         "Require the use of PnetCDF"                   ON)
 

src/externals/pio2/Makefile.am

@@ -10,7 +10,6 @@ if BUILD_DOCS
 DOC = doc
 endif
 
-SUBDIRS = src tests examples ${DOC} scripts
-
-EXTRA_DIST = CMakeLists.txt set_flags.am COPYRIGHT
+SUBDIRS = src tests examples ${DOC} scripts cmake
 
+EXTRA_DIST = CMakeLists.txt set_flags.am COPYRIGHT cmake_config.h.in

src/externals/pio2/cmake/Makefile.am

@@ -0,0 +1,13 @@
+## This is the automake file for the cmake directory of the PIO
+## libraries. This directory holds files needed for the CMake build,
+## but not the autotools build.
+
+# Ed Hartnett 8/19/19
+
+# Cmake needs all these extra files to build.
+EXTRA_DIST = FindGPTL.cmake FindHDF5.cmake FindLIBRT.cmake		\
+FindLIBZ.cmake FindMPE.cmake FindMPISERIAL.cmake FindNetCDF.cmake	\
+FindPAPI.cmake FindPnetCDF.cmake FindSZIP.cmake LibCheck.cmake		\
+LibFind.cmake LibMPI.cmake Makefile.am mpiexec.alcf mpiexec.ncsa	\
+mpiexec.nersc mpiexec.nwscla mpiexec.olcf TryHDF5_HAS_SZIP.c		\
+TryNetCDF_DAP.c TryNetCDF_PARALLEL.c TryNetCDF_PNETCDF.c

src/externals/pio2/configure.ac

@@ -1,21 +1,20 @@
 ## This is the autoconf file for the PIO library.
 ## Ed Hartnett 8/16/17
 
-
 # Initialize autoconf and automake.
-AC_INIT(pio, 2.4.4-development)
+AC_INIT(pio, 2.5.0)
 AC_CONFIG_SRCDIR(src/clib/pio_darray.c)
 AM_INIT_AUTOMAKE([foreign serial-tests])
 
 # The PIO version, again.
 AC_DEFINE([PIO_VERSION_MAJOR], [2], [PIO major version])
-AC_DEFINE([PIO_VERSION_MINOR], [4], [PIO minor version])
-AC_DEFINE([PIO_VERSION_PATCH], [4], [PIO patch version])
+AC_DEFINE([PIO_VERSION_MINOR], [5], [PIO minor version])
+AC_DEFINE([PIO_VERSION_PATCH], [0], [PIO patch version])
 
 # Once more for the documentation.
 AC_SUBST([VERSION_MAJOR], [2])
-AC_SUBST([VERSION_MINOR], [4])
-AC_SUBST([VERSION_PATCH], [4])
+AC_SUBST([VERSION_MINOR], [5])
+AC_SUBST([VERSION_PATCH], [0])
 
 # The m4 directory holds macros for autoconf.
 AC_CONFIG_MACRO_DIR([m4])
@@ -314,4 +313,5 @@ AC_OUTPUT(Makefile
           examples/Makefile
           examples/c/Makefile
           examples/f03/Makefile
+          cmake/Makefile
           scripts/Makefile)

src/externals/pio2/doc/source/base.txt

@@ -29,4 +29,5 @@ releases.
  - @ref users_guide
  - @ref api
  - @ref c_api
+ - @ref netcdf_integration
 */

src/externals/pio2/doc/source/netcdf_integration.txt

@@ -9,14 +9,160 @@ In order to use netCDF integration:
 
 * The PIO configure must use the option --enable-netcdf-integration.
 
-* The latest master of the netcdf-c library must be built and used,
-and the PIO build must include the netcdf-c/include directory in its
-CPPFLAGS, in order to find internal netCDF header files needed for
-netCDF integration. (The netcdf-c library is being modified to export
-publically everything needed, so future releases of netcdf-c will no
-longer require this.)
+* Version 4.7.2 or later of the netCDF C library.
 
 Once PIO is build for netCDF integration, it provides the nc_* and
-nf_* functions required to fully integrate PIO and netCDF.
+nf_* functions required to fully integrate PIO and netCDF. Users must
+include the PIO header files in their C or Fortran programs, and link
+to the PIO and the netCDF libraries (and, optionally, the
+parallel-netcdf and HDF5 libraries).
+
+# Initializing the IO System
+
+IO system initialization is required before anyother PIO functionality
+is used in netCDF calls. The IO system defines how many processors are
+used for I/O, and how many for computation.
+
+The IO system may be initialized in one of two modes:
+
+* Intercomm Mode - All processors are involved in computation. A
+  subset does I/O (and also computation). To initialize in intercomm
+  mode, use nc_def_iosystem().
+
+* Async Mode - Some processors are dedicated to IO, and do no
+  computation. Other processors are organized into computational
+  unit. Each computational unit runs its own code. All computational
+  units which do netCDF/PIO calls will channel all IO through the
+  dedicated I/O nodes. To initialize in async mode, use
+  nc_def_async().
+
+Once defined, these functions return one or (in the case of async)
+more IO system IDs, usually abreviated in the code as 'iosysid'.
+
+For intercomm mode, there is one iosysid. For async mode, there is an
+array of iosysids, one for each computational unit.
+
+# The Default IO System
+
+The IO system ID (iosysid) must be provided for PIO calls. When using
+netCDF integration, there is no parameter in the functions available
+to pass the iosysid. Instead, there is a default iosysid. When
+creating a new IO system, the default iosysid is set, so that
+subsequent netCDF calls can know which IO system to use.
+
+In the (rare) cases where the user may wish to use multiple IO systems
+within the same section of code, the functions nc_set_iosystem() and
+nc_get_iosystem() will allow the user to set and get the defauly
+iosysid.
+
+# Opening/Creating a File
+
+To open a file, use nc_open()/nf_open(), providing the NC_PIO flag.
+
+In C:
+
+\code{.c}
+        /* Initialize the intracomm. */
+        if (nc_def_iosystemm(MPI_COMM_WORLD, 1, 1, 0, 0, &iosysid)) PERR;
+
+        /* Create a file with a 3D record var. */
+        if (nc_create(FILE_NAME, NC_PIO, &ncid)) PERR;
+        if (nc_def_dim(ncid, DIM_NAME_UNLIMITED, dimlen[0], &dimid[0])) PERR;
+        if (nc_def_dim(ncid, DIM_NAME_X, dimlen[1], &dimid[1])) PERR;
+        if (nc_def_dim(ncid, DIM_NAME_Y, dimlen[2], &dimid[2])) PERR;
+        if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM3, dimid, &varid)) PERR;
+\endcode
+
+Resources associated with the IO system must be released with
+nc_free_iosystem()/nf_free_iosystem().
+
+In Fortran:
+
+\code{.F90}
+  ! Define an IOSystem.
+  ierr = nf_def_iosystem(my_rank, MPI_COMM_WORLD, niotasks, numAggregator, &
+       stride, PIO_rearr_box, iosysid, base)
+  if (ierr .ne. nf_noerr) call handle_err(ierr)
+
+  ! Create a file.
+  ierr = nf_create(FILE_NAME, 64, ncid)
+  if (ierr .ne. nf_noerr) call handle_err(ierr)
+\endcode
+
+# Defining a Decomposition
+
+To define a decompositon for a distributed array use
+nc_def_decomp()/nf_def_decomp().
+
+In C:
+\code{.c}
+        /* Calculate a decomposition for distributed arrays. */
+        elements_per_pe = DIM_LEN_X * DIM_LEN_Y / ntasks;
+        if (!(compdof = malloc(elements_per_pe * sizeof(size_t))))
+            PERR;
+        for (i = 0; i < elements_per_pe; i++)
+            compdof[i] = my_rank * elements_per_pe + i;
+
+        /* Create the PIO decomposition for this test. */
+        if (nc_def_decomp(iosysid, PIO_INT, NDIM2, &dimlen[1], elements_per_pe,
+                          compdof, &ioid, 1, NULL, NULL)) PERR;
+        free(compdof);
+\endcode
+
+In Fortran:
+\code{.F90}
+  allocate(compdof(maplen))
+  allocate(data_buffer(maplen))
+  ! Row decomposition. Recall that my_rank is 0-based, even
+  ! in fortran. Also recall that compdof is 1-based for fortran.
+  do i = 1, maplen
+     compdof(i) = i + my_rank * maplen
+     data_buffer(i) = my_rank * 10 + i
+  end do
+  print *, 'compdof', my_rank, compdof
+  ierr = nf_def_decomp(iosysid, PIO_int, dims, compdof, decompid)
+  if (ierr .ne. nf_noerr) call handle_err(ierr)
+\endcode
+
+When a decomposition is defined, a decomposition ID is returned, and
+must be used later when accessing data using this decomposition.
+
+The resources associated with a decomposition must be freed with
+nc_free_decomp()/nf_free_decomp().
+
+# Reading and Writing Distributed Arrays
+
+Once a decomposition has been defined, it may be used to read and
+write distributed arrays. This allows the code to be written in terms
+of local storage only. That is, the array allocated and indexed in the
+code is the array of data that this processor works with. When the
+data is read/written, the library will map the local data into the
+global array space.
+
+To read distributed data, use the nc_get_vard() function, or one of the
+type-specific versions (ex. nc_get_vard_int()).
+
+To write distributed data, use the nc_put_vard() function, or one of the
+type-specific versions (ex. nc_put_vard_int()).
+
+In C:
+\code{.c}
+        /* Write some data with distributed arrays. */
+        if (nc_put_vard_int(ncid, varid, ioid, 0, my_data)) PERR;
+        if (nc_close(ncid)) PERR;
+\endcode
+
+In Fortran:
+\code{.F90}
+  ! Write 1st record with distributed arrays.
+  ierr = nf_put_vard_int(ncid, varid, decompid, 1, data_buffer)
+  if (ierr .ne. nf_noerr) call handle_err(ierr)
+\endcode
+
+# Examples in C and Fortran
+
+For examples using the netCDF integration features, look in the
+tests/ncint directory (for C) or the tests/fncint directory (for
+Fortran).
 
 */

src/externals/pio2/examples/Makefile.am

@@ -7,3 +7,5 @@ F03 = f03
 endif # BUILD_FORTRAN
 
 SUBDIRS = c ${F03}
+
+EXTRA_DIST = CMakeLists.txt

src/externals/pio2/examples/c/CMakeLists.txt

@@ -37,9 +37,9 @@ ADD_EXECUTABLE(darray_no_async darray_no_async.c)
 TARGET_LINK_LIBRARIES(darray_no_async pioc)
 add_dependencies(tests darray_no_async)
 
-ADD_EXECUTABLE(darray_async darray_async.c)
-TARGET_LINK_LIBRARIES(darray_async pioc)
-add_dependencies(tests darray_async)
+# ADD_EXECUTABLE(darray_async darray_async.c)
+# TARGET_LINK_LIBRARIES(darray_async pioc)
+# add_dependencies(tests darray_async)
 
 if (PIO_USE_MPISERIAL)
   add_test(NAME examplePio COMMAND examplePio)

src/externals/pio2/examples/c/Makefile.am

@@ -16,7 +16,7 @@ TESTS = run_tests.sh
 endif # RUN_TESTS
 
 # Distribute the test script.
-EXTRA_DIST = run_tests.sh
+EXTRA_DIST = run_tests.sh CMakeLists.txt example2.c
 
 # Clean up files produced during testing.
 CLEANFILES = *.nc *.log *.clog2 *.slog2

src/externals/pio2/examples/c/darray_async.c

@@ -249,11 +249,11 @@ int main(int argc, char* argv[])
         ERR(ret);
 
 
-    /* The rest of the code executes on computation tasks only. As
-     * PIO functions are called on the computation tasks, the
-     * async system will call them on the IO task. When the
-     * computation tasks call PIO_finalize(), the IO task will get
-     * a message to shut itself down. */
+    /* The rest of the code executes on computation tasks only. As PIO
+     * functions are called on the computation tasks, the async system
+     * will call them on the IO task. When the computation tasks call
+     * PIO_finalize(), the IO task will get a message to shut itself
+     * down. */
     if (comp_task)
     {
         /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */

src/externals/pio2/src/clib/CMakeLists.txt

@@ -49,8 +49,8 @@ endif()
 # Library
 install (TARGETS pioc DESTINATION lib)
 
-# Include/Header File
-install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pio.h DESTINATION include)
+# Include/Header Files
+install (FILES pio.h uthash.h DESTINATION include)
 
 #==============================================================================
 #  DEFINE THE DEPENDENCIES

src/externals/pio2/src/clib/Makefile.am

@@ -13,7 +13,7 @@ endif # BUILD_NCINT
 # These linker flags specify libtool version info.
 # See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
 # for information regarding incrementing `-version-info`.
-libpioc_la_LDFLAGS = -version-info 2:0:1
+libpioc_la_LDFLAGS = -version-info 4:0:3
 
 # The library header file will be installed in include dir.
 include_HEADERS = pio.h uthash.h
@@ -25,4 +25,4 @@ pioc_support.c pio_darray_int.c pio_get_nc.c pio_lists.c pio_nc4.c	\
 pio_put_nc.c pio_spmd.c pio_get_vard.c pio_put_vard.c pio_internal.h	\
 bget.h uthash.h pio_error.h
 
-EXTRA_DIST = CMakeLists.txt
+EXTRA_DIST = CMakeLists.txt topology.c