Merge pull request ESMCI#1556 from NCAR/ejh_pio_netcdf

PIO/NetCDF integration

configure.ac

@@ -262,6 +262,28 @@ if test "x$enable_timing" = xyes; then
 fi
 AM_CONDITIONAL([HAVE_PAPI], [test "x$have_papi" = xyes])
 
+# Does the user want to build netcdf-c integration layer?
+AC_MSG_CHECKING([whether netcdf-c integration layer should be build])
+AC_ARG_ENABLE([netcdf-integration],
+              [AS_HELP_STRING([--enable-netcdf-integration],
+                              [enable building of netCDF C API integration.])])
+test "x$enable_netcdf_integration" = xyes || enable_netcdf_integration=no
+AC_MSG_RESULT([$enable_netcdf_integration])
+if test "x$enable_netcdf_integration" = xyes -a "x$enable_timing" = xyes; then
+   AC_MSG_ERROR([Cannot use GPTL timing library with netCDF interation.])
+fi
+if test "x$enable_netcdf_integration" = xyes -a "x$have_netcdf_par" = xno; then
+   AC_MSG_ERROR([Cannot use netCDF integration unless netCDF library was built for parallel I/O.])
+fi
+# These are needed by ncdispatch.h. Only build with HDF5 parallel
+# versions of netCDF. */
+if test "x$enable_netcdf_integration" = xyes; then
+  AC_DEFINE([HDF5_PARALLEL],[1],[Does HDF5 library provide parallel access])
+  AC_DEFINE([USE_NETCDF4],[1],[Does HDF5 library provide parallel access])
+fi
+
+AM_CONDITIONAL(BUILD_NCINT, [test "x$enable_netcdf_integration" = xyes])
+
 AC_CONFIG_FILES([tests/general/pio_tutil.F90:tests/general/util/pio_tutil.F90])
 
 AC_CONFIG_LINKS([tests/unit/input.nl:tests/unit/input.nl])
@@ -273,10 +295,12 @@ AC_CONFIG_HEADERS([config.h])
 AC_OUTPUT(Makefile
           src/Makefile
           src/clib/Makefile
+          src/ncint/Makefile
           src/flib/Makefile
           src/gptl/Makefile
           tests/Makefile
           tests/cunit/Makefile
+          tests/ncint/Makefile
           tests/unit/Makefile
           tests/general/Makefile
           tests/general/util/Makefile

examples/c/darray_no_async.c

@@ -104,7 +104,7 @@ int check_file(int iosysid, int ntasks, char *filename, int iotype,
     /* int expected[DIM_LEN_X];        /\* Data values we expect to find. *\/ */
 
     /* Open the file. */
-    if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0)))
+    if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0, 0)))
         return ret;
     /* printf("opened file %s ncid = %d\n", filename, ncid); */
 

src/Makefile.am

@@ -7,10 +7,17 @@ if BUILD_FORTRAN
 FLIB = flib
 endif
 
+# Are we building with the GPTL timing library?
 if USE_GPTL
 GPTL = gptl
 endif
 
-SUBDIRS = clib ${GPTL} $(FLIB)
+# Are we building with netCDF integration?
+if BUILD_NCINT
+NCINT = ncint
+endif # BUILD_NCINT
+
+# Build these subdirectories.
+SUBDIRS = ${NCINT} clib ${GPTL} $(FLIB)
 
 EXTRA_DIST = CMakeLists.txt

src/clib/Makefile.am

@@ -4,6 +4,11 @@
 # The library we are building.
 lib_LTLIBRARIES = libpioc.la
 
+# Are we building with netCDF integration?
+if BUILD_NCINT
+libpioc_la_LIBADD = ../ncint/libncint.la
+endif # BUILD_NCINT
+
 # These linker flags specify libtool version info.
 # See http://www.gnu.org/software/libtool/manual/libtool.html#Libtool-versioning
 # for information regarding incrementing `-version-info`.

src/clib/pio.h

@@ -1245,6 +1245,23 @@ extern "C" {
                                const long long *op);
     int PIOc_put_vard_ulonglong(int ncid, int varid, int decompid, const PIO_Offset recnum,
                                 const unsigned long long *op);
+
+    /* These functions are for the netCDF integration layer. */
+    int nc_init_intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr,
+                          int *iosysidp);
+
+    int nc_free_iosystem(int iosysid);
+
+    int nc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen,
+                       int maplen, const size_t *compmap, int *ioidp,
+                       int rearranger, const size_t *iostart,
+                       const size_t *iocount);
+
+    int nc_free_decomp(int ioid);
+
+    int nc_put_vard_int(int ncid, int varid, int decompid, const size_t recnum,
+                        const int *op);
+
 #if defined(__cplusplus)
 }
 #endif

src/clib/pio_file.c

@@ -51,12 +51,13 @@ extern int event_num[2][NUM_EVENTS];
  * @ingroup PIO_open_file_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *filename,
+int
+PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *filename,
                   int mode)
 {
     PLOG((1, "PIOc_openfile iosysid %d *iotype %d filename %s mode %d", iosysid,
           iotype ? *iotype: 0, filename, mode));
-    return PIOc_openfile_retry(iosysid, ncidp, iotype, filename, mode, 1);
+    return PIOc_openfile_retry(iosysid, ncidp, iotype, filename, mode, 1, 0);
 }
 
 /**
@@ -78,12 +79,13 @@ int PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *filename,
  * @ingroup PIO_open_file_c
  * @author Ed Hartnett
  */
-int PIOc_openfile2(int iosysid, int *ncidp, int *iotype, const char *filename,
+int
+PIOc_openfile2(int iosysid, int *ncidp, int *iotype, const char *filename,
                    int mode)
 {
     PLOG((1, "PIOc_openfile2 iosysid %d *iotype %d filename %s mode %d", iosysid,
           iotype ? *iotype : 0, filename, mode));
-    return PIOc_openfile_retry(iosysid, ncidp, iotype, filename, mode, 0);
+    return PIOc_openfile_retry(iosysid, ncidp, iotype, filename, mode, 0, 0);
 }
 
 /**
@@ -99,31 +101,26 @@ int PIOc_openfile2(int iosysid, int *ncidp, int *iotype, const char *filename,
  * @ingroup PIO_open_file_c
  * @author Ed Hartnett
  */
-int PIOc_open(int iosysid, const char *path, int mode, int *ncidp)
+int
+PIOc_open(int iosysid, const char *path, int mode, int *ncidp)
 {
     int iotype;
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ret;
 
     PLOG((1, "PIOc_open iosysid = %d path = %s mode = %x", iosysid, path, mode));
 
-    /* Figure out the iotype. */
-    if (mode & NC_NETCDF4)
-    {
-        if (mode & NC_MPIIO || mode & NC_MPIPOSIX)
-            iotype = PIO_IOTYPE_NETCDF4P;
-        else
-            iotype = PIO_IOTYPE_NETCDF4C;
-    }
-    else
-    {
-        if (mode & NC_PNETCDF || mode & NC_MPIIO)
-            iotype = PIO_IOTYPE_PNETCDF;
-        else
-            iotype = PIO_IOTYPE_NETCDF;
-    }
+    /* Get the IO system info from the id. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Find the IOTYPE from the mode flag. */
+    if ((ret = find_iotype_from_omode(mode, &iotype)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
     /* Open the file. If the open fails, do not retry as serial
      * netCDF. Just return the error code. */
-    return PIOc_openfile_retry(iosysid, ncidp, &iotype, path, mode, 0);
+    return PIOc_openfile_retry(iosysid, ncidp, &iotype, path, mode, 0, 0);
 }
 
 /**
@@ -144,7 +141,8 @@ int PIOc_open(int iosysid, const char *path, int mode, int *ncidp)
  * @ingroup PIO_create_file_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename,
+int
+PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename,
                     int mode)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
@@ -158,7 +156,7 @@ int PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename,
           iosysid, *iotype, filename, mode));
 
     /* Create the file. */
-    if ((ret = PIOc_createfile_int(iosysid, ncidp, iotype, filename, mode)))
+    if ((ret = PIOc_createfile_int(iosysid, ncidp, iotype, filename, mode, 0)))
         return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
     /* Run this on all tasks if async is not in use, but only on
@@ -188,27 +186,25 @@ int PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename,
  * @ingroup PIO_create_file_c
  * @author Ed Hartnett
  */
-int PIOc_create(int iosysid, const char *filename, int cmode, int *ncidp)
+int
+PIOc_create(int iosysid, const char *path, int cmode, int *ncidp)
 {
     int iotype;            /* The PIO IO type. */
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ret;
 
-    /* Figure out the iotype. */
-    if (cmode & NC_NETCDF4)
-    {
-        if (cmode & NC_MPIIO || cmode & NC_MPIPOSIX)
-            iotype = PIO_IOTYPE_NETCDF4P;
-        else
-            iotype = PIO_IOTYPE_NETCDF4C;
-    }
-    else
-    {
-        if (cmode & NC_PNETCDF || cmode & NC_MPIIO)
-            iotype = PIO_IOTYPE_PNETCDF;
-        else
-            iotype = PIO_IOTYPE_NETCDF;
-    }
+    PLOG((1, "PIOc_create iosysid = %d path = %s cmode = %x", iosysid, path,
+          cmode));
+
+    /* Get the IO system info from the id. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Find the IOTYPE from the mode flag. */
+    if ((ret = find_iotype_from_cmode(cmode, &iotype)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
 
-    return PIOc_createfile_int(iosysid, ncidp, &iotype, filename, cmode);
+    return PIOc_createfile_int(iosysid, ncidp, &iotype, path, cmode, 0);
 }
 
 /**
@@ -219,7 +215,8 @@ int PIOc_create(int iosysid, const char *filename, int cmode, int *ncidp)
  * @ingroup PIO_close_file_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_closefile(int ncid)
+int
+PIOc_closefile(int ncid)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */
@@ -318,7 +315,8 @@ int PIOc_closefile(int ncid)
  * @returns PIO_NOERR for success, error code otherwise.
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_deletefile(int iosysid, const char *filename)
+int
+PIOc_deletefile(int iosysid, const char *filename)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     int ierr = PIO_NOERR;  /* Return code from function calls. */
@@ -395,7 +393,8 @@ int PIOc_deletefile(int iosysid, const char *filename)
  * @ingroup PIO_sync_file_c
  * @author Jim Edwards, Ed Hartnett
  */
-int PIOc_sync(int ncid)
+int
+PIOc_sync(int ncid)
 {
     iosystem_desc_t *ios;  /* Pointer to io system information. */
     file_desc_t *file;     /* Pointer to file information. */

src/clib/pio_internal.h

@@ -187,12 +187,19 @@ extern "C" {
     int delete_var_desc(int varid, var_desc_t **varlist);
 
     /* Create a file (internal function). */
-    int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, int mode);
+    int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
+                            int mode, int use_ext_ncid);
 
     /* Open a file with optional retry as netCDF-classic if first
      * iotype does not work. */
     int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename, int mode,
-                            int retry);
+                            int retry, int use_ext_ncid);
+
+    /* Give the mode flag from an open, determine the IOTYPE. */
+    int find_iotype_from_omode(int mode, int *iotype);
+
+    /* Give the mode flag from an nc_create call, determine the IOTYPE. */
+    int find_iotype_from_cmode(int cmode, int *iotype);
 
     /* Given PIO type, find MPI type and type size. */
     int find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size);

src/clib/pio_msg.c

@@ -1972,9 +1972,9 @@ int open_file_handler(iosystem_desc_t *ios)
     PLOG((2, "open_file_handler got parameters len = %d filename = %s iotype = %d mode = %d",
           len, filename, iotype, mode));
 
-    /* Call the open file function. Errors are handling within
+    /* Call the open file function. Errors are handled within
      * function, so return code can be ignored. */
-    PIOc_openfile_retry(ios->iosysid, &ncid, &iotype, filename, mode, 0);
+    PIOc_openfile_retry(ios->iosysid, &ncid, &iotype, filename, mode, 0, 0);
 
     return PIO_NOERR;
 }