Merge pull request ESMCI#1497 from NCAR/ejh_mpe_3

Basic MPE logging capability

configure.ac

@@ -21,6 +21,7 @@ AC_SUBST([VERSION_PATCH], [4])
 AC_CONFIG_MACRO_DIR([m4])
 
 # Libtool initialisation.
+LD=ld # Required for MPE to work.
 LT_INIT
 
 # Find and learn about the C compiler.
@@ -52,22 +53,6 @@ if test "x$enable_logging" = xyes; then
    AC_DEFINE([PIO_ENABLE_LOGGING], 1, [If true, turn on logging.])
 fi
 
-# Does the user want to use MPE library?
-AC_MSG_CHECKING([whether use of MPE library is enabled])
-AC_ARG_ENABLE([mpe],
-              [AS_HELP_STRING([--enable-mpe],
-                              [enable use of MPE library for timing and diagnostic info (may negatively impact performance).])])
-test "x$enable_mpe" = xyes || enable_mpe=no
-AC_MSG_RESULT([$enable_mpe])
-if test "x$enable_mpe" = xyes; then
-   AC_SEARCH_LIBS([MPE_Log_get_event_number], [mpe], [HAVE_LIBMPE=yes], [HAVE_LIBMPE=no], [-lpthread -lm])
-   AC_CHECK_HEADERS([mpe.h], [HAVE_MPE=yes], [HAVE_MPE=no])
-   if test "x$HAVE_LIBMPE" = xno -o "x$HAVE_MPE" = xno; then
-      AC_MSG_ERROR([MPE not found but --enable-mpe used.])
-   fi
-   AC_DEFINE([USE_MPE], 1, [If true, use MPE timing library.])
-fi
-
 # Does the user want to enable timing?
 AC_MSG_CHECKING([whether GPTL timing library is used])
 AC_ARG_ENABLE([timing],
@@ -104,6 +89,32 @@ test "x$enable_fortran" = xyes || enable_fortran=no
 AC_MSG_RESULT([$enable_fortran])
 AM_CONDITIONAL(BUILD_FORTRAN, [test "x$enable_fortran" = xyes])
 
+# Does the user want to use MPE library?
+AC_MSG_CHECKING([whether use of MPE library is enabled])
+AC_ARG_ENABLE([mpe],
+              [AS_HELP_STRING([--enable-mpe],
+                              [enable use of MPE library for timing and diagnostic info (may negatively impact performance).])])
+test "x$enable_mpe" = xyes || enable_mpe=no
+AC_MSG_RESULT([$enable_mpe])
+if test "x$enable_mpe" = xyes; then
+
+   AC_SEARCH_LIBS([pthread_setspecific], [pthread], [], [], [])
+   dnl AC_SEARCH_LIBS([MPE_Log_get_event_number], [mpe], [HAVE_LIBMPE=yes], [HAVE_LIBMPE=no], [])
+   dnl AC_SEARCH_LIBS([MPE_Init_mpi_core], [lmpe], [HAVE_LIBLMPE=yes], [HAVE_LIBLMPE=no], [])
+   AC_CHECK_HEADERS([mpe.h], [HAVE_MPE=yes], [HAVE_MPE=no])
+   dnl if test "x$HAVE_LIBMPE" != xyes; then
+   dnl    AC_MSG_ERROR([-lmpe not found but --enable-mpe used.])
+   dnl fi
+   dnl if test "x$HAVE_LIBLMPE" != xyes; then
+   dnl    AC_MSG_ERROR([-llmpe not found but --enable-mpe used.])
+   dnl fi
+   if test $enable_fortran = yes; then
+      AC_MSG_ERROR([MPE not implemented in Fortran tests and examples. Build without --enable-fortran])
+   fi
+   AC_DEFINE([USE_MPE], 1, [If true, use MPE timing library.])
+
+fi
+
 # Does the user want to disable pnetcdf?
 AC_MSG_CHECKING([whether pnetcdf is to be used])
 AC_ARG_ENABLE([pnetcdf],

examples/c/Makefile.am

@@ -4,7 +4,7 @@
 # Ed Hartnett 5/7/18
 
 # Link to our assembled library.
-AM_LDFLAGS = ${top_builddir}/src/clib/libpio.la
+LDADD = ${top_builddir}/src/clib/libpio.la
 AM_CPPFLAGS = -I$(top_srcdir)/src/clib
 
 # Build the tests for make check.
@@ -19,4 +19,4 @@ endif # RUN_TESTS
 EXTRA_DIST = run_tests.sh
 
 # Clean up files produced during testing.
-CLEANFILES = *.nc *.log
+CLEANFILES = *.nc *.log *.clog2 *.slog2

examples/c/darray_no_async.c

@@ -21,6 +21,9 @@
 #include <gptl.h>
 #endif
 
+/* The name of this program. */
+#define TEST_NAME "darray_no_async"
+
 /* The number of possible output netCDF output flavors available to
  * the ParallelIO library. */
 #define NUM_NETCDF_FLAVORS 4
@@ -119,7 +122,7 @@ int check_file(int iosysid, int ntasks, char *filename, int iotype,
     /* Open the file. */
     if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0)))
         return ret;
-    printf("opened file %s ncid = %d\n", filename, ncid);
+    /* printf("opened file %s ncid = %d\n", filename, ncid); */
 
     /* Check the metadata. */
     if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
@@ -250,11 +253,17 @@ int main(int argc, char* argv[])
     if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
         MPIERR(ret);
 
+#ifdef USE_MPE
+    /* If MPE logging is being used, then initialize it. */
+    if ((ret = MPE_Init_log()))
+        return ret;
+#endif /* USE_MPE */
+
     /* Check that a valid number of processors was specified. */
     if (ntasks != TARGET_NTASKS)
         fprintf(stderr, "Number of processors must be 16!\n");
-    printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n",
-           my_rank, ntasks);
+    /* printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n", */
+    /*        my_rank, ntasks); */
 
     /* Turn on logging. */
     if ((ret = PIOc_set_log_level(LOG_LEVEL)))
@@ -281,8 +290,8 @@ int main(int argc, char* argv[])
     /* Create the PIO decomposition for this example. Since this
      * is a variable with an unlimited dimension, we want to
      * create a 2-D composition which represents one record. */
-    printf("rank: %d Creating decomposition, elements_per_pe %lld...\n", my_rank,
-           elements_per_pe);
+    /* printf("rank: %d Creating decomposition, elements_per_pe %lld...\n", my_rank, */
+    /*        elements_per_pe); */
     if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe,
                                 compdof, &ioid, PIO_REARR_SUBSET, NULL, NULL)))
         ERR(ret);
@@ -310,13 +319,13 @@ int main(int argc, char* argv[])
         sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]);
 
         /* Create the netCDF output file. */
-        printf("rank: %d Creating sample file %s with format %d...\n",
-               my_rank, filename, format[fmt]);
+        /* printf("rank: %d Creating sample file %s with format %d...\n", */
+        /*        my_rank, filename, format[fmt]); */
         if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER)))
             ERR(ret);
 
         /* Define netCDF dimension and variable. */
-        printf("rank: %d Defining netCDF metadata...\n", my_rank);
+        /* printf("rank: %d Defining netCDF metadata...\n", my_rank); */
         for (int d = 0; d < NDIM3; d++)
             if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d])))
                 ERR(ret);
@@ -336,7 +345,7 @@ int main(int argc, char* argv[])
                 buffer[i] = 100 * t + START_DATA_VAL + my_rank;
 
             /* Write data to the file. */
-            printf("rank: %d Writing sample data...\n", my_rank);
+            /* printf("rank: %d Writing sample data...\n", my_rank); */
             if ((ret = PIOc_setframe(ncid, varid, t)))
                 ERR(ret);
             if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL)))
@@ -348,7 +357,7 @@ int main(int argc, char* argv[])
             ERR(ret);
 
         /* Close the netCDF file. */
-        printf("rank: %d Closing the sample data file...\n", my_rank);
+        /* printf("rank: %d Closing the sample data file...\n", my_rank); */
         if ((ret = PIOc_closefile(ncid)))
             ERR(ret);
 
@@ -359,12 +368,12 @@ int main(int argc, char* argv[])
     }
 
     /* Free the PIO decomposition. */
-    printf("rank: %d Freeing PIO decomposition...\n", my_rank);
+    /* printf("rank: %d Freeing PIO decomposition...\n", my_rank); */
     if ((ret = PIOc_freedecomp(iosysid, ioid)))
         ERR(ret);
 
     /* Finalize the IO system. */
-    printf("rank: %d Freeing PIO resources...\n", my_rank);
+    /* printf("rank: %d Freeing PIO resources...\n", my_rank); */
     if ((ret = PIOc_free_iosystem(iosysid)))
         ERR(ret);
 
@@ -377,6 +386,7 @@ int main(int argc, char* argv[])
         return ret;
 #endif
 
-    printf("rank: %d SUCCESS!\n", my_rank);
+    if (!my_rank)
+        printf("rank: %d SUCCESS!\n", my_rank);
     return 0;
 }

examples/c/example1.c

@@ -20,6 +20,12 @@
 #ifdef TIMING
 #include <gptl.h>
 #endif
+#ifdef USE_MPE
+#include <mpe.h>
+#endif /* USE_MPE */
+
+/** The name of this program. */
+#define TEST_NAME "example1"
 
 /** The number of possible output netCDF output flavors available to
  * the ParallelIO library. */
@@ -303,7 +309,13 @@ int check_file(int ntasks, char *filename) {
 	    printf("%d: ParallelIO Library example1 running on %d processors.\n",
 		   my_rank, ntasks);
 
-	/* keep things simple - 1 iotask per MPI process */    
+#ifdef USE_MPE
+        /* If MPE logging is being used, then initialize it. */
+        if ((ret = MPE_Init_log()))
+            return ret;
+#endif /* USE_MPE */
+
+        /* keep things simple - 1 iotask per MPI process */
 	niotasks = ntasks;
 
         /* Turn on logging if available. */
@@ -424,7 +436,7 @@ int check_file(int ntasks, char *filename) {
 	    return ret;
 #endif    
 
-	if (verbose)
+        if (verbose)
 	    printf("rank: %d SUCCESS!\n", my_rank);
 	return 0;
     }

examples/c/examplePio.c

@@ -20,6 +20,12 @@
 #ifdef TIMING
 #include <gptl.h>
 #endif
+#ifdef USE_MPE
+#include <mpe.h>
+#endif /* USE_MPE */
+
+/** The name of this program. */
+#define TEST_NAME "examplePio"
 
 /** The length of our 1-d data array. */
 static const int LEN = 16;
@@ -179,7 +185,16 @@ struct examplePioClass* epc_init( struct examplePioClass* this )
 	  this->ntasks == 8 || this->ntasks == 16))
 	this->errorHandler(this, "Number of processors must be 1, 2, 4, 8, or 16!",
 			   ERR_CODE);
-
+
+/* #ifdef USE_MPE */
+/*     /\* If MPE logging is being used, then initialize it. *\/ */
+/*     { */
+/*         int ret; */
+/*         if ((ret = MPE_Init_log())) */
+/*             return NULL; */
+/*     } */
+/* #endif /\* USE_MPE *\/ */
+
     /*
     ** set up PIO for rest of example
     */
@@ -458,6 +473,7 @@ struct examplePioClass* epc_cleanUp( struct examplePioClass* this )
 
     PIOc_freedecomp(this->pioIoSystem, this->iodescNCells);
     PIOc_free_iosystem(this->pioIoSystem);
+
     MPI_Finalize();
 
     return this;
@@ -587,14 +603,20 @@ int main(int argc, char* argv[])
     if ((ret = GPTLinitialize ()))
       return ret;
 #endif    
-    
+
     pioExInst->init(pioExInst);
     pioExInst->createDecomp(pioExInst);
     pioExInst->createFile(pioExInst);
     pioExInst->defineVar(pioExInst);
     pioExInst->writeVar(pioExInst);
     pioExInst->readVar(pioExInst);
     pioExInst->closeFile(pioExInst);
+
+/* #ifdef USE_MPE */
+/*     if ((ret = MPE_Finish_log("examplePio"))) */
+/*         return ret; */
+/* #endif /\* USE_MPE *\/ */
+
     pioExInst->cleanUp(pioExInst);
 
 #ifdef TIMING    

src/clib/pio_darray.c

@@ -38,6 +38,11 @@ PIO_Offset maxusage = 0;
  * indicate that data are being written. */
 #define DARRAY_DATA 0
 
+#ifdef USE_MPE
+/* The event numbers for MPE logging. */
+extern int event_num[2][NUM_EVENTS];
+#endif /* USE_MPE */
+
 /**
  * Set the PIO IO node data buffer size limit.
  *
@@ -129,6 +134,10 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
     int ierr;              /* Return code. */
     void *tmparray;
 
+/* #ifdef USE_MPE */
+/*     pio_start_mpe_log(DARRAY_WRITE); */
+/* #endif /\* USE_MPE *\/ */
+
     /* Get the file info. */
     if ((ierr = pio_get_file(ncid, &file)))
         return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
@@ -406,6 +415,10 @@ PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         if ((ierr = flush_output_buffer(file, flushtodisk, 0)))
             return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
+/* #ifdef USE_MPE */
+/*     pio_stop_mpe_log(DARRAY_WRITE, __func__); */
+/* #endif /\* USE_MPE *\/ */
+
     return PIO_NOERR;
 }
 
@@ -643,6 +656,9 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra
 
     LOG((1, "PIOc_write_darray ncid = %d varid = %d ioid = %d arraylen = %d",
          ncid, varid, ioid, arraylen));
+#ifdef USE_MPE
+    pio_start_mpe_log(DARRAY_WRITE);
+#endif /* USE_MPE */
 
     /* Get the file info. */
     if ((ierr = pio_get_file(ncid, &file)))
@@ -841,6 +857,10 @@ PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *arra
         wmb->frame[wmb->num_arrays] = vdesc->record;
     wmb->num_arrays++;
 
+#ifdef USE_MPE
+    pio_stop_mpe_log(DARRAY_WRITE, __func__);
+#endif /* USE_MPE */
+
     LOG((2, "wmb->num_arrays = %d iodesc->maxbytes / iodesc->mpitype_size = %d "
          "iodesc->ndof = %d iodesc->llen = %d", wmb->num_arrays,
          iodesc->maxbytes / iodesc->mpitype_size, iodesc->ndof, iodesc->llen));
@@ -879,6 +899,10 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
     int ierr;              /* Return code. */
     void *tmparray;        /* unsorted copy of array buf if required */
 
+#ifdef USE_MPE
+    pio_start_mpe_log(DARRAY_READ);
+#endif /* USE_MPE */
+
     /* Get the file info. */
     if ((ierr = pio_get_file(ncid, &file)))
         return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
@@ -942,5 +966,9 @@ PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
     if (rlen > 0)
         brel(iobuf);
 
+#ifdef USE_MPE
+    pio_stop_mpe_log(DARRAY_READ, __func__);
+#endif /* USE_MPE */
+
     return PIO_NOERR;
 }

src/clib/pio_file.c

@@ -28,6 +28,11 @@
    netCDF-4/HDF5 (starts at 65xxx). */
 int pio_next_ncid = 16;
 
+#ifdef USE_MPE
+/* The event numbers for MPE logging. */
+extern int event_num[2][NUM_EVENTS];
+#endif /* USE_MPE */
+
 /**
  * Open an existing file using PIO library.
  *
@@ -221,6 +226,10 @@ int PIOc_closefile(int ncid)
     int ierr = PIO_NOERR;  /* Return code from function calls. */
     int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
 
+#ifdef USE_MPE
+    pio_start_mpe_log(CLOSE);
+#endif /* USE_MPE */
+
     LOG((1, "PIOc_closefile ncid = %d", ncid));
     /* Find the info about this file. */
     if ((ierr = pio_get_file(ncid, &file)))
@@ -294,6 +303,10 @@ int PIOc_closefile(int ncid)
     if ((ierr = pio_delete_file_from_list(ncid)))
         return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
+#ifdef USE_MPE
+    pio_stop_mpe_log(CLOSE, __func__);
+#endif /* USE_MPE */
+
     return ierr;
 }