Merge branch 'worleyph/atm/mpi_rsend_disable-by-default' (PR #1199)

Disable MPI_RSEND by default

Recent versions of the Cray MPI library are causing MPI communication
algorithms in PIO that use MPI_RSEND to hang. This is not the first time
that MPI_RSEND (and MPI_IRSEND) support has been broken in vendor MPI
libraries. Currently, CPP defines in CAM, MCT, and PIO will redefine
MPI_RSEND as MPI_SEND and MPI_IRSEND as MPI_ISEND when the CPP token
_NO_MPI_RSEND is defined during the build process. For improved
reliability, this modification makes this the default, always redefining
MPI_RSEND as MPI_SEND and MPI_IRSEND as MPI_ISEND, unless the CPP
token _USE_MPI_RSEND is defined during the build. Separate PRs have been
submitted for MCT and PIO. This one includes the changes for CAM.

BFB
Addresses Issue #1157

* worleyph/atm/mpi_rsend_disable-by-default:
  Disable MPI_RSEND by default

components/cam/src/control/wrap_mpi.F90

@@ -44,9 +44,13 @@
 !
 
 !
-! Performance bug work around for Gemini interconnect
+! Disable the use of the MPI ready send protocol by default, to
+! address recurrent issues with poor performance or incorrect 
+! functionality in MPI libraries. When support is known to be robust,
+! or for experimentation, can be re-enabled by defining the CPP token
+! _USE_MPI_RSEND during the build process.
 !
-#ifdef _NO_MPI_RSEND
+#ifndef _USE_MPI_RSEND
 #define mpi_rsend mpi_send
 #define mpi_irsend mpi_isend
 #endif

components/cam/src/utils/pilgrim/mod_comm.F90

@@ -130,9 +130,13 @@ module mod_comm
 #endif
 
 !
-! Performance bug work around for Gemini interconnect
+! Disable the use of the MPI ready send protocol by default, to
+! address recurrent issues with poor performance or incorrect 
+! functionality in MPI libraries. When support is known to be robust,
+! or for experimentation, can be re-enabled by defining the CPP token
+! _USE_MPI_RSEND during the build process.
 !
-#ifdef _NO_MPI_RSEND
+#ifndef _USE_MPI_RSEND
 #define MPI_RSEND MPI_SEND
 #define mpi_rsend mpi_send
 #define MPI_IRSEND MPI_ISEND

components/cam/src/utils/spmd_utils.F90

@@ -18,9 +18,13 @@ module spmd_utils
 !-----------------------------------------------------------------------
 
 !
-! Performance bug work around for Gemini interconnect
+! Disable the use of the MPI ready send protocol by default, to
+! address recurrent issues with poor performance or incorrect 
+! functionality in MPI libraries. When support is known to be robust,
+! or for experimentation, can be re-enabled by defining the CPP token
+! _USE_MPI_RSEND during the build process.
 !
-#ifdef _NO_MPI_RSEND
+#ifndef _USE_MPI_RSEND
 #define mpi_rsend mpi_send
 #define mpi_irsend mpi_isend
 #endif