ESMCI · jedwards4b · Apr 21, 2017 · Apr 19, 2017 · Apr 19, 2017 · Apr 19, 2017
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,13 +2,15 @@ cmake_minimum_required(VERSION 2.8)
 include(ExternalProject)
 set(CIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}")
 
+list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY})
+include(CIME_initial_setup)
+
 project(cime_tests Fortran C)
 
 # We rely on pio for cmake utilities like findnetcdf.cmake, so that we don't
 # need to duplicate this cmake code
 list(APPEND CMAKE_MODULE_PATH "${CIME_ROOT}/src/externals/pio2/cmake")
 
-list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY})
 include(CIME_utils)
 find_package(NetCDF COMPONENTS C Fortran)
 include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS})
@@ -21,14 +23,21 @@ include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS})
 # ------------------------------------------------------------------------
 set(MCT_ROOT "${CIME_ROOT}/src/externals/mct")
 
+if (USE_MPI_SERIAL)
+  set(ENABLE_MPI_SERIAL "--enable-mpiserial")
+else()
+  set(ENABLE_MPI_SERIAL "")
+endif()
+
 ExternalProject_add(mct_project
   PREFIX ${CMAKE_CURRENT_BINARY_DIR}
   SOURCE_DIR ${MCT_ROOT}
   BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/mct
-  CONFIGURE_COMMAND ${MCT_ROOT}/configure --enable-debugging --prefix=${CMAKE_CURRENT_BINARY_DIR} CFLAGS=${CFLAGS} FCFLAGS=${FFLAGS} SRCDIR=${MCT_ROOT} DEBUG="-g"
-  BUILD_COMMAND $(MAKE)
+  CONFIGURE_COMMAND ${MCT_ROOT}/configure ${ENABLE_MPI_SERIAL} --enable-debugging --prefix=${CMAKE_CURRENT_BINARY_DIR} CC=${CMAKE_C_COMPILER} FC=${CMAKE_Fortran_COMPILER} CFLAGS=${CFLAGS} FCFLAGS=${FFLAGS} SRCDIR=${MCT_ROOT} DEBUG="-g"
+  BUILD_COMMAND $(MAKE) SRCDIR=${MCT_ROOT}
   # Leave things in <BINARY_DIR> rather than "installing", because we have
-  # no need to move things around inside of the CMake binary directory.
+  # no need to move things around inside of the CMake binary directory. Also,
+  # mpi-serial doesn't install properly in the out-of-source build
   INSTALL_COMMAND :
   )
 # This copy_makefiles step is needed because mct currently doesn't support an
@@ -43,12 +52,27 @@ ExternalProject_add_step(mct_project copy_makefiles
   COMMAND mkdir -p mpeu
   COMMAND cp -p <SOURCE_DIR>/mpeu/Makefile mpeu/
   )
+if (USE_MPI_SERIAL)
+  ExternalProject_add_step(mct_project copy_mpi_serial_files
+    DEPENDEES configure
+    DEPENDERS build
+    WORKING_DIRECTORY <BINARY_DIR>
+    COMMAND mkdir -p mpi-serial
+    COMMAND cp -p <SOURCE_DIR>/mpi-serial/Makefile mpi-serial/
+    COMMAND cp <SOURCE_DIR>/mpi-serial/mpif.h mpi-serial/
+    COMMAND cp <SOURCE_DIR>/mpi-serial/mpi.h mpi-serial/
+    )
+endif()
 
 # Tell cmake to look for libraries & mod files here, because this is where we built libraries
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct)
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu)
 link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct)
 link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu)
+if (USE_MPI_SERIAL)
+  include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial)
+  link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial)
+endif()
 
 # ------------------------------------------------------------------------
 # Done MCT build

diff --git a/README.unit_testing b/README.unit_testing
@@ -1,6 +1,5 @@
-# To run all CIME unit tests on caldera, run the following command:
-# (Note that this must be done from an interactive caldera session, not from yellowstone)
-# Note also that this requires module load all-python-libs
+# To run all the CIME Fortran unit tests, run the following command:
+# On yellowstone, this requires module load all-python-libs
 #
 # The creation of a temporary directory ensures that you are doing a completely
 # clean build of the unit tests. (The use of the --clean flag to run_tests.py

diff --git a/config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml
@@ -811,7 +811,6 @@ for mct, etc.
               -L$(LAPACK_LIBDIR) -Wl,-rpath=$(LAPACK_LIBDIR) \
               -L$(BLAS_LIBDIR) -Wl,-rpath=$(BLAS_LIBDIR)
       </ADD_LDFLAGS>
-      <PFUNIT_PATH></PFUNIT_PATH>
 </compiler>
 
 <compiler COMPILER="intel" MACH="titan">
@@ -920,7 +919,6 @@ for mct, etc.
   <LD> mpixlf77_r </LD>
   <NETCDF_PATH>/soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/</NETCDF_PATH>
   <PNETCDF_PATH>/soft/libraries/pnetcdf/1.6.0/cnk-xl/current/</PNETCDF_PATH>
-  <PFUNIT_PATH>/home/santos/pFUnit/pFUnit_IBM</PFUNIT_PATH>
   <HDF5_PATH>/soft/libraries/hdf5/1.8.14/cnk-xl/current/</HDF5_PATH>
   <ADD_SLIBS>-L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf -L$(HDF5_PATH)/lib -lhdf5_hl -lhdf5 -L/soft/libraries/alcf/current/xl/ZLIB/lib -lz -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack -L/soft/libraries/alcf/current/xl/BLAS/lib -lblas -L/bgsys/drivers/ppcfloor/comm/sys/lib </ADD_SLIBS>
   <ADD_SLIBS compile_threaded="true"> -L$(IBM_MAIN_DIR)/xlf/bg/14.1/bglib64 -lxlfmath -lxlf90_r -lxlopt -lxl -L$(IBM_MAIN_DIR)/xlsmp/bg/3.1/bglib64 -lxlsmp </ADD_SLIBS>
@@ -943,7 +941,6 @@ for mct, etc.
   <LD> mpixlf77_r </LD>
   <NETCDF_PATH>/soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/</NETCDF_PATH>
   <PNETCDF_PATH>/soft/libraries/pnetcdf/1.6.0/cnk-xl/current/</PNETCDF_PATH>
-  <PFUNIT_PATH>/home/santos/pFUnit/pFUnit_IBM</PFUNIT_PATH>
   <HDF5_PATH>/soft/libraries/hdf5/1.8.14/cnk-xl/current/</HDF5_PATH>
   <ADD_SLIBS>-L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf -L$(HDF5_PATH)/lib -lhdf5_hl -lhdf5 -L/soft/libraries/alcf/current/xl/ZLIB/lib -lz -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack -L/soft/libraries/alcf/current/xl/BLAS/lib -lblas -L/bgsys/drivers/ppcfloor/comm/sys/lib </ADD_SLIBS>
   <ADD_SLIBS compile_threaded="true"> -L$(IBM_MAIN_DIR)/xlf/bg/14.1/bglib64 -lxlfmath -lxlf90_r -lxlopt -lxl -L$(IBM_MAIN_DIR)/xlsmp/bg/3.1/bglib64 -lxlsmp </ADD_SLIBS>

diff --git a/config/acme/machines/userdefined_laptop_template/config_compilers.xml b/config/acme/machines/userdefined_laptop_template/config_compilers.xml
@@ -29,7 +29,7 @@
       <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
       <NETCDF_PATH>/usr/local</NETCDF_PATH>
       <ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate</ADD_SLIBS>
-      <PFUNIT_PATH>$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
+      <PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
    </compiler>
 
    <compiler COMPILER="gnu" MACH="example-osx-macports">
@@ -57,6 +57,6 @@
       <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
       <NETCDF_PATH>/opt/local</NETCDF_PATH>
       <ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate</ADD_SLIBS>
-      <PFUNIT_PATH>$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
+      <PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
    </compiler>
 </config_compilers>
diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml
@@ -687,10 +687,10 @@ using a fortran linker.
     <append> -Wl,-rpath,$ENV{MPI_PATH}/lib</append>
     <append> -lifcore</append>
   </LDFLAGS>
-  <PFUNIT_PATH>/home/santos/pFUnit/pFUnit_Intel_3_0</PFUNIT_PATH>
   <SLIBS>
     <append MPILIB="mvapich2"> -mkl=cluster </append>
   </SLIBS>
+  <PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">/fs/cgd/csm/tools/pFUnit/pFUnit3.2.8_hobart_Intel15.0.2_noMPI_noOpenMP</PFUNIT_PATH>
 </compiler>
 
 <compiler MACH="hobart" COMPILER="nag">
@@ -701,7 +701,6 @@ using a fortran linker.
   <LDFLAGS>
     <append> -lpthread</append>
   </LDFLAGS>
-  <PFUNIT_PATH>/home/santos/pFUnit/pFUnit_NAG_3_0</PFUNIT_PATH>
 </compiler>
 
 <compiler MACH="hobart" COMPILER="pgi">
@@ -767,7 +766,6 @@ using a fortran linker.
   <MPICC> /home/pkcoff/mpich-sandboxes/master/install-production/bin/mpixlc_r </MPICC>
   <MPIFC> /home/pkcoff/mpich-sandboxes/master/install-production/bin/mpixlf2003_r </MPIFC>
   <NETCDF_PATH>/soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/</NETCDF_PATH>
-  <PFUNIT_PATH>/home/santos/pFUnit/pFUnit_IBM</PFUNIT_PATH>
   <PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
   <PNETCDF_PATH>/soft/libraries/pnetcdf/1.6.1/cnk-xl/current/</PNETCDF_PATH>
   <SCC> /home/pkcoff/mpich-sandboxes/master/install-production/bin/mpixlc_r </SCC>
@@ -930,7 +928,6 @@ using a fortran linker.
   <MPICC> mpixlc_r </MPICC>
   <MPIFC> mpixlf2003_r </MPIFC>
   <NETCDF_PATH>/soft/libraries/netcdf/4.3.0-f4.2/cnk-xl/V1R2M0-20131211/</NETCDF_PATH>
-  <PFUNIT_PATH>/home/santos/pFUnit/pFUnit_IBM</PFUNIT_PATH>
   <PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
   <PNETCDF_PATH>/soft/libraries/pnetcdf/1.3.1/cnk-xl/current/</PNETCDF_PATH>
   <SCC> mpixlc_r </SCC>
@@ -992,7 +989,8 @@ using a fortran linker.
   <CMAKE_OPTS>
     <append DEBUG="TRUE"> -DPIO_ENABLE_LOGGING=ON </append>
   </CMAKE_OPTS>
-  <PFUNIT_PATH>$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.2.8_cheyenne_Intel17.0.1_MPI_openMP</PFUNIT_PATH>
+  <PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.2.8_cheyenne_Intel17.0.1_noMPI_noOpenMP</PFUNIT_PATH>
+  <PFUNIT_PATH MPILIB="mpt" compile_threaded="true">$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.2.8_cheyenne_Intel17.0.1_MPI_openMP</PFUNIT_PATH>
   <!-- Bug in the intel/17.0.1 compiler requires this, remove this line when compiler is updated -->
   <HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
 </compiler>
@@ -1037,7 +1035,8 @@ using a fortran linker.
   <MPICXX MPILIB="mpich2">mpiicpc</MPICXX>
   <PAPI_INC MPILIB="mpich2"> /glade/apps/opt/papi/5.3.0/intel/12.1.5/include/</PAPI_INC>
   <PAPI_LIB MPILIB="mpich2">/glade/apps/opt/papi/5.3.0/intel/12.1.5/lib64 </PAPI_LIB>
-  <PFUNIT_PATH>$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.1_Intel15.0.1_MPI</PFUNIT_PATH>
+  <PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.1_yellowstone_Intel15.0.1_noMPI_noOpenMP</PFUNIT_PATH>
+  <PFUNIT_PATH MPILIB="mpich2" compile_threaded="true">$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.1_Intel15.0.1_MPI</PFUNIT_PATH>
   <!-- Needed due to the way that netcdf is loaded on yellowstone -->
   <SCC MPILIB="mpi-serial">icc</SCC>
   <SFC MPILIB="mpi-serial">ifort</SFC>

diff --git a/config/cesm/machines/userdefined_laptop_template/config_compilers.xml b/config/cesm/machines/userdefined_laptop_template/config_compilers.xml
@@ -29,7 +29,7 @@
       <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
       <NETCDF_PATH>/usr/local</NETCDF_PATH>
       <ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate</ADD_SLIBS>
-      <PFUNIT_PATH>$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
+      <PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
    </compiler>
 
    <compiler COMPILER="gnu" MACH="example-osx-macports">
@@ -57,6 +57,6 @@
       <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
       <NETCDF_PATH>/opt/local</NETCDF_PATH>
       <ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate</ADD_SLIBS>
-      <PFUNIT_PATH>$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
+      <PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
    </compiler>
 </config_compilers>
diff --git a/scripts/lib/CIME/BuildTools/cmakemacroswriter.py b/scripts/lib/CIME/BuildTools/cmakemacroswriter.py
@@ -89,7 +89,8 @@ def set_variable(self, name, value):
         >>> s.getvalue()
         u'set(foo "bar")\\n'
         """
-        self.write_line("set(" + name + ' "' + value + '")')
+        value_transformed = self._transform_value(value)
+        self.write_line("set(" + name + ' "' + value_transformed + '")')
 
     def start_ifeq(self, left, right):
         """Write out a statement to start a conditional block.
@@ -117,3 +118,22 @@ def end_ifeq(self):
         """
         self.indent_left()
         self.write_line("endif()")
+
+    def _transform_value(self, value):
+        """Some elements need their values transformed in some way for CMake to handle them properly.
+        This method does those transformations.
+
+        Args:
+        - value (str): value of element
+
+        Returns transformed value
+        """
+
+        # Not all variables need leading & trailing whitespace removed, but some
+        # do. In particular, compiler variables (MPICC, MPICXX, MPIFC, SCC,
+        # SCXX, SFC) are only handled correctly if leading & trailing whitespace
+        # is removed. It doesn't seem to hurt to remove whitespace from all
+        # variables.
+        value_transformed = value.strip()
+
+        return value_transformed
diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
@@ -181,7 +181,9 @@ def setUpClass(cls):
     def _has_unit_test_support(self):
         default_compiler = MACHINE.get_default_compiler()
         compiler = Compilers(MACHINE, compiler=default_compiler)
-        pfunit_path = compiler.get_optional_compiler_node("PFUNIT_PATH")
+        attrs = {'MPILIB': 'mpi-serial', 'compile_threaded': 'false'}
+        pfunit_path = compiler.get_optional_compiler_node("PFUNIT_PATH",
+                                                          attributes=attrs)
         if pfunit_path is None:
             return False
         else:

diff --git a/src/drivers/mct/unit_test/CMakeLists.txt b/src/drivers/mct/unit_test/CMakeLists.txt
@@ -11,6 +11,11 @@ add_definitions(
   -DNUM_COMP_INST_ESP=1
   )
 
+# The following definitions are needed when building with the mpi-serial library
+if (USE_MPI_SERIAL)
+  add_definitions(-DNO_MPI2 -DNO_MPIMOD)
+endif()
+
 # Add source directories from stubs. This should be done first, so that in the
 # case of name collisions, the drv versions take precedence (when there are two
 # files with the same name, the one added later wins).
@@ -46,6 +51,9 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 # tests need all of these libraries, but it's easiest just to set the same list
 # for everyone.
 set(DRV_UNIT_TEST_LIBS drv;csm_share;esmf_wrf_timemgr;mct;mpeu)
+if (USE_MPI_SERIAL)
+  list(APPEND DRV_UNIT_TEST_LIBS mpi-serial)
+endif()
 list(APPEND DRV_UNIT_TEST_LIBS ${NETCDF_LIBRARIES})
 
 # Add the test directories

diff --git a/src/externals/CMake/CIME_initial_setup.cmake b/src/externals/CMake/CIME_initial_setup.cmake
@@ -0,0 +1,16 @@
+# Module used for CIME testing.
+#
+# This module does some initial setup that must be done BEFORE the 'project'
+# line in the main CMakeLists.txt file.
+
+include(${CMAKE_BINARY_DIR}/Macros.cmake RESULT_VARIABLE FOUND)
+if(NOT FOUND)
+  message(FATAL_ERROR "You must generate a Macros.cmake file using CIME's configure")
+endif()
+if("$ENV{MPILIB}" STREQUAL "mpi-serial")
+  set(CMAKE_C_COMPILER ${SCC})
+  set(CMAKE_Fortran_COMPILER ${SFC})
+else()
+  set(CMAKE_C_COMPILER ${MPICC})
+  set(CMAKE_Fortran_COMPILER ${MPIFC})
+endif()
diff --git a/src/externals/CMake/CIME_utils.cmake b/src/externals/CMake/CIME_utils.cmake
@@ -3,6 +3,10 @@
 # This module contains statements that would otherwise be boilerplate in
 # most CIME tests. It enables CTest testing, handles the USE_COLOR and
 # ENABLE_GENF90 arguments, and includes several other modules.
+#
+# Some of the things done here must be done AFTER the 'project' line in the main
+# CMakeLists.txt file. This assumes that CIME_initial_setup has already been
+# included.
 
 #==========================================================================
 # Copyright (c) 2013-2014, University Corporation for Atmospheric Research
@@ -30,16 +34,10 @@ set(CMAKE_COLOR_MAKEFILE "${USE_COLOR}")
 # Compiler info
 #=================================================
 
-include(${CMAKE_BINARY_DIR}/Macros.cmake RESULT_VARIABLE FOUND)
 list(APPEND CMAKE_MODULE_PATH "../pio2/cmake")
-set(CMAKE_C_COMPILER ${MPICC})
-set(CMAKE_Fortran_COMPILER  ${MPIF90})
 set(CMAKE_C_FLAGS "${CPPDEFS} ${CFLAGS}")
 set(CMAKE_Fortran_FLAGS "${CPPDEFS} ${FFLAGS}")
-set (CMAKE_EXE_LINKER_FLAGS ${LDFLAGS})
-if(NOT FOUND)
-       message(FATAL_ERROR "You must generate a Macros.cmake file using CIME's configure")
-endif()
+set(CMAKE_EXE_LINKER_FLAGS "${LDFLAGS} ${SLIBS}")
 
 include(Compilers)
 

diff --git a/src/externals/CMake/README.md b/src/externals/CMake/README.md
@@ -28,6 +28,12 @@ Sourcelist_utils - Use source file lists defined over multiple directories.
 
 Modules that are CESM-specific and/or incomplete:
 
-CIME_utils - Handles a few options, and includes several other modules.
+CIME\_initial\_setup - Handles setup that must be done before the 'project'
+line. This must be included before the 'project' line in the main CMakeLists.txt
+file.
+
+CIME_utils - Handles a few options, and includes several other modules. This
+must be included after the 'project' line in the main CMakeLists.txt file, and
+after the inclusion of CIME\_initial\_setup.
 
 Compilers - Specify compiler-specific behavior, add build types for CESM.
diff --git a/tools/unit_testing/Examples/circle_area/tests/CMakeLists.txt b/tools/unit_testing/Examples/circle_area/tests/CMakeLists.txt
@@ -1,7 +1,9 @@
 cmake_minimum_required(VERSION 2.8)
-project(circle_area_tests Fortran C)
 
 list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY})
+include(CIME_initial_setup)
+
+project(circle_area_tests Fortran C)
 
 include(CIME_utils)
 

diff --git a/tools/unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt b/tools/unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt
@@ -1,7 +1,10 @@
 cmake_minimum_required(VERSION 2.8)
-project(interpolate_1d_tests Fortran C)
 
 list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY})
+include(CIME_initial_setup)
+
+project(interpolate_1d_tests Fortran C)
+
 include(CIME_utils)
 
 add_subdirectory(../src interpolate_1d)