Merge pull request #2589 from mnlevy1981/mlevy/mapping_on_geyser

Mlevy/mapping on geyser
Updated the wrappers for the ESMF mapping tools to work on NCAR machines. Note that currently only the serial modules are working - I have an issue ticket in with CISL to get the ESMF tools running in parallel, at which point I'll submit another pull request.

I also removed all reference to yellowstone and jaguar from the mapping tools.

Test suite: None
Test baseline: N/A
Test namelist changes: N/A
Test status: I verified that I could use my scripts to generate mapping files on cheyenne (both login node and compute node), geyser, caldera, and pronghorn.

Fixes #2469

User interface changes?: None

Update gh-pages html (Y/N)?: N

Code review: jedwards

config/cesm/machines/config_machines.xml

@@ -2111,7 +2111,6 @@
       <modules compiler="intel">
 	<command name="load">intel/15.0.3</command>
 	<command name="load">mkl/11.1.2</command>
-	<command name="load">trilinos/11.10.2</command>
 	<command name="load">esmf</command>
       </modules>
       <modules compiler="intel" mpilib="!mpi-serial" DEBUG="TRUE">

tools/mapping/check_maps/README

@@ -36,12 +36,12 @@ BUILD
 =====
 
 To compile this tool, you need to have the ESMFMKFILE environment variable set.
-The easiest way to do that is to load the ESMF modules. On yellowstone and
-cheyenne, you must already have the intel module loaded. Then load the version
-of ESMF you want to use (as of April 2017, the most recent version available is
-7.0.0):
+The easiest way to do that is to load the ESMF modules. On cheyenne, caldera,
+geyser, or pronghorn you must already have the intel module loaded. Then load the
+version of ESMF you want to use (as of May 2018, the most recent version on the
+NCAR machines is 7.0.0):
 
-Yellowstone:
+Caldera, Geyser, Pronghorn:
 $ module load esmf
 $ module load esmf-7.0.0-ncdfio-uni-O
 
@@ -78,6 +78,6 @@ listed in FILELIST
   --help, -h        Output this usage information
 
 Notes:
-  1) For use on cheyenne, yellowstone, geyser, or caldera only!
+  1) For use on cheyenne, geyser, caldera, or pronghorn only!
   2) Need to set ESMFMKFILE (see comments in Makefile) or compilation will fail
   3) If -rc option is not enabled, -v flag is ignored and verbose / concise will depend on previous compilation

tools/mapping/check_maps/check_map.sh

@@ -25,7 +25,7 @@ usage() {
 	echo '  --help, -h        Output this usage information'
 	echo ''
 	echo 'Notes:'
-	echo '  1) For use on yellowstone, geyser, or caldera only!'
+	echo '  1) For use on cheyenne, caldera, geyser, or pronghorn only!'
 	echo '  2) Need to set ESMFMKFILE (see comments in Makefile)'\
 	          'or compilation will fail'
 	echo '  3) If -rc option is not enabled, -v flag is ignored and verbose /'\
@@ -95,7 +95,7 @@ do
 	if [ -e $MAP ]; then
 		n=n+1
 		echo "${n}: ${MAP}"
-		$EXE $MAP
+		$EXE $MAP || exit $?
 		echo "-----"
 	else
 		echo "File not found: $MAP"

tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90

@@ -761,9 +761,9 @@ program OfflineTester
       ! destroy and deallocate
         call ESMF_ArrayDestroy(srcArray, rc=status)
         call ESMF_ArrayDestroy(dstArray, rc=status)
-        if (ESMF_LogFoundError(rcToCheck=status, msg=ESMF_LOGERR_PASSTHRU, &
-                               line=__LINE__, file=__FILE__, rcToReturn=rc)) &
-          call ESMF_Finalize(endflag=ESMF_END_ABORT)
+!        if (ESMF_LogFoundError(rcToCheck=status, msg=ESMF_LOGERR_PASSTHRU, &
+!                               line=__LINE__, file=__FILE__, rcToReturn=rc)) &
+!          call ESMF_Finalize(endflag=ESMF_END_ABORT)
 
       call ESMF_Finalize()
 

tools/mapping/check_maps/src/Makefile

@@ -8,11 +8,11 @@
 
 ################################################################################
 ### Finding and including esmf.mk                                         ######
-### NOTE: On Yellowstone, ESMF is only available with the intel compilers ######
+### NOTE: On Cheyenne, ESMF is only available with the intel compilers    ######
 ###       And require running the following:                              ######
-###           $ module load esmf                                          ######
-###           $ module load esmf-6.3.0r-ncdfio-uni-O                      ######
-###       The executable should run on the yellowstone login node         ######
+###           $ module load esmf_libs/7.0.0                               ######
+###           $ module load esmf-7.0.0-ncdfio-uni-O                       ######
+###       The executable should run on the cheyenne login node            ######
 ################################################################################
 
 # Don't require ESMF to be loaded to run "make clean":

tools/mapping/gen_domain_files/src/.gitignore

@@ -4,3 +4,4 @@ Depends*
 Macros.make
 *.o
 *.mod
+*.optrpt

tools/mapping/gen_domain_files/src/Makefile

@@ -143,7 +143,7 @@ $(EXENAME): $(OBJS)
 	$(LINKER) -o $@ $(OBJS) $(SLIBS) $(LDFLAGS)
 
 clean:
-	$(RM) -f $(OBJS) $(EXENAME) *.mod
+	$(RM) -f $(OBJS) $(EXENAME) *.mod *.optrpt
 
 distclean: clean
 	../../../configure --clean

tools/mapping/gen_mapping_files/.gitignore

@@ -0,0 +1,2 @@
+*.Log
+*.nc

tools/mapping/gen_mapping_files/README

@@ -64,10 +64,12 @@ gen_cesm_maps.sh
   --fileocn|-focn  input ocn_grid_filename
   --filelnd|-flnd  input lnd_grid_filename
   --filertm|-frtm  input rtm_grid_filename
+  --fileglc|-fglc  input glc_grid_filename
   --nameocn|-nocn  output ocn_name in mapping file
   --nameatm|-natm  output atm_name in mapping file
   --namelnd|-nlnd  output lnd_name in mapping file
   --namertm|-nrtm  output rtm_name in mapping file
+  --nameglc|-nglc  output glc_name in mapping file
   [ --typeocn|tocn ] [regional|global]
   [ --typeatm|tatm ] [regional|global]
   [ --nogridcheck ]
@@ -84,6 +86,8 @@ where
    SCRIP grid format land filename (full pathname), must be global
  --filertm (or -frtm)
    SCRIP grid format runoff filename (full pathname)
+ --fileglc (or -fglc)
+   SCRIP grid format glc filename (full pathname), assumed to be regional
  --nameatm (or -natm)
    Shortname to use for atm in mapping filename
  --nameocn (or -nocn)
@@ -92,6 +96,8 @@ where
    Shortname to use for lnd in mapping filename
  --namertm (or -nrtm)
    Shortname to use for rtm in mapping filename
+ --nameglc (or -nglc)
+   Shortname to use for glc in mapping filename
  --typeocn (or -tocn)
    ocean grid type,  valid values are regional or global
    default is global
@@ -102,57 +108,20 @@ where
  --nogridcheck
    By default, script will run consistency check on new
    maps; this flag disables these checks
- --batch (or -b)
-   Toggles batch mode usage. If you want to run in batch mode
-   you need to have a separate batch script for a supported machine
-   that calls this script interactively - you cannot submit this
-   script directly to the batch system
+ --serial
+   Run the ESMF tools in serial rather than parallel
  -rc
    Pass the "--recompile" flag to the ESMF tool
    (Only necessary if nothing has been built in ../check_maps/)
- -d
-   toggle debug-only
  --help or -h
    displays this help message
 
-Note: if rtm is specified and lnd is not, then this tool will
+Note: if rtm or glc are specified and lnd is not, then this tool will
       assume lnd and atm are on the same grid.
 
 You can also set the following env variables:
   ESMFBIN_PATH - Path to ESMF binaries
-                 (Leave unset on cheyenne, yellowstone, and caldera and the
-                 tool will be loaded from modules)
+                 (Known machines will load tools from modules)
   MPIEXEC ------ Name of mpirun executable
-                 (default is mpirun.lsf on yellowstone and caldera; if
-                 you run interactively on yellowstone, mpi is not used)
-  REGRID_PROC -- Number of MPI processors to use
-                 (default is 8)
-
-=================
-USAGE: BATCH MODE
-=================
-
-Currently, batch mode is only set up for yellowstone. This uses the file
-regridbatch.yellowstone.sh as a wrapper to gen_cesm_maps.sh.
-
-IMPORTANT NOTE: check_maps is NOT run when you run in batch mode. This must be
-run as a separate step after you have created the mapping files. See below for
-an example of how to do this.
-
-(1) Modify the #BSUB lines at the top of regridbatch.yellowstone.sh. You need to
-at least specify a project number; you may want to change some of the other
-settings, too.
-
-(2) Modify the variables in the section labeled "Set user-defined parameters
-here". See the documentation for gen_cesm_maps.sh above for more details on
-the parameters that can be set here.
-
-(3) Submit with:
-
-bsub < regridbatch.yellowstone.sh
-
-(4) Run the check_maps tool on the generated mapping files; e.g.:
-
-    files=*.nc
-    ../check_maps/check_map.sh $files
+                 (currently tools only run in serial due to module issues)
 

tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore

@@ -1,2 +1,5 @@
+.env_mach*
+env_mach*
+Depends*
 *.Log
 *.nc

tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README

@@ -60,20 +60,14 @@ where
    default is global
  --pass2esmf
    pass options directly to the ESMF tool.
- --batch (or -b)
-   Toggles batch mode usage. If you want to run in batch mode
-   you need to have a separate batch script for a supported machine
-   that calls this script interactively - you cannot submit this
-   script directly to the batch system
  --clm_name
    Use the CLM naming convention
  --serial
-   For yellowstone batch jobs only! Load the serial ESMF tools rather
-   than the parallel tools (necessary for mapping grids with a single
-   point).
+   Load the serial ESMF tools rather than the parallel tools
+   (necessary for mapping grids with a single point).
  --machine (or -mach)
-   Name of the machine you are running on. Currently supports yellowstone,
-   geyser, caldera, pronghorn, and jaguar. Note that this script will
+   Name of the machine you are running on. Currently supports cheyenne,
+   geyser, caldera, and pronghorn. Note that this script will
    determine the machine name automatically from the hostfile command.
  -d
    toggle debug-only
@@ -82,36 +76,9 @@ where
 
 You can also set the following env variables:
   ESMFBIN_PATH - Path to ESMF binaries
-                 (Leave unset on cheyenne, yellowstone, and caldera and the
-                 tool will be loaded from modules)
+                 (Leave unset on cheyenne/caldera/geyser/pronghorn
+                 and the tool will be loaded from modules)
   MPIEXEC ------ Name of mpirun executable
-                 (default is mpirun.lsf on yellowstone and caldera; if
-                 you run interactively on yellowstone, mpi is not used)
-  REGRID_PROC -- Number of MPI processors to use (jaguar only!)
-                 (default is 8)
-
-=================
-USAGE: BATCH MODE
-=================
-
-Currently, batch mode is only set up for yellowstone. This uses the file
-regridbatch.yellowstone.sh as a wrapper to create_ESMF_map.sh. Before submitting,
-make sure you do the following:
-
-(1) Modify the #BSUB lines at the top of regridbatch.yellowstone.sh. You need to
-at least specify a project number; you may want to change some of the other
-settings, too.
-
-(2) Modify the variables in the section labeled "Set user-defined parameters
-here". See the documentation for create_ESMF_map.sh above for more details on
-the parameters that can be set here.
-
-(3) Submit with:
-
-bsub < regridbatch.yellowstone.sh
-
-(4) Run the check_maps tool on the generated mapping file:
-
-    out_file=*.nc
-    ../../check_maps/check_map.sh ${out_file}
+                 (ignored if --serial, which is default on cheyenne
+                 login nodes