NOAA machine pass of base_suite #155 (#372)

* update to include a 'high_Sierra' machine with gnu compilers,
only marginally modified from the travisCI env and Macros.

* Syncing theia and upstream/master branch

* theia execution

* intel compiler version tests -- 15.0.0 or greater

* working on variations for round robin test

* syncing to main

* sync

* testing

* end test

* successful on phase2 for all regression tests

* start on phase3 execution

* add phase3 system

* syncing and phase3 modules

* update configs manually from master

* env edit

* builds on phase3 now

* test

* trying to pull/merge cice master

* theia

* successful theia build

* garbage

* update theia intel env

* mods for running in NOAA

* start towards hera

* tmp

* theia passes with most recent of all modules

* moving to hera

* clean regression run on hera

* cleanup

* phase3 work

* minor manual move towards master copy

* re-fix phase3 script

* nearing full phase3

* update path for data

* actual final version of phase3

* from master

* cicecore updating from master

* from consortium/master

* more from master

* retry sync

* manual add/subtrace

* script perms and large diff files

* retrieve hera

* docs and cicecore sync to master

* machine syncing

* couple more

* some more

* finally the right template

* finally correct template

* typography in master

* restore working batch.csh

* towards matching consortium master

* last? add for syncing

* Revert icepack to 1.1.2

Co-authored-by: Tony Craig <[email protected]>

.gitignore

@@ -8,6 +8,7 @@ doc/build
 # Ignore macOS cache files
 .DS_Store
 
+
 # Ignore testsuite file/directories
 testsuite*
 caselist*

configuration/scripts/cice.batch.csh

@@ -213,7 +213,20 @@ cat >> ${jobfile} << EOFB
 #SBATCH -N ${nnodes}
 #SBATCH -e slurm%j.err
 #SBATCH -o slurm%j.out
-#SBATCH --mail-type END,FAIL
+#SBATCH --mail-type FAIL
+#SBATCH [email protected]
+EOFB
+
+else if (${ICE_MACHINE} =~ hera*) then
+cat >> ${jobfile} << EOFB
+#SBATCH -J ${ICE_CASENAME}
+#SBATCH -t `echo ${batchtime} | cut -f1-2 -d:`
+#SBATCH -q batch
+#SBATCH -A marine-cpu
+#SBATCH -N ${nnodes}
+#SBATCH -e slurm%j.err
+#SBATCH -o slurm%j.out
+#SBATCH --mail-type FAIL
 #SBATCH [email protected]
 EOFB
 
@@ -223,8 +236,25 @@ cat >> ${jobfile} << EOFB
 EOFB
 
 else if (${ICE_MACHINE} =~ phase3*) then
+if ( ${nnodes} > 15) then
+  setenv p3tile 16
+  setenv mem `expr 100 \* 1024 / $nnodes`
+else
+  setenv p3tile ${nnodes}
+  setenv mem 8192
+endif
+echo mem = ${mem} nnodes and p3tiles ${nnodes} ${p3tile} p3tile must be le nnodes
 cat >> ${jobfile} << EOFB
-# nothing to do
+#BSUB -J ${ICE_CASENAME}
+#BSUB -q "dev_shared"
+#BSUB -P RTO-T2O
+#BSUB -W `echo ${batchtime} | cut -f1-2 -d:`
+#BSUB -n ${nnodes}
+#BSUB -R "affinity[core]"
+#BSUB -R "span[ptile=${p3tile}]"
+#BSUB -R "rusage[mem=${mem}]"
+#BSUB -o /u/Robert.Grumbine/${ICE_CASENAME}.out.%J
+#BSUB -e /u/Robert.Grumbine/${ICE_CASENAME}.err.%J
 EOFB
 
 else if (${ICE_MACHINE} =~ high_Sierra*) then

configuration/scripts/cice.launch.csh

@@ -197,6 +197,20 @@ cat >> ${jobfile} << EOFR
 ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 
+#=======
+else if (${ICE_MACHINE} =~ phase3*) then
+cat >> ${jobfile} << EOFR
+mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+
+#=======
+else if (${ICE_MACHINE} =~ phase2*) then
+cat >> ${jobfile} << EOFR
+mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+
+=======
+
 #=======
 else if (${ICE_MACHINE} =~ travisCI*) then
 if (${ICE_COMMDIR} =~ serial*) then

configuration/scripts/machines/Macros.hera_intel

@@ -0,0 +1,60 @@
+#==============================================================================
+# Makefile macros for theia - intel and openmpi compilers
+#==============================================================================
+
+CPP        := cpp
+CPPDEFS    := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS}
+CFLAGS     := -c -O2  -xHost
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback   -xHost
+
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+#RG: this looks more like gfortran options:  
+##  FFLAGS     += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+  FFLAGS     += -O0 -g -check-bounds -implicit-none -fpe3
+else
+  FFLAGS     += -O2
+endif
+
+#Note that on theia mpif90 refers to gfortran even though it is in the intel bin
+FC := mpiifort
+MPIFC:= mpiifort
+LD:= $(MPIFC)
+
+NETCDF_PATH := $(NETCDF)
+
+ifeq ($(ICE_IOTYPE), netcdf)
+    NETCDF_PATH := $(shell nc-config --prefix)
+    INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include
+    LIB_NETCDF := $(NETCDF_PATH)/lib
+    LIB_PNETCDF := 
+    SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+else
+    SLIBS   := 
+endif
+
+LIB_MPI := 
+
+
+SCC   := icc 
+SFC   := ifort
+MPICC := mpiicc
+MPIFC := mpiifort
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+endif

configuration/scripts/machines/env.hera_intel

@@ -0,0 +1,26 @@
+#!/bin/csh -f
+
+source /etc/profile.d/modules.csh
+#module list
+module purge
+module load intel/18.0.5.274
+module load impi/2018.0.4
+module load esmf/7.1.0r
+module load hdf5/1.10.4  netcdf/4.6.1 
+module load wgrib wgrib2
+#echo renewed modules:
+#module list
+
+setenv ICE_MACHINE_ENVNAME hera
+setenv ICE_MACHINE_COMPILER intel
+setenv ICE_MACHINE_MAKE make
+setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/rgdev/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/rgdev/
+setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/rgdev/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "sbatch"
+setenv ICE_MACHINE_TPNODE 4
+setenv ICE_MACHINE_ACCT P0000000
+setenv ICE_MACHINE_QUEUE "batch"
+setenv ICE_MACHINE_BLDTHRDS 1
+setenv ICE_MACHINE_QSTAT " "
+#setenv ICE_MACHINE_QUIETMODE true

configuration/scripts/machines/env.high_Sierra_gnu

@@ -4,7 +4,7 @@ setenv ICE_MACHINE_ENVNAME high_Sierra
 setenv ICE_MACHINE_COMPILER gnu
 setenv ICE_MACHINE_MAKE make
 setenv ICE_MACHINE_WKDIR /Volumes/ncep/CICE_RUNS
-setenv ICE_MACHINE_INPUTDATA /Volumes/Data/CICE_data
+setenv ICE_MACHINE_INPUTDATA /Volumes/Data/
 setenv ICE_MACHINE_BASELINE /Volumes/ncep/CICE_BASELINE
 setenv ICE_MACHINE_SUBMIT " "
 setenv ICE_MACHINE_TPNODE 4

configuration/scripts/machines/env.phase3_intel

@@ -1,8 +1,12 @@
-#!/bin/csh -f --login
+#!/bin/csh -f
 
-source /etc/profile.d/lmod.csh
+#. $MODULESHOME/init/csh
+source $MODULESHOME/init/csh
+echo done with init.csh
+module use -a $MODULESHOME/modulefiles
 
 module purge
+module load lsf/10.1
 module load ips/18.0.1.163
 module load impi/18.0.1
 module load NetCDF/4.5.0 
@@ -15,10 +19,10 @@ setenv ICE_MACHINE_MAKE gmake
 setenv ICE_MACHINE_WKDIR /u/Robert.Grumbine/noscrub/CICE_RUNS
 setenv ICE_MACHINE_INPUTDATA /u/Robert.Grumbine/noscrub/
 setenv ICE_MACHINE_BASELINE /u/Robert.Grumbine/noscrub/CICE_BASELINE
-setenv ICE_MACHINE_SUBMIT " "
+setenv ICE_MACHINE_SUBMIT "bsub < "
 setenv ICE_MACHINE_TPNODE 4
 setenv ICE_MACHINE_QUEUE "default"
 setenv ICE_MACHINE_ACCT P0000000
 setenv ICE_MACHINE_BLDTHRDS 1
-setenv ICE_MACHINE_QSTAT "qstat "
+setenv ICE_MACHINE_QSTAT "bjobs "