From b08a97c6e047274c4bcba20b444f7209691ac5ad Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Fri, 15 May 2020 12:12:01 -0400 Subject: [PATCH] NOAA machine pass of base_suite #155 (#372) * update to include a 'high_Sierra' machine with gnu compilers, only marginally modified from the travisCI env and Macros. * Syncing theia and upstream/master branch * theia execution * intel compiler version tests -- 15.0.0 or greater * working on variations for round robin test * syncing to main * sync * testing * end test * successful on phase2 for all regression tests * start on phase3 execution * add phase3 system * syncing and phase3 modules * update configs manually from master * env edit * builds on phase3 now * test * trying to pull/merge cice master * theia * successful theia build * garbage * update theia intel env * mods for running in NOAA * start towards hera * tmp * theia passes with most recent of all modules * moving to hera * clean regression run on hera * cleanup * phase3 work * minor manual move towards master copy * re-fix phase3 script * nearing full phase3 * update path for data * actual final version of phase3 * from master * cicecore updating from master * from consortium/master * more from master * retry sync * manual add/subtrace * script perms and large diff files * retrieve hera * docs and cicecore sync to master * machine syncing * couple more * some more * finally the right template * finally correct template * typography in master * restore working batch.csh * towards matching consortium master * last? add for syncing * Revert icepack to 1.1.2 Co-authored-by: Tony Craig --- .gitignore | 1 + configuration/scripts/cice.batch.csh | 34 ++++++++++- configuration/scripts/cice.launch.csh | 14 +++++ .../scripts/machines/Macros.hera_intel | 60 +++++++++++++++++++ configuration/scripts/machines/env.hera_intel | 26 ++++++++ .../scripts/machines/env.high_Sierra_gnu | 2 +- .../scripts/machines/env.phase3_intel | 12 ++-- 7 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 configuration/scripts/machines/Macros.hera_intel create mode 100755 configuration/scripts/machines/env.hera_intel diff --git a/.gitignore b/.gitignore index f32b5b0ea..2a2a5fdb6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ doc/build # Ignore macOS cache files .DS_Store + # Ignore testsuite file/directories testsuite* caselist* diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 6797a9f61..73e50bf47 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -213,7 +213,20 @@ cat >> ${jobfile} << EOFB #SBATCH -N ${nnodes} #SBATCH -e slurm%j.err #SBATCH -o slurm%j.out -#SBATCH --mail-type END,FAIL +#SBATCH --mail-type FAIL +#SBATCH --mail-user=robert.grumbine@noaa.gov +EOFB + +else if (${ICE_MACHINE} =~ hera*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t `echo ${batchtime} | cut -f1-2 -d:` +#SBATCH -q batch +#SBATCH -A marine-cpu +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +#SBATCH --mail-type FAIL #SBATCH --mail-user=robert.grumbine@noaa.gov EOFB @@ -223,8 +236,25 @@ cat >> ${jobfile} << EOFB EOFB else if (${ICE_MACHINE} =~ phase3*) then +if ( ${nnodes} > 15) then + setenv p3tile 16 + setenv mem `expr 100 \* 1024 / $nnodes` +else + setenv p3tile ${nnodes} + setenv mem 8192 +endif +echo mem = ${mem} nnodes and p3tiles ${nnodes} ${p3tile} p3tile must be le nnodes cat >> ${jobfile} << EOFB -# nothing to do +#BSUB -J ${ICE_CASENAME} +#BSUB -q "dev_shared" +#BSUB -P RTO-T2O +#BSUB -W `echo ${batchtime} | cut -f1-2 -d:` +#BSUB -n ${nnodes} +#BSUB -R "affinity[core]" +#BSUB -R "span[ptile=${p3tile}]" +#BSUB -R "rusage[mem=${mem}]" +#BSUB -o /u/Robert.Grumbine/${ICE_CASENAME}.out.%J +#BSUB -e /u/Robert.Grumbine/${ICE_CASENAME}.err.%J EOFB else if (${ICE_MACHINE} =~ high_Sierra*) then diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 296878914..48d6c0062 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -197,6 +197,20 @@ cat >> ${jobfile} << EOFR ./cice >&! \$ICE_RUNLOG_FILE EOFR +#======= +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + +#======= +else if (${ICE_MACHINE} =~ phase2*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + +======= + #======= else if (${ICE_MACHINE} =~ travisCI*) then if (${ICE_COMMDIR} =~ serial*) then diff --git a/configuration/scripts/machines/Macros.hera_intel b/configuration/scripts/machines/Macros.hera_intel new file mode 100644 index 000000000..38cc62ed8 --- /dev/null +++ b/configuration/scripts/machines/Macros.hera_intel @@ -0,0 +1,60 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpiifort +MPIFC:= mpiifort +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := + + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.hera_intel b/configuration/scripts/machines/env.hera_intel new file mode 100755 index 000000000..ef3afdb7a --- /dev/null +++ b/configuration/scripts/machines/env.hera_intel @@ -0,0 +1,26 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +#module list +module purge +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load esmf/7.1.0r +module load hdf5/1.10.4 netcdf/4.6.1 +module load wgrib wgrib2 +#echo renewed modules: +#module list + +setenv ICE_MACHINE_ENVNAME hera +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/rgdev/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/rgdev/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/rgdev/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "batch" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.high_Sierra_gnu b/configuration/scripts/machines/env.high_Sierra_gnu index cabd66944..db8bc01a3 100755 --- a/configuration/scripts/machines/env.high_Sierra_gnu +++ b/configuration/scripts/machines/env.high_Sierra_gnu @@ -4,7 +4,7 @@ setenv ICE_MACHINE_ENVNAME high_Sierra setenv ICE_MACHINE_COMPILER gnu setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR /Volumes/ncep/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /Volumes/Data/CICE_data +setenv ICE_MACHINE_INPUTDATA /Volumes/Data/ setenv ICE_MACHINE_BASELINE /Volumes/ncep/CICE_BASELINE setenv ICE_MACHINE_SUBMIT " " setenv ICE_MACHINE_TPNODE 4 diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel index 03aa7083f..4c8f65eee 100755 --- a/configuration/scripts/machines/env.phase3_intel +++ b/configuration/scripts/machines/env.phase3_intel @@ -1,8 +1,12 @@ -#!/bin/csh -f --login +#!/bin/csh -f -source /etc/profile.d/lmod.csh +#. $MODULESHOME/init/csh +source $MODULESHOME/init/csh +echo done with init.csh +module use -a $MODULESHOME/modulefiles module purge +module load lsf/10.1 module load ips/18.0.1.163 module load impi/18.0.1 module load NetCDF/4.5.0 @@ -15,10 +19,10 @@ setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /u/Robert.Grumbine/noscrub/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /u/Robert.Grumbine/noscrub/ setenv ICE_MACHINE_BASELINE /u/Robert.Grumbine/noscrub/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_SUBMIT "bsub < " setenv ICE_MACHINE_TPNODE 4 setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_BLDTHRDS 1 -setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QSTAT "bjobs "