From 3f7753b553260cf737dc14e181ebb6514809ab5a Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sun, 22 Apr 2018 21:24:10 -0400 Subject: [PATCH 01/54] update to include a 'high_Sierra' machine with gnu compilers, only marginally modified from the travisCI env and Macros. --- configuration/scripts/cice.batch.csh | 6 +++ configuration/scripts/cice.launch.csh | 5 ++ .../scripts/machines/Macros.cheyenne_intel | 0 .../scripts/machines/Macros.fram_intel | 0 .../scripts/machines/Macros.high_Sierra_gnu | 47 +++++++++++++++++++ .../scripts/machines/Macros.testmachine_intel | 0 .../scripts/machines/env.high_Sierra_gnu | 14 ++++++ 7 files changed, 72 insertions(+) mode change 100755 => 100644 configuration/scripts/machines/Macros.cheyenne_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.fram_intel create mode 100644 configuration/scripts/machines/Macros.high_Sierra_gnu mode change 100755 => 100644 configuration/scripts/machines/Macros.testmachine_intel create mode 100755 configuration/scripts/machines/env.high_Sierra_gnu diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 009471807..99f025e1e 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -148,6 +148,12 @@ cat >> ${jobfile} << EOFB # nothing to do EOFB +else if (${ICE_MACHINE} =~ high_Sierra*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + + else echo "${0} ERROR: ${ICE_MACHINE} unknown" exit -1 diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index b81d22d26..7008ec21c 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -62,6 +62,11 @@ cat >> ${jobfile} << EOFR mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR +else if (${ICE_MACHINE} =~ high_Sierra*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + #cat >> ${jobfile} << EOFR #srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE #EOFR diff --git a/configuration/scripts/machines/Macros.cheyenne_intel b/configuration/scripts/machines/Macros.cheyenne_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.fram_intel b/configuration/scripts/machines/Macros.fram_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.high_Sierra_gnu b/configuration/scripts/machines/Macros.high_Sierra_gnu new file mode 100644 index 000000000..b1600d7e0 --- /dev/null +++ b/configuration/scripts/machines/Macros.high_Sierra_gnu @@ -0,0 +1,47 @@ +#============================================================================== +# Makefile macros for Travis-CI - GCC and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise -xHost + +FIXEDFLAGS := -132 +FFLAGS := -O2 -ffree-line-length-none -fconvert=big-endian -finit-real=nan +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + #FFLAGS += -O0 -g -Wextra -fbacktrace -fbounds-check -ffpe-trap=zero,overflow + FFLAGS += -O0 -g -std=f2008 -fbacktrace -fbounds-check -ffpe-trap=zero,overflow +else + FFLAGS += -O2 +endif + +FC := mpif90 + +MPICC:= + +MPIFC:= mpif90 +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:= +SFC:= + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.testmachine_intel b/configuration/scripts/machines/Macros.testmachine_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/env.high_Sierra_gnu b/configuration/scripts/machines/env.high_Sierra_gnu new file mode 100755 index 000000000..565d01344 --- /dev/null +++ b/configuration/scripts/machines/env.high_Sierra_gnu @@ -0,0 +1,14 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME high_Sierra +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA ~ +setenv ICE_MACHINE_BASELINE ~/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +setenv ICE_MACHINE_QUIETMODE false From 9fe8b2be5f938f2731e8e6e41474b52824915f40 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 5 Oct 2018 14:03:31 +0000 Subject: [PATCH 02/54] Syncing theia and upstream/master branch --- .../scripts/machines/Macros.cheyenne_intel | 0 .../scripts/machines/Macros.fram_intel | 0 .../scripts/machines/Macros.testmachine_intel | 0 .../scripts/machines/Macros.theia_gnu | 48 +++++++++++++++++ .../scripts/machines/Macros.theia_intel | 51 +++++++++++++++++++ configuration/scripts/machines/env.theia_gnu | 15 ++++++ .../scripts/machines/env.theia_intel | 36 +++++++++++++ 7 files changed, 150 insertions(+) mode change 100644 => 100755 configuration/scripts/machines/Macros.cheyenne_intel mode change 100644 => 100755 configuration/scripts/machines/Macros.fram_intel mode change 100644 => 100755 configuration/scripts/machines/Macros.testmachine_intel create mode 100644 configuration/scripts/machines/Macros.theia_gnu create mode 100644 configuration/scripts/machines/Macros.theia_intel create mode 100755 configuration/scripts/machines/env.theia_gnu create mode 100755 configuration/scripts/machines/env.theia_intel diff --git a/configuration/scripts/machines/Macros.cheyenne_intel b/configuration/scripts/machines/Macros.cheyenne_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/Macros.fram_intel b/configuration/scripts/machines/Macros.fram_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/Macros.testmachine_intel b/configuration/scripts/machines/Macros.testmachine_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/Macros.theia_gnu b/configuration/scripts/machines/Macros.theia_gnu new file mode 100644 index 000000000..63cce4b99 --- /dev/null +++ b/configuration/scripts/machines/Macros.theia_gnu @@ -0,0 +1,48 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FFLAGS := -fconvert=big-endian +#FFLAGS := -h bytwswapio +#FFLAGS := +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpif90 + +MPICC:= + +MPIFC:= mpif90 +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:= +SFC:= + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.theia_intel b/configuration/scripts/machines/Macros.theia_intel new file mode 100644 index 000000000..fa7b71140 --- /dev/null +++ b/configuration/scripts/machines/Macros.theia_intel @@ -0,0 +1,51 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpiifort + +MPICC:= + +MPIFC:= mpiifort +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:= +SFC:= + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.theia_gnu b/configuration/scripts/machines/env.theia_gnu new file mode 100755 index 000000000..974be35e0 --- /dev/null +++ b/configuration/scripts/machines/env.theia_gnu @@ -0,0 +1,15 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME theia +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/save/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel new file mode 100755 index 000000000..b26faae52 --- /dev/null +++ b/configuration/scripts/machines/env.theia_intel @@ -0,0 +1,36 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +module list +module purge +module load hpss +module load intel +module load impi +module load esmf +module load netcdf +module load hdf5 +module load wgrib +module load wgrib2 +echo renewed modules: +module list + +# 1) hpss/hpss 6) hdf5/1.8.14 11) gcc/4.9.1 +# 2) intel/14.0.2 7) imagemagick/7.0.5 12) cudatoolkit/8.0.44 +# 3) impi/5.1.2.150 8) wgrib/1.8.1.0b 13) mvapich2_gnu/2.2rc1.0.3_noslurm +# 4) esmf/7.0.2 9) wgrib2/0.1.9.5.1 +# 5) netcdf/4.3.0 10) grads/2.0.1a + + +setenv ICE_MACHINE_ENVNAME theia +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /scratch3/NCEPDEV/marine/save/Robert.Grumbine/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true From e9e08187e59c79d336c5f77fe7b1ec5c7f5b6295 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 5 Oct 2018 16:29:48 +0000 Subject: [PATCH 03/54] theia execution --- .gitignore | 3 ++ configuration/scripts/cice.batch.csh | 5 ++ configuration/scripts/cice.launch.csh | 5 ++ configuration/scripts/cice.settings | 2 +- .../scripts/machines/Macros.theia_pgi | 48 +++++++++++++++++++ .../scripts/machines/env.theia_intel | 7 +-- configuration/scripts/machines/env.theia_pgi | 27 +++++++++++ 7 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 configuration/scripts/machines/Macros.theia_pgi create mode 100755 configuration/scripts/machines/env.theia_pgi diff --git a/.gitignore b/.gitignore index 9574d2b2b..99521e901 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ doc/build # Ignore macOS cache files .DS_Store + +# Ignore testsuite directories +testsuite.* diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index b7fc82f7a..d684b4706 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -159,6 +159,11 @@ cat >> ${jobfile} << EOFB # nothing to do EOFB +else if (${ICE_MACHINE} =~ theia*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + else echo "${0} ERROR: ${ICE_MACHINE} unknown" diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 4b52c896e..e5f65b2cc 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -82,6 +82,11 @@ cat >> ${jobfile} << EOFR mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR +else if (${ICE_MACHINE} =~ theia*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + #cat >> ${jobfile} << EOFR #srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE #EOFR diff --git a/configuration/scripts/cice.settings b/configuration/scripts/cice.settings index f5cd9c7e8..65d8ca3ef 100755 --- a/configuration/scripts/cice.settings +++ b/configuration/scripts/cice.settings @@ -29,7 +29,7 @@ setenv ICE_TESTNAME undefined setenv ICE_BASELINE undefined setenv ICE_BASEGEN undefined setenv ICE_BASECOM undefined -setenv ICE_BFBCOMP undefined +setenv ICE_BFBCOMP BFB setenv ICE_SPVAL undefined setenv ICE_RUNLENGTH 0 setenv ICE_ACCOUNT undefined diff --git a/configuration/scripts/machines/Macros.theia_pgi b/configuration/scripts/machines/Macros.theia_pgi new file mode 100644 index 000000000..83bcc264d --- /dev/null +++ b/configuration/scripts/machines/Macros.theia_pgi @@ -0,0 +1,48 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +FC := mpif90 +MPICC:= +MPIFC:= mpif90 +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:= +SFC:= + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel index b26faae52..43f92a453 100755 --- a/configuration/scripts/machines/env.theia_intel +++ b/configuration/scripts/machines/env.theia_intel @@ -3,14 +3,11 @@ source /etc/profile.d/modules.csh module list module purge -module load hpss module load intel module load impi module load esmf -module load netcdf -module load hdf5 -module load wgrib -module load wgrib2 +module load hdf5 netcdf +module load wgrib wgrib2 echo renewed modules: module list diff --git a/configuration/scripts/machines/env.theia_pgi b/configuration/scripts/machines/env.theia_pgi new file mode 100755 index 000000000..2c4229c0e --- /dev/null +++ b/configuration/scripts/machines/env.theia_pgi @@ -0,0 +1,27 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +module list +module purge +module load pgi +module load impi +module load esmf +module load hdf5 netcdf/4.4.0 +module load wgrib wgrib2 +echo renewed modules: +module list + + +setenv ICE_MACHINE_ENVNAME theia +setenv ICE_MACHINE_COMPILER pgi +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /scratch3/NCEPDEV/marine/save/Robert.Grumbine/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true From a9cbb030a82da4c64254ffeefdd6445fcac6e406 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 5 Oct 2018 20:07:02 +0000 Subject: [PATCH 04/54] intel compiler version tests -- 15.0.0 or greater --- .../scripts/machines/env.theia_intel | 21 ++++++++----------- configuration/scripts/tests/base_suite.ts | 14 ++++++------- configuration/scripts/tests/rgvers_suite.ts | 9 ++++++++ icepack | 2 +- 4 files changed, 26 insertions(+), 20 deletions(-) create mode 100644 configuration/scripts/tests/rgvers_suite.ts diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel index 43f92a453..c775d7e73 100755 --- a/configuration/scripts/machines/env.theia_intel +++ b/configuration/scripts/machines/env.theia_intel @@ -1,23 +1,20 @@ #!/bin/csh -f source /etc/profile.d/modules.csh -module list +#module list module purge -module load intel +#module load intel/18.1.163 Works, at least w. nc 4.4.0, +#14.0.2 w nc4.3.0 does not module load intel/18.1.163 +# ok w nc4.3.0: module load intel/16.0.1.150 +# ok w nc4.3.0: module load intel/15.3.187 +module load intel/15.0.0 module load impi module load esmf -module load hdf5 netcdf +module load hdf5 netcdf/4.3.0 module load wgrib wgrib2 -echo renewed modules: -module list +#echo renewed modules: +#module list -# 1) hpss/hpss 6) hdf5/1.8.14 11) gcc/4.9.1 -# 2) intel/14.0.2 7) imagemagick/7.0.5 12) cudatoolkit/8.0.44 -# 3) impi/5.1.2.150 8) wgrib/1.8.1.0b 13) mvapich2_gnu/2.2rc1.0.3_noslurm -# 4) esmf/7.0.2 9) wgrib2/0.1.9.5.1 -# 5) netcdf/4.3.0 10) grads/2.0.1a - - setenv ICE_MACHINE_ENVNAME theia setenv ICE_MACHINE_COMPILER intel setenv ICE_MACHINE_MAKE make diff --git a/configuration/scripts/tests/base_suite.ts b/configuration/scripts/tests/base_suite.ts index 644a7d033..012adfe4b 100644 --- a/configuration/scripts/tests/base_suite.ts +++ b/configuration/scripts/tests/base_suite.ts @@ -1,22 +1,22 @@ # Test Grid PEs Sets BFB-compare +smoke gx3 8x2 diag1,run5day +smoke gx3 4x2 diag1,run5day smoke_gx3_8x2_diag1_run5day +smoke gx3 4x1 diag1,run5day,thread smoke_gx3_8x2_diag1_run5day smoke gx3 1x1 debug,diag1,run2day smoke gx3 1x4 debug,diag1,run2day smoke gx3 4x1 debug,diag1,run5day -restart gx3 8x2 debug -smoke gx3 8x2 diag1,run5day smoke gx3 8x2 diag24,run1year,medium -smoke gx3 4x2 diag1,run5day smoke_gx3_8x2_diag1_run5day -smoke gx3 4x1 diag1,run5day,thread smoke_gx3_8x2_diag1_run5day decomp gx3 4x2x25x29x5 -restart gx1 40x4 droundrobin,short restart gx3 4x4 none restart gx3 4x4 iobinary +restart gx3 8x2 debug +restart gx1 40x4 droundrobin,short restart gx3 6x2 alt01 restart gx3 8x2 alt02 restart gx3 4x2 alt03 restart gx3 4x4 alt04 restart gx3 4x4 alt05 -restart gbox128 4x2 none restart gbox128 4x2 boxdyn -restart gbox128 2x2 boxadv +restart gbox128 4x2 none restart gbox128 4x4 boxrestore +restart gbox128 2x2 boxadv diff --git a/configuration/scripts/tests/rgvers_suite.ts b/configuration/scripts/tests/rgvers_suite.ts new file mode 100644 index 000000000..f9b28a63e --- /dev/null +++ b/configuration/scripts/tests/rgvers_suite.ts @@ -0,0 +1,9 @@ +# Test Grid PEs Sets BFB-compare +restart gx3 4x4 none +restart gx3 4x4 iobinary +restart gx3 8x2 debug +restart gx3 6x2 alt01 +restart gx3 8x2 alt02 +restart gx3 4x2 alt03 +restart gx3 4x4 alt04 +restart gx3 4x4 alt05 diff --git a/icepack b/icepack index e922ad03f..0a8139fd3 160000 --- a/icepack +++ b/icepack @@ -1 +1 @@ -Subproject commit e922ad03ff32018ae47bfd7e4d73d1554c29ecfd +Subproject commit 0a8139fd38052aa323492e0b64f986d1cdc94515 From 1519154a886f7184505c4cb86b6a1e758ee4b73f Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Tue, 18 Dec 2018 15:07:34 +0000 Subject: [PATCH 05/54] working on variations for round robin test --- configuration/scripts/tests/rgvers_suite.ts | 25 ++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/configuration/scripts/tests/rgvers_suite.ts b/configuration/scripts/tests/rgvers_suite.ts index f9b28a63e..dc85613db 100644 --- a/configuration/scripts/tests/rgvers_suite.ts +++ b/configuration/scripts/tests/rgvers_suite.ts @@ -1,9 +1,18 @@ # Test Grid PEs Sets BFB-compare -restart gx3 4x4 none -restart gx3 4x4 iobinary -restart gx3 8x2 debug -restart gx3 6x2 alt01 -restart gx3 8x2 alt02 -restart gx3 4x2 alt03 -restart gx3 4x4 alt04 -restart gx3 4x4 alt05 +#restart gx1 4x4 droundrobin,short +#restart gx1 8x4 droundrobin,short +#restart gx1 12x4 droundrobin,short +#restart gx1 16x4 droundrobin,short +#restart gx1 20x4 droundrobin,short +#restart gx1 24x4 droundrobin,short +#restart gx1 28x4 droundrobin,short +#restart gx1 32x4 droundrobin,short +restart gx1 32x2 droundrobin,short +restart gx1 32x3 droundrobin,short +restart gx1 32x1 droundrobin,short + +restart gx1 33x4 droundrobin,short +restart gx1 34x4 droundrobin,short +restart gx1 35x4 droundrobin,short +restart gx1 36x4 droundrobin,short +#restart gx1 40x4 droundrobin,short From 363bcac721cd85beb33ab051af386d1a28836dfc Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 1 Mar 2019 14:15:42 +0000 Subject: [PATCH 06/54] syncing to main --- configuration/scripts/cice.launch.csh | 120 ++++++++++++++++++++++ configuration/scripts/tests/base_suite.ts | 63 ++++++++++++ 2 files changed, 183 insertions(+) create mode 100755 configuration/scripts/cice.launch.csh create mode 100644 configuration/scripts/tests/base_suite.ts diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh new file mode 100755 index 000000000..ae5e3ecc4 --- /dev/null +++ b/configuration/scripts/cice.launch.csh @@ -0,0 +1,120 @@ +#! /bin/csh -f + +#echo ${0} +echo "running cice.launch.csh" + +#source ./cice.settings +#source ${ICE_CASEDIR}/env.${ICE_MACHCOMP} || exit 2 + +set jobfile = $1 + +set ntasks = ${ICE_NTASKS} +set nthrds = ${ICE_NTHRDS} +set maxtpn = ${ICE_MACHINE_TPNODE} + +@ ncores = ${ntasks} * ${nthrds} +@ taskpernode = ${maxtpn} / $nthrds +@ nnodes = ${ntasks} / ${taskpernode} +if (${nnodes} * ${taskpernode} < ${ntasks}) @ nnodes = $nnodes + 1 +set taskpernodelimit = ${taskpernode} +if (${taskpernodelimit} > ${ntasks}) set taskpernodelimit = ${ntasks} +@ corespernode = ${taskpernodelimit} * ${nthrds} + +#========================================== + +if (${ICE_MACHINE} =~ cheyenne*) then +cat >> ${jobfile} << EOFR +mpiexec_mpt -n ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ hobart*) then +cat >> ${jobfile} << EOFR +mpiexec -n ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ thunder*) then +cat >> ${jobfile} << EOFR +mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif +#======= +else if (${ICE_MACHINE} =~ onyx*) then +cat >> ${jobfile} << EOFR +aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif +#======= +else if (${ICE_MACHINE} =~ cori*) then +cat >> ${jobfile} << EOFR +srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ badger*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ fram*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ cesium*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ testmachine*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ travisCI*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +<<<<<<< HEAD + +else if (${ICE_MACHINE} =~ high_Sierra*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + +else if (${ICE_MACHINE} =~ theia*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + +======= +>>>>>>> 38cad4468c706be015d70720c2b4b5fecdd38720 +#cat >> ${jobfile} << EOFR +#srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +#EOFR +#======= +else + echo "${0} ERROR ${ICE_MACHINE} unknown" + exit -1 +endif +#======= + +exit 0 diff --git a/configuration/scripts/tests/base_suite.ts b/configuration/scripts/tests/base_suite.ts new file mode 100644 index 000000000..e7f8864d6 --- /dev/null +++ b/configuration/scripts/tests/base_suite.ts @@ -0,0 +1,63 @@ +# Test Grid PEs Sets BFB-compare +smoke gx3 8x2 diag1,run5day +<<<<<<< HEAD +smoke gx3 4x2 diag1,run5day smoke_gx3_8x2_diag1_run5day +smoke gx3 4x1 diag1,run5day,thread smoke_gx3_8x2_diag1_run5day +smoke gx3 1x1 debug,diag1,run2day +smoke gx3 1x4 debug,diag1,run2day +smoke gx3 4x1 debug,diag1,run5day +smoke gx3 8x2 diag24,run1year,medium +decomp gx3 4x2x25x29x5 +======= +smoke gx3 1x1 debug,diag1,run2day +smoke gx3 1x4 debug,diag1,run2day +smoke gx3 4x1 debug,diag1,run5day +restart gx3 8x2 debug +smoke gx3 8x2 diag24,run1year,medium +decomp gx3 4x2x25x29x5 +smoke gx3 4x2 diag1,run5day smoke_gx3_8x2_diag1_run5day +smoke gx3 4x1 diag1,run5day,thread smoke_gx3_8x2_diag1_run5day +<<<<<<< HEAD +restart gx1 40x4 droundrobin,short +restart tx1 40x4 dsectrobin,short +>>>>>>> upstream/master +======= +restart gx1 40x4 droundrobin,medium +restart tx1 40x4 dsectrobin,medium +>>>>>>> 38cad4468c706be015d70720c2b4b5fecdd38720 +restart gx3 4x4 none +restart gx3 4x4 iobinary +restart gx3 8x2 debug +restart gx1 40x4 droundrobin,short +restart gx3 6x2 alt01 +restart gx3 8x2 alt02 +restart gx3 4x2 alt03 +restart gx3 4x4 alt04 +restart gx3 4x4 alt05 +<<<<<<< HEAD +restart gbox128 4x2 boxdyn +restart gbox128 4x2 none +restart gbox128 4x4 boxrestore +restart gbox128 2x2 boxadv +======= +restart gx3 6x2 alt01,debug,short +restart gx3 8x2 alt02,debug,short +restart gx3 4x2 alt03,debug,short +smoke gx3 4x4 alt04,debug,short +smoke gx3 4x4 alt05,debug,short +restart gbox128 4x2 none +restart gbox128 4x2 boxdyn +restart gbox128 4x2 boxdyn,debug +restart gbox128 2x2 boxadv,short +smoke gbox128 2x2 boxadv,short,debug +restart gbox128 4x4 boxrestore +smoke gbox128 4x4 boxrestore,debug +restart gbox80 1x1 box2001 +smoke gbox80 1x1 boxslotcyl +smoke gx3 8x2 bgcz +smoke gx3 8x2 bgcz,debug +smoke gx3 8x1 bgcskl,debug +#smoke gx3 4x1 bgcz,thread smoke_gx3_8x2_bgcz +restart gx1 4x2 bgcsklclim,medium +restart gx1 8x1 bgczclim,medium +>>>>>>> 38cad4468c706be015d70720c2b4b5fecdd38720 From 883b0bd52a039a96c7251be3b2900e6045d7e1c8 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 1 Mar 2019 14:54:20 +0000 Subject: [PATCH 07/54] sync --- configuration/scripts/tests/base_suite.ts | 24 ----------------------- 1 file changed, 24 deletions(-) diff --git a/configuration/scripts/tests/base_suite.ts b/configuration/scripts/tests/base_suite.ts index e7f8864d6..19d073a97 100644 --- a/configuration/scripts/tests/base_suite.ts +++ b/configuration/scripts/tests/base_suite.ts @@ -1,14 +1,5 @@ # Test Grid PEs Sets BFB-compare smoke gx3 8x2 diag1,run5day -<<<<<<< HEAD -smoke gx3 4x2 diag1,run5day smoke_gx3_8x2_diag1_run5day -smoke gx3 4x1 diag1,run5day,thread smoke_gx3_8x2_diag1_run5day -smoke gx3 1x1 debug,diag1,run2day -smoke gx3 1x4 debug,diag1,run2day -smoke gx3 4x1 debug,diag1,run5day -smoke gx3 8x2 diag24,run1year,medium -decomp gx3 4x2x25x29x5 -======= smoke gx3 1x1 debug,diag1,run2day smoke gx3 1x4 debug,diag1,run2day smoke gx3 4x1 debug,diag1,run5day @@ -17,29 +8,15 @@ smoke gx3 8x2 diag24,run1year,medium decomp gx3 4x2x25x29x5 smoke gx3 4x2 diag1,run5day smoke_gx3_8x2_diag1_run5day smoke gx3 4x1 diag1,run5day,thread smoke_gx3_8x2_diag1_run5day -<<<<<<< HEAD -restart gx1 40x4 droundrobin,short -restart tx1 40x4 dsectrobin,short ->>>>>>> upstream/master -======= restart gx1 40x4 droundrobin,medium restart tx1 40x4 dsectrobin,medium ->>>>>>> 38cad4468c706be015d70720c2b4b5fecdd38720 restart gx3 4x4 none restart gx3 4x4 iobinary -restart gx3 8x2 debug -restart gx1 40x4 droundrobin,short restart gx3 6x2 alt01 restart gx3 8x2 alt02 restart gx3 4x2 alt03 restart gx3 4x4 alt04 restart gx3 4x4 alt05 -<<<<<<< HEAD -restart gbox128 4x2 boxdyn -restart gbox128 4x2 none -restart gbox128 4x4 boxrestore -restart gbox128 2x2 boxadv -======= restart gx3 6x2 alt01,debug,short restart gx3 8x2 alt02,debug,short restart gx3 4x2 alt03,debug,short @@ -60,4 +37,3 @@ smoke gx3 8x1 bgcskl,debug #smoke gx3 4x1 bgcz,thread smoke_gx3_8x2_bgcz restart gx1 4x2 bgcsklclim,medium restart gx1 8x1 bgczclim,medium ->>>>>>> 38cad4468c706be015d70720c2b4b5fecdd38720 From 56940e9060c2b74c9fa4febbfa461cc06332bd5b Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Tue, 30 Apr 2019 14:56:52 +0000 Subject: [PATCH 08/54] testing --- fred | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 fred diff --git a/fred b/fred new file mode 100644 index 000000000..e69de29bb From 3626ad510912b1e97a2e44348033808f58c8e1ee Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Tue, 30 Apr 2019 14:57:14 +0000 Subject: [PATCH 09/54] end test --- fred | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 fred diff --git a/fred b/fred deleted file mode 100644 index e69de29bb..000000000 From c97f570e34bc834fd230e2ee02a5f054c5cb95d7 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Tue, 30 Apr 2019 15:10:40 +0000 Subject: [PATCH 10/54] successful on phase2 for all regression tests --- configuration/scripts/cice.batch.csh | 13 +--- configuration/scripts/cice.launch.csh | 78 ++++++++++++++++--- .../scripts/machines/Macros.phase2_intel | 57 ++++++++++++++ .../scripts/machines/env.phase2_intel | 14 ++++ 4 files changed, 143 insertions(+), 19 deletions(-) create mode 100755 configuration/scripts/machines/Macros.phase2_intel create mode 100755 configuration/scripts/machines/env.phase2_intel diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index fb2cb12bd..2ea5825e1 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -18,6 +18,7 @@ set acct = ${ICE_ACCOUNT} @ ncores = ${ntasks} * ${nthrds} @ taskpernode = ${maxtpn} / $nthrds +if (${taskpernode} == 0) set taskpernode = 1 @ nnodes = ${ntasks} / ${taskpernode} if (${nnodes} * ${taskpernode} < ${ntasks}) @ nnodes = $nnodes + 1 set taskpernodelimit = ${taskpernode} @@ -149,27 +150,21 @@ cat >> ${jobfile} << EOFB #SBATCH --qos=standby EOFB -else if (${ICE_MACHINE} =~ testmachine*) then +else if (${ICE_MACHINE} =~ phase2*) then cat >> ${jobfile} << EOFB # nothing to do EOFB -else if (${ICE_MACHINE} =~ travisCI*) then -cat >> ${jobfile} << EOFB -# nothing to do -EOFB - -else if (${ICE_MACHINE} =~ high_Sierra*) then +else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFB # nothing to do EOFB -else if (${ICE_MACHINE} =~ theia*) then +else if (${ICE_MACHINE} =~ travisCI*) then cat >> ${jobfile} << EOFB # nothing to do EOFB - else echo "${0} ERROR: ${ICE_MACHINE} unknown" exit -1 diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index ae5e3ecc4..e2d3ebf59 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -14,6 +14,7 @@ set maxtpn = ${ICE_MACHINE_TPNODE} @ ncores = ${ntasks} * ${nthrds} @ taskpernode = ${maxtpn} / $nthrds +if (${taskpernode} == 0) set taskpernode = 1 @ nnodes = ${ntasks} / ${taskpernode} if (${nnodes} * ${taskpernode} < ${ntasks}) @ nnodes = $nnodes + 1 set taskpernodelimit = ${taskpernode} @@ -21,21 +22,41 @@ if (${taskpernodelimit} > ${ntasks}) set taskpernodelimit = ${ntasks} @ corespernode = ${taskpernodelimit} * ${nthrds} #========================================== - if (${ICE_MACHINE} =~ cheyenne*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else cat >> ${jobfile} << EOFR mpiexec_mpt -n ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR +endif + #======= else if (${ICE_MACHINE} =~ hobart*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else cat >> ${jobfile} << EOFR mpiexec -n ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR +endif + #======= else if (${ICE_MACHINE} =~ thunder*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else cat >> ${jobfile} << EOFR mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$ICE_RUNLOG_FILE EOFR +endif + #======= else if (${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr*) then if (${ICE_COMMDIR} =~ serial*) then @@ -47,11 +68,19 @@ cat >> ${jobfile} << EOFR mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$ICE_RUNLOG_FILE EOFR endif + #======= else if (${ICE_MACHINE} =~ onyx*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else cat >> ${jobfile} << EOFR aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE EOFR +endif + #======= else if (${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad*) then if (${ICE_COMMDIR} =~ serial*) then @@ -63,53 +92,82 @@ cat >> ${jobfile} << EOFR aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE EOFR endif + #======= else if (${ICE_MACHINE} =~ cori*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else cat >> ${jobfile} << EOFR srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE EOFR +endif + #======= else if (${ICE_MACHINE} =~ badger*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else cat >> ${jobfile} << EOFR mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR +endif + #======= else if (${ICE_MACHINE} =~ fram*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else cat >> ${jobfile} << EOFR mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR +endif + #======= else if (${ICE_MACHINE} =~ cesium*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else cat >> ${jobfile} << EOFR mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR +endif + #======= else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFR ./cice >&! \$ICE_RUNLOG_FILE EOFR + #======= -else if (${ICE_MACHINE} =~ travisCI*) then +else if (${ICE_MACHINE} =~ phase2*) then cat >> ${jobfile} << EOFR mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR -<<<<<<< HEAD -else if (${ICE_MACHINE} =~ high_Sierra*) then +#======= +else if (${ICE_MACHINE} =~ travisCI*) then +if (${ICE_COMMDIR} =~ serial*) then cat >> ${jobfile} << EOFR -mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +./cice >&! \$ICE_RUNLOG_FILE EOFR - -else if (${ICE_MACHINE} =~ theia*) then +else cat >> ${jobfile} << EOFR mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR - -======= ->>>>>>> 38cad4468c706be015d70720c2b4b5fecdd38720 +endif #cat >> ${jobfile} << EOFR #srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE #EOFR + #======= else echo "${0} ERROR ${ICE_MACHINE} unknown" diff --git a/configuration/scripts/machines/Macros.phase2_intel b/configuration/scripts/machines/Macros.phase2_intel new file mode 100755 index 000000000..6bfea6b19 --- /dev/null +++ b/configuration/scripts/machines/Macros.phase2_intel @@ -0,0 +1,57 @@ +#============================================================================== +# Makefile macros for generic test machine, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +#CFLAGS := -c -O2 -fp-model precise -xHost +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) $(NETCDF_INCLUDE) + +LIB_NETCDF := $(NETCDF_PATH)/lib +LIB_MPI := $(IMPILIBDIR) + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(compile_threaded), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +### if using parallel I/O, load all 3 libraries. PIO must be first! +#ifeq ($(IO_TYPE), pio) +# PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib +# INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include +# SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof +#endif diff --git a/configuration/scripts/machines/env.phase2_intel b/configuration/scripts/machines/env.phase2_intel new file mode 100755 index 000000000..2457d4721 --- /dev/null +++ b/configuration/scripts/machines/env.phase2_intel @@ -0,0 +1,14 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME phase2 +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR ~/noscrub/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA ~/noscrub/ +setenv ICE_MACHINE_BASELINE ~/noscrub/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "qstat " From c0c9916d4fb8619f2370b204c55d5c8f6b8d2549 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Tue, 30 Apr 2019 15:16:04 +0000 Subject: [PATCH 11/54] start on phase3 execution --- .../scripts/machines/Macros.phase3_intel | 57 +++++++++++++++++++ .../scripts/machines/env.phase3_intel | 14 +++++ 2 files changed, 71 insertions(+) create mode 100755 configuration/scripts/machines/Macros.phase3_intel create mode 100755 configuration/scripts/machines/env.phase3_intel diff --git a/configuration/scripts/machines/Macros.phase3_intel b/configuration/scripts/machines/Macros.phase3_intel new file mode 100755 index 000000000..6bfea6b19 --- /dev/null +++ b/configuration/scripts/machines/Macros.phase3_intel @@ -0,0 +1,57 @@ +#============================================================================== +# Makefile macros for generic test machine, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +#CFLAGS := -c -O2 -fp-model precise -xHost +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) $(NETCDF_INCLUDE) + +LIB_NETCDF := $(NETCDF_PATH)/lib +LIB_MPI := $(IMPILIBDIR) + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(compile_threaded), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +### if using parallel I/O, load all 3 libraries. PIO must be first! +#ifeq ($(IO_TYPE), pio) +# PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib +# INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include +# SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof +#endif diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel new file mode 100755 index 000000000..6d61f4cb5 --- /dev/null +++ b/configuration/scripts/machines/env.phase3_intel @@ -0,0 +1,14 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME phase3 +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR ~/noscrub/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA ~/noscrub/ +setenv ICE_MACHINE_BASELINE ~/noscrub/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "qstat " From 62b2d2fa5ca43d1de4c18b3751617bbea4e5535d Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Tue, 30 Apr 2019 15:17:24 +0000 Subject: [PATCH 12/54] add phase3 system --- configuration/scripts/cice.batch.csh | 5 +++++ configuration/scripts/cice.launch.csh | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 2ea5825e1..8ff13dec4 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -155,6 +155,11 @@ cat >> ${jobfile} << EOFB # nothing to do EOFB +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFB # nothing to do diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index e2d3ebf59..c05c44976 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -147,6 +147,12 @@ cat >> ${jobfile} << EOFR ./cice >&! \$ICE_RUNLOG_FILE EOFR +#======= +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR + #======= else if (${ICE_MACHINE} =~ phase2*) then cat >> ${jobfile} << EOFR From c7991a7a88383f7dca861d13e45676a9313eaf19 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Tue, 30 Apr 2019 16:50:59 +0000 Subject: [PATCH 13/54] syncing and phase3 modules --- .../infrastructure/comm/mpi/ice_reprosum.F90 | 1481 ++++++++++++++++ .../comm/serial/ice_reprosum.F90 | 1482 +++++++++++++++++ .../infrastructure/ice_shr_reprosum86.c | 83 + .../scripts/machines/env.phase3_intel | 15 +- .../scripts/options/set_nml.reprosum | 1 + configuration/scripts/tests/comparelog.csh | 70 + configuration/scripts/tests/first_suite.ts | 5 + configuration/scripts/tests/reprosum_suite.ts | 11 + .../scripts/tests/test_logbfb.script | 36 + 9 files changed, 3183 insertions(+), 1 deletion(-) create mode 100644 cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 create mode 100644 cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 create mode 100644 cicecore/cicedynB/infrastructure/ice_shr_reprosum86.c create mode 100644 configuration/scripts/options/set_nml.reprosum create mode 100755 configuration/scripts/tests/comparelog.csh create mode 100644 configuration/scripts/tests/first_suite.ts create mode 100644 configuration/scripts/tests/reprosum_suite.ts create mode 100644 configuration/scripts/tests/test_logbfb.script diff --git a/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 b/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 new file mode 100644 index 000000000..ab1c5969a --- /dev/null +++ b/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 @@ -0,0 +1,1481 @@ + +!> Reproducible sum method from P. Worley + +MODULE ice_reprosum + +!----------------------------------------------------------------------- +! +! Purpose: +!> Compute reproducible global sums of a set of arrays across an MPI +!> subcommunicator +! +! Methods: +!> Compute using either or both a scalable, reproducible algorithm and a +!> scalable, nonreproducible algorithm: +!> * Reproducible (scalable): +!> Convert to fixed point (integer vector representation) to enable +!> reproducibility when using MPI_Allreduce +!> * Alternative usually reproducible (scalable): +!> Use parallel double-double algorithm due to Helen He and +!> Chris Ding, based on David Bailey's/Don Knuth's DDPDD algorithm +!> * Nonreproducible (scalable): +!> Floating point and MPI_Allreduce based. +!> If computing both reproducible and nonreproducible sums, compare +!> these and report relative difference (if absolute difference +!> less than sum) or absolute difference back to calling routine. +! +! Author: P. Worley (based on suggestions from J. White for fixed +! point algorithm and on He/Ding paper for ddpdd +! algorithm) +! +! Modified by T.Craig for CICE, March 2019 based on the public version in +! Oasis3-MCT_4.0. +! +!----------------------------------------------------------------------- + +!----------------------------------------------------------------------- +!- use statements ------------------------------------------------------ +!----------------------------------------------------------------------- +#if ( defined noI8 ) + ! Workaround for when shr_kind_i8 is not supported. + use ice_kinds_mod, only: r8 => dbl_kind, i8 => int_kind +#else + use ice_kinds_mod, only: r8 => dbl_kind, i8 => int8_kind +#endif + use ice_kinds_mod, only: char_len_long + use ice_fileunits, only: nu_diag + use ice_exit, only: abort_ice + +! internal timers not yet implemented, need to revisit if needed +! use ice_mpi, only: xicex_mpi_barrier +! use ice_timer, only: xicex_timer_start, xicex_timer_stop + +!----------------------------------------------------------------------- +!- module boilerplate -------------------------------------------------- +!----------------------------------------------------------------------- + implicit none + private + +!----------------------------------------------------------------------- +!- include statements -------------------------------------------------- +!----------------------------------------------------------------------- +#ifndef SERIAL_REMOVE_MPI +#include +#endif + +!----------------------------------------------------------------------- +! Public interfaces ---------------------------------------------------- +!----------------------------------------------------------------------- + public :: & + ice_reprosum_setopts, &! set runtime options + ice_reprosum_calc, &! calculate distributed sum + ice_reprosum_tolExceeded ! utility function to check relative + ! differences against the tolerance + +!----------------------------------------------------------------------- +! Public data ---------------------------------------------------------- +!----------------------------------------------------------------------- + logical, public :: ice_reprosum_recompute = .false. + + real(r8), public :: ice_reprosum_reldiffmax = -1.0_r8 + +!----------------------------------------------------------------------- +! Private interfaces --------------------------------------------------- +!----------------------------------------------------------------------- + private :: & + ddpdd, &! double-double sum routine + split_indices ! split indices among OMP threads + +!----------------------------------------------------------------------- +! Private data --------------------------------------------------------- +!----------------------------------------------------------------------- + + logical :: repro_sum_use_ddpdd = .false. + logical :: detailed_timing = .false. + character(len=char_len_long) :: tmpstr + + CONTAINS + +!======================================================================== +!----------------------------------------------------------------------- +! Purpose: +!> Set runtime options +! Author: P. Worley +!----------------------------------------------------------------------- + + subroutine ice_reprosum_setopts(repro_sum_use_ddpdd_in, & + repro_sum_rel_diff_max_in, & + repro_sum_recompute_in, & + repro_sum_master, & + repro_sum_logunit ) + +!------------------------------Arguments-------------------------------- + logical, intent(in), optional :: repro_sum_use_ddpdd_in + !< Use DDPDD algorithm instead of fixed precision algorithm + real(r8), intent(in), optional :: repro_sum_rel_diff_max_in + !< maximum permissible difference between reproducible and + !< nonreproducible sums + logical, intent(in), optional :: repro_sum_recompute_in + !< recompute using different algorithm when difference between + !< reproducible and nonreproducible sums is too great + logical, intent(in), optional :: repro_sum_master + !< flag indicating whether this process should output + !< log messages + integer, intent(in), optional :: repro_sum_logunit + !< unit number for log messages +!---------------------------Local Workspace----------------------------- + integer llogunit ! unit number for log messages + logical master ! local master? + logical,save :: firstcall = .true. ! first call + character(len=*),parameter :: subname = '(ice_reprosum_setopts)' +!----------------------------------------------------------------------- + + if ( present(repro_sum_master) ) then + master = repro_sum_master + else + master = .false. + endif + + if ( present(repro_sum_logunit) ) then + llogunit = repro_sum_logunit + else + llogunit = nu_diag + endif + + if (.not. firstcall) then + write(tmpstr,*) subname//' ERROR: can only be called once' + call abort_ice(tmpstr,file=__FILE__,line=__LINE__) + endif + firstcall = .false. + + if ( present(repro_sum_use_ddpdd_in) ) then + repro_sum_use_ddpdd = repro_sum_use_ddpdd_in + endif + if ( present(repro_sum_rel_diff_max_in) ) then + ice_reprosum_reldiffmax = repro_sum_rel_diff_max_in + endif + if ( present(repro_sum_recompute_in) ) then + ice_reprosum_recompute = repro_sum_recompute_in + endif + if (master) then + if ( repro_sum_use_ddpdd ) then + write(llogunit,*) subname, & + 'Using double-double-based (scalable) usually reproducible ', & + 'distributed sum algorithm' + else + write(llogunit,*) subname, & + 'Using fixed-point-based (scalable) reproducible ', & + 'distributed sum algorithm' + endif + + if (ice_reprosum_reldiffmax >= 0._r8) then + write(llogunit,*) subname, & + ' with a maximum relative error tolerance of ', & + ice_reprosum_reldiffmax + if (ice_reprosum_recompute) then + write(llogunit,*) subname, & + 'If tolerance exceeded, sum is recomputed using ', & + 'a serial algorithm.' + else + write(llogunit,*) subname, & + 'If tolerance exceeded, fixed-precision is sum used ', & + 'but a warning is output.' + endif + else + write(llogunit,*) subname, & + 'and not comparing with floating point algorithms.' + endif + + endif + end subroutine ice_reprosum_setopts + +!======================================================================== + +! +! Purpose: +!> Compute the global sum of each field in "arr" using the indicated +!> communicator with a reproducible yet scalable implementation based +!> on a fixed point algorithm. An alternative is to use an "almost +!> always reproducible" floating point algorithm. +! +! The accuracy of the fixed point algorithm is controlled by the +! number of "levels" of integer expansion. The algorithm will calculate +! the number of levels that is required for the sum to be essentially +! exact. The optional parameter arr_max_levels can be used to override +! the calculated value. The optional parameter arr_max_levels_out can be +! used to return the values used. +! +! The algorithm also requires an upper bound on +! the maximum summand (in absolute value) for each field, and will +! calculate this internally. However, if the optional parameters +! arr_max_levels and arr_gbl_max are both set, then the algorithm will +! use the values in arr_gbl_max for the upper bounds instead. If these +! are not upper bounds, or if the upper bounds are not tight enough +! to achieve the requisite accuracy, and if the optional parameter +! repro_sum_validate is NOT set to .false., the algorithm will repeat the +! computation with appropriate upper bounds. If only arr_gbl_max is present, +! then the maxima are computed internally (and the specified values are +! ignored). The optional parameter arr_gbl_max_out can be +! used to return the values used. +! +! Finally, the algorithm requires an upper bound on the number of +! local summands across all processes. This will be calculated internally, +! using an MPI collective, but the value in the optional argument +! gbl_max_nsummands will be used instead if (1) it is present, (2) +! it is > 0, and (3) the maximum value and required number of levels +! are also specified. (If the maximum value is calculated, the same +! MPI collective is used to determine the maximum number of local +! summands.) The accuracy of the user-specified value is not checked. +! However, if set to < 1, the value will instead be calculated. If the +! optional parameter gbl_max_nsummands_out is present, then the value +! used (gbl_max_nsummands if >= 1; calculated otherwise) will be +! returned. +! +! If requested (by setting ice_reprosum_reldiffmax >= 0.0 and passing in +! the optional rel_diff parameter), results are compared with a +! nonreproducible floating point algorithm. +! +! Note that the cost of the algorithm is not strongly correlated with +! the number of levels, which primarily shows up as a (modest) increase +! in cost of the MPI_Allreduce as a function of vector length. Rather the +! cost is more a function of (a) the number of integers required to +! represent an individual summand and (b) the number of MPI_Allreduce +! calls. The number of integers required to represent an individual +! summand is 1 or 2 when using 8-byte integers for 8-byte real summands +! when the number of local summands is not too large. As the number of +! local summands increases, the number of integers required increases. +! The number of MPI_Allreduce calls is either 2 (specifying nothing) or +! 1 (specifying gbl_max_nsummands, arr_max_levels, and arr_gbl_max +! correctly). When specifying arr_max_levels and arr_gbl_max +! incorrectly, 3 or 4 MPI_Allreduce calls will be required. +! +! The alternative algorithm is a minor modification of a parallel +! implementation of David Bailey's routine DDPDD by Helen He +! and Chris Ding. Bailey uses the Knuth trick to implement quadruple +! precision summation of double precision values with 10 double +! precision operations. The advantage of this algorithm is that +! it requires a single MPI_Allreduce and is less expensive per summand +! than is the fixed precision algorithm. The disadvantage is that it +! is not guaranteed to be reproducible (though it is reproducible +! much more often than is the standard algorithm). This alternative +! is used when the optional parameter ddpdd_sum is set to .true. It is +! also used if the fixed precision algorithm radix assumption does not +! hold. + +!---------------------------------------------------------------------- + subroutine ice_reprosum_calc (arr, arr_gsum, nsummands, dsummands, & + nflds, ddpdd_sum, & + arr_gbl_max, arr_gbl_max_out, & + arr_max_levels, arr_max_levels_out, & + gbl_max_nsummands, gbl_max_nsummands_out,& + gbl_count, repro_sum_validate, & + repro_sum_stats, rel_diff, commid ) +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: nsummands !< number of local summands + integer, intent(in) :: dsummands !< declared first dimension + integer, intent(in) :: nflds !< number of fields + real(r8), intent(in) :: arr(dsummands,nflds) + !< input array + + real(r8), intent(out):: arr_gsum(nflds) + !< global means + + logical, intent(in), optional :: ddpdd_sum + !< use ddpdd algorithm instead + !< of fixed precision algorithm + + real(r8), intent(in), optional :: arr_gbl_max(nflds) + !< upper bound on max(abs(arr)) + + real(r8), intent(out), optional :: arr_gbl_max_out(nflds) + !< calculated upper bound on + !< max(abs(arr)) + + integer, intent(in), optional :: arr_max_levels(nflds) + !< maximum number of levels of + !< integer expansion to use + + integer, intent(out), optional :: arr_max_levels_out(nflds) + !< output of number of levels of + !< integer expansion to used + + integer, intent(in), optional :: gbl_max_nsummands + !< maximum of nsummand over all + !< processes + + integer, intent(out), optional :: gbl_max_nsummands_out + !< calculated maximum nsummands + !< over all processes + + integer, intent(in), optional :: gbl_count + !< was total number of summands; + !< now is ignored; use + !< gbl_max_nsummands instead + + logical, intent(in), optional :: repro_sum_validate + !< flag enabling/disabling testing that gmax and max_levels are + !< accurate/sufficient. Default is enabled. + + integer, intent(inout), optional :: repro_sum_stats(5) + !< increment running totals for + !< (1) one-reduction repro_sum + !< (2) two-reduction repro_sum + !< (3) both types in one call + !< (4) nonrepro_sum + !< (5) global max nsummands reduction + + real(r8), intent(out), optional :: rel_diff(2,nflds) + !< relative and absolute + !< differences between fixed + !< and floating point sums + + integer, intent(in), optional :: commid + !< MPI communicator + +! Local workspace + + logical :: use_ddpdd_sum ! flag indicating whether to + ! use ice_reprosum_ddpdd or not + logical :: recompute ! flag indicating need to + ! determine gmax/gmin before + ! computing sum + logical :: validate ! flag indicating need to + ! verify gmax and max_levels + ! are accurate/sufficient + integer :: omp_nthreads ! number of OpenMP threads + integer :: mpi_comm ! MPI subcommunicator + integer :: tasks ! number of MPI processes + integer :: mype ! MPI task rank + integer :: ierr ! MPI error return + integer :: ifld, isum, ithread ! loop variables + integer :: max_nsummands ! max nsummands over all processes + ! or threads (used in both ways) + + integer, allocatable :: isum_beg(:), isum_end(:) + ! range of summand indices for each + ! OpenMP thread + integer, allocatable :: arr_tlmin_exp(:,:) + ! per thread local exponent minima + integer, allocatable :: arr_tlmax_exp(:,:) + ! per thread local exponent maxima + integer :: arr_exp, arr_exp_tlmin, arr_exp_tlmax + ! summand exponent and working min/max + integer :: arr_lmin_exp(nflds) ! local exponent minima + integer :: arr_lmax_exp(nflds) ! local exponent maxima + integer :: arr_lextremes(0:nflds,2)! local exponent extrema + integer :: arr_gextremes(0:nflds,2)! global exponent extrema + + integer :: arr_gmax_exp(nflds) ! global exponents maxima + integer :: arr_gmin_exp(nflds) ! global exponents minima + integer :: arr_max_shift ! maximum safe exponent for + ! value < 1 (so that sum does + ! not overflow) + integer :: max_levels(nflds) ! maximum number of levels of + ! integer expansion to use + integer :: max_level ! maximum value in max_levels + integer :: gbl_max_red ! global max local sum reduction? (0/1) + integer :: repro_sum_fast ! 1 reduction repro_sum? (0/1) + integer :: repro_sum_slow ! 2 reduction repro_sum? (0/1) + integer :: repro_sum_both ! both fast and slow? (0/1) + integer :: nonrepro_sum ! nonrepro_sum? (0/1) + + real(r8) :: xmax_nsummands ! dble of max_nsummands + real(r8) :: arr_lsum(nflds) ! local sums + real(r8) :: arr_gsum_fast(nflds) ! global sum calculated using + ! fast, nonreproducible, + ! floating point alg. + real(r8) :: abs_diff ! absolute difference between + ! fixed and floating point + ! sums +#ifdef _OPENMP + integer omp_get_max_threads + external omp_get_max_threads +#endif + character(len=*),parameter :: subname = '(ice_reprosum_calc)' + +!----------------------------------------------------------------------- + +! check whether should use ice_reprosum_ddpdd algorithm + use_ddpdd_sum = repro_sum_use_ddpdd + if ( present(ddpdd_sum) ) then + use_ddpdd_sum = ddpdd_sum + endif + +! check whether intrinsic-based algorithm will work on this system +! (requires floating point and integer bases to be the same) +! If not, always use ddpdd. + use_ddpdd_sum = use_ddpdd_sum .or. (radix(0._r8) /= radix(0_i8)) + +! initialize local statistics variables + gbl_max_red = 0 + repro_sum_fast = 0 + repro_sum_slow = 0 + repro_sum_both = 0 + nonrepro_sum = 0 + +! set MPI communicator + if ( present(commid) ) then + mpi_comm = commid + else +#ifdef SERIAL_REMOVE_MPI + mpi_comm = 0 +#else + mpi_comm = MPI_COMM_WORLD +#endif + endif + +! if (detailed_timing) then +! call xicex_timer_start('xicex_reprosum_prebarrier') +! call xicex_mpi_barrier(mpi_comm,subname) +! call xicex_timer_stop ('xicex_reprosum_prebarrier') +! endif + + if ( use_ddpdd_sum ) then + +! if (detailed_timing) call xicex_timer_start('ice_reprosum_ddpdd') + + call ice_reprosum_ddpdd(arr, arr_gsum, nsummands, dsummands, & + nflds, mpi_comm) + repro_sum_fast = 1 + +! if (detailed_timing) call xicex_timer_stop('ice_reprosum_ddpdd') + + else + +! if (detailed_timing) call xicex_timer_start('ice_reprosum_int') + +! get number of MPI tasks +#ifdef SERIAL_REMOVE_MPI + tasks = 1 + mype = 0 +#else + call mpi_comm_size(mpi_comm, tasks, ierr) + call mpi_comm_rank(mpi_comm, mype, ierr) +#endif + +! get number of OpenMP threads +#ifdef _OPENMP + omp_nthreads = omp_get_max_threads() +#else + omp_nthreads = 1 +#endif + +! see if have sufficient information to not require max/min allreduce + recompute = .true. + validate = .false. + if ( present(arr_gbl_max) .and. present(arr_max_levels) ) then + recompute = .false. + +! setting lower bound on max_level*nflds to be 64 to improve OpenMP +! performance for loopb in ice_reprosum_int + max_level = (64/nflds) + 1 + do ifld=1,nflds + if ((arr_gbl_max(ifld) .ge. 0.0_r8) .and. & + (arr_max_levels(ifld) > 0)) then + + arr_gmax_exp(ifld) = exponent(arr_gbl_max(ifld)) + if (max_level < arr_max_levels(ifld)) & + max_level = arr_max_levels(ifld) + + else + recompute = .true. + endif + enddo + + if (.not. recompute) then + +! determine maximum number of summands in local phases of the +! algorithm +! if (detailed_timing) call xicex_timer_start("repro_sum_allr_max") + if ( present(gbl_max_nsummands) ) then + if (gbl_max_nsummands < 1) then +#ifdef SERIAL_REMOVE_MPI + max_nsummands = nsummands +#else + call mpi_allreduce (nsummands, max_nsummands, 1, & + MPI_INTEGER, MPI_MAX, mpi_comm, ierr) +#endif + gbl_max_red = 1 + else + max_nsummands = gbl_max_nsummands + endif + else +#ifdef SERIAL_REMOVE_MPI + max_nsummands = nsummands +#else + call mpi_allreduce (nsummands, max_nsummands, 1, & + MPI_INTEGER, MPI_MAX, mpi_comm, ierr) +#endif + gbl_max_red = 1 + endif +! if (detailed_timing) call xicex_timer_stop("repro_sum_allr_max") + +! determine maximum shift. Shift needs to be small enough that summation +! does not exceed maximum number of digits in i8. + +! if requested, return max_nsummands before it is redefined + if ( present( gbl_max_nsummands_out) ) then + gbl_max_nsummands_out = max_nsummands + endif + +! summing within each thread first + max_nsummands = (max_nsummands/omp_nthreads) + 1 +! then over threads and tasks + max_nsummands = max(max_nsummands, tasks*omp_nthreads) + + xmax_nsummands = dble(max_nsummands) + arr_max_shift = digits(0_i8) - (exponent(xmax_nsummands) + 1) + if (arr_max_shift < 2) then + write(tmpstr,*) subname//' ERROR: number of summands too large for fixed precision algorithm' + call abort_ice(tmpstr,file=__FILE__,line=__LINE__) + endif + +! calculate sum + if (present(repro_sum_validate)) then + validate = repro_sum_validate + else + validate = .true. + endif + call ice_reprosum_int(arr, arr_gsum, nsummands, dsummands, & + nflds, arr_max_shift, arr_gmax_exp, & + arr_max_levels, max_level, validate, & + recompute, omp_nthreads, mpi_comm) + +! record statistics, etc. + repro_sum_fast = 1 + if (recompute) then + repro_sum_both = 1 + else +! if requested, return specified levels and upper bounds on maxima + if ( present(arr_max_levels_out) ) then + do ifld=1,nflds + arr_max_levels_out(ifld) = arr_max_levels(ifld) + enddo + endif + if ( present(arr_gbl_max_out) ) then + do ifld=1,nflds + arr_gbl_max_out(ifld) = arr_gbl_max(ifld) + enddo + endif + endif + endif + endif + +! do not have sufficient information; calculate global max/min and +! use to compute required number of levels + if (recompute) then + +! record statistic + repro_sum_slow = 1 + +! determine maximum and minimum (non-zero) summand values and +! maximum number of local summands + +! allocate thread-specific work space + allocate(arr_tlmax_exp(nflds,omp_nthreads)) + allocate(arr_tlmin_exp(nflds,omp_nthreads)) + allocate(isum_beg(omp_nthreads)) + allocate(isum_end(omp_nthreads)) + +! split summand index range over OpenMP threads + call split_indices(nsummands, omp_nthreads, isum_beg, isum_end) + +!$omp parallel do & +!$omp default(shared) & +!$omp private(ithread, ifld, isum, arr_exp, arr_exp_tlmin, arr_exp_tlmax) + do ithread=1,omp_nthreads +! if (detailed_timing) call xicex_timer_start('repro_sum_loopa') + do ifld=1,nflds + arr_exp_tlmin = MAXEXPONENT(1._r8) + arr_exp_tlmax = MINEXPONENT(1._r8) + do isum=isum_beg(ithread),isum_end(ithread) + if (arr(isum,ifld) .ne. 0.0_r8) then + arr_exp = exponent(arr(isum,ifld)) + arr_exp_tlmin = min(arr_exp,arr_exp_tlmin) + arr_exp_tlmax = max(arr_exp,arr_exp_tlmax) + endif + end do + arr_tlmin_exp(ifld,ithread) = arr_exp_tlmin + arr_tlmax_exp(ifld,ithread) = arr_exp_tlmax + end do +! if (detailed_timing) call xicex_timer_stop('repro_sum_loopa') + end do + + do ifld=1,nflds + arr_lmax_exp(ifld) = maxval(arr_tlmax_exp(ifld,:)) + arr_lmin_exp(ifld) = minval(arr_tlmin_exp(ifld,:)) + end do + deallocate(arr_tlmin_exp,arr_tlmax_exp,isum_beg,isum_end) + + arr_lextremes(0,:) = -nsummands + arr_lextremes(1:nflds,1) = -arr_lmax_exp(:) + arr_lextremes(1:nflds,2) = arr_lmin_exp(:) +! if (detailed_timing) call xicex_timer_start("repro_sum_allr_minmax") +#ifdef SERIAL_REMOVE_MPI + arr_gextremes = arr_lextremes +#else + call mpi_allreduce (arr_lextremes, arr_gextremes, 2*(nflds+1), & + MPI_INTEGER, MPI_MIN, mpi_comm, ierr) +#endif +! if (detailed_timing) call xicex_timer_stop("repro_sum_allr_minmax") + max_nsummands = -arr_gextremes(0,1) + arr_gmax_exp(:) = -arr_gextremes(1:nflds,1) + arr_gmin_exp(:) = arr_gextremes(1:nflds,2) + +! if a field is identically zero, arr_gmin_exp still equals MAXEXPONENT +! and arr_gmax_exp still equals MINEXPONENT. In this case, set +! arr_gmin_exp = arr_gmax_exp = MINEXPONENT + do ifld=1,nflds + arr_gmin_exp(ifld) = min(arr_gmax_exp(ifld),arr_gmin_exp(ifld)) + enddo + +! if requested, return upper bounds on observed maxima + if ( present(arr_gbl_max_out) ) then + do ifld=1,nflds + arr_gbl_max_out(ifld) = scale(1.0_r8,arr_gmax_exp(ifld)) + enddo + endif + +! if requested, return max_nsummands before it is redefined + if ( present( gbl_max_nsummands_out) ) then + gbl_max_nsummands_out = max_nsummands + endif + +! determine maximum shift (same as in previous branch, but with calculated +! max_nsummands). Shift needs to be small enough that summation does not +! exceed maximum number of digits in i8. + +! summing within each thread first + max_nsummands = (max_nsummands/omp_nthreads) + 1 +! then over threads and tasks + max_nsummands = max(max_nsummands, tasks*omp_nthreads) + + xmax_nsummands = dble(max_nsummands) + arr_max_shift = digits(0_i8) - (exponent(xmax_nsummands) + 1) + if (arr_max_shift < 2) then + write(tmpstr,*) subname//' ERROR: number of summands too large for fixed precision algorithm' + call abort_ice(tmpstr,file=__FILE__,line=__LINE__) + endif + +! determine maximum number of levels required for each field +! ((digits(0_i8) + (arr_gmax_exp(ifld)-arr_gmin_exp(ifld))) / arr_max_shift) +! + 1 because first truncation probably does not involve a maximal shift +! + 1 to guarantee that the integer division rounds up (not down) +! (setting lower bound on max_level*nflds to be 64 to improve OpenMP +! performance for loopb in ice_reprosum_int) + max_level = (64/nflds) + 1 + do ifld=1,nflds + max_levels(ifld) = 2 + & + ((digits(0_i8) + (arr_gmax_exp(ifld)-arr_gmin_exp(ifld))) & + / arr_max_shift) + if ( present(arr_max_levels) .and. (.not. validate) ) then +! if validate true, then computation with arr_max_levels failed +! previously + if ( arr_max_levels(ifld) > 0 ) then + max_levels(ifld) = & + min(arr_max_levels(ifld),max_levels(ifld)) + endif + endif + if (max_level < max_levels(ifld)) & + max_level = max_levels(ifld) + enddo + +! if requested, return calculated levels + if ( present(arr_max_levels_out) ) then + do ifld=1,nflds + arr_max_levels_out(ifld) = max_levels(ifld) + enddo + endif + +! calculate sum + validate = .false. + call ice_reprosum_int(arr, arr_gsum, nsummands, dsummands, nflds, & + arr_max_shift, arr_gmax_exp, max_levels, & + max_level, validate, recompute, & + omp_nthreads, mpi_comm) + + endif + +! if (detailed_timing) call xicex_timer_stop('ice_reprosum_int') + + endif + +! compare fixed and floating point results + if ( present(rel_diff) ) then + if (ice_reprosum_reldiffmax >= 0.0_r8) then + +! if (detailed_timing) then +! call xicex_timer_start('xicex_nonreprosum_prebarrier') +! call xicex_mpi_barrier(mpi_comm,subname) +! call xicex_timer_stop ('xicex_nonreprosum_prebarrier') +! endif + +! if (detailed_timing) call xicex_timer_start('nonrepro_sum') +! record statistic + nonrepro_sum = 1 +! compute nonreproducible sum + arr_lsum(:) = 0._r8 +!$omp parallel do & +!$omp default(shared) & +!$omp private(ifld, isum) + do ifld=1,nflds + do isum=1,nsummands + arr_lsum(ifld) = arr(isum,ifld) + arr_lsum(ifld) + end do + end do + +#ifdef SERIAL_REMOVE_MPI + arr_gsum_fast = arr_lsum +#else + call mpi_allreduce (arr_lsum, arr_gsum_fast, nflds, & + MPI_REAL8, MPI_SUM, mpi_comm, ierr) +#endif + +! if (detailed_timing) call xicex_timer_stop('nonrepro_sum') + +! determine differences +!$omp parallel do & +!$omp default(shared) & +!$omp private(ifld, abs_diff) + do ifld=1,nflds + abs_diff = abs(arr_gsum_fast(ifld)-arr_gsum(ifld)) + if (abs(arr_gsum(ifld)) > abs_diff) then + rel_diff(1,ifld) = abs_diff/abs(arr_gsum(ifld)) + else + rel_diff(1,ifld) = abs_diff + endif + rel_diff(2,ifld) = abs_diff + enddo + else + rel_diff(:,:) = 0.0_r8 + endif + endif + +! return statistics + if ( present(repro_sum_stats) ) then + repro_sum_stats(1) = repro_sum_stats(1) + repro_sum_fast + repro_sum_stats(2) = repro_sum_stats(2) + repro_sum_slow + repro_sum_stats(3) = repro_sum_stats(3) + repro_sum_both + repro_sum_stats(4) = repro_sum_stats(4) + nonrepro_sum + repro_sum_stats(5) = repro_sum_stats(5) + gbl_max_red + endif + + + end subroutine ice_reprosum_calc + +!======================================================================== +! +! Purpose: +!> Compute the global sum of each field in "arr" using the indicated +!> communicator with a reproducible yet scalable implementation based +!> on a fixed point algorithm. The accuracy of the fixed point algorithm +!> is controlled by the number of "levels" of integer expansion, the +!> maximum value of which is specified by max_level. +! +!---------------------------------------------------------------------- + + subroutine ice_reprosum_int (arr, arr_gsum, nsummands, dsummands, nflds, & + arr_max_shift, arr_gmax_exp, max_levels, & + max_level, validate, recompute, & + omp_nthreads, mpi_comm ) + +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: nsummands !< number of local summands + integer, intent(in) :: dsummands !< declared first dimension + integer, intent(in) :: nflds !< number of fields + integer, intent(in) :: arr_max_shift !< maximum safe exponent for + !< value < 1 (so that sum + !< does not overflow) + integer, intent(in) :: arr_gmax_exp(nflds) + !< exponents of global maxima + integer, intent(in) :: max_levels(nflds) + !< maximum number of levels + !< of integer expansion + integer, intent(in) :: max_level !< maximum value in + !< max_levels + integer, intent(in) :: omp_nthreads !< number of OpenMP threads + integer, intent(in) :: mpi_comm !< MPI subcommunicator + + real(r8), intent(in) :: arr(dsummands,nflds) + !< input array + + logical, intent(in):: validate + !< flag indicating that accuracy of solution generated from + !< arr_gmax_exp and max_levels should be tested + + logical, intent(out):: recompute + !< flag indicating that either the upper bounds are inaccurate, + !< or max_levels and arr_gmax_exp do not generate accurate + !< enough sums + + real(r8), intent(out):: arr_gsum(nflds) !< global means + +! Local workspace + + integer, parameter :: max_jlevel = & + 1 + (digits(0_i8)/digits(0.0_r8)) + + integer(i8) :: i8_arr_tlsum_level(0:max_level,nflds,omp_nthreads) + ! integer vector representing local + ! sum (per thread, per field) + integer(i8) :: i8_arr_lsum_level((max_level+3)*nflds) + ! integer vector representing local + ! sum + integer(i8) :: i8_arr_level ! integer part of summand for current + ! expansion level + integer(i8) :: i8_arr_gsum_level((max_level+3)*nflds) + ! integer vector representing global + ! sum + integer(i8) :: IX_8 ! integer representation of current + ! jlevels of X_8 ('part' of + ! i8_arr_gsum_level) + integer(i8) :: i8_sign ! sign global sum + integer(i8) :: i8_radix ! radix for i8 variables + + integer :: max_error(nflds,omp_nthreads) + ! accurate upper bound on data? + integer :: not_exact(nflds,omp_nthreads) + ! max_levels sufficient to + ! capture all digits? + integer :: isum_beg(omp_nthreads), isum_end(omp_nthreads) + ! range of summand indices for each + ! OpenMP thread + integer :: ifld, isum, ithread + ! loop variables + integer :: arr_exp ! exponent of summand + integer :: arr_shift ! exponent used to generate integer + ! for current expansion level + integer :: ilevel ! current integer expansion level + integer :: offset(nflds) ! beginning location in + ! i8_arr_{g,l}sum_level for integer + ! expansion of current ifld + integer :: voffset ! modification to offset used to + ! include validation metrics + integer :: ioffset ! offset(ifld) + integer :: jlevel ! number of floating point 'pieces' + ! extracted from a given i8 integer + integer :: ierr ! MPI error return + integer :: LX(max_jlevel) ! exponent of X_8 (see below) + integer :: veclth ! total length of i8_arr_lsum_level + integer :: sum_digits ! lower bound on number of significant + ! in integer expansion of sum + integer :: curr_exp ! exponent of partial sum during + ! reconstruction from integer vector + integer :: corr_exp ! exponent of current summand in + ! reconstruction from integer vector + + real(r8) :: arr_frac ! fraction of summand + real(r8) :: arr_remainder ! part of summand remaining after + ! current level of integer expansion + real(r8) :: X_8(max_jlevel) ! r8 vector representation of current + ! i8_arr_gsum_level + real(r8) :: RX_8 ! r8 representation of difference + ! between current i8_arr_gsum_level + ! and current jlevels of X_8 + ! (== IX_8). Also used in final + ! scaling step + + logical :: first ! flag used to indicate that just + ! beginning reconstruction of sum + ! from integer vector + + character(len=*),parameter :: subname = '(ice_reprosum_int)' + +!----------------------------------------------------------------------- +! Save radix of i8 variables in an i8 variable + i8_radix = radix(IX_8) + +! If validating upper bounds, reserve space for validation metrics +! In both cases, reserve an extra level for overflow from the top level + if (validate) then + voffset = 3 + else + voffset = 1 + endif + +! compute offsets for each field + offset(1) = voffset + do ifld=2,nflds + offset(ifld) = offset(ifld-1) & + + (max_levels(ifld-1) + voffset) + enddo + veclth = offset(nflds) + max_levels(nflds) + +! split summand index range over OpenMP threads + call split_indices(nsummands, omp_nthreads, isum_beg, isum_end) + +! convert local summands to vector of integers and sum +! (Using scale instead of set_exponent because arr_remainder may not be +! "normal" after level 1 calculation) + i8_arr_lsum_level(:) = 0_i8 + +!$omp parallel do & +!$omp default(shared) & +!$omp private(ithread, ifld, ioffset, isum, arr_frac, arr_exp, & +!$omp arr_shift, ilevel, i8_arr_level, arr_remainder, RX_8, IX_8) + do ithread=1,omp_nthreads +! if (detailed_timing) call xicex_timer_start('repro_sum_loopb') + do ifld=1,nflds + ioffset = offset(ifld) + + max_error(ifld,ithread) = 0 + not_exact(ifld,ithread) = 0 + + i8_arr_tlsum_level(:,ifld,ithread) = 0_i8 + do isum=isum_beg(ithread),isum_end(ithread) + arr_remainder = 0.0_r8 + + if (arr(isum,ifld) .ne. 0.0_r8) then + arr_exp = exponent(arr(isum,ifld)) + arr_frac = fraction(arr(isum,ifld)) + +! test that global maximum upper bound is an upper bound + if (arr_exp > arr_gmax_exp(ifld)) then + max_error(ifld,ithread) = 1 + exit + endif + +! calculate first shift + arr_shift = arr_max_shift - (arr_gmax_exp(ifld)-arr_exp) + +! determine first (probably) nonzero level (assuming initial fraction is +! 'normal' - algorithm still works if this is not true) +! NOTE: this is critical; scale will set to zero if min exponent is too small. + if (arr_shift < 1) then + ilevel = (1 + (arr_gmax_exp(ifld)-arr_exp))/arr_max_shift + arr_shift = ilevel*arr_max_shift - (arr_gmax_exp(ifld)-arr_exp) + + do while (arr_shift < 1) + arr_shift = arr_shift + arr_max_shift + ilevel = ilevel + 1 + enddo + else + ilevel = 1 + endif + + if (ilevel .le. max_levels(ifld)) then +! apply first shift/truncate, add it to the relevant running +! sum, and calculate the remainder. + arr_remainder = scale(arr_frac,arr_shift) + i8_arr_level = int(arr_remainder,i8) + i8_arr_tlsum_level(ilevel,ifld,ithread) = & + i8_arr_tlsum_level(ilevel,ifld,ithread) + i8_arr_level + arr_remainder = arr_remainder - i8_arr_level + +! while the remainder is non-zero, continue to shift, truncate, +! sum, and calculate new remainder + do while ((arr_remainder .ne. 0.0_r8) & + .and. (ilevel < max_levels(ifld))) + ilevel = ilevel + 1 + arr_remainder = scale(arr_remainder,arr_max_shift) + i8_arr_level = int(arr_remainder,i8) + i8_arr_tlsum_level(ilevel,ifld,ithread) = & + i8_arr_tlsum_level(ilevel,ifld,ithread) + i8_arr_level + arr_remainder = arr_remainder - i8_arr_level + enddo + + endif + endif + + if (arr_remainder .ne. 0.0_r8) then + not_exact(ifld,ithread) = 1 + endif + + enddo + +! postprocess integer vector to eliminate potential for overlap in the following +! sums over threads and processes: if value larger than or equal to +! (radix(IX_8)**arr_max_shift), add this 'overlap' to next larger integer in +! vector, resulting in nonoverlapping ranges for each component. Note that +! "ilevel-1==0" corresponds to an extra level used to guarantee that the sums +! over threads and processes do not overflow for ilevel==1. + do ilevel=max_levels(ifld),1,-1 + RX_8 = i8_arr_tlsum_level(ilevel,ifld,ithread) + IX_8 = int(scale(RX_8,-arr_max_shift),i8) + if (IX_8 .ne. 0_i8) then + i8_arr_tlsum_level(ilevel-1,ifld,ithread) = & + i8_arr_tlsum_level(ilevel-1,ifld,ithread) + IX_8 + IX_8 = IX_8*(i8_radix**arr_max_shift) + i8_arr_tlsum_level(ilevel,ifld,ithread) = & + i8_arr_tlsum_level(ilevel,ifld,ithread) - IX_8 + endif + enddo + enddo +! if (detailed_timing) call xicex_timer_stop('repro_sum_loopb') + enddo + +! sum contributions from different threads + do ifld=1,nflds + ioffset = offset(ifld) + do ithread = 1,omp_nthreads + do ilevel = 0,max_levels(ifld) + i8_arr_lsum_level(ioffset+ilevel) = & + i8_arr_lsum_level(ioffset+ilevel) & + + i8_arr_tlsum_level(ilevel,ifld,ithread) + enddo + enddo + enddo + +! record if upper bound was inaccurate or if level expansion stopped +! before full accuracy was achieved + if (validate) then + do ifld=1,nflds + ioffset = offset(ifld) + i8_arr_lsum_level(ioffset-voffset+1) = maxval(max_error(ifld,:)) + i8_arr_lsum_level(ioffset-voffset+2) = maxval(not_exact(ifld,:)) + enddo + endif + +! sum integer vector element-wise +#ifdef SERIAL_REMOVE_MPI + i8_arr_gsum_level = i8_arr_lsum_level +#else +#if ( defined noI8 ) + ! Workaround for when i8 is not supported. +! if (detailed_timing) call xicex_timer_start("repro_sum_allr_i4") + call mpi_allreduce (i8_arr_lsum_level, i8_arr_gsum_level, & + veclth, MPI_INTEGER, MPI_SUM, mpi_comm, ierr) +! if (detailed_timing) call xicex_timer_stop("repro_sum_allr_i4") +#else +! if (detailed_timing) call xicex_timer_start("repro_sum_allr_i8") + call mpi_allreduce (i8_arr_lsum_level, i8_arr_gsum_level, & + veclth, MPI_INTEGER8, MPI_SUM, mpi_comm, ierr) +! if (detailed_timing) call xicex_timer_stop("repro_sum_allr_i8") +#endif +#endif + +! Construct global sum from integer vector representation: +! 1) arr_max_shift is the shift applied to fraction(arr_gmax) . +! When shifting back, need to "add back in" true arr_gmax exponent. This was +! removed implicitly by working only with the fraction . +! 2) want to add levels into sum in reverse order (smallest to largest). However, +! even this can generate floating point rounding errors if signs of integers +! alternate. To avoid this, do some arithmetic with integer vectors so that all +! components have the same sign. This should keep relative difference between +! using different integer sizes (e.g. i8 and i4) to machine epsilon +! 3) assignment to X_8 will usually lose accuracy since maximum integer +! size is greater than the max number of 'digits' in r8 value (if xmax_nsummands +! correction not very large). Calculate remainder and add in first (since +! smaller). One correction is sufficient for r8 (53 digits) and i8 (63 digits). +! For r4 (24 digits) may need to correct twice. Code is written in a general +! fashion, to work no matter how many corrections are necessary (assuming +! max_jlevel parameter calculation is correct). + + recompute = .false. + do ifld=1,nflds + arr_gsum(ifld) = 0.0_r8 + ioffset = offset(ifld) + +! if validate is .true., test whether the summand upper bound +! was exceeded on any of the processes + if (validate) then + if (i8_arr_gsum_level(ioffset-voffset+1) .ne. 0_i8) then + recompute = .true. + endif + endif + + if (.not. recompute) then + +! preprocess integer vector: +! a) if value larger than or equal to (radix(IX_8)**arr_max_shift), add this 'overlap' +! to next larger integer in vector, resulting in nonoverlapping ranges for each +! component. Note that have "ilevel-1=0" level here as described above. + do ilevel=max_levels(ifld),1,-1 + RX_8 = i8_arr_gsum_level(ioffset+ilevel) + IX_8 = int(scale(RX_8,-arr_max_shift),i8) + if (IX_8 .ne. 0_i8) then + i8_arr_gsum_level(ioffset+ilevel-1) = i8_arr_gsum_level(ioffset+ilevel-1) & + + IX_8 + IX_8 = IX_8*(i8_radix**arr_max_shift) + i8_arr_gsum_level(ioffset+ilevel) = i8_arr_gsum_level(ioffset+ilevel) & + - IX_8 + endif + enddo +! b) subtract +/- 1 from larger and add +/- 1 to smaller when necessary +! so that all vector components have the same sign (eliminating loss +! of accuracy arising from difference of large values when +! reconstructing r8 sum from integer vector) + ilevel = 0 + do while ((i8_arr_gsum_level(ioffset+ilevel) .eq. 0_i8) & + .and. (ilevel < max_levels(ifld))) + ilevel = ilevel + 1 + enddo +! + if (ilevel < max_levels(ifld)) then + if (i8_arr_gsum_level(ioffset+ilevel) > 0_i8) then + i8_sign = 1_i8 + else + i8_sign = -1_i8 + endif + do jlevel=ilevel,max_levels(ifld)-1 + if (sign(1_i8,i8_arr_gsum_level(ioffset+jlevel)) & + .ne. sign(1_i8,i8_arr_gsum_level(ioffset+jlevel+1))) then + i8_arr_gsum_level(ioffset+jlevel) = i8_arr_gsum_level(ioffset+jlevel) & + - i8_sign + i8_arr_gsum_level(ioffset+jlevel+1) = i8_arr_gsum_level(ioffset+jlevel+1) & + + i8_sign*(i8_radix**arr_max_shift) + endif + enddo + endif + +! start with maximum shift, and work up to larger values + arr_shift = arr_gmax_exp(ifld) & + - max_levels(ifld)*arr_max_shift + curr_exp = 0 + first = .true. + do ilevel=max_levels(ifld),0,-1 + + if (i8_arr_gsum_level(ioffset+ilevel) .ne. 0_i8) then + jlevel = 1 + +! r8 representation of higher order bits in integer + X_8(jlevel) = i8_arr_gsum_level(ioffset+ilevel) + LX(jlevel) = exponent(X_8(jlevel)) + +! calculate remainder + IX_8 = int(X_8(jlevel),i8) + RX_8 = (i8_arr_gsum_level(ioffset+ilevel) - IX_8) + +! repeat using remainder + do while (RX_8 .ne. 0.0_r8) + jlevel = jlevel + 1 + X_8(jlevel) = RX_8 + LX(jlevel) = exponent(RX_8) + IX_8 = IX_8 + int(RX_8,i8) + RX_8 = (i8_arr_gsum_level(ioffset+ilevel) - IX_8) + enddo + +! add in contributions, smaller to larger, rescaling for each +! addition to guarantee that exponent of working summand is always +! larger than minexponent + do while (jlevel > 0) + if (first) then + curr_exp = LX(jlevel) + arr_shift + arr_gsum(ifld) = fraction(X_8(jlevel)) + first = .false. + else + corr_exp = curr_exp - (LX(jlevel) + arr_shift) + arr_gsum(ifld) = fraction(X_8(jlevel)) & + + scale(arr_gsum(ifld),corr_exp) + curr_exp = LX(jlevel) + arr_shift + endif + jlevel = jlevel - 1 + enddo + + endif + + arr_shift = arr_shift + arr_max_shift + enddo + +! apply final exponent correction, scaling first if exponent is too small +! to apply directly + corr_exp = curr_exp + exponent(arr_gsum(ifld)) + if (corr_exp .ge. MINEXPONENT(1._r8)) then + arr_gsum(ifld) = set_exponent(arr_gsum(ifld),corr_exp) + else + RX_8 = set_exponent(arr_gsum(ifld), & + corr_exp-MINEXPONENT(1._r8)) + arr_gsum(ifld) = scale(RX_8,MINEXPONENT(1._r8)) + endif + +! if validate is .true. and some precision lost, test whether 'too much' +! was lost, due to too loose an upper bound, too stringent a limit on number +! of levels of expansion, cancellation, .... Calculated by comparing lower +! bound on number of sigificant digits with number of digits in 1.0_r8 . + if (validate) then + if (i8_arr_gsum_level(ioffset-voffset+2) .ne. 0_i8) then + +! find first nonzero level and use exponent for this level, then assume all +! subsequent levels contribute arr_max_shift digits. + sum_digits = 0 + do ilevel=0,max_levels(ifld) + if (sum_digits .eq. 0) then + if (i8_arr_gsum_level(ioffset+ilevel) .ne. 0_i8) then + X_8(1) = i8_arr_gsum_level(ioffset+ilevel) + LX(1) = exponent(X_8(1)) + sum_digits = LX(1) + endif + else + sum_digits = sum_digits + arr_max_shift + endif + enddo + + if (sum_digits < digits(1.0_r8)) then + recompute = .true. + endif + endif + endif + + endif + + enddo + + + end subroutine ice_reprosum_int + +!======================================================================== +! +! Purpose: +!> Test whether distributed sum exceeds tolerance and print out a +!> warning message. +! +!---------------------------------------------------------------------- + + logical function ice_reprosum_tolExceeded (name, nflds, master, & + logunit, rel_diff ) +!---------------------------------------------------------------------- + +! Arguments + + character(len=*), intent(in) :: name !< distributed sum identifier + integer, intent(in) :: nflds !< number of fields + logical, intent(in) :: master !< process that will write + !< warning messages? + integer, optional, intent(in) :: logunit!< unit warning messages + !< written to + real(r8), intent(in) :: rel_diff(2,nflds) + !< relative and absolute + !< differences between fixed + !< and floating point sums + +! Local workspace + + integer :: llogunit ! local log unit + integer :: ifld ! field index + integer :: exceeds_limit ! number of fields whose + ! sum exceeds tolerance + real(r8) :: max_rel_diff ! maximum relative difference + integer :: max_rel_diff_idx ! field index for max. rel. diff. + real(r8) :: max_abs_diff ! maximum absolute difference + integer :: max_abs_diff_idx ! field index for max. abs. diff. + character(len=*),parameter :: subname = '(ice_reprosum_tolExceeded)' + +!----------------------------------------------------------------------- + + ice_reprosum_tolExceeded = .false. + if (ice_reprosum_reldiffmax < 0.0_r8) return + + if ( present(logunit) ) then + llogunit = logunit + else + llogunit = nu_diag + endif + + ! check that "fast" reproducible sum is accurate enough. + exceeds_limit = 0 + max_rel_diff = 0.0_r8 + max_abs_diff = 0.0_r8 + do ifld=1,nflds + if (rel_diff(1,ifld) > ice_reprosum_reldiffmax) then + exceeds_limit = exceeds_limit + 1 + if (rel_diff(1,ifld) > max_rel_diff) then + max_rel_diff = rel_diff(1,ifld) + max_rel_diff_idx = ifld + endif + if (rel_diff(2,ifld) > max_abs_diff) then + max_abs_diff = rel_diff(2,ifld) + max_abs_diff_idx = ifld + endif + endif + enddo + + if (exceeds_limit > 0) then + if (master) then + write(llogunit,*) subname,trim(name), & + ': difference in fixed and floating point sums ', & + ' exceeds tolerance in ', exceeds_limit, & + ' fields.' + write(llogunit,*) subname,' Maximum relative diff: (rel)', & + rel_diff(1,max_rel_diff_idx), ' (abs) ', & + rel_diff(2,max_rel_diff_idx) + write(llogunit,*) subname,' Maximum absolute diff: (rel)', & + rel_diff(1,max_abs_diff_idx), ' (abs) ', & + rel_diff(2,max_abs_diff_idx) + endif + ice_reprosum_tolExceeded = .true. + endif + + + end function ice_reprosum_tolExceeded + +!======================================================================== +! +! Purpose: +!> Compute the global sum of each field in "arr" using the indicated +!> communicator with a reproducible yet scalable implementation based +!> on He and Ding's implementation of the double-double algorithm. +! +!---------------------------------------------------------------------- + + subroutine ice_reprosum_ddpdd (arr, arr_gsum, nsummands, dsummands, & + nflds, mpi_comm ) +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: nsummands !< number of local summands + integer, intent(in) :: dsummands !< declared first dimension + integer, intent(in) :: nflds !< number of fields + real(r8), intent(in) :: arr(dsummands,nflds) + !< input array + integer, intent(in) :: mpi_comm !< MPI subcommunicator + + real(r8), intent(out):: arr_gsum(nflds) + !< global sums + + +! Local workspace + + integer :: old_cw ! for x86 processors, save + ! current arithmetic mode + integer :: ifld, isum ! loop variables + integer :: ierr ! MPI error return + + real(r8) :: e, t1, t2 ! temporaries + complex(r8) :: arr_lsum_dd(nflds) ! local sums (in double-double + ! format) + complex(r8) :: arr_gsum_dd(nflds) ! global sums (in double-double + ! format) + + integer, save :: mpi_sumdd + logical, save :: first_time = .true. + character(len=*),parameter :: subname = '(ice_reprosum_ddpdd)' + +!----------------------------------------------------------------------- + + call ice_shr_reprosumx86_fix_start (old_cw) + + if (first_time) then +#ifdef SERIAL_REMOVE_MPI + mpi_sumdd = 0 +#else + call mpi_op_create(ddpdd, .true., mpi_sumdd, ierr) +#endif + first_time = .false. + endif + + do ifld=1,nflds + arr_lsum_dd(ifld) = (0.0_r8,0.0_r8) + + do isum=1,nsummands + + ! Compute arr(isum,ifld) + arr_lsum_dd(ifld) using Knuth''s + ! trick. + t1 = arr(isum,ifld) + real(arr_lsum_dd(ifld)) + e = t1 - arr(isum,ifld) + t2 = ((real(arr_lsum_dd(ifld)) - e) & + + (arr(isum,ifld) - (t1 - e))) & + + aimag(arr_lsum_dd(ifld)) + + ! The result is t1 + t2, after normalization. + arr_lsum_dd(ifld) = cmplx ( t1 + t2, t2 - ((t1 + t2) - t1), r8 ) + enddo + + enddo + +#ifdef SERIAL_REMOVE_MPI + arr_gsum_dd = arr_lsum_dd +#else + call mpi_allreduce (arr_lsum_dd, arr_gsum_dd, nflds, & + MPI_COMPLEX16, mpi_sumdd, mpi_comm, ierr) +#endif + do ifld=1,nflds + arr_gsum(ifld) = real(arr_gsum_dd(ifld)) + enddo + + call ice_shr_reprosumx86_fix_end (old_cw) + + end subroutine ice_reprosum_ddpdd + +!----------------------------------------------------------------------- + + subroutine DDPDD (dda, ddb, len, itype) +!---------------------------------------------------------------------- +! +! Purpose: +! Modification of original codes written by David H. Bailey +! This subroutine computes ddb(i) = dda(i)+ddb(i) +! +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: len ! array length + complex(r8), intent(in) :: dda(len) ! input + complex(r8), intent(inout) :: ddb(len) ! result + integer, intent(in) :: itype ! unused + +! Local workspace + + real(r8) e, t1, t2 + integer i + character(len=*),parameter :: subname = '(ice_reprosum_mod:DDPDD)' + +!----------------------------------------------------------------------- + + do i = 1, len +! Compute dda + ddb using Knuth's trick. + t1 = real(dda(i)) + real(ddb(i)) + e = t1 - real(dda(i)) + t2 = ((real(ddb(i)) - e) + (real(dda(i)) - (t1 - e))) & + + aimag(dda(i)) + aimag(ddb(i)) + +! The result is t1 + t2, after normalization. + ddb(i) = cmplx ( t1 + t2, t2 - ((t1 + t2) - t1), r8 ) + enddo + + + end subroutine DDPDD + +!----------------------------------------------------------------------- + + subroutine split_indices(total,num_pieces,ibeg,iend) +!---------------------------------------------------------------------- +! +! Purpose: +! Split range into 'num_pieces' +! +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: total + integer, intent(in) :: num_pieces + integer, intent(out) :: ibeg(num_pieces), iend(num_pieces) + +! Local workspace + + integer :: itmp1, itmp2, ioffset, i + character(len=*),parameter :: subname = '(ice_reprosum_mod:split_indices)' + +!----------------------------------------------------------------------- + + itmp1 = total/num_pieces + itmp2 = mod(total,num_pieces) + ioffset = 0 + do i=1,itmp2 + ibeg(i) = ioffset + 1 + iend(i) = ioffset + (itmp1+1) + ioffset = iend(i) + enddo + do i=itmp2+1,num_pieces + ibeg(i) = ioffset + 1 + if (ibeg(i) > total) then + iend(i) = ibeg(i) - 1 + else + iend(i) = ioffset + itmp1 + ioffset = iend(i) + endif + enddo + + end subroutine split_indices + +!======================================================================== + +end module ice_reprosum diff --git a/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 b/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 new file mode 100644 index 000000000..490b16b14 --- /dev/null +++ b/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 @@ -0,0 +1,1482 @@ + +!> Reproducible sum method from P. Worley +#define SERIAL_REMOVE_MPI + +MODULE ice_reprosum + +!----------------------------------------------------------------------- +! +! Purpose: +!> Compute reproducible global sums of a set of arrays across an MPI +!> subcommunicator +! +! Methods: +!> Compute using either or both a scalable, reproducible algorithm and a +!> scalable, nonreproducible algorithm: +!> * Reproducible (scalable): +!> Convert to fixed point (integer vector representation) to enable +!> reproducibility when using MPI_Allreduce +!> * Alternative usually reproducible (scalable): +!> Use parallel double-double algorithm due to Helen He and +!> Chris Ding, based on David Bailey's/Don Knuth's DDPDD algorithm +!> * Nonreproducible (scalable): +!> Floating point and MPI_Allreduce based. +!> If computing both reproducible and nonreproducible sums, compare +!> these and report relative difference (if absolute difference +!> less than sum) or absolute difference back to calling routine. +! +! Author: P. Worley (based on suggestions from J. White for fixed +! point algorithm and on He/Ding paper for ddpdd +! algorithm) +! +! Modified by T.Craig for CICE, March 2019 based on the public version in +! Oasis3-MCT_4.0. +! +!----------------------------------------------------------------------- + +!----------------------------------------------------------------------- +!- use statements ------------------------------------------------------ +!----------------------------------------------------------------------- +#if ( defined noI8 ) + ! Workaround for when shr_kind_i8 is not supported. + use ice_kinds_mod, only: r8 => dbl_kind, i8 => int_kind +#else + use ice_kinds_mod, only: r8 => dbl_kind, i8 => int8_kind +#endif + use ice_kinds_mod, only: char_len_long + use ice_fileunits, only: nu_diag + use ice_exit, only: abort_ice + +! internal timers not yet implemented, need to revisit if needed +! use ice_mpi, only: xicex_mpi_barrier +! use ice_timer, only: xicex_timer_start, xicex_timer_stop + +!----------------------------------------------------------------------- +!- module boilerplate -------------------------------------------------- +!----------------------------------------------------------------------- + implicit none + private + +!----------------------------------------------------------------------- +!- include statements -------------------------------------------------- +!----------------------------------------------------------------------- +#ifndef SERIAL_REMOVE_MPI +#include +#endif + +!----------------------------------------------------------------------- +! Public interfaces ---------------------------------------------------- +!----------------------------------------------------------------------- + public :: & + ice_reprosum_setopts, &! set runtime options + ice_reprosum_calc, &! calculate distributed sum + ice_reprosum_tolExceeded ! utility function to check relative + ! differences against the tolerance + +!----------------------------------------------------------------------- +! Public data ---------------------------------------------------------- +!----------------------------------------------------------------------- + logical, public :: ice_reprosum_recompute = .false. + + real(r8), public :: ice_reprosum_reldiffmax = -1.0_r8 + +!----------------------------------------------------------------------- +! Private interfaces --------------------------------------------------- +!----------------------------------------------------------------------- + private :: & + ddpdd, &! double-double sum routine + split_indices ! split indices among OMP threads + +!----------------------------------------------------------------------- +! Private data --------------------------------------------------------- +!----------------------------------------------------------------------- + + logical :: repro_sum_use_ddpdd = .false. + logical :: detailed_timing = .false. + character(len=char_len_long) :: tmpstr + + CONTAINS + +!======================================================================== +!----------------------------------------------------------------------- +! Purpose: +!> Set runtime options +! Author: P. Worley +!----------------------------------------------------------------------- + + subroutine ice_reprosum_setopts(repro_sum_use_ddpdd_in, & + repro_sum_rel_diff_max_in, & + repro_sum_recompute_in, & + repro_sum_master, & + repro_sum_logunit ) + +!------------------------------Arguments-------------------------------- + logical, intent(in), optional :: repro_sum_use_ddpdd_in + !< Use DDPDD algorithm instead of fixed precision algorithm + real(r8), intent(in), optional :: repro_sum_rel_diff_max_in + !< maximum permissible difference between reproducible and + !< nonreproducible sums + logical, intent(in), optional :: repro_sum_recompute_in + !< recompute using different algorithm when difference between + !< reproducible and nonreproducible sums is too great + logical, intent(in), optional :: repro_sum_master + !< flag indicating whether this process should output + !< log messages + integer, intent(in), optional :: repro_sum_logunit + !< unit number for log messages +!---------------------------Local Workspace----------------------------- + integer llogunit ! unit number for log messages + logical master ! local master? + logical,save :: firstcall = .true. ! first call + character(len=*),parameter :: subname = '(ice_reprosum_setopts)' +!----------------------------------------------------------------------- + + if ( present(repro_sum_master) ) then + master = repro_sum_master + else + master = .false. + endif + + if ( present(repro_sum_logunit) ) then + llogunit = repro_sum_logunit + else + llogunit = nu_diag + endif + + if (.not. firstcall) then + write(tmpstr,*) subname//' ERROR: can only be called once' + call abort_ice(tmpstr,file=__FILE__,line=__LINE__) + endif + firstcall = .false. + + if ( present(repro_sum_use_ddpdd_in) ) then + repro_sum_use_ddpdd = repro_sum_use_ddpdd_in + endif + if ( present(repro_sum_rel_diff_max_in) ) then + ice_reprosum_reldiffmax = repro_sum_rel_diff_max_in + endif + if ( present(repro_sum_recompute_in) ) then + ice_reprosum_recompute = repro_sum_recompute_in + endif + if (master) then + if ( repro_sum_use_ddpdd ) then + write(llogunit,*) subname, & + 'Using double-double-based (scalable) usually reproducible ', & + 'distributed sum algorithm' + else + write(llogunit,*) subname, & + 'Using fixed-point-based (scalable) reproducible ', & + 'distributed sum algorithm' + endif + + if (ice_reprosum_reldiffmax >= 0._r8) then + write(llogunit,*) subname, & + ' with a maximum relative error tolerance of ', & + ice_reprosum_reldiffmax + if (ice_reprosum_recompute) then + write(llogunit,*) subname, & + 'If tolerance exceeded, sum is recomputed using ', & + 'a serial algorithm.' + else + write(llogunit,*) subname, & + 'If tolerance exceeded, fixed-precision is sum used ', & + 'but a warning is output.' + endif + else + write(llogunit,*) subname, & + 'and not comparing with floating point algorithms.' + endif + + endif + end subroutine ice_reprosum_setopts + +!======================================================================== + +! +! Purpose: +!> Compute the global sum of each field in "arr" using the indicated +!> communicator with a reproducible yet scalable implementation based +!> on a fixed point algorithm. An alternative is to use an "almost +!> always reproducible" floating point algorithm. +! +! The accuracy of the fixed point algorithm is controlled by the +! number of "levels" of integer expansion. The algorithm will calculate +! the number of levels that is required for the sum to be essentially +! exact. The optional parameter arr_max_levels can be used to override +! the calculated value. The optional parameter arr_max_levels_out can be +! used to return the values used. +! +! The algorithm also requires an upper bound on +! the maximum summand (in absolute value) for each field, and will +! calculate this internally. However, if the optional parameters +! arr_max_levels and arr_gbl_max are both set, then the algorithm will +! use the values in arr_gbl_max for the upper bounds instead. If these +! are not upper bounds, or if the upper bounds are not tight enough +! to achieve the requisite accuracy, and if the optional parameter +! repro_sum_validate is NOT set to .false., the algorithm will repeat the +! computation with appropriate upper bounds. If only arr_gbl_max is present, +! then the maxima are computed internally (and the specified values are +! ignored). The optional parameter arr_gbl_max_out can be +! used to return the values used. +! +! Finally, the algorithm requires an upper bound on the number of +! local summands across all processes. This will be calculated internally, +! using an MPI collective, but the value in the optional argument +! gbl_max_nsummands will be used instead if (1) it is present, (2) +! it is > 0, and (3) the maximum value and required number of levels +! are also specified. (If the maximum value is calculated, the same +! MPI collective is used to determine the maximum number of local +! summands.) The accuracy of the user-specified value is not checked. +! However, if set to < 1, the value will instead be calculated. If the +! optional parameter gbl_max_nsummands_out is present, then the value +! used (gbl_max_nsummands if >= 1; calculated otherwise) will be +! returned. +! +! If requested (by setting ice_reprosum_reldiffmax >= 0.0 and passing in +! the optional rel_diff parameter), results are compared with a +! nonreproducible floating point algorithm. +! +! Note that the cost of the algorithm is not strongly correlated with +! the number of levels, which primarily shows up as a (modest) increase +! in cost of the MPI_Allreduce as a function of vector length. Rather the +! cost is more a function of (a) the number of integers required to +! represent an individual summand and (b) the number of MPI_Allreduce +! calls. The number of integers required to represent an individual +! summand is 1 or 2 when using 8-byte integers for 8-byte real summands +! when the number of local summands is not too large. As the number of +! local summands increases, the number of integers required increases. +! The number of MPI_Allreduce calls is either 2 (specifying nothing) or +! 1 (specifying gbl_max_nsummands, arr_max_levels, and arr_gbl_max +! correctly). When specifying arr_max_levels and arr_gbl_max +! incorrectly, 3 or 4 MPI_Allreduce calls will be required. +! +! The alternative algorithm is a minor modification of a parallel +! implementation of David Bailey's routine DDPDD by Helen He +! and Chris Ding. Bailey uses the Knuth trick to implement quadruple +! precision summation of double precision values with 10 double +! precision operations. The advantage of this algorithm is that +! it requires a single MPI_Allreduce and is less expensive per summand +! than is the fixed precision algorithm. The disadvantage is that it +! is not guaranteed to be reproducible (though it is reproducible +! much more often than is the standard algorithm). This alternative +! is used when the optional parameter ddpdd_sum is set to .true. It is +! also used if the fixed precision algorithm radix assumption does not +! hold. + +!---------------------------------------------------------------------- + subroutine ice_reprosum_calc (arr, arr_gsum, nsummands, dsummands, & + nflds, ddpdd_sum, & + arr_gbl_max, arr_gbl_max_out, & + arr_max_levels, arr_max_levels_out, & + gbl_max_nsummands, gbl_max_nsummands_out,& + gbl_count, repro_sum_validate, & + repro_sum_stats, rel_diff, commid ) +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: nsummands !< number of local summands + integer, intent(in) :: dsummands !< declared first dimension + integer, intent(in) :: nflds !< number of fields + real(r8), intent(in) :: arr(dsummands,nflds) + !< input array + + real(r8), intent(out):: arr_gsum(nflds) + !< global means + + logical, intent(in), optional :: ddpdd_sum + !< use ddpdd algorithm instead + !< of fixed precision algorithm + + real(r8), intent(in), optional :: arr_gbl_max(nflds) + !< upper bound on max(abs(arr)) + + real(r8), intent(out), optional :: arr_gbl_max_out(nflds) + !< calculated upper bound on + !< max(abs(arr)) + + integer, intent(in), optional :: arr_max_levels(nflds) + !< maximum number of levels of + !< integer expansion to use + + integer, intent(out), optional :: arr_max_levels_out(nflds) + !< output of number of levels of + !< integer expansion to used + + integer, intent(in), optional :: gbl_max_nsummands + !< maximum of nsummand over all + !< processes + + integer, intent(out), optional :: gbl_max_nsummands_out + !< calculated maximum nsummands + !< over all processes + + integer, intent(in), optional :: gbl_count + !< was total number of summands; + !< now is ignored; use + !< gbl_max_nsummands instead + + logical, intent(in), optional :: repro_sum_validate + !< flag enabling/disabling testing that gmax and max_levels are + !< accurate/sufficient. Default is enabled. + + integer, intent(inout), optional :: repro_sum_stats(5) + !< increment running totals for + !< (1) one-reduction repro_sum + !< (2) two-reduction repro_sum + !< (3) both types in one call + !< (4) nonrepro_sum + !< (5) global max nsummands reduction + + real(r8), intent(out), optional :: rel_diff(2,nflds) + !< relative and absolute + !< differences between fixed + !< and floating point sums + + integer, intent(in), optional :: commid + !< MPI communicator + +! Local workspace + + logical :: use_ddpdd_sum ! flag indicating whether to + ! use ice_reprosum_ddpdd or not + logical :: recompute ! flag indicating need to + ! determine gmax/gmin before + ! computing sum + logical :: validate ! flag indicating need to + ! verify gmax and max_levels + ! are accurate/sufficient + integer :: omp_nthreads ! number of OpenMP threads + integer :: mpi_comm ! MPI subcommunicator + integer :: tasks ! number of MPI processes + integer :: mype ! MPI task rank + integer :: ierr ! MPI error return + integer :: ifld, isum, ithread ! loop variables + integer :: max_nsummands ! max nsummands over all processes + ! or threads (used in both ways) + + integer, allocatable :: isum_beg(:), isum_end(:) + ! range of summand indices for each + ! OpenMP thread + integer, allocatable :: arr_tlmin_exp(:,:) + ! per thread local exponent minima + integer, allocatable :: arr_tlmax_exp(:,:) + ! per thread local exponent maxima + integer :: arr_exp, arr_exp_tlmin, arr_exp_tlmax + ! summand exponent and working min/max + integer :: arr_lmin_exp(nflds) ! local exponent minima + integer :: arr_lmax_exp(nflds) ! local exponent maxima + integer :: arr_lextremes(0:nflds,2)! local exponent extrema + integer :: arr_gextremes(0:nflds,2)! global exponent extrema + + integer :: arr_gmax_exp(nflds) ! global exponents maxima + integer :: arr_gmin_exp(nflds) ! global exponents minima + integer :: arr_max_shift ! maximum safe exponent for + ! value < 1 (so that sum does + ! not overflow) + integer :: max_levels(nflds) ! maximum number of levels of + ! integer expansion to use + integer :: max_level ! maximum value in max_levels + integer :: gbl_max_red ! global max local sum reduction? (0/1) + integer :: repro_sum_fast ! 1 reduction repro_sum? (0/1) + integer :: repro_sum_slow ! 2 reduction repro_sum? (0/1) + integer :: repro_sum_both ! both fast and slow? (0/1) + integer :: nonrepro_sum ! nonrepro_sum? (0/1) + + real(r8) :: xmax_nsummands ! dble of max_nsummands + real(r8) :: arr_lsum(nflds) ! local sums + real(r8) :: arr_gsum_fast(nflds) ! global sum calculated using + ! fast, nonreproducible, + ! floating point alg. + real(r8) :: abs_diff ! absolute difference between + ! fixed and floating point + ! sums +#ifdef _OPENMP + integer omp_get_max_threads + external omp_get_max_threads +#endif + character(len=*),parameter :: subname = '(ice_reprosum_calc)' + +!----------------------------------------------------------------------- + +! check whether should use ice_reprosum_ddpdd algorithm + use_ddpdd_sum = repro_sum_use_ddpdd + if ( present(ddpdd_sum) ) then + use_ddpdd_sum = ddpdd_sum + endif + +! check whether intrinsic-based algorithm will work on this system +! (requires floating point and integer bases to be the same) +! If not, always use ddpdd. + use_ddpdd_sum = use_ddpdd_sum .or. (radix(0._r8) /= radix(0_i8)) + +! initialize local statistics variables + gbl_max_red = 0 + repro_sum_fast = 0 + repro_sum_slow = 0 + repro_sum_both = 0 + nonrepro_sum = 0 + +! set MPI communicator + if ( present(commid) ) then + mpi_comm = commid + else +#ifdef SERIAL_REMOVE_MPI + mpi_comm = 0 +#else + mpi_comm = MPI_COMM_WORLD +#endif + endif + +! if (detailed_timing) then +! call xicex_timer_start('xicex_reprosum_prebarrier') +! call xicex_mpi_barrier(mpi_comm,subname) +! call xicex_timer_stop ('xicex_reprosum_prebarrier') +! endif + + if ( use_ddpdd_sum ) then + +! if (detailed_timing) call xicex_timer_start('ice_reprosum_ddpdd') + + call ice_reprosum_ddpdd(arr, arr_gsum, nsummands, dsummands, & + nflds, mpi_comm) + repro_sum_fast = 1 + +! if (detailed_timing) call xicex_timer_stop('ice_reprosum_ddpdd') + + else + +! if (detailed_timing) call xicex_timer_start('ice_reprosum_int') + +! get number of MPI tasks +#ifdef SERIAL_REMOVE_MPI + tasks = 1 + mype = 0 +#else + call mpi_comm_size(mpi_comm, tasks, ierr) + call mpi_comm_rank(mpi_comm, mype, ierr) +#endif + +! get number of OpenMP threads +#ifdef _OPENMP + omp_nthreads = omp_get_max_threads() +#else + omp_nthreads = 1 +#endif + +! see if have sufficient information to not require max/min allreduce + recompute = .true. + validate = .false. + if ( present(arr_gbl_max) .and. present(arr_max_levels) ) then + recompute = .false. + +! setting lower bound on max_level*nflds to be 64 to improve OpenMP +! performance for loopb in ice_reprosum_int + max_level = (64/nflds) + 1 + do ifld=1,nflds + if ((arr_gbl_max(ifld) .ge. 0.0_r8) .and. & + (arr_max_levels(ifld) > 0)) then + + arr_gmax_exp(ifld) = exponent(arr_gbl_max(ifld)) + if (max_level < arr_max_levels(ifld)) & + max_level = arr_max_levels(ifld) + + else + recompute = .true. + endif + enddo + + if (.not. recompute) then + +! determine maximum number of summands in local phases of the +! algorithm +! if (detailed_timing) call xicex_timer_start("repro_sum_allr_max") + if ( present(gbl_max_nsummands) ) then + if (gbl_max_nsummands < 1) then +#ifdef SERIAL_REMOVE_MPI + max_nsummands = nsummands +#else + call mpi_allreduce (nsummands, max_nsummands, 1, & + MPI_INTEGER, MPI_MAX, mpi_comm, ierr) +#endif + gbl_max_red = 1 + else + max_nsummands = gbl_max_nsummands + endif + else +#ifdef SERIAL_REMOVE_MPI + max_nsummands = nsummands +#else + call mpi_allreduce (nsummands, max_nsummands, 1, & + MPI_INTEGER, MPI_MAX, mpi_comm, ierr) +#endif + gbl_max_red = 1 + endif +! if (detailed_timing) call xicex_timer_stop("repro_sum_allr_max") + +! determine maximum shift. Shift needs to be small enough that summation +! does not exceed maximum number of digits in i8. + +! if requested, return max_nsummands before it is redefined + if ( present( gbl_max_nsummands_out) ) then + gbl_max_nsummands_out = max_nsummands + endif + +! summing within each thread first + max_nsummands = (max_nsummands/omp_nthreads) + 1 +! then over threads and tasks + max_nsummands = max(max_nsummands, tasks*omp_nthreads) + + xmax_nsummands = dble(max_nsummands) + arr_max_shift = digits(0_i8) - (exponent(xmax_nsummands) + 1) + if (arr_max_shift < 2) then + write(tmpstr,*) subname//' ERROR: number of summands too large for fixed precision algorithm' + call abort_ice(tmpstr,file=__FILE__,line=__LINE__) + endif + +! calculate sum + if (present(repro_sum_validate)) then + validate = repro_sum_validate + else + validate = .true. + endif + call ice_reprosum_int(arr, arr_gsum, nsummands, dsummands, & + nflds, arr_max_shift, arr_gmax_exp, & + arr_max_levels, max_level, validate, & + recompute, omp_nthreads, mpi_comm) + +! record statistics, etc. + repro_sum_fast = 1 + if (recompute) then + repro_sum_both = 1 + else +! if requested, return specified levels and upper bounds on maxima + if ( present(arr_max_levels_out) ) then + do ifld=1,nflds + arr_max_levels_out(ifld) = arr_max_levels(ifld) + enddo + endif + if ( present(arr_gbl_max_out) ) then + do ifld=1,nflds + arr_gbl_max_out(ifld) = arr_gbl_max(ifld) + enddo + endif + endif + endif + endif + +! do not have sufficient information; calculate global max/min and +! use to compute required number of levels + if (recompute) then + +! record statistic + repro_sum_slow = 1 + +! determine maximum and minimum (non-zero) summand values and +! maximum number of local summands + +! allocate thread-specific work space + allocate(arr_tlmax_exp(nflds,omp_nthreads)) + allocate(arr_tlmin_exp(nflds,omp_nthreads)) + allocate(isum_beg(omp_nthreads)) + allocate(isum_end(omp_nthreads)) + +! split summand index range over OpenMP threads + call split_indices(nsummands, omp_nthreads, isum_beg, isum_end) + +!$omp parallel do & +!$omp default(shared) & +!$omp private(ithread, ifld, isum, arr_exp, arr_exp_tlmin, arr_exp_tlmax) + do ithread=1,omp_nthreads +! if (detailed_timing) call xicex_timer_start('repro_sum_loopa') + do ifld=1,nflds + arr_exp_tlmin = MAXEXPONENT(1._r8) + arr_exp_tlmax = MINEXPONENT(1._r8) + do isum=isum_beg(ithread),isum_end(ithread) + if (arr(isum,ifld) .ne. 0.0_r8) then + arr_exp = exponent(arr(isum,ifld)) + arr_exp_tlmin = min(arr_exp,arr_exp_tlmin) + arr_exp_tlmax = max(arr_exp,arr_exp_tlmax) + endif + end do + arr_tlmin_exp(ifld,ithread) = arr_exp_tlmin + arr_tlmax_exp(ifld,ithread) = arr_exp_tlmax + end do +! if (detailed_timing) call xicex_timer_stop('repro_sum_loopa') + end do + + do ifld=1,nflds + arr_lmax_exp(ifld) = maxval(arr_tlmax_exp(ifld,:)) + arr_lmin_exp(ifld) = minval(arr_tlmin_exp(ifld,:)) + end do + deallocate(arr_tlmin_exp,arr_tlmax_exp,isum_beg,isum_end) + + arr_lextremes(0,:) = -nsummands + arr_lextremes(1:nflds,1) = -arr_lmax_exp(:) + arr_lextremes(1:nflds,2) = arr_lmin_exp(:) +! if (detailed_timing) call xicex_timer_start("repro_sum_allr_minmax") +#ifdef SERIAL_REMOVE_MPI + arr_gextremes = arr_lextremes +#else + call mpi_allreduce (arr_lextremes, arr_gextremes, 2*(nflds+1), & + MPI_INTEGER, MPI_MIN, mpi_comm, ierr) +#endif +! if (detailed_timing) call xicex_timer_stop("repro_sum_allr_minmax") + max_nsummands = -arr_gextremes(0,1) + arr_gmax_exp(:) = -arr_gextremes(1:nflds,1) + arr_gmin_exp(:) = arr_gextremes(1:nflds,2) + +! if a field is identically zero, arr_gmin_exp still equals MAXEXPONENT +! and arr_gmax_exp still equals MINEXPONENT. In this case, set +! arr_gmin_exp = arr_gmax_exp = MINEXPONENT + do ifld=1,nflds + arr_gmin_exp(ifld) = min(arr_gmax_exp(ifld),arr_gmin_exp(ifld)) + enddo + +! if requested, return upper bounds on observed maxima + if ( present(arr_gbl_max_out) ) then + do ifld=1,nflds + arr_gbl_max_out(ifld) = scale(1.0_r8,arr_gmax_exp(ifld)) + enddo + endif + +! if requested, return max_nsummands before it is redefined + if ( present( gbl_max_nsummands_out) ) then + gbl_max_nsummands_out = max_nsummands + endif + +! determine maximum shift (same as in previous branch, but with calculated +! max_nsummands). Shift needs to be small enough that summation does not +! exceed maximum number of digits in i8. + +! summing within each thread first + max_nsummands = (max_nsummands/omp_nthreads) + 1 +! then over threads and tasks + max_nsummands = max(max_nsummands, tasks*omp_nthreads) + + xmax_nsummands = dble(max_nsummands) + arr_max_shift = digits(0_i8) - (exponent(xmax_nsummands) + 1) + if (arr_max_shift < 2) then + write(tmpstr,*) subname//' ERROR: number of summands too large for fixed precision algorithm' + call abort_ice(tmpstr,file=__FILE__,line=__LINE__) + endif + +! determine maximum number of levels required for each field +! ((digits(0_i8) + (arr_gmax_exp(ifld)-arr_gmin_exp(ifld))) / arr_max_shift) +! + 1 because first truncation probably does not involve a maximal shift +! + 1 to guarantee that the integer division rounds up (not down) +! (setting lower bound on max_level*nflds to be 64 to improve OpenMP +! performance for loopb in ice_reprosum_int) + max_level = (64/nflds) + 1 + do ifld=1,nflds + max_levels(ifld) = 2 + & + ((digits(0_i8) + (arr_gmax_exp(ifld)-arr_gmin_exp(ifld))) & + / arr_max_shift) + if ( present(arr_max_levels) .and. (.not. validate) ) then +! if validate true, then computation with arr_max_levels failed +! previously + if ( arr_max_levels(ifld) > 0 ) then + max_levels(ifld) = & + min(arr_max_levels(ifld),max_levels(ifld)) + endif + endif + if (max_level < max_levels(ifld)) & + max_level = max_levels(ifld) + enddo + +! if requested, return calculated levels + if ( present(arr_max_levels_out) ) then + do ifld=1,nflds + arr_max_levels_out(ifld) = max_levels(ifld) + enddo + endif + +! calculate sum + validate = .false. + call ice_reprosum_int(arr, arr_gsum, nsummands, dsummands, nflds, & + arr_max_shift, arr_gmax_exp, max_levels, & + max_level, validate, recompute, & + omp_nthreads, mpi_comm) + + endif + +! if (detailed_timing) call xicex_timer_stop('ice_reprosum_int') + + endif + +! compare fixed and floating point results + if ( present(rel_diff) ) then + if (ice_reprosum_reldiffmax >= 0.0_r8) then + +! if (detailed_timing) then +! call xicex_timer_start('xicex_nonreprosum_prebarrier') +! call xicex_mpi_barrier(mpi_comm,subname) +! call xicex_timer_stop ('xicex_nonreprosum_prebarrier') +! endif + +! if (detailed_timing) call xicex_timer_start('nonrepro_sum') +! record statistic + nonrepro_sum = 1 +! compute nonreproducible sum + arr_lsum(:) = 0._r8 +!$omp parallel do & +!$omp default(shared) & +!$omp private(ifld, isum) + do ifld=1,nflds + do isum=1,nsummands + arr_lsum(ifld) = arr(isum,ifld) + arr_lsum(ifld) + end do + end do + +#ifdef SERIAL_REMOVE_MPI + arr_gsum_fast = arr_lsum +#else + call mpi_allreduce (arr_lsum, arr_gsum_fast, nflds, & + MPI_REAL8, MPI_SUM, mpi_comm, ierr) +#endif + +! if (detailed_timing) call xicex_timer_stop('nonrepro_sum') + +! determine differences +!$omp parallel do & +!$omp default(shared) & +!$omp private(ifld, abs_diff) + do ifld=1,nflds + abs_diff = abs(arr_gsum_fast(ifld)-arr_gsum(ifld)) + if (abs(arr_gsum(ifld)) > abs_diff) then + rel_diff(1,ifld) = abs_diff/abs(arr_gsum(ifld)) + else + rel_diff(1,ifld) = abs_diff + endif + rel_diff(2,ifld) = abs_diff + enddo + else + rel_diff(:,:) = 0.0_r8 + endif + endif + +! return statistics + if ( present(repro_sum_stats) ) then + repro_sum_stats(1) = repro_sum_stats(1) + repro_sum_fast + repro_sum_stats(2) = repro_sum_stats(2) + repro_sum_slow + repro_sum_stats(3) = repro_sum_stats(3) + repro_sum_both + repro_sum_stats(4) = repro_sum_stats(4) + nonrepro_sum + repro_sum_stats(5) = repro_sum_stats(5) + gbl_max_red + endif + + + end subroutine ice_reprosum_calc + +!======================================================================== +! +! Purpose: +!> Compute the global sum of each field in "arr" using the indicated +!> communicator with a reproducible yet scalable implementation based +!> on a fixed point algorithm. The accuracy of the fixed point algorithm +!> is controlled by the number of "levels" of integer expansion, the +!> maximum value of which is specified by max_level. +! +!---------------------------------------------------------------------- + + subroutine ice_reprosum_int (arr, arr_gsum, nsummands, dsummands, nflds, & + arr_max_shift, arr_gmax_exp, max_levels, & + max_level, validate, recompute, & + omp_nthreads, mpi_comm ) + +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: nsummands !< number of local summands + integer, intent(in) :: dsummands !< declared first dimension + integer, intent(in) :: nflds !< number of fields + integer, intent(in) :: arr_max_shift !< maximum safe exponent for + !< value < 1 (so that sum + !< does not overflow) + integer, intent(in) :: arr_gmax_exp(nflds) + !< exponents of global maxima + integer, intent(in) :: max_levels(nflds) + !< maximum number of levels + !< of integer expansion + integer, intent(in) :: max_level !< maximum value in + !< max_levels + integer, intent(in) :: omp_nthreads !< number of OpenMP threads + integer, intent(in) :: mpi_comm !< MPI subcommunicator + + real(r8), intent(in) :: arr(dsummands,nflds) + !< input array + + logical, intent(in):: validate + !< flag indicating that accuracy of solution generated from + !< arr_gmax_exp and max_levels should be tested + + logical, intent(out):: recompute + !< flag indicating that either the upper bounds are inaccurate, + !< or max_levels and arr_gmax_exp do not generate accurate + !< enough sums + + real(r8), intent(out):: arr_gsum(nflds) !< global means + +! Local workspace + + integer, parameter :: max_jlevel = & + 1 + (digits(0_i8)/digits(0.0_r8)) + + integer(i8) :: i8_arr_tlsum_level(0:max_level,nflds,omp_nthreads) + ! integer vector representing local + ! sum (per thread, per field) + integer(i8) :: i8_arr_lsum_level((max_level+3)*nflds) + ! integer vector representing local + ! sum + integer(i8) :: i8_arr_level ! integer part of summand for current + ! expansion level + integer(i8) :: i8_arr_gsum_level((max_level+3)*nflds) + ! integer vector representing global + ! sum + integer(i8) :: IX_8 ! integer representation of current + ! jlevels of X_8 ('part' of + ! i8_arr_gsum_level) + integer(i8) :: i8_sign ! sign global sum + integer(i8) :: i8_radix ! radix for i8 variables + + integer :: max_error(nflds,omp_nthreads) + ! accurate upper bound on data? + integer :: not_exact(nflds,omp_nthreads) + ! max_levels sufficient to + ! capture all digits? + integer :: isum_beg(omp_nthreads), isum_end(omp_nthreads) + ! range of summand indices for each + ! OpenMP thread + integer :: ifld, isum, ithread + ! loop variables + integer :: arr_exp ! exponent of summand + integer :: arr_shift ! exponent used to generate integer + ! for current expansion level + integer :: ilevel ! current integer expansion level + integer :: offset(nflds) ! beginning location in + ! i8_arr_{g,l}sum_level for integer + ! expansion of current ifld + integer :: voffset ! modification to offset used to + ! include validation metrics + integer :: ioffset ! offset(ifld) + integer :: jlevel ! number of floating point 'pieces' + ! extracted from a given i8 integer + integer :: ierr ! MPI error return + integer :: LX(max_jlevel) ! exponent of X_8 (see below) + integer :: veclth ! total length of i8_arr_lsum_level + integer :: sum_digits ! lower bound on number of significant + ! in integer expansion of sum + integer :: curr_exp ! exponent of partial sum during + ! reconstruction from integer vector + integer :: corr_exp ! exponent of current summand in + ! reconstruction from integer vector + + real(r8) :: arr_frac ! fraction of summand + real(r8) :: arr_remainder ! part of summand remaining after + ! current level of integer expansion + real(r8) :: X_8(max_jlevel) ! r8 vector representation of current + ! i8_arr_gsum_level + real(r8) :: RX_8 ! r8 representation of difference + ! between current i8_arr_gsum_level + ! and current jlevels of X_8 + ! (== IX_8). Also used in final + ! scaling step + + logical :: first ! flag used to indicate that just + ! beginning reconstruction of sum + ! from integer vector + + character(len=*),parameter :: subname = '(ice_reprosum_int)' + +!----------------------------------------------------------------------- +! Save radix of i8 variables in an i8 variable + i8_radix = radix(IX_8) + +! If validating upper bounds, reserve space for validation metrics +! In both cases, reserve an extra level for overflow from the top level + if (validate) then + voffset = 3 + else + voffset = 1 + endif + +! compute offsets for each field + offset(1) = voffset + do ifld=2,nflds + offset(ifld) = offset(ifld-1) & + + (max_levels(ifld-1) + voffset) + enddo + veclth = offset(nflds) + max_levels(nflds) + +! split summand index range over OpenMP threads + call split_indices(nsummands, omp_nthreads, isum_beg, isum_end) + +! convert local summands to vector of integers and sum +! (Using scale instead of set_exponent because arr_remainder may not be +! "normal" after level 1 calculation) + i8_arr_lsum_level(:) = 0_i8 + +!$omp parallel do & +!$omp default(shared) & +!$omp private(ithread, ifld, ioffset, isum, arr_frac, arr_exp, & +!$omp arr_shift, ilevel, i8_arr_level, arr_remainder, RX_8, IX_8) + do ithread=1,omp_nthreads +! if (detailed_timing) call xicex_timer_start('repro_sum_loopb') + do ifld=1,nflds + ioffset = offset(ifld) + + max_error(ifld,ithread) = 0 + not_exact(ifld,ithread) = 0 + + i8_arr_tlsum_level(:,ifld,ithread) = 0_i8 + do isum=isum_beg(ithread),isum_end(ithread) + arr_remainder = 0.0_r8 + + if (arr(isum,ifld) .ne. 0.0_r8) then + arr_exp = exponent(arr(isum,ifld)) + arr_frac = fraction(arr(isum,ifld)) + +! test that global maximum upper bound is an upper bound + if (arr_exp > arr_gmax_exp(ifld)) then + max_error(ifld,ithread) = 1 + exit + endif + +! calculate first shift + arr_shift = arr_max_shift - (arr_gmax_exp(ifld)-arr_exp) + +! determine first (probably) nonzero level (assuming initial fraction is +! 'normal' - algorithm still works if this is not true) +! NOTE: this is critical; scale will set to zero if min exponent is too small. + if (arr_shift < 1) then + ilevel = (1 + (arr_gmax_exp(ifld)-arr_exp))/arr_max_shift + arr_shift = ilevel*arr_max_shift - (arr_gmax_exp(ifld)-arr_exp) + + do while (arr_shift < 1) + arr_shift = arr_shift + arr_max_shift + ilevel = ilevel + 1 + enddo + else + ilevel = 1 + endif + + if (ilevel .le. max_levels(ifld)) then +! apply first shift/truncate, add it to the relevant running +! sum, and calculate the remainder. + arr_remainder = scale(arr_frac,arr_shift) + i8_arr_level = int(arr_remainder,i8) + i8_arr_tlsum_level(ilevel,ifld,ithread) = & + i8_arr_tlsum_level(ilevel,ifld,ithread) + i8_arr_level + arr_remainder = arr_remainder - i8_arr_level + +! while the remainder is non-zero, continue to shift, truncate, +! sum, and calculate new remainder + do while ((arr_remainder .ne. 0.0_r8) & + .and. (ilevel < max_levels(ifld))) + ilevel = ilevel + 1 + arr_remainder = scale(arr_remainder,arr_max_shift) + i8_arr_level = int(arr_remainder,i8) + i8_arr_tlsum_level(ilevel,ifld,ithread) = & + i8_arr_tlsum_level(ilevel,ifld,ithread) + i8_arr_level + arr_remainder = arr_remainder - i8_arr_level + enddo + + endif + endif + + if (arr_remainder .ne. 0.0_r8) then + not_exact(ifld,ithread) = 1 + endif + + enddo + +! postprocess integer vector to eliminate potential for overlap in the following +! sums over threads and processes: if value larger than or equal to +! (radix(IX_8)**arr_max_shift), add this 'overlap' to next larger integer in +! vector, resulting in nonoverlapping ranges for each component. Note that +! "ilevel-1==0" corresponds to an extra level used to guarantee that the sums +! over threads and processes do not overflow for ilevel==1. + do ilevel=max_levels(ifld),1,-1 + RX_8 = i8_arr_tlsum_level(ilevel,ifld,ithread) + IX_8 = int(scale(RX_8,-arr_max_shift),i8) + if (IX_8 .ne. 0_i8) then + i8_arr_tlsum_level(ilevel-1,ifld,ithread) = & + i8_arr_tlsum_level(ilevel-1,ifld,ithread) + IX_8 + IX_8 = IX_8*(i8_radix**arr_max_shift) + i8_arr_tlsum_level(ilevel,ifld,ithread) = & + i8_arr_tlsum_level(ilevel,ifld,ithread) - IX_8 + endif + enddo + enddo +! if (detailed_timing) call xicex_timer_stop('repro_sum_loopb') + enddo + +! sum contributions from different threads + do ifld=1,nflds + ioffset = offset(ifld) + do ithread = 1,omp_nthreads + do ilevel = 0,max_levels(ifld) + i8_arr_lsum_level(ioffset+ilevel) = & + i8_arr_lsum_level(ioffset+ilevel) & + + i8_arr_tlsum_level(ilevel,ifld,ithread) + enddo + enddo + enddo + +! record if upper bound was inaccurate or if level expansion stopped +! before full accuracy was achieved + if (validate) then + do ifld=1,nflds + ioffset = offset(ifld) + i8_arr_lsum_level(ioffset-voffset+1) = maxval(max_error(ifld,:)) + i8_arr_lsum_level(ioffset-voffset+2) = maxval(not_exact(ifld,:)) + enddo + endif + +! sum integer vector element-wise +#ifdef SERIAL_REMOVE_MPI + i8_arr_gsum_level = i8_arr_lsum_level +#else +#if ( defined noI8 ) + ! Workaround for when i8 is not supported. +! if (detailed_timing) call xicex_timer_start("repro_sum_allr_i4") + call mpi_allreduce (i8_arr_lsum_level, i8_arr_gsum_level, & + veclth, MPI_INTEGER, MPI_SUM, mpi_comm, ierr) +! if (detailed_timing) call xicex_timer_stop("repro_sum_allr_i4") +#else +! if (detailed_timing) call xicex_timer_start("repro_sum_allr_i8") + call mpi_allreduce (i8_arr_lsum_level, i8_arr_gsum_level, & + veclth, MPI_INTEGER8, MPI_SUM, mpi_comm, ierr) +! if (detailed_timing) call xicex_timer_stop("repro_sum_allr_i8") +#endif +#endif + +! Construct global sum from integer vector representation: +! 1) arr_max_shift is the shift applied to fraction(arr_gmax) . +! When shifting back, need to "add back in" true arr_gmax exponent. This was +! removed implicitly by working only with the fraction . +! 2) want to add levels into sum in reverse order (smallest to largest). However, +! even this can generate floating point rounding errors if signs of integers +! alternate. To avoid this, do some arithmetic with integer vectors so that all +! components have the same sign. This should keep relative difference between +! using different integer sizes (e.g. i8 and i4) to machine epsilon +! 3) assignment to X_8 will usually lose accuracy since maximum integer +! size is greater than the max number of 'digits' in r8 value (if xmax_nsummands +! correction not very large). Calculate remainder and add in first (since +! smaller). One correction is sufficient for r8 (53 digits) and i8 (63 digits). +! For r4 (24 digits) may need to correct twice. Code is written in a general +! fashion, to work no matter how many corrections are necessary (assuming +! max_jlevel parameter calculation is correct). + + recompute = .false. + do ifld=1,nflds + arr_gsum(ifld) = 0.0_r8 + ioffset = offset(ifld) + +! if validate is .true., test whether the summand upper bound +! was exceeded on any of the processes + if (validate) then + if (i8_arr_gsum_level(ioffset-voffset+1) .ne. 0_i8) then + recompute = .true. + endif + endif + + if (.not. recompute) then + +! preprocess integer vector: +! a) if value larger than or equal to (radix(IX_8)**arr_max_shift), add this 'overlap' +! to next larger integer in vector, resulting in nonoverlapping ranges for each +! component. Note that have "ilevel-1=0" level here as described above. + do ilevel=max_levels(ifld),1,-1 + RX_8 = i8_arr_gsum_level(ioffset+ilevel) + IX_8 = int(scale(RX_8,-arr_max_shift),i8) + if (IX_8 .ne. 0_i8) then + i8_arr_gsum_level(ioffset+ilevel-1) = i8_arr_gsum_level(ioffset+ilevel-1) & + + IX_8 + IX_8 = IX_8*(i8_radix**arr_max_shift) + i8_arr_gsum_level(ioffset+ilevel) = i8_arr_gsum_level(ioffset+ilevel) & + - IX_8 + endif + enddo +! b) subtract +/- 1 from larger and add +/- 1 to smaller when necessary +! so that all vector components have the same sign (eliminating loss +! of accuracy arising from difference of large values when +! reconstructing r8 sum from integer vector) + ilevel = 0 + do while ((i8_arr_gsum_level(ioffset+ilevel) .eq. 0_i8) & + .and. (ilevel < max_levels(ifld))) + ilevel = ilevel + 1 + enddo +! + if (ilevel < max_levels(ifld)) then + if (i8_arr_gsum_level(ioffset+ilevel) > 0_i8) then + i8_sign = 1_i8 + else + i8_sign = -1_i8 + endif + do jlevel=ilevel,max_levels(ifld)-1 + if (sign(1_i8,i8_arr_gsum_level(ioffset+jlevel)) & + .ne. sign(1_i8,i8_arr_gsum_level(ioffset+jlevel+1))) then + i8_arr_gsum_level(ioffset+jlevel) = i8_arr_gsum_level(ioffset+jlevel) & + - i8_sign + i8_arr_gsum_level(ioffset+jlevel+1) = i8_arr_gsum_level(ioffset+jlevel+1) & + + i8_sign*(i8_radix**arr_max_shift) + endif + enddo + endif + +! start with maximum shift, and work up to larger values + arr_shift = arr_gmax_exp(ifld) & + - max_levels(ifld)*arr_max_shift + curr_exp = 0 + first = .true. + do ilevel=max_levels(ifld),0,-1 + + if (i8_arr_gsum_level(ioffset+ilevel) .ne. 0_i8) then + jlevel = 1 + +! r8 representation of higher order bits in integer + X_8(jlevel) = i8_arr_gsum_level(ioffset+ilevel) + LX(jlevel) = exponent(X_8(jlevel)) + +! calculate remainder + IX_8 = int(X_8(jlevel),i8) + RX_8 = (i8_arr_gsum_level(ioffset+ilevel) - IX_8) + +! repeat using remainder + do while (RX_8 .ne. 0.0_r8) + jlevel = jlevel + 1 + X_8(jlevel) = RX_8 + LX(jlevel) = exponent(RX_8) + IX_8 = IX_8 + int(RX_8,i8) + RX_8 = (i8_arr_gsum_level(ioffset+ilevel) - IX_8) + enddo + +! add in contributions, smaller to larger, rescaling for each +! addition to guarantee that exponent of working summand is always +! larger than minexponent + do while (jlevel > 0) + if (first) then + curr_exp = LX(jlevel) + arr_shift + arr_gsum(ifld) = fraction(X_8(jlevel)) + first = .false. + else + corr_exp = curr_exp - (LX(jlevel) + arr_shift) + arr_gsum(ifld) = fraction(X_8(jlevel)) & + + scale(arr_gsum(ifld),corr_exp) + curr_exp = LX(jlevel) + arr_shift + endif + jlevel = jlevel - 1 + enddo + + endif + + arr_shift = arr_shift + arr_max_shift + enddo + +! apply final exponent correction, scaling first if exponent is too small +! to apply directly + corr_exp = curr_exp + exponent(arr_gsum(ifld)) + if (corr_exp .ge. MINEXPONENT(1._r8)) then + arr_gsum(ifld) = set_exponent(arr_gsum(ifld),corr_exp) + else + RX_8 = set_exponent(arr_gsum(ifld), & + corr_exp-MINEXPONENT(1._r8)) + arr_gsum(ifld) = scale(RX_8,MINEXPONENT(1._r8)) + endif + +! if validate is .true. and some precision lost, test whether 'too much' +! was lost, due to too loose an upper bound, too stringent a limit on number +! of levels of expansion, cancellation, .... Calculated by comparing lower +! bound on number of sigificant digits with number of digits in 1.0_r8 . + if (validate) then + if (i8_arr_gsum_level(ioffset-voffset+2) .ne. 0_i8) then + +! find first nonzero level and use exponent for this level, then assume all +! subsequent levels contribute arr_max_shift digits. + sum_digits = 0 + do ilevel=0,max_levels(ifld) + if (sum_digits .eq. 0) then + if (i8_arr_gsum_level(ioffset+ilevel) .ne. 0_i8) then + X_8(1) = i8_arr_gsum_level(ioffset+ilevel) + LX(1) = exponent(X_8(1)) + sum_digits = LX(1) + endif + else + sum_digits = sum_digits + arr_max_shift + endif + enddo + + if (sum_digits < digits(1.0_r8)) then + recompute = .true. + endif + endif + endif + + endif + + enddo + + + end subroutine ice_reprosum_int + +!======================================================================== +! +! Purpose: +!> Test whether distributed sum exceeds tolerance and print out a +!> warning message. +! +!---------------------------------------------------------------------- + + logical function ice_reprosum_tolExceeded (name, nflds, master, & + logunit, rel_diff ) +!---------------------------------------------------------------------- + +! Arguments + + character(len=*), intent(in) :: name !< distributed sum identifier + integer, intent(in) :: nflds !< number of fields + logical, intent(in) :: master !< process that will write + !< warning messages? + integer, optional, intent(in) :: logunit!< unit warning messages + !< written to + real(r8), intent(in) :: rel_diff(2,nflds) + !< relative and absolute + !< differences between fixed + !< and floating point sums + +! Local workspace + + integer :: llogunit ! local log unit + integer :: ifld ! field index + integer :: exceeds_limit ! number of fields whose + ! sum exceeds tolerance + real(r8) :: max_rel_diff ! maximum relative difference + integer :: max_rel_diff_idx ! field index for max. rel. diff. + real(r8) :: max_abs_diff ! maximum absolute difference + integer :: max_abs_diff_idx ! field index for max. abs. diff. + character(len=*),parameter :: subname = '(ice_reprosum_tolExceeded)' + +!----------------------------------------------------------------------- + + ice_reprosum_tolExceeded = .false. + if (ice_reprosum_reldiffmax < 0.0_r8) return + + if ( present(logunit) ) then + llogunit = logunit + else + llogunit = nu_diag + endif + + ! check that "fast" reproducible sum is accurate enough. + exceeds_limit = 0 + max_rel_diff = 0.0_r8 + max_abs_diff = 0.0_r8 + do ifld=1,nflds + if (rel_diff(1,ifld) > ice_reprosum_reldiffmax) then + exceeds_limit = exceeds_limit + 1 + if (rel_diff(1,ifld) > max_rel_diff) then + max_rel_diff = rel_diff(1,ifld) + max_rel_diff_idx = ifld + endif + if (rel_diff(2,ifld) > max_abs_diff) then + max_abs_diff = rel_diff(2,ifld) + max_abs_diff_idx = ifld + endif + endif + enddo + + if (exceeds_limit > 0) then + if (master) then + write(llogunit,*) subname,trim(name), & + ': difference in fixed and floating point sums ', & + ' exceeds tolerance in ', exceeds_limit, & + ' fields.' + write(llogunit,*) subname,' Maximum relative diff: (rel)', & + rel_diff(1,max_rel_diff_idx), ' (abs) ', & + rel_diff(2,max_rel_diff_idx) + write(llogunit,*) subname,' Maximum absolute diff: (rel)', & + rel_diff(1,max_abs_diff_idx), ' (abs) ', & + rel_diff(2,max_abs_diff_idx) + endif + ice_reprosum_tolExceeded = .true. + endif + + + end function ice_reprosum_tolExceeded + +!======================================================================== +! +! Purpose: +!> Compute the global sum of each field in "arr" using the indicated +!> communicator with a reproducible yet scalable implementation based +!> on He and Ding's implementation of the double-double algorithm. +! +!---------------------------------------------------------------------- + + subroutine ice_reprosum_ddpdd (arr, arr_gsum, nsummands, dsummands, & + nflds, mpi_comm ) +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: nsummands !< number of local summands + integer, intent(in) :: dsummands !< declared first dimension + integer, intent(in) :: nflds !< number of fields + real(r8), intent(in) :: arr(dsummands,nflds) + !< input array + integer, intent(in) :: mpi_comm !< MPI subcommunicator + + real(r8), intent(out):: arr_gsum(nflds) + !< global sums + + +! Local workspace + + integer :: old_cw ! for x86 processors, save + ! current arithmetic mode + integer :: ifld, isum ! loop variables + integer :: ierr ! MPI error return + + real(r8) :: e, t1, t2 ! temporaries + complex(r8) :: arr_lsum_dd(nflds) ! local sums (in double-double + ! format) + complex(r8) :: arr_gsum_dd(nflds) ! global sums (in double-double + ! format) + + integer, save :: mpi_sumdd + logical, save :: first_time = .true. + character(len=*),parameter :: subname = '(ice_reprosum_ddpdd)' + +!----------------------------------------------------------------------- + + call ice_shr_reprosumx86_fix_start (old_cw) + + if (first_time) then +#ifdef SERIAL_REMOVE_MPI + mpi_sumdd = 0 +#else + call mpi_op_create(ddpdd, .true., mpi_sumdd, ierr) +#endif + first_time = .false. + endif + + do ifld=1,nflds + arr_lsum_dd(ifld) = (0.0_r8,0.0_r8) + + do isum=1,nsummands + + ! Compute arr(isum,ifld) + arr_lsum_dd(ifld) using Knuth''s + ! trick. + t1 = arr(isum,ifld) + real(arr_lsum_dd(ifld)) + e = t1 - arr(isum,ifld) + t2 = ((real(arr_lsum_dd(ifld)) - e) & + + (arr(isum,ifld) - (t1 - e))) & + + aimag(arr_lsum_dd(ifld)) + + ! The result is t1 + t2, after normalization. + arr_lsum_dd(ifld) = cmplx ( t1 + t2, t2 - ((t1 + t2) - t1), r8 ) + enddo + + enddo + +#ifdef SERIAL_REMOVE_MPI + arr_gsum_dd = arr_lsum_dd +#else + call mpi_allreduce (arr_lsum_dd, arr_gsum_dd, nflds, & + MPI_COMPLEX16, mpi_sumdd, mpi_comm, ierr) +#endif + do ifld=1,nflds + arr_gsum(ifld) = real(arr_gsum_dd(ifld)) + enddo + + call ice_shr_reprosumx86_fix_end (old_cw) + + end subroutine ice_reprosum_ddpdd + +!----------------------------------------------------------------------- + + subroutine DDPDD (dda, ddb, len, itype) +!---------------------------------------------------------------------- +! +! Purpose: +! Modification of original codes written by David H. Bailey +! This subroutine computes ddb(i) = dda(i)+ddb(i) +! +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: len ! array length + complex(r8), intent(in) :: dda(len) ! input + complex(r8), intent(inout) :: ddb(len) ! result + integer, intent(in) :: itype ! unused + +! Local workspace + + real(r8) e, t1, t2 + integer i + character(len=*),parameter :: subname = '(ice_reprosum_mod:DDPDD)' + +!----------------------------------------------------------------------- + + do i = 1, len +! Compute dda + ddb using Knuth's trick. + t1 = real(dda(i)) + real(ddb(i)) + e = t1 - real(dda(i)) + t2 = ((real(ddb(i)) - e) + (real(dda(i)) - (t1 - e))) & + + aimag(dda(i)) + aimag(ddb(i)) + +! The result is t1 + t2, after normalization. + ddb(i) = cmplx ( t1 + t2, t2 - ((t1 + t2) - t1), r8 ) + enddo + + + end subroutine DDPDD + +!----------------------------------------------------------------------- + + subroutine split_indices(total,num_pieces,ibeg,iend) +!---------------------------------------------------------------------- +! +! Purpose: +! Split range into 'num_pieces' +! +!---------------------------------------------------------------------- + +! Arguments + + integer, intent(in) :: total + integer, intent(in) :: num_pieces + integer, intent(out) :: ibeg(num_pieces), iend(num_pieces) + +! Local workspace + + integer :: itmp1, itmp2, ioffset, i + character(len=*),parameter :: subname = '(ice_reprosum_mod:split_indices)' + +!----------------------------------------------------------------------- + + itmp1 = total/num_pieces + itmp2 = mod(total,num_pieces) + ioffset = 0 + do i=1,itmp2 + ibeg(i) = ioffset + 1 + iend(i) = ioffset + (itmp1+1) + ioffset = iend(i) + enddo + do i=itmp2+1,num_pieces + ibeg(i) = ioffset + 1 + if (ibeg(i) > total) then + iend(i) = ibeg(i) - 1 + else + iend(i) = ioffset + itmp1 + ioffset = iend(i) + endif + enddo + + end subroutine split_indices + +!======================================================================== + +end module ice_reprosum diff --git a/cicecore/cicedynB/infrastructure/ice_shr_reprosum86.c b/cicecore/cicedynB/infrastructure/ice_shr_reprosum86.c new file mode 100644 index 000000000..8bae08853 --- /dev/null +++ b/cicecore/cicedynB/infrastructure/ice_shr_reprosum86.c @@ -0,0 +1,83 @@ +/* + * src/x86.c + * + * This work was supported by the Director, Office of Science, Division + * of Mathematical, Information, and Computational Sciences of the + * U.S. Department of Energy under contract number DE-AC03-76SF00098. + * + * Copyright (c) 2000-2001 + * + * Contains functions to set and restore the round-to-double flag in the + * control word of a x86 FPU. + */ + +#define _NO_CHANGE 0 +#define _UPPER_CASE 1 +#define _ADD_UNDERSCORE 2 +#define _ADD_TWO_UNDERSCORES 3 + +#ifdef FORTRANUNDERSCORE +#define NAMING _ADD_UNDERSCORE +#endif + +#ifdef FORTRANDOUBLEUNDERSCORE +#define NAMING _ADD_TWO_UNDERSCORES +#endif + +#ifdef FORTRANCAPS +#define NAMING _UPPER_CASE +#endif + +#ifndef NAMING +#define NAMING _NO_CHANGE +#endif + +#if (NAMING == _ADD_UNDERSCORE) +#define ice_shr_reprosumx86_fix_start ice_shr_reprosumx86_fix_start_ +#define ice_shr_reprosumx86_fix_end ice_shr_reprosumx86_fix_end_ +#endif + +#if (NAMING == _ADD_TWO_UNDERSCORES) +#define ice_shr_reprosumx86_fix_start ice_shr_reprosumx86_fix_start__ +#define ice_shr_reprosumx86_fix_end ice_shr_reprosumx86_fix_end__ +#endif + +#if (NAMING == _UPPER_CASE) +#define ice_shr_reprosumx86_fix_start ICE_SHR_REPROSUMX86_FIX_START +#define ice_shr_reprosumx86_fix_end ICE_SHR_REPROSUMX86_FIX_END +#endif + +#ifdef x86 +#ifndef _FPU_GETCW +#define _FPU_GETCW(x) asm volatile ("fnstcw %0":"=m" (x)); +#endif + +#ifndef _FPU_SETCW +#define _FPU_SETCW(x) asm volatile ("fldcw %0": :"m" (x)); +#endif + +#ifndef _FPU_EXTENDED +#define _FPU_EXTENDED 0x0300 +#endif + +#ifndef _FPU_DOUBLE +#define _FPU_DOUBLE 0x0200 +#endif +#endif /* x86 */ + +void ice_shr_reprosumx86_fix_start(unsigned short *old_cw) { +#ifdef x86 + unsigned short new_cw; + + _FPU_GETCW(*old_cw); + new_cw = (*old_cw & ~_FPU_EXTENDED) | _FPU_DOUBLE; + _FPU_SETCW(new_cw); +#endif +} + +void ice_shr_reprosumx86_fix_end(unsigned short *old_cw) { +#ifdef x86 + _FPU_SETCW(*old_cw); +#endif +} + diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel index 6d61f4cb5..1060febd7 100755 --- a/configuration/scripts/machines/env.phase3_intel +++ b/configuration/scripts/machines/env.phase3_intel @@ -1,4 +1,17 @@ -#!/bin/csh -f +#!/bin/csh -f --login + +source /etc/profile.d/lmod.csh + +module purge +module load ips/18.0.1.163 +module load impi/18.0.1 +module load NetCDF/4.5.0 +module load ESMF/7_1_0r +module list +#module load intel/18.1.163 Works, at least w. nc 4.4.0, +# module load esmf +# module load wgrib wgrib2 +# setenv ICE_MACHINE_ENVNAME phase3 setenv ICE_MACHINE_COMPILER intel diff --git a/configuration/scripts/options/set_nml.reprosum b/configuration/scripts/options/set_nml.reprosum new file mode 100644 index 000000000..6526f0d8c --- /dev/null +++ b/configuration/scripts/options/set_nml.reprosum @@ -0,0 +1 @@ +bfbflag = 'reprosum' diff --git a/configuration/scripts/tests/comparelog.csh b/configuration/scripts/tests/comparelog.csh new file mode 100755 index 000000000..8c1ff3a3c --- /dev/null +++ b/configuration/scripts/tests/comparelog.csh @@ -0,0 +1,70 @@ +#!/bin/csh -f + +# Compare prognostic output in two log files +#----------------------------------------------------------- + +# usage: comparelog.csh base_file test_file +# does diff of two files +# +# Return Codes (depends on quality of error checking) +# 0 = pass +# 1 = fail +# 2 = missing data +# 9 = error + +set filearg = 0 +if ( $#argv == 2 ) then + set filearg = 1 + set base_data = $argv[1] + set test_data = $argv[2] +else + echo "Error in ${0}" + echo "Usage: ${0} " + echo " does diff of two files" + exit 9 +endif + +set failure = 0 +set base_out = "comparelog_base_out_file.log" +set test_out = "comparelog_test_out_file.log" + +if ($filearg == 1) then + echo "base_data: $base_data" + echo "test_data: $test_data" + if ( -f ${base_data} && -f ${test_data}) then + if (${base_data} == ${test_data}) then + set failure = 9 + echo " input data are same" + else + + touch ${base_out} + cat ${base_data} | grep -A 99999999 istep1: | grep -e istep1: -e = | grep -iv "min, max, sum" >&! ${base_out} + touch ${test_out} + cat ${test_data} | grep -A 99999999 istep1: | grep -e istep1: -e = | grep -iv "min, max, sum" >&! ${test_out} + + set basenum = `cat ${base_out} | wc -l` + set testnum = `cat ${base_out} | wc -l` + set filediff = `diff -w ${base_out} ${test_out} | wc -l` + + if (${basenum} > 0 && ${testnum} > 0) then + if ($filediff == 0) then + echo " compare OK" + else + set failure = 1 + echo " compare FAIL" + endif + else + set failure = 9 + echo " compare on no output" + endif + rm ${base_out} + rm ${test_out} + endif + else + set failure = 2 + echo " missing data" + endif +endif + +exit ${failure} + diff --git a/configuration/scripts/tests/first_suite.ts b/configuration/scripts/tests/first_suite.ts new file mode 100644 index 000000000..b06ee7e0b --- /dev/null +++ b/configuration/scripts/tests/first_suite.ts @@ -0,0 +1,5 @@ +# Test Grid PEs Sets BFB-compare +smoke gx3 8x2 diag1,run5day +restart gx3 4x2x25x29x4 dslenderX2 +logbfb gx3 4x2x25x29x4 dslenderX2,diag1,reprosum +smoke gx3 1x2 run2day diff --git a/configuration/scripts/tests/reprosum_suite.ts b/configuration/scripts/tests/reprosum_suite.ts new file mode 100644 index 000000000..34cf51a80 --- /dev/null +++ b/configuration/scripts/tests/reprosum_suite.ts @@ -0,0 +1,11 @@ +# Test Grid PEs Sets BFB-compare +logbfb gx3 4x2x25x29x4 dslenderX2,diag1,reprosum +#logbfb gx3 4x2x25x29x4 dslenderX2,diag1 +sleep 60 +logbfb gx3 1x1x50x58x4 droundrobin,diag1,thread,maskhalo,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +logbfb gx3 4x1x25x116x1 dslenderX1,diag1,thread,maskhalo,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +logbfb gx3 1x20x5x29x80 dsectrobin,diag1,short,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +logbfb gx3 8x2x8x10x20 droundrobin,diag1,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +logbfb gx3 6x2x50x58x1 droundrobin,diag1,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +logbfb gx3 6x2x4x29x18 dspacecurve,diag1,maskhalo,reprosum logbfb_gx3_4x2x25x29x4_diag1_dslenderX2_reprosum +#logbfb gx3 8x2x8x10x20 droundrobin,diag1 logbfb_gx3_4x2x25x29x4_diag1_dslenderX2 diff --git a/configuration/scripts/tests/test_logbfb.script b/configuration/scripts/tests/test_logbfb.script new file mode 100644 index 000000000..fbce5d918 --- /dev/null +++ b/configuration/scripts/tests/test_logbfb.script @@ -0,0 +1,36 @@ + +#---------------------------------------------------- +# Run the CICE model +# This is identical to a smoke test, but triggers bfbcompare with log files instead of restarts +# cice.run returns -1 if run did not complete successfully + +mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev +cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output +rm -f ${ICE_CASEDIR}/test_output.prev +echo "RUN ${ICE_TESTNAME} run " >> ${ICE_CASEDIR}/test_output + +./cice.run +set res="$status" + +set log_file = `ls -t1 ${ICE_RUNDIR}/cice.runlog* | head -1` +set ttimeloop = `grep TimeLoop ${log_file} | grep Timer | cut -c 22-32` +set tdynamics = `grep Dynamics ${log_file} | grep Timer | cut -c 22-32` +set tcolumn = `grep Column ${log_file} | grep Timer | cut -c 22-32` +if (${ttimeloop} == "") set ttimeloop = -1 +if (${tdynamics} == "") set tdynamics = -1 +if (${tcolumn} == "") set tcolumn = -1 + +mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev +cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} run" >! ${ICE_CASEDIR}/test_output +mv -f ${ICE_CASEDIR}/test_output ${ICE_CASEDIR}/test_output.prev +cat ${ICE_CASEDIR}/test_output.prev | grep -iv "${ICE_TESTNAME} test" >! ${ICE_CASEDIR}/test_output +rm -f ${ICE_CASEDIR}/test_output.prev + +set grade = FAIL +if ( $res == 0 ) then + set grade = PASS +endif + +echo "$grade ${ICE_TESTNAME} run ${ttimeloop} ${tdynamics} ${tcolumn}" >> ${ICE_CASEDIR}/test_output +echo "$grade ${ICE_TESTNAME} test " >> ${ICE_CASEDIR}/test_output + From f606b2289c9f7cfc01c69833e8f2fd52fa92febb Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Wed, 1 May 2019 13:37:58 +0000 Subject: [PATCH 14/54] update configs manually from master --- .../scripts/machines/Macros.badger_intel | 23 ++++++----- .../scripts/machines/Macros.cesium_intel | 34 ++++++++--------- .../scripts/machines/Macros.cheyenne_intel | 32 ++++++---------- .../scripts/machines/Macros.conrad_cray | 23 ++++++----- .../scripts/machines/Macros.conrad_gnu | 22 +++++------ .../scripts/machines/Macros.conrad_intel | 23 ++++++----- .../scripts/machines/Macros.conrad_pgi | 23 ++++++----- .../scripts/machines/Macros.cori_intel | 23 ++++++----- .../scripts/machines/Macros.fram_intel | 29 ++++++-------- .../scripts/machines/Macros.gaffney_gnu | 23 ++++++----- .../scripts/machines/Macros.gaffney_intel | 23 ++++++----- .../scripts/machines/Macros.gordon_cray | 28 +++++++------- .../scripts/machines/Macros.gordon_gnu | 22 +++++------ .../scripts/machines/Macros.gordon_intel | 23 ++++++----- .../scripts/machines/Macros.gordon_pgi | 23 ++++++----- .../scripts/machines/Macros.hobart_intel | 38 ++++++------------- .../scripts/machines/Macros.hobart_nag | 25 ++++++------ .../scripts/machines/Macros.koehr_intel | 23 ++++++----- .../scripts/machines/Macros.loft_gnu | 19 ++++++---- .../scripts/machines/Macros.onyx_cray | 23 ++++++----- .../scripts/machines/Macros.onyx_gnu | 22 +++++------ .../scripts/machines/Macros.onyx_intel | 23 ++++++----- .../scripts/machines/Macros.phase3_intel | 3 +- .../scripts/machines/Macros.testmachine_intel | 32 ++++++---------- .../scripts/machines/Macros.thunder_intel | 24 ++++++------ .../scripts/machines/Macros.travisCI_gnu | 25 +++++++----- .../scripts/machines/env.cesium_intel | 12 ++++-- .../scripts/machines/env.cheyenne_intel | 2 +- configuration/scripts/machines/env.fram_intel | 1 + .../scripts/machines/env.phase3_intel | 7 ++-- 30 files changed, 306 insertions(+), 347 deletions(-) diff --git a/configuration/scripts/machines/Macros.badger_intel b/configuration/scripts/machines/Macros.badger_intel index 54cdb1493..2873f97fb 100644 --- a/configuration/scripts/machines/Macros.badger_intel +++ b/configuration/scripts/machines/Macros.badger_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -17,16 +17,19 @@ else FFLAGS += -O2 endif +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 + ifeq ($(ICE_COMMDIR), mpi) - FC := mpif90 + FC := $(MPIFC) + CC := $(MPICC) else - FC := ifort + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= mpicc - -MPIFC:= mpif90 -LD:= $(MPIFC) +LD:= $(FC) # set in Macros file NETCDF_PATH := /usr/projects/climate/SHARED_CLIMATE/software/conejo/netcdf/3.6.3/intel-13.0.1 @@ -49,10 +52,6 @@ else SLIBS := endif -SCC:= icc - -SFC:= ifort - ifeq ($(ICE_THREADED), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp diff --git a/configuration/scripts/machines/Macros.cesium_intel b/configuration/scripts/machines/Macros.cesium_intel index 454ba859a..aca33b9fd 100644 --- a/configuration/scripts/machines/Macros.cesium_intel +++ b/configuration/scripts/machines/Macros.cesium_intel @@ -5,7 +5,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${CICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise #-xHost @@ -16,23 +16,27 @@ FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceb FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays +# -heap-arrays 1024 else FFLAGS += -O2 endif -ifeq ($(CICE_COMMDIR), mpi) - FC := s.f90 -mpi +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) else - FC := s.f90 + FC := $(SFC) + CC := $(SCC) endif +LD:= $(FC) -MPICC:= s.cc -mpi - -MPIFC:= s.f90 -mpi -LD:= $(MPIFC) - -NETCDF_PATH := $(NETCDF) +NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs @@ -48,20 +52,12 @@ LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -SCC:= s.cc - -SFC:= s.f90 - ifeq ($(compile_threaded), true) LDFLAGS += -openmp CFLAGS += -openmp FFLAGS += -openmp endif -ifeq ($(DITTO), yes) - CPPDEFS := $(CPPDEFS) -DREPRODUCIBLE -endif - ### if using parallel I/O, load all 3 libraries. PIO must be first! ifeq ($(IO_TYPE), pio) PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib diff --git a/configuration/scripts/machines/Macros.cheyenne_intel b/configuration/scripts/machines/Macros.cheyenne_intel index 3709160fb..e2b80d023 100755 --- a/configuration/scripts/machines/Macros.cheyenne_intel +++ b/configuration/scripts/machines/Macros.cheyenne_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -17,16 +17,19 @@ else FFLAGS += -O2 endif +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 + ifeq ($(ICE_COMMDIR), mpi) - FC := mpif90 + FC := $(MPIFC) + CC := $(MPICC) else - FC := ifort + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= mpicc - -MPIFC:= mpif90 -LD:= $(MPIFC) +LD:= $(FC) NETCDF_PATH := $(NETCDF) @@ -44,29 +47,16 @@ LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= icc - -SFC:= ifort - ifeq ($(compile_threaded), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp FFLAGS += -qopenmp endif -ifeq ($(DITTO), yes) - CPPDEFS := $(CPPDEFS) -DREPRODUCIBLE -endif - ### if using parallel I/O, load all 3 libraries. PIO must be first! ifeq ($(IO_TYPE), pio) PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof - - CPPDEFS := $(CPPDEFS) -Dncdf endif -ifeq ($(IO_TYPE), netcdf) - CPPDEFS := $(CPPDEFS) -Dncdf -endif diff --git a/configuration/scripts/machines/Macros.conrad_cray b/configuration/scripts/machines/Macros.conrad_cray index b626c2be2..f76652bed 100644 --- a/configuration/scripts/machines/Macros.conrad_cray +++ b/configuration/scripts/machines/Macros.conrad_cray @@ -3,7 +3,7 @@ #============================================================================== CPP := ftn -e P -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -h fp0 FIXEDFLAGS := -132 @@ -19,16 +19,19 @@ else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -46,10 +49,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= cc - -SFC:= ftn - ifeq ($(ICE_THREADED), false) LDFLAGS += -hnoomp CFLAGS += -hnoomp diff --git a/configuration/scripts/machines/Macros.conrad_gnu b/configuration/scripts/machines/Macros.conrad_gnu index 285ab0c42..2f54d753e 100644 --- a/configuration/scripts/machines/Macros.conrad_gnu +++ b/configuration/scripts/machines/Macros.conrad_gnu @@ -3,7 +3,7 @@ #============================================================================== CPP := ftn -E -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 FIXEDFLAGS := -ffixed-line-length-132 @@ -17,16 +17,19 @@ else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -44,9 +47,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= gcc -SFC:= gfortran - ifeq ($(ICE_THREADED), true) LDFLAGS += -fopenmp CFLAGS += -fopenmp diff --git a/configuration/scripts/machines/Macros.conrad_intel b/configuration/scripts/machines/Macros.conrad_intel index c0b48821f..a5cb37a5c 100644 --- a/configuration/scripts/machines/Macros.conrad_intel +++ b/configuration/scripts/machines/Macros.conrad_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -18,16 +18,19 @@ else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -45,10 +48,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= icc - -SFC:= ifort - ifeq ($(ICE_THREADED), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp diff --git a/configuration/scripts/machines/Macros.conrad_pgi b/configuration/scripts/machines/Macros.conrad_pgi index be6fb1ef2..29bcda360 100644 --- a/configuration/scripts/machines/Macros.conrad_pgi +++ b/configuration/scripts/machines/Macros.conrad_pgi @@ -3,7 +3,7 @@ #============================================================================== CPP := pgcc -E -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS} CFLAGS := -c -O2 -Kieee FIXEDFLAGS := -Mextend @@ -17,16 +17,19 @@ else FFLAGS += -O -g endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -44,10 +47,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= cc - -SFC:= ftn - ifeq ($(ICE_THREADED), true) LDFLAGS += -mp CFLAGS += -mp diff --git a/configuration/scripts/machines/Macros.cori_intel b/configuration/scripts/machines/Macros.cori_intel index cd788649f..8e006e4cc 100644 --- a/configuration/scripts/machines/Macros.cori_intel +++ b/configuration/scripts/machines/Macros.cori_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -17,16 +17,19 @@ else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -44,10 +47,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= icc - -SFC:= ifort - ifeq ($(ICE_THREADED), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp diff --git a/configuration/scripts/machines/Macros.fram_intel b/configuration/scripts/machines/Macros.fram_intel index 5181a9e18..947c7545c 100755 --- a/configuration/scripts/machines/Macros.fram_intel +++ b/configuration/scripts/machines/Macros.fram_intel @@ -5,7 +5,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${CICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise #-xHost @@ -15,16 +15,19 @@ FFLAGS := -O2 -fp-model precise -convert big_endian -assume byterecl -ftz -t #-xHost FFLAGS_NOOPT:= -O0 -ifeq ($(CICE_COMMDIR), mpi) - FC := s.f90 -mpi +SCC := s.cc +SFC := s.f90 +MPICC := s.cc -mpi +MPIFC := s.f90 -mpi + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) else - FC := s.f90 + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= s.cc -mpi - -MPIFC:= s.f90 -mpi -LD:= $(MPIFC) +LD:= $(FC) NETCDF_PATH := $(NETCDF) @@ -42,20 +45,12 @@ LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -SCC:= s.cc - -SFC:= s.f90 - ifeq ($(compile_threaded), true) LDFLAGS += -openmp CFLAGS += -openmp FFLAGS += -openmp endif -ifeq ($(DITTO), yes) - CPPDEFS := $(CPPDEFS) -DREPRODUCIBLE -endif - ### if using parallel I/O, load all 3 libraries. PIO must be first! ifeq ($(IO_TYPE), pio) PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib diff --git a/configuration/scripts/machines/Macros.gaffney_gnu b/configuration/scripts/machines/Macros.gaffney_gnu index c2aaf8ba4..b2f178247 100644 --- a/configuration/scripts/machines/Macros.gaffney_gnu +++ b/configuration/scripts/machines/Macros.gaffney_gnu @@ -3,7 +3,7 @@ #============================================================================== CPP := ftn -E -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 FIXEDFLAGS := -ffixed-line-length-132 @@ -17,17 +17,20 @@ else FFLAGS += -O2 endif +SCC := gcc +SFC := gfortran +MPICC := gcc +MPIFC := gfortran + ifeq ($(ICE_COMMDIR), mpi) - FC := mpif90 + FC := $(MPIFC) + CC := $(MPICC) LDFLAGS += -lmpi else - FC := gfortran + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= mpicc -MPIFC:= mpif90 -#LD:= $(MPIFC) -LD:= gfortran +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -45,10 +48,6 @@ LIB_NETCDF := $(NETCDF_PATH)/lib #LIB_MPI := $(IMPILIBDIR) SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= icc - -SFC:= ifort - ifeq ($(ICE_THREADED), true) LDFLAGS += -fopenmp CFLAGS += -fopenmp diff --git a/configuration/scripts/machines/Macros.gaffney_intel b/configuration/scripts/machines/Macros.gaffney_intel index 465d7d0e1..bfad4b54c 100644 --- a/configuration/scripts/machines/Macros.gaffney_intel +++ b/configuration/scripts/machines/Macros.gaffney_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -18,17 +18,20 @@ else FFLAGS += -O2 endif +SCC := icc +SFC := ifort +MPICC := icc +MPIFC := ifort + ifeq ($(ICE_COMMDIR), mpi) - FC := ifort + FC := $(MPIFC) + CC := $(MPICC) LDFLAGS += -lmpi else - FC := ifort + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= icc - -MPIFC:= ifort -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -46,10 +49,6 @@ LIB_NETCDF := $(NETCDF_PATH)/lib #LIB_MPI := $(IMPILIBDIR) SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= icc - -SFC:= ifort - ifeq ($(ICE_THREADED), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp diff --git a/configuration/scripts/machines/Macros.gordon_cray b/configuration/scripts/machines/Macros.gordon_cray index fb46c2641..4f9c43da1 100644 --- a/configuration/scripts/machines/Macros.gordon_cray +++ b/configuration/scripts/machines/Macros.gordon_cray @@ -1,33 +1,37 @@ #============================================================================== -# Macros file for NAVYDSRC gordon, intel compiler +# Macros file for NAVYDSRC gordon, cray compiler #============================================================================== CPP := ftn -e P -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -h fp0 FIXEDFLAGS := -132 FREEFLAGS := FFLAGS := -h fp0 -h byteswapio FFLAGS_NOOPT:= -O0 -LDFLAGS = -h byteswapio +LDFLAGS := -h byteswapio ifeq ($(ICE_BLDDEBUG), true) FFLAGS += -O0 -g -Rbcdps +# FFLAGS += -O0 -g -Rbcdps -ei else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -45,10 +49,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= cc - -SFC:= ftn - ifeq ($(ICE_THREADED), false) LDFLAGS += -hnoomp CFLAGS += -hnoomp diff --git a/configuration/scripts/machines/Macros.gordon_gnu b/configuration/scripts/machines/Macros.gordon_gnu index c23be2b76..131f539c1 100644 --- a/configuration/scripts/machines/Macros.gordon_gnu +++ b/configuration/scripts/machines/Macros.gordon_gnu @@ -3,7 +3,7 @@ #============================================================================== CPP := ftn -E -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 FIXEDFLAGS := -ffixed-line-length-132 @@ -17,16 +17,19 @@ else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -44,9 +47,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= gcc -SFC:= gfortran - ifeq ($(ICE_THREADED), true) LDFLAGS += -fopenmp CFLAGS += -fopenmp diff --git a/configuration/scripts/machines/Macros.gordon_intel b/configuration/scripts/machines/Macros.gordon_intel index d1ed9cda7..73214ec26 100644 --- a/configuration/scripts/machines/Macros.gordon_intel +++ b/configuration/scripts/machines/Macros.gordon_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -17,16 +17,19 @@ else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -44,10 +47,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= icc - -SFC:= ifort - ifeq ($(ICE_THREADED), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp diff --git a/configuration/scripts/machines/Macros.gordon_pgi b/configuration/scripts/machines/Macros.gordon_pgi index 1403cf13a..45d03c64f 100644 --- a/configuration/scripts/machines/Macros.gordon_pgi +++ b/configuration/scripts/machines/Macros.gordon_pgi @@ -3,7 +3,7 @@ #============================================================================== CPP := pgcc -Mcpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS} CFLAGS := -c -O2 -Kieee FIXEDFLAGS := -Mextend @@ -17,16 +17,19 @@ else FFLAGS += -O -g endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -44,10 +47,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= cc - -SFC:= ftn - ifeq ($(ICE_THREADED), true) LDFLAGS += -mp CFLAGS += -mp diff --git a/configuration/scripts/machines/Macros.hobart_intel b/configuration/scripts/machines/Macros.hobart_intel index f4176eaf1..7e4984a16 100755 --- a/configuration/scripts/machines/Macros.hobart_intel +++ b/configuration/scripts/machines/Macros.hobart_intel @@ -4,8 +4,8 @@ CPP := /usr/bin/cpp CPPFLAGS := $(CFLAGS) -lifcore -CPPDEFS := $(CPPDEFS) -DFORTRANUNDERSCORE -DCPRINTEL -CFLAGS := -qno-opt-dynamic-align -fp-model precise -std=gnu99 +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -qno-opt-dynamic-align -fp-model precise -std=gnu99 FIXEDFLAGS := -fixed -132 FREEFLAGS := -free @@ -18,16 +18,19 @@ ifeq ($(ICE_BLDDEBUG), true) FFLAGS := -qno-opt-dynamic-align -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created endif +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 + ifeq ($(ICE_COMMDIR), mpi) - FC := mpif90 + FC := $(MPIFC) + CC := $(MPICC) else - FC := ifort + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= mpicc - -MPIFC:= mpif90 -LD:= $(MPIFC) +LD:= $(FC) INCLDIR := -I$(NETCDF_PATH)/include -I$(MPI_PATH)/include @@ -37,27 +40,10 @@ LIB_MPI := $(IMPILIBDIR) SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L/usr/lib64 -llapack -mkl=cluster -SCC:= icc - -SFC:= ifort - -ifeq ($(DITTO), yes) - CPPDEFS := $(CPPDEFS) -DREPRODUCIBLE -endif - -ifeq ($(IO_TYPE), netcdf) - CPPDEFS := $(CPPDEFS) -Dncdf -endif - ## if using parallel I/O, load all 3 libraries. PIO must be first! ifeq ($(IO_TYPE), pio) PIO_PATH:= INCLDIR += -I SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS - - CPPDEFS := $(CPPDEFS) -Dncdf endif -ifeq ($(IO_TYPE), netcdf) - CPPDEFS := $(CPPDEFS) -Dncdf -endif diff --git a/configuration/scripts/machines/Macros.hobart_nag b/configuration/scripts/machines/Macros.hobart_nag index a7f973a69..02a4bbe65 100755 --- a/configuration/scripts/machines/Macros.hobart_nag +++ b/configuration/scripts/machines/Macros.hobart_nag @@ -4,8 +4,8 @@ CPP := /usr/bin/cpp CPPFLAGS := -P -traditional -CPPDEFS := -DFORTRANUNDERSCORE -DNO_CRAY_POINTERS -DNO_SHR_VMATH -DCPRNAG $(ICE_CPPDEFS) -CFLAGS := -std=gnu99 +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 $(ICE_CPPDEFS) +CFLAGS := -c -std=gnu99 FIXEDFLAGS := -fixed FREEFLAGS := -free @@ -18,16 +18,19 @@ ifeq ($(ICE_BLDDEBUG), true) FFLAGS := -Wp,-macro=no_com -convert=BIG_ENDIAN -wmismatch=mpi_bcast,mpi_isend,mpi_irecv,mpi_send,mpi_recv,mpi_allreduce -gline -C=all -g -time -f2003 -ieee=stop endif +SCC := cc +SFC := nagfor +MPICC := mpicc +MPIFC := mpif90 + ifeq ($(ICE_COMMDIR), mpi) - FC := mpif90 + FC := $(MPIFC) + CC := $(MPICC) else - FC := nagfor + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= mpicc - -MPIFC:= mpif90 -LD:= $(MPIFC) +LD:= $(FC) INCLDIR := -I$(NETCDF_PATH)/include -I$(MPI_PATH)/include @@ -37,10 +40,6 @@ LIB_MPI := $(IMPILIBDIR) SLIBS := -L/usr/local/nag-6.2/lib/NAG_Fortran -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L/usr/lib64 -llapack -lblas -SCC:= nagcc - -SFC:= nagfor - ## if using parallel I/O, load all 3 libraries. PIO must be first! ifeq ($(IO_TYPE), pio) PIO_PATH:= diff --git a/configuration/scripts/machines/Macros.koehr_intel b/configuration/scripts/machines/Macros.koehr_intel index 90482b618..d77e90768 100644 --- a/configuration/scripts/machines/Macros.koehr_intel +++ b/configuration/scripts/machines/Macros.koehr_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -18,17 +18,20 @@ else FFLAGS += -O2 endif +SCC := icc +SFC := ifort +MPICC := icc +MPIFC := ifort + ifeq ($(ICE_COMMDIR), mpi) - FC := ifort + FC := $(MPIFC) + CC := $(MPICC) LDFLAGS += -lmpi else - FC := ifort + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= icc - -MPIFC:= ifort -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -46,10 +49,6 @@ LIB_NETCDF := $(NETCDF_PATH)/lib #LIB_MPI := $(IMPILIBDIR) SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= icc - -SFC:= ifort - ifeq ($(ICE_THREADED), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp diff --git a/configuration/scripts/machines/Macros.loft_gnu b/configuration/scripts/machines/Macros.loft_gnu index e5e38b810..12158350e 100644 --- a/configuration/scripts/machines/Macros.loft_gnu +++ b/configuration/scripts/machines/Macros.loft_gnu @@ -4,7 +4,7 @@ CPP := /usr/bin/cpp #CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -27,11 +27,18 @@ else FFLAGS += -O2 endif - FC := gfortran +SCC := gcc +SFC := gfortran +MPICC := gcc +MPIFC := gfortran -MPICC:= - -MPIFC:= +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif LD:= $(FC) ifeq ($(ICE_IOTYPE), netcdf) @@ -45,8 +52,6 @@ else endif LIB_MPI := -SCC:= -SFC:= ifeq ($(ICE_THREADED), true) LDFLAGS += -openmp diff --git a/configuration/scripts/machines/Macros.onyx_cray b/configuration/scripts/machines/Macros.onyx_cray index c0d14eb5a..aa185e2e5 100644 --- a/configuration/scripts/machines/Macros.onyx_cray +++ b/configuration/scripts/machines/Macros.onyx_cray @@ -3,7 +3,7 @@ #============================================================================== CPP := ftn -e P -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -h fp0 FIXEDFLAGS := -132 @@ -18,16 +18,19 @@ else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -45,10 +48,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= cc - -SFC:= ftn - ifeq ($(ICE_THREADED), false) LDFLAGS += -hnoomp CFLAGS += -hnoomp diff --git a/configuration/scripts/machines/Macros.onyx_gnu b/configuration/scripts/machines/Macros.onyx_gnu index b87878d1e..14784e625 100644 --- a/configuration/scripts/machines/Macros.onyx_gnu +++ b/configuration/scripts/machines/Macros.onyx_gnu @@ -3,7 +3,7 @@ #============================================================================== CPP := ftn -E -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 FIXEDFLAGS := -ffixed-line-length-132 @@ -17,16 +17,19 @@ else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -44,9 +47,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= gcc -SFC:= gfortran - ifeq ($(ICE_THREADED), true) LDFLAGS += -fopenmp CFLAGS += -fopenmp diff --git a/configuration/scripts/machines/Macros.onyx_intel b/configuration/scripts/machines/Macros.onyx_intel index 900b46de7..b7f685ac1 100644 --- a/configuration/scripts/machines/Macros.onyx_intel +++ b/configuration/scripts/machines/Macros.onyx_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -17,16 +17,19 @@ else FFLAGS += -O2 endif +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + ifeq ($(ICE_COMMDIR), mpi) - FC := ftn + FC := $(MPIFC) + CC := $(MPICC) else - FC := ftn + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= cc - -MPIFC:= ftn -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -44,10 +47,6 @@ INCLDIR := $(INCLDIR) #LIB_MPI := $(IMPILIBDIR) #SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= icc - -SFC:= ifort - ifeq ($(ICE_THREADED), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp diff --git a/configuration/scripts/machines/Macros.phase3_intel b/configuration/scripts/machines/Macros.phase3_intel index 6bfea6b19..e71d92c34 100755 --- a/configuration/scripts/machines/Macros.phase3_intel +++ b/configuration/scripts/machines/Macros.phase3_intel @@ -3,7 +3,8 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -DnoI8 -DNOI8 + #CFLAGS := -c -O2 -fp-model precise -xHost CFLAGS := -c -O2 diff --git a/configuration/scripts/machines/Macros.testmachine_intel b/configuration/scripts/machines/Macros.testmachine_intel index 6b9a31b77..0d9121667 100755 --- a/configuration/scripts/machines/Macros.testmachine_intel +++ b/configuration/scripts/machines/Macros.testmachine_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -17,16 +17,19 @@ else FFLAGS += -O2 endif +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 + ifeq ($(ICE_COMMDIR), mpi) - FC := mpif90 + FC := $(MPIFC) + CC := $(MPICC) else - FC := ifort + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= mpicc - -MPIFC:= mpif90 -LD:= $(MPIFC) +LD:= $(FC) NETCDF_PATH := $(NETCDF) @@ -43,29 +46,16 @@ LIB_MPI := $(IMPILIBDIR) SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl -SCC:= icc - -SFC:= ifort - ifeq ($(compile_threaded), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp FFLAGS += -qopenmp endif -ifeq ($(DITTO), yes) - CPPDEFS := $(CPPDEFS) -DREPRODUCIBLE -endif - ### if using parallel I/O, load all 3 libraries. PIO must be first! ifeq ($(IO_TYPE), pio) PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof - - CPPDEFS := $(CPPDEFS) -Dncdf endif -ifeq ($(IO_TYPE), netcdf) - CPPDEFS := $(CPPDEFS) -Dncdf -endif diff --git a/configuration/scripts/machines/Macros.thunder_intel b/configuration/scripts/machines/Macros.thunder_intel index 343b1e99c..581e31909 100644 --- a/configuration/scripts/machines/Macros.thunder_intel +++ b/configuration/scripts/machines/Macros.thunder_intel @@ -3,7 +3,7 @@ #============================================================================== CPP := fpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} CFLAGS := -c -O2 -fp-model precise -xHost FIXEDFLAGS := -132 @@ -17,16 +17,20 @@ else FFLAGS += -O2 endif +SCC := icc +SFC := ifort +MPICC := icc +MPIFC := ifort + ifeq ($(ICE_COMMDIR), mpi) - FC := mpif90 + FC := $(MPIFC) + CC := $(MPICC) + LDFLAGS += -lmpi else - FC := ifort + FC := $(SFC) + CC := $(SCC) endif - -MPICC:= mpicc - -MPIFC:= mpif90 -LD:= $(MPIFC) +LD:= $(FC) # defined by module #NETCDF_PATH := $(NETCDF) @@ -44,10 +48,6 @@ LIB_PNETCDF := $(PNETCDF_PATH)/lib LIB_MPI := $(IMPILIBDIR) SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -SCC:= icc - -SFC:= ifort - ifeq ($(ICE_THREADED), true) LDFLAGS += -qopenmp CFLAGS += -qopenmp diff --git a/configuration/scripts/machines/Macros.travisCI_gnu b/configuration/scripts/machines/Macros.travisCI_gnu index c9f0b19da..66fb30a07 100644 --- a/configuration/scripts/machines/Macros.travisCI_gnu +++ b/configuration/scripts/machines/Macros.travisCI_gnu @@ -3,8 +3,8 @@ #============================================================================== CPP := cpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} -CFLAGS := -c -O2 -xHost +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 FIXEDFLAGS := -ffixed-line-length-132 FREEFLAGS := -ffree-form @@ -17,13 +17,22 @@ else FFLAGS += -O2 endif -FC := mpif90 +SCC := gcc +SFC := gfortran +MPICC := mpicc +MPIFC := mpif90 -MPICC:= - -MPIFC:= mpif90 +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif LD:= $(FC) +LIB_MPI := + NETCDF_PATH := $(NETCDF) ifeq ($(ICE_IOTYPE), netcdf) @@ -36,10 +45,6 @@ else SLIBS := endif -LIB_MPI := -SCC:= -SFC:= - ifeq ($(ICE_THREADED), true) LDFLAGS += -fopenmp CFLAGS += -fopenmp diff --git a/configuration/scripts/machines/env.cesium_intel b/configuration/scripts/machines/env.cesium_intel index b4c72820a..1b8717897 100644 --- a/configuration/scripts/machines/env.cesium_intel +++ b/configuration/scripts/machines/env.cesium_intel @@ -1,11 +1,14 @@ #!/bin/csh -f -#. ssmuse-sh -d /fs/ssm/eccc/mrd/rpn/OCEAN/cncpt-3.1.2 -#source NEMO_compiler.ksh +set ssmuse=/fs/ssm/main/env/20180430/all/bin/ssmuse-csh # package loader +source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156 # intel compiler +source /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156/intelcomp_2016.1.156_multi/bin/compilervars.csh intel64 # should be sourced by above domain, but bug in csh script +source $ssmuse -d /fs/ssm/main/opt/openmpi/openmpi-1.6.5/intelcomp-2016.1.156 # openmpi +source $ssmuse -d /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150 # netcdf (and openmpi) setenv ICE_MACHINE_ENVNAME cesium setenv ICE_MACHINE_COMPILER intel -setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_MAKE colormake-short setenv ICE_MACHINE_WKDIR /users/dor/afsg/phb/local/CICEDIRS/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /users/dor/afsg/phb/local/FORCING setenv ICE_MACHINE_BASELINE /users/dor/afsg/phb/local/CICEDIRS/CICE_BASELINE @@ -13,7 +16,8 @@ setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_TPNODE 36 setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_QUEUE "debug" -setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_BLDTHRDS 4 +setenv ICE_MACHINE_QSTAT "qstat " if (-e ~/.cice_proj) then set account_name = `head -1 ~/.cice_proj` diff --git a/configuration/scripts/machines/env.cheyenne_intel b/configuration/scripts/machines/env.cheyenne_intel index 1bb276f15..ae9236766 100755 --- a/configuration/scripts/machines/env.cheyenne_intel +++ b/configuration/scripts/machines/env.cheyenne_intel @@ -14,7 +14,7 @@ module load ncarenv/1.2 module load intel/17.0.1 module load mpt/2.19 module load ncarcompilers/0.4.1 -module load netcdf-mpi/4.6.1 +module load netcdf/4.6.1 endif diff --git a/configuration/scripts/machines/env.fram_intel b/configuration/scripts/machines/env.fram_intel index 5f71b21d9..8e9fc1935 100755 --- a/configuration/scripts/machines/env.fram_intel +++ b/configuration/scripts/machines/env.fram_intel @@ -14,6 +14,7 @@ setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_TPNODE 36 setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "qstat " if (-e ~/.cice_proj) then set account_name = `head -1 ~/.cice_proj` diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel index 1060febd7..4c424727f 100755 --- a/configuration/scripts/machines/env.phase3_intel +++ b/configuration/scripts/machines/env.phase3_intel @@ -8,10 +8,8 @@ module load impi/18.0.1 module load NetCDF/4.5.0 module load ESMF/7_1_0r module list -#module load intel/18.1.163 Works, at least w. nc 4.4.0, -# module load esmf -# module load wgrib wgrib2 -# + +setenv noI8 true setenv ICE_MACHINE_ENVNAME phase3 setenv ICE_MACHINE_COMPILER intel @@ -25,3 +23,4 @@ setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_BLDTHRDS 1 setenv ICE_MACHINE_QSTAT "qstat " + From 8834eba445fbb3c5070bfb60b16eb7df23199cdc Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Thu, 2 May 2019 17:32:33 +0000 Subject: [PATCH 15/54] env edit --- configuration/scripts/machines/env.phase3_intel | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel index 4c424727f..98d0efdfb 100755 --- a/configuration/scripts/machines/env.phase3_intel +++ b/configuration/scripts/machines/env.phase3_intel @@ -14,9 +14,9 @@ setenv noI8 true setenv ICE_MACHINE_ENVNAME phase3 setenv ICE_MACHINE_COMPILER intel setenv ICE_MACHINE_MAKE gmake -setenv ICE_MACHINE_WKDIR ~/noscrub/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA ~/noscrub/ -setenv ICE_MACHINE_BASELINE ~/noscrub/CICE_BASELINE +setenv ICE_MACHINE_WKDIR /u/Robert.Grumbine/noscrub/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /u/Robert.Grumbine/noscrub/ +setenv ICE_MACHINE_BASELINE /u/Robert.Grumbine/noscrub/CICE_BASELINE setenv ICE_MACHINE_SUBMIT " " setenv ICE_MACHINE_TPNODE 4 setenv ICE_MACHINE_QUEUE "default" From 7305a29db1356032e8d0e1d00b65fb9ae9e87f8b Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Mon, 13 May 2019 19:07:22 +0000 Subject: [PATCH 16/54] builds on phase3 now --- configuration/scripts/Makefile | 4 ++-- configuration/scripts/cice.build | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configuration/scripts/Makefile b/configuration/scripts/Makefile index a346cf37e..d6f481ed3 100644 --- a/configuration/scripts/Makefile +++ b/configuration/scripts/Makefile @@ -98,7 +98,7 @@ db_files: db_flags: @echo " " @echo "* cpp := $(CPP) $(CPPFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR)" - @echo "* cc := cc -c $(CFLAGS) $(INCS) $(INCLDIR)" + @echo "* cc := $(CC) -c $(CFLAGS) $(INCS) $(INCLDIR)" @echo "* .F.o := $(FC) -c $(FFLAGS) $(FIXEDFLAGS) $(INCS) $(INCLDIR)" @echo "* .F90.o := $(FC) -c $(FFLAGS) $(FREEFLAGS) $(INCS) $(INCLDIR)" @@ -110,7 +110,7 @@ $(EXEC): $(OBJS) $(LD) -o $(EXEC) $(LDFLAGS) $(OBJS) $(ULIBS) $(SLIBS) .c.o: - cc $(CFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR) $< + $(CC) $(CFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR) $< .F.o: $(FC) -c $(FFLAGS) $(FIXEDFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR) $< diff --git a/configuration/scripts/cice.build b/configuration/scripts/cice.build index e7a9e42a9..a641dc179 100755 --- a/configuration/scripts/cice.build +++ b/configuration/scripts/cice.build @@ -57,7 +57,7 @@ cd ${ICE_OBJDIR} #setenv ICE_CPPDEFS "-DNXGLOB=${ICE_NXGLOB} -DNYGLOB=${ICE_NYGLOB} -DBLCKX=${ICE_BLCKX} -DBLCKY=${ICE_BLCKY} -DMXBLCKS=${ICE_MXBLCKS} -DNICELYR=${NICELYR} -DNSNWLYR=${NSNWLYR} -DNICECAT=${NICECAT} -DTRAGE=${TRAGE} -DTRFY=${TRFY} -DTRLVL=${TRLVL} -DTRPND=${TRPND} -DTRBRI=${TRBRI} -DNTRAERO=${NTRAERO} -DTRZS=${TRZS} -DNBGCLYR=${NBGCLYR} -DTRALG=${TRALG} -DTRBGCZ=${TRBGCZ} -DTRDOC=${TRDOC} -DTRDOC=${TRDOC} -DTRDIC=${TRDIC} -DTRDON=${TRDON} -DTRFED=${TRFED} -DTRFEP=${TRFEP} -DTRZAERO=${TRZAERO} -DTRBGCS=${TRBGCS} -DNUMIN=${NUMIN} -DNUMAX=${NUMAX}" #setenv ICE_CPPDEFS "-DNICELYR=${NICELYR} -DNSNWLYR=${NSNWLYR} -DNICECAT=${NICECAT} -DTRAGE=${TRAGE} -DTRFY=${TRFY} -DTRLVL=${TRLVL} -DTRPND=${TRPND} -DTRBRI=${TRBRI} -DNTRAERO=${NTRAERO} -DTRZS=${TRZS} -DNBGCLYR=${NBGCLYR} -DTRALG=${TRALG} -DTRBGCZ=${TRBGCZ} -DTRDOC=${TRDOC} -DTRDOC=${TRDOC} -DTRDIC=${TRDIC} -DTRDON=${TRDON} -DTRFED=${TRFED} -DTRFEP=${TRFEP} -DTRZAERO=${TRZAERO} -DTRBGCS=${TRBGCS} -DNUMIN=${NUMIN} -DNUMAX=${NUMAX}" #setenv ICE_CPPDEFS "-DNUMIN=${NUMIN} -DNUMAX=${NUMAX}" -setenv ICE_CPPDEFS " " +setenv ICE_CPPDEFS "-DNOI8 -DnoI8 -DNAMING=_ADD_UNDERSCORE " if ($DITTO == 'yes') then setenv ICE_CPPDEFS "${ICE_CPPDEFS} -DREPRODUCIBLE" From 2703c619d2a5b4c0591d1c3e289406f4e8cbb94a Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 24 May 2019 12:11:47 +0000 Subject: [PATCH 17/54] test --- alpha | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 alpha diff --git a/alpha b/alpha new file mode 100644 index 000000000..bbe440257 --- /dev/null +++ b/alpha @@ -0,0 +1,27 @@ +[![Build Status](https://travis-ci.org/CICE-Consortium/CICE.svg?branch=master)](https://travis-ci.org/CICE-Consortium/CICE) +[![Documentation Status](https://readthedocs.org/projects/cice-consortium-cice/badge/?version=master)](http://cice-consortium-cice.readthedocs.io/en/master/?badge=master) + +## Overview +This repository contains the files and code needed to run the CICE sea ice numerical model starting with version 6. CICE is maintained by the CICE Consortium. Versions prior to v6 are found in the [CICE-svn-trunk repository](https://github.com/CICE-Consortium/CICE-svn-trunk). + +CICE consists of a top level driver and dynamical core plus the Icepack column physics code, which is included in CICE as a git submodule. Because Icepack is a submodule of CICE, Icepack and CICE development are handled independently with respect to the github repositories even though development and testing may be done together. + +If you expect to make any changes to the code, we recommend that you first fork both the CICE and Icepack repositories. Basic instructions for working with CICE and Icepack are found in the Git Workflow Guidance, linked from the Resource Index (below). In order to incorporate your developments into the Consortium code it is +imperative you follow the guidance for Pull Requests and requisite testing. + +## Useful links +* **CICE wiki**: https://github.com/CICE-Consortium/CICE/wiki + + Information about the CICE model + +* **CICE Version Index**: https://github.com/CICE-Consortium/CICE/wiki/CICE-Version-Index + + Numbered CICE releases since version 6 with associated documentation and DOIs. + +* **Consortium Community Bulletin Board**: https://bb.cgd.ucar.edu/forums/cice-consortium-model-development + + First point of contact for discussing model development including bugs, diagnostics, and future directions. + +* **Resource Index**: https://github.com/CICE-Consortium/About-Us/wiki/Resource-Index + + List of resources for information about the Consortium and its repositories as well as model documentation, testing, and development. From f843af73b50bbf07d0c08d6b549180215982af3c Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 24 May 2019 13:12:07 +0000 Subject: [PATCH 18/54] trying to pull/merge cice master --- .../io/io_netcdf/ice_restart.F90 | 4 - configuration/scripts/cice.batch.csh | 181 +++++++++++++++++ configuration/scripts/cice.build | 151 ++++++++++++++ configuration/scripts/cice.launch.csh | 184 ++++++++++++++++++ configuration/scripts/cice.settings | 2 +- 5 files changed, 517 insertions(+), 5 deletions(-) create mode 100755 configuration/scripts/cice.batch.csh create mode 100755 configuration/scripts/cice.build create mode 100755 configuration/scripts/cice.launch.csh diff --git a/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 b/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 index 64cde26cf..83f9bee1a 100644 --- a/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 @@ -149,7 +149,6 @@ subroutine init_restart_write(filename_spec) dimid_ncat, & ! iflag, & ! netCDF creation flag status ! status variable from netCDF routine - integer (kind=int_kind) :: oldMode character (len=3) :: nchar, ncharb @@ -602,9 +601,6 @@ subroutine init_restart_write(filename_spec) endif !z_tracers deallocate(dims) -! From Denise Worthen for theia 11 Oct 2018: - !Prevent prefilling of arrays with _FillValue - status = nf90_set_fill(ncid, nf90_nofill, oldMode) status = nf90_enddef(ncid) write(nu_diag,*) 'Writing ',filename(1:lenstr(filename)) diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh new file mode 100755 index 000000000..07b1f06e8 --- /dev/null +++ b/configuration/scripts/cice.batch.csh @@ -0,0 +1,181 @@ +#! /bin/csh -f + +if ( $1 != "" ) then + echo ${0:t} ${1} +else + echo ${0:t} +endif + +#source ./cice.settings +#source ${ICE_CASEDIR}/env.${ICE_MACHCOMP} || exit 2 + +set jobfile = $1 + +set ntasks = ${ICE_NTASKS} +set nthrds = ${ICE_NTHRDS} +set maxtpn = ${ICE_MACHINE_TPNODE} +set acct = ${ICE_ACCOUNT} + +@ ncores = ${ntasks} * ${nthrds} +@ taskpernode = ${maxtpn} / $nthrds +if (${taskpernode} == 0) set taskpernode = 1 +@ nnodes = ${ntasks} / ${taskpernode} +if (${nnodes} * ${taskpernode} < ${ntasks}) @ nnodes = $nnodes + 1 +set taskpernodelimit = ${taskpernode} +if (${taskpernodelimit} > ${ntasks}) set taskpernodelimit = ${ntasks} +@ corespernode = ${taskpernodelimit} * ${nthrds} + +set ptile = $taskpernode +if ($ptile > ${maxtpn} / 2) @ ptile = ${maxtpn} / 2 + +set queue = "${ICE_QUEUE}" +set batchtime = "00:15:00" +if (${ICE_RUNLENGTH} > 0) set batchtime = "00:29:00" +if (${ICE_RUNLENGTH} == 1) set batchtime = "00:59:00" +if (${ICE_RUNLENGTH} == 2) set batchtime = "2:00:00" +if (${ICE_RUNLENGTH} == 3) set batchtime = "3:00:00" +if (${ICE_RUNLENGTH} == 4) set batchtime = "4:00:00" +if (${ICE_RUNLENGTH} == 5) set batchtime = "5:00:00" +if (${ICE_RUNLENGTH} == 6) set batchtime = "6:00:00" +if (${ICE_RUNLENGTH} == 7) set batchtime = "7:00:00" +if (${ICE_RUNLENGTH} >= 8) set batchtime = "8:00:00" + +set shortcase = `echo ${ICE_CASENAME} | cut -c1-15` + +#========================================== + +cat >! ${jobfile} << EOF0 +#!/bin/csh -f +EOF0 + +#========================================== + +if (${ICE_MACHINE} =~ cheyenne*) then +cat >> ${jobfile} << EOFB +#PBS -j oe +###PBS -m ae +#PBS -V +#PBS -q ${queue} +#PBS -N ${ICE_CASENAME} +#PBS -A ${acct} +#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds} +#PBS -l walltime=${batchtime} +EOFB + +else if (${ICE_MACHINE} =~ hobart*) then +cat >> ${jobfile} << EOFB +#PBS -j oe +###PBS -m ae +#PBS -V +#PBS -q short +#PBS -N ${ICE_CASENAME} +#PBS -l nodes=1:ppn=24 +EOFB + +else if (${ICE_MACHINE} =~ thunder* || ${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad* || ${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr*) then +cat >> ${jobfile} << EOFB +#PBS -N ${shortcase} +#PBS -q ${queue} +#PBS -A ${acct} +#PBS -l select=${nnodes}:ncpus=${maxtpn}:mpiprocs=${taskpernode} +#PBS -l walltime=${batchtime} +#PBS -j oe +###PBS -M username@domain.com +###PBS -m be +EOFB + +else if (${ICE_MACHINE} =~ onyx*) then +cat >> ${jobfile} << EOFB +#PBS -N ${ICE_CASENAME} +#PBS -q ${queue} +#PBS -A ${acct} +#PBS -l select=${nnodes}:ncpus=${maxtpn}:mpiprocs=${taskpernode} +#PBS -l walltime=${batchtime} +#PBS -j oe +###PBS -M username@domain.com +###PBS -m be +EOFB + +else if (${ICE_MACHINE} =~ cori*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -p ${queue} +###SBATCH -A ${acct} +#SBATCH -n ${ncores} +#SBATCH -t ${batchtime} +#SBATCH -L SCRATCH +#SBATCH -C haswell +###SBATCH -e filename +###SBATCH -o filename +###SBATCH --mail-type FAIL +###SBATCH --mail-user username@domain.com +EOFB + +else if (${ICE_MACHINE} =~ badger*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t ${batchtime} +#SBATCH -A ${acct} +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +###SBATCH --mail-type END,FAIL +###SBATCH --mail-user=eclare@lanl.gov +#SBATCH --qos=standby +EOFB + +else if (${ICE_MACHINE} =~ fram*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t ${batchtime} +#SBATCH -A ${acct} +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +###SBATCH --mail-type END,FAIL +###SBATCH --mail-user=armnjfl@ec.gc.ca +#SBATCH --qos=standby +EOFB + +else if (${ICE_MACHINE} =~ cesium*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t ${batchtime} +#SBATCH -A ${acct} +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +###SBATCH --mail-type END,FAIL +###SBATCH --mail-user=philippe.blain@canada.ca +#SBATCH --qos=standby +EOFB + +else if (${ICE_MACHINE} =~ millikan*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t ${batchtime} +#SBATCH -A ${acct} +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +###SBATCH --mail-type END,FAIL +###SBATCH --mail-user=amelie.bouchat@canada.ca +#SBATCH --qos=standby +EOFB + +else if (${ICE_MACHINE} =~ testmachine*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + +else if (${ICE_MACHINE} =~ travisCI*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + +else + echo "${0} ERROR: ${ICE_MACHINE} unknown" + exit -1 +endif + +exit 0 diff --git a/configuration/scripts/cice.build b/configuration/scripts/cice.build new file mode 100755 index 000000000..7d958d309 --- /dev/null +++ b/configuration/scripts/cice.build @@ -0,0 +1,151 @@ +#! /bin/csh -f + +#==================================== +# If the cice binary is passed as an argument and the file exists, +# copy it into the run directory and don't build the model. + +set ciceexe = "UnDEFineD" +if ($#argv == 1) then + set ciceexe = $1 + echo "${0}: ciceexe = ${ciceexe}" + if (-e ${ciceexe}) then + + source ./cice.settings + source ${ICE_CASEDIR}/env.${ICE_MACHCOMP} -nomodules || exit 2 + if !(-d ${ICE_RUNDIR}) mkdir -p ${ICE_RUNDIR} + cp -p ${ciceexe} ${ICE_RUNDIR}/cice + + echo "`date` ${0}:${ICE_CASENAME} build copied ${ciceexe}" >> ${ICE_CASEDIR}/README.case + if ( ${ICE_TEST} != ${ICE_SPVAL} ) then + echo "#---" >! ${ICE_CASEDIR}/test_output + echo "COPY ${ICE_TESTNAME} build" >> ${ICE_CASEDIR}/test_output + echo "PEND ${ICE_TESTNAME} run" >> ${ICE_CASEDIR}/test_output + endif + + exit 0 + + endif +endif + +#==================================== + +source ./cice.settings +source ${ICE_CASEDIR}/env.${ICE_MACHCOMP} || exit 2 + +if (${ICE_MACHINE} != ${ICE_MACHINE_ENVNAME}) then + echo "WARNING, is ICE_MACHINE setting OK, ${ICE_MACHINE}, ${ICE_MACHINE_ENVNAME}" +endif + +if (${ICE_COMPILER} != ${ICE_MACHINE_COMPILER}) then + echo "WARNING, is ICE_COMPILER setting OK, ${ICE_COMPILER}, ${ICE_MACHINE_COMPILER}" +endif + +echo " " +echo "${0}:" + +set stamp = `date '+%y%m%d-%H%M%S'` +set ICE_BLDLOG_FILE = "cice.bldlog.${stamp}" +set quiet = ${ICE_QUIETMODE} + +if (${ICE_CLEANBUILD} == 'true') then + echo "cleaning objdir" + rm -r -f ${ICE_OBJDIR} +endif +if !(-d ${ICE_OBJDIR}) mkdir -p ${ICE_OBJDIR} +cd ${ICE_OBJDIR} + +setenv ICE_CPPDEFS " " + +if (${ICE_IOTYPE} == 'netcdf') then + set IODIR = io_netcdf + setenv ICE_CPPDEFS "${ICE_CPPDEFS} -Dncdf" +else if (${ICE_IOTYPE} == 'pio') then + set IODIR = io_pio + setenv ICE_CPPDEFS "${ICE_CPPDEFS} -Dncdf" +else + set IODIR = io_binary +endif + +### List of source code directories (in order of importance). +cat >! Filepath << EOF +${ICE_SANDBOX}/cicecore/drivers/${ICE_DRVOPT} +${ICE_SANDBOX}/cicecore/cicedynB/dynamics +${ICE_SANDBOX}/cicecore/cicedynB/general +${ICE_SANDBOX}/cicecore/cicedynB/analysis +${ICE_SANDBOX}/cicecore/cicedynB/infrastructure +${ICE_SANDBOX}/cicecore/cicedynB/infrastructure/io/$IODIR +${ICE_SANDBOX}/cicecore/cicedynB/infrastructure/comm/${ICE_COMMDIR} +${ICE_SANDBOX}/cicecore/shared +${ICE_SANDBOX}/icepack/columnphysics +EOF + +echo " " +echo ICE_GRID = ${ICE_GRID} +echo ICE_NTASK = ${ICE_NTASKS} +echo ICE_NTHRD = ${ICE_NTHRDS} +echo "ICE_CPPDEFS = ${ICE_CPPDEFS}" +echo "Filepath = " +cat ${ICE_OBJDIR}/Filepath +echo " " + +echo "building makdep" +${ICE_MACHINE_MAKE} \ + -f ${ICE_CASEDIR}/Makefile MACFILE=${ICE_CASEDIR}/Macros.${ICE_MACHCOMP} makdep || exit 2 + +echo "building cice > ${ICE_OBJDIR}/${ICE_BLDLOG_FILE}" + +if !(-d ${ICE_RUNDIR}) mkdir -p ${ICE_RUNDIR} +if !($?ICE_MACHINE_BLDTHRDS) then + set ICE_MACHINE_BLDTHRDS = 1 +endif +if (-e ${ICE_BLDLOG_FILE}) rm ${ICE_BLDLOG_FILE} + +if (${ICE_CLEANBUILD} == 'true') then + echo "gmake clean" + if (${quiet} == "true") then + ${ICE_MACHINE_MAKE} VPFILE=Filepath EXEC=${ICE_RUNDIR}/cice \ + -f ${ICE_CASEDIR}/Makefile MACFILE=${ICE_CASEDIR}/Macros.${ICE_MACHCOMP} clean >& ${ICE_BLDLOG_FILE} + else + ${ICE_MACHINE_MAKE} VPFILE=Filepath EXEC=${ICE_RUNDIR}/cice \ + -f ${ICE_CASEDIR}/Makefile MACFILE=${ICE_CASEDIR}/Macros.${ICE_MACHCOMP} clean | tee ${ICE_BLDLOG_FILE} + endif +endif + +echo "gmake cice" +if (${quiet} == "true") then + echo " quiet mode on... patience" + ${ICE_MACHINE_MAKE} -j ${ICE_MACHINE_BLDTHRDS} VPFILE=Filepath EXEC=${ICE_RUNDIR}/cice \ + -f ${ICE_CASEDIR}/Makefile MACFILE=${ICE_CASEDIR}/Macros.${ICE_MACHCOMP} >& ${ICE_BLDLOG_FILE} + set bldstat = ${status} +else + ${ICE_MACHINE_MAKE} -j ${ICE_MACHINE_BLDTHRDS} VPFILE=Filepath EXEC=${ICE_RUNDIR}/cice \ + -f ${ICE_CASEDIR}/Makefile MACFILE=${ICE_CASEDIR}/Macros.${ICE_MACHCOMP} | tee ${ICE_BLDLOG_FILE} + set bldstat = ${status} +endif + +if !(-d ${ICE_LOGDIR}) mkdir -p ${ICE_LOGDIR} +cp -p ${ICE_BLDLOG_FILE} ${ICE_LOGDIR}/ + +if (${bldstat} != 0) then + echo "${0}: COMPILE FAILED, see" + echo " cat ${ICE_OBJDIR}/${ICE_BLDLOG_FILE}" + if (${quiet} == "true") then + tail -10 ${ICE_OBJDIR}/${ICE_BLDLOG_FILE} + endif + if ( ${ICE_TEST} != ${ICE_SPVAL} ) then + # This is a test case. Write output to test_output file + echo "#---" >! ${ICE_CASEDIR}/test_output + echo "FAIL ${ICE_TESTNAME} build" >> ${ICE_CASEDIR}/test_output + echo "FAIL ${ICE_TESTNAME} run" >> ${ICE_CASEDIR}/test_output + endif + exit 99 +endif + +echo "`date` ${0}:${ICE_CASENAME} build completed ${ICE_BLDLOG_FILE}" >> ${ICE_CASEDIR}/README.case +echo "${0}: COMPILE SUCCESSFUL, ${ICE_LOGDIR}/${ICE_BLDLOG_FILE}" +if ( ${ICE_TEST} != ${ICE_SPVAL} ) then + echo "#---" >! ${ICE_CASEDIR}/test_output + echo "PASS ${ICE_TESTNAME} build" >> ${ICE_CASEDIR}/test_output + echo "PEND ${ICE_TESTNAME} run" >> ${ICE_CASEDIR}/test_output +endif + diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh new file mode 100755 index 000000000..86ede7049 --- /dev/null +++ b/configuration/scripts/cice.launch.csh @@ -0,0 +1,184 @@ +#! /bin/csh -f + +#echo ${0} +echo "running cice.launch.csh" + +#source ./cice.settings +#source ${ICE_CASEDIR}/env.${ICE_MACHCOMP} || exit 2 + +set jobfile = $1 + +set ntasks = ${ICE_NTASKS} +set nthrds = ${ICE_NTHRDS} +set maxtpn = ${ICE_MACHINE_TPNODE} + +@ ncores = ${ntasks} * ${nthrds} +@ taskpernode = ${maxtpn} / $nthrds +if (${taskpernode} == 0) set taskpernode = 1 +@ nnodes = ${ntasks} / ${taskpernode} +if (${nnodes} * ${taskpernode} < ${ntasks}) @ nnodes = $nnodes + 1 +set taskpernodelimit = ${taskpernode} +if (${taskpernodelimit} > ${ntasks}) set taskpernodelimit = ${ntasks} +@ corespernode = ${taskpernodelimit} * ${nthrds} + +#========================================== +if (${ICE_MACHINE} =~ cheyenne*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpiexec_mpt -n ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ hobart*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpiexec -n ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ thunder*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ onyx*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +aprun -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ cori*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ badger*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ fram*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ cesium*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ millikan*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif + +#======= +else if (${ICE_MACHINE} =~ testmachine*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR + +#======= +else if (${ICE_MACHINE} =~ travisCI*) then +if (${ICE_COMMDIR} =~ serial*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +else +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +EOFR +endif +#cat >> ${jobfile} << EOFR +#srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +#EOFR + +#======= +else + echo "${0} ERROR ${ICE_MACHINE} unknown" + exit -1 +endif +#======= + +exit 0 diff --git a/configuration/scripts/cice.settings b/configuration/scripts/cice.settings index 34ae94468..5f6388a2e 100755 --- a/configuration/scripts/cice.settings +++ b/configuration/scripts/cice.settings @@ -25,7 +25,7 @@ setenv ICE_TESTID undefined setenv ICE_BASELINE undefined setenv ICE_BASEGEN undefined setenv ICE_BASECOM undefined -setenv ICE_BFBCOMP BFB +setenv ICE_BFBCOMP undefined setenv ICE_SPVAL undefined setenv ICE_RUNLENGTH 0 setenv ICE_ACCOUNT undefined From 64081e76614c415cf5043953703513252a4c7a84 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 24 May 2019 13:23:29 +0000 Subject: [PATCH 19/54] theia --- configuration/scripts/cice.batch.csh | 15 +++++++++++++++ configuration/scripts/cice.launch.csh | 15 +++++++++++++++ .../scripts/machines/Macros.theia_intel | 19 ++++++++++++++----- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 07b1f06e8..68395b219 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -163,6 +163,21 @@ cat >> ${jobfile} << EOFB #SBATCH --qos=standby EOFB +else if (${ICE_MACHINE} =~ theia*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + +else if (${ICE_MACHINE} =~ phase2*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFB # nothing to do diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index 86ede7049..4c1c6c3a3 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -153,6 +153,21 @@ mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR endif +#======= +else if (${ICE_MACHINE} =~ theia*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ phase2*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFR +./cice >&! \$ICE_RUNLOG_FILE +EOFR #======= else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFR diff --git a/configuration/scripts/machines/Macros.theia_intel b/configuration/scripts/machines/Macros.theia_intel index fa7b71140..31f2afa64 100644 --- a/configuration/scripts/machines/Macros.theia_intel +++ b/configuration/scripts/machines/Macros.theia_intel @@ -22,9 +22,6 @@ endif #Note that on theia mpif90 refers to gfortran even though it is in the intel bin FC := mpiifort - -MPICC:= - MPIFC:= mpiifort LD:= $(MPIFC) @@ -41,8 +38,20 @@ else endif LIB_MPI := -SCC:= -SFC:= + + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif ifeq ($(ICE_THREADED), true) LDFLAGS += -fopenmp From 97179e1cee0dcbfa7c634f6231089c41fa3f9629 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 24 May 2019 14:15:14 +0000 Subject: [PATCH 20/54] successful theia build --- configuration/scripts/machines/Macros.theia_intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/scripts/machines/Macros.theia_intel b/configuration/scripts/machines/Macros.theia_intel index 31f2afa64..38cc62ed8 100644 --- a/configuration/scripts/machines/Macros.theia_intel +++ b/configuration/scripts/machines/Macros.theia_intel @@ -42,7 +42,7 @@ LIB_MPI := SCC := icc SFC := ifort -MPICC := mpicc +MPICC := mpiicc MPIFC := mpiifort ifeq ($(ICE_COMMDIR), mpi) From 70acceaef3a291567ee36bc96664958eaa983f0c Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 24 May 2019 15:27:09 +0000 Subject: [PATCH 21/54] garbage --- alpha | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 alpha diff --git a/alpha b/alpha deleted file mode 100644 index bbe440257..000000000 --- a/alpha +++ /dev/null @@ -1,27 +0,0 @@ -[![Build Status](https://travis-ci.org/CICE-Consortium/CICE.svg?branch=master)](https://travis-ci.org/CICE-Consortium/CICE) -[![Documentation Status](https://readthedocs.org/projects/cice-consortium-cice/badge/?version=master)](http://cice-consortium-cice.readthedocs.io/en/master/?badge=master) - -## Overview -This repository contains the files and code needed to run the CICE sea ice numerical model starting with version 6. CICE is maintained by the CICE Consortium. Versions prior to v6 are found in the [CICE-svn-trunk repository](https://github.com/CICE-Consortium/CICE-svn-trunk). - -CICE consists of a top level driver and dynamical core plus the Icepack column physics code, which is included in CICE as a git submodule. Because Icepack is a submodule of CICE, Icepack and CICE development are handled independently with respect to the github repositories even though development and testing may be done together. - -If you expect to make any changes to the code, we recommend that you first fork both the CICE and Icepack repositories. Basic instructions for working with CICE and Icepack are found in the Git Workflow Guidance, linked from the Resource Index (below). In order to incorporate your developments into the Consortium code it is -imperative you follow the guidance for Pull Requests and requisite testing. - -## Useful links -* **CICE wiki**: https://github.com/CICE-Consortium/CICE/wiki - - Information about the CICE model - -* **CICE Version Index**: https://github.com/CICE-Consortium/CICE/wiki/CICE-Version-Index - - Numbered CICE releases since version 6 with associated documentation and DOIs. - -* **Consortium Community Bulletin Board**: https://bb.cgd.ucar.edu/forums/cice-consortium-model-development - - First point of contact for discussing model development including bugs, diagnostics, and future directions. - -* **Resource Index**: https://github.com/CICE-Consortium/About-Us/wiki/Resource-Index - - List of resources for information about the Consortium and its repositories as well as model documentation, testing, and development. From f859b242404e3a811a2f912c6f27c1796c761f95 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Tue, 28 May 2019 15:42:14 +0000 Subject: [PATCH 22/54] update theia intel env --- configuration/scripts/machines/env.theia_intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel index c775d7e73..40df5110b 100755 --- a/configuration/scripts/machines/env.theia_intel +++ b/configuration/scripts/machines/env.theia_intel @@ -7,7 +7,7 @@ module purge #14.0.2 w nc4.3.0 does not module load intel/18.1.163 # ok w nc4.3.0: module load intel/16.0.1.150 # ok w nc4.3.0: module load intel/15.3.187 -module load intel/15.0.0 +module load intel/17.0.5.239 module load impi module load esmf module load hdf5 netcdf/4.3.0 From 40d1cf0c63894fd11a75bec21f6a32a7ebfb177f Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Fri, 31 May 2019 11:14:13 -0400 Subject: [PATCH 23/54] mods for running in NOAA --- configuration/scripts/cice.batch.csh | 28 +++++++++ configuration/scripts/cice.launch.csh | 27 +++++++++ .../scripts/machines/Macros.high_Sierra_gnu | 47 +++++++++++++++ .../scripts/machines/Macros.phase2_intel | 57 ++++++++++++++++++ .../scripts/machines/Macros.phase3_intel | 58 ++++++++++++++++++ .../scripts/machines/Macros.theia_intel | 60 +++++++++++++++++++ .../scripts/machines/env.high_Sierra_gnu | 15 +++++ .../scripts/machines/env.phase2_intel | 14 +++++ .../scripts/machines/env.phase3_intel | 24 ++++++++ .../scripts/machines/env.theia_intel | 30 ++++++++++ 10 files changed, 360 insertions(+) create mode 100644 configuration/scripts/machines/Macros.high_Sierra_gnu create mode 100755 configuration/scripts/machines/Macros.phase2_intel create mode 100755 configuration/scripts/machines/Macros.phase3_intel create mode 100644 configuration/scripts/machines/Macros.theia_intel create mode 100755 configuration/scripts/machines/env.high_Sierra_gnu create mode 100755 configuration/scripts/machines/env.phase2_intel create mode 100755 configuration/scripts/machines/env.phase3_intel create mode 100755 configuration/scripts/machines/env.theia_intel diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 4fc083703..0417c43a3 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -165,6 +165,34 @@ cat >> ${jobfile} << EOFB #SBATCH --qos=standby EOFB +else if (${ICE_MACHINE} =~ theia*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t ${batchtime} +#SBATCH -q batch +#SBATCH -A marine-cpu +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +#SBATCH --mail-type END,FAIL +#SBATCH --mail-user=robert.grumbine@noaa.gov +EOFB + +else if (${ICE_MACHINE} =~ phase2*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + +else if (${ICE_MACHINE} =~ high_Sierra*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFB # nothing to do diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index c3aafba21..ce328f2a6 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -153,6 +153,33 @@ mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR endif +#======= +else if (${ICE_MACHINE} =~ theia*) then +cat >> ${jobfile} << EOFR +#mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ high_Sierra*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +EOFR + +#======= +else if (${ICE_MACHINE} =~ phase2*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +EOFR + +#======= +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +EOFR #======= else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFR diff --git a/configuration/scripts/machines/Macros.high_Sierra_gnu b/configuration/scripts/machines/Macros.high_Sierra_gnu new file mode 100644 index 000000000..960779da1 --- /dev/null +++ b/configuration/scripts/machines/Macros.high_Sierra_gnu @@ -0,0 +1,47 @@ +#============================================================================== +# Makefile macros for Travis-CI - GCC and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FFLAGS := -O2 -ffree-line-length-none -fconvert=big-endian -finit-real=nan +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + #FFLAGS += -O0 -g -Wextra -fbacktrace -fbounds-check -ffpe-trap=zero,overflow + FFLAGS += -O0 -g -std=f2008 -fbacktrace -fbounds-check -ffpe-trap=zero,overflow +else + FFLAGS += -O2 +endif + +FC := mpif90 + +MPICC:= + +MPIFC:= mpif90 +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:=gcc +SFC:= + +#ifeq ($(ICE_THREADED), true) +# LDFLAGS += -fopenmp +# CFLAGS += -fopenmp +# FFLAGS += -fopenmp +#endif diff --git a/configuration/scripts/machines/Macros.phase2_intel b/configuration/scripts/machines/Macros.phase2_intel new file mode 100755 index 000000000..6bfea6b19 --- /dev/null +++ b/configuration/scripts/machines/Macros.phase2_intel @@ -0,0 +1,57 @@ +#============================================================================== +# Makefile macros for generic test machine, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +#CFLAGS := -c -O2 -fp-model precise -xHost +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) $(NETCDF_INCLUDE) + +LIB_NETCDF := $(NETCDF_PATH)/lib +LIB_MPI := $(IMPILIBDIR) + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(compile_threaded), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +### if using parallel I/O, load all 3 libraries. PIO must be first! +#ifeq ($(IO_TYPE), pio) +# PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib +# INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include +# SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof +#endif diff --git a/configuration/scripts/machines/Macros.phase3_intel b/configuration/scripts/machines/Macros.phase3_intel new file mode 100755 index 000000000..c5a8b7a7e --- /dev/null +++ b/configuration/scripts/machines/Macros.phase3_intel @@ -0,0 +1,58 @@ +#============================================================================== +# Makefile macros for generic test machine, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} + +#CFLAGS := -c -O2 -fp-model precise -xHost +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) $(NETCDF_INCLUDE) + +LIB_NETCDF := $(NETCDF_PATH)/lib +LIB_MPI := $(IMPILIBDIR) + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(compile_threaded), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +### if using parallel I/O, load all 3 libraries. PIO must be first! +#ifeq ($(IO_TYPE), pio) +# PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib +# INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include +# SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof +#endif diff --git a/configuration/scripts/machines/Macros.theia_intel b/configuration/scripts/machines/Macros.theia_intel new file mode 100644 index 000000000..38cc62ed8 --- /dev/null +++ b/configuration/scripts/machines/Macros.theia_intel @@ -0,0 +1,60 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpiifort +MPIFC:= mpiifort +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := + + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.high_Sierra_gnu b/configuration/scripts/machines/env.high_Sierra_gnu new file mode 100755 index 000000000..cabd66944 --- /dev/null +++ b/configuration/scripts/machines/env.high_Sierra_gnu @@ -0,0 +1,15 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME high_Sierra +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /Volumes/ncep/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /Volumes/Data/CICE_data +setenv ICE_MACHINE_BASELINE /Volumes/ncep/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +setenv ICE_MACHINE_QUIETMODE false +setenv ICE_MACHINE_QUEUE " " diff --git a/configuration/scripts/machines/env.phase2_intel b/configuration/scripts/machines/env.phase2_intel new file mode 100755 index 000000000..2457d4721 --- /dev/null +++ b/configuration/scripts/machines/env.phase2_intel @@ -0,0 +1,14 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME phase2 +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR ~/noscrub/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA ~/noscrub/ +setenv ICE_MACHINE_BASELINE ~/noscrub/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel new file mode 100755 index 000000000..03aa7083f --- /dev/null +++ b/configuration/scripts/machines/env.phase3_intel @@ -0,0 +1,24 @@ +#!/bin/csh -f --login + +source /etc/profile.d/lmod.csh + +module purge +module load ips/18.0.1.163 +module load impi/18.0.1 +module load NetCDF/4.5.0 +module load ESMF/7_1_0r +module list + +setenv ICE_MACHINE_ENVNAME phase3 +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /u/Robert.Grumbine/noscrub/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /u/Robert.Grumbine/noscrub/ +setenv ICE_MACHINE_BASELINE /u/Robert.Grumbine/noscrub/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "qstat " + diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel new file mode 100755 index 000000000..0438238d9 --- /dev/null +++ b/configuration/scripts/machines/env.theia_intel @@ -0,0 +1,30 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +#module list +module purge +#module load intel/18.1.163 Works, at least w. nc 4.4.0, +#14.0.2 w nc4.3.0 does not module load intel/18.1.163 +# ok w nc4.3.0: module load intel/16.0.1.150 +# ok w nc4.3.0: module load intel/15.3.187 +module load intel/17.0.5.239 +module load impi +module load esmf +module load hdf5 netcdf/4.3.0 +module load wgrib wgrib2 +#echo renewed modules: +#module list + +setenv ICE_MACHINE_ENVNAME theia +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /scratch3/NCEPDEV/marine/save/Robert.Grumbine/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "batch" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true From e15fcd399b78b4a6844fa46ede8626d814b1b584 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 30 Aug 2019 16:21:51 +0000 Subject: [PATCH 24/54] start towards hera --- configuration/scripts/machines/env.hera_intel | 26 +++++++++++++++++++ .../scripts/machines/env.theia_intel | 5 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 configuration/scripts/machines/env.hera_intel diff --git a/configuration/scripts/machines/env.hera_intel b/configuration/scripts/machines/env.hera_intel new file mode 100755 index 000000000..85b0f3f38 --- /dev/null +++ b/configuration/scripts/machines/env.hera_intel @@ -0,0 +1,26 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +#module list +module purge +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load esmf/7.1.0r +module load hdf5/1.10.4 netcdf/4.6.1 +module load wgrib wgrib2 +#echo renewed modules: +#module list + +setenv ICE_MACHINE_ENVNAME hera +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/hera.rgdev/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/hera.rgdev/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/hera.rgdev/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "batch" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel index 0438238d9..122b4033a 100755 --- a/configuration/scripts/machines/env.theia_intel +++ b/configuration/scripts/machines/env.theia_intel @@ -7,10 +7,11 @@ module purge #14.0.2 w nc4.3.0 does not module load intel/18.1.163 # ok w nc4.3.0: module load intel/16.0.1.150 # ok w nc4.3.0: module load intel/15.3.187 -module load intel/17.0.5.239 +#old theia module load intel/17.0.5.239 +module load intel/18.0.5.274 module load impi module load esmf -module load hdf5 netcdf/4.3.0 +module load hdf5/1.10.4 netcdf/4.6.1 module load wgrib wgrib2 #echo renewed modules: #module list From a8b26672c88008b52a5ea70af532a31d28e69b88 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 30 Aug 2019 16:23:25 +0000 Subject: [PATCH 25/54] tmp --- configuration/scripts/machines/env.theia_intel | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel index 0438238d9..a106f3471 100755 --- a/configuration/scripts/machines/env.theia_intel +++ b/configuration/scripts/machines/env.theia_intel @@ -7,10 +7,11 @@ module purge #14.0.2 w nc4.3.0 does not module load intel/18.1.163 # ok w nc4.3.0: module load intel/16.0.1.150 # ok w nc4.3.0: module load intel/15.3.187 -module load intel/17.0.5.239 -module load impi -module load esmf -module load hdf5 netcdf/4.3.0 +module load intel/18.0.1.163 +module load impi/5.1.2.150 +module load esmf/7.0.2 +module load hdf5/1.10.4 +module load netcdf/4.6.1 module load wgrib wgrib2 #echo renewed modules: #module list From 4970900e1e72ccf70ae17d2973e57b9602f74726 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 30 Aug 2019 17:38:54 +0000 Subject: [PATCH 26/54] theia passes with most recent of all modules --- configuration/scripts/machines/env.theia_intel | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel index a106f3471..28db8fb87 100755 --- a/configuration/scripts/machines/env.theia_intel +++ b/configuration/scripts/machines/env.theia_intel @@ -7,8 +7,10 @@ module purge #14.0.2 w nc4.3.0 does not module load intel/18.1.163 # ok w nc4.3.0: module load intel/16.0.1.150 # ok w nc4.3.0: module load intel/15.3.187 -module load intel/18.0.1.163 -module load impi/5.1.2.150 +#module load intel/18.0.1.163 +#module load impi/5.1.2.150 +module load intel/18.1.163 +module load impi/2018.0.1 module load esmf/7.0.2 module load hdf5/1.10.4 module load netcdf/4.6.1 From 96ffc20945657601d8acde00c6fa96286e1b4173 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Wed, 18 Sep 2019 13:58:34 +0000 Subject: [PATCH 27/54] moving to hera --- configuration/scripts/machines/env.theia_intel | 4 ++-- icepack | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel index 122b4033a..3cfa7965f 100755 --- a/configuration/scripts/machines/env.theia_intel +++ b/configuration/scripts/machines/env.theia_intel @@ -9,8 +9,8 @@ module purge # ok w nc4.3.0: module load intel/15.3.187 #old theia module load intel/17.0.5.239 module load intel/18.0.5.274 -module load impi -module load esmf +module load impi/2018.0.4 +module load esmf/7.1.0r module load hdf5/1.10.4 netcdf/4.6.1 module load wgrib wgrib2 #echo renewed modules: diff --git a/icepack b/icepack index ac6195df8..51ea326f1 160000 --- a/icepack +++ b/icepack @@ -1 +1 @@ -Subproject commit ac6195df8f3b5ae9a8e030b28c5a727fb5c11d62 +Subproject commit 51ea326f12578355d4d51b0cea29b45cc9890cd1 From 710683962c95fd3f7f7cc33cd538b427b7a3a7f1 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Thu, 19 Sep 2019 16:47:35 +0000 Subject: [PATCH 28/54] clean regression run on hera --- configuration/scripts/cice.batch.csh | 15 ++++- configuration/scripts/cice.launch.csh | 7 +++ .../scripts/machines/Macros.hera_intel | 60 +++++++++++++++++++ configuration/scripts/machines/env.hera_intel | 6 +- 4 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 configuration/scripts/machines/Macros.hera_intel diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 0417c43a3..bb2bcabf9 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -174,7 +174,20 @@ cat >> ${jobfile} << EOFB #SBATCH -N ${nnodes} #SBATCH -e slurm%j.err #SBATCH -o slurm%j.out -#SBATCH --mail-type END,FAIL +#SBATCH --mail-type FAIL +#SBATCH --mail-user=robert.grumbine@noaa.gov +EOFB + +else if (${ICE_MACHINE} =~ hera*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t ${batchtime} +#SBATCH -q batch +#SBATCH -A marine-cpu +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +#SBATCH --mail-type FAIL #SBATCH --mail-user=robert.grumbine@noaa.gov EOFB diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index ce328f2a6..bcfc05bd5 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -161,6 +161,13 @@ srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE #./cice >&! \$ICE_RUNLOG_FILE EOFR #======= +else if (${ICE_MACHINE} =~ hera*) then +cat >> ${jobfile} << EOFR +#mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= else if (${ICE_MACHINE} =~ high_Sierra*) then cat >> ${jobfile} << EOFR mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE diff --git a/configuration/scripts/machines/Macros.hera_intel b/configuration/scripts/machines/Macros.hera_intel new file mode 100644 index 000000000..38cc62ed8 --- /dev/null +++ b/configuration/scripts/machines/Macros.hera_intel @@ -0,0 +1,60 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpiifort +MPIFC:= mpiifort +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := + + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.hera_intel b/configuration/scripts/machines/env.hera_intel index 85b0f3f38..ef3afdb7a 100755 --- a/configuration/scripts/machines/env.hera_intel +++ b/configuration/scripts/machines/env.hera_intel @@ -14,9 +14,9 @@ module load wgrib wgrib2 setenv ICE_MACHINE_ENVNAME hera setenv ICE_MACHINE_COMPILER intel setenv ICE_MACHINE_MAKE make -setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/hera.rgdev/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/hera.rgdev/ -setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/hera.rgdev/CICE_BASELINE +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/rgdev/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/rgdev/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/rgdev/CICE_BASELINE setenv ICE_MACHINE_SUBMIT "sbatch" setenv ICE_MACHINE_TPNODE 4 setenv ICE_MACHINE_ACCT P0000000 From 011bdf742dd0e6bcc17f0fb1b99b709425d754a3 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Fri, 20 Sep 2019 01:07:30 +0000 Subject: [PATCH 29/54] cleanup --- .../scripts/machines/Macros.theia_gnu | 48 ------------------- .../scripts/machines/Macros.theia_pgi | 48 ------------------- configuration/scripts/machines/env.theia_gnu | 15 ------ configuration/scripts/machines/env.theia_pgi | 27 ----------- configuration/scripts/tests/rgvers_suite.ts | 18 ------- 5 files changed, 156 deletions(-) delete mode 100644 configuration/scripts/machines/Macros.theia_gnu delete mode 100644 configuration/scripts/machines/Macros.theia_pgi delete mode 100755 configuration/scripts/machines/env.theia_gnu delete mode 100755 configuration/scripts/machines/env.theia_pgi delete mode 100644 configuration/scripts/tests/rgvers_suite.ts diff --git a/configuration/scripts/machines/Macros.theia_gnu b/configuration/scripts/machines/Macros.theia_gnu deleted file mode 100644 index 63cce4b99..000000000 --- a/configuration/scripts/machines/Macros.theia_gnu +++ /dev/null @@ -1,48 +0,0 @@ -#============================================================================== -# Makefile macros for theia - intel and openmpi compilers -#============================================================================== - -CPP := cpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} -CFLAGS := -c -O2 -xHost - -FFLAGS := -fconvert=big-endian -#FFLAGS := -h bytwswapio -#FFLAGS := -FFLAGS_NOOPT:= -O0 - -ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow -else - FFLAGS += -O2 -endif - -#Note that on theia mpif90 refers to gfortran even though it is in the intel bin -FC := mpif90 - -MPICC:= - -MPIFC:= mpif90 -LD:= $(MPIFC) - -NETCDF_PATH := $(NETCDF) - -ifeq ($(ICE_IOTYPE), netcdf) - NETCDF_PATH := $(shell nc-config --prefix) - INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include - LIB_NETCDF := $(NETCDF_PATH)/lib - LIB_PNETCDF := - SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -else - SLIBS := -endif - -LIB_MPI := -SCC:= -SFC:= - -ifeq ($(ICE_THREADED), true) - LDFLAGS += -fopenmp - CFLAGS += -fopenmp - FFLAGS += -fopenmp -endif diff --git a/configuration/scripts/machines/Macros.theia_pgi b/configuration/scripts/machines/Macros.theia_pgi deleted file mode 100644 index 83bcc264d..000000000 --- a/configuration/scripts/machines/Macros.theia_pgi +++ /dev/null @@ -1,48 +0,0 @@ -#============================================================================== -# Makefile macros for theia - intel and openmpi compilers -#============================================================================== - -CPP := cpp -CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} -CFLAGS := -c -O2 -xHost - -FIXEDFLAGS := -132 -FREEFLAGS := -FR -FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost - -FFLAGS_NOOPT:= -O0 - -ifeq ($(ICE_BLDDEBUG), true) -#RG: this looks more like gfortran options: -## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow - FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 -else - FFLAGS += -O2 -endif - -FC := mpif90 -MPICC:= -MPIFC:= mpif90 -LD:= $(MPIFC) - -NETCDF_PATH := $(NETCDF) - -ifeq ($(ICE_IOTYPE), netcdf) - NETCDF_PATH := $(shell nc-config --prefix) - INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include - LIB_NETCDF := $(NETCDF_PATH)/lib - LIB_PNETCDF := - SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -else - SLIBS := -endif - -LIB_MPI := -SCC:= -SFC:= - -ifeq ($(ICE_THREADED), true) - LDFLAGS += -fopenmp - CFLAGS += -fopenmp - FFLAGS += -fopenmp -endif diff --git a/configuration/scripts/machines/env.theia_gnu b/configuration/scripts/machines/env.theia_gnu deleted file mode 100755 index 974be35e0..000000000 --- a/configuration/scripts/machines/env.theia_gnu +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/csh -f - -setenv ICE_MACHINE_ENVNAME theia -setenv ICE_MACHINE_COMPILER gnu -setenv ICE_MACHINE_MAKE make -setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/save/ -setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT " " -setenv ICE_MACHINE_TPNODE 4 -setenv ICE_MACHINE_ACCT P0000000 -setenv ICE_MACHINE_QUEUE "default" -setenv ICE_MACHINE_BLDTHRDS 1 -setenv ICE_MACHINE_QSTAT " " -setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.theia_pgi b/configuration/scripts/machines/env.theia_pgi deleted file mode 100755 index 2c4229c0e..000000000 --- a/configuration/scripts/machines/env.theia_pgi +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/csh -f - -source /etc/profile.d/modules.csh -module list -module purge -module load pgi -module load impi -module load esmf -module load hdf5 netcdf/4.4.0 -module load wgrib wgrib2 -echo renewed modules: -module list - - -setenv ICE_MACHINE_ENVNAME theia -setenv ICE_MACHINE_COMPILER pgi -setenv ICE_MACHINE_MAKE make -setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /scratch3/NCEPDEV/marine/save/Robert.Grumbine/ -setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT " " -setenv ICE_MACHINE_TPNODE 4 -setenv ICE_MACHINE_ACCT P0000000 -setenv ICE_MACHINE_QUEUE "default" -setenv ICE_MACHINE_BLDTHRDS 1 -setenv ICE_MACHINE_QSTAT " " -#setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/tests/rgvers_suite.ts b/configuration/scripts/tests/rgvers_suite.ts deleted file mode 100644 index dc85613db..000000000 --- a/configuration/scripts/tests/rgvers_suite.ts +++ /dev/null @@ -1,18 +0,0 @@ -# Test Grid PEs Sets BFB-compare -#restart gx1 4x4 droundrobin,short -#restart gx1 8x4 droundrobin,short -#restart gx1 12x4 droundrobin,short -#restart gx1 16x4 droundrobin,short -#restart gx1 20x4 droundrobin,short -#restart gx1 24x4 droundrobin,short -#restart gx1 28x4 droundrobin,short -#restart gx1 32x4 droundrobin,short -restart gx1 32x2 droundrobin,short -restart gx1 32x3 droundrobin,short -restart gx1 32x1 droundrobin,short - -restart gx1 33x4 droundrobin,short -restart gx1 34x4 droundrobin,short -restart gx1 35x4 droundrobin,short -restart gx1 36x4 droundrobin,short -#restart gx1 40x4 droundrobin,short From ffe9272b59c2ddd9b80d76a0e2306da90eee2d82 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Thu, 10 Oct 2019 17:59:02 +0000 Subject: [PATCH 30/54] phase3 work --- .../cicedynB/infrastructure/ice_domain.F90 | 4 +- cicecore/shared/ice_arrays_column.F90 | 2 +- cicecore/shared/ice_constants.F90 | 2 +- cicecore/shared/ice_spacecurve.F90 | 84 +++++++++---------- configuration/scripts/cice.batch.csh | 13 ++- .../scripts/machines/env.phase3_intel | 8 +- icepack | 2 +- 7 files changed, 63 insertions(+), 52 deletions(-) diff --git a/cicecore/cicedynB/infrastructure/ice_domain.F90 b/cicecore/cicedynB/infrastructure/ice_domain.F90 index b4b4c4ab2..236f6f179 100644 --- a/cicecore/cicedynB/infrastructure/ice_domain.F90 +++ b/cicecore/cicedynB/infrastructure/ice_domain.F90 @@ -478,12 +478,12 @@ subroutine init_domain_distribution(KMTG,ULATG) if (this_block%j_glob(j) > 0) then do i=this_block%ilo,this_block%ihi if (this_block%i_glob(i) > 0) then - ig = this_block%i_glob(i) + ig = this_block%i_glob(i) jg = this_block%j_glob(j) if (KMTG(ig,jg) > puny .and. & (ULATG(ig,jg) < shlat/rad_to_deg .or. & ULATG(ig,jg) > nhlat/rad_to_deg) ) & - nocn(n) = nocn(n) + flat(ig,jg) + nocn(n) = nocn(n) + flat(ig,jg) endif end do endif diff --git a/cicecore/shared/ice_arrays_column.F90 b/cicecore/shared/ice_arrays_column.F90 index 71d03ed94..c430ddd13 100644 --- a/cicecore/shared/ice_arrays_column.F90 +++ b/cicecore/shared/ice_arrays_column.F90 @@ -268,7 +268,7 @@ module ice_arrays_column real (kind=dbl_kind), dimension(:), allocatable, public :: & R_C2N , & ! algal C to N (mole/mole) R_chl2N , & ! 3 algal chlorophyll to N (mg/mmol) - R_Si2N ! silica to nitrogen mole ratio for algal groups + R_Si2N ! silica to nitrogen mole ratio for algal groups !======================================================================= diff --git a/cicecore/shared/ice_constants.F90 b/cicecore/shared/ice_constants.F90 index c49732e35..15581e787 100644 --- a/cicecore/shared/ice_constants.F90 +++ b/cicecore/shared/ice_constants.F90 @@ -60,7 +60,7 @@ module ice_constants c180 = 180.0_dbl_kind, & c360 = 360.0_dbl_kind, & c365 = 365.0_dbl_kind, & - c400 = 400.0_dbl_kind, & + c400 = 400.0_dbl_kind, & c1000= 1000.0_dbl_kind, & c3600= 3600.0_dbl_kind, & p001 = 0.001_dbl_kind, & diff --git a/cicecore/shared/ice_spacecurve.F90 b/cicecore/shared/ice_spacecurve.F90 index 78b256b8f..ed108e1c7 100644 --- a/cicecore/shared/ice_spacecurve.F90 +++ b/cicecore/shared/ice_spacecurve.F90 @@ -31,7 +31,7 @@ module ice_spacecurve ! !PUBLIC MEMBER FUNCTIONS: public :: GenSpaceCurve, & - IsLoadBalanced + IsLoadBalanced public :: Factor, & IsFactorable, & @@ -41,24 +41,24 @@ module ice_spacecurve ! !PRIVATE MEMBER FUNCTIONS: - private :: map, & - PeanoM, & - Hilbert, & - Cinco, & + private :: map, & + PeanoM, & + Hilbert, & + Cinco, & GenCurve private :: FirstFactor, & FindandMark integer(int_kind), dimension(:,:), allocatable :: & - dir, &! direction to move along each level + dir, &! direction to move along each level ordered ! the ordering integer(int_kind), dimension(:), allocatable :: & - pos ! position along each of the axes + pos ! position along each of the axes integer(int_kind) :: & - maxdim, &! dimensionality of entire space - vcnt ! visitation count + maxdim, &! dimensionality of entire space + vcnt ! visitation count logical :: verbose=.FALSE. @@ -80,7 +80,7 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) ! !DESCRIPTION: ! This subroutine implements a Cinco space-filling curve. ! Cinco curves connect a Nb x Nb block of points where -! +! ! Nb = 5^p ! ! !REVISION HISTORY: @@ -91,12 +91,12 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) ! !INPUT PARAMETERS integer(int_kind), intent(in) :: & - l, & ! level of the space-filling curve + l, & ! level of the space-filling curve type, & ! type of SFC curve - ma, & ! Major axis [0,1] - md, & ! direction of major axis [-1,1] - ja, & ! joiner axis [0,1] - jd ! direction of joiner axis [-1,1] + ma, & ! Major axis [0,1] + md, & ! direction of major axis [-1,1] + ja, & ! joiner axis [0,1] + jd ! direction of joiner axis [-1,1] ! !OUTPUT PARAMETERS @@ -111,12 +111,12 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) !----------------------------------------------------------------------- integer(int_kind) :: & - lma, &! local major axis (next level) - lmd, &! local major direction (next level) - lja, &! local joiner axis (next level) - ljd, &! local joiner direction (next level) - ltype, &! type of SFC on next level - ll ! next level down + lma, &! local major axis (next level) + lmd, &! local major direction (next level) + lja, &! local joiner axis (next level) + ljd, &! local joiner direction (next level) + ltype, &! type of SFC on next level + ll ! next level down logical :: debug = .FALSE. @@ -963,8 +963,8 @@ function IncrementCurve(ja,jd) result(ierr) ! !INPUT PARAMETERS: integer(int_kind) :: & - ja, &! axis to increment - jd ! direction along axis + ja, &! axis to increment + jd ! direction along axis ! !OUTPUT PARAMETERS: integer(int_kind) :: ierr ! error return code @@ -975,7 +975,7 @@ function IncrementCurve(ja,jd) result(ierr) ! mark the newly visited point !----------------------------- ordered(pos(0)+1,pos(1)+1) = vcnt - + !------------------------------------ ! increment curve and update position !------------------------------------ @@ -1064,12 +1064,12 @@ function IsLoadBalanced(nelem,npart) ! !INTPUT PARAMETERS: integer(int_kind), intent(in) :: & - nelem, & ! number of blocks/elements to partition - npart ! size of partition + nelem, & ! number of blocks/elements to partition + npart ! size of partition ! !OUTPUT PARAMETERS: logical :: IsLoadBalanced ! .TRUE. if a perfectly load balanced - ! partition is possible + ! partition is possible !EOP !BOC !----------------------------------------------------------------------- @@ -1077,7 +1077,7 @@ function IsLoadBalanced(nelem,npart) ! local variables ! !----------------------------------------------------------------------- - + integer(int_kind) :: tmp1 ! temporary int character(len=*),parameter :: subname='(IsLoadBalanced)' @@ -1085,10 +1085,10 @@ function IsLoadBalanced(nelem,npart) !----------------------------------------------------------------------- tmp1 = nelem/npart - if (npart*tmp1 == nelem ) then - IsLoadBalanced=.TRUE. + if(npart*tmp1 == nelem ) then + IsLoadBalanced=.TRUE. else - IsLoadBalanced=.FALSE. + IsLoadBalanced=.FALSE. endif !EOP @@ -1285,7 +1285,7 @@ function Factor(num) result(res) !----------------------------------------------------------------------- integer(int_kind) :: & - tmp,tmp2,tmp3,tmp5 ! tempories for the factorization algorithm + tmp,tmp2,tmp3,tmp5 ! tempories for the factorization algorithm integer(int_kind) :: i,n ! loop tempories logical :: found ! logical temporary character(len=*),parameter :: subname='(Factor)' @@ -1437,9 +1437,9 @@ subroutine map(l) !----------------------------------------------------------------------- integer(int_kind) :: & - d, & ! dimension of curve only 2D is supported - type, & ! type of space-filling curve to start off - ierr ! error return code + d, & ! dimension of curve only 2D is supported + type, & ! type of space-filling curve to start off + ierr ! error return code character(len=*),parameter :: subname='(map)' d = SIZE(pos) @@ -1484,8 +1484,8 @@ subroutine PrintCurve(Mesh) ! !----------------------------------------------------------------------- integer(int_kind) :: & - gridsize, &! order of space-filling curve - i ! loop temporary + gridsize, &! order of space-filling curve + i ! loop temporary character(len=*),parameter :: subname='(PrintCurve)' !----------------------------------------------------------------------- @@ -1523,7 +1523,7 @@ subroutine PrintCurve(Mesh) write (*,*) "------------------------------------------" do i=1,gridsize write(*,6) Mesh(1,i),Mesh(2,i),Mesh(3,i), & - Mesh(4,i),Mesh(5,i),Mesh(6,i) + Mesh(4,i),Mesh(5,i),Mesh(6,i) enddo else if(gridsize == 8) then write (*,*) "A Level 3 Hilbert Curve:" @@ -1615,7 +1615,7 @@ subroutine PrintCurve(Mesh) Mesh(13,i),Mesh(14,i),Mesh(15,i),Mesh(16,i), & Mesh(17,i),Mesh(18,i),Mesh(19,i),Mesh(20,i), & Mesh(21,i),Mesh(22,i),Mesh(23,i),Mesh(24,i), & - Mesh(25,i) + Mesh(25,i) enddo else if(gridsize == 27) then write (*,*) "A Level 3 Peano Meandering Curve:" @@ -1683,7 +1683,7 @@ subroutine GenSpaceCurve(Mesh) ! !INPUT/OUTPUT PARAMETERS: integer(int_kind), target,intent(inout) :: & - Mesh(:,:) ! The SFC ordering in 2D array + Mesh(:,:) ! The SFC ordering in 2D array !EOP !BOC @@ -1694,8 +1694,8 @@ subroutine GenSpaceCurve(Mesh) !----------------------------------------------------------------------- integer(int_kind) :: & - level, &! Level of space-filling curve - dim ! dimension of SFC... currently limited to 2D + level, &! Level of space-filling curve + dim ! dimension of SFC... currently limited to 2D integer(int_kind) :: gridsize ! number of points on a side diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index bb2bcabf9..7e87fd2dd 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -181,7 +181,7 @@ EOFB else if (${ICE_MACHINE} =~ hera*) then cat >> ${jobfile} << EOFB #SBATCH -J ${ICE_CASENAME} -#SBATCH -t ${batchtime} +#SBATCH -t `echo ${batchtime} | cut -f1-2 -d:` #SBATCH -q batch #SBATCH -A marine-cpu #SBATCH -N ${nnodes} @@ -198,7 +198,16 @@ EOFB else if (${ICE_MACHINE} =~ phase3*) then cat >> ${jobfile} << EOFB -# nothing to do +#BSUB -J ${ICE_CASENAME} +#BSUB -q "dev" +#BSUB -P RTO-T2O +#BSUB -W 00:29 +#BSUB -n ${nnodes} +#BSUB -R "affinity[core]" +#BSUB -R "span[ptile=${nnodes}]" +#BSUB -R "rusage[mem=16384]" +#BSUB -o /u/Robert.Grumbine/${ICE_CASENAME}.out.%J +#BSUB -e /u/Robert.Grumbine/${ICE_CASENAME}.err.%J EOFB else if (${ICE_MACHINE} =~ high_Sierra*) then diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel index 03aa7083f..01f119f3c 100755 --- a/configuration/scripts/machines/env.phase3_intel +++ b/configuration/scripts/machines/env.phase3_intel @@ -1,8 +1,10 @@ #!/bin/csh -f --login -source /etc/profile.d/lmod.csh +. $MODULESHOME/init/csh +echo done with init.csh module purge +module load lsf/10.1 module load ips/18.0.1.163 module load impi/18.0.1 module load NetCDF/4.5.0 @@ -15,10 +17,10 @@ setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /u/Robert.Grumbine/noscrub/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /u/Robert.Grumbine/noscrub/ setenv ICE_MACHINE_BASELINE /u/Robert.Grumbine/noscrub/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_SUBMIT "bsub < " setenv ICE_MACHINE_TPNODE 4 setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_BLDTHRDS 1 -setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QSTAT "bjobs " diff --git a/icepack b/icepack index 51ea326f1..ac6195df8 160000 --- a/icepack +++ b/icepack @@ -1 +1 @@ -Subproject commit 51ea326f12578355d4d51b0cea29b45cc9890cd1 +Subproject commit ac6195df8f3b5ae9a8e030b28c5a727fb5c11d62 From 571a6e42a87f210c577424fe06f17846c23371a7 Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Thu, 10 Oct 2019 19:06:40 +0000 Subject: [PATCH 31/54] minor manual move towards master copy --- .../cicedynB/analysis/ice_diagnostics.F90 | 2 + .../cicedynB/dynamics/ice_transport_remap.F90 | 2 +- .../comm/mpi/ice_gather_scatter.F90 | 1 - .../comm/serial/ice_boundary.F90 | 3 +- cicecore/drivers/cesm/ice_scam.F90 | 1 - cicecore/shared/ice_init_column.F90 | 17 +- .../scripts/machines/Macros.brooks_intel | 78 +++++ .../scripts/machines/Macros.izumi_gnu | 52 +++ .../scripts/machines/Macros.izumi_intel | 52 +++ .../scripts/machines/Macros.izumi_nag | 53 ++++ .../scripts/machines/Macros.izumi_pgi | 52 +++ .../scripts/machines/Macros.theia_gnu | 48 +++ .../scripts/machines/Macros.theia_pgi | 48 +++ .../scripts/machines/env.brooks_intel | 26 ++ .../scripts/machines/env.hobart_intel | 2 +- configuration/scripts/machines/env.izumi_gnu | 32 ++ .../scripts/machines/env.izumi_intel | 32 ++ configuration/scripts/machines/env.izumi_nag | 32 ++ configuration/scripts/machines/env.izumi_pgi | 32 ++ configuration/scripts/machines/env.theia_gnu | 15 + configuration/scripts/machines/env.theia_pgi | 27 ++ .../scripts/machines/env.travisCI_gnu | 4 +- configuration/scripts/timeseries.csh | 127 ++++++++ configuration/scripts/timeseries.py | 297 ++++++++++++++++++ 24 files changed, 1018 insertions(+), 17 deletions(-) create mode 100644 configuration/scripts/machines/Macros.brooks_intel create mode 100644 configuration/scripts/machines/Macros.izumi_gnu create mode 100644 configuration/scripts/machines/Macros.izumi_intel create mode 100644 configuration/scripts/machines/Macros.izumi_nag create mode 100644 configuration/scripts/machines/Macros.izumi_pgi create mode 100644 configuration/scripts/machines/Macros.theia_gnu create mode 100644 configuration/scripts/machines/Macros.theia_pgi create mode 100755 configuration/scripts/machines/env.brooks_intel create mode 100755 configuration/scripts/machines/env.izumi_gnu create mode 100755 configuration/scripts/machines/env.izumi_intel create mode 100755 configuration/scripts/machines/env.izumi_nag create mode 100755 configuration/scripts/machines/env.izumi_pgi create mode 100755 configuration/scripts/machines/env.theia_gnu create mode 100755 configuration/scripts/machines/env.theia_pgi create mode 100755 configuration/scripts/timeseries.csh create mode 100755 configuration/scripts/timeseries.py diff --git a/cicecore/cicedynB/analysis/ice_diagnostics.F90 b/cicecore/cicedynB/analysis/ice_diagnostics.F90 index 4557b4075..5f8c56264 100644 --- a/cicecore/cicedynB/analysis/ice_diagnostics.F90 +++ b/cicecore/cicedynB/analysis/ice_diagnostics.F90 @@ -124,6 +124,7 @@ subroutine runtime_diags (dt) use ice_global_reductions, only: global_sum, global_sum_prod, global_maxval use ice_grid, only: lmask_n, lmask_s, tarean, tareas use ice_state ! everything +! tcraig, this is likely to cause circular dependency because ice_prescribed_mod is high level routine #ifdef CESMCOUPLED use ice_prescribed_mod, only: prescribed_ice #endif @@ -839,6 +840,7 @@ subroutine runtime_diags (dt) if (print_global) then ! global diags for conservations checks +! tcraig, this is likely to cause circular dependency because ice_prescribed_mod is high level routine #ifdef CESMCOUPLED if (prescribed_ice) then write (nu_diag,*) '----------------------------' diff --git a/cicecore/cicedynB/dynamics/ice_transport_remap.F90 b/cicecore/cicedynB/dynamics/ice_transport_remap.F90 index bf9d0f373..b641104ed 100644 --- a/cicecore/cicedynB/dynamics/ice_transport_remap.F90 +++ b/cicecore/cicedynB/dynamics/ice_transport_remap.F90 @@ -260,7 +260,7 @@ subroutine init_remap ! xxxav, xxyav, xyyav, yyyav integer (kind=int_kind) :: & - i, j, iblk ! standard indices + i, j, iblk ! standard indices character(len=*), parameter :: subname = '(init_remap)' diff --git a/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90 b/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90 index afdfa2dfd..e9eb49db0 100644 --- a/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90 +++ b/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90 @@ -2820,7 +2820,6 @@ subroutine scatter_global_ext(ARRAY, ARRAY_G, src_task, dst_dist) iblk, jblk, &! block indices iglb, jglb, &! global indices nrecvs, &! actual number of messages received - isrc, jsrc, &! source addresses dst_block, &! location of block in dst array ierr ! MPI error flag diff --git a/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90 b/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90 index 9775a9dad..9c2cfd9fc 100644 --- a/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90 +++ b/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90 @@ -3434,8 +3434,7 @@ subroutine ice_HaloUpdate_stress(array1, array2, halo, & ! local variables integer (int_kind) :: & - i,j,n,nmsg, &! dummy loop indices - ierr, &! error or status flag for MPI,alloc + nmsg, &! dummy loop indices nxGlobal, &! global domain size in x (tripole) iSrc,jSrc, &! source addresses for message iDst,jDst, &! dest addresses for message diff --git a/cicecore/drivers/cesm/ice_scam.F90 b/cicecore/drivers/cesm/ice_scam.F90 index 86a56d19c..f5280b259 100644 --- a/cicecore/drivers/cesm/ice_scam.F90 +++ b/cicecore/drivers/cesm/ice_scam.F90 @@ -12,4 +12,3 @@ module ice_scam end module ice_scam - diff --git a/cicecore/shared/ice_init_column.F90 b/cicecore/shared/ice_init_column.F90 index 3f3a5bfe6..f86984195 100644 --- a/cicecore/shared/ice_init_column.F90 +++ b/cicecore/shared/ice_init_column.F90 @@ -444,11 +444,11 @@ subroutine init_shortwave !---------------------------------------------------------------- if (runtype == 'initial' .and. .not. restart) then scale_factor(i,j,iblk) = & - swvdr(i,j,iblk)*(c1 - alvdr_ai(i,j,iblk)) & - + swvdf(i,j,iblk)*(c1 - alvdf_ai(i,j,iblk)) & - + swidr(i,j,iblk)*(c1 - alidr_ai(i,j,iblk)) & - + swidf(i,j,iblk)*(c1 - alidf_ai(i,j,iblk)) - endif + swvdr(i,j,iblk)*(c1 - alvdr_ai(i,j,iblk)) & + + swvdf(i,j,iblk)*(c1 - alvdf_ai(i,j,iblk)) & + + swidr(i,j,iblk)*(c1 - alidr_ai(i,j,iblk)) & + + swidf(i,j,iblk)*(c1 - alidf_ai(i,j,iblk)) + endif enddo ! i enddo ! j @@ -895,8 +895,7 @@ subroutine input_zbgc integer (kind=int_kind) :: & nml_error, & ! namelist i/o error flag - k, mm , & ! loop index - ierr, abort_flag + abort_flag character(len=*), parameter :: subname='(input_zbgc)' @@ -1626,7 +1625,6 @@ subroutine count_tracers ! local variables integer (kind=int_kind) :: & - n , & ! loop index k, mm , & ! loop index nk , & ! layer index nk_bgc ! layer index @@ -2116,7 +2114,6 @@ end subroutine count_tracers subroutine init_zbgc - use ice_broadcast, only: broadcast_scalar use ice_state, only: trcr_base, trcr_depend, n_trcr_strata, & nt_strata use ice_arrays_column, only: R_C2N, R_chl2N, R_C2N_DON, R_Si2N @@ -2186,7 +2183,7 @@ subroutine init_zbgc tr_bgc_DMS, tr_bgc_PON, & tr_bgc_N, tr_bgc_C, tr_bgc_chl, & tr_bgc_DON, tr_bgc_Fe, tr_zaero, & - tr_bgc_hum, tr_aero + tr_bgc_hum real (kind=dbl_kind) :: & initbio_frac, & diff --git a/configuration/scripts/machines/Macros.brooks_intel b/configuration/scripts/machines/Macros.brooks_intel new file mode 100644 index 000000000..5bd725143 --- /dev/null +++ b/configuration/scripts/machines/Macros.brooks_intel @@ -0,0 +1,78 @@ +#============================================================================== +# Makefile macros for ECCC brooks +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 +#-xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +#NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib + +INCLDIR := $(INCLDIR) + +LIB_NETCDF := $(NETCDF_PATH)/lib +LIB_PNETCDF := $(PNETCDF_PATH)/lib +LIB_MPI := $(IMPILIBDIR) + +#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl +#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +### if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib + INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include + SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof + + CPPDEFS := $(CPPDEFS) -Dncdf +endif + +ifeq ($(IO_TYPE), netcdf) + CPPDEFS := $(CPPDEFS) -Dncdf +endif diff --git a/configuration/scripts/machines/Macros.izumi_gnu b/configuration/scripts/machines/Macros.izumi_gnu new file mode 100644 index 000000000..0d48f1013 --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_gnu @@ -0,0 +1,52 @@ +#============================================================================== +# Makefile macros for NCAR izumi, gnu compiler +#============================================================================== + +CPP := /usr/bin/cpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 + +FIXEDFLAGS := -ffixed-line-length-132 +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +SCC := gcc +SFC := gfortran +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS +endif + diff --git a/configuration/scripts/machines/Macros.izumi_intel b/configuration/scripts/machines/Macros.izumi_intel new file mode 100644 index 000000000..502f7b218 --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_intel @@ -0,0 +1,52 @@ +#============================================================================== +# Makefile macros for NCAR izumi, intel compiler +#============================================================================== + +CPP := /usr/bin/cpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -qno-opt-dynamic-align -fp-model precise + +FIXEDFLAGS := -fixed -132 +FREEFLAGS := -free +FFLAGS := -qno-opt-dynamic-align -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -mkl + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS +endif + diff --git a/configuration/scripts/machines/Macros.izumi_nag b/configuration/scripts/machines/Macros.izumi_nag new file mode 100644 index 000000000..8e42df4aa --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_nag @@ -0,0 +1,53 @@ +#============================================================================== +# Makefile macros for NCAR izumi, NAG compiler +#============================================================================== + +CPP := /usr/bin/cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 $(ICE_CPPDEFS) +CFLAGS := -c + +FIXEDFLAGS := -fixed +FREEFLAGS := -free +FFLAGS := -Wp,-macro=no_com -convert=BIG_ENDIAN -wmismatch=mpi_bcast,mpi_isend,mpi_irecv,mpi_send,mpi_recv,mpi_allreduce +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -C=all -g -time -f2003 -ieee=stop +else + FFLAGS += -O2 -ieee=full +endif + +SCC := cc +SFC := nagfor +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -openmp + CFLAGS += -fopenmp + FFLAGS += -openmp +else + FFLAGS += -gline +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiof +endif diff --git a/configuration/scripts/machines/Macros.izumi_pgi b/configuration/scripts/machines/Macros.izumi_pgi new file mode 100644 index 000000000..8a8ca4b97 --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_pgi @@ -0,0 +1,52 @@ +#============================================================================== +# Makefile macros for NCAR izumi, pgi compiler +#============================================================================== + +CPP := pgcc -E +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS} +CFLAGS := -c -O2 -Kieee + +FIXEDFLAGS := -Mextend +FREEFLAGS := -Mfree +FFLAGS := -Kieee -Mbyteswapio -traceback +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -Mbounds -Mchkptr +else + FFLAGS += -O2 +endif + +SCC := pgcc +SFC := pgf90 +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -mp + CFLAGS += -mp + FFLAGS += -mp +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS +endif + diff --git a/configuration/scripts/machines/Macros.theia_gnu b/configuration/scripts/machines/Macros.theia_gnu new file mode 100644 index 000000000..63cce4b99 --- /dev/null +++ b/configuration/scripts/machines/Macros.theia_gnu @@ -0,0 +1,48 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FFLAGS := -fconvert=big-endian +#FFLAGS := -h bytwswapio +#FFLAGS := +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpif90 + +MPICC:= + +MPIFC:= mpif90 +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:= +SFC:= + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.theia_pgi b/configuration/scripts/machines/Macros.theia_pgi new file mode 100644 index 000000000..83bcc264d --- /dev/null +++ b/configuration/scripts/machines/Macros.theia_pgi @@ -0,0 +1,48 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +FC := mpif90 +MPICC:= +MPIFC:= mpif90 +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:= +SFC:= + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.brooks_intel b/configuration/scripts/machines/env.brooks_intel new file mode 100755 index 000000000..4a51b5bb1 --- /dev/null +++ b/configuration/scripts/machines/env.brooks_intel @@ -0,0 +1,26 @@ +#!/bin/csh -f + +source /opt/modules/default/init/csh +module load PrgEnv-intel # Intel compiler +module load cray-mpich # MPI (Cray MPICH) +module load cray-netcdf # NetCDF +module load cray-hdf5 # HDF5 + +setenv ICE_MACHINE_ENVNAME brooks +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/brooks/cice/runs +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/phb001/ +setenv ICE_MACHINE_BASELINE ~/data/brooks/cice/baselines +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_TPNODE 36 +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "development" +setenv ICE_MACHINE_BLDTHRDS 4 +setenv ICE_MACHINE_QSTAT "qstat " + +if (-e ~/.cice_proj) then + set account_name = `head -1 ~/.cice_proj` + setenv CICE_ACCT ${account_name} +endif diff --git a/configuration/scripts/machines/env.hobart_intel b/configuration/scripts/machines/env.hobart_intel index 76f4e5e55..80092297a 100755 --- a/configuration/scripts/machines/env.hobart_intel +++ b/configuration/scripts/machines/env.hobart_intel @@ -15,7 +15,7 @@ module load compiler/intel/18.0.3 endif setenv ICE_MACHINE_ENVNAME hobart -setenv ICE_MACHINE_COMPILER ifort +setenv ICE_MACHINE_COMPILER intel setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata diff --git a/configuration/scripts/machines/env.izumi_gnu b/configuration/scripts/machines/env.izumi_gnu new file mode 100755 index 000000000..9f9938d68 --- /dev/null +++ b/configuration/scripts/machines/env.izumi_gnu @@ -0,0 +1,32 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/gnu/8.2.0 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "short" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_intel b/configuration/scripts/machines/env.izumi_intel new file mode 100755 index 000000000..63d175990 --- /dev/null +++ b/configuration/scripts/machines/env.izumi_intel @@ -0,0 +1,32 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/intel/19.0.1 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "short" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_nag b/configuration/scripts/machines/env.izumi_nag new file mode 100755 index 000000000..785cc410a --- /dev/null +++ b/configuration/scripts/machines/env.izumi_nag @@ -0,0 +1,32 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/nag/6.2 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER nag +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "short" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_pgi b/configuration/scripts/machines/env.izumi_pgi new file mode 100755 index 000000000..b89eafeb8 --- /dev/null +++ b/configuration/scripts/machines/env.izumi_pgi @@ -0,0 +1,32 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/pgi/18.10 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER pgi +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "short" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.theia_gnu b/configuration/scripts/machines/env.theia_gnu new file mode 100755 index 000000000..974be35e0 --- /dev/null +++ b/configuration/scripts/machines/env.theia_gnu @@ -0,0 +1,15 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME theia +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/save/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.theia_pgi b/configuration/scripts/machines/env.theia_pgi new file mode 100755 index 000000000..2c4229c0e --- /dev/null +++ b/configuration/scripts/machines/env.theia_pgi @@ -0,0 +1,27 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +module list +module purge +module load pgi +module load impi +module load esmf +module load hdf5 netcdf/4.4.0 +module load wgrib wgrib2 +echo renewed modules: +module list + + +setenv ICE_MACHINE_ENVNAME theia +setenv ICE_MACHINE_COMPILER pgi +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /scratch3/NCEPDEV/marine/save/Robert.Grumbine/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.travisCI_gnu b/configuration/scripts/machines/env.travisCI_gnu index 687c4ba07..f3c9da26b 100755 --- a/configuration/scripts/machines/env.travisCI_gnu +++ b/configuration/scripts/machines/env.travisCI_gnu @@ -7,7 +7,9 @@ setenv ICE_MACHINE_WKDIR ~/CICE_RUNS setenv ICE_MACHINE_INPUTDATA ~ setenv ICE_MACHINE_BASELINE ~/CICE_BASELINE setenv ICE_MACHINE_SUBMIT " " -setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_TPNODE 4 # maximum tasks per node +setenv ICE_MACHINE_MAXPES 4 # maximum total pes (tasks * threads) available +setenv ICE_MACHINE_MAXRUNLENGTH 1 # maximum batch wall time limit in hours (integer) setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/timeseries.csh b/configuration/scripts/timeseries.csh new file mode 100755 index 000000000..cdd025efc --- /dev/null +++ b/configuration/scripts/timeseries.csh @@ -0,0 +1,127 @@ +#!/bin/csh + +# Check to see if test case directory was passed +if ( $1 == "-h" ) then + echo "To generate timeseries plots, this script can be passed a directory" + echo "containing a logs/ subdirectory, or it can be run in the directory with" + echo "the log files, without being passed a directory." + echo "Example: ./timeseries.csh ./annual_gx3_conrad_4x1.t00" + echo "Example: ./timeseries.csh" + echo "It will pull the diagnostic data from the most recently modified log file." + exit -1 +endif +set basename = `echo $1 | sed -e 's#/$##' | sed -e 's/^\.\///'` + +# Set x-axis limits + # Manuallyl set x-axis limits +#set xrange = 'set xrange ["19980101":"19981231"]' + # Let gnuplot determine x-alis limits +set xrange = '' + +# Determine if BASELINE dataset exists +if ( $1 == "" ) then +set basefile_dir = "IGNORE" +else +source $1/cice.settings +set basefile_dir = "$ICE_BASELINE/$ICE_BASECOM/$ICE_TESTNAME" +endif + +if ( -d $basefile_dir ) then + set num_basefile = `ls $basefile_dir | grep cice.runlog | wc -l` + if ( $num_basefile > 0 ) then + set baseline_exists = 1 + foreach file ($basefile_dir/cice.runlog.*) + set base_logfile = $file + end + else + set baseline_exists = 0 + endif +else + set baseline_exists = 0 +endif + +set fieldlist=("total ice area (km^2)" \ + "total ice extent(km^2)" \ + "total ice volume (m^3)" \ + "total snw volume (m^3)" \ + "rms ice speed (m/s)" ) + +# Get the filename for the latest log +if ( $1 == "" ) then +foreach file (./cice.runlog.*) + set logfile = $file +end +else +foreach file ($1/logs/cice.runlog.*) + set logfile = $file +end +endif + +# Loop through each field and create the plot +foreach field ($fieldlist:q) + set fieldname = `echo "$field" | sed -e 's/([^()]*)//g'` + set search = "'$fieldname'\|istep1" + rm -f data.txt + foreach line ("`egrep $search $logfile`") + if ("$line" =~ *"istep1"*) then + set argv = ( $line ) + set date = $4 + @ hour = ( $6 / 3600 ) + else + set data1 = `echo $line | rev | cut -d ' ' -f2 | rev` + set data2 = `echo $line | rev | cut -d ' ' -f1 | rev` + echo "$date-$hour,$data1,$data2" >> data.txt + endif + end + set format = "%Y%m%d-%H" + + set output = `echo $fieldname | sed 's/ /_/g'` + set output = "${output}_${ICE_CASENAME}.png" + + echo "Plotting data for '$fieldname' and saving to $output" + +# Call the plotting routine, which uses the data in the data.txt file +gnuplot << EOF > $output +# Plot style +set style data points + +set datafile separator "," + +# Term type and background color, canvas size +set terminal png size 1920,960 + +# x-axis +set xdata time +set timefmt "$format" +set format x "%Y/%m/%d" + +# Axis tick marks +set xtics rotate + +set title "$field (Diagnostic Output)" +set ylabel "$field" +set xlabel "Simulation Day" + +set key left top + +# Set x-axlis limits +$xrange + +if ( $baseline_exists == 1 ) \ + plot "data_baseline.txt" using (timecolumn(1)):2 with lines lw 2 lt 2 lc 2 title \ + "Arctic - Baseline", \ + "" using (timecolumn(1)):3 with lines lw 2 lt 2 lc 5 title "Antarctic - Baseline", \ + "data.txt" using (timecolumn(1)):2 with lines lw 2 lt 1 lc 1 title "Arctic", \ + "" using (timecolumn(1)):3 with lines lw 2 lt 1 lc 3 title "Antarctic"; \ +else \ + plot "data.txt" using (timecolumn(1)):2 with lines lw 2 lt 1 lc 1 title "Arctic", \ + "" using (timecolumn(1)):3 with lines lw 2 lt 1 lc 3 title "Antarctic" \ + +EOF + +# Delete the data file +rm -f data.txt +if ( $baseline_exists ) then + rm -f data_baseline.txt +endif +end diff --git a/configuration/scripts/timeseries.py b/configuration/scripts/timeseries.py new file mode 100755 index 000000000..2b50c373a --- /dev/null +++ b/configuration/scripts/timeseries.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python + +''' +This script generates timeseries plots of CICE diagnostic output. +It is generated to replicate the previous timeseries.csh script. + +Written by: Matthew Turner +Date: August, 2019 +''' + +import os +import sys +import logging +import numpy as np + +def find_logfile(log_dir): + ''' + This function searches for the most recently created log file in the provided directory. + ''' + + logger.debug('Getting a list of files in {}'.format(log_dir)) + try: + path = '{}/logs'.format(log_dir.rstrip('/')) + files = [os.path.join(path,f) for f in os.listdir('{}/logs'.format(log_dir)) \ + if f.startswith('cice.runlog')] + except: + path = log_dir + files = [os.path.join(path,f) for f in os.listdir(log_dir) if f.startswith('cice.runlog')] + + # Check if any files were found. If not, exit + if len(files) == 0: + logger.error('No cice.runlog* files found. Please make sure you are passing the \ + correct directory.') + sys.exit(1) + + # Get the most recently created file + outfile = max(files, key = os.path.getctime) + + logger.debug('List of files = {}'.format([f for f in files])) + logger.debug('Most recent file is {}'.format(outfile)) + + return outfile + +def get_data(logfile,field): + ''' + This function extracts data from a CICE log file for the specific field. + ''' + import datetime + import re + + logger.debug('Extracting data for {}'.format(field)) + + # Build the regular expression to extract the data + field_regex = field.replace('(','\(').replace('^','\^').replace(')','\)') + number_regex = '[-+]?\d+\.?\d+([eE][-+]?\d+)?' + my_regex = '{}\s+=\s+({})\s+({})'.format(field_regex,number_regex,number_regex) + + dtg = [] + arctic = [] + antarctic = [] + with open(logfile) as f: + for line in f.readlines(): + m1 = re.search('istep1:\s+(\d+)\s+idate:\s+(\d+)\s+sec:\s+(\d+)', line) + if m1: + # Extract the current date-time group from the file + date = m1.group(2) + seconds = int(m1.group(3)) + hours = seconds // 3600 + minutes = (seconds - hours*3600) // 60 + leftover = seconds - hours*3600 - minutes*60 + curr_date = '{}-{:02d}:{:02d}:{:02d}'.format(date,hours,minutes,leftover) + dtg.append(datetime.datetime.strptime(curr_date, '%Y%m%d-%H:%M:%S')) + logger.debug('Currently on timestep {}'.format(dtg[-1])) + + m = re.search(my_regex, line) + if m: + # Extract the data from the file + if 'E' in m.group(1) or 'e' in m.group(1): + expon = True + else: + expon = False + arctic.append(float(m.group(1))) + antarctic.append(float(m.group(3))) + logger.debug(' Arctic = {}, Antarctic = {}'.format(arctic[-1], antarctic[-1])) + + return dtg, arctic, antarctic, expon + +def latexit(string): + s = string[::-1].replace('(','($',1) + return (s.replace(')','$)',1))[::-1] + +def plot_timeseries(log, field, dtg, arctic, antarctic, expon, dtg_base=None, arctic_base=None, \ + antarctic_base=None, base_dir=None, grid=False): + ''' + Plot the timeseries data from the CICE log file + ''' + + casename = os.path.abspath(log).rstrip('/').rstrip('/logs').split('/')[-1] + if base_dir: + base_casename = os.path.abspath(base_dir).rstrip('/').rstrip('/logs').split('/')[-1] + + # Load the plotting libraries, but set the logging level for matplotlib + # to WARNING so that matplotlib debugging info is not printed when running + # with '-v' + logging.getLogger('matplotlib').setLevel(logging.WARNING) + import matplotlib.pyplot as plt + import matplotlib.dates as mdates + import matplotlib.ticker as ticker + + fig = plt.figure(figsize=(12,8)) + ax = fig.add_axes([0.05,0.08,0.9,0.9]) + + # Add the arctic data to the plot + ax.plot(dtg,arctic,label='Arctic') + # Add the baseline arctic data to the plot, if available + if arctic_base: + ax.plot(dtg_base,arctic_base,label='Baseline Arctic') + + # Add the antarctic data to the plot + ax.plot(dtg,antarctic,label='Antarctic') + # Add the baseline antarctic data to the plot, if available + if antarctic_base: + ax.plot(dtg_base,antarctic_base,label='Baseline Antarctic') + + ax.set_xlabel('') + ax.set_title('{} Diagnostic Output'.format(latexit(field))) + ax.set_ylabel(latexit(field)) + + # Format the x-axis labels + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y/%m/%d')) + ax.xaxis.set_minor_locator(mdates.MonthLocator()) + + # Add a text box that prints the test case name and the baseline case name (if given) + try: + text_field = "Test/Case: {}\nBaseline: {}".format(casename,base_casename) + from matplotlib.offsetbox import AnchoredText + anchored_text = AnchoredText(text_field,loc=2) + ax.add_artist(anchored_text) + except: + text_field = "Test/Case: {}".format(casename) + from matplotlib.offsetbox import AnchoredText + anchored_text = AnchoredText(text_field,loc=2) + ax.add_artist(anchored_text) + + ax.legend(loc='upper right') + + # Add grid lines if the `--grid` argument was passed at the command line. + if grid: + ax.grid(ls='--') + + # Reduce the number of ticks on the y axis + nbins = 10 + try: + minval = min( \ + min(min(arctic), min(antarctic)), \ + min(min(arctic_base), min(antarctic_base))) + maxval = max( \ + max(max(arctic), max(antarctic)), \ + max(max(arctic_base), max(antarctic_base))) + except: + minval = min(min(arctic), min(antarctic)) + maxval = max(max(arctic), max(antarctic)) + step = (maxval-minval)/nbins + ax.yaxis.set_ticks(np.arange(minval, maxval+step, step)) + + # Format the y-axis tick labels, based on whether or not the values in the log file + # are in scientific notation or float notation. + if expon: + ax.yaxis.set_major_formatter(ticker.FormatStrFormatter('%0.3e')) + else: + ax.yaxis.set_major_formatter(ticker.FormatStrFormatter('%0.5f')) + + # Rotate and right align the x labels + for tick in ax.get_xticklabels(): + tick.set_rotation(45) + + # Create an output file and save the figure + field_tmp = field.split('(')[0].rstrip() + try: + outfile = '{}_{}_base-{}.png'.format(field_tmp.replace(' ','_'), casename,base_casename) + except: + outfile = '{}_{}.png'.format(field_tmp.replace(' ','_'), casename) + logger.info('Saving file to {}'.format(outfile)) + plt.savefig(outfile,dpi=300,bbox_inches='tight') + +def main(): + import argparse + parser = argparse.ArgumentParser(description="To generate timeseries plots, this script \ + can be passed a directory containing a logs/ subdirectory, \ + or it can be run in the directory with the log files, \ + without being passed a directory. It will pull the \ + diagnostic data from the most recently modified log file.\ + \ + If no flags are passed selecting the variables to plot, \ + then plots will be created for all available variables.") + parser.add_argument('log_dir', nargs='?', default=os.getcwd(), \ + help="Path to diagnostic output log file. A specific log file can \ + be passed, or a case directory. If a directory is passed, \ + the most recent log file will be used. If no directory or \ + file is passed, the script will look for a log file in the \ + current directory.") + parser.add_argument('--bdir',dest='base_dir', help='Path to the the log file for a baseline \ + dataset, if desired. A specific log file or case directory can \ + be passed. If a directory is passed, the most recent log file \ + will be used.') + parser.add_argument('-v', '--verbose', dest='verbose', help='Print debug output?', \ + action='store_true') + parser.add_argument('--area', dest='area', help='Create a plot for total ice area?', \ + action='store_true') + parser.add_argument('--extent', dest='extent', help='Create a plot for total ice extent?', \ + action='store_true') + parser.add_argument('--volume', dest='ice_volume', help='Create a plot for total ice volume?', \ + action='store_true') + parser.add_argument('--snw_vol', dest='snow_volume', help='Create a plot for total snow \ + volume?', action='store_true') + parser.add_argument('--speed', dest='speed', help='Create a plot for rms ice speed?', \ + action='store_true') + parser.add_argument('--grid',dest='grid', help='Add grid lines to the figures?', \ + action='store_true') + + # Set the defaults for the command line options + parser.set_defaults(verbose=False) + parser.set_defaults(area=False) + parser.set_defaults(extent=False) + parser.set_defaults(ice_volume=False) + parser.set_defaults(snow_volume=False) + parser.set_defaults(speed=False) + parser.set_defaults(grid=False) + + args = parser.parse_args() + + # If no fields are passed, plot all fields + if not ( args.area or args.extent or args.ice_volume or args.snow_volume or args.speed ): + args.area = True + args.extent = True + args.ice_volume = True + args.snow_volume = True + args.speed = True + + # Build the fieldlist based on which fields are passed + fieldlist = [] + if args.area: + fieldlist.append('total ice area (km^2)') + if args.extent: + fieldlist.append('total ice extent(km^2)') + if args.ice_volume: + fieldlist.append('total ice volume (m^3)') + if args.snow_volume: + fieldlist.append('total snw volume (m^3)') + if args.speed: + fieldlist.append('rms ice speed (m/s)') + + # Setup the logger + global logger + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + + # Find the test and baseline log files, based on the input directories. + if os.path.isdir(args.log_dir): + logger.debug('{} is a directory'.format(args.log_dir)) + log = find_logfile(args.log_dir) + log_dir = args.log_dir + else: + logger.debug('{} is a file'.format(args.log_dir)) + log = args.log_dir + log_dir = args.log_dir.rsplit('/',1)[0] + logger.info('Log file = {}'.format(log)) + if args.base_dir: + if os.path.isdir(args.base_dir): + base_log = find_logfile(args.base_dir) + base_dir = args.base_dir + else: + base_log = args.base_dir + base_dir = args.base_dir.rsplit('/',1)[0] + logger.info('Base Log file = {}'.format(base_log)) + + # Loop through each field and create the plot + for field in fieldlist: + logger.debug('Current field = {}'.format(field)) + + # Get the data from the log files + dtg, arctic, antarctic, expon = get_data(log, field) + if args.base_dir: + dtg_base, arctic_base, antarctic_base, expon_base = get_data(base_log,field) + + # Plot the data + if args.base_dir: + plot_timeseries(log_dir, field, dtg, arctic, antarctic, expon, dtg_base, \ + arctic_base, antarctic_base, base_dir, grid=args.grid) + else: + plot_timeseries(log_dir, field, dtg, arctic, antarctic, expon, grid=args.grid) + +if __name__ == "__main__": + main() From 088dc11f5bf5d13369a5ab61a000c0a1cef4191f Mon Sep 17 00:00:00 2001 From: "Robert.Grumbine" Date: Thu, 10 Oct 2019 20:19:09 +0000 Subject: [PATCH 32/54] re-fix phase3 script --- configuration/scripts/machines/env.phase3_intel | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel index 01f119f3c..4c8f65eee 100755 --- a/configuration/scripts/machines/env.phase3_intel +++ b/configuration/scripts/machines/env.phase3_intel @@ -1,7 +1,9 @@ -#!/bin/csh -f --login +#!/bin/csh -f -. $MODULESHOME/init/csh +#. $MODULESHOME/init/csh +source $MODULESHOME/init/csh echo done with init.csh +module use -a $MODULESHOME/modulefiles module purge module load lsf/10.1 From 6727489c66a77d14a8ebc05e0f7745f11f968798 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Fri, 18 Oct 2019 16:02:14 +0000 Subject: [PATCH 33/54] nearing full phase3 --- configuration/scripts/cice.batch.csh | 15 +++++++++++---- .../scripts/machines/Macros.cheyenne_intel | 0 .../scripts/machines/Macros.hobart_intel | 0 configuration/scripts/machines/Macros.hobart_nag | 0 .../scripts/machines/Macros.phase2_intel | 0 .../scripts/machines/Macros.phase3_intel | 0 .../scripts/machines/Macros.testmachine_intel | 0 configuration/scripts/machines/env.cesium_intel | 0 configuration/scripts/machines/env.millikan_intel | 0 9 files changed, 11 insertions(+), 4 deletions(-) mode change 100755 => 100644 configuration/scripts/machines/Macros.cheyenne_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.hobart_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.hobart_nag mode change 100755 => 100644 configuration/scripts/machines/Macros.phase2_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.phase3_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.testmachine_intel mode change 100644 => 100755 configuration/scripts/machines/env.cesium_intel mode change 100644 => 100755 configuration/scripts/machines/env.millikan_intel diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 7e87fd2dd..2f21054dd 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -197,15 +197,22 @@ cat >> ${jobfile} << EOFB EOFB else if (${ICE_MACHINE} =~ phase3*) then +if ( ${nnodes} > 15) then + setenv ptile 16 +else if ( ${nnodes} > 3) then + setenv ptile 4 +else + setenv ptile 1 +endif cat >> ${jobfile} << EOFB #BSUB -J ${ICE_CASENAME} -#BSUB -q "dev" +#BSUB -q "dev_shared" #BSUB -P RTO-T2O -#BSUB -W 00:29 +#BSUB -W `echo ${batchtime} | cut -f1-2 -d:` #BSUB -n ${nnodes} #BSUB -R "affinity[core]" -#BSUB -R "span[ptile=${nnodes}]" -#BSUB -R "rusage[mem=16384]" +#BSUB -R "span[ptile=$ptile]" +#BSUB -R "rusage[mem=2048]" #BSUB -o /u/Robert.Grumbine/${ICE_CASENAME}.out.%J #BSUB -e /u/Robert.Grumbine/${ICE_CASENAME}.err.%J EOFB diff --git a/configuration/scripts/machines/Macros.cheyenne_intel b/configuration/scripts/machines/Macros.cheyenne_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.hobart_intel b/configuration/scripts/machines/Macros.hobart_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.hobart_nag b/configuration/scripts/machines/Macros.hobart_nag old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.phase2_intel b/configuration/scripts/machines/Macros.phase2_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.phase3_intel b/configuration/scripts/machines/Macros.phase3_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.testmachine_intel b/configuration/scripts/machines/Macros.testmachine_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/env.cesium_intel b/configuration/scripts/machines/env.cesium_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/env.millikan_intel b/configuration/scripts/machines/env.millikan_intel old mode 100644 new mode 100755 From 3e27ea182919f547c786b7599e3699ea465016bc Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Fri, 18 Oct 2019 17:09:50 -0400 Subject: [PATCH 34/54] update path for data --- configuration/scripts/machines/env.high_Sierra_gnu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/scripts/machines/env.high_Sierra_gnu b/configuration/scripts/machines/env.high_Sierra_gnu index cabd66944..db8bc01a3 100755 --- a/configuration/scripts/machines/env.high_Sierra_gnu +++ b/configuration/scripts/machines/env.high_Sierra_gnu @@ -4,7 +4,7 @@ setenv ICE_MACHINE_ENVNAME high_Sierra setenv ICE_MACHINE_COMPILER gnu setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR /Volumes/ncep/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /Volumes/Data/CICE_data +setenv ICE_MACHINE_INPUTDATA /Volumes/Data/ setenv ICE_MACHINE_BASELINE /Volumes/ncep/CICE_BASELINE setenv ICE_MACHINE_SUBMIT " " setenv ICE_MACHINE_TPNODE 4 From 536b50c481e8c4dc3a57e788600867e0e6dce073 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 13:33:47 +0000 Subject: [PATCH 35/54] actual final version of phase3 --- configuration/scripts/cice.batch.csh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 2f21054dd..30fe08714 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -198,12 +198,13 @@ EOFB else if (${ICE_MACHINE} =~ phase3*) then if ( ${nnodes} > 15) then - setenv ptile 16 -else if ( ${nnodes} > 3) then - setenv ptile 4 + setenv p3tile 16 + setenv mem `expr 100 \* 1024 / $nnodes` else - setenv ptile 1 + setenv p3tile ${nnodes} + setenv mem 8192 endif +echo mem = ${mem} nnodes and p3tiles ${nnodes} ${p3tile} p3tile must be le nnodes cat >> ${jobfile} << EOFB #BSUB -J ${ICE_CASENAME} #BSUB -q "dev_shared" @@ -211,8 +212,8 @@ cat >> ${jobfile} << EOFB #BSUB -W `echo ${batchtime} | cut -f1-2 -d:` #BSUB -n ${nnodes} #BSUB -R "affinity[core]" -#BSUB -R "span[ptile=$ptile]" -#BSUB -R "rusage[mem=2048]" +#BSUB -R "span[ptile=${p3tile}]" +#BSUB -R "rusage[mem=${mem}]" #BSUB -o /u/Robert.Grumbine/${ICE_CASENAME}.out.%J #BSUB -e /u/Robert.Grumbine/${ICE_CASENAME}.err.%J EOFB From 6141edca4cfa124efaeed583188bdbba6e80b790 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 21:11:46 +0000 Subject: [PATCH 36/54] from master --- cice.setup | 141 +++++++--- doc/source/conf.py | 6 +- doc/source/developer_guide/dg_about.rst | 11 + doc/source/developer_guide/dg_dynamics.rst | 14 +- doc/source/developer_guide/dg_other.rst | 64 ++--- doc/source/developer_guide/dg_scripts.rst | 26 +- doc/source/intro/copyright.rst | 2 +- doc/source/intro/quickstart.rst | 2 + doc/source/master_list.bib | 17 ++ doc/source/science_guide/sg_dynamics.rst | 10 +- doc/source/user_guide/ug_case_settings.rst | 3 +- doc/source/user_guide/ug_implementation.rst | 134 ++++----- doc/source/user_guide/ug_running.rst | 276 ++++++++++++++++++- doc/source/user_guide/ug_testing.rst | 69 +---- doc/source/user_guide/ug_troubleshooting.rst | 3 + 15 files changed, 570 insertions(+), 208 deletions(-) diff --git a/cice.setup b/cice.setup index d8c67f4ca..13ba00f85 100755 --- a/cice.setup +++ b/cice.setup @@ -377,7 +377,6 @@ else exit -1 endif cp -f ${ICE_SCRIPTS}/tests/report_results.csh ${tsdir} - cp -f ${ICE_SCRIPTS}/tests/timeseries.csh ${tsdir} cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir} foreach file (${tsdir}/suite.run ${tsdir}/suite.submit) @@ -433,6 +432,7 @@ if (${doabort} == true) then exit -1 endif +# Create a new sets_base variable to store sets passed to cice.setup set sets_base = "${sets}" set bfbcomp_base = "$bfbcomp" foreach compiler ( $ncompilers ) @@ -458,13 +458,14 @@ EOF continue endif + source ${ICE_SCRIPTS}/machines/env.${machcomp} -nomodules || exit 2 + # Obtain the test name, sets, grid, and PE information from .ts file set test = `echo $line | cut -d' ' -f1` set grid = `echo $line | cut -d' ' -f2` set pesx = `echo $line | cut -d' ' -f3` set sets_tmp = `echo $line | cut -d' ' -f4` set bfbcomp_tmp = `echo $line | cut -d' ' -f5` - # Create a new sets_base variable to store sets passed to cice.setup # Append sets from .ts file to the $sets variable set sets = "$sets_base,$sets_tmp" @@ -482,6 +483,107 @@ EOF set fbfbcomp = ${machcomp}_${bfbcomp} endif + #------------------------------------------------------------ + # Parse pesx with strict checking, limit pes for machine + + set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'` + if (${chck} == OK) then + set task = `echo ${pesx} | cut -d x -f 1` + set thrd = `echo ${pesx} | cut -d x -f 2` + set blckx = `echo ${pesx} | cut -d x -f 3` + set blcky = `echo ${pesx} | cut -d x -f 4` + set mblck = `echo ${pesx} | cut -d x -f 5` + if (${task} == 0 || ${thrd} == 0 || ${blckx} == 0 || ${blcky} == 0 || ${mblck} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 + endif + else + set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'` + if (${chck} == OK) then + set task = `echo ${pesx} | cut -d x -f 1` + set thrd = `echo ${pesx} | cut -d x -f 2` + set blckx = `echo ${pesx} | cut -d x -f 3` + set blcky = `echo ${pesx} | cut -d x -f 4` + set mblck = 0 + if (${task} == 0 || ${thrd} == 0 || ${blckx} == 0 || ${blcky} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 + endif + else + set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*$/OK/'` + if (${chck} == OK) then + set task = `echo ${pesx} | cut -d x -f 1` + set thrd = `echo ${pesx} | cut -d x -f 2` + set blckx = 0 + set blcky = 0 + set mblck = 0 + if (${task} == 0 || ${thrd} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 + endif + else + set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*$/OK/'` + if (${chck} == OK) then + set task = `echo ${pesx} | cut -d x -f 1` + set thrd = 1 + set blckx = 0 + set blcky = 0 + set mblck = 0 + if (${task} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 + endif + else + echo "${0}: ERROR in -p argument, ${pesx}, must be [m], [m]x[n], [m]x[n]x[bx]x[by], or [m]x[n]x[bx]x[by]x[mb] " + exit -1 + endif + endif + endif + endif + + @ blkreq = ${task} * ${mblck} + # check max threads, reduce threads, increase tasks + if ($?ICE_MACHINE_MAXTHREADS) then + if (${thrd} > ${ICE_MACHINE_MAXTHREADS}) then + @ pesreq = ${task} * ${thrd} + @ task = ${pesreq} / ${ICE_MACHINE_MAXTHREADS} + @ thrd = ${ICE_MACHINE_MAXTHREADS} + @ peschk = ${task} * ${thrd} + if (${pesreq} > ${peschk}) then + @ task = ${task} + 1 + endif +# echo "tcx1 reset to $task $thrd $mblck" + endif + endif + + # check max pes, reduce tasks + if ($?ICE_MACHINE_MAXPES) then + @ pesreq = ${task} * ${thrd} + if (${pesreq} > ${ICE_MACHINE_MAXPES}) then + @ task = ${ICE_MACHINE_MAXPES} / ${thrd} +# echo "tcx2 reset to $task $thrd $mblck" + endif + endif + + # check max blocks and adjust as needed + if ($mblck > 0) then + @ mblck = ${blkreq} / ${task} + @ blkchk = ${task} * ${mblck} + if (${blkreq} > ${blkchk}) then + @ mblck = $mblck + 1 + endif +# echo "tcx3 reset to $task $thrd $mblck" + endif + + # update pesx based on use defined settings and machine limits to reflect actual value + set pesx = ${task}x${thrd}x${blckx}x${blcky}x${mblck} + if (${mblck} == 0) then + set pesx = ${task}x${thrd}x${blckx}x${blcky} + endif + if (${blckx} == 0 || ${blcky} == 0) then + set pesx = ${task}x${thrd} + endif + set testname_noid = ${spval} # create case for test cases if (${docase} == 0) then @@ -533,7 +635,7 @@ EOF endif # from basic script dir to case - foreach file (cice.build cice.settings Makefile ice_in makdep.c setup_run_dirs.csh) + foreach file (cice.build cice.settings Makefile ice_in makdep.c setup_run_dirs.csh timeseries.csh timeseries.py) if !(-e ${ICE_SCRIPTS}/$file) then echo "${0}: ERROR, ${ICE_SCRIPTS}/$file not found" exit -1 @@ -560,7 +662,6 @@ EOF end cd ${casedir} - source ./env.${machcomp} -nomodules || exit 2 set quietmode = false if ($?ICE_MACHINE_QUIETMODE) then @@ -586,36 +687,6 @@ EOF #------------------------------------------------------------ # Compute a default blocksize - set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'` - if (${chck} == OK) then - set task = `echo ${pesx} | cut -d x -f 1` - set thrd = `echo ${pesx} | cut -d x -f 2` - set blckx = `echo ${pesx} | cut -d x -f 3` - set blcky = `echo ${pesx} | cut -d x -f 4` - set mblck = `echo ${pesx} | cut -d x -f 5` - else - set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'` - if (${chck} == OK) then - set task = `echo ${pesx} | cut -d x -f 1` - set thrd = `echo ${pesx} | cut -d x -f 2` - set blckx = `echo ${pesx} | cut -d x -f 3` - set blcky = `echo ${pesx} | cut -d x -f 4` - set mblck = 0 - else - set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*$/OK/'` - if (${chck} == OK) then - set task = `echo ${pesx} | cut -d x -f 1` - set thrd = `echo ${pesx} | cut -d x -f 2` - set blckx = 0 - set blcky = 0 - set mblck = 0 - else - echo "${0}: ERROR in -p argument, ${pesx}, must be [m]x[n], [m]x[n]x[bx]x[by], or [m]x[n]x[bx]x[by]x[mb] " - exit -1 - endif - endif - endif - setenv ICE_DECOMP_GRID ${grid} setenv ICE_DECOMP_NTASK ${task} setenv ICE_DECOMP_NTHRD ${thrd} @@ -847,7 +918,7 @@ echo "${testname_base}" cd ${testname_base} source ./cice.settings set ciceexe = "../ciceexe.\${ICE_COMPILER}.\${ICE_COMMDIR}.\${ICE_BLDDEBUG}.\${ICE_THREADED}.\${ICE_IOTYPE}" -./cice.build \${ciceexe} +./cice.build --exe \${ciceexe} if !(-e \${ciceexe}) cp -p \${ICE_RUNDIR}/cice \${ciceexe} EOF end diff --git a/doc/source/conf.py b/doc/source/conf.py index 2599d0151..4404ecbb1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -54,7 +54,7 @@ # General information about the project. project = u'CICE' -copyright = u'2018, Triad National Security, LLC (code) and National Center for Atmospheric Research (documentation)' +copyright = u'2019, Triad National Security, LLC (code) and National Center for Atmospheric Research (documentation)' author = u'CICE-Consortium' # The version info for the project you're documenting, acts as replacement for @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = u'6.0.0' +version = u'6.0.2' # The full version, including alpha/beta/rc tags. -version = u'6.0.0' +version = u'6.0.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/developer_guide/dg_about.rst b/doc/source/developer_guide/dg_about.rst index 0fae7db05..bd06d7a3f 100755 --- a/doc/source/developer_guide/dg_about.rst +++ b/doc/source/developer_guide/dg_about.rst @@ -16,3 +16,14 @@ Guiding principles for the creation of CICE include the following: on individual gridcells, and contain no underlying infrastructure. CICE must call into Icepack using interfaces and approaches specified by Icepack. + +Git workflow and Pull Requests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There is extensive Information for Developers documentation available. +See https://github.com/CICE-Consortium/About-Us/wiki/Resource-Index#information-for-developers +for information on: + - Contributing to model development + - Software development practices guide + - git Workflow Guide - including extensive information about the Pull Request process and requirements + - Documentation Workflow Guide diff --git a/doc/source/developer_guide/dg_dynamics.rst b/doc/source/developer_guide/dg_dynamics.rst index 83933e591..21790f199 100755 --- a/doc/source/developer_guide/dg_dynamics.rst +++ b/doc/source/developer_guide/dg_dynamics.rst @@ -37,8 +37,18 @@ The dynamics solvers are found in **cicecore/cicedynB/dynamics/**. A couple of available including EVP, revised EVP, and EAP. The dynamics solver is specified in namelist with the ``kdyn`` variable. ``kdyn=1`` is evp, ``kdyn=2`` is eap, and revised evp requires the ``revised_evp`` namelist flag be set to true. -A vectorized version of EVP is available through the namelist flag ``evp_kernel_ver``. Default is "normal" -EVP as usual ``evp_kernel_ver=0``, whereas an vectorized version (ver.2) is available ``evp_kernel_ver=2``. + +Multiple evp solvers are supported thru the namelist flag ``kevp_kernel``. The standard implementation +and current default is ``kevp_kernel=0``. In this case, the stress is solved on the regular decomposition +via subcycling and calls to subroutine stress and subroutine stepu with MPI global sums required in each +subcycling call. With ``kevp_kernel=2``, the data required to compute the stress is gathered to the root +MPI process and the stress calculation is performed on the root task without any MPI global sums. OpenMP +parallelism is supported in ``kevp_kernel=2``. The solutions with ``kevp_kernel`` set to 0 or 2 will +not be bit-for-bit +identical but should be the same to roundoff and produce the same climate. ``kevp_kernel=2`` may perform +better for some configurations, some machines, and some pe counts. ``kevp_kernel=2`` is not supported +with the tripole grid and is still being validated. Until ``kevp_kernel=2`` is fully validated, it will +abort if set. To override the abort, use value 102 for testing. Transport diff --git a/doc/source/developer_guide/dg_other.rst b/doc/source/developer_guide/dg_other.rst index a8c959820..2a46186bd 100644 --- a/doc/source/developer_guide/dg_other.rst +++ b/doc/source/developer_guide/dg_other.rst @@ -160,57 +160,57 @@ Additional information about tracers can be found in the To add a tracer, follow these steps using one of the existing tracers as a pattern. -#. **icepack\_tracers.F90** and **icepack\_[tracer].F90**: declare tracers, -add flags and indices, and create physics routines as described in the -`Icepack documentation `_ + 1) **icepack\_tracers.F90** and **icepack\_[tracer].F90**: declare tracers, + add flags and indices, and create physics routines as described in the + `Icepack documentation `_ -#. **ice_arrays_column.F90**: declare arrays + 2) **ice_arrays_column.F90**: declare arrays -#. **ice_init_column.F90**: initialize arrays + 3) **ice_init_column.F90**: initialize arrays -#. **ice\_init.F90**: (some of this may be done in **icepack\_[tracer].F90** - instead) + 4) **ice\_init.F90**: (some of this may be done in **icepack\_[tracer].F90** + instead) - - declare ``tr_[tracer]`` and ``nt_[tracer]`` as needed + - declare ``tr_[tracer]`` and ``nt_[tracer]`` as needed - - add logical namelist variables ``tr_[tracer]``, ``restart_[tracer]`` + - add logical namelist variables ``tr_[tracer]``, ``restart_[tracer]`` - - initialize and broadcast namelist variables + - initialize and broadcast namelist variables - - check for potential conflicts, aborting if any occur + - check for potential conflicts, aborting if any occur - - print namelist variables to diagnostic output file + - print namelist variables to diagnostic output file - - initialize tracer flags etc in icepack (call *icepack_init_tracer_flags* etc) + - initialize tracer flags etc in icepack (call *icepack_init_tracer_flags* etc) - - increment number of tracers in use based on namelist input (``ntrcr``) + - increment number of tracers in use based on namelist input (``ntrcr``) - - define tracer dependencies + - define tracer dependencies -#. **CICE\_InitMod.F90**: initialize tracer (includes reading restart file) + 5) **CICE\_InitMod.F90**: initialize tracer (includes reading restart file) -#. **CICE\_RunMod.F90**, **ice\_step\_mod.F90** (and elsewhere as needed): + 6) **CICE\_RunMod.F90**, **ice\_step\_mod.F90** (and elsewhere as needed): - - call routine to write tracer restart data + - call routine to write tracer restart data - - call Icepack or other routines to update tracer value - (often called from **ice\_step\_mod.F90**) + - call Icepack or other routines to update tracer value + (often called from **ice\_step\_mod.F90**) -#. **ice\_restart.F90**: define restart variables (for binary, netCDF and PIO) + 7) **ice\_restart.F90**: define restart variables (for binary, netCDF and PIO) -#. **ice\_restart\_column.F90**: create routines to read, write tracer restart data + 8) **ice\_restart\_column.F90**: create routines to read, write tracer restart data -#. **ice\_fileunits.F90**: add new dump and restart file units + 9) **ice\_fileunits.F90**: add new dump and restart file units -#. **ice\_history\_[tracer].F90**: add history variables - (Section :ref:`addhist`) + 10) **ice\_history\_[tracer].F90**: add history variables + (Section :ref:`addhist`) -#. **ice\_in**: add namelist variables to *tracer\_nml* and - *icefields\_nml*. Best practice is to set the namelist values so that the - new capability is turned off, and create an option file with your preferred - configuration in **configuration/scripts/options**. + 11) **ice\_in**: add namelist variables to *tracer\_nml* and + *icefields\_nml*. Best practice is to set the namelist values so that the + new capability is turned off, and create an option file with your preferred + configuration in **configuration/scripts/options**. -#. If strict conservation is necessary, add diagnostics as noted for - topo ponds in the `Icepack documentation `_. + 12) If strict conservation is necessary, add diagnostics as noted for + topo ponds in the `Icepack documentation `_. -#. Update documentation, including **cice_index.rst** and **ug_case_settings.rst** + 13) Update documentation, including **cice_index.rst** and **ug_case_settings.rst** diff --git a/doc/source/developer_guide/dg_scripts.rst b/doc/source/developer_guide/dg_scripts.rst index 185616eab..da5ef7d24 100755 --- a/doc/source/developer_guide/dg_scripts.rst +++ b/doc/source/developer_guide/dg_scripts.rst @@ -31,6 +31,8 @@ The directory structure under configure/scripts is as follows. | **parse_settings.sh** replaces settings with command-line configuration | **setup_run_dirs.csh** creates the case run directories | **set_version_number.csh** updates the model version number from the **cice.setup** command line +| **timeseries.csh** generates PNG timeseries plots from output files, using GNUPLOT +| **timeseries.py** generates PNG timeseries plots from output files, using Python | **tests/** scripts for configuring and running basic tests .. _dev_strategy: @@ -78,6 +80,25 @@ with appropriate names and syntax. The set_nml file syntax is the same as namel syntax and the set_env files are consistent with csh setenv syntax. See other files for examples of the syntax. +.. _build: + +Build Scripts +-------------- + +CICE uses GNU Make to build the model. There is a common **Makefile** for all machines. +Each machine provides a Macros file to define some Makefile variables +and and an env file to specify the modules/software stack for each compiler. +The machine is built by the cice.build script which invokes Make. +There is a special trap for circular dependencies in the cice.build script to +highlight this error when it occurs. + +The **cice.build** script has some additional features including the ability to +pass a Makefile target. This is documented in :ref:`cicebuild`. In addition, there +is a hidden feature in the **cice.build** script that allows for reuse of +executables. This is used by the test suites to significantly reduce cost of +building the model. It is invoked with the ``--exe`` argument to **cice.build** +and should not be invoked by users interactively. + .. _dev_machines: Machines @@ -122,9 +143,8 @@ setup the various tests, such as smoke and restart tests (**test_smoke.script**, and the files that describe with options files are needed for each test (ie. **test_smoke.files**, **test_restart.files**). A baseline test script (**baseline.script**) is also there to setup the general regression and comparison testing. That directory also contains the preset test suites -(ie. **base_suite.ts**) and a file that supports post-processing on the model -output (**timeseries.csh**). There is also a script **report_results.csh** that pushes results -from test suites back to the CICE-Consortium test results wiki page. +(ie. **base_suite.ts**) and a script (**report_results.csh**) that pushes results from +test suites back to the CICE-Consortium test results wiki page. To add a new test (for example newtest), several files may be needed, diff --git a/doc/source/intro/copyright.rst b/doc/source/intro/copyright.rst index 1fa535a70..57b190f44 100755 --- a/doc/source/intro/copyright.rst +++ b/doc/source/intro/copyright.rst @@ -5,7 +5,7 @@ Copyright ============================= -© Copyright 2018, Triad National Security LLC. All rights reserved. +© Copyright 2019, Triad National Security LLC. All rights reserved. This software was produced under U.S. Government contract 89233218CNA000001 for Los Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC for the U.S. Department diff --git a/doc/source/intro/quickstart.rst b/doc/source/intro/quickstart.rst index deded30a5..547d6ef20 100755 --- a/doc/source/intro/quickstart.rst +++ b/doc/source/intro/quickstart.rst @@ -14,6 +14,8 @@ found in the `CICE Git and Workflow Guide `_ or :ref:`force`. +Software requirements are noted in this :ref:`software` section. + From your main CICE directory, execute:: ./cice.setup -c ~/mycase1 -g gx3 -m testmachine -s diag1,thread -p 8x1 diff --git a/doc/source/master_list.bib b/doc/source/master_list.bib index b80b36d41..11bb3e887 100644 --- a/doc/source/master_list.bib +++ b/doc/source/master_list.bib @@ -694,6 +694,14 @@ @Article{Hunke09 issue = {CB}, url = {http://dx.doi.org/10.1029/2008JC005186} } +@Article{Large09 + author = "W.G. Large and S.G. Yeager", + title = "{The global climatology of an interannually varying air-sea flux data set}", + journal = OM, + year = {2009}, + volume = {33}, + url = {http://dx.doi.org/10.1007/s00382-008-0441-3} +} @Article{Weiss09 author = "J. Weiss and E.M. Schulson", title = "{Coulombic faulting from the grain scale to the geophysical scale: lessons from ice}", @@ -920,6 +928,15 @@ @Article{Roberts18 volume = {376}, url = {http://dx.doi.org/10.1098/rsta.2017.0344} } +@Article{Tsujino18, + author = "H. Tsujino and S. Urakawa and R.J. Small and W.M. Kim and S.G. Yeager and et al.", + title = "{JRA‐55 based surface dataset for driving ocean–sea‐ice models (JRA55‐do)}", + journal = OM, + year = {2018}, + volume = {130}, + pages = {79-139}, + url = {http://dx.doi.org/10.1016/j.ocemod.2018.07.002} +} % ********************************************** % For new entries, see example entry in BIB_TEMPLATE.txt % ********************************************** diff --git a/doc/source/science_guide/sg_dynamics.rst b/doc/source/science_guide/sg_dynamics.rst index f5db205c6..8117fd58d 100644 --- a/doc/source/science_guide/sg_dynamics.rst +++ b/doc/source/science_guide/sg_dynamics.rst @@ -190,7 +190,11 @@ above hydrostatic balance and the value of :math:`k_2`. It is, however, the para The value of :math:`k_1` can be changed at runtime using the namelist variable ``k1``. The grounding scheme can be turned on or off using the namelist logical basalstress. Note that the user must provide a bathymetry field for using this grounding -scheme. Grounding occurs up to water depth of ~25 m. It is suggested to have a bathymetry field with water depths larger than 5 m that represents well shallow water regions such as the Laptev Sea and the East Siberian Sea. +scheme. It is suggested to have a bathymetry field with water depths larger than +5 m that represents well shallow water regions such as the Laptev Sea and the +East Siberian Sea. To prevent unrealistic grounding, :math:`T_b` is set to zero when :math:`h_{wu}` +is larger than 30 m. This maximum value is chosen based on observations of large +keels in the Arctic Ocean :cite:`Amundrud04`. .. _internal-stress: @@ -553,7 +557,7 @@ Introducing another numerical parameter :math:`\alpha=2T \Delta t_e ^{-1}` :cite where as opposed to the classic EVP, the second term in each equation is at iteration :math:`k` :cite:`Bouillon13`. Also, as opposed to the classic EVP, :math:`\Delta t_e` times the number of subcycles (or iterations) does not need to be equal to the advective time step :math:`\Delta t`. -A last difference between the classic EVP and the revised approach is that the latter one initializes the stresses to 0 at the beginning of each time step, -while the classic EVP approach uses the previous time level value. The revised EVP is activated by setting the namelist parameter `revised\_evp` = true. +Finally, as with the classic EVP approach, the stresses are initialized using the previous time level values. +The revised EVP is activated by setting the namelist parameter `revised\_evp` = true. In the code :math:`\alpha = arlx` and :math:`\beta = brlx`. The values of :math:`arlx` and :math:`brlx` can be set in the namelist. It is recommended to use large values of these parameters and to set :math:`arlx=brlx` :cite:`Kimmritz15`. diff --git a/doc/source/user_guide/ug_case_settings.rst b/doc/source/user_guide/ug_case_settings.rst index 8986bc9d8..c0ee9331d 100755 --- a/doc/source/user_guide/ug_case_settings.rst +++ b/doc/source/user_guide/ug_case_settings.rst @@ -337,7 +337,8 @@ Table of namelist options "\*","``trestore``", "integer", "sst restoring time scale (days)", "" "","``restore_ice``", "true/false", "restore ice state along lateral boundaries", "" "\*","``atm_data_type``", "``default``", "constant values defined in the code", "" - "","", "``LYq``", "AOMIP/Large-Yeager forcing data", "" + "","", "``LYq``", "COREII Large-Yeager (AOMIP) forcing data", ":cite:`Large09`" + "","", "``JRA55``", "JRA55 forcing data :cite:`Tsujino18`", "" "","", "``monthly``", "monthly forcing data", "" "","", "``ncar``", "NCAR bulk forcing data", "" "","", "``box2001``", "forcing data for :cite:`Hunke01` box problem", "" diff --git a/doc/source/user_guide/ug_implementation.rst b/doc/source/user_guide/ug_implementation.rst index e2a144ab3..eea7d8310 100644 --- a/doc/source/user_guide/ug_implementation.rst +++ b/doc/source/user_guide/ug_implementation.rst @@ -106,15 +106,15 @@ Big Endian files. In CESM, the sea ice model may exchange coupling fluxes using a different grid than the computational grid. This functionality is -activated using the namelist variable ``gridcpl\_file``. +activated using the namelist variable ``gridcpl_file``. *********************** Grid domains and blocks *********************** In general, the global gridded domain is -``nx\_global`` :math:`\times`\ ``ny\_global``, while the subdomains used in the -block distribution are ``nx\_block`` :math:`\times`\ ``ny\_block``. The +``nx_global`` :math:`\times`\ ``ny_global``, while the subdomains used in the +block distribution are ``nx_block`` :math:`\times`\ ``ny_block``. The physical portion of a subdomain is indexed as [``ilo:ihi``, ``jlo:jhi``], with nghost “ghost” or “halo" cells outside the domain used for boundary conditions. These parameters are illustrated in :ref:`fig-grid` in one @@ -141,14 +141,14 @@ Figure :ref:`fig-grid` shows the grid parameters for a sample one-dimensional, 2 global domain decomposed into four local subdomains. Each local domain has one ghost (halo) cell on each side, and the physical portion of the local domains are labeled ``ilo:ihi``. The parameter -``nx\_block`` is the total number of cells in the local domain, including +``nx_block`` is the total number of cells in the local domain, including ghost cells, and the same numbering system is applied to each of the four subdomains. The user sets the ``NTASKS`` and ``NTHRDS`` settings in **cice.settings** -and chooses a block size ``block\_size\_x`` :math:`\times`\ ``block\_size\_y``, -``max\_blocks``, and decomposition information ``distribution\_type``, ``processor\_shape``, -and ``distribution\_type`` in **ice\_in**. That information is used to +and chooses a block size ``block_size_x`` :math:`\times`\ ``block_size_y``, +``max_blocks``, and decomposition information ``distribution_type``, ``processor_shape``, +and ``distribution_type`` in **ice\_in**. That information is used to determine how the blocks are distributed across the processors, and how the processors are distributed across the grid domain. Recommended combinations of these @@ -159,8 +159,8 @@ but the user can overwrite the defaults by manually changing the values in information to the log file, and if the block size or max blocks is inconsistent with the task and thread size, the model will abort. The code will also print a warning if the maximum number of blocks is too large. -Although this is not fatal, it does use extra memory. If ``max\_blocks`` is -set to -1, the code will compute a ``max\_blocks`` on the fly. +Although this is not fatal, it does use extra memory. If ``max_blocks`` is +set to -1, the code will compute a ``max_blocks`` on the fly. A loop at the end of routine *create\_blocks* in module **ice\_blocks.F90** will print the locations for all of the blocks on @@ -173,10 +173,10 @@ manually set in the code in each case (independently of the dbug flag in **ice\_in**), as there may be hundreds or thousands of blocks to print and this information should be needed only rarely. This information is much easier to look at using a debugger such as Totalview. There is also -an output field that can be activated in `icefields\_nml`, ``f\_blkmask``, +an output field that can be activated in `icefields\_nml`, ``f_blkmask``, that prints out the variable ``blkmask`` to the history file and which labels the blocks in the grid decomposition according to ``blkmask = -my\_task + iblk/100``. +my_task + iblk/100``. ************* Tripole grids @@ -198,11 +198,11 @@ poles and the cells between them can be grid T cells, making a “T-fold.” Both of these options are also supported by the OPA/NEMO ocean model, which calls the U-fold an “f-fold” (because it uses the Arakawa C-grid in which U cells are on T-rows). The choice of tripole grid is given by -the namelist variable ``ns\_boundary\_type``, ‘tripole’ for the U-fold and +the namelist variable ``ns_boundary_type``, ‘tripole’ for the U-fold and ‘tripoleT’ for the T-fold grid. In the U-fold tripole grid, the poles have U-index -:math:`{\tt nx\_global}/2` and ``nx\_global`` on the top U-row of the +:math:`{\tt nx\_global}/2` and ``nx_global`` on the top U-row of the physical grid, and points with U-index i and :math:`{\tt nx\_global-i}` are coincident. Let the fold have U-row index :math:`n` on the global grid; this will also be the T-row index of the T-row to the south of the @@ -267,7 +267,7 @@ masked by land, periodic conditions wrap the domain around the globe. CICE can be run on regional grids with open boundary conditions; except for variables describing grid lengths, non-land halo cells along the grid edge must be filled by restoring them to specified values. The -namelist variable ``restore\_ice`` turns this functionality on and off; the +namelist variable ``restore_ice`` turns this functionality on and off; the restoring timescale ``trestore`` may be used (it is also used for restoring ocean sea surface temperature in stand-alone ice runs). This implementation is only intended to provide the “hooks" for a more @@ -279,7 +279,7 @@ allow Neumann boundary conditions, which must be set explicitly. This has been done in an unreleased branch of the code; contact Elizabeth for more information. -For exact restarts using restoring, set ``restart\_ext`` = true in namelist +For exact restarts using restoring, set ``restart_ext`` = true in namelist to use the extended-grid subroutines. On tripole grids, the order of operations used for calculating elements @@ -308,27 +308,27 @@ The logical masks ``tmask`` and ``umask`` (which correspond to the real masks In addition to the land masks, two other masks are implemented in *dyn\_prep* in order to reduce the dynamics component’s work on a global -grid. At each time step the logical masks ``ice\_tmask`` and ``ice\_umask`` are +grid. At each time step the logical masks ``ice_tmask`` and ``ice_umask`` are determined from the current ice extent, such that they have the value “true” wherever ice exists. They also include a border of cells around the ice pack for numerical purposes. These masks are used in the dynamics component to prevent unnecessary calculations on grid points where there is no ice. They are not used in the thermodynamics component, so that ice may form in previously ice-free cells. Like the -land masks ``hm`` and ``uvm``, the ice extent masks ``ice\_tmask`` and ``ice\_umask`` +land masks ``hm`` and ``uvm``, the ice extent masks ``ice_tmask`` and ``ice_umask`` are for T cells and U cells, respectively. Improved parallel performance may result from utilizing halo masks for boundary updates of the full ice state, incremental remapping transport, or for EVP or EAP dynamics. These options are accessed through the -logical namelist flags ``maskhalo\_bound``, ``maskhalo\_remap``, and -``maskhalo\_dyn``, respectively. Only the halo cells containing needed +logical namelist flags ``maskhalo_bound``, ``maskhalo_remap``, and +``maskhalo_dyn``, respectively. Only the halo cells containing needed information are communicated. -Two additional masks are created for the user’s convenience: ``lmask\_n`` -and ``lmask\_s`` can be used to compute or write data only for the northern +Two additional masks are created for the user’s convenience: ``lmask_n`` +and ``lmask_s`` can be used to compute or write data only for the northern or southern hemispheres, respectively. Special constants (``spval`` and -``spval\_dbl``, each equal to :math:`10^{30}`) are used to indicate land +``spval_dbl``, each equal to :math:`10^{30}`) are used to indicate land points in the history files and diagnostics. @@ -338,13 +338,13 @@ points in the history files and diagnostics. Performance *************** -Namelist options (*domain\_nml*) provide considerable flexibility for +Namelist options (*domain_nml*) provide considerable flexibility for finding efficient processor and block configuration. Some of these choices are illustrated in :ref:`fig-distrb`. Users have control -of many aspects of the decomposition such as the block size (``block\_size\_x``, -``block\_size\_y``), the ``distribution\_type``, the ``distribution\_wght``, -the ``distribution\_wght\_file`` (when ``distribution\_type`` = ``wghtfile``), -and the ``processor\_shape`` (when ``distribution\_type`` = ``cartesian``). +of many aspects of the decomposition such as the block size (``block_size_x``, +``block_size_y``), the ``distribution_type``, the ``distribution_wght``, +the ``distribution_wght_file`` (when ``distribution_type`` = ``wghtfile``), +and the ``processor_shape`` (when ``distribution_type`` = ``cartesian``). The user specifies the total number of tasks and threads in **cice.settings** and the block size and decompostion in the namelist file. The main trades @@ -361,7 +361,7 @@ volume-to-surface ratio important for communication cost. Often 3 to 8 blocks per processor provide the decompositions flexiblity to create reasonable load balance configurations. -The ``distribution\_type`` options allow standard cartesian distributions +The ``distribution_type`` options allow standard cartesian distributions of blocks, redistribution via a ‘rake’ algorithm for improved load balancing across processors, and redistribution based on space-filling curves. There are also additional distribution types @@ -395,7 +395,7 @@ Figure :ref:`fig-distrbB` shows sample decompositions for (a) spiral center and (b) wghtfile for an Arctic polar grid. (c) is the weight field in the input file use to drive the decompostion in (b). -``processor\_shape`` is used with the ``distribution\_type`` cartesian option, +``processor_shape`` is used with the ``distribution_type`` cartesian option, and it allocates blocks to processors in various groupings such as tall, thin processor domains (``slenderX1`` or ``slenderX2``, often better for sea ice simulations on global grids where nearly all of @@ -405,14 +405,14 @@ which maximize the volume to surface ratio (and therefore on-processor computations to message passing, if there were ice in every grid cell). In cases where the number of processors is not a perfect square (4, 9, 16...), the -``processor\_shape`` namelist variable allows the user to choose how the +``processor_shape`` namelist variable allows the user to choose how the processors are arranged. Here again, it is better in the sea ice model to have more processors in x than in y, for example, 8 processors arranged 4x2 (``square-ice``) rather than 2x4 (``square-pop``). The latter option is offered for direct-communication compatibility with POP, in which this is the default. -``distribution\_wght`` chooses how the work-per-block estimates are +``distribution_wght`` chooses how the work-per-block estimates are weighted. The ‘block’ option is the default in POP and it weights each block equally. This is useful in POP which always has work in each block and is written with a lot of @@ -422,7 +422,7 @@ direct-communication compatibility with POP. The ‘latitude’ option weights the blocks based on latitude and the number of ocean grid cells they contain. Many of the non-cartesian decompositions support automatic land block elimination and provide alternative ways to -decompose blocks without needing the ``distribution\_wght``. +decompose blocks without needing the ``distribution_wght``. The rake distribution type is initialized as a standard, Cartesian distribution. Using the work-per-block estimates, blocks are “raked" @@ -549,7 +549,7 @@ layers and the ice thickness distribution defined by ``kcatbound`` = 0. Restart information for some tracers is also included in the netCDF restart files. -Three namelist variables control model initialization, ``ice\_ic``, ``runtype``, +Three namelist variables control model initialization, ``ice_ic``, ``runtype``, and ``restart``, as described in :ref:`tab-ic`. It is possible to do an initial run from a file **filename** in two ways: (1) set runtype = ‘initial’, restart = true and ice\_ic = **filename**, or (2) runtype = @@ -562,7 +562,7 @@ true or false, depending on whether the tracer restart data exist. With the second option, tracer restart flags are set to ‘continue’ for all active tracers. -An additional namelist option, ``restart\_ext`` specifies whether halo cells +An additional namelist option, ``restart_ext`` specifies whether halo cells are included in the restart files. This option is useful for tripole and regional grids, but can not be used with PIO. @@ -577,8 +577,8 @@ her own routines. Whether the code is to be run in stand-alone or coupled mode is determined at compile time, as described below. Table :ref:`tab-ic` shows ice initial state resulting from combinations of -``ice\_ic``, ``runtype`` and ``restart``. :math:`^a`\ If false, restart is reset to -true. :math:`^b`\ restart is reset to false. :math:`^c`\ ice\_ic is +``ice_ic``, ``runtype`` and ``restart``. :math:`^a`\ If false, restart is reset to +true. :math:`^b`\ restart is reset to false. :math:`^c`\ ice_ic is reset to ‘none.’ .. _tab-ic: @@ -607,9 +607,9 @@ The time step is chosen based on stability of the transport component (both horizontal and in thickness space) and on resolution of the physical forcing. CICE allows the dynamics, advection and ridging portion of the code to be run with a shorter timestep, -:math:`\Delta t_{dyn}` (``dt\_dyn``), than the thermodynamics timestep +:math:`\Delta t_{dyn}` (``dt_dyn``), than the thermodynamics timestep :math:`\Delta t` (``dt``). In this case, ``dt`` and the integer ndtd are -specified, and ``dt\_dyn`` = ``dt/ndtd``. +specified, and ``dt_dyn`` = ``dt/ndtd``. A conservative estimate of the horizontal transport time step bound, or CFL condition, under remapping yields @@ -632,8 +632,8 @@ As discussed in :cite:`Lipscomb07`, the maximum time step in practice is usually determined by the time scale for large changes in the ice strength (which depends in part on wind strength). Using the strength parameterization of :cite:`Rothrock75`, limits the time step to :math:`\sim`\ 30 -minutes for the old ridging scheme (``krdg\_partic`` = 0), and to -:math:`\sim`\ 2 hours for the new scheme (``krdg\_partic`` = 1), assuming +minutes for the old ridging scheme (``krdg_partic`` = 0), and to +:math:`\sim`\ 2 hours for the new scheme (``krdg_partic`` = 1), assuming :math:`\Delta x` = 10 km. Practical limits may be somewhat less, depending on the strength of the atmospheric winds. @@ -646,7 +646,7 @@ growth rate. For the 5-category ice thickness distribution used as the default in this distribution, this is not a stringent limitation: :math:`\Delta t < 19.4` hr, assuming :math:`\max(f) = 40` cm/day. -In the classic EVP or EAP approach (``kdyn`` = 1 or 2, ``revised\_evp`` = false), +In the classic EVP or EAP approach (``kdyn`` = 1 or 2, ``revised_evp`` = false), the dynamics component is subcycled ndte (:math:`N`) times per dynamics time step so that the elastic waves essentially disappear before the next time step. The subcycling time step (:math:`\Delta @@ -657,7 +657,7 @@ t_e`) is thus A second parameter, :math:`E_\circ` (``eyc``), defines the elastic wave damping timescale :math:`T`, described in Section :ref:`dynam`, as -``eyc\ * dt\_dyn``. The forcing terms are not updated during the subcycling. +``eyc * dt_dyn``. The forcing terms are not updated during the subcycling. Given the small step (``dte``) at which the EVP dynamics model is subcycled, the elastic parameter :math:`E` is also limited by stability constraints, as discussed in :cite:`Hunke97`. Linear stability @@ -683,7 +683,7 @@ temperature :math:`T_{sfc}` is computed internally. The numerical constraint on the thermodynamics time step is associated with the transport scheme rather than the thermodynamic solver. -For the revised EVP approach (``kdyn`` = 1, ``revised\_evp`` = true), the +For the revised EVP approach (``kdyn`` = 1, ``revised_evp`` = true), the relaxation parameter ``arlx1i`` effectively sets the damping timescale in the problem, and ``brlx`` represents the effective subcycling :cite:`Bouillon13` (see Section :ref:`revp`). @@ -699,16 +699,16 @@ History files ************* Model output data is averaged over the period(s) given by ``histfreq`` and -``histfreq\_n``, and written to binary or netCDF files prepended by ``history\_file`` -in **ice\_in**. These settings for history files are set in the -**setup\_nml** section of **ice\_in** (see :ref:`tabnamelist`). -If ``history\_file`` = ‘iceh’ then the +``histfreq_n``, and written to binary or netCDF files prepended by ``history_file`` +in **ice_in**. These settings for history files are set in the +**setup_nml** section of **ice_in** (see :ref:`tabnamelist`). +If ``history_file`` = ‘iceh’ then the filenames will have the form **iceh.[timeID].nc** or **iceh.[timeID].da**, depending on the output file format chosen in **cice.settings** (set -``ICE\_IOTYPE``). The netCDF history files are CF-compliant; header information for +``ICE_IOTYPE``). The netCDF history files are CF-compliant; header information for data contained in the netCDF files is displayed with the command ``ncdump -h filename.nc``. Parallel netCDF output is available using the PIO library; the -attribute ``io\_flavor`` distinguishes output files written with PIO from +attribute ``io_flavor`` distinguishes output files written with PIO from those written with standard netCDF. With binary files, a separate header file is written with equivalent information. Standard fields are output according to settings in the **icefields\_nml** section of **ice\_in** @@ -735,7 +735,7 @@ monthly) via its namelist flag, `f\_` :math:`\left<{var}\right>`, which is now a character string corresponding to ``histfreq`` or ‘x’ for none. (Grid variable flags are still logicals, since they are written to all files, no matter what the frequency is.) If there are no namelist flags -with a given ``histfreq`` value, or if an element of ``histfreq\_n`` is 0, then +with a given ``histfreq`` value, or if an element of ``histfreq_n`` is 0, then no file will be written at that frequency. The output period can be discerned from the filenames. @@ -766,7 +766,7 @@ The history variable names must be unique for netCDF, so in cases where a variable is written at more than one frequency, the variable name is appended with the frequency in files after the first one. In the example above, ``meltb`` is called ``meltb`` in the monthly file (for backward -compatibility with the default configuration) and ``meltb\_h`` in the +compatibility with the default configuration) and ``meltb_h`` in the 6-hourly file. Using the same frequency twice in ``histfreq`` will have unexpected @@ -774,7 +774,7 @@ consequences and currently will cause the code to abort. It is not possible at the moment to output averages once a month and also once every 3 months, for example. -If ``write\_ic`` is set to true in **ice\_in**, a snapshot of the same set +If ``write_ic`` is set to true in **ice\_in**, a snapshot of the same set of history fields at the start of the run will be written to the history directory in **iceh\_ic.[timeID].nc(da)**. Several history variables are hard-coded for instantaneous output regardless of the averaging flag, at @@ -811,16 +811,16 @@ Diagnostic files Like ``histfreq``, the parameter ``diagfreq`` can be used to regulate how often output is written to a log file. The log file unit to which diagnostic -output is written is set in **ice\_fileunits.F90**. If ``diag\_type`` = +output is written is set in **ice\_fileunits.F90**. If ``diag_type`` = ‘stdout’, then it is written to standard out (or to **ice.log.[ID]** if you redirect standard out as in **cice.run**); otherwise it is written -to the file given by ``diag\_file``. In addition to the standard diagnostic +to the file given by ``diag_file``. In addition to the standard diagnostic output (maximum area-averaged thickness, velocity, average albedo, total ice area, and total ice and snow volumes), the namelist options -``print\_points`` and ``print\_global`` cause additional diagnostic information -to be computed and written. ``print\_global`` outputs global sums that are +``print_points`` and ``print_global`` cause additional diagnostic information +to be computed and written. ``print_global`` outputs global sums that are useful for checking global conservation of mass and energy. -``print\_points`` writes data for two specific grid points. Currently, one +``print_points`` writes data for two specific grid points. Currently, one point is near the North Pole and the other is in the Weddell Sea; these may be changed in **ice\_in**. @@ -888,27 +888,27 @@ Restart files ************* CICE provides restart data in binary unformatted or netCDF formats, via -the ``ICE\_IOTYPE`` flag in **cice.settings** and namelist variable -``restart\_format``. Restart and history files must use the same format. As +the ``ICE_IOTYPE`` flag in **cice.settings** and namelist variable +``restart_format``. Restart and history files must use the same format. As with the history output, there is also an option for writing parallel netCDF restart files using PIO. The restart files created by CICE contain all of the variables needed for a full, exact restart. The filename begins with the character string ‘iced.’, and the restart dump frequency is given by the namelist -variables ``dumpfreq`` and ``dumpfreq\_n``. The pointer to the filename from +variables ``dumpfreq`` and ``dumpfreq_n``. The pointer to the filename from which the restart data is to be read for a continuation run is set in -``pointer\_file``. The code assumes that auxiliary binary tracer restart +``pointer_file``. The code assumes that auxiliary binary tracer restart files will be identified using the same pointer and file name prefix, but with an additional character string in the file name that is associated with each tracer set. All variables are included in netCDF restart files. Additional namelist flags provide further control of restart behavior. -``dump\_last`` = true causes a set of restart files to be written at the end +``dump_last`` = true causes a set of restart files to be written at the end of a run when it is otherwise not scheduled to occur. The flag -``use\_restart\_time`` enables the user to choose to use the model date -provided in the restart files. If ``use\_restart\_time`` = false then the +``use_restart_time`` enables the user to choose to use the model date +provided in the restart files. If ``use_restart_time`` = false then the initial model date stamp is determined from the namelist parameters. lcdf64 = true sets 64-bit netCDF output, allowing larger file sizes. @@ -917,7 +917,7 @@ of the “extended" global grid, including the physical domain and ghost (halo) cells around the outer edges, allow exact restarts on regional grids with open boundary conditions, and they will also simplify restarts on the various tripole grids. They are accessed by setting -``restart\_ext`` = true in namelist. Extended grid restarts are not +``restart_ext`` = true in namelist. Extended grid restarts are not available when using PIO; in this case extra halo update calls fill ghost cells for tripole grids (do not use PIO for regional grids). @@ -929,5 +929,5 @@ initialized with no ice. The gx3 case was run for 1 year using the 1997 forcing data provided with the code. The gx1 case was run for 20 years, so that the date of restart in the file is 1978-01-01. Note that the restart dates provided in the restart files can be overridden using the -namelist variables ``use\_restart\_time``, ``year\_init`` and ``istep0``. The -forcing time can also be overridden using ``fyear\_init``. +namelist variables ``use_restart_time``, ``year_init`` and ``istep0``. The +forcing time can also be overridden using ``fyear_init``. diff --git a/doc/source/user_guide/ug_running.rst b/doc/source/user_guide/ug_running.rst index b22e79628..4abd52d0a 100644 --- a/doc/source/user_guide/ug_running.rst +++ b/doc/source/user_guide/ug_running.rst @@ -7,6 +7,71 @@ Running CICE Quick-start instructions are provided in the :ref:`quickstart` section. +.. _software: + +Software Requirements +---------------------- + +To run stand-alone, CICE requires + +- gmake (GNU Make) +- Fortran and C compilers (Intel, PGI, GNU, Cray, and NAG have been tested) +- NetCDF +- MPI (this is actually optional but without it you can only run on 1 processor) + +Below are lists of software versions that the Consortium has tested at some point. There is no +guarantee that all compiler versions work with all CICE model versions. At any given +point, the Consortium is regularly testing on several different compilers, but not +necessarily on all possible versions or combinations. A CICE goal is to be relatively portable +across different hardware, compilers, and other software. As a result, the coding +implementation tends to be on the conservative side at times. If there are problems +porting to a particular system, please let the Consortium know. + +The Consortium has tested the following compilers at some point, + +- Intel 15.0.3.187 +- Intel 16.0.1.150 +- Intel 17.0.1.132 +- Intel 17.0.2.174 +- Intel 17.0.5.239 +- Intel 18.0.1.163 +- Intel 19.0.2 +- Intel 19.0.3.199 +- PGI 16.10.0 +- GNU 6.3.0 +- GNU 7.2.0 +- GNU 7.3.0 +- Cray 8.5.8 +- Cray 8.6.4 +- NAG 6.2 + +The Consortium has tested the following mpi versions, + +- MPICH 7.3.2 +- MPICH 7.5.3 +- MPICH 7.6.2 +- MPICH 7.6.3 +- MPICH 7.7.6 +- Intel MPI 18.0.1 +- MPT 2.14 +- MPT 2.17 +- MPT 2.18 +- MPT 2.19 +- OpenMPI 1.6.5 + +The NetCDF implementation is relatively general and should work with any version of NetCDF 3 or 4. The Consortium has tested + +- NetCDF 4.3.0 +- NetCDF 4.3.2 +- NetCDF 4.4.0 +- NetCDF 4.4.1.1.32 +- NetCDF 4.4.1.1 +- NetCDF 4.4.2 +- NetCDF 4.5.0 +- NetCDF 4.6.1.3 + +Please email the Consortium if this list can be extended. + .. _scripts: Scripts @@ -56,7 +121,7 @@ Once a case/test is created, several files are placed in the case directory - **makdep.c** is a tool that will automatically generate the make dependencies - **Macros.[machine]** defines the Makefile macros - **Makefile** is the makefile used to build the model -- **cice.build** is a script that builds and compiles the model +- **cice.build** is a script that calls the Makefile and compiles the model - **ice\_in** is the namelist input file - **setup\_run\_dirs.csh** is a script that will create the run directories. This will be called automatically from the **cice.run** script if the user does not invoke it. - **cice.run** is a batch run script @@ -82,7 +147,7 @@ Some hints: - To change batch settings, manually edit the top of the **cice.run** or **cice.test** (if running a test) file - To turn on the debug compiler flags, set ``ICE_BLDDEBUG`` in **cice.setttings** to true. It is also possible to use the ``debug`` option (``-s debug``) when creating the case with **cice.setup** to set this option automatically. - To change compiler options, manually edit the Macros file -- To clean the build before each compile, set ``ICE_CLEANBUILD`` in **cice.settings** to true (this is the default value), or use the ``buildclean`` option (``-s buildclean``) when creating the case with **cice.setup**. To not clean before the build, set ``ICE_CLEANBUILD`` in **cice.settings** to false, or use the ``buildincremental`` option (``-s buildincremental``) when creating the case with **cice.setup**. +- To clean the build before each compile, set ``ICE_CLEANBUILD`` in **cice.settings** to true (this is the default value), or use the ``buildclean`` option (``-s buildclean``) when creating the case with **cice.setup**. To not clean before the build, set ``ICE_CLEANBUILD`` in **cice.settings** to false, or use the ``buildincremental`` option (``-s buildincremental``) when creating the case with **cice.setup**. It is recommended that the ``ICE_CLEANBUILD`` be set to true if there are any questions about whether the build is proceeding properly. To build and run:: @@ -96,8 +161,8 @@ case directory, NOT the run directory. .. _case_options: -Command Line Options -~~~~~~~~~~~~~~~~~~~~ +**cice.setup** Command Line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``cice.setup -h`` provides a summary of the command line options. There are three different modes, ``--case``, ``--test``, and ``--suite``. This section provides details about the relevant options for setting up cases with examples. Testing will be described in greater detail in the :ref:`testing` section. @@ -153,8 +218,8 @@ by doing ``cice.setup --help``. The default CICE namelist and CICE settings are specified in the files **configuration/scripts/ice_in** and -**configuration/scripts/cice.settings** respectively. When picking a -preset setting (option), the set_env.setting and set_nml.setting will be used to +**configuration/scripts/cice.settings** respectively. When picking +settings (options), the set_env.setting and set_nml.setting will be used to change the defaults. This is done as part of the ``cice.setup`` and the modifications are resolved in the **cice.settings** and **ice_in** file placed in the case directory. If multiple options are chosen and then conflict, then the last @@ -202,6 +267,84 @@ To add some optional settings, one might do:: Once the cases are created, users are free to modify the cice.settings and ice_in namelist to further modify their setup. +.. _cicebuild: + +More about **cice.build** +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**cice.build** is copied into the case directory and should be run interactively from the +case directory to build the model. CICE is built with make and there is a generic +Makefile and a machine specific Macros file in the case directory. **cice.build** +is a wrapper for a call to make that includes several other features. + +CICE is built as follows. First, the makdep binary is created by compiling a small +C program. The makdep binary is then run and dependency files are created. The dependency +files are included into the Makefile automatically. As a result, make dependencies do not +need to be explicitly defined by the user. In the next step, make compiles the CICE +code and generates the cice binary. + +The standard and recommended way to run is with +no arguments +:: + + cice.build + +However, **cice.build** does support a couple other use modes. +:: + + cice.build [-h|--help] + +provides a summary of the usage. +:: + + cice.build [make arguments] [target] + +turns off most of the features of the cice.build script and turns it into a wrapper +for the make call. The arguments and/or target are passed to make and invoked more +or less like make [make arguments] [target]. This will be the case if either or +both the arguments or target are passed to cice.build. Some examples of that are +:: + + cice.build --version + +which will pass --version to make. +:: + + cice.build targets + +is a valid target of the CICE Makefile and simply echos all the valid +targets of the Makefile. +:: + + cice.build cice + +or :: + + cice.build all + +are largely equivalent to running **cice.build** without an argument, +although as noted earlier, many of the extra features of the cice.build script +are turned off when calling cice.build with a target or an argument. Any of the +full builds will compile makdep, generate the source code dependencies, and +compile the source code. +:: + + cice.build [clean|realclean] + cice.build [db_files|db_flags] + cice.build [makdep|depends] + +are other valid options for cleaning the build, writing out information about +the Makefile setup, and building just the makdep tool or the dependency file. +It is also possible to target a particular CICE object file. + +Finally, there is one important parameter in **cice.settings**. The ``ICE_CLEANBUILD`` +variable defines whether the model is cleaned before a build is carried out. By +default, this variable is true which means each invokation of **cice.build** will +automatically clean the prior build. If incremental builds are desired to save +time during development, the ``ICE_CLEANBUILD`` setting in **cice.settings** should +be modified. + + .. _porting: Porting @@ -235,6 +378,36 @@ directory back to **configuration/scripts/machines/** and update the **configuration/scripts/cice.batch.csh** file, retest, and then add and commit the updated machine files to the repository. +.. _machvars: + +Machine variables +~~~~~~~~~~~~~~~~~~~~~~ + +There are several machine specific variables defined in the **env.$[machine]**. These +variables are used to generate working cases for a given machine, compiler, and batch +system. Some variables are optional. + +.. csv-table:: *Machine Settings* + :header: "variable", "format", "description" + :widths: 15, 15, 25 + + "ICE_MACHINE_ENVNAME", "string", "machine name" + "ICE_MACHINE_COMPILER", "string", "compiler" + "ICE_MACHINE_MAKE", "string", "make command" + "ICE_MACHINE_WKDIR", "string", "root work directory" + "ICE_MACHINE_INPUTDATA", "string", "root input data directory" + "ICE_MACHINE_BASELINE", "string", "root regression baseline directory" + "ICE_MACHINE_SUBMIT", "string", "batch job submission command" + "ICE_MACHINE_TPNODE", "integer", "machine maximum MPI tasks per node" + "ICE_MACHINE_MAXPES", "integer", "machine maximum total processors per job (optional)" + "ICE_MACHINE_MAXTHREADS", "integer", "machine maximum threads per mpi task (optional)" + "ICE_MACHINE_MAXRUNLENGTH", "integer", "batch wall time limit in hours (optional)" + "ICE_MACHINE_ACCT", "string", "batch default account" + "ICE_MACHINE_QUEUE", "string", "batch default queue" + "ICE_MACHINE_BLDTHRDS", "integer", "number of threads used during build" + "ICE_MACHINE_QSTAT", "string", "batch job status command (optional)" + "ICE_MACHINE_QUIETMODE", "true/false", "flag to reduce build output (optional)" + .. _cross_compiling: Cross-compiling @@ -305,3 +478,94 @@ should be rebuilt before being resubmitted. It is always recommended that users modify the scripts and input settings in the case directory, NOT the run directory. In general, files in the run directory are overwritten by versions in the case directory when the model is built, submitted, and run. + +.. _timeseries: + +Timeseries Plotting +------------------- + +The CICE scripts include two scripts that will generate timeseries figures from a +diagnostic output file, a Python version (``timeseries.py``) and a csh version +(``timeseries.csh``). Both scripts create the same set of plots, but the Python +script has more capabilities, and it's likely that the csh +script will be removed in the future. + +To use the ``timeseries.py`` script, the following requirements must be met: + +* Python v2.7 or later +* numpy Python package +* matplotlib Python package +* datetime Python package + +See :ref:`CodeCompliance` for additional information about how to setup the Python +environment, but we recommend using ``pip`` as follows: :: + + pip install --user numpy + pip install --user matplotlib + pip install --user datetime + +When creating a case or test via ``cice.setup``, the ``timeseries.csh`` and +``timeseries.py`` scripts are automatically copied to the case directory. +Alternatively, the plotting scripts can be found in ``./configuration/scripts``, and can be +run from any directory. + +The Python script can be passed a directory, a specific log file, or no directory at all: + + - If a directory is passed, the script will look either in that directory or in + directory/logs for a filename like cice.run*. As such, users can point the script + to either a case directory or the ``logs`` directory directly. The script will use + the file with the most recent creation time. + - If a specific file is passed the script parses that file, assuming that the file + matches the same form of cice.run* files. + - If nothing is passed, the script will look for log files or a ``logs`` directory in the + directory from where the script was run. + +For example: + +Run the timeseries script on the desired case. :: + +$ python timeseries.py /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/ + +or :: + +$ python timeseries.py /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/logs + +The output figures are placed in the directory where the ``timeseries.py`` script is run. + +The plotting script will plot the following variables by default, but you can also select +specific plots to create via the optional command line arguments. + + - total ice area (:math:`km^2`) + - total ice extent (:math:`km^2`) + - total ice volume (:math:`m^3`) + - total snow volume (:math:`m^3`) + - RMS ice speed (:math:`m/s`) + +For example, to plot only total ice volume and total snow volume :: + +$ python timeseries.py /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/ --volume --snw_vol + +To generate plots for all of the cases within a suite with a testid, create and run a script such as :: + + #!/bin/csh + foreach dir (`ls -1 | grep testid`) + echo $dir + python timeseries.py $dir + end + +Plots are only made for a single output file at a time. The ability to plot output from +a series of cice.run* files is not currently possible, but may be added in the future. +However, using the ``--bdir`` option will plot two datasets (from log files) on the +same figure. + +For the latest help information for the script, run :: + +$ python timeseries.py -h + +The ``timeseries.csh`` script works basically the same way as the Python version, however it +does not include all of the capabilities present in the Python version. + +To use the C-Shell version of the script, :: + +$ ./timeseries.csh /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/ + diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 27393e3de..7a17e67ea 100644 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -819,6 +819,7 @@ hemispheres, and must exceed a critical value nominally set to test and the Two-Stage test described in the previous section are provided in :cite:`Hunke18`. +.. _CodeCompliance: Code Compliance Testing Procedure ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -846,7 +847,7 @@ To install the necessary Python packages, the ``pip`` Python utility can be used pip install --user matplotlib To run the compliance test, setup a baseline run with the original baseline model and then -a perturbation run based on recent model changes. Use ``--sets qc`` in both runs in addition +a perturbation run based on recent model changes. Use ``--set qc`` in both runs in addition to other settings needed. Then use the QC script to compare history output, .. code-block:: bash @@ -866,15 +867,17 @@ The script will produce output similar to: Additionally, the exit code from the test (``echo $?``) will be 0 if the test passed, and 1 if the test failed. -Implementation notes: 1) Provide a pass/fail on each of the confidence -intervals, 2) Facilitate output of a bitmap for each test so that -locations of failures can be identified. - -The cice.t-test.py requires memory to store multiple two-dimensional fields spanning +The ``cice.t-test.py`` requires memory to store multiple two-dimensional fields spanning 1825 unique timesteps, a total of several GB. An appropriate resource is needed to run the script. If the script runs out of memory on an interactive resource, try logging into a batch resource or finding a large memory node. +The ``cice.t-test.py`` script will also attempt to generate plots of the mean ice thickness +for both the baseline and test cases. Additionally, if the 2-stage test fails then the +script will attempt to plot a map showing the grid cells that failed the test. For a +full list of options, run ``python cice.t-test.py -h``. + + End-To-End Testing Procedure ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -888,7 +891,7 @@ Below is an example of a step-by-step procedure for testing a code change that m # Create a baseline dataset (only necessary if no baseline exists on the system) # git clone the baseline code - ./cice.setup -m onyx -e intel --suite base_suite --testid base0 -bgen cice.my.baseline + ./cice.setup -m onyx -e intel --suite base_suite --testid base0 --bgen cice.my.baseline # Run the test suite with the new code # git clone the new code @@ -900,6 +903,8 @@ Below is an example of a step-by-step procedure for testing a code change that m cd testsuite.test0 ./results.csh + # Note which tests failed and determine which namelist options are responsible for the failures + .. If the regression comparisons fail, then you may want to run the QC test, @@ -913,6 +918,7 @@ If the regression comparisons fail, then you may want to run the QC test, ./cice.setup -m onyx -e intel --test smoke -g gx1 -p 44x1 --testid qc_base -s qc,medium cd onyx_intel_smoke_gx1_44x1_medium_qc.qc_base + # modify ice_in to activate the namelist options that were determined above ./cice.build ./cice.submit @@ -921,6 +927,7 @@ If the regression comparisons fail, then you may want to run the QC test, ./cice.setup -m onyx -e intel --test smoke -g gx1 -p 44x1 -testid qc_test -s qc,medium cd onyx_intel_smoke_gx1_44x1_medium_qc.qc_test + # modify ice_in to activate the namelist options that were determined above ./cice.build ./cice.submit @@ -940,51 +947,3 @@ If the regression comparisons fail, then you may want to run the QC test, INFO:__main__:Quality Control Test PASSED -.. _testplotting: - -Test Plotting ----------------- - -The CICE scripts include a script (``timeseries.csh``) that will generate timeseries -figures from a diagnostic output file. -When running a test suite, the ``timeseries.csh`` script is automatically copied to the suite directory. -If the ``timeseries.csh`` script is to be used on a test or case that is not a part of a test suite, -users will need to run the ``timeseries.csh`` script from the tests directory -(``./configuration/scripts/tests/timeseries.csh ./path/``), or copy it to a local directory. -When used with the test suites or given a path, it needs to be run in the directory -above the particular case being plotted, but it can also be run on isolated log files in the same directory, -without a path. - -For example: - -Run the test suite. :: - -$ ./cice.setup -m conrad -e intel --suite base_suite --testid t00 - -Wait for suite to finish then go to the directory. :: - -$ cd testsuite.t00 - -Run the timeseries script on the desired case. :: - -$ ./timeseries.csh /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/ - -The output figures are placed in the directory where the ``timeseries.csh`` script is run. - -To generate plots for all of the cases within a suite with a testid, create and run a script such as :: - - #!/bin/csh - foreach dir (`ls -1 | grep testid`) - echo $dir - timeseries.csh $dir - end - - -This plotting script can be used to plot the following variables: - - - total ice area (:math:`km^2`) - - total ice extent (:math:`km^2`) - - total ice volume (:math:`m^3`) - - total snow volume (:math:`m^3`) - - RMS ice speed (:math:`m/s`) - diff --git a/doc/source/user_guide/ug_troubleshooting.rst b/doc/source/user_guide/ug_troubleshooting.rst index 7998eef6b..e34d1261b 100644 --- a/doc/source/user_guide/ug_troubleshooting.rst +++ b/doc/source/user_guide/ug_troubleshooting.rst @@ -190,6 +190,9 @@ Known bugs - Latitude and longitude fields in the history output may be wrong when using padding. +- History and restart files will not be written on the first timestep in + some cases. + Interpretation of albedos ---------------------------------------- From bfd04dc7afc61d98db6862e94b4968bd8f6f3af8 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 21:14:56 +0000 Subject: [PATCH 37/54] cicecore updating from master --- .../cicedynB/analysis/ice_history_bgc.F90 | 12 +- .../cicedynB/analysis/ice_history_shared.F90 | 16 +- cicecore/cicedynB/dynamics/ice_dyn_eap.F90 | 8 +- cicecore/cicedynB/dynamics/ice_dyn_evp.F90 | 40 +- cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 | 622 ++++++++++++------ cicecore/cicedynB/dynamics/ice_dyn_shared.F90 | 67 +- cicecore/cicedynB/general/ice_forcing.F90 | 373 +++++++++-- cicecore/cicedynB/general/ice_forcing_bgc.F90 | 60 +- cicecore/cicedynB/general/ice_init.F90 | 56 +- cicecore/cicedynB/general/ice_step_mod.F90 | 6 - .../infrastructure/comm/mpi/ice_reprosum.F90 | 2 +- .../comm/serial/ice_reprosum.F90 | 2 +- .../cicedynB/infrastructure/ice_domain.F90 | 2 + cicecore/cicedynB/infrastructure/ice_grid.F90 | 102 ++- .../infrastructure/ice_read_write.F90 | 30 +- .../cicedynB/infrastructure/ice_restoring.F90 | 2 +- .../io/io_binary/ice_history_write.F90 | 2 +- .../io/io_binary/ice_restart.F90 | 15 +- .../io/io_netcdf/ice_restart.F90 | 9 +- .../io/io_pio/ice_history_write.F90 | 8 +- cicecore/drivers/cesm/CICE_copyright.txt | 4 +- cicecore/drivers/cesm/ice_comp_esmf.F90 | 12 +- cicecore/drivers/cesm/ice_comp_mct.F90 | 14 +- cicecore/drivers/cesm/ice_prescribed_mod.F90 | 2 +- cicecore/drivers/cice/CICE.F90 | 4 +- cicecore/drivers/cice/CICE_InitMod.F90 | 4 +- cicecore/drivers/cice/CICE_RunMod.F90 | 2 +- cicecore/drivers/hadgem3/CICE.F90 | 4 +- cicecore/shared/ice_arrays_column.F90 | 3 +- cicecore/shared/ice_calendar.F90 | 8 +- cicecore/shared/ice_restart_column.F90 | 3 - cicecore/version.txt | 2 +- 32 files changed, 1045 insertions(+), 451 deletions(-) mode change 100644 => 100755 cicecore/cicedynB/general/ice_forcing.F90 diff --git a/cicecore/cicedynB/analysis/ice_history_bgc.F90 b/cicecore/cicedynB/analysis/ice_history_bgc.F90 index ae5bccb2f..c27683423 100644 --- a/cicecore/cicedynB/analysis/ice_history_bgc.F90 +++ b/cicecore/cicedynB/analysis/ice_history_bgc.F90 @@ -112,10 +112,10 @@ module ice_history_bgc f_peakval = 'x', f_bgc_Fed_ml = 'x', & f_bgc_Fep_ml = 'x', f_bgc_hum_ml = 'x', & f_bgc_N_cat1 = 'x', f_bgc_DOC_cat1 = 'x', & - f_bgc_DIC_cat1 = 'x', f_bgc_Nit_cat1 = 'x', & + f_bgc_DIC_cat1 = 'x', f_bgc_Nit_cat1 = 'x', & f_bgc_Am_cat1 = 'x', f_bgc_Sil_cat1 = 'x', & f_bgc_DMSPd_cat1= 'x', f_bgc_DMS_cat1 = 'x', & - f_bgc_DON_cat1 = 'x', f_bgc_Fed_cat1 = 'x', & + f_bgc_DON_cat1 = 'x', f_bgc_Fed_cat1 = 'x', & f_bgc_hum_cat1 = 'x', f_bgc_Fep_cat1 = 'x', & f_bgc_PON_cat1 = 'x' @@ -226,11 +226,11 @@ module ice_history_bgc n_upNO , n_upNH , & n_bTin , n_bphi , & n_iDi , n_iki , & - n_bgc_PON , n_bgc_PON_ml , & + n_bgc_PON , & n_fbri , n_hbri , & n_zfswin , n_Nitnet , & n_Amnet , n_Silnet , & - n_humnet , & + n_humnet , & n_DMSPpnet , n_DMSPdnet , & n_DMSnet , n_PONnet , & n_Nitsnow , n_Amsnow , & @@ -238,8 +238,8 @@ module ice_history_bgc n_DMSPpsnow , n_DMSPdsnow , & n_DMSsnow , n_PONsnow , & n_Nitfrac , n_Amfrac , & - n_Silfrac , n_zbgc_frac , & - n_humfrac , & + n_Silfrac , & + n_humfrac , & n_DMSPpfrac , n_DMSPdfrac , & n_DMSfrac , n_PONfrac , & n_grownet , n_PPnet , & diff --git a/cicecore/cicedynB/analysis/ice_history_shared.F90 b/cicecore/cicedynB/analysis/ice_history_shared.F90 index d8335e1f7..64a82e9ae 100644 --- a/cicecore/cicedynB/analysis/ice_history_shared.F90 +++ b/cicecore/cicedynB/analysis/ice_history_shared.F90 @@ -452,7 +452,7 @@ module ice_history_shared f_e22, & f_s11, f_s12, & f_s22, & - f_yieldstress11, & + f_yieldstress11, & f_yieldstress12, & f_yieldstress22 @@ -608,13 +608,13 @@ module ice_history_shared n_keffn_top , & n_Tinz , n_Sinz , & n_Tsnz , & - n_a11 , n_a12 , & - n_e11 , n_e12 , & - n_e22 , & - n_s11 , n_s12 , & - n_s22 , & - n_yieldstress11, n_yieldstress12, & - n_yieldstress22 + n_a11 , n_a12 , & + n_e11 , n_e12 , & + n_e22 , & + n_s11 , n_s12 , & + n_s22 , & + n_yieldstress11, n_yieldstress12, & + n_yieldstress22 interface accum_hist_field ! generic interface module procedure accum_hist_field_2D, & diff --git a/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 b/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 index 944d80607..bc42e4855 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 @@ -119,8 +119,8 @@ subroutine eap (dt) use ice_constants, only: field_loc_center, field_loc_NEcorner, & field_type_scalar, field_type_vector use ice_domain, only: nblocks, blocks_ice, halo_info, maskhalo_dyn - use ice_dyn_shared, only: fcor_blk, ndte, dtei, a_min, m_min, & - cosw, sinw, denom1, uvel_init, vvel_init, arlx1i, & + use ice_dyn_shared, only: fcor_blk, ndte, dtei, & + denom1, uvel_init, vvel_init, arlx1i, & dyn_prep1, dyn_prep2, stepu, dyn_finish, & basal_stress_coeff, basalstress use ice_flux, only: rdg_conv, strairxT, strairyT, & @@ -1615,9 +1615,9 @@ subroutine update_stress_rdg (ksub, ndte, divu, tension, & invstressconviso, & Angle_denom_gamma, Angle_denom_alpha, & Tany_1, Tany_2, & - gamma, alpha, x, y, dx, dy, da, & + x, y, dx, dy, da, & invdx, invdy, invda, invsin, & - invleng, dtemp1, dtemp2, atempprime, & + dtemp1, dtemp2, atempprime, & kxw, kyw, kaw, & puny, pi, pi2, piq, pih diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 index d26f2d4c9..002341667 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 @@ -34,10 +34,11 @@ module ice_dyn_evp use ice_kinds_mod + use ice_communicate, only: my_task use ice_constants, only: field_loc_center, field_loc_NEcorner, & field_type_scalar, field_type_vector - use ice_constants, only: c0, c4, p027, p055, p111, p166, & - p2, p222, p25, p333, p5, c1 + use ice_constants, only: c0, p027, p055, p111, p166, & + p222, p25, p333, p5, c1 use ice_dyn_shared, only: stepu, dyn_prep1, dyn_prep2, dyn_finish, & ndte, yield_curve, ecci, denom1, arlx1i, fcor_blk, uvel_init, & vvel_init, basal_stress_coeff, basalstress, Ktens, revp @@ -91,8 +92,9 @@ subroutine evp (dt) aice_init, aice0, aicen, vicen, strength use ice_timers, only: timer_dynamics, timer_bound, & ice_timer_start, ice_timer_stop, timer_evp_1d, timer_evp_2d - use ice_dyn_evp_1d - use ice_dyn_shared, only: evp_kernel_ver + use ice_dyn_evp_1d, only: ice_dyn_evp_1d_copyin, ice_dyn_evp_1d_kernel, & + ice_dyn_evp_1d_copyout + use ice_dyn_shared, only: kevp_kernel real (kind=dbl_kind), intent(in) :: & dt ! time step @@ -346,14 +348,14 @@ subroutine evp (dt) !$OMP END PARALLEL DO endif call ice_timer_start(timer_evp_2d) - if (evp_kernel_ver > 0) then - !write(*,*)'Entering evp_kernel version ',evp_kernel_ver + if (kevp_kernel > 0) then +! if (my_task == 0) write(nu_diag,*) subname,' Entering kevp_kernel version ',kevp_kernel if (trim(grid_type) == 'tripole') then - call abort_ice('(ice_dyn_evp): & - & Kernel not tested on tripole grid. Set evp_kernel_ver=0') + call abort_ice(trim(subname)//' & + & Kernel not tested on tripole grid. Set kevp_kernel=0') endif - call evp_copyin( & - nx_block,ny_block,nblocks,nx_global+2*nghost,ny_global+2*nghost,& + call ice_dyn_evp_1d_copyin( & + nx_block,ny_block,nblocks,nx_global+2*nghost,ny_global+2*nghost, & HTE,HTN, & !v1 dxhy,dyhx,cyp,cxp,cym,cxm,tinyarea, & !v1 waterx,watery, & @@ -364,17 +366,17 @@ subroutine evp (dt) stressp_1 ,stressp_2, stressp_3, stressp_4, & stressm_1 ,stressm_2, stressm_3, stressm_4, & stress12_1,stress12_2,stress12_3,stress12_4 ) - if (evp_kernel_ver == 2) then + if (kevp_kernel == 2) then call ice_timer_start(timer_evp_1d) - call evp_kernel_v2() + call ice_dyn_evp_1d_kernel() call ice_timer_stop(timer_evp_1d) -!v1 else if (evp_kernel_ver == 1) then +!v1 else if (kevp_kernel == 1) then !v1 call evp_kernel_v1() else - write(*,*)'Kernel: evp_kernel_ver = ',evp_kernel_ver - call abort_ice('(ice_dyn_evp): Kernel not implemented.') + if (my_task == 0) write(nu_diag,*) subname,' ERROR: kevp_kernel = ',kevp_kernel + call abort_ice(subname//' kevp_kernel not supported.') endif - call evp_copyout( & + call ice_dyn_evp_1d_copyout( & nx_block,ny_block,nblocks,nx_global+2*nghost,ny_global+2*nghost,& !strocn uvel,vvel, strocnx,strocny, strintx,strinty, & uvel,vvel, strintx,strinty, & @@ -383,7 +385,7 @@ subroutine evp (dt) stress12_1,stress12_2,stress12_3,stress12_4, & divu,rdg_conv,rdg_shear,shear,taubx,tauby ) - else ! evp_kernel_ver == 0 (Standard CICE) + else ! kevp_kernel == 0 (Standard CICE) do ksub = 1,ndte ! subcycling @@ -461,7 +463,7 @@ subroutine evp (dt) !$OMP END PARALLEL DO enddo ! subcycling - endif ! evp_kernel_ver + endif ! kevp_kernel call ice_timer_stop(timer_evp_2d) deallocate(fld2) @@ -640,7 +642,7 @@ subroutine stress (nx_block, ny_block, & tensionne, tensionnw, tensionse, tensionsw, & ! tension shearne, shearnw, shearse, shearsw , & ! shearing Deltane, Deltanw, Deltase, Deltasw , & ! Delt - puny , & ! puny +! puny , & ! puny c0ne, c0nw, c0se, c0sw , & ! useful combinations c1ne, c1nw, c1se, c1sw , & ssigpn, ssigps, ssigpe, ssigpw , & diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 index 3ba03e0a8..0cb5d9102 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 @@ -1,12 +1,21 @@ +! ice_dyn_evp_1d +! +! Contained 3 Fortran modules, +! * dmi_omp +! * bench_v2 +! * ice_dyn_evp_1d +! These were merged into one module, ice_dyn_evp_1d to support some +! coupled build systems. +! ! Modules used for: ! * convert 2D arrays into 1D vectors ! * Do stress/stepu/halo_update interations ! * convert 1D vectors into 2D matrices ! ! Call from ice_dyn_evp.F90: -! call evp_copyin(...) -! call evp_kernel() -! call evp_copyout(...) +! call ice_dyn_evp_1d_copyin(...) +! call ice_dyn_evp_1d_kernel() +! call ice_dyn_evp_1d_copyout(...) ! ! * REAL4 internal version: ! mv evp_kernel1d.F90 evp_kernel1d_r8.F90 @@ -20,52 +29,111 @@ !=============================================================================== !=============================================================================== -module dmi_omp - !- directives ---------------------------------------------------------------- + +!-- One dimension representation of EVP 2D arrays used for EVP kernels +module ice_dyn_evp_1d + use ice_kinds_mod + use ice_fileunits, only: nu_diag + use ice_exit, only: abort_ice + implicit none private - INTEGER, PARAMETER :: JPIM = SELECTED_INT_KIND(9) + public :: ice_dyn_evp_1d_copyin, ice_dyn_evp_1d_copyout, ice_dyn_evp_1d_kernel - !- interfaces ---------------------------------------------------------------- - interface domp_get_domain - module procedure domp_get_domain_rlu + interface ice_dyn_evp_1d_copyin +! module procedure evp_copyin_v1 + module procedure evp_copyin_v2 end interface - integer(int_kind), private :: domp_iam, domp_nt + interface ice_dyn_evp_1d_kernel +! module procedure evp_kernel_v1 + module procedure evp_kernel_v2 + end interface - private :: domp_get_domain_rlu + interface ice_dyn_evp_1d_copyout + module procedure evp_copyout + end interface - !- public vars & methods ----------------------------------------------------- - public :: domp_init, domp_get_domain, domp_get_thread_no + interface convert_2d_1d +! module procedure convert_2d_1d_v1 + module procedure convert_2d_1d_v2 + end interface + + integer(kind=int_kind) :: & + NA_len, NAVEL_len + logical(kind=log_kind), dimension(:), allocatable :: & + skipucell + integer(kind=int_kind), dimension(:), allocatable :: & + ee,ne,se,nw,sw,sse,indi,indj,indij , halo_parent + real (kind=dbl_kind), dimension(:), allocatable :: & + cdn_ocn,aiu,uocn,vocn,forcex,forcey,Tbu,tarear, & + umassdti,fm,uarear,strintx,strinty,uvel_init,vvel_init + real (kind=dbl_kind), dimension(:), allocatable :: & + strength,uvel,vvel,dxt,dyt, & +!v1 dxhy,dyhx,cyp,cxp,cym,cxm,tinyarea, & +!v1 waterx,watery, & + stressp_1, stressp_2, stressp_3, stressp_4, & + stressm_1, stressm_2, stressm_3, stressm_4, & + stress12_1,stress12_2,stress12_3,stress12_4, & + divu,rdg_conv,rdg_shear,shear,taubx,tauby + real (kind=DBL_KIND), dimension(:), allocatable :: & + str1, str2, str3, str4, str5, str6, str7, str8 + real (kind=dbl_kind), dimension(:), allocatable :: & + HTE,HTN, & + HTEm1,HTNm1 + logical(kind=log_kind),parameter :: dbug = .false. + + +!--- dmi_omp --------------------------- + interface domp_get_domain + module procedure domp_get_domain_rlu + end interface + + INTEGER, PARAMETER :: JPIM = SELECTED_INT_KIND(9) + integer(int_kind) :: domp_iam, domp_nt #if defined (_OPENMP) ! Please note, this constant will create a compiler info for a constant ! expression in IF statements: - real(kind=dbl_kind), private :: rdomp_iam, rdomp_nt + real(kind=dbl_kind) :: rdomp_iam, rdomp_nt !$OMP THREADPRIVATE(domp_iam,domp_nt,rdomp_iam,rdomp_nt) #endif +!--- dmi_omp --------------------------- -contains - - ! ---------------------------------------------------------------------------- +!--- bench_v2 -------------------------- + interface evp1d_stress + module procedure stress_i + module procedure stress_l + end interface + interface evp1d_stepu + module procedure stepu_iter + module procedure stepu_last + end interface +!--- bench_v2 -------------------------- + + contains + +!=============================================================================== +!former module dmi_omp subroutine domp_init(nt_out) #if defined (_OPENMP) use omp_lib, only : omp_get_thread_num, omp_get_num_threads #endif - use ice_forcing, only : dbug - !- argument(s) ------------------------------------------------------------- integer(int_kind), intent(out) :: nt_out + character(len=*), parameter :: subname = '(domp_init)' + !--------------------------------------- + !$OMP PARALLEL DEFAULT(none) #if defined (_OPENMP) domp_iam = omp_get_thread_num() - rdomp_iam = real(domp_iam,8) + rdomp_iam = real(domp_iam,dbl_kind) domp_nt = omp_get_num_threads() - rdomp_nt = real(domp_nt,8) + rdomp_nt = real(domp_nt,dbl_kind) #else domp_iam = 0 domp_nt = 1 @@ -74,21 +142,21 @@ subroutine domp_init(nt_out) if (dbug) then #if defined (_OPENACC) - write(*,'(a27)') 'Build with openACC support' + write(nu_diag,'(2a)') subname,' Build with openACC support' !#elif defined (_OPENMP) -! write(*,'(a26)') 'Build with openMP support' +! write(nu_diag,'(2a)') subname,' Build with openMP support' !#else -! write(*,'(a41)') 'Build without openMP and openACC support' +! write(nu_diag,'(2a)') subname,' Build without openMP and openACC support' #endif !- echo #threads: if (domp_nt > 1) then - write(*,'(a20,i5,a8)') 'Running openMP with ', domp_nt, ' threads' + write(nu_diag,'(2a,i5,a)') subname,' Running openMP with ', domp_nt, ' threads' else #if defined (_OPENMP) - write(*,'(a35)') 'Running openMP with a single thread' + write(nu_diag,'(2a)') subname,' Running openMP with a single thread' #else - write(*,'(a22)') 'Running without openMP' + write(nu_diag,'(2a)') subname,' Running without openMP' #endif endif endif @@ -98,24 +166,26 @@ subroutine domp_init(nt_out) end subroutine domp_init - ! ---------------------------------------------------------------------------- +!---------------------------------------------------------------------------- subroutine domp_get_domain_rlu(lower,upper,d_lower,d_upper) + #if defined (_OPENMP) use omp_lib, only : omp_in_parallel -#endif use ice_constants, only: p5 +#endif - !- arguments --------------------------------------------------------------- integer(KIND=JPIM), intent(in) :: lower,upper integer(KIND=JPIM), intent(out) :: d_lower,d_upper #if defined (_OPENMP) - ! local variables --------------------------------------------------------- + !-- local variables real(kind=dbl_kind) :: dlen - integer(int_kind) :: lr, ur #endif + character(len=*), parameter :: subname = '(domp_get_domain_rlu)' + !--------------------------------------- + ! proper action in "null" cases: if (upper <= 0 .or. upper < lower) then d_lower = 0 @@ -129,41 +199,41 @@ subroutine domp_get_domain_rlu(lower,upper,d_lower,d_upper) #if defined (_OPENMP) if (omp_in_parallel()) then - dlen = real(upper-lower+1, 8) - d_lower = lower + floor((rdomp_iam*dlen+p5)/rdomp_nt, 4) - d_upper = lower -1 + floor((rdomp_iam*dlen+dlen+p5)/rdomp_nt, 4) + dlen = real(upper-lower+1, dbl_kind) + d_lower = lower + floor((rdomp_iam*dlen+p5)/rdomp_nt, JPIM) + d_upper = lower -1 + floor((rdomp_iam*dlen+dlen+p5)/rdomp_nt, JPIM) endif #endif if (.false.) then - write(*,'(a14,i3,a24,i10,i10)') 'openMP thread ', domp_iam, & + write(nu_diag,'(2a,i3,a,2i10)') subname,' openMP thread ', domp_iam, & ' handles range: ', d_lower, d_upper endif end subroutine domp_get_domain_rlu +!---------------------------------------------------------------------------- + subroutine domp_get_thread_no (tnum) - implicit none + implicit none integer(int_kind), intent(out) :: tnum + character(len=*), parameter :: subname = '(domp_get_thread_no)' tnum = domp_iam + 1 end subroutine domp_get_thread_no - ! ---------------------------------------------------------------------------- -end module dmi_omp + +!---------------------------------------------------------------------------- + +!former end module dmi_omp + !=============================================================================== -module bench_v2 - !- interfaces ---------------------------------------------------------------- - interface stress - module procedure stress_i - module procedure stress_l - end interface - interface stepu - module procedure stepu_iter - module procedure stepu_last - end interface - contains + +!former module bench_v2 + +!---------------------------------------------------------------------------- + subroutine stress_i(NA_len, & ee,ne,se,lb,ub,uvel,vvel,dxt,dyt, & hte,htn,htem1,htnm1, & @@ -171,15 +241,14 @@ subroutine stress_i(NA_len, & stressm_1,stressm_2,stressm_3,stressm_4,stress12_1, & stress12_2,stress12_3,stress12_4,str1,str2,str3,str4,str5, & str6,str7,str8) - !- modules ------------------------------------------------------------------- + use ice_kinds_mod - use dmi_omp, only : domp_get_domain use ice_constants, only: p027, p055, p111, p166, p222, p25, p333, p5, c1p5, c1 use icepack_parameters, only: puny use ice_dyn_shared, only: ecci, denom1, arlx1i, Ktens, revp - !- directives ---------------------------------------------------------------- + implicit none - ! arguments ------------------------------------------------------------------ + integer (kind=int_kind), intent(in) :: NA_len integer (kind=int_kind), intent(in) :: lb,ub integer (kind=int_kind), dimension(:), intent(in), contiguous :: ee,ne,se @@ -191,7 +260,9 @@ subroutine stress_i(NA_len, & stressm_3,stressm_4, stress12_1,stress12_2,stress12_3, stress12_4 real (kind=DBL_KIND), dimension(:), intent(out), contiguous :: & str1,str2,str3,str4,str5,str6,str7,str8 - ! local variables ------------------------------------------------------------ + + !-- local variables + integer (kind=int_kind) :: iw,il,iu real (kind=DBL_KIND) :: & divune, divunw, divuse, divusw,tensionne, tensionnw, tensionse, tensionsw, & @@ -205,6 +276,9 @@ subroutine stress_i(NA_len, & tmp_vvel_ne, tmp_uvel_ne, tmp_uvel_se real (kind=DBL_KIND) :: dxhy,dyhx,cxp,cyp,cxm,cym,tinyarea + character(len=*), parameter :: subname = '(stress_i)' + !--------------------------------------- + #ifdef _OPENACC !$acc parallel & !$acc present(ee,ne,se,strength,uvel,vvel,dxt,dyt, & @@ -419,8 +493,11 @@ subroutine stress_i(NA_len, & #ifdef _OPENACC !$acc end parallel #endif + end subroutine stress_i +!---------------------------------------------------------------------------- + subroutine stress_l(NA_len, tarear, & ee,ne,se,lb,ub,uvel,vvel,dxt,dyt, & hte,htn,htem1,htnm1, & @@ -429,15 +506,14 @@ subroutine stress_l(NA_len, tarear, & stress12_2,stress12_3,stress12_4, & divu,rdg_conv,rdg_shear,shear, & str1,str2,str3,str4,str5,str6,str7,str8 ) - !- modules ------------------------------------------------------------------- + use ice_kinds_mod - use dmi_omp, only : domp_get_domain use ice_constants, only: p027, p055, p111, p166, p222, p25, p333, p5, c1p5, c0, c1 use icepack_parameters, only: puny use ice_dyn_shared, only: ecci, denom1, arlx1i, Ktens, revp - !- directives ---------------------------------------------------------------- + implicit none - ! arguments ------------------------------------------------------------------ + integer (kind=int_kind), intent(in) :: NA_len integer (kind=int_kind), intent(in) :: lb,ub integer (kind=int_kind), dimension(:), intent(in), contiguous :: ee,ne,se @@ -451,7 +527,9 @@ subroutine stress_l(NA_len, tarear, & str1,str2,str3,str4,str5,str6,str7,str8 real (kind=dbl_kind), dimension(:), intent(out), contiguous :: & divu,rdg_conv,rdg_shear,shear - ! local variables ------------------------------------------------------------ + + !-- local variables + integer (kind=int_kind) :: iw,il,iu real (kind=DBL_KIND) :: & divune, divunw, divuse, divusw,tensionne, tensionnw, tensionse, tensionsw, & @@ -465,6 +543,9 @@ subroutine stress_l(NA_len, tarear, & tmp_vvel_ne, tmp_uvel_ne, tmp_uvel_se real (kind=DBL_KIND) :: dxhy,dyhx,cxp,cyp,cxm,cym,tinyarea + character(len=*), parameter :: subname = '(stress_l)' + !--------------------------------------- + #ifdef _OPENACC !$acc parallel & !$acc present(ee,ne,se,strength,uvel,vvel,dxt,dyt,tarear, & @@ -687,18 +768,19 @@ subroutine stress_l(NA_len, tarear, & #endif end subroutine stress_l +!---------------------------------------------------------------------------- + subroutine stepu_iter(NA_len,rhow, & lb,ub,Cw,aiu,uocn,vocn,forcex,forcey,umassdti,fm,uarear,Tbu, & uvel_init,vvel_init,uvel,vvel, & str1,str2,str3,str4,str5,str6,str7,str8, nw,sw,se,skipme) - !- modules ------------------------------------------------------------------- + use ice_kinds_mod - use dmi_omp, only : domp_get_domain use ice_dyn_shared, only: brlx, revp use ice_constants, only: c0, c1 - !- directives ---------------------------------------------------------------- + implicit none - ! arguments ------------------------------------------------------------------ + integer (kind=int_kind), intent(in) :: NA_len real (kind=dbl_kind), intent(in) :: rhow integer(kind=int_kind),intent(in) :: lb,ub @@ -714,7 +796,9 @@ subroutine stepu_iter(NA_len,rhow, & real (kind=dbl_kind), parameter :: & cosw = c1 , & ! cos(ocean turning angle) ! turning angle = 0 sinw = c0 - ! local variables + + !-- local variables + integer (kind=int_kind) :: iw,il,iu real (kind=dbl_kind) :: uold, vold, vrel,cca,ccb,ab2,cc1,cc2,taux,tauy,Cb real (kind=dbl_kind) :: tmp_str2_nw,tmp_str3_se,tmp_str4_sw, tmp_strintx @@ -722,6 +806,9 @@ subroutine stepu_iter(NA_len,rhow, & real (kind=dbl_kind) :: waterx,watery real (kind=dbl_kind) :: u0 = 5.e-5_dbl_kind ! residual velocity for basal stress (m/s) + character(len=*), parameter :: subname = '(stepu_iter)' + !--------------------------------------- + #ifdef _OPENACC !$acc parallel & !$acc present(Cw,aiu,uocn,vocn,forcex,forcey,umassdti,fm,uarear,Tbu, & @@ -770,20 +857,21 @@ subroutine stepu_iter(NA_len,rhow, & end subroutine stepu_iter +!---------------------------------------------------------------------------- + subroutine stepu_last(NA_len, rhow, & lb,ub,Cw,aiu,uocn,vocn,forcex,forcey,umassdti,fm,uarear,Tbu, & strintx,strinty,taubx,tauby, & uvel_init,vvel_init,uvel,vvel, & str1,str2,str3,str4,str5,str6,str7,str8, nw,sw,se,skipme) - !- modules ------------------------------------------------------------------- + use ice_kinds_mod - use dmi_omp, only : domp_get_domain use ice_constants, only: c0, c1 use icepack_parameters, only: puny use ice_dyn_shared, only: brlx, revp, basalstress - !- directives ---------------------------------------------------------------- + implicit none - ! arguments ------------------------------------------------------------------ + integer (kind=int_kind), intent(in) :: NA_len real (kind=dbl_kind), intent(in) :: rhow logical(kind=log_kind),intent(in), dimension(:) :: skipme @@ -799,7 +887,9 @@ subroutine stepu_last(NA_len, rhow, & real (kind=dbl_kind), parameter :: & cosw = c1 , & ! cos(ocean turning angle) ! turning angle = 0 sinw = c0 - ! local variables + + !-- local variables + integer (kind=int_kind) :: iw,il,iu real (kind=dbl_kind) :: uold, vold, vrel,cca,ccb,ab2,cc1,cc2,taux,tauy,Cb real (kind=dbl_kind) :: tmp_str2_nw,tmp_str3_se,tmp_str4_sw @@ -807,6 +897,9 @@ subroutine stepu_last(NA_len, rhow, & real (kind=dbl_kind) :: waterx,watery real (kind=dbl_kind) :: u0 = 5.e-5_dbl_kind ! residual velocity for basal stress (m/s) + character(len=*), parameter :: subname = '(stepu_last)' + !--------------------------------------- + #ifdef _OPENACC !$acc parallel & !$acc present(Cw,aiu,uocn,vocn,forcex,forcey,umassdti,fm,uarear,Tbu, & @@ -857,22 +950,29 @@ subroutine stepu_last(NA_len, rhow, & #ifdef _OPENACC !$acc end parallel #endif + end subroutine stepu_last - subroutine halo_update(NAVEL_len,lb,ub,uvel,vvel, halo_parent) - !- modules ------------------------------------------------------------------- +!---------------------------------------------------------------------------- + + subroutine evp1d_halo_update(NAVEL_len,lb,ub,uvel,vvel, halo_parent) + use ice_kinds_mod - use dmi_omp, only : domp_get_domain - !- directives ---------------------------------------------------------------- + implicit none - ! arguments ------------------------------------------------------------------ + integer (kind=int_kind), intent(in) :: NAVEL_len integer(kind=int_kind),intent(in) :: lb,ub integer(kind=int_kind),dimension(:), intent(in), contiguous :: halo_parent real(kind=dbl_kind),dimension(:), intent(inout), contiguous :: uvel,vvel - ! local variables + + !-- local variables + integer (kind=int_kind) :: iw,il,iu + character(len=*), parameter :: subname = '(evp1d_halo_update)' + !--------------------------------------- + #ifdef _OPENACC !$acc parallel & !$acc present(uvel,vvel) & @@ -889,58 +989,26 @@ subroutine halo_update(NAVEL_len,lb,ub,uvel,vvel, halo_parent) #ifdef _OPENACC !$acc end parallel #endif - end subroutine halo_update -end module bench_v2 + end subroutine evp1d_halo_update + +!---------------------------------------------------------------------------- + +!former end module bench_v2 !=============================================================================== - -!-- One dimension representation of EVP 2D arrays used for EVP kernels -module ice_dyn_evp_1d - use ice_kinds_mod - use ice_exit, only: abort_ice - !-- BEGIN: specific for the KERNEL - use ice_dyn_shared, only: revp, ecci, denom1, arlx1i, brlx - !-- END: specific for the KERNEL - implicit none - interface evp_copyin -! module procedure evp_copyin_v1 - module procedure evp_copyin_v2 - end interface - interface convert_2d_1d -! module procedure convert_2d_1d_v1 - module procedure convert_2d_1d_v2 - end interface - public :: evp_copyin, evp_copyout, evp_kernel_v2 - private - save - integer(kind=int_kind) :: & - NA_len, NAVEL_len - logical(kind=log_kind), dimension(:), allocatable :: & - skipucell - integer(kind=int_kind), dimension(:), allocatable :: & - ee,ne,se,nw,sw,sse,indi,indj,indij , halo_parent - real (kind=dbl_kind), dimension(:), allocatable :: & - cdn_ocn,aiu,uocn,vocn,forcex,forcey,Tbu,tarear, & - umassdti,fm,uarear,strintx,strinty,uvel_init,vvel_init - real (kind=dbl_kind), dimension(:), allocatable :: & - strength,uvel,vvel,dxt,dyt, & -!v1 dxhy,dyhx,cyp,cxp,cym,cxm,tinyarea, & -!v1 waterx,watery, & - stressp_1, stressp_2, stressp_3, stressp_4, & - stressm_1, stressm_2, stressm_3, stressm_4, & - stress12_1,stress12_2,stress12_3,stress12_4, & - divu,rdg_conv,rdg_shear,shear,taubx,tauby - real (kind=DBL_KIND), dimension(:), allocatable :: & - str1, str2, str3, str4, str5, str6, str7, str8 - real (kind=dbl_kind), dimension(:), allocatable :: & - HTE,HTN, & - HTEm1,HTNm1 - contains +!---------------------------------------------------------------------------- + subroutine alloc1d(na) + implicit none + integer(kind=int_kind),intent(in) :: na integer(kind=int_kind) :: ierr,nb + + character(len=*), parameter :: subname = '(alloc1d)' + !--------------------------------------- + nb=na allocate( & ! U+T cells @@ -968,22 +1036,43 @@ subroutine alloc1d(na) uvel_init(1:nb),vvel_init(1:nb), & taubx(1:nb),tauby(1:nb), & stat=ierr) - if (ierr/=0) call abort_ice('(ice_dyn_evp_1d) : Error allocating 1D') + + if (ierr/=0) call abort_ice(subname//': ERROR allocating 1D') + end subroutine alloc1d + +!---------------------------------------------------------------------------- + subroutine alloc1d_navel(navel) + implicit none + integer(kind=int_kind),intent(in) :: navel integer(kind=int_kind) :: ierr + + character(len=*), parameter :: subname = '(alloc1d_navel)' + !--------------------------------------- + allocate( & uvel(1:navel),vvel(1:navel), indij(1:navel), halo_parent(1:navel), & str1(1:navel),str2(1:navel),str3(1:navel),str4(1:navel), & str5(1:navel),str6(1:navel),str7(1:navel),str8(1:navel), & stat=ierr) - if (ierr/=0) call abort_ice('(ice_dyn_evp_1d) : Error allocating 1D navel') + if (ierr/=0) call abort_ice(subname// ': Error allocating 1D navel') + end subroutine alloc1d_navel + +!---------------------------------------------------------------------------- + subroutine dealloc1d + implicit none + integer(kind=int_kind) :: ierr + + character(len=*), parameter :: subname = '(dealloc1d)' + !--------------------------------------- + deallocate( & ! U+T cells ! Helper index for neighbours @@ -1009,24 +1098,29 @@ subroutine dealloc1d ! NAVEL uvel,vvel, indij, halo_parent, & stat=ierr) - if (ierr/=0) call abort_ice('(ice_dyn_evp_1d) : Error de-allocating 1D') + + if (ierr/=0) call abort_ice(subname//': Error de-allocating 1D') + !v1 if (allocated(tinyarea)) then !v1 deallocate( & !v1 dxhy,dyhx,cyp,cxp,cym,cxm,tinyarea, & !v1 waterx,watery, & !v1 stat=ierr) -!v1 if (ierr/=0) call abort_ice('(ice_dyn_evp_1d) : Error de-allocating 1D, v1') +!v1 if (ierr/=0) call abort_ice(subname//': Error de-allocating 1D, v1') !v1 endif + if (allocated(HTE)) then deallocate( & ! Grid distances: HTE,HTN + "-1 neighbours" HTE,HTN, HTEm1,HTNm1, & stat=ierr) - if (ierr/=0) call abort_ice('(ice_dyn_evp_1d) : Error de-allocating 1D, v2') + if (ierr/=0) call abort_ice(subname//': Error de-allocating 1D, v2') endif + end subroutine dealloc1d -!=============================================================================== -!=============================================================================== + +!---------------------------------------------------------------------------- + subroutine evp_copyin_v2(nx,ny,nblk,nx_glob,ny_glob, & I_HTE,I_HTN, & !v1 I_dxhy,I_dyhx,I_cyp,I_cxp,I_cym,I_cxm,I_tinyarea, & @@ -1038,11 +1132,13 @@ subroutine evp_copyin_v2(nx,ny,nblk,nx_glob,ny_glob, I_stressp_1 ,I_stressp_2, I_stressp_3, I_stressp_4, & I_stressm_1 ,I_stressm_2, I_stressm_3, I_stressm_4, & I_stress12_1,I_stress12_2,I_stress12_3,I_stress12_4 ) + use ice_gather_scatter, only: gather_global_ext use ice_domain, only: distrb_info use ice_communicate, only: my_task, master_task - use ice_constants, only: c0,c1,p5 + implicit none + integer(int_kind), intent(in) :: nx, ny, nblk, nx_glob, ny_glob integer (kind=int_kind),dimension (nx,ny,nblk), intent(in) :: I_icetmask logical (kind=log_kind),dimension (nx,ny,nblk), intent(in) :: I_iceumask @@ -1056,7 +1152,9 @@ subroutine evp_copyin_v2(nx,ny,nblk,nx_glob,ny_glob, I_stressp_1, I_stressp_2, I_stressp_3, I_stressp_4, & I_stressm_1, I_stressm_2, I_stressm_3, I_stressm_4, & I_stress12_1,I_stress12_2,I_stress12_3,I_stress12_4 - ! local variables + + !-- local variables + integer (kind=int_kind),dimension (nx_glob,ny_glob) :: G_icetmask logical (kind=log_kind),dimension (nx_glob,ny_glob) :: G_iceumask real (kind=dbl_kind), dimension(nx_glob,ny_glob) :: & @@ -1070,7 +1168,11 @@ subroutine evp_copyin_v2(nx,ny,nblk,nx_glob,ny_glob, G_stressm_1, G_stressm_2, G_stressm_3, G_stressm_4, & G_stress12_1,G_stress12_2,G_stress12_3,G_stress12_4 integer(kind=int_kind) :: na, navel + + character(len=*), parameter :: subname = '(evp_copyin_v2)' + !--------------------------------------- !-- Gather data into one single block -- + call gather_global_ext(G_icetmask, I_icetmask, master_task, distrb_info) call gather_global_ext(G_iceumask, I_iceumask, master_task, distrb_info) call gather_global_ext(G_HTE, I_HTE, master_task, distrb_info) @@ -1116,9 +1218,9 @@ subroutine evp_copyin_v2(nx,ny,nblk,nx_glob,ny_glob, call gather_global_ext(G_stress12_2, I_stress12_2, master_task, distrb_info) call gather_global_ext(G_stress12_3, I_stress12_3, master_task, distrb_info) call gather_global_ext(G_stress12_4, I_stress12_4, master_task, distrb_info) - ! END: Gather data !-- All calculations has to be done on the master-task -- + if (my_task == master_task) then !-- Find number of active points and allocate vectors -- call calc_na(nx_glob,ny_glob,na,G_icetmask) @@ -1144,32 +1246,38 @@ subroutine evp_copyin_v2(nx,ny,nblk,nx_glob,ny_glob, NA_len=na NAVEL_len=navel endif - !-- write check + + !-- write check !if (1 == 1) then -! write(*,*)'MHRI: INDICES start: evp-copyin' -! write(*,*) 'na,navel ', na,navel -! write(*,*) 'Min/max ee', minval(ee(1:na)), maxval(ee(1:na)) -! write(*,*) 'Min/max ne', minval(ne(1:na)), maxval(ne(1:na)) -! write(*,*) 'Min/max se', minval(se(1:na)), maxval(se(1:na)) -! write(*,*) 'Min/max nw', minval(nw(1:na)), maxval(nw(1:na)) -! write(*,*) 'Min/max sw', minval(sw(1:na)), maxval(sw(1:na)) -! write(*,*) 'Min/max sse', minval(sse(1:na)), maxval(sse(1:na)) -! write(*,*)'MHRI: INDICES end: evp-copyin' +! write(nu_diag,*) subname,' MHRI: INDICES start:' +! write(nu_diag,*) 'na,navel ', na,navel +! write(nu_diag,*) 'Min/max ee', minval(ee(1:na)), maxval(ee(1:na)) +! write(nu_diag,*) 'Min/max ne', minval(ne(1:na)), maxval(ne(1:na)) +! write(nu_diag,*) 'Min/max se', minval(se(1:na)), maxval(se(1:na)) +! write(nu_diag,*) 'Min/max nw', minval(nw(1:na)), maxval(nw(1:na)) +! write(nu_diag,*) 'Min/max sw', minval(sw(1:na)), maxval(sw(1:na)) +! write(nu_diag,*) 'Min/max sse', minval(sse(1:na)), maxval(sse(1:na)) +! write(nu_diag,*) subname,' MHRI: INDICES end:' !endif + end subroutine evp_copyin_v2 - !=============================================================================== + +!---------------------------------------------------------------------------- + subroutine evp_copyout(nx,ny,nblk,nx_glob,ny_glob, & I_uvel,I_vvel, I_strintx,I_strinty, & I_stressp_1, I_stressp_2, I_stressp_3, I_stressp_4, & I_stressm_1, I_stressm_2, I_stressm_3, I_stressm_4, & I_stress12_1,I_stress12_2,I_stress12_3,I_stress12_4, & I_divu,I_rdg_conv,I_rdg_shear,I_shear,I_taubx,I_tauby ) - use ice_constants, only : c0, field_loc_center, field_loc_NEcorner, & - field_type_scalar, field_type_vector - use ice_gather_scatter, only: scatter_global_ext, scatter_global + + use ice_constants, only : c0 + use ice_gather_scatter, only: scatter_global_ext use ice_domain, only: distrb_info use ice_communicate, only: my_task, master_task + implicit none + integer(int_kind), intent(in) :: nx,ny,nblk, nx_glob,ny_glob real(dbl_kind), dimension(nx,ny,nblk), intent(out) :: & I_uvel,I_vvel, I_strintx,I_strinty, & @@ -1177,7 +1285,9 @@ subroutine evp_copyout(nx,ny,nblk,nx_glob,ny_glob, & I_stressm_1, I_stressm_2, I_stressm_3, I_stressm_4, & I_stress12_1,I_stress12_2,I_stress12_3,I_stress12_4, & I_divu,I_rdg_conv, I_rdg_shear,I_shear, I_taubx,I_tauby - ! local variables + + !-- local variables + real(dbl_kind), dimension(nx_glob,ny_glob) :: & G_uvel,G_vvel, G_strintx,G_strinty, & G_stressp_1, G_stressp_2, G_stressp_3, G_stressp_4, & @@ -1185,8 +1295,12 @@ subroutine evp_copyout(nx,ny,nblk,nx_glob,ny_glob, & G_stress12_1,G_stress12_2,G_stress12_3,G_stress12_4, & G_divu,G_rdg_conv, G_rdg_shear,G_shear, G_taubx,G_tauby integer(int_kind) :: i,j,iw, nx_block + + character(len=*), parameter :: subname = '(evp_copyout)' + !--------------------------------------- ! Remap 1d to 2d and fill in nx_block=nx_glob ! Total block size in x-dir + if (my_task == master_task) then G_uvel = c0 G_vvel = c0 @@ -1251,7 +1365,7 @@ subroutine evp_copyout(nx,ny,nblk,nx_glob,ny_glob, & !-- Scatter data into blocks -- !-- has to be done on all tasks -- - ! BEGIN: Scatter data + call scatter_global_ext(I_uvel, G_uvel, master_task, distrb_info) call scatter_global_ext(I_vvel, G_vvel, master_task, distrb_info) call scatter_global_ext(I_strintx, G_strintx, master_task, distrb_info) @@ -1274,22 +1388,29 @@ subroutine evp_copyout(nx,ny,nblk,nx_glob,ny_glob, & call scatter_global_ext(I_shear, G_shear, master_task, distrb_info) call scatter_global_ext(I_taubx, G_taubx, master_task, distrb_info) call scatter_global_ext(I_tauby, G_tauby, master_task, distrb_info) + end subroutine evp_copyout - !=============================================================================== + +!---------------------------------------------------------------------------- + subroutine evp_kernel_v2 + use ice_constants, only : c0 use ice_dyn_shared, only: ndte - use bench_v2, only : stress, stepu, halo_update - use dmi_omp, only : domp_init use icepack_intfc, only: icepack_query_parameters use ice_communicate, only: my_task, master_task implicit none + real(kind=dbl_kind) :: rhow - integer (kind=int_kind) :: ierr, lun, i, nthreads + integer (kind=int_kind) :: i, nthreads integer (kind=int_kind) :: na,nb,navel + character(len=*), parameter :: subname = '(evp_kernel_v2)' + !--------------------------------------- !-- All calculations has to be done on one single node (choose master-task) -- + if (my_task == master_task) then + !- Read constants... call icepack_query_parameters(rhow_out=rhow) na=NA_len @@ -1309,10 +1430,11 @@ subroutine evp_kernel_v2 str7=c0 str8=c0 - if (ndte<2) STOP 'ndte must be 2 or higher for this kernel' + if (ndte<2) call abort_ice(subname//' ERROR: ndte must be 2 or higher for this kernel') + !$OMP PARALLEL PRIVATE(i) do i = 1, ndte-1 - call stress (NA_len, & + call evp1d_stress(NA_len, & ee,ne,se,1,na,uvel,vvel,dxt,dyt, & hte,htn,htem1,htnm1, & strength,stressp_1,stressp_2,stressp_3,stressp_4, & @@ -1320,15 +1442,16 @@ subroutine evp_kernel_v2 stress12_2,stress12_3,stress12_4,str1,str2,str3, & str4,str5,str6,str7,str8) !$OMP BARRIER - call stepu(NA_len, rhow, & + call evp1d_stepu(NA_len, rhow, & 1,nb,cdn_ocn,aiu,uocn,vocn,forcex,forcey,umassdti,fm,uarear,Tbu,& uvel_init,vvel_init,uvel,vvel, & str1,str2,str3,str4,str5,str6,str7,str8, nw,sw,sse,skipucell) !$OMP BARRIER - call halo_update(NA_len,1,navel,uvel,vvel, halo_parent) + call evp1d_halo_update(NA_len,1,navel,uvel,vvel, halo_parent) !$OMP BARRIER enddo - call stress (NA_len, tarear, & + + call evp1d_stress(NA_len, tarear, & ee,ne,se,1,na,uvel,vvel,dxt,dyt, & hte,htn,htem1,htnm1, & strength,stressp_1,stressp_2,stressp_3,stressp_4, & @@ -1337,25 +1460,35 @@ subroutine evp_kernel_v2 divu,rdg_conv,rdg_shear,shear, & str1,str2,str3,str4,str5,str6,str7,str8) !$OMP BARRIER - call stepu (NA_len, rhow, & + call evp1d_stepu(NA_len, rhow, & 1,nb,cdn_ocn,aiu,uocn,vocn,forcex,forcey,umassdti,fm,uarear,Tbu,& strintx,strinty,taubx,tauby, & uvel_init,vvel_init,uvel,vvel, & str1,str2,str3,str4,str5,str6,str7,str8, nw,sw,sse,skipucell) !$OMP BARRIER - call halo_update(NA_len,1,navel,uvel,vvel, halo_parent) + call evp1d_halo_update(NA_len,1,navel,uvel,vvel, halo_parent) !$OMP END PARALLEL + endif + end subroutine evp_kernel_v2 - !=============================================================================== + +!---------------------------------------------------------------------------- + subroutine calc_na(nx,ny,na,icetmask) ! Calculate number of active points (na) use ice_blocks, only: nghost + implicit none + integer(int_kind),intent(in) :: nx,ny integer(int_kind),intent(out) :: na integer (kind=int_kind),dimension (nx,ny), intent(in) :: icetmask integer(int_kind) :: i,j + + character(len=*), parameter :: subname = '(calc_na)' + !--------------------------------------- + na = 0 ! Note: The icellt mask includes north and east ghost cells. (ice_dyn_shared.F90) do j = 1+nghost, ny ! -nghost @@ -1365,14 +1498,25 @@ subroutine calc_na(nx,ny,na,icetmask) endif enddo enddo + end subroutine calc_na + +!---------------------------------------------------------------------------- + subroutine calc_2d_indices(nx,ny,na,icetmask,iceumask) + use ice_blocks, only: nghost + implicit none + integer(int_kind),intent(in) :: nx,ny,na integer (kind=int_kind),dimension (nx,ny), intent(in) :: icetmask logical (kind=log_kind),dimension (nx,ny), intent(in) :: iceumask integer(int_kind) :: i,j,Nmaskt + + character(len=*), parameter :: subname = '(calc_2d_indices)' + !--------------------------------------- + skipucell(:)=.false. indi=0 indj=0 @@ -1392,21 +1536,32 @@ subroutine calc_2d_indices(nx,ny,na,icetmask,iceumask) enddo enddo if (Nmaskt.ne.na) then - write(*,*)'Nmaskt,na: ',Nmaskt,na - call abort_ice('(ice_dyn_evp_1d) : Problem Nmaskt != na') + write(nu_diag,*) subname,' Nmaskt,na: ',Nmaskt,na + call abort_ice(subname//': ERROR Problem Nmaskt != na') endif if (Nmaskt==0) then - write(*,*)'WARNING: NO ICE' + write(nu_diag,*) subname,' WARNING: NO ICE' endif + end subroutine calc_2d_indices + +!---------------------------------------------------------------------------- + subroutine calc_navel(nx_block,ny_block,na,navel) ! Calculate number of active points including needed halo points (navel) + implicit none + integer(int_kind),intent(in) :: nx_block,ny_block,na integer(int_kind),intent(out) :: navel + integer(int_kind) :: iw,i,j integer(int_kind),dimension(1:na) :: Iin,Iee,Ine,Ise,Inw,Isw,Isse integer(int_kind),dimension(1:7*na) :: util1,util2 + + character(len=*), parameter :: subname = '(calc_navel)' + + !--------------------------------------- ! Additional indices used for finite differences (FD) do iw=1,na i=indi(iw) @@ -1419,6 +1574,7 @@ subroutine calc_navel(nx_block,ny_block,na,navel) Isw(iw) = i+1 + (j-0)*nx_block ! (+1,+1) Isse(iw)= i + (j-0)*nx_block ! ( 0,+1) enddo + !-- Find number of points needed for finite difference calculations call union(Iin, Iee,na,na,util1,i) call union(util1,Ine, i,na,util2,j) @@ -1426,15 +1582,20 @@ subroutine calc_navel(nx_block,ny_block,na,navel) call union(util1,Inw, i,na,util2,j) call union(util2,Isw, j,na,util1,i) call union(util1,Isse,i,na,util2,navel) + !-- Check bounds do iw=1,navel if (util2(iw)>nx_block*ny_block .or. util2(iw)<1) then - write(*,*)'nx_block,ny_block,nx_block*ny_block: ',nx_block,ny_block,nx_block*ny_block - write(*,*)'na,navel,iw,util2(iw): ',na,navel,iw,util2(iw) - call abort_ice('(ice_dyn_evp_1d) : Problem with boundary. Check halo zone values') + write(nu_diag,*) subname,' nx_block,ny_block,nx_block*ny_block: ',nx_block,ny_block,nx_block*ny_block + write(nu_diag,*) subname,' na,navel,iw,util2(iw): ',na,navel,iw,util2(iw) + call abort_ice(subname//': Problem with boundary. Check halo zone values') endif enddo + end subroutine calc_navel + +!---------------------------------------------------------------------------- + subroutine convert_2d_1d_v2(nx,ny, na,navel, & I_HTE,I_HTN, & !v1 I_dxhy,I_dyhx,I_cyp,I_cxp,I_cym,I_cxm,I_tinyarea, & @@ -1446,7 +1607,9 @@ subroutine convert_2d_1d_v2(nx,ny, na,navel, & I_stressp_1 ,I_stressp_2, I_stressp_3, I_stressp_4, & I_stressm_1 ,I_stressm_2, I_stressm_3, I_stressm_4, & I_stress12_1,I_stress12_2,I_stress12_3,I_stress12_4 ) + implicit none + integer(int_kind),intent(in) :: nx,ny,na,navel real (kind=dbl_kind), dimension(nx,ny), intent(in) :: & I_HTE,I_HTN, & @@ -1459,10 +1622,15 @@ subroutine convert_2d_1d_v2(nx,ny, na,navel, & I_stressp_1 ,I_stressp_2, I_stressp_3, I_stressp_4, & I_stressm_1 ,I_stressm_2, I_stressm_3, I_stressm_4, & I_stress12_1,I_stress12_2,I_stress12_3,I_stress12_4 + integer(int_kind) :: iw,i,j, nx_block integer(int_kind),dimension(1:na) :: Iin,Iee,Ine,Ise,Inw,Isw,Isse integer(int_kind),dimension(1:7*na) :: util1,util2 integer(int_kind) :: nachk + + character(len=*), parameter :: subname = '(convert_2d_1d_v2)' + + !--------------------------------------- ! Additional indices used for finite differences (FD) nx_block=nx ! Total block size in x-dir do iw=1,na @@ -1476,6 +1644,7 @@ subroutine convert_2d_1d_v2(nx,ny, na,navel, & Isw(iw) = i+1 + (j-0)*nx_block ! (+1,+1) Isse(iw)= i + (j-0)*nx_block ! ( 0,+1) enddo + !-- Find number of points needed for finite difference calculations call union(Iin, Iee,na,na,util1,i) call union(util1,Ine, i,na,util2,j) @@ -1483,15 +1652,17 @@ subroutine convert_2d_1d_v2(nx,ny, na,navel, & call union(util1,Inw, i,na,util2,j) call union(util2,Isw, j,na,util1,i) call union(util1,Isse,i,na,util2,nachk) + if (nachk .ne. navel) then - write(*,*)'ERROR: navel badly chosen: na,navel,nachk = ',na,navel,nachk - call abort_ice('(ice_dyn_evp_1d) : ERROR: navel badly chosen') + write(nu_diag,*) subname,' ERROR: navel badly chosen: na,navel,nachk = ',na,navel,nachk + call abort_ice(subname//': ERROR: navel badly chosen') endif ! indij: vector with target points (sorted) ... do iw=1,na indij(iw)=Iin(iw) enddo + ! indij: ... followed by extra points (sorted) call setdiff(util2,Iin,navel,na,util1,j) do iw=na+1,navel @@ -1508,14 +1679,14 @@ subroutine convert_2d_1d_v2(nx,ny, na,navel, & !-- write check !if (1 == 2) then -! write(*,*)'MHRI: INDICES start: convert_2d_1d_v2' -! write(*,*) 'Min/max ee', minval(ee), maxval(ee) -! write(*,*) 'Min/max ne', minval(ne), maxval(ne) -! write(*,*) 'Min/max se', minval(se), maxval(se) -! write(*,*) 'Min/max nw', minval(nw), maxval(nw) -! write(*,*) 'Min/max sw', minval(sw), maxval(sw) -! write(*,*) 'Min/max sse',minval(sse),maxval(sse) -! write(*,*)'MHRI: INDICES end: convert_2d_1d_v2' +! write(nu_diag,*) subname,' MHRI: INDICES start:' +! write(nu_diag,*) 'Min/max ee', minval(ee), maxval(ee) +! write(nu_diag,*) 'Min/max ne', minval(ne), maxval(ne) +! write(nu_diag,*) 'Min/max se', minval(se), maxval(se) +! write(nu_diag,*) 'Min/max nw', minval(nw), maxval(nw) +! write(nu_diag,*) 'Min/max sw', minval(sw), maxval(sw) +! write(nu_diag,*) 'Min/max sse',minval(sse),maxval(sse) +! write(nu_diag,*) subname,' MHRI: INDICES end:' !endif ! Write 1D data from 2D: Here only extra FD part, the rest follows... @@ -1580,20 +1751,33 @@ subroutine convert_2d_1d_v2(nx,ny, na,navel, & HTNm1(iw) = I_HTN(i,j-1) enddo !$OMP END PARALLEL DO + end subroutine convert_2d_1d_v2 + +!---------------------------------------------------------------------------- + subroutine calc_halo_parent(nx,ny,na,navel, I_icetmask) + implicit none + integer(kind=int_kind),intent(in) :: nx,ny,na,navel integer(kind=int_kind), dimension(nx,ny), intent(in) :: I_icetmask + integer(kind=int_kind) :: iw,i,j !,masku,maskt integer(kind=int_kind),dimension(1:navel) :: Ihalo + + character(len=*), parameter :: subname = '(calc_halo_parent)' + + !--------------------------------------- ! Indices for halo update: ! 0: no halo point ! >0: index for halo point parent. Finally related to indij vector ! TODO: ONLY for nghost==1 ! TODO: ONLY for circular grids - NOT tripole grids + Ihalo(:)=0 halo_parent(:)=0 + !$OMP PARALLEL DO PRIVATE(iw,i,j) do iw=1,navel j=int((indij(iw)-1)/(nx))+1 @@ -1605,13 +1789,14 @@ subroutine calc_halo_parent(nx,ny,na,navel, I_icetmask) if (j==1 .and. I_icetmask(i,ny-1)==1) Ihalo(iw)= i+(ny-2)*nx enddo !$OMP END PARALLEL DO + ! Relate halo indices to indij vector call findXinY_halo(Ihalo,indij,navel,navel,halo_parent) !-- write check !if (1 == 1) then ! integer(kind=int_kind) :: iiw,ii,jj !,masku,maskt MHRI -! write(*,*)'MHRI: halo boundary start: calc_halo_parent ' +! write(nu_diag,*) subname,' MHRI: halo boundary start:' ! do iw=1,navel ! if (halo_parent(iw)>0) then ! iiw=halo_parent(iw) @@ -1621,23 +1806,33 @@ subroutine calc_halo_parent(nx,ny,na,navel, I_icetmask) ! jj=j ! j=int((indij(iw)-1)/(nx))+1 ! i=indij(iw)-(j-1)*nx -! write(*,*)iw,i,j,iiw,ii,jj +! write(nu_diag,*)iw,i,j,iiw,ii,jj ! endif ! enddo -! write(*,*)'MHRI: halo boundary end: calc_halo_parent ' +! write(nu_diag,*) subname,' MHRI: halo boundary end:' !endif + end subroutine calc_halo_parent - !======================================================================= + +!---------------------------------------------------------------------------- + subroutine union(x,y,nx,ny,xy,nxy) ! Find union (xy) of two sorted integer vectors (x and y) ! ie. Combined values of the two vectors with no repetitions. !use ice_kinds_mod + implicit none + integer (int_kind) :: i,j,k integer (int_kind),intent(in) :: nx,ny integer (int_kind),intent(in) :: x(1:nx),y(1:ny) integer (int_kind),intent(out) :: xy(1:nx+ny) integer (int_kind),intent(out) :: nxy + + character(len=*), parameter :: subname = '(union)' + + !--------------------------------------- + i=1 j=1 k=1 @@ -1655,6 +1850,7 @@ subroutine union(x,y,nx,ny,xy,nxy) endif k=k+1 enddo + ! The rest do while (i<=nx) xy(k)=x(i) @@ -1667,18 +1863,27 @@ subroutine union(x,y,nx,ny,xy,nxy) k=k+1 enddo nxy=k-1 + end subroutine union - !======================================================================= + +!---------------------------------------------------------------------------- + subroutine setdiff(x,y,nx,ny,xy,nxy) ! Find element (xy) of two sorted integer vectors (x and y) ! that are in x, but not in y ... or in y, but not in x !use ice_kinds_mod + implicit none + integer (int_kind) :: i,j,k integer (int_kind),intent(in) :: nx,ny integer (int_kind),intent(in) :: x(1:nx),y(1:ny) integer (int_kind),intent(out) :: xy(1:nx+ny) integer (int_kind),intent(out) :: nxy + + character(len=*), parameter :: subname = '(setdiff)' + !--------------------------------------- + i=1 j=1 k=1 @@ -1696,6 +1901,7 @@ subroutine setdiff(x,y,nx,ny,xy,nxy) j=j+1 endif enddo + ! The rest do while (i<=nx) xy(k)=x(i) @@ -1708,8 +1914,11 @@ subroutine setdiff(x,y,nx,ny,xy,nxy) k=k+1 enddo nxy=k-1 + end subroutine setdiff - !======================================================================= + +!---------------------------------------------------------------------------- + subroutine findXinY(x,y,nx,ny,indx) ! Find indx vector so that x(1:na)=y(indx(1:na)) ! @@ -1722,11 +1931,17 @@ subroutine findXinY(x,y,nx,ny,indx) ! Return: indx(1:na) ! !use ice_kinds_mod + implicit none + integer (int_kind),intent(in) :: nx,ny integer (int_kind),intent(in) :: x(1:nx),y(1:ny) integer (int_kind),intent(out) :: indx(1:nx) integer (int_kind) :: i,j1,j2 + + character(len=*), parameter :: subname = '(findXinY)' + !--------------------------------------- + i=1 j1=1 j2=nx+1 @@ -1744,13 +1959,17 @@ subroutine findXinY(x,y,nx,ny,indx) else if (x(i)>y(j2) ) then !.and. j22) then ! Stop for inf. loop. This check should not be necessary for halo - write(*,*)'nx,ny: ',nx,ny - write(*,*)'i,j1: ',i,j1 - write(*,*)'x(i),y(j1): ',x(i),y(j1) - call abort_ice('(ice_dyn_evp_1d) : ERROR in findXinY_halo: too many loops') + write(nu_diag,*) subname,' nx,ny: ',nx,ny + write(nu_diag,*) subname,' i,j1: ',i,j1 + write(nu_diag,*) subname,' x(i),y(j1): ',x(i),y(j1) + call abort_ice(subname//': ERROR too many loops') endif endif endif end do + end subroutine findXinY_halo - !======================================================================= +!---------------------------------------------------------------------------- + subroutine numainit(l,u,uu) - !- modules ----------------------------------------------------------------- - use dmi_omp, only : domp_get_domain + use ice_constants, only: c0 + implicit none + integer(kind=int_kind),intent(in) :: l,u,uu + integer(kind=int_kind) :: lo,up + + character(len=*), parameter :: subname = '(numainit)' + !--------------------------------------- + call domp_get_domain(l,u,lo,up) ee(lo:up)=0 ne(lo:up)=0 @@ -1877,8 +2110,11 @@ subroutine numainit(l,u,uu) str6(lo:up)=c0 str7(lo:up)=c0 str8(lo:up)=c0 + end subroutine numainit - !======================================================================= +!---------------------------------------------------------------------------- +!=============================================================================== + end module ice_dyn_evp_1d diff --git a/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 b/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 index dc929ddee..fec4c6823 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 @@ -11,7 +11,8 @@ module ice_dyn_shared use ice_kinds_mod use ice_communicate, only: my_task, master_task - use ice_constants, only: c0, c1, p01, p001 + use ice_constants, only: c0, c1, c2, p01, p001 + use ice_constants, only: omega, spval_dbl, p5, c4 use ice_blocks, only: nx_block, ny_block use ice_domain_size, only: max_blocks use ice_fileunits, only: nu_diag @@ -40,10 +41,10 @@ module ice_dyn_shared revised_evp ! if true, use revised evp procedure integer (kind=int_kind), public :: & - evp_kernel_ver ! 0 = 2D org version - ! 1 = 1D representation raw (not implemented) - ! 2 = 1D + calculate distances inline (implemented) - ! 3 = 1D + calculate distances inline + real*4 internal (not implemented yet) + kevp_kernel ! 0 = 2D org version + ! 1 = 1D representation raw (not implemented) + ! 2 = 1D + calculate distances inline (implemented) + ! 3 = 1D + calculate distances inline + real*4 internal (not implemented yet) ! other EVP parameters character (len=char_len), public :: & @@ -115,7 +116,6 @@ end subroutine alloc_dyn_shared subroutine init_evp (dt) use ice_blocks, only: nx_block, ny_block - use ice_constants, only: c0, c2, omega use ice_domain, only: nblocks use ice_domain_size, only: max_blocks use ice_flux, only: rdg_conv, rdg_shear, iceumask, & @@ -205,12 +205,6 @@ end subroutine init_evp subroutine set_evp_parameters (dt) - use ice_communicate, only: my_task, master_task - use ice_constants, only: p25, c1, c2, p5 - use ice_domain, only: distrb_info - use ice_global_reductions, only: global_minval - use ice_grid, only: dxt, dyt, tmask - real (kind=dbl_kind), intent(in) :: & dt ! time step @@ -275,8 +269,6 @@ subroutine dyn_prep1 (nx_block, ny_block, & strairx, strairy, & tmass, icetmask) - use ice_constants, only: c0 - integer (kind=int_kind), intent(in) :: & nx_block, ny_block, & ! block dimensions ilo,ihi,jlo,jhi ! beginning and end of physical domain @@ -409,8 +401,6 @@ subroutine dyn_prep2 (nx_block, ny_block, & uvel, vvel, & Tbu) - use ice_constants, only: c0, c1 - integer (kind=int_kind), intent(in) :: & nx_block, ny_block, & ! block dimensions ilo,ihi,jlo,jhi ! beginning and end of physical domain @@ -503,20 +493,7 @@ subroutine dyn_prep2 (nx_block, ny_block, & taubx (i,j) = c0 tauby (i,j) = c0 - if (revp==1) then ! revised evp - stressp_1 (i,j) = c0 - stressp_2 (i,j) = c0 - stressp_3 (i,j) = c0 - stressp_4 (i,j) = c0 - stressm_1 (i,j) = c0 - stressm_2 (i,j) = c0 - stressm_3 (i,j) = c0 - stressm_4 (i,j) = c0 - stress12_1(i,j) = c0 - stress12_2(i,j) = c0 - stress12_3(i,j) = c0 - stress12_4(i,j) = c0 - else if (icetmask(i,j)==0) then ! classic evp + if (icetmask(i,j)==0) then stressp_1 (i,j) = c0 stressp_2 (i,j) = c0 stressp_3 (i,j) = c0 @@ -529,7 +506,7 @@ subroutine dyn_prep2 (nx_block, ny_block, & stress12_2(i,j) = c0 stress12_3(i,j) = c0 stress12_4(i,j) = c0 - endif ! revp + endif enddo ! i enddo ! j @@ -890,8 +867,6 @@ subroutine basal_stress_coeff (nx_block, ny_block, & vice, aice, & hwater, Tbu) - use ice_constants, only: c0, c1 - integer (kind=int_kind), intent(in) :: & nx_block, ny_block, & ! block dimensions icellu ! no. of cells where icetmask = 1 @@ -913,8 +888,10 @@ subroutine basal_stress_coeff (nx_block, ny_block, & hu, & ! volume per unit area of ice at u location (mean thickness) hwu, & ! water depth at u location hcu, & ! critical thickness at u location - k2 = 15.0_dbl_kind , & ! second free parameter (N/m^3) for landfast parametrization - alphab = 20.0_dbl_kind ! alphab=Cb factor in Lemieux et al 2015 + k2 = 15.0_dbl_kind , & ! second free parameter (N/m^3) for landfast parametrization + alphab = 20.0_dbl_kind, & ! alphab=Cb factor in Lemieux et al 2015 + threshold_hw = 30.0_dbl_kind ! max water depth for grounding + ! see keel data from Amundrud et al. 2004 (JGR) integer (kind=int_kind) :: & i, j, ij @@ -926,15 +903,21 @@ subroutine basal_stress_coeff (nx_block, ny_block, & j = indxuj(ij) ! convert quantities to u-location - au = max(aice(i,j),aice(i+1,j),aice(i,j+1),aice(i+1,j+1)) + hwu = min(hwater(i,j),hwater(i+1,j),hwater(i,j+1),hwater(i+1,j+1)) - hu = max(vice(i,j),vice(i+1,j),vice(i,j+1),vice(i+1,j+1)) - ! 1- calculate critical thickness - hcu = au * hwu / k1 + if (hwu < threshold_hw) then + + au = max(aice(i,j),aice(i+1,j),aice(i,j+1),aice(i+1,j+1)) + hu = max(vice(i,j),vice(i+1,j),vice(i,j+1),vice(i+1,j+1)) - ! 2- calculate basal stress factor - Tbu(i,j) = k2 * max(c0,(hu - hcu)) * exp(-alphab * (c1 - au)) + ! 1- calculate critical thickness + hcu = au * hwu / k1 + + ! 2- calculate basal stress factor + Tbu(i,j) = k2 * max(c0,(hu - hcu)) * exp(-alphab * (c1 - au)) + + endif enddo ! ij @@ -953,8 +936,6 @@ subroutine principal_stress(nx_block, ny_block, & sig1, sig2, & sigP) - use ice_constants, only: spval_dbl, p5, c4 - integer (kind=int_kind), intent(in) :: & nx_block, ny_block ! block dimensions diff --git a/cicecore/cicedynB/general/ice_forcing.F90 b/cicecore/cicedynB/general/ice_forcing.F90 old mode 100644 new mode 100755 index 4baa88fff..53eb0ec49 --- a/cicecore/cicedynB/general/ice_forcing.F90 +++ b/cicecore/cicedynB/general/ice_forcing.F90 @@ -30,9 +30,9 @@ module ice_forcing use ice_timers, only: ice_timer_start, ice_timer_stop, timer_readwrite, & timer_bound use ice_arrays_column, only: oceanmixed_ice, restore_bgc - use ice_constants, only: c0, c1, c2, c3, c4, c5, c10, c12, c20, & - c180, c365, c1000, c3600 - use ice_constants, only: p001, p01, p1, p25, p5, p6 + use ice_constants, only: c0, c1, c2, c3, c4, c5, c8, c10, c12, c15, c20, & + c180, c360, c365, c1000, c3600 + use ice_constants, only: p001, p01, p1, p2, p25, p5, p6 use ice_constants, only: cm_to_m use ice_constants, only: field_loc_center, field_type_scalar, & field_type_vector, field_loc_NEcorner @@ -115,7 +115,8 @@ module ice_forcing atm_data_format, & ! 'bin'=binary or 'nc'=netcdf ocn_data_format, & ! 'bin'=binary or 'nc'=netcdf atm_data_type, & ! 'default', 'monthly', 'ncar', - ! 'LYq' or 'hadgem' or 'oned' + ! 'LYq' or 'hadgem' or 'oned' or + ! 'JRA55' bgc_data_type, & ! 'default', 'clim' ocn_data_type, & ! 'default', 'clim', 'ncar', 'oned', ! 'hadgem_sst' or 'hadgem_sst_uvocn' @@ -211,6 +212,8 @@ subroutine init_forcing_atmo ! Determine the current and final year of the forcing cycle based on ! namelist input; initialize the atmospheric forcing data filenames. + use ice_calendar, only: use_leap_years + character(len=*), parameter :: subname = '(init_forcing_atmo)' ! Allocate forcing arrays @@ -235,6 +238,14 @@ subroutine init_forcing_atmo file=__FILE__, line=__LINE__) endif + if (use_leap_years .and. (trim(atm_data_type) /= 'JRA55' .and. & + trim(atm_data_type) /= 'default' .and. & + trim(atm_data_type) /= 'box2001')) then + write(nu_diag,*) 'use_leap_years option is currently only supported for' + write(nu_diag,*) 'JRA55, default , and box2001 atmospheric data' + call abort_ice(error_message=subname, file=__FILE__, line=__LINE__) + endif + !------------------------------------------------------------------- ! Get filenames for input forcing data !------------------------------------------------------------------- @@ -244,6 +255,8 @@ subroutine init_forcing_atmo call NCAR_files(fyear) elseif (trim(atm_data_type) == 'LYq') then call LY_files(fyear) + elseif (trim(atm_data_type) == 'JRA55') then + call JRA55_files(fyear) elseif (trim(atm_data_type) == 'hadgem') then call hadgem_files(fyear) elseif (trim(atm_data_type) == 'monthly') then @@ -414,11 +427,11 @@ subroutine init_forcing_ocn(dt) if (trim(ocn_data_type) == 'hadgem_sst' .or. & trim(ocn_data_type) == 'hadgem_sst_uvocn') then - diag = .true. ! write diagnostic information + diag = .true. ! write diagnostic information sst_file = trim (ocn_data_dir)//'/MONTHLY/sst.1997.nc' - if (my_task == master_task) then + if (my_task == master_task) then write (nu_diag,*) ' ' write (nu_diag,*) 'Initial SST file:', trim(sst_file) @@ -539,6 +552,8 @@ subroutine get_forcing_atmo call ncar_data elseif (trim(atm_data_type) == 'LYq') then call LY_data + elseif (trim(atm_data_type) == 'JRA55') then + call JRA55_data(fyear) elseif (trim(atm_data_type) == 'hadgem') then call hadgem_data elseif (trim(atm_data_type) == 'monthly') then @@ -1376,6 +1391,10 @@ subroutine file_year (data_file, yr) i = index(data_file,'.nc') - 5 tmpname = data_file write(data_file,'(a,i4.4,a)') tmpname(1:i), yr, '.nc' + elseif (trim(atm_data_type) == 'JRA55') then ! netcdf + i = index(data_file,'.nc') - 5 + tmpname = data_file + write(data_file,'(a,i4.4,a)') tmpname(1:i), yr, '.nc' else ! LANL/NCAR naming convention i = index(data_file,'.dat') - 5 tmpname = data_file @@ -2002,6 +2021,22 @@ subroutine LY_files (yr) endif ! master_task end subroutine LY_files + subroutine JRA55_files(yr) +! + integer (kind=int_kind), intent(in) :: & + yr ! current forcing year + + character(len=*), parameter :: subname = '(JRA55_files)' + + uwind_file = & + trim(atm_data_dir)//'/8XDAILY/JRA55_03hr_forcing_2005.nc' + call file_year(uwind_file,yr) + if (my_task == master_task) then + write (nu_diag,*) ' ' + write (nu_diag,*) 'Atmospheric data files:' + write (nu_diag,*) trim(uwind_file) + endif + end subroutine JRA55_files !======================================================================= ! @@ -2225,6 +2260,239 @@ subroutine LY_data end subroutine LY_data + subroutine JRA55_data (yr) + + use ice_blocks, only: block, get_block + use ice_global_reductions, only: global_minval, global_maxval + use ice_domain, only: nblocks, distrb_info, blocks_ice + use ice_flux, only: fsnow, Tair, uatm, vatm, Qa, fsw, flw + use ice_grid, only: hm, tlon, tlat, tmask, umask + use ice_state, only: aice + use ice_calendar, only: days_per_year, use_leap_years + + + integer (kind=int_kind) :: & + ncid , & ! netcdf file id + i, j , & + ixm,ixx,ixp , & ! record numbers for neighboring months + recnum , & ! record number + maxrec , & ! maximum record number + recslot , & ! spline slot for current record + midmonth , & ! middle day of month + dataloc , & ! = 1 for data located in middle of time interval + ! = 2 for date located at end of time interval + iblk , & ! block index + ilo,ihi,jlo,jhi, & ! beginning and end of physical domain + yr ! current forcing year + + real (kind=dbl_kind) :: & + sec3hr , & ! number of seconds in 3 hours + secday , & ! number of seconds in day + Tffresh , & + vmin, vmax + + logical (kind=log_kind) :: readm, read6,debug_n_d + + type (block) :: & + this_block ! block information for current block + + character(len=64) :: fieldname !netcdf field name + character(len=*), parameter :: subname = '(JRA55_data)' + + debug_n_d = .false. !usually false + + call icepack_query_parameters(Tffresh_out=Tffresh) + call icepack_query_parameters(secday_out=secday) + call icepack_warnings_flush(nu_diag) + if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & + file=__FILE__, line=__LINE__) + + !------------------------------------------------------------------- + ! 3-hourly data + ! + ! Assume that the 3-hourly value is located at the end of the + ! 3-hour period. This is the convention for NCEP reanalysis data. + ! E.g. record 1 gives conditions at 3 am GMT on 1 January. + !------------------------------------------------------------------- + + dataloc = 2 ! data located at end of interval + sec3hr = secday/c8 ! seconds in 3 hours + !maxrec = 2920 ! 365*8; for leap years = 366*8 + + if(use_leap_years) days_per_year = 366 !overrides setting of 365 in ice_calendar + maxrec = days_per_year*8 + + if(days_per_year == 365 .and. (mod(yr, 4) == 0)) then + call abort_ice('days_per_year should be set to 366 for leap years') + end if + + ! current record number + recnum = 8*int(yday) - 7 + int(real(sec,kind=dbl_kind)/sec3hr) + + ! Compute record numbers for surrounding data (2 on each side) + + ixm = mod(recnum+maxrec-2,maxrec) + 1 + ixx = mod(recnum-1, maxrec) + 1 + + ! Compute interpolation coefficients + ! If data is located at the end of the time interval, then the + ! data value for the current record goes in slot 2 + + recslot = 2 + ixp = -99 + call interp_coeff (recnum, recslot, sec3hr, dataloc) + + ! Read + read6 = .false. + if (istep==1 .or. oldrecnum .ne. recnum) read6 = .true. + !------------------------------------------------------------------- + ! File is NETCDF with winds in NORTH and EAST direction + ! file variable names are: + ! glbrad (shortwave W/m^2) + ! dlwsfc (longwave W/m^2) + ! wndewd (eastward wind m/s) + ! wndnwd (northward wind m/s) + ! airtmp (air temperature K) + ! spchmd (specific humidity kg/kg) + ! ttlpcp (precipitation kg/m s-1) + !------------------------------------------------------------------- + call ice_open_nc(uwind_file,ncid) + + fieldname = 'airtmp' + call ice_read_nc(ncid,recnum,fieldname,Tair_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,Tair_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'wndewd' + call ice_read_nc(ncid,recnum,fieldname,uatm_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,uatm_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'wndnwd' + call ice_read_nc(ncid,recnum,fieldname,vatm_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,vatm_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'spchmd' + call ice_read_nc(ncid,recnum,fieldname,Qa_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,Qa_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'glbrad' + call ice_read_nc(ncid,recnum,fieldname,fsw_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,fsw_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'dlwsfc' + call ice_read_nc(ncid,recnum,fieldname,flw_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,flw_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'ttlpcp' + call ice_read_nc(ncid,recnum,fieldname,fsnow_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,fsnow_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + call ice_close_nc(ncid) + + + ! Interpolate + call interpolate_data (Tair_data, Tair) + call interpolate_data (uatm_data, uatm) + call interpolate_data (vatm_data, vatm) + call interpolate_data (Qa_data, Qa) + call interpolate_data (fsw_data, fsw) + call interpolate_data (flw_data, flw) + call interpolate_data (fsnow_data, fsnow) + + !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block) + do iblk = 1, nblocks + ! limit summer Tair values where ice is present + do j = 1, ny_block + do i = 1, nx_block + if (aice(i,j,iblk) > p1) Tair(i,j,iblk) = min(Tair(i,j,iblk), Tffresh+p1) + enddo + enddo + + do j = 1, ny_block + do i = 1, nx_block + Qa (i,j,iblk) = Qa (i,j,iblk) * hm(i,j,iblk) + Tair(i,j,iblk) = Tair(i,j,iblk) * hm(i,j,iblk) + uatm(i,j,iblk) = uatm(i,j,iblk) * hm(i,j,iblk) + vatm(i,j,iblk) = vatm(i,j,iblk) * hm(i,j,iblk) + fsw (i,j,iblk) = fsw (i,j,iblk) * hm(i,j,iblk) + flw (i,j,iblk) = flw (i,j,iblk) * hm(i,j,iblk) + fsnow(i,j,iblk) = fsnow (i,j,iblk) * hm(i,j,iblk) + enddo + enddo + + enddo ! iblk + !$OMP END PARALLEL DO + + ! Save record number + oldrecnum = recnum + + if (dbug) then + if (my_task == master_task) write (nu_diag,*) 'JRA55_bulk_data' + vmin = global_minval(fsw,distrb_info,tmask) + + vmax = global_maxval(fsw,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'fsw',vmin,vmax + vmin = global_minval(flw,distrb_info,tmask) + vmax = global_maxval(flw,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'flw',vmin,vmax + vmin =global_minval(fsnow,distrb_info,tmask) + vmax =global_maxval(fsnow,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'fsnow',vmin,vmax + vmin = global_minval(Tair,distrb_info,tmask) + vmax = global_maxval(Tair,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'Tair',vmin,vmax + vmin = global_minval(uatm,distrb_info,umask) + vmax = global_maxval(uatm,distrb_info,umask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'uatm',vmin,vmax + vmin = global_minval(vatm,distrb_info,umask) + vmax = global_maxval(vatm,distrb_info,umask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'vatm',vmin,vmax + vmin = global_minval(Qa,distrb_info,tmask) + vmax = global_maxval(Qa,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'Qa',vmin,vmax + if (my_task.eq.master_task) & + write (nu_diag,*) 'maxrec',maxrec + write (nu_diag,*) 'days_per_year', days_per_year + + endif ! dbug + + end subroutine JRA55_data + + !======================================================================= ! ! AOMIP shortwave forcing @@ -2260,6 +2528,7 @@ subroutine compute_shortwave(nx_block, ny_block, & sw0 , & secday , & pi , & + lontmp , & deg2rad integer (kind=int_kind) :: & @@ -2275,8 +2544,18 @@ subroutine compute_shortwave(nx_block, ny_block, & do j=jlo,jhi do i=ilo,ihi deg2rad = pi/c180 +! solar_time = mod(real(sec,kind=dbl_kind),secday)/c3600 & +! + c12*sin(p5*TLON(i,j)) + +! Convert longitude to range of -180 to 180 for LST calculation + + lontmp = mod(TLON(i,j)/deg2rad,c360) + if (lontmp .gt. c180) lontmp = lontmp - c360 + if (lontmp .lt. -c180) lontmp = lontmp + c360 + solar_time = mod(real(sec,kind=dbl_kind),secday)/c3600 & - + c12*sin(p5*TLON(i,j)) + + lontmp/c15 + if (solar_time .ge. 24._dbl_kind) solar_time = solar_time - 24._dbl_kind hour_angle = (c12 - solar_time)*pi/c12 declin = 23.44_dbl_kind*cos((172._dbl_kind-yday) & * c2*pi/c365)*deg2rad ! use dayyr instead of c365??? @@ -3325,6 +3604,7 @@ subroutine ocn_data_ncar_init 'T', 'S', 'hblt', 'U', 'V', & 'dhdx', 'dhdy', 'qdp' / +#ifdef ncdf integer (kind=int_kind) :: & fid , & ! file id dimid ! dimension id @@ -3333,6 +3613,7 @@ subroutine ocn_data_ncar_init status , & ! status flag nlat , & ! number of longitudes of data nlon ! number of latitudes of data +#endif real (kind=dbl_kind), dimension (nx_block,ny_block,max_blocks) :: & work1 @@ -3466,6 +3747,7 @@ subroutine ocn_data_ncar_init_3D use netcdf #endif +#ifdef ncdf integer (kind=int_kind) :: & n , & ! field index m , & ! month index @@ -3488,6 +3770,7 @@ subroutine ocn_data_ncar_init_3D real (kind=dbl_kind), dimension (nx_block,ny_block,max_blocks) :: & work1, work2 +#endif character(len=*), parameter :: subname = '(ocn_data_ncar_init_3D)' @@ -3804,8 +4087,8 @@ subroutine ocn_data_hadgem(dt) ! Reads in HadGEM ocean forcing data as required from netCDF files ! Current options (selected by ocn_data_type) -! hadgem_sst: read in sst only -! hadgem_sst_uvocn: read in sst plus uocn and vocn +! hadgem_sst: read in sst only +! hadgem_sst_uvocn: read in sst plus uocn and vocn ! authors: Ann Keen, Met Office @@ -3831,10 +4114,10 @@ subroutine ocn_data_hadgem(dt) logical (kind=log_kind) :: readm character (char_len) :: & - fieldname ! field name in netcdf file + fieldname ! field name in netcdf file character (char_len_long) :: & - filename ! name of netCDF file + filename ! name of netCDF file character(len=*), parameter :: subname = '(ocn_data_hadgem)' @@ -3887,7 +4170,7 @@ subroutine ocn_data_hadgem(dt) ! ----------------------------------------------------------- ! SST ! ----------------------------------------------------------- - sst_file = trim(ocn_data_dir)//'/MONTHLY/sst.1997.nc' + sst_file = trim(ocn_data_dir)//'/MONTHLY/sst.1997.nc' fieldname='sst' call read_data_nc (readm, 0, fyear, ixm, month, ixp, & maxrec, sst_file, fieldname, sst_data, & @@ -3921,23 +4204,23 @@ subroutine ocn_data_hadgem(dt) if (trim(ocn_data_type)=='hadgem_sst_uvocn') then - filename = trim(ocn_data_dir)//'/MONTHLY/uocn.1997.nc' - fieldname='uocn' - call read_data_nc (readm, 0, fyear, ixm, month, ixp, & + filename = trim(ocn_data_dir)//'/MONTHLY/uocn.1997.nc' + fieldname='uocn' + call read_data_nc (readm, 0, fyear, ixm, month, ixp, & maxrec, filename, fieldname, uocn_data, & field_loc_center, field_type_vector) - ! Interpolate to current time step - call interpolate_data (uocn_data, uocn) + ! Interpolate to current time step + call interpolate_data (uocn_data, uocn) - filename = trim(ocn_data_dir)//'/MONTHLY/vocn.1997.nc' - fieldname='vocn' - call read_data_nc (readm, 0, fyear, ixm, month, ixp, & + filename = trim(ocn_data_dir)//'/MONTHLY/vocn.1997.nc' + fieldname='vocn' + call read_data_nc (readm, 0, fyear, ixm, month, ixp, & maxrec, filename, fieldname, vocn_data, & field_loc_center, field_type_vector) - ! Interpolate to current time step - call interpolate_data (vocn_data, vocn) + ! Interpolate to current time step + call interpolate_data (vocn_data, vocn) !----------------------------------------------------------------- ! Rotate zonal/meridional vectors to local coordinates, @@ -3959,9 +4242,9 @@ subroutine ocn_data_hadgem(dt) uocn(i,j,iblk) = uocn(i,j,iblk) * cm_to_m vocn(i,j,iblk) = vocn(i,j,iblk) * cm_to_m - enddo ! i - enddo ! j - enddo ! nblocks + enddo ! i + enddo ! j + enddo ! nblocks !$OMP END PARALLEL DO !----------------------------------------------------------------- @@ -3983,7 +4266,6 @@ subroutine ocn_data_hycom_init ! + rename/link file use ice_blocks, only: nx_block, ny_block use ice_domain, only: nblocks - use ice_domain_size, only: max_blocks use ice_flux, only: sss, sst, Tf #ifdef ncdf use netcdf @@ -4442,13 +4724,11 @@ subroutine ISPOL_data !local parameters +#ifdef ncdf character (char_len_long) :: & met_file, & ! netcdf filename fieldname ! field name in netcdf file - integer (kind=int_kind) :: & - status ! status flag - real (kind=dbl_kind), dimension(2), save :: & Tair_data_p , & ! air temperature (K) for interpolation Qa_data_p, fsnow_data_p, & @@ -4468,30 +4748,33 @@ subroutine ISPOL_data ! for interpolation of hourly data integer (kind=int_kind) :: & - ixm,ixx,ixp , & ! record numbers for neighboring months - recnum , & ! record number - recnum4X , & ! record number - maxrec , & ! maximum record number - recslot , & ! spline slot for current record - dataloc ! = 1 for data located in middle of time interval + ixm,ixx,ixp , & ! record numbers for neighboring months + maxrec , & ! maximum record number + recslot , & ! spline slot for current record + dataloc ! = 1 for data located in middle of time interval ! = 2 for date located at end of time interval - real (kind=dbl_kind) :: & + real (kind=dbl_kind) :: & secday , & Qa_pnt real (kind=dbl_kind) :: & - sec1hr ! number of seconds in 1 hour + sec1hr ! number of seconds in 1 hour logical (kind=log_kind) :: read1 +#endif + + integer (kind=int_kind) :: & + recnum , & ! record number + recnum4X ! record number character(len=*), parameter :: subname = '(ISPOL_data)' +#ifdef ncdf call icepack_query_parameters(secday_out=secday) call icepack_warnings_flush(nu_diag) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & file=__FILE__, line=__LINE__) -#ifdef ncdf if (trim(atm_data_format) == 'nc') then ! read nc file !------------------------------------------------------------------- @@ -4636,9 +4919,9 @@ subroutine ISPOL_data ! fixed data ! May not be needed !----------------------------- - rhoa (:,:,:) = 1.3_dbl_kind ! air density (kg/m^3) - cldf(:,:,:) = c1 !0.25_dbl_kind ! cloud fraction - frain(:,:,:) = c0 ! this is available in hourlymet_rh file + rhoa (:,:,:) = 1.3_dbl_kind ! air density (kg/m^3) + cldf(:,:,:) = c1 !0.25_dbl_kind ! cloud fraction + frain(:,:,:) = c0 ! this is available in hourlymet_rh file ! Save record number for next time step oldrecnum = recnum @@ -4675,6 +4958,7 @@ subroutine ocn_data_ispol_init use netcdf #endif +#ifdef ncdf integer (kind=int_kind) :: & n , & ! field index m ! month index @@ -4686,13 +4970,14 @@ subroutine ocn_data_ispol_init 'dhdx', 'dhdy', 'qdp' / real (kind=dbl_kind) :: & - work + work integer (kind=int_kind) :: & fid ! file id integer (kind=int_kind) :: & status ! status flag +#endif character(len=*), parameter :: subname = '(ocn_data_ispol_init)' @@ -4755,10 +5040,8 @@ subroutine box2001_data ! authors: Elizabeth Hunke, LANL use ice_domain, only: nblocks - use ice_constants, only: c0, c1, c2, c3, c4, c5, p2 use ice_blocks, only: nx_block, ny_block, nghost use ice_flux, only: uocn, vocn, uatm, vatm, wind, rhoa, strax, stray - use ice_fileunits, only: nu_diag, nu_forcing use ice_grid, only: uvm ! local parameters @@ -4767,7 +5050,7 @@ subroutine box2001_data iblk, i,j ! loop indices real (kind=dbl_kind) :: & - secday, pi , c10, c12, c20, puny, period, pi2, tau + secday, pi , puny, period, pi2, tau call icepack_query_parameters(pi_out=pi, pi2_out=pi2, puny_out=puny) call icepack_query_parameters(secday_out=secday) diff --git a/cicecore/cicedynB/general/ice_forcing_bgc.F90 b/cicecore/cicedynB/general/ice_forcing_bgc.F90 index 0b34accb1..6e543a056 100644 --- a/cicecore/cicedynB/general/ice_forcing_bgc.F90 +++ b/cicecore/cicedynB/general/ice_forcing_bgc.F90 @@ -77,7 +77,7 @@ subroutine get_forcing_bgc use ice_domain, only: nblocks, blocks_ice use ice_arrays_column, only: ocean_bio_all use ice_calendar, only: yday - use ice_flux, only: sss +! use ice_flux, only: sss use ice_flux_bgc, only: sil, nit use ice_forcing, only: trestore, trest, fyear, & read_clim_data_nc, interpolate_data, & @@ -202,7 +202,7 @@ subroutine get_forcing_bgc if (istep == 1 .or. .NOT. restore_bgc) then !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -210,8 +210,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi sil(i,j,iblk) = sildat(i,j,iblk) ks = 2*icepack_max_algae + icepack_max_doc + 3 + icepack_max_dic @@ -224,7 +224,7 @@ subroutine get_forcing_bgc elseif (restore_bgc) then !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -232,8 +232,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi sil(i,j,iblk) = sil(i,j,iblk) & + (sildat(i,j,iblk)-sil(i,j,iblk))*dt/trest @@ -246,7 +246,7 @@ subroutine get_forcing_bgc endif !restore elseif (tr_bgc_Sil) then ! bgc_data_type /= 'clim' !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -254,10 +254,10 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi - sil(i,j,iblk) = 25.0_dbl_kind + sil(i,j,iblk) = 25.0_dbl_kind ks = 2*icepack_max_algae + icepack_max_doc + 3 + icepack_max_dic ocean_bio_all(i,j,ks,iblk) = sil(i,j,iblk) !Sil enddo @@ -283,7 +283,7 @@ subroutine get_forcing_bgc if (istep == 1 .or. .NOT. restore_bgc) then !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -291,10 +291,10 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi - nit(i,j,iblk) = nitdat(i,j,iblk) + nit(i,j,iblk) = nitdat(i,j,iblk) ks = icepack_max_algae + 1 ocean_bio_all(i,j,ks,iblk) = nit(i,j,iblk) !nit ks = 2*icepack_max_algae + icepack_max_doc + 7 + icepack_max_dic @@ -305,7 +305,7 @@ subroutine get_forcing_bgc !$OMP END PARALLEL DO elseif (restore_bgc ) then !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -313,8 +313,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi nit(i,j,iblk) = nit(i,j,iblk) & + (nitdat(i,j,iblk)-nit(i,j,iblk))*dt/trest @@ -330,7 +330,7 @@ subroutine get_forcing_bgc ! elseif (trim(nit_data_type) == 'sss' .AND. tr_bgc_Nit) then ! !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) -! do iblk = 1, nblocks +! do iblk = 1, nblocks ! this_block = get_block(blocks_ice(iblk),iblk) ! ilo = this_block%ilo @@ -353,7 +353,7 @@ subroutine get_forcing_bgc elseif (tr_bgc_Nit) then ! bgc_data_type /= 'clim' !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -361,8 +361,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi nit(i,j,iblk) = 12.0_dbl_kind ks = icepack_max_algae + 1 @@ -444,7 +444,7 @@ subroutine get_forcing_bgc endif !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -452,8 +452,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi ks = 2*icepack_max_algae + icepack_max_doc + 3 + icepack_max_dic ocean_bio_all(i,j,ks,iblk) = sil(i,j,iblk) !Sil @@ -791,7 +791,7 @@ subroutine init_bgc_data (fed1,fep1) !------------------------------------------------------------------- if (trim(fe_data_type) == 'clim') then - diag = .true. ! write diagnostic information + diag = .true. ! write diagnostic information iron_file = trim(bgc_data_dir)//'dFe_50m_annual_Tagliabue_gx1.nc' if (my_task == master_task) then @@ -808,7 +808,7 @@ subroutine init_bgc_data (fed1,fep1) if (my_task == master_task) call ice_close_nc(fid) - diag = .true. ! write diagnostic information + diag = .true. ! write diagnostic information iron_file = trim(bgc_data_dir)//'pFe_bathy_gx1.nc' if (my_task == master_task) then @@ -858,6 +858,9 @@ subroutine faero_optics ! local parameters + logical (kind=log_kind) :: modal_aero + +#ifdef ncdf integer (kind=int_kind) :: & varid , & ! variable id status , & ! status output from netcdf routines @@ -869,11 +872,10 @@ subroutine faero_optics integer (kind=int_kind) :: & fid ! file id for netCDF file - logical (kind=log_kind) :: modal_aero - character (char_len_long) :: & optics_file, & ! netcdf filename fieldname ! field name in netcdf file +#endif character(len=*), parameter :: subname = '(faero_optics)' diff --git a/cicecore/cicedynB/general/ice_init.F90 b/cicecore/cicedynB/general/ice_init.F90 index e71d96933..1d3aedf6e 100644 --- a/cicecore/cicedynB/general/ice_init.F90 +++ b/cicecore/cicedynB/general/ice_init.F90 @@ -19,14 +19,14 @@ module ice_init use ice_fileunits, only: nu_nml, nu_diag, nml_filename, diag_type, & ice_stdout, get_fileunit, release_fileunit, bfbflag, flush_fileunit, & ice_IOUnitsMinUnit, ice_IOUnitsMaxUnit +#ifdef CESMCOUPLED use ice_fileunits, only: inst_suffix +#endif use icepack_intfc, only: icepack_warnings_flush, icepack_warnings_aborted use icepack_intfc, only: icepack_aggregate use icepack_intfc, only: icepack_init_trcr use icepack_intfc, only: icepack_init_parameters use icepack_intfc, only: icepack_init_tracer_flags - use icepack_intfc, only: icepack_init_tracer_numbers - use icepack_intfc, only: icepack_init_tracer_indices use icepack_intfc, only: icepack_query_tracer_flags use icepack_intfc, only: icepack_query_tracer_numbers use icepack_intfc, only: icepack_query_tracer_indices @@ -92,7 +92,7 @@ subroutine input_data grid_type, grid_format, & dxrect, dyrect use ice_dyn_shared, only: ndte, kdyn, revised_evp, yield_curve, & - evp_kernel_ver, & + kevp_kernel, & basalstress, k1, Ktens, e_ratio, coriolis, & kridge, ktransport, brlx, arlx use ice_transport_driver, only: advection @@ -107,8 +107,6 @@ subroutine input_data nml_error, & ! namelist i/o error flag n ! loop index - character (len=6) :: chartmp - logical :: exists real (kind=dbl_kind) :: ustar_min, albicev, albicei, albsnowv, albsnowi, & @@ -127,8 +125,6 @@ subroutine input_data logical (kind=log_kind) :: tr_iage, tr_FY, tr_lvl, tr_pond, tr_aero logical (kind=log_kind) :: tr_pond_cesm, tr_pond_lvl, tr_pond_topo - integer (kind=int_kind) :: nt_Tsfc, nt_sice, nt_qice, nt_qsno, nt_iage, nt_FY - integer (kind=int_kind) :: nt_alvl, nt_vlvl, nt_apnd, nt_hpnd, nt_ipnd, nt_aero integer (kind=int_kind) :: numin, numax ! unit number limits integer (kind=int_kind) :: rpcesm, rplvl, rptopo @@ -180,7 +176,7 @@ subroutine input_data namelist /dynamics_nml/ & kdyn, ndte, revised_evp, yield_curve, & - evp_kernel_ver, & + kevp_kernel, & brlx, arlx, & advection, coriolis, kridge, ktransport, & kstrength, krdg_partic, krdg_redist, mu_rdg, & @@ -283,7 +279,7 @@ subroutine input_data kdyn = 1 ! type of dynamics (-1, 0 = off, 1 = evp, 2 = eap) ndtd = 1 ! dynamic time steps per thermodynamic time step ndte = 120 ! subcycles per dynamics timestep: ndte=dt_dyn/dte - evp_kernel_ver = 0 ! EVP kernel (0 = 2D, >0: 1D. Only ver. 2 is implemented yet) + kevp_kernel = 0 ! EVP kernel (0 = 2D, >0: 1D. Only ver. 2 is implemented yet) brlx = 300.0_dbl_kind ! revised_evp values. Otherwise overwritten in ice_dyn_shared arlx = 300.0_dbl_kind ! revised_evp values. Otherwise overwritten in ice_dyn_shared revised_evp = .false. ! if true, use revised procedure for evp dynamics @@ -546,7 +542,7 @@ subroutine input_data call broadcast_scalar(kdyn, master_task) call broadcast_scalar(ndtd, master_task) call broadcast_scalar(ndte, master_task) - call broadcast_scalar(evp_kernel_ver, master_task) + call broadcast_scalar(kevp_kernel, master_task) call broadcast_scalar(brlx, master_task) call broadcast_scalar(arlx, master_task) call broadcast_scalar(revised_evp, master_task) @@ -996,10 +992,8 @@ subroutine input_data endif write(nu_diag,1020) ' ndtd = ', ndtd write(nu_diag,1020) ' ndte = ', ndte - write(nu_diag,1010) ' revised_evp = ', & - revised_evp - write(nu_diag,1020) ' evp_kernel_ver = ', & - evp_kernel_ver + write(nu_diag,1010) ' revised_evp = ', revised_evp + write(nu_diag,1020) ' kevp_kernel = ', kevp_kernel write(nu_diag,1005) ' brlx = ', brlx write(nu_diag,1005) ' arlx = ', arlx if (kdyn == 1) & @@ -1190,8 +1184,23 @@ subroutine input_data abort_flag = 20 endif + ! check for valid kevp_kernel + ! tcraig, kevp_kernel=2 is not validated, do not allow use + ! use "102" to test "2" for now + if (kevp_kernel /= 0) then + if (kevp_kernel == 102) then + kevp_kernel = 2 + else + if (my_task == master_task) write(nu_diag,*) subname//' ERROR: kevp_kernel = ',kevp_kernel + if (kevp_kernel == 2) then + if (my_task == master_task) write(nu_diag,*) subname//' kevp_kernel=2 not validated, use kevp_kernel=102 for testing until it is validated' + endif + abort_flag = 21 + endif + endif + if (abort_flag /= 0) then - call flush_fileunit(nu_diag) + call flush_fileunit(nu_diag) endif call ice_barrier() if (abort_flag /= 0) then @@ -1873,7 +1882,7 @@ subroutine boxslotcyl_data_aice(aicen, i, j, & ! Geometric configuration of the slotted cylinder diam = p3 *dxrect*(nx_global-1) center_x = p5 *dxrect*(nx_global-1) - center_y = p75*dxrect*(ny_global-1) + center_y = p75*dyrect*(ny_global-1) radius = p5*diam width = p166*diam length = c5*p166*diam @@ -1886,11 +1895,11 @@ subroutine boxslotcyl_data_aice(aicen, i, j, & ! check if grid point is inside slotted cylinder in_slot = (dxrect*real(iglob(i)-1, kind=dbl_kind) >= slot_x(1)) .and. & (dxrect*real(iglob(i)-1, kind=dbl_kind) <= slot_x(2)) .and. & - (dxrect*real(jglob(j)-1, kind=dbl_kind) >= slot_y(1)) .and. & - (dxrect*real(jglob(j)-1, kind=dbl_kind) <= slot_y(2)) + (dyrect*real(jglob(j)-1, kind=dbl_kind) >= slot_y(1)) .and. & + (dyrect*real(jglob(j)-1, kind=dbl_kind) <= slot_y(2)) in_cyl = sqrt((dxrect*real(iglob(i)-1, kind=dbl_kind) - center_x)**c2 + & - (dxrect*real(jglob(j)-1, kind=dbl_kind) - center_y)**c2) <= radius + (dyrect*real(jglob(j)-1, kind=dbl_kind) - center_y)**c2) <= radius in_slotted_cyl = in_cyl .and. .not. in_slot @@ -1914,9 +1923,10 @@ subroutine boxslotcyl_data_vel(i, j, & iglob, jglob, & uvel, vvel) - use ice_constants, only: c1, c4, c2, c12, p5, cm_to_m + use ice_constants, only: c2, c12, p5, cm_to_m use ice_domain_size, only: nx_global, ny_global use ice_grid, only: dxrect + use icepack_parameters, only: secday, pi integer (kind=int_kind), intent(in) :: & i, j, & ! local indices @@ -1933,15 +1943,11 @@ subroutine boxslotcyl_data_vel(i, j, & max_vel , & ! max velocity domain_length , & ! physical domain length period ! rotational period - - real (kind=dbl_kind), parameter :: & - pi = c4*atan(c1), & ! pi - days_to_s = 86400._dbl_kind character(len=*), parameter :: subname = '(boxslotcyl_data_vel)' domain_length = dxrect*cm_to_m*nx_global - period = c12*days_to_s ! 12 days rotational period + period = c12*secday ! 12 days rotational period max_vel = pi*domain_length/period uvel(i,j) = c2*max_vel*(real(jglob(j), kind=dbl_kind) - p5) & diff --git a/cicecore/cicedynB/general/ice_step_mod.F90 b/cicecore/cicedynB/general/ice_step_mod.F90 index fb759cb27..4bbf0aa3b 100644 --- a/cicecore/cicedynB/general/ice_step_mod.F90 +++ b/cicecore/cicedynB/general/ice_step_mod.F90 @@ -947,8 +947,6 @@ subroutine ocean_mixed_layer (dt, iblk) use ice_arrays_column, only: Cdn_atm, Cdn_atm_ratio use ice_blocks, only: nx_block, ny_block - use ice_blocks, only: block, get_block - use ice_domain, only: blocks_ice use ice_flux, only: sst, Tf, Qa, uatm, vatm, wind, potT, rhoa, zlvl, & frzmlt, fhocn, fswthru, flw, flwout_ocn, fsens_ocn, flat_ocn, evap_ocn, & alvdr_ocn, alidr_ocn, alvdf_ocn, alidf_ocn, swidf, swvdf, swidr, swvdr, & @@ -970,7 +968,6 @@ subroutine ocean_mixed_layer (dt, iblk) frzmlt_max = c1000 ! max magnitude of frzmlt (W/m^2) integer (kind=int_kind) :: & - ilo,ihi,jlo,jhi, & ! beginning and end of physical domain i, j , & ! horizontal indices ij ! combined ij index @@ -986,9 +983,6 @@ subroutine ocean_mixed_layer (dt, iblk) integer (kind=int_kind), dimension(nx_block*ny_block) :: & indxi, indxj ! compressed indices for ocean cells - type (block) :: & - this_block ! block information for current block - character(len=*), parameter :: subname = '(ocn_mixed_layer)' !----------------------------------------------------------------- diff --git a/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 b/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 index ab1c5969a..8f2992eb4 100644 --- a/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 +++ b/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 @@ -60,7 +60,7 @@ MODULE ice_reprosum !- include statements -------------------------------------------------- !----------------------------------------------------------------------- #ifndef SERIAL_REMOVE_MPI -#include +include 'mpif.h' #endif !----------------------------------------------------------------------- diff --git a/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 b/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 index 490b16b14..d52e58add 100644 --- a/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 +++ b/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 @@ -61,7 +61,7 @@ MODULE ice_reprosum !- include statements -------------------------------------------------- !----------------------------------------------------------------------- #ifndef SERIAL_REMOVE_MPI -#include +include 'mpif.h' #endif !----------------------------------------------------------------------- diff --git a/cicecore/cicedynB/infrastructure/ice_domain.F90 b/cicecore/cicedynB/infrastructure/ice_domain.F90 index 236f6f179..ae20e5855 100644 --- a/cicecore/cicedynB/infrastructure/ice_domain.F90 +++ b/cicecore/cicedynB/infrastructure/ice_domain.F90 @@ -300,9 +300,11 @@ subroutine init_domain_distribution(KMTG,ULATG) i,j,n ,&! dummy loop indices ig,jg ,&! global indices work_unit ,&! size of quantized work unit +#ifdef ncdf fid ,&! file id varid ,&! var id status ,&! netcdf return code +#endif tblocks_tmp ,&! total number of blocks nblocks_tmp ,&! temporary value of nblocks nblocks_max ! max blocks on proc diff --git a/cicecore/cicedynB/infrastructure/ice_grid.F90 b/cicecore/cicedynB/infrastructure/ice_grid.F90 index 35c3003a4..7aa967a6c 100644 --- a/cicecore/cicedynB/infrastructure/ice_grid.F90 +++ b/cicecore/cicedynB/infrastructure/ice_grid.F90 @@ -15,18 +15,21 @@ module ice_grid use ice_kinds_mod + use ice_broadcast, only: broadcast_scalar, broadcast_array use ice_boundary, only: ice_HaloUpdate, ice_HaloExtrapolate use ice_communicate, only: my_task, master_task use ice_blocks, only: block, get_block, nx_block, ny_block, nghost use ice_domain_size, only: nx_global, ny_global, max_blocks use ice_domain, only: blocks_ice, nblocks, halo_info, distrb_info, & ew_boundary_type, ns_boundary_type, init_domain_distribution - use ice_fileunits, only: nu_diag, nu_grid, nu_kmt + use ice_fileunits, only: nu_diag, nu_grid, nu_kmt, & + get_fileunit, release_fileunit use ice_gather_scatter, only: gather_global, scatter_global use ice_read_write, only: ice_read, ice_read_nc, ice_read_global, & ice_read_global_nc, ice_open, ice_open_nc, ice_close_nc use ice_timers, only: timer_bound, ice_timer_start, ice_timer_stop use ice_exit, only: abort_ice + use ice_global_reductions, only: global_minval, global_maxval use icepack_intfc, only: icepack_warnings_flush, icepack_warnings_aborted use icepack_intfc, only: icepack_query_parameters @@ -540,7 +543,11 @@ subroutine init_grid2 ! bathymetry !----------------------------------------------------------------- +#ifdef RASM_MODS + call get_bathymetry_popfile +#else call get_bathymetry +#endif !---------------------------------------------------------------- ! Corner coordinates for CF compliant history files @@ -1662,7 +1669,6 @@ subroutine Tlatlon use ice_constants, only: c0, c1, c2, c4, & field_loc_center, field_type_scalar - use ice_global_reductions, only: global_minval, global_maxval integer (kind=int_kind) :: & i, j, iblk , & ! horizontal indices @@ -2357,6 +2363,98 @@ subroutine get_bathymetry end subroutine get_bathymetry +!======================================================================= +! with use_bathymetry = false, vertical depth profile generated for max KMT +! with use_bathymetry = true, expects to read in pop vert_grid file + + subroutine get_bathymetry_popfile + + integer (kind=int_kind) :: & + i, j, k, iblk ! loop indices + + integer (kind=int_kind) :: & + ntmp, nlevel , & ! number of levels (max KMT) + k1 , & ! levels + ierr , & ! error tag + fid ! fid unit number + + real (kind=dbl_kind), dimension(:),allocatable :: & + depth , & ! total depth, m + thick ! layer thickness, cm -> m + + character(len=*), parameter :: subname = '(get_bathymetry_popfile)' + + ntmp = maxval(KMT) + nlevel = global_maxval(ntmp,distrb_info) + + if (my_task==master_task) then + write(nu_diag,*) subname,' KMT max = ',nlevel + endif + + allocate(depth(nlevel),thick(nlevel)) + thick = -999999. + depth = -999999. + + if (use_bathymetry) then + + write (nu_diag,*) subname,' Bathymetry file = ', trim(bathymetry_file) + if (my_task == master_task) then + call get_fileunit(fid) + open(fid,file=bathymetry_file,form='formatted',iostat=ierr) + if (ierr/=0) call abort_ice(subname//' open error') + do k = 1,nlevel + read(fid,*,iostat=ierr) thick(k) + if (ierr/=0) call abort_ice(subname//' read error') + enddo + call release_fileunit(fid) + endif + + call broadcast_array(thick,master_task) + + else + + ! create thickness profile + k1 = min(5,nlevel) + do k = 1,k1 + thick(k) = max(10000._dbl_kind/float(nlevel),500.) + enddo + do k = k1+1,nlevel + thick(k) = min(thick(k-1)*1.2_dbl_kind,20000._dbl_kind) + enddo + + endif + + ! convert thick from cm to m + thick = thick / 100._dbl_kind + + ! convert to total depth + depth(1) = thick(1) + do k = 2, nlevel + depth(k) = depth(k-1) + thick(k) + if (depth(k) < 0.) call abort_ice(subname//' negative depth error') + enddo + + if (my_task==master_task) then + do k = 1,nlevel + write(nu_diag,'(2a,i6,2f13.7)') subname,' k, thick(m), depth(m) = ',k,thick(k),depth(k) + enddo + endif + + bathymetry = 0._dbl_kind + do iblk = 1, nblocks + do j = 1, ny_block + do i = 1, nx_block + k = kmt(i,j,iblk) + if (k > nlevel) call abort_ice(subname//' kmt/nlevel error') + if (k > 0) bathymetry(i,j,iblk) = depth(k) + enddo + enddo + enddo + + deallocate(depth,thick) + + end subroutine get_bathymetry_popfile + !======================================================================= ! Read bathymetry data for basal stress calculation (grounding scheme for diff --git a/cicecore/cicedynB/infrastructure/ice_read_write.F90 b/cicecore/cicedynB/infrastructure/ice_read_write.F90 index ff05f4fe3..22d3f8260 100644 --- a/cicecore/cicedynB/infrastructure/ice_read_write.F90 +++ b/cicecore/cicedynB/infrastructure/ice_read_write.F90 @@ -1516,12 +1516,10 @@ subroutine ice_read_nc_z(fid, nrec, varname, work, diag, & ! local variables +#ifdef ncdf real (kind=dbl_kind), dimension(:), allocatable :: & - work_z - - character(len=*), parameter :: subname = '(ice_read_nc_z)' + work_z -#ifdef ncdf ! netCDF file diagnostics: integer (kind=int_kind) :: & varid, & ! netcdf id for field @@ -1532,6 +1530,11 @@ subroutine ice_read_nc_z(fid, nrec, varname, work, diag, & character (char_len) :: & dimname ! dimension name +#endif + + character(len=*), parameter :: subname = '(ice_read_nc_z)' + +#ifdef ncdf allocate(work_z(nilyr)) @@ -1625,8 +1628,8 @@ subroutine ice_write_nc_xy(fid, nrec, varid, work, diag, & amin, amax, asum ! min, max values and sum of input array character (char_len) :: & - lvarname, & ! variable name - dimname ! dimension name + lvarname ! variable name +! dimname ! dimension name real (kind=dbl_kind), dimension(:,:), allocatable :: & work_g1 @@ -1745,8 +1748,8 @@ subroutine ice_write_nc_xyz(fid, nrec, varid, work, diag, & amin, amax, asum ! min, max values and sum of input array character (char_len) :: & - lvarname, & ! variable name - dimname ! dimension name + lvarname ! variable name +! dimname ! dimension name real (kind=dbl_kind), dimension(:,:,:), allocatable :: & work_g1 @@ -2136,15 +2139,11 @@ subroutine ice_read_vec_nc (fid, nrec, varname, work_g, diag) ! netCDF file diagnostics: integer (kind=int_kind) :: & varid, & ! netcdf id for field - status, & ! status output from netcdf routines - nvar ! sizes of netcdf vector + status ! status output from netcdf routines real (kind=dbl_kind) :: & amin, amax ! min, max values of input vector - character (char_len) :: & - dimname ! dimension name -! work_g(:) = c0 if (my_task == master_task) then @@ -2194,10 +2193,15 @@ subroutine ice_get_ncvarsize(fid,varname,recsize) varname ! field name in netcdf file integer (kind=int_kind), intent(out) :: & recsize ! Number of records in file + + ! local variables + +#ifdef ncdf integer (kind=int_kind) :: & ndims, i, status character (char_len) :: & cvar +#endif character(len=*), parameter :: subname = '(ice_get_ncvarsize)' #ifdef ncdf diff --git a/cicecore/cicedynB/infrastructure/ice_restoring.F90 b/cicecore/cicedynB/infrastructure/ice_restoring.F90 index 799b263d6..a7851ef07 100644 --- a/cicecore/cicedynB/infrastructure/ice_restoring.F90 +++ b/cicecore/cicedynB/infrastructure/ice_restoring.F90 @@ -57,7 +57,7 @@ subroutine ice_HaloRestore_init use ice_domain, only: ew_boundary_type, ns_boundary_type, & nblocks, blocks_ice use ice_grid, only: tmask, hm - use ice_flux, only: sst, Tf, Tair, salinz, Tmltz + use ice_flux, only: Tf, Tair, salinz, Tmltz use ice_restart_shared, only: restart_ext integer (int_kind) :: & diff --git a/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90 b/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90 index 73540f5a9..d1564fab4 100644 --- a/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90 @@ -56,7 +56,7 @@ subroutine ice_write_hist(ns) ! local variables - integer (kind=int_kind) :: i,k,n,nn,nrec,nbits + integer (kind=int_kind) :: k,n,nn,nrec,nbits character (char_len) :: title character (char_len_long) :: ncfile(max_nstrm), hdrfile diff --git a/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90 b/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90 index a9aaef525..4d13540e5 100644 --- a/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90 @@ -636,12 +636,10 @@ subroutine read_restart_field(nu,nrec,work,atype,vname,ndim3, & ! local variables integer (kind=int_kind) :: & - n, & ! number of dimensions for variable - varid, & ! variable id - status ! status variable from netCDF routine + n ! number of dimensions for variable real (kind=dbl_kind), dimension(nx_block,ny_block,max_blocks) :: & - work2 ! input array (real, 8-byte) + work2 ! input array (real, 8-byte) character(len=*), parameter :: subname = '(read_restart_field)' @@ -699,12 +697,10 @@ subroutine write_restart_field(nu,nrec,work,atype,vname,ndim3,diag) ! local variables integer (kind=int_kind) :: & - n, & ! dimension counter - varid, & ! variable id - status ! status variable from netCDF routine + n ! dimension counter real (kind=dbl_kind), dimension(nx_block,ny_block,max_blocks) :: & - work2 ! input array (real, 8-byte) + work2 ! input array (real, 8-byte) character(len=*), parameter :: subname = '(write_restart_field)' @@ -735,8 +731,7 @@ subroutine final_restart() tr_pond_topo, tr_pond_lvl, tr_brine integer (kind=int_kind) :: & - nbtrcr, & ! number of bgc tracers - status + nbtrcr ! number of bgc tracers character(len=*), parameter :: subname = '(final_restart)' diff --git a/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 b/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 index 83f9bee1a..29efa67f9 100644 --- a/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 @@ -134,9 +134,9 @@ subroutine init_restart_write(filename_spec) tr_bgc_hum integer (kind=int_kind) :: & - k, n, & ! index + k, n, & ! index nx, ny, & ! global array size - iyear, imonth, iday, & ! year, month, day + iyear, & ! year nbtrcr ! number of bgc tracers character(len=char_len_long) :: filename @@ -643,11 +643,6 @@ subroutine read_restart_field(nu,nrec,work,atype,vname,ndim3, & ! local variables - integer (kind=int_kind) :: & - n, & ! number of dimensions for variable - varid, & ! variable id - status ! status variable from netCDF routine - real (kind=dbl_kind), dimension(nx_block,ny_block,max_blocks) :: & work2 ! input array (real, 8-byte) diff --git a/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 b/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 index ff3d2d6fd..43e5dab74 100644 --- a/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 @@ -57,7 +57,7 @@ subroutine ice_write_hist (ns) use ice_restart_shared, only: runid, lcdf64 use netcdf #endif - use ice_pio + use ice_pio use pio integer (kind=int_kind), intent(in) :: ns @@ -161,7 +161,7 @@ subroutine ice_write_hist (ns) File%fh=-1 call ice_pio_init(mode='write', filename=trim(filename), File=File, & - clobber=.true., cdf64=lcdf64) + clobber=.true., cdf64=lcdf64) call ice_pio_initdecomp(iodesc=iodesc2d) call ice_pio_initdecomp(ndim3=ncat_hist, iodesc=iodesc3dc) @@ -349,7 +349,7 @@ subroutine ice_write_hist (ns) dimidex(4)=kmtidb dimidex(5)=kmtida - do i = 1, nvarz + do i = 1, nvarz if (igrdz(i)) then status = pio_def_var(File, trim(var_nz(i)%short_name), pio_real, & (/dimidex(i)/), varid) @@ -782,7 +782,7 @@ subroutine ice_write_hist (ns) ! Extra dimensions (NCAT, VGRD*) - do i = 1, nvarz + do i = 1, nvarz if (igrdz(i)) then status = pio_inq_varid(File, var_nz(i)%short_name, varid) SELECT CASE (var_nz(i)%short_name) diff --git a/cicecore/drivers/cesm/CICE_copyright.txt b/cicecore/drivers/cesm/CICE_copyright.txt index 7564fa80f..959a3ce32 100644 --- a/cicecore/drivers/cesm/CICE_copyright.txt +++ b/cicecore/drivers/cesm/CICE_copyright.txt @@ -1,7 +1,7 @@ -! Copyright (c) 2018, Triad National Security, LLC +! Copyright (c) 2019, Triad National Security, LLC ! All rights reserved. ! -! Copyright 2018. Triad National Security, LLC. This software was +! Copyright 2019. Triad National Security, LLC. This software was ! produced under U.S. Government contract DE-AC52-06NA25396 for Los ! Alamos National Laboratory (LANL), which is operated by Triad ! National Security, LLC for the U.S. Department of Energy. The U.S. diff --git a/cicecore/drivers/cesm/ice_comp_esmf.F90 b/cicecore/drivers/cesm/ice_comp_esmf.F90 index 8f682ebe5..8ae80abdc 100644 --- a/cicecore/drivers/cesm/ice_comp_esmf.F90 +++ b/cicecore/drivers/cesm/ice_comp_esmf.F90 @@ -16,7 +16,7 @@ module ice_comp_esmf use shr_sys_mod, only : shr_sys_abort, shr_sys_flush ! use shr_mem_mod, only : shr_get_memusage, shr_init_memusage use shr_file_mod, only : shr_file_getlogunit, shr_file_getloglevel, & - shr_file_setloglevel, shr_file_setlogunit + shr_file_setloglevel, shr_file_setlogunit use mct_mod #ifdef USE_ESMF_LIB use esmf @@ -26,10 +26,10 @@ module ice_comp_esmf use seq_flds_mod use seq_infodata_mod,only : seq_infodata_start_type_cont, & - seq_infodata_start_type_brnch, seq_infodata_start_type_start + seq_infodata_start_type_brnch, seq_infodata_start_type_start use seq_timemgr_mod, only : seq_timemgr_eclockgetdata, & seq_timemgr_restartalarmison, & - seq_timemgr_eclockdateinsync, & + seq_timemgr_eclockdateinsync, & seq_timemgr_stopalarmison use seq_comm_mct, only : seq_comm_suffix, seq_comm_inst, seq_comm_name use perf_mod, only : t_startf, t_stopf, t_barrierf @@ -44,12 +44,12 @@ module ice_comp_esmf use ice_domain, only : nblocks, blocks_ice, halo_info, distrb_info use ice_blocks, only : block, get_block, nx_block, ny_block use ice_grid, only : tlon, tlat, tarea, tmask, anglet, hm, & - grid_type, t2ugrid_vector, gridcpl_file, ocn_gridcell_frac + grid_type, t2ugrid_vector, gridcpl_file, ocn_gridcell_frac use ice_constants, only : c0, c1, spval_dbl, rad_to_deg, radius, secday use ice_communicate, only : my_task, master_task, MPI_COMM_ICE use ice_calendar, only : istep, istep1, force_restart_now, write_ic,& idate, idate0, mday, time, month, daycal, & - sec, dt, dt_dyn, calendar, & + sec, dt, dt_dyn, calendar, & calendar_type, nextsw_cday, days_per_year, & nyr, new_year, time2sec, year_init use icepack_orbital, only : eccen, obliqr, lambm0, mvelpp @@ -694,7 +694,7 @@ subroutine ice_run_esmf(comp, import_state, export_state, EClock, rc) force_restart_now = seq_timemgr_RestartAlarmIsOn(EClock) - if (calendar_type .eq. "GREGORIAN") then + if (calendar_type .eq. "GREGORIAN") then nyrp = nyr nyr = (curr_ymd/10000)+1 ! integer year of basedate if (nyr /= nyrp) then diff --git a/cicecore/drivers/cesm/ice_comp_mct.F90 b/cicecore/drivers/cesm/ice_comp_mct.F90 index da86f91be..7162d6397 100644 --- a/cicecore/drivers/cesm/ice_comp_mct.F90 +++ b/cicecore/drivers/cesm/ice_comp_mct.F90 @@ -14,7 +14,7 @@ module ice_comp_mct use shr_sys_mod, only : shr_sys_abort, shr_sys_flush ! use shr_mem_mod, only : shr_get_memusage, shr_init_memusage use shr_file_mod, only : shr_file_getlogunit, shr_file_getloglevel, & - shr_file_setloglevel, shr_file_setlogunit + shr_file_setloglevel, shr_file_setlogunit use shr_const_mod use mct_mod #ifdef USE_ESMF_LIB @@ -26,11 +26,11 @@ module ice_comp_mct use seq_flds_mod use seq_cdata_mod, only : seq_cdata, seq_cdata_setptrs use seq_infodata_mod,only : seq_infodata_type, seq_infodata_getdata, & - seq_infodata_putdata, seq_infodata_start_type_cont, & - seq_infodata_start_type_brnch, seq_infodata_start_type_start + seq_infodata_putdata, seq_infodata_start_type_cont, & + seq_infodata_start_type_brnch, seq_infodata_start_type_start use seq_timemgr_mod, only : seq_timemgr_eclockgetdata, & seq_timemgr_restartalarmison, & - seq_timemgr_eclockdateinsync, & + seq_timemgr_eclockdateinsync, & seq_timemgr_stopalarmison use seq_comm_mct, only : seq_comm_suffix, seq_comm_inst, seq_comm_name use perf_mod, only : t_startf, t_stopf, t_barrierf @@ -42,13 +42,13 @@ module ice_comp_mct use ice_domain, only : nblocks, blocks_ice, halo_info, distrb_info use ice_blocks, only : block, get_block, nx_block, ny_block use ice_grid, only : tlon, tlat, tarea, tmask, anglet, hm, & - grid_type, t2ugrid_vector, gridcpl_file, ocn_gridcell_frac + grid_type, t2ugrid_vector, gridcpl_file, ocn_gridcell_frac use ice_constants, only : c0, c1, spval_dbl, radius use ice_constants, only : ice_init_constants use ice_communicate, only : my_task, master_task, MPI_COMM_ICE use ice_calendar, only : istep, istep1, force_restart_now, write_ic,& idate, idate0, mday, time, month, daycal, & - sec, dt, dt_dyn, calendar, & + sec, dt, dt_dyn, calendar, & calendar_type, nextsw_cday, days_per_year, & nyr, new_year, time2sec, year_init use ice_timers @@ -579,7 +579,7 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i ) force_restart_now = seq_timemgr_RestartAlarmIsOn(EClock) -! if (calendar_type .eq. "GREGORIAN") then +! if (calendar_type .eq. "GREGORIAN") then ! nyrp = nyr ! nyr = (curr_ymd/10000)+1 ! integer year of basedate ! if (nyr /= nyrp) then diff --git a/cicecore/drivers/cesm/ice_prescribed_mod.F90 b/cicecore/drivers/cesm/ice_prescribed_mod.F90 index 10c71cc84..eeb217577 100644 --- a/cicecore/drivers/cesm/ice_prescribed_mod.F90 +++ b/cicecore/drivers/cesm/ice_prescribed_mod.F90 @@ -138,7 +138,7 @@ subroutine ice_prescribed_init(compid, gsmap, dom) namelist /ice_prescribed_nml/ & prescribed_ice, & model_year_align, & - stream_year_first , & + stream_year_first , & stream_year_last , & stream_fldVarName , & stream_fldFileName, & diff --git a/cicecore/drivers/cice/CICE.F90 b/cicecore/drivers/cice/CICE.F90 index 88ee77438..ec850b128 100644 --- a/cicecore/drivers/cice/CICE.F90 +++ b/cicecore/drivers/cice/CICE.F90 @@ -1,8 +1,8 @@ !======================================================================= -! Copyright (c) 2018, Triad National Security, LLC +! Copyright (c) 2019, Triad National Security, LLC ! All rights reserved. ! -! Copyright 2018. Triad National Security, LLC. This software was +! Copyright 2019. Triad National Security, LLC. This software was ! produced under U.S. Government contract DE-AC52-06NA25396 for Los ! Alamos National Laboratory (LANL), which is operated by Triad ! National Security, LLC for the U.S. Department of Energy. The U.S. diff --git a/cicecore/drivers/cice/CICE_InitMod.F90 b/cicecore/drivers/cice/CICE_InitMod.F90 index 1acbcb873..ddc966b55 100644 --- a/cicecore/drivers/cice/CICE_InitMod.F90 +++ b/cicecore/drivers/cice/CICE_InitMod.F90 @@ -72,12 +72,12 @@ subroutine cice_init use ice_flux, only: init_coupler_flux, init_history_therm, & init_history_dyn, init_flux_atm, init_flux_ocn, alloc_flux use ice_forcing, only: init_forcing_ocn, init_forcing_atmo, & - get_forcing_atmo, get_forcing_ocn, alloc_forcing + get_forcing_atmo, get_forcing_ocn use ice_forcing_bgc, only: get_forcing_bgc, get_atm_bgc, & faero_default, faero_optics, alloc_forcing_bgc use ice_grid, only: init_grid1, init_grid2, alloc_grid use ice_history, only: init_hist, accum_hist - use ice_restart_shared, only: restart, runid, runtype + use ice_restart_shared, only: restart, runtype use ice_init, only: input_data, init_state use ice_init_column, only: init_thermo_vertical, init_shortwave, init_zbgc, input_zbgc, count_tracers use ice_kinds_mod diff --git a/cicecore/drivers/cice/CICE_RunMod.F90 b/cicecore/drivers/cice/CICE_RunMod.F90 index a9f5a9707..af3e681b9 100644 --- a/cicecore/drivers/cice/CICE_RunMod.F90 +++ b/cicecore/drivers/cice/CICE_RunMod.F90 @@ -44,7 +44,7 @@ module CICE_RunMod subroutine CICE_Run use ice_calendar, only: istep, istep1, time, dt, stop_now, calendar - use ice_forcing, only: get_forcing_atmo, get_forcing_ocn, atm_data_type + use ice_forcing, only: get_forcing_atmo, get_forcing_ocn use ice_forcing_bgc, only: get_forcing_bgc, get_atm_bgc, & faero_default use ice_flux, only: init_flux_atm, init_flux_ocn diff --git a/cicecore/drivers/hadgem3/CICE.F90 b/cicecore/drivers/hadgem3/CICE.F90 index 40eee5676..ccc9177fb 100644 --- a/cicecore/drivers/hadgem3/CICE.F90 +++ b/cicecore/drivers/hadgem3/CICE.F90 @@ -1,8 +1,8 @@ !======================================================================= -! Copyright (c) 2018, Triad National Security, LLC +! Copyright (c) 2019, Triad National Security, LLC ! All rights reserved. ! -! Copyright 2018. Triad National Security, LLC. This software was +! Copyright 2019. Triad National Security, LLC. This software was ! produced under U.S. Government contract DE-AC52-06NA25396 for Los ! Alamos National Laboratory (LANL), which is operated by Triad ! National Security, LLC for the U.S. Department of Energy. The U.S. diff --git a/cicecore/shared/ice_arrays_column.F90 b/cicecore/shared/ice_arrays_column.F90 index c430ddd13..e6917f51f 100644 --- a/cicecore/shared/ice_arrays_column.F90 +++ b/cicecore/shared/ice_arrays_column.F90 @@ -279,13 +279,12 @@ module ice_arrays_column subroutine alloc_arrays_column ! Allocate column arrays use ice_exit, only: abort_ice - integer (int_kind) :: nspint, max_nbtrcr, max_algae, max_aero, & + integer (int_kind) :: max_nbtrcr, max_algae, max_aero, & nmodal1, nmodal2, max_don integer (int_kind) :: ierr, ntrcr character(len=*),parameter :: subname='(alloc_arrays_column)' -! call icepack_query_parameters(nspint_out=nspint) call icepack_query_tracer_numbers(ntrcr_out=ntrcr) call icepack_query_tracer_sizes( max_nbtrcr_out=max_nbtrcr, & max_algae_out=max_algae, max_aero_out=max_aero, & diff --git a/cicecore/shared/ice_calendar.F90 b/cicecore/shared/ice_calendar.F90 index a767bdfd7..e7107f42a 100644 --- a/cicecore/shared/ice_calendar.F90 +++ b/cicecore/shared/ice_calendar.F90 @@ -54,10 +54,10 @@ module ice_calendar data daycal366/ 0,31, 60, 91,121,152,182,213,244,274,305,335,366/ real (kind=dbl_kind), parameter :: & - days_per_4c = 146097.0_dbl_kind, & - days_per_c = 36524.0_dbl_kind, & - days_per_4y = 1461.0_dbl_kind, & - days_per_y = 365.0_dbl_kind + days_per_4c = 146097.0_dbl_kind, & + days_per_c = 36524.0_dbl_kind, & + days_per_4y = 1461.0_dbl_kind, & + days_per_y = 365.0_dbl_kind integer (kind=int_kind), public :: & istep , & ! local step counter for time loop diff --git a/cicecore/shared/ice_restart_column.F90 b/cicecore/shared/ice_restart_column.F90 index 63d095f11..b46e6fdcd 100644 --- a/cicecore/shared/ice_restart_column.F90 +++ b/cicecore/shared/ice_restart_column.F90 @@ -786,9 +786,6 @@ subroutine write_restart_bgc() type (block) :: & this_block ! block information for current block - integer (kind=int_kind) :: & - ipoint - character(len=*),parameter :: subname='(write_restart_bgc)' call icepack_query_parameters(skl_bgc_out=skl_bgc, solve_zsal_out=solve_zsal) diff --git a/cicecore/version.txt b/cicecore/version.txt index fb024cb80..c6a84bfc7 100644 --- a/cicecore/version.txt +++ b/cicecore/version.txt @@ -1 +1 @@ -CICE 6.0.0 +CICE 6.0.2 From 3c708200f5f2c1b537ad6cf8f6741ca582350629 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 21:51:17 +0000 Subject: [PATCH 38/54] from consortium/master --- .../scripts/options/set_nml.boxrestore | 4 +- configuration/scripts/options/set_nml.jra55 | 17 ++ .../scripts/options/set_nml.jra55_2008 | 17 ++ .../scripts/options/set_nml.run90day | 5 + .../scripts/options/test_nml.restart1 | 1 + .../scripts/options/test_nml.restart2 | 2 +- configuration/scripts/tests/QC/cice.t-test.py | 157 +++++++++++++++--- configuration/scripts/tests/base_suite.ts | 2 + configuration/scripts/tests/nothread_suite.ts | 61 +++++++ 9 files changed, 235 insertions(+), 31 deletions(-) create mode 100755 configuration/scripts/options/set_nml.jra55 create mode 100755 configuration/scripts/options/set_nml.jra55_2008 create mode 100644 configuration/scripts/options/set_nml.run90day mode change 100644 => 100755 configuration/scripts/tests/base_suite.ts create mode 100644 configuration/scripts/tests/nothread_suite.ts diff --git a/configuration/scripts/options/set_nml.boxrestore b/configuration/scripts/options/set_nml.boxrestore index 294d507aa..6789e1ff8 100644 --- a/configuration/scripts/options/set_nml.boxrestore +++ b/configuration/scripts/options/set_nml.boxrestore @@ -2,11 +2,9 @@ nilyr = 1 ice_ic = 'default' restart = .false. restart_ext = .true. -use_leap_years = .true. +use_leap_years = .false. ndtd = 2 kcatbound = 1 -distribution_type = 'cartesian' -processor_shape = 'slenderX1' ew_boundary_type = 'cyclic' ns_boundary_type = 'open' histfreq = 'd','x','x','x','x' diff --git a/configuration/scripts/options/set_nml.jra55 b/configuration/scripts/options/set_nml.jra55 new file mode 100755 index 000000000..d0112a857 --- /dev/null +++ b/configuration/scripts/options/set_nml.jra55 @@ -0,0 +1,17 @@ +year_init = 2005 +ice_ic = 'ICE_MACHINE_INPUTDATA/CICE_data/ic/gx1/iced_gx1_v5.nc' +grid_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx1/grid_gx1.bin' +kmt_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx1/kmt_gx1.bin' +bathymetry_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx1/global_gx1.bathy.nc' +use_leap_years = .true. +use_restart_time = .false. +maskhalo_dyn = .true. +maskhalo_remap = .true. +maskhalo_bound = .true. +fyear_init = 2005 +atm_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1/JRA55' +atm_data_format = 'nc' +atm_data_type = 'JRA55' +precip_units = 'mks' +ocn_data_dir = 'default' +bgc_data_dir = 'default' diff --git a/configuration/scripts/options/set_nml.jra55_2008 b/configuration/scripts/options/set_nml.jra55_2008 new file mode 100755 index 000000000..042431fc0 --- /dev/null +++ b/configuration/scripts/options/set_nml.jra55_2008 @@ -0,0 +1,17 @@ +year_init = 2008 +ice_ic = 'ICE_MACHINE_INPUTDATA/CICE_data/ic/gx1/iced_gx1_v5.nc' +grid_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx1/grid_gx1.bin' +kmt_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx1/kmt_gx1.bin' +bathymetry_file = 'ICE_MACHINE_INPUTDATA/CICE_data/grid/gx1/global_gx1.bathy.nc' +use_leap_years = .true. +use_restart_time = .false. +maskhalo_dyn = .true. +maskhalo_remap = .true. +maskhalo_bound = .true. +fyear_init = 2008 +atm_data_dir = 'ICE_MACHINE_INPUTDATA/CICE_data/forcing/gx1/JRA55' +atm_data_format = 'nc' +atm_data_type = 'JRA55' +precip_units = 'mks' +ocn_data_dir = 'default' +bgc_data_dir = 'default' diff --git a/configuration/scripts/options/set_nml.run90day b/configuration/scripts/options/set_nml.run90day new file mode 100644 index 000000000..06db1a3d8 --- /dev/null +++ b/configuration/scripts/options/set_nml.run90day @@ -0,0 +1,5 @@ +npt = 2160 +dumpfreq = 'd' +dumpfreq_n = 30 +histfreq = 'd','x','x','x','x' +f_aice = 'd' diff --git a/configuration/scripts/options/test_nml.restart1 b/configuration/scripts/options/test_nml.restart1 index 4874c25a5..82f934720 100644 --- a/configuration/scripts/options/test_nml.restart1 +++ b/configuration/scripts/options/test_nml.restart1 @@ -2,3 +2,4 @@ npt = 240 dumpfreq = 'd' dumpfreq_n = 5 runtype = 'initial' +use_restart_time = .false. diff --git a/configuration/scripts/options/test_nml.restart2 b/configuration/scripts/options/test_nml.restart2 index 5981918a8..4ae10c5a6 100644 --- a/configuration/scripts/options/test_nml.restart2 +++ b/configuration/scripts/options/test_nml.restart2 @@ -3,4 +3,4 @@ dumpfreq = 'd' dumpfreq_n = 5 runtype = 'continue' restart = .true. - +use_restart_time = .true. diff --git a/configuration/scripts/tests/QC/cice.t-test.py b/configuration/scripts/tests/QC/cice.t-test.py index 0954cddf8..86938d8e8 100755 --- a/configuration/scripts/tests/QC/cice.t-test.py +++ b/configuration/scripts/tests/QC/cice.t-test.py @@ -109,12 +109,16 @@ def calc_diff(data_a, data_b): np.all(np.equal(data_d, 0.), axis=0), np.all(data_a < 0.01, axis=0))\ , np.all(data_b < 0.01, axis=0)) mask_array_a = np.zeros_like(data_d) + for x, value in np.ndenumerate(mask_d): i, j = x mask_array_a[:, i, j] = value + del mask_d + data_a = ma.masked_array(data_a, mask=mask_array_a) data_b = ma.masked_array(data_b, mask=mask_array_a) data_d = ma.masked_array(data_d, mask=mask_array_a) + del mask_array_a return data_a, data_b, data_d @@ -145,10 +149,10 @@ def stage_one(data_d, num_files, mean_d, variance_d): r1_den2 = np.zeros_like(mean_d) for i in np.arange(np.size(data_a, axis=0)-1): r1_num = r1_num + (data_d[i, :, :]-mean_nm1_d[:, :])*(data_d[i+1, :, :]-mean_2n_d[:, :]) - r1_den1 = r1_den1 + np.power(data_d[i, :, :]-mean_nm1_d[:, :], 2) + r1_den1 = r1_den1 + np.square(data_d[i, :, :]-mean_nm1_d[:, :]) for i in np.arange(1, np.size(data_a, axis=0)): - r1_den2 = r1_den2 + np.power(data_d[i, :, :] - mean_2n_d[:, :], 2) + r1_den2 = r1_den2 + np.square(data_d[i, :, :] - mean_2n_d[:, :]) r1 = r1_num / np.sqrt(r1_den1*r1_den2) @@ -174,6 +178,13 @@ def stage_one(data_d, num_files, mean_d, variance_d): for x in maenumerate(data_d): min_val = np.min(np.abs(df[x]-df_table)) idx = np.where(np.abs(df[x]-df_table) == min_val) + # Handle the cases where the data point falls exactly half way between + # 2 critical T-values (i.e., idx has more than 1 value in it) + while True: + try: + idx = idx[0] + except: + break t_crit[x] = t_crit_table[idx] # Create an array of Pass / Fail values for each grid cell @@ -183,7 +194,14 @@ def stage_one(data_d, num_files, mean_d, variance_d): # Calculate the mean of the difference mean_d = np.mean(data_d, axis=0) - variance_d = np.sum(np.power(data_d - mean_d, 2)) / (num_files - 1) + + # Loop through each timestep and calculate the square of the difference. + # This is required (instead of just np.square(data_d - mean_d) to reduce + # the memory footprint of the script. + tmp1 = np.zeros_like(data_d) + for i in np.arange(np.shape(data_d)[0]): + tmp1[i,:,:] = np.square(data_d[i,:,:] - mean_d[:,:]) + variance_d = np.sum(tmp1) / float(num_files - 1) n_eff, H1, r1, t_crit = stage_one(data_d, num_files, mean_d, variance_d) @@ -220,6 +238,13 @@ def stage_one(data_d, num_files, mean_d, variance_d): for x in maenumerate(data_d): min_val = np.min(np.abs(r1[x]-r1_table)) idx = np.where(np.abs(r1[x]-r1_table) == min_val) + # Handle the cases where the data point falls exactly half way between + # 2 critical T-values (i.e., idx has more than 1 value in it) + while True: + try: + idx = idx[0] + except: + break t_crit[x] = t_crit_table[idx] # Create an array showing locations of Pass / Fail grid cells @@ -300,8 +325,8 @@ def skill_test(path_a, fname, data_a, data_b, num_files, hemisphere): area_var_a = 0 area_var_b = 0 for t in np.arange(num_files): - area_var_a = area_var_a + np.sum(area_weight*np.power(data_a[t, :, :]-weighted_mean_a, 2)) - area_var_b = area_var_b + np.sum(area_weight*np.power(data_b[t, :, :]-weighted_mean_b, 2)) + area_var_a = area_var_a + np.sum(area_weight*np.square(data_a[t, :, :]-weighted_mean_a)) + area_var_b = area_var_b + np.sum(area_weight*np.square(data_b[t, :, :]-weighted_mean_b)) area_var_a = nonzero_weights / (num_files * nonzero_weights - 1.) * area_var_a area_var_b = nonzero_weights / (num_files * nonzero_weights - 1.) * area_var_b @@ -317,8 +342,8 @@ def skill_test(path_a, fname, data_a, data_b, num_files, hemisphere): weighted_r = combined_cov / (std_a*std_b) - s = np.power((1+weighted_r)*(std_a*std_b)/\ - (area_var_a + area_var_b), 2) + s = np.square((1+weighted_r)*(std_a*std_b)/\ + (area_var_a + area_var_b)) logger.debug('%s Hemisphere skill score = %f', hemisphere, s) @@ -333,48 +358,102 @@ def skill_test(path_a, fname, data_a, data_b, num_files, hemisphere): logger.info('Quadratic Skill Test Failed for %s Hemisphere', hemisphere) return False -def plot_data(data, lat, lon, units): - '''This function plots CICE data and creates a .png file (ice_thickness_map.png).''' +def plot_data(data, lat, lon, units, case, plot_type): + '''This function plots CICE data and creates a .png file.''' try: - logger.info('Creating map of the data (ice_thickness_map.png)') # Load the necessary plotting libraries import matplotlib.pyplot as plt from mpl_toolkits.basemap import Basemap from mpl_toolkits.axes_grid1 import make_axes_locatable except ImportError: logger.warning('Error loading necessary Python modules in plot_data function') + return # Suppress Matplotlib deprecation warnings import warnings warnings.filterwarnings("ignore", category=UserWarning) # Create the figure and axis - fig = plt.figure(figsize=(12, 8)) - ax = fig.add_axes([0.05, 0.08, 0.9, 0.9]) + fig, axes = plt.subplots(nrows=1, ncols=2,figsize=(14, 8)) + # Plot the northern hemisphere data as a scatter plot # Create the basemap, and draw boundaries - m = Basemap(projection='kav7', lon_0=180., resolution='l') - m.drawmapboundary(fill_color='white') + plt.sca(axes[0]) + m = Basemap(projection='npstere', boundinglat=35,lon_0=270, resolution='l') m.drawcoastlines() + m.fillcontinents() m.drawcountries() - # Plot the data as a scatter plot - x, y = m(lon, lat) - sc = m.scatter(x, y, c=data, cmap='jet', lw=0) + if plot_type == 'scatter': + x, y = m(lon,lat) + sc = m.scatter(x, y, c=data, cmap='jet', lw=0, s=4) + else: + # Create new arrays to add 1 additional longitude value to prevent a + # small amount of whitespace around longitude of 0/360 degrees. + lon_cyc = np.zeros((lon.shape[0],lon.shape[1]+1)) + mask = np.zeros((data.shape[0],data.shape[1]+1)) + lat_cyc = np.zeros((lat.shape[0],lat.shape[1]+1)) + + mask[:,0:-1] = data.mask[:,:] + mask[:,-1] = data.mask[:,0] + lon_cyc[:,0:-1] = lon[:,:]; lon_cyc[:,-1] = lon[:,0] + lat_cyc[:,0:-1] = lat[:,:]; lat_cyc[:,-1] = lat[:,0] + + lon1 = np.ma.masked_array(lon_cyc, mask=mask) + lat1 = np.ma.masked_array(lat_cyc, mask=mask) + + d = np.zeros((data.shape[0],data.shape[1]+1)) + d[:,0:-1] = data[:,:] + d[:,-1] = data[:,0] + d1 = np.ma.masked_array(d,mask=mask) + + x, y = m(lon1.data, lat1.data) + + if plot_type == 'contour': + sc = m.contourf(x, y, d1, cmap='jet') + else: # pcolor + sc = m.pcolor(x, y, d1, cmap='jet') + + m.drawparallels(np.arange(-90.,120.,15.),labels=[1,0,0,0]) # draw parallels + m.drawmeridians(np.arange(0.,420.,30.),labels=[1,1,1,1]) # draw meridians + + # Plot the southern hemisphere data as a scatter plot + plt.sca(axes[1]) + m = Basemap(projection='spstere', boundinglat=-45,lon_0=270, resolution='l') + m.drawcoastlines() + m.fillcontinents() + m.drawcountries() - m.drawmeridians(np.arange(0, 360, 60), labels=[0, 0, 0, 1], fontsize=10) - m.drawparallels(np.arange(-90, 90, 30), labels=[1, 0, 0, 0], fontsize=10) + if plot_type == 'scatter': + x, y = m(lon,lat) + sc = m.scatter(x, y, c=data, cmap='jet', lw=0, s=4) + else: + x, y = m(lon1.data, lat1.data) + + # Bandaid for a bug in the version of Basemap used during development + outside = (x <= m.xmin) | (x >= m.xmax) | (y <= m.ymin) | (y >= m.ymax) + tmp = np.ma.masked_where(outside,d1) + + if plot_type == 'contour': + sc = m.contourf(x, y, tmp, cmap='jet') + else: # pcolor + sc = m.pcolor(x, y, tmp, cmap='jet') - plt.title('CICE Ice Thickness') + m.drawparallels(np.arange(-90.,120.,15.),labels=[1,0,0,0]) # draw parallels + m.drawmeridians(np.arange(0.,420.,30.),labels=[1,1,1,1]) # draw meridians - # Create the colorbar and add Pass / Fail labels - divider = make_axes_locatable(ax) - cax = divider.append_axes("bottom", size="5%", pad=0.5) - cb = plt.colorbar(sc, cax=cax, orientation="horizontal", format="%.2f") + plt.suptitle('CICE Mean Ice Thickness\n{}'.format(case), y=0.95) + + # Make some room at the bottom of the figure, and create a colorbar + fig.subplots_adjust(bottom=0.2) + cbar_ax = fig.add_axes([0.11,0.1,0.8,0.05]) + cb = plt.colorbar(sc, cax=cbar_ax, orientation="horizontal", format="%.2f") cb.set_label(units, x=1.0) - plt.savefig('ice_thickness_map.png', dpi=300) + outfile = 'ice_thickness_{}.png'.format(case.replace('\n- ','_minus_')) + logger.info('Creating map of the data ({})'.format(outfile)) + plt.savefig(outfile, dpi=300, bbox_inches='tight') def plot_two_stage_failures(data, lat, lon): '''This function plots each grid cell and whether or not it Passed or Failed @@ -403,7 +482,7 @@ def plot_two_stage_failures(data, lat, lon): ax = fig.add_axes([0.05, 0.08, 0.9, 0.9]) # Create the basemap, and draw boundaries - m = Basemap(projection='kav7', lon_0=180., resolution='l') + m = Basemap(projection='moll', lon_0=0., resolution='l') m.drawmapboundary(fill_color='white') m.drawcoastlines() m.drawcountries() @@ -415,7 +494,7 @@ def plot_two_stage_failures(data, lat, lon): # Plot the data as a scatter plot x, y = m(lon, lat) - sc = m.scatter(x, y, c=int_data, cmap=cm, lw=0, vmin=0, vmax=1) + sc = m.scatter(x, y, c=int_data, cmap=cm, lw=0, vmin=0, vmax=1, s=4) m.drawmeridians(np.arange(0, 360, 60), labels=[0, 0, 0, 1], fontsize=10) m.drawparallels(np.arange(-90, 90, 30), labels=[1, 0, 0, 0], fontsize=10) @@ -457,8 +536,11 @@ def main(): help='Path to the test history (iceh_inst*) files. REQUIRED') parser.add_argument('-v', '--verbose', dest='verbose', help='Print debug output?', \ action='store_true') + parser.add_argument('-pt','--plot_type', dest='plot_type', help='Specify type of plot \ + to create', choices=['scatter','contour','pcolor']) parser.set_defaults(verbose=False) + parser.set_defaults(plot_type='pcolor') # If no arguments are provided, print the help message if len(sys.argv) == 1: @@ -503,6 +585,17 @@ def main(): # If test failed, attempt to create a plot of the failure locations if not PASSED: plot_two_stage_failures(H1_array, t_lat, t_lon) + + # Create plots of mean ice thickness + baseDir = os.path.abspath(args.base_dir).rstrip('history/').rstrip(\ + 'history').split('/')[-1] + testDir = os.path.abspath(args.test_dir).rstrip('history/').rstrip( \ + 'history').split('/')[-1] + plot_data(np.mean(data_base,axis=0), t_lat, t_lon, 'm', baseDir, args.plot_type) + plot_data(np.mean(data_test,axis=0), t_lat, t_lon, 'm', testDir, args.plot_type) + plot_data(np.mean(data_base-data_test,axis=0), t_lat, t_lon, 'm', '{}\n- {}'.\ + format(baseDir,testDir), args.plot_type) + logger.error('Quality Control Test FAILED') sys.exit(-1) @@ -534,6 +627,16 @@ def main(): PASSED_SKILL = PASSED_NH and PASSED_SH + # Plot the ice thickness data for the base and test cases + baseDir = os.path.abspath(args.base_dir).rstrip('history/').rstrip( \ + 'history').split('/')[-1] + testDir = os.path.abspath(args.test_dir).rstrip('history/').rstrip( \ + 'history').split('/')[-1] + plot_data(np.mean(data_base,axis=0), t_lat, t_lon, 'm', baseDir, args.plot_type) + plot_data(np.mean(data_test,axis=0), t_lat, t_lon, 'm', testDir, args.plot_type) + plot_data(np.mean(data_base-data_test,axis=0), t_lat, t_lon, 'm', '{}\n- {}'.\ + format(baseDir,testDir), args.plot_type) + logger.info('') if not PASSED_SKILL: logger.error('Quality Control Test FAILED') diff --git a/configuration/scripts/tests/base_suite.ts b/configuration/scripts/tests/base_suite.ts old mode 100644 new mode 100755 index 413047e7a..39f68a5f2 --- a/configuration/scripts/tests/base_suite.ts +++ b/configuration/scripts/tests/base_suite.ts @@ -37,3 +37,5 @@ smoke gx3 8x1 bgcskl,debug #smoke gx3 4x1 bgcz,thread smoke_gx3_8x2_bgcz restart gx1 4x2 bgcsklclim,medium restart gx1 8x1 bgczclim,medium +smoke gx1 24x1 jra55_2008,medium,run90day +restart gx1 24x1 jra55,short diff --git a/configuration/scripts/tests/nothread_suite.ts b/configuration/scripts/tests/nothread_suite.ts new file mode 100644 index 000000000..d9ab722c0 --- /dev/null +++ b/configuration/scripts/tests/nothread_suite.ts @@ -0,0 +1,61 @@ +# Test Grid PEs Sets BFB-compare + +restart gx3 8x1x25x29x2 dslenderX2 +logbfb gx3 8x1x25x29x2 dslenderX2,diag1,reprosum + +smoke gx3 16x1 diag1,run5day +smoke gx3 1x1 debug,diag1,run2day +smoke gx3 4x1 debug,diag1,run5day,thread +restart gx3 16x1 thread +smoke gx3 16x1 diag24,run1year,medium + +restart gx1 160x1 droundrobin,medium +restart tx1 160x1 dsectrobin,medium +restart gx3 16x1 none +restart gx3 16x1 iobinary + +restart gx3 12x1 alt01 +restart gx3 16x1 alt02 +restart gx3 8x1 alt03 +restart gx3 16x1 alt04 +restart gx3 16x1 alt05 +restart gx3 18x1 alt01,debug,short +restart gx3 20x1 alt02,debug,short +restart gx3 24x1 alt03,debug,short +smoke gx3 24x1 alt04,debug,short +smoke gx3 32x1 alt05,debug,short + +restart gbox128 8x1 short +restart gbox128 16x1 boxdyn,short +restart gbox128 24x1 boxdyn,short,debug +restart gbox128 12x1 boxadv,short +smoke gbox128 20x1 boxadv,short,debug +restart gbox128 32x1 boxrestore,short +smoke gbox128 24x1 boxrestore,short,debug +restart gbox80 1x1 box2001 +smoke gbox80 1x1 boxslotcyl + +smoke gx3 16x1 bgcz +smoke gx3 16x1 bgcz,debug +smoke gx3 24x1 bgcskl,debug +restart gx1 128x1 bgcsklclim,medium +restart gx1 256x1 bgczclim,medium + +decomp gx3 8x1x5x29x20 +restart gx3 1x1x50x58x4 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 4x1x25x116x1 dslenderX1 restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 12x1x4x29x9 dspacecurve restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 16x1x8x10x10 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 6x1x50x58x1 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 8x1x19x19x5 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 20x1x5x29x20 dsectrobin,short restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 32x1x5x10x10 drakeX2 restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 16x1x8x10x10 droundrobin,maskhalo restart_gx3_8x1x25x29x2_dslenderX2 +restart gx3 4x1x25x29x4 droundrobin restart_gx3_8x1x25x29x2_dslenderX2 + +logbfb gx3 1x1x50x58x4 droundrobin,diag1,maskhalo,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 4x1x25x116x1 dslenderX1,diag1,maskhalo,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 20x1x5x29x20 dsectrobin,diag1,short,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 16x1x8x10x10 droundrobin,diag1,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 6x1x50x58x1 droundrobin,diag1,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum +logbfb gx3 12x1x4x29x9 dspacecurve,diag1,maskhalo,reprosum logbfb_gx3_8x1x25x29x2_diag1_dslenderX2_reprosum From e30ace6c394aae25f0b04fc497806a8915d156cc Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 22:05:00 +0000 Subject: [PATCH 39/54] more from master --- .gitattributes | 1 + .travis.yml | 4 +++- .../scripts/machines/Macros.badger_intel | 12 ++++-------- .../scripts/machines/Macros.fram_intel | 17 ++++++++++++----- configuration/scripts/machines/env.badger_intel | 12 ++++++++---- .../scripts/machines/env.cheyenne_intel | 6 +++--- configuration/scripts/machines/env.conrad_intel | 4 +++- configuration/scripts/machines/env.cori_intel | 10 ++++++---- configuration/scripts/machines/env.fram_intel | 14 +++++++++----- 9 files changed, 49 insertions(+), 31 deletions(-) create mode 100644 .gitattributes mode change 100755 => 100644 configuration/scripts/machines/Macros.fram_intel diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..069709970 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.F90 diff=fortran diff --git a/.travis.yml b/.travis.yml index 12b42a345..b8098f3df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ language: cpp +dist: xenial + sudo: false addons: @@ -9,7 +11,7 @@ addons: packages: - tcsh - pkg-config - - netcdf-bin libnetcdf-dev #libnetcdff-dev (only required on Debian) + - netcdf-bin libnetcdf-dev libnetcdff-dev - gfortran - gcc - openmpi-bin libopenmpi-dev diff --git a/configuration/scripts/machines/Macros.badger_intel b/configuration/scripts/machines/Macros.badger_intel index 2873f97fb..9a92262c2 100644 --- a/configuration/scripts/machines/Macros.badger_intel +++ b/configuration/scripts/machines/Macros.badger_intel @@ -31,23 +31,19 @@ else endif LD:= $(FC) -# set in Macros file -NETCDF_PATH := /usr/projects/climate/SHARED_CLIMATE/software/conejo/netcdf/3.6.3/intel-13.0.1 -PNETCDF_PATH := /usr/projects/climate/SHARED_CLIMATE/software/conejo/parallel-netcdf/1.3.1/intel-13.0.1/openmpi-1.6.3 -#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib +NETCDF_PATH := /usr/projects/hpcsoft/toss3/common/netcdf/4.4.0_intel-18.0.5 +PNETCDF_PATH := /usr/projects/hpcsoft/toss3/badger/netcdf/4.4.0_intel-18.0.5_openmpi-2.1.2 PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs ifeq ($(ICE_IOTYPE), netcdf) - INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include -# INCLDIR := $(INCLDIR) -I/usr/projects/climate/SHARED_CLIMATE/software/conejo/netcdf/3.6.3/intel-13.0.1/include + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include -I$(PNETCDF_PATH)/include LIB_NETCDF := $(NETCDF_PATH)/lib LIB_PNETCDF := $(PNETCDF_PATH)/lib LIB_MPI := $(IMPILIBDIR) - #SLIBS := -L$(LIB_NETCDF) -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -L$(LAPACK_LIBDIR) -llapack -lblas - SLIBS := -L$(LIB_NETCDF) -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf + SLIBS := -L$(LIB_NETCDF) -lnetcdf -L$(LIB_PNETCDF) -lnetcdff else SLIBS := endif diff --git a/configuration/scripts/machines/Macros.fram_intel b/configuration/scripts/machines/Macros.fram_intel old mode 100755 new mode 100644 index 947c7545c..406f3b6c3 --- a/configuration/scripts/machines/Macros.fram_intel +++ b/configuration/scripts/machines/Macros.fram_intel @@ -15,10 +15,17 @@ FFLAGS := -O2 -fp-model precise -convert big_endian -assume byterecl -ftz -t #-xHost FFLAGS_NOOPT:= -O0 -SCC := s.cc -SFC := s.f90 -MPICC := s.cc -mpi -MPIFC := s.f90 -mpi +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 ifeq ($(ICE_COMMDIR), mpi) FC := $(MPIFC) @@ -29,7 +36,7 @@ else endif LD:= $(FC) -NETCDF_PATH := $(NETCDF) +NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs diff --git a/configuration/scripts/machines/env.badger_intel b/configuration/scripts/machines/env.badger_intel index a184eba51..3e7bb4f8c 100755 --- a/configuration/scripts/machines/env.badger_intel +++ b/configuration/scripts/machines/env.badger_intel @@ -12,10 +12,14 @@ if ("$inp" != "-nomodules") then #module purge #module load intel #module load openmpi - -setenv NETCDF_PATH /usr/projects/climate/SHARED_CLIMATE/software/conejo/netcdf/3.6.3/intel-13.0.1 -setenv PNETCDF_PATH /usr/projects/climate/SHARED_CLIMATE/software/conejo/parallel-netcdf/1.3.1/intel-13.0.1/openmpi-1.6.3 -#setenv LAPACK_LIBDIR /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib +module unload hdf5-serial +module unload hdf5-parallel +module unload netcdf-serial +module unload netcdf-h5parallel +module load hdf5-serial +module load netcdf-serial/4.4.0 +module load hdf5-parallel +module load netcdf-h5parallel/4.4.0 #setenv OMP_STACKSIZE 256M #setenv MP_LABELIO yes diff --git a/configuration/scripts/machines/env.cheyenne_intel b/configuration/scripts/machines/env.cheyenne_intel index ae9236766..e05c51db2 100755 --- a/configuration/scripts/machines/env.cheyenne_intel +++ b/configuration/scripts/machines/env.cheyenne_intel @@ -11,10 +11,10 @@ source /glade/u/apps/ch/opt/lmod/7.2.1/lmod/7.2.1/init/csh module purge module load ncarenv/1.2 -module load intel/17.0.1 +module load intel/19.0.2 module load mpt/2.19 -module load ncarcompilers/0.4.1 -module load netcdf/4.6.1 +module load ncarcompilers/0.5.0 +module load netcdf/4.6.3 endif diff --git a/configuration/scripts/machines/env.conrad_intel b/configuration/scripts/machines/env.conrad_intel index 2411ec703..f3f2c2529 100755 --- a/configuration/scripts/machines/env.conrad_intel +++ b/configuration/scripts/machines/env.conrad_intel @@ -50,6 +50,8 @@ setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE setenv ICE_MACHINE_SUBMIT "qsub " setenv ICE_MACHINE_ACCT P00000000 setenv ICE_MACHINE_QUEUE "debug" -setenv ICE_MACHINE_TPNODE 32 # tasks per node +setenv ICE_MACHINE_TPNODE 32 # tasks per node +setenv ICE_MACHINE_MAXPES 8000 # maximum total pes (tasks * threads) available +setenv ICE_MACHINE_MAXRUNLENGTH 168 # maximum batch wall time limit in hours (integer) setenv ICE_MACHINE_BLDTHRDS 4 setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.cori_intel b/configuration/scripts/machines/env.cori_intel index f865c1f50..703177ef7 100755 --- a/configuration/scripts/machines/env.cori_intel +++ b/configuration/scripts/machines/env.cori_intel @@ -13,14 +13,16 @@ module unload PrgEnv-cray module unload PrgEnv-gnu module unload PrgEnv-intel module unload PrgEnv-pgi -module load PrgEnv-intel/6.0.4 +module load PrgEnv-intel/6.0.5 module unload intel -module load intel/18.0.1.163 +module load intel/19.0.3.199 +module unload gcc +module load gcc/8.2.0 module unload cray-mpich module unload cray-mpich-abi -module load cray-mpich/7.7.3 +module load cray-mpich/7.7.6 module unload cray-hdf5 module unload cray-hdf5-parallel @@ -32,7 +34,7 @@ module load cray-netcdf/4.6.1.3 module unload cray-libsci module unload craype -module load craype/2.5.15 +module load craype/2.5.18 setenv NETCDF_PATH ${NETCDF_DIR} setenv OMP_PROC_BIND true diff --git a/configuration/scripts/machines/env.fram_intel b/configuration/scripts/machines/env.fram_intel index 8e9fc1935..8b1326913 100755 --- a/configuration/scripts/machines/env.fram_intel +++ b/configuration/scripts/machines/env.fram_intel @@ -1,14 +1,18 @@ #!/bin/csh -f -#. ssmuse-sh -d /fs/ssm/eccc/mrd/rpn/OCEAN/cncpt-3.1.2 -#source NEMO_compiler.ksh +source /fs/ssm/main/opt/ssmuse/ssmuse-1.7/ssmuse_1.7_all/bin/ssmuse-boot.csh >& /dev/null # package loader +set ssmuse=`which ssmuse-csh` +source $ssmuse -d /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156 # intel compiler +source /fs/ssm/main/opt/intelcomp/intelcomp-2016.1.156/intelcomp_2016.1.156_multi/bin/compilervars.csh intel64 # should be sourced by above domain, but bug in csh script +source $ssmuse -d /fs/ssm/main/opt/openmpi/openmpi-1.6.5/intelcomp-2016.1.156 # openmpi +source $ssmuse -d /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150 # netcdf (and openmpi) setenv ICE_MACHINE_ENVNAME fram setenv ICE_MACHINE_COMPILER intel setenv ICE_MACHINE_MAKE make -setenv ICE_MACHINE_WKDIR /home/dormrb01/zephyr4/armn/jfl/local1/Minor_modif4july2018/CICE/tests/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /users/dor/armn/jfl/local1/CICE6/CICE/configuration/data/gx3Ncar -setenv ICE_MACHINE_BASELINE /home/dormrb01/zephyr4/armn/jfl/local1/Minor_modif4july2018/CICE/tests/CICE_BASELINE +setenv ICE_MACHINE_WKDIR /home/dormrb01/zephyr4/armn/jfl/local1/CICEDIRS/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /users/dor/armn/jfl/local1/FORCING +setenv ICE_MACHINE_BASELINE /home/dormrb01/zephyr4/armn/jfl/local1/CICEDIRS/CICE_BASELINE setenv ICE_MACHINE_SUBMIT "qsub" setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_TPNODE 36 From 3da094d19b189da3c39528b4c7925bfbcd4b2164 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 22:25:24 +0000 Subject: [PATCH 40/54] retry sync --- configuration/scripts/machines/Macros.cheyenne_intel | 0 configuration/scripts/machines/Macros.hobart_intel | 0 configuration/scripts/machines/Macros.hobart_nag | 0 configuration/scripts/machines/Macros.phase2_intel | 0 configuration/scripts/machines/Macros.testmachine_intel | 0 configuration/scripts/machines/env.cesium_intel | 0 configuration/scripts/machines/env.millikan_intel | 0 7 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 configuration/scripts/machines/Macros.cheyenne_intel mode change 100644 => 100755 configuration/scripts/machines/Macros.hobart_intel mode change 100644 => 100755 configuration/scripts/machines/Macros.hobart_nag mode change 100644 => 100755 configuration/scripts/machines/Macros.phase2_intel mode change 100644 => 100755 configuration/scripts/machines/Macros.testmachine_intel mode change 100755 => 100644 configuration/scripts/machines/env.cesium_intel mode change 100755 => 100644 configuration/scripts/machines/env.millikan_intel diff --git a/configuration/scripts/machines/Macros.cheyenne_intel b/configuration/scripts/machines/Macros.cheyenne_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/Macros.hobart_intel b/configuration/scripts/machines/Macros.hobart_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/Macros.hobart_nag b/configuration/scripts/machines/Macros.hobart_nag old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/Macros.phase2_intel b/configuration/scripts/machines/Macros.phase2_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/Macros.testmachine_intel b/configuration/scripts/machines/Macros.testmachine_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/env.cesium_intel b/configuration/scripts/machines/env.cesium_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/env.millikan_intel b/configuration/scripts/machines/env.millikan_intel old mode 100755 new mode 100644 From 9b5ff3c6e72439058d4076df073959937bb2f0ac Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 22:56:30 +0000 Subject: [PATCH 41/54] manual add/subtrace --- alpha | 27 -- .../scripts/machines/Macros.brooks_intel | 78 +++++ .../scripts/machines/Macros.izumi_gnu | 52 +++ .../scripts/machines/Macros.izumi_intel | 52 +++ .../scripts/machines/Macros.izumi_nag | 53 ++++ .../scripts/machines/Macros.izumi_pgi | 52 +++ .../scripts/machines/env.brooks_intel | 26 ++ configuration/scripts/machines/env.izumi_gnu | 32 ++ .../scripts/machines/env.izumi_intel | 32 ++ configuration/scripts/machines/env.izumi_nag | 32 ++ configuration/scripts/machines/env.izumi_pgi | 32 ++ configuration/scripts/timeseries.csh | 127 ++++++++ configuration/scripts/timeseries.py | 297 ++++++++++++++++++ 13 files changed, 865 insertions(+), 27 deletions(-) delete mode 100644 alpha create mode 100644 configuration/scripts/machines/Macros.brooks_intel create mode 100644 configuration/scripts/machines/Macros.izumi_gnu create mode 100644 configuration/scripts/machines/Macros.izumi_intel create mode 100644 configuration/scripts/machines/Macros.izumi_nag create mode 100644 configuration/scripts/machines/Macros.izumi_pgi create mode 100755 configuration/scripts/machines/env.brooks_intel create mode 100755 configuration/scripts/machines/env.izumi_gnu create mode 100755 configuration/scripts/machines/env.izumi_intel create mode 100755 configuration/scripts/machines/env.izumi_nag create mode 100755 configuration/scripts/machines/env.izumi_pgi create mode 100755 configuration/scripts/timeseries.csh create mode 100755 configuration/scripts/timeseries.py diff --git a/alpha b/alpha deleted file mode 100644 index bbe440257..000000000 --- a/alpha +++ /dev/null @@ -1,27 +0,0 @@ -[![Build Status](https://travis-ci.org/CICE-Consortium/CICE.svg?branch=master)](https://travis-ci.org/CICE-Consortium/CICE) -[![Documentation Status](https://readthedocs.org/projects/cice-consortium-cice/badge/?version=master)](http://cice-consortium-cice.readthedocs.io/en/master/?badge=master) - -## Overview -This repository contains the files and code needed to run the CICE sea ice numerical model starting with version 6. CICE is maintained by the CICE Consortium. Versions prior to v6 are found in the [CICE-svn-trunk repository](https://github.com/CICE-Consortium/CICE-svn-trunk). - -CICE consists of a top level driver and dynamical core plus the Icepack column physics code, which is included in CICE as a git submodule. Because Icepack is a submodule of CICE, Icepack and CICE development are handled independently with respect to the github repositories even though development and testing may be done together. - -If you expect to make any changes to the code, we recommend that you first fork both the CICE and Icepack repositories. Basic instructions for working with CICE and Icepack are found in the Git Workflow Guidance, linked from the Resource Index (below). In order to incorporate your developments into the Consortium code it is -imperative you follow the guidance for Pull Requests and requisite testing. - -## Useful links -* **CICE wiki**: https://github.com/CICE-Consortium/CICE/wiki - - Information about the CICE model - -* **CICE Version Index**: https://github.com/CICE-Consortium/CICE/wiki/CICE-Version-Index - - Numbered CICE releases since version 6 with associated documentation and DOIs. - -* **Consortium Community Bulletin Board**: https://bb.cgd.ucar.edu/forums/cice-consortium-model-development - - First point of contact for discussing model development including bugs, diagnostics, and future directions. - -* **Resource Index**: https://github.com/CICE-Consortium/About-Us/wiki/Resource-Index - - List of resources for information about the Consortium and its repositories as well as model documentation, testing, and development. diff --git a/configuration/scripts/machines/Macros.brooks_intel b/configuration/scripts/machines/Macros.brooks_intel new file mode 100644 index 000000000..5bd725143 --- /dev/null +++ b/configuration/scripts/machines/Macros.brooks_intel @@ -0,0 +1,78 @@ +#============================================================================== +# Makefile macros for ECCC brooks +#============================================================================== +# For use with intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise +#-xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model source -convert big_endian -assume byterecl -ftz -traceback -diag-disable 5140 +#-xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check -fpe0 -ftrapuv -fp-model except -check noarg_temp_created +# -heap-arrays 1024 +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +#NETCDF_PATH := /fs/ssm/hpco/tmp/eccc/201402/04/intel-2016.1.150/ubuntu-14.04-amd64-64/ + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib + +INCLDIR := $(INCLDIR) + +LIB_NETCDF := $(NETCDF_PATH)/lib +LIB_PNETCDF := $(PNETCDF_PATH)/lib +LIB_MPI := $(IMPILIBDIR) + +#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf -L$(LIB_PNETCDF) -lpnetcdf -lgptl +#SLIBS := -L$(LIB_NETCDF) -lnetcdff -lnetcdf + +ifeq ($(ICE_IOTYPE), netcdf) + INCLDIR += $(shell nf-config --fflags) + SLIBS := $(shell nf-config --flibs) +endif + + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +### if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib + INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include + SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof + + CPPDEFS := $(CPPDEFS) -Dncdf +endif + +ifeq ($(IO_TYPE), netcdf) + CPPDEFS := $(CPPDEFS) -Dncdf +endif diff --git a/configuration/scripts/machines/Macros.izumi_gnu b/configuration/scripts/machines/Macros.izumi_gnu new file mode 100644 index 000000000..0d48f1013 --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_gnu @@ -0,0 +1,52 @@ +#============================================================================== +# Makefile macros for NCAR izumi, gnu compiler +#============================================================================== + +CPP := /usr/bin/cpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 + +FIXEDFLAGS := -ffixed-line-length-132 +FREEFLAGS := -ffree-form +FFLAGS := -fconvert=big-endian -fbacktrace -ffree-line-length-none +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +SCC := gcc +SFC := gfortran +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS +endif + diff --git a/configuration/scripts/machines/Macros.izumi_intel b/configuration/scripts/machines/Macros.izumi_intel new file mode 100644 index 000000000..502f7b218 --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_intel @@ -0,0 +1,52 @@ +#============================================================================== +# Makefile macros for NCAR izumi, intel compiler +#============================================================================== + +CPP := /usr/bin/cpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -qno-opt-dynamic-align -fp-model precise + +FIXEDFLAGS := -fixed -132 +FREEFLAGS := -free +FFLAGS := -qno-opt-dynamic-align -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -mkl + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS +endif + diff --git a/configuration/scripts/machines/Macros.izumi_nag b/configuration/scripts/machines/Macros.izumi_nag new file mode 100644 index 000000000..8e42df4aa --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_nag @@ -0,0 +1,53 @@ +#============================================================================== +# Makefile macros for NCAR izumi, NAG compiler +#============================================================================== + +CPP := /usr/bin/cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 $(ICE_CPPDEFS) +CFLAGS := -c + +FIXEDFLAGS := -fixed +FREEFLAGS := -free +FFLAGS := -Wp,-macro=no_com -convert=BIG_ENDIAN -wmismatch=mpi_bcast,mpi_isend,mpi_irecv,mpi_send,mpi_recv,mpi_allreduce +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -C=all -g -time -f2003 -ieee=stop +else + FFLAGS += -O2 -ieee=full +endif + +SCC := cc +SFC := nagfor +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -openmp + CFLAGS += -fopenmp + FFLAGS += -openmp +else + FFLAGS += -gline +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiof +endif diff --git a/configuration/scripts/machines/Macros.izumi_pgi b/configuration/scripts/machines/Macros.izumi_pgi new file mode 100644 index 000000000..8a8ca4b97 --- /dev/null +++ b/configuration/scripts/machines/Macros.izumi_pgi @@ -0,0 +1,52 @@ +#============================================================================== +# Makefile macros for NCAR izumi, pgi compiler +#============================================================================== + +CPP := pgcc -E +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS} +CFLAGS := -c -O2 -Kieee + +FIXEDFLAGS := -Mextend +FREEFLAGS := -Mfree +FFLAGS := -Kieee -Mbyteswapio -traceback +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -Mbounds -Mchkptr +else + FFLAGS += -O2 +endif + +SCC := pgcc +SFC := pgf90 +MPICC := mpicc +MPIFC := mpif90 + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +INCLDIR := -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -mp + CFLAGS += -mp + FFLAGS += -mp +endif + +## if using parallel I/O, load all 3 libraries. PIO must be first! +ifeq ($(IO_TYPE), pio) + PIO_PATH:= + INCLDIR += -I + SLIBS := $(SLIB) -L$(PIO_PATH) -lpiofS +endif + diff --git a/configuration/scripts/machines/env.brooks_intel b/configuration/scripts/machines/env.brooks_intel new file mode 100755 index 000000000..4a51b5bb1 --- /dev/null +++ b/configuration/scripts/machines/env.brooks_intel @@ -0,0 +1,26 @@ +#!/bin/csh -f + +source /opt/modules/default/init/csh +module load PrgEnv-intel # Intel compiler +module load cray-mpich # MPI (Cray MPICH) +module load cray-netcdf # NetCDF +module load cray-hdf5 # HDF5 + +setenv ICE_MACHINE_ENVNAME brooks +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR ~/data/brooks/cice/runs +setenv ICE_MACHINE_INPUTDATA /home/ords/cmdd/cmde/phb001/ +setenv ICE_MACHINE_BASELINE ~/data/brooks/cice/baselines +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_TPNODE 36 +setenv ICE_MACHINE_MAXRUNLENGTH 3 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "development" +setenv ICE_MACHINE_BLDTHRDS 4 +setenv ICE_MACHINE_QSTAT "qstat " + +if (-e ~/.cice_proj) then + set account_name = `head -1 ~/.cice_proj` + setenv CICE_ACCT ${account_name} +endif diff --git a/configuration/scripts/machines/env.izumi_gnu b/configuration/scripts/machines/env.izumi_gnu new file mode 100755 index 000000000..9f9938d68 --- /dev/null +++ b/configuration/scripts/machines/env.izumi_gnu @@ -0,0 +1,32 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/gnu/8.2.0 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "short" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_intel b/configuration/scripts/machines/env.izumi_intel new file mode 100755 index 000000000..63d175990 --- /dev/null +++ b/configuration/scripts/machines/env.izumi_intel @@ -0,0 +1,32 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/intel/19.0.1 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "short" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_nag b/configuration/scripts/machines/env.izumi_nag new file mode 100755 index 000000000..785cc410a --- /dev/null +++ b/configuration/scripts/machines/env.izumi_nag @@ -0,0 +1,32 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/nag/6.2 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER nag +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "short" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/machines/env.izumi_pgi b/configuration/scripts/machines/env.izumi_pgi new file mode 100755 index 000000000..b89eafeb8 --- /dev/null +++ b/configuration/scripts/machines/env.izumi_pgi @@ -0,0 +1,32 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source /usr/share/Modules/init/csh + +module purge +module load compiler/pgi/18.10 + +setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_ENVNAME izumi +setenv ICE_MACHINE_COMPILER pgi +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata +setenv ICE_MACHINE_BASELINE /scratch/cluster/$user/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QUEUE "short" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_TPNODE 48 +setenv ICE_MACHINE_MAXPES 384 +setenv ICE_MACHINE_MAXTHREADS 1 +setenv ICE_MACHINE_BLDTHRDS 1 diff --git a/configuration/scripts/timeseries.csh b/configuration/scripts/timeseries.csh new file mode 100755 index 000000000..cdd025efc --- /dev/null +++ b/configuration/scripts/timeseries.csh @@ -0,0 +1,127 @@ +#!/bin/csh + +# Check to see if test case directory was passed +if ( $1 == "-h" ) then + echo "To generate timeseries plots, this script can be passed a directory" + echo "containing a logs/ subdirectory, or it can be run in the directory with" + echo "the log files, without being passed a directory." + echo "Example: ./timeseries.csh ./annual_gx3_conrad_4x1.t00" + echo "Example: ./timeseries.csh" + echo "It will pull the diagnostic data from the most recently modified log file." + exit -1 +endif +set basename = `echo $1 | sed -e 's#/$##' | sed -e 's/^\.\///'` + +# Set x-axis limits + # Manuallyl set x-axis limits +#set xrange = 'set xrange ["19980101":"19981231"]' + # Let gnuplot determine x-alis limits +set xrange = '' + +# Determine if BASELINE dataset exists +if ( $1 == "" ) then +set basefile_dir = "IGNORE" +else +source $1/cice.settings +set basefile_dir = "$ICE_BASELINE/$ICE_BASECOM/$ICE_TESTNAME" +endif + +if ( -d $basefile_dir ) then + set num_basefile = `ls $basefile_dir | grep cice.runlog | wc -l` + if ( $num_basefile > 0 ) then + set baseline_exists = 1 + foreach file ($basefile_dir/cice.runlog.*) + set base_logfile = $file + end + else + set baseline_exists = 0 + endif +else + set baseline_exists = 0 +endif + +set fieldlist=("total ice area (km^2)" \ + "total ice extent(km^2)" \ + "total ice volume (m^3)" \ + "total snw volume (m^3)" \ + "rms ice speed (m/s)" ) + +# Get the filename for the latest log +if ( $1 == "" ) then +foreach file (./cice.runlog.*) + set logfile = $file +end +else +foreach file ($1/logs/cice.runlog.*) + set logfile = $file +end +endif + +# Loop through each field and create the plot +foreach field ($fieldlist:q) + set fieldname = `echo "$field" | sed -e 's/([^()]*)//g'` + set search = "'$fieldname'\|istep1" + rm -f data.txt + foreach line ("`egrep $search $logfile`") + if ("$line" =~ *"istep1"*) then + set argv = ( $line ) + set date = $4 + @ hour = ( $6 / 3600 ) + else + set data1 = `echo $line | rev | cut -d ' ' -f2 | rev` + set data2 = `echo $line | rev | cut -d ' ' -f1 | rev` + echo "$date-$hour,$data1,$data2" >> data.txt + endif + end + set format = "%Y%m%d-%H" + + set output = `echo $fieldname | sed 's/ /_/g'` + set output = "${output}_${ICE_CASENAME}.png" + + echo "Plotting data for '$fieldname' and saving to $output" + +# Call the plotting routine, which uses the data in the data.txt file +gnuplot << EOF > $output +# Plot style +set style data points + +set datafile separator "," + +# Term type and background color, canvas size +set terminal png size 1920,960 + +# x-axis +set xdata time +set timefmt "$format" +set format x "%Y/%m/%d" + +# Axis tick marks +set xtics rotate + +set title "$field (Diagnostic Output)" +set ylabel "$field" +set xlabel "Simulation Day" + +set key left top + +# Set x-axlis limits +$xrange + +if ( $baseline_exists == 1 ) \ + plot "data_baseline.txt" using (timecolumn(1)):2 with lines lw 2 lt 2 lc 2 title \ + "Arctic - Baseline", \ + "" using (timecolumn(1)):3 with lines lw 2 lt 2 lc 5 title "Antarctic - Baseline", \ + "data.txt" using (timecolumn(1)):2 with lines lw 2 lt 1 lc 1 title "Arctic", \ + "" using (timecolumn(1)):3 with lines lw 2 lt 1 lc 3 title "Antarctic"; \ +else \ + plot "data.txt" using (timecolumn(1)):2 with lines lw 2 lt 1 lc 1 title "Arctic", \ + "" using (timecolumn(1)):3 with lines lw 2 lt 1 lc 3 title "Antarctic" \ + +EOF + +# Delete the data file +rm -f data.txt +if ( $baseline_exists ) then + rm -f data_baseline.txt +endif +end diff --git a/configuration/scripts/timeseries.py b/configuration/scripts/timeseries.py new file mode 100755 index 000000000..2b50c373a --- /dev/null +++ b/configuration/scripts/timeseries.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python + +''' +This script generates timeseries plots of CICE diagnostic output. +It is generated to replicate the previous timeseries.csh script. + +Written by: Matthew Turner +Date: August, 2019 +''' + +import os +import sys +import logging +import numpy as np + +def find_logfile(log_dir): + ''' + This function searches for the most recently created log file in the provided directory. + ''' + + logger.debug('Getting a list of files in {}'.format(log_dir)) + try: + path = '{}/logs'.format(log_dir.rstrip('/')) + files = [os.path.join(path,f) for f in os.listdir('{}/logs'.format(log_dir)) \ + if f.startswith('cice.runlog')] + except: + path = log_dir + files = [os.path.join(path,f) for f in os.listdir(log_dir) if f.startswith('cice.runlog')] + + # Check if any files were found. If not, exit + if len(files) == 0: + logger.error('No cice.runlog* files found. Please make sure you are passing the \ + correct directory.') + sys.exit(1) + + # Get the most recently created file + outfile = max(files, key = os.path.getctime) + + logger.debug('List of files = {}'.format([f for f in files])) + logger.debug('Most recent file is {}'.format(outfile)) + + return outfile + +def get_data(logfile,field): + ''' + This function extracts data from a CICE log file for the specific field. + ''' + import datetime + import re + + logger.debug('Extracting data for {}'.format(field)) + + # Build the regular expression to extract the data + field_regex = field.replace('(','\(').replace('^','\^').replace(')','\)') + number_regex = '[-+]?\d+\.?\d+([eE][-+]?\d+)?' + my_regex = '{}\s+=\s+({})\s+({})'.format(field_regex,number_regex,number_regex) + + dtg = [] + arctic = [] + antarctic = [] + with open(logfile) as f: + for line in f.readlines(): + m1 = re.search('istep1:\s+(\d+)\s+idate:\s+(\d+)\s+sec:\s+(\d+)', line) + if m1: + # Extract the current date-time group from the file + date = m1.group(2) + seconds = int(m1.group(3)) + hours = seconds // 3600 + minutes = (seconds - hours*3600) // 60 + leftover = seconds - hours*3600 - minutes*60 + curr_date = '{}-{:02d}:{:02d}:{:02d}'.format(date,hours,minutes,leftover) + dtg.append(datetime.datetime.strptime(curr_date, '%Y%m%d-%H:%M:%S')) + logger.debug('Currently on timestep {}'.format(dtg[-1])) + + m = re.search(my_regex, line) + if m: + # Extract the data from the file + if 'E' in m.group(1) or 'e' in m.group(1): + expon = True + else: + expon = False + arctic.append(float(m.group(1))) + antarctic.append(float(m.group(3))) + logger.debug(' Arctic = {}, Antarctic = {}'.format(arctic[-1], antarctic[-1])) + + return dtg, arctic, antarctic, expon + +def latexit(string): + s = string[::-1].replace('(','($',1) + return (s.replace(')','$)',1))[::-1] + +def plot_timeseries(log, field, dtg, arctic, antarctic, expon, dtg_base=None, arctic_base=None, \ + antarctic_base=None, base_dir=None, grid=False): + ''' + Plot the timeseries data from the CICE log file + ''' + + casename = os.path.abspath(log).rstrip('/').rstrip('/logs').split('/')[-1] + if base_dir: + base_casename = os.path.abspath(base_dir).rstrip('/').rstrip('/logs').split('/')[-1] + + # Load the plotting libraries, but set the logging level for matplotlib + # to WARNING so that matplotlib debugging info is not printed when running + # with '-v' + logging.getLogger('matplotlib').setLevel(logging.WARNING) + import matplotlib.pyplot as plt + import matplotlib.dates as mdates + import matplotlib.ticker as ticker + + fig = plt.figure(figsize=(12,8)) + ax = fig.add_axes([0.05,0.08,0.9,0.9]) + + # Add the arctic data to the plot + ax.plot(dtg,arctic,label='Arctic') + # Add the baseline arctic data to the plot, if available + if arctic_base: + ax.plot(dtg_base,arctic_base,label='Baseline Arctic') + + # Add the antarctic data to the plot + ax.plot(dtg,antarctic,label='Antarctic') + # Add the baseline antarctic data to the plot, if available + if antarctic_base: + ax.plot(dtg_base,antarctic_base,label='Baseline Antarctic') + + ax.set_xlabel('') + ax.set_title('{} Diagnostic Output'.format(latexit(field))) + ax.set_ylabel(latexit(field)) + + # Format the x-axis labels + ax.xaxis.set_major_formatter(mdates.DateFormatter('%Y/%m/%d')) + ax.xaxis.set_minor_locator(mdates.MonthLocator()) + + # Add a text box that prints the test case name and the baseline case name (if given) + try: + text_field = "Test/Case: {}\nBaseline: {}".format(casename,base_casename) + from matplotlib.offsetbox import AnchoredText + anchored_text = AnchoredText(text_field,loc=2) + ax.add_artist(anchored_text) + except: + text_field = "Test/Case: {}".format(casename) + from matplotlib.offsetbox import AnchoredText + anchored_text = AnchoredText(text_field,loc=2) + ax.add_artist(anchored_text) + + ax.legend(loc='upper right') + + # Add grid lines if the `--grid` argument was passed at the command line. + if grid: + ax.grid(ls='--') + + # Reduce the number of ticks on the y axis + nbins = 10 + try: + minval = min( \ + min(min(arctic), min(antarctic)), \ + min(min(arctic_base), min(antarctic_base))) + maxval = max( \ + max(max(arctic), max(antarctic)), \ + max(max(arctic_base), max(antarctic_base))) + except: + minval = min(min(arctic), min(antarctic)) + maxval = max(max(arctic), max(antarctic)) + step = (maxval-minval)/nbins + ax.yaxis.set_ticks(np.arange(minval, maxval+step, step)) + + # Format the y-axis tick labels, based on whether or not the values in the log file + # are in scientific notation or float notation. + if expon: + ax.yaxis.set_major_formatter(ticker.FormatStrFormatter('%0.3e')) + else: + ax.yaxis.set_major_formatter(ticker.FormatStrFormatter('%0.5f')) + + # Rotate and right align the x labels + for tick in ax.get_xticklabels(): + tick.set_rotation(45) + + # Create an output file and save the figure + field_tmp = field.split('(')[0].rstrip() + try: + outfile = '{}_{}_base-{}.png'.format(field_tmp.replace(' ','_'), casename,base_casename) + except: + outfile = '{}_{}.png'.format(field_tmp.replace(' ','_'), casename) + logger.info('Saving file to {}'.format(outfile)) + plt.savefig(outfile,dpi=300,bbox_inches='tight') + +def main(): + import argparse + parser = argparse.ArgumentParser(description="To generate timeseries plots, this script \ + can be passed a directory containing a logs/ subdirectory, \ + or it can be run in the directory with the log files, \ + without being passed a directory. It will pull the \ + diagnostic data from the most recently modified log file.\ + \ + If no flags are passed selecting the variables to plot, \ + then plots will be created for all available variables.") + parser.add_argument('log_dir', nargs='?', default=os.getcwd(), \ + help="Path to diagnostic output log file. A specific log file can \ + be passed, or a case directory. If a directory is passed, \ + the most recent log file will be used. If no directory or \ + file is passed, the script will look for a log file in the \ + current directory.") + parser.add_argument('--bdir',dest='base_dir', help='Path to the the log file for a baseline \ + dataset, if desired. A specific log file or case directory can \ + be passed. If a directory is passed, the most recent log file \ + will be used.') + parser.add_argument('-v', '--verbose', dest='verbose', help='Print debug output?', \ + action='store_true') + parser.add_argument('--area', dest='area', help='Create a plot for total ice area?', \ + action='store_true') + parser.add_argument('--extent', dest='extent', help='Create a plot for total ice extent?', \ + action='store_true') + parser.add_argument('--volume', dest='ice_volume', help='Create a plot for total ice volume?', \ + action='store_true') + parser.add_argument('--snw_vol', dest='snow_volume', help='Create a plot for total snow \ + volume?', action='store_true') + parser.add_argument('--speed', dest='speed', help='Create a plot for rms ice speed?', \ + action='store_true') + parser.add_argument('--grid',dest='grid', help='Add grid lines to the figures?', \ + action='store_true') + + # Set the defaults for the command line options + parser.set_defaults(verbose=False) + parser.set_defaults(area=False) + parser.set_defaults(extent=False) + parser.set_defaults(ice_volume=False) + parser.set_defaults(snow_volume=False) + parser.set_defaults(speed=False) + parser.set_defaults(grid=False) + + args = parser.parse_args() + + # If no fields are passed, plot all fields + if not ( args.area or args.extent or args.ice_volume or args.snow_volume or args.speed ): + args.area = True + args.extent = True + args.ice_volume = True + args.snow_volume = True + args.speed = True + + # Build the fieldlist based on which fields are passed + fieldlist = [] + if args.area: + fieldlist.append('total ice area (km^2)') + if args.extent: + fieldlist.append('total ice extent(km^2)') + if args.ice_volume: + fieldlist.append('total ice volume (m^3)') + if args.snow_volume: + fieldlist.append('total snw volume (m^3)') + if args.speed: + fieldlist.append('rms ice speed (m/s)') + + # Setup the logger + global logger + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + + # Find the test and baseline log files, based on the input directories. + if os.path.isdir(args.log_dir): + logger.debug('{} is a directory'.format(args.log_dir)) + log = find_logfile(args.log_dir) + log_dir = args.log_dir + else: + logger.debug('{} is a file'.format(args.log_dir)) + log = args.log_dir + log_dir = args.log_dir.rsplit('/',1)[0] + logger.info('Log file = {}'.format(log)) + if args.base_dir: + if os.path.isdir(args.base_dir): + base_log = find_logfile(args.base_dir) + base_dir = args.base_dir + else: + base_log = args.base_dir + base_dir = args.base_dir.rsplit('/',1)[0] + logger.info('Base Log file = {}'.format(base_log)) + + # Loop through each field and create the plot + for field in fieldlist: + logger.debug('Current field = {}'.format(field)) + + # Get the data from the log files + dtg, arctic, antarctic, expon = get_data(log, field) + if args.base_dir: + dtg_base, arctic_base, antarctic_base, expon_base = get_data(base_log,field) + + # Plot the data + if args.base_dir: + plot_timeseries(log_dir, field, dtg, arctic, antarctic, expon, dtg_base, \ + arctic_base, antarctic_base, base_dir, grid=args.grid) + else: + plot_timeseries(log_dir, field, dtg, arctic, antarctic, expon, grid=args.grid) + +if __name__ == "__main__": + main() From 07d4a1d4f4345760367ee001f8d554a8abb82d5c Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 23:09:23 +0000 Subject: [PATCH 42/54] script perms and large diff files --- cice.setup | 141 +++++-- cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 | 188 ++++----- cicecore/cicedynB/general/ice_forcing.F90 | 373 +++++++++++++++--- cicecore/cicedynB/infrastructure/ice_grid.F90 | 102 ++++- .../scripts/machines/Macros.cheyenne_intel | 0 .../scripts/machines/Macros.hobart_intel | 0 .../scripts/machines/Macros.hobart_nag | 0 .../scripts/machines/Macros.phase2_intel | 0 .../scripts/machines/Macros.phase3_intel | 0 .../scripts/machines/Macros.testmachine_intel | 0 .../scripts/machines/env.cesium_intel | 0 .../scripts/machines/env.millikan_intel | 0 doc/source/user_guide/ug_running.rst | 276 ++++++++++++- 13 files changed, 889 insertions(+), 191 deletions(-) mode change 100644 => 100755 cicecore/cicedynB/general/ice_forcing.F90 mode change 100755 => 100644 configuration/scripts/machines/Macros.cheyenne_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.hobart_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.hobart_nag mode change 100755 => 100644 configuration/scripts/machines/Macros.phase2_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.phase3_intel mode change 100755 => 100644 configuration/scripts/machines/Macros.testmachine_intel mode change 100644 => 100755 configuration/scripts/machines/env.cesium_intel mode change 100644 => 100755 configuration/scripts/machines/env.millikan_intel diff --git a/cice.setup b/cice.setup index d8c67f4ca..13ba00f85 100755 --- a/cice.setup +++ b/cice.setup @@ -377,7 +377,6 @@ else exit -1 endif cp -f ${ICE_SCRIPTS}/tests/report_results.csh ${tsdir} - cp -f ${ICE_SCRIPTS}/tests/timeseries.csh ${tsdir} cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir} foreach file (${tsdir}/suite.run ${tsdir}/suite.submit) @@ -433,6 +432,7 @@ if (${doabort} == true) then exit -1 endif +# Create a new sets_base variable to store sets passed to cice.setup set sets_base = "${sets}" set bfbcomp_base = "$bfbcomp" foreach compiler ( $ncompilers ) @@ -458,13 +458,14 @@ EOF continue endif + source ${ICE_SCRIPTS}/machines/env.${machcomp} -nomodules || exit 2 + # Obtain the test name, sets, grid, and PE information from .ts file set test = `echo $line | cut -d' ' -f1` set grid = `echo $line | cut -d' ' -f2` set pesx = `echo $line | cut -d' ' -f3` set sets_tmp = `echo $line | cut -d' ' -f4` set bfbcomp_tmp = `echo $line | cut -d' ' -f5` - # Create a new sets_base variable to store sets passed to cice.setup # Append sets from .ts file to the $sets variable set sets = "$sets_base,$sets_tmp" @@ -482,6 +483,107 @@ EOF set fbfbcomp = ${machcomp}_${bfbcomp} endif + #------------------------------------------------------------ + # Parse pesx with strict checking, limit pes for machine + + set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'` + if (${chck} == OK) then + set task = `echo ${pesx} | cut -d x -f 1` + set thrd = `echo ${pesx} | cut -d x -f 2` + set blckx = `echo ${pesx} | cut -d x -f 3` + set blcky = `echo ${pesx} | cut -d x -f 4` + set mblck = `echo ${pesx} | cut -d x -f 5` + if (${task} == 0 || ${thrd} == 0 || ${blckx} == 0 || ${blcky} == 0 || ${mblck} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 + endif + else + set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'` + if (${chck} == OK) then + set task = `echo ${pesx} | cut -d x -f 1` + set thrd = `echo ${pesx} | cut -d x -f 2` + set blckx = `echo ${pesx} | cut -d x -f 3` + set blcky = `echo ${pesx} | cut -d x -f 4` + set mblck = 0 + if (${task} == 0 || ${thrd} == 0 || ${blckx} == 0 || ${blcky} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 + endif + else + set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*$/OK/'` + if (${chck} == OK) then + set task = `echo ${pesx} | cut -d x -f 1` + set thrd = `echo ${pesx} | cut -d x -f 2` + set blckx = 0 + set blcky = 0 + set mblck = 0 + if (${task} == 0 || ${thrd} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 + endif + else + set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*$/OK/'` + if (${chck} == OK) then + set task = `echo ${pesx} | cut -d x -f 1` + set thrd = 1 + set blckx = 0 + set blcky = 0 + set mblck = 0 + if (${task} == 0) then + echo "${0}: ERROR in -p argument, cannot have zeros" + exit -1 + endif + else + echo "${0}: ERROR in -p argument, ${pesx}, must be [m], [m]x[n], [m]x[n]x[bx]x[by], or [m]x[n]x[bx]x[by]x[mb] " + exit -1 + endif + endif + endif + endif + + @ blkreq = ${task} * ${mblck} + # check max threads, reduce threads, increase tasks + if ($?ICE_MACHINE_MAXTHREADS) then + if (${thrd} > ${ICE_MACHINE_MAXTHREADS}) then + @ pesreq = ${task} * ${thrd} + @ task = ${pesreq} / ${ICE_MACHINE_MAXTHREADS} + @ thrd = ${ICE_MACHINE_MAXTHREADS} + @ peschk = ${task} * ${thrd} + if (${pesreq} > ${peschk}) then + @ task = ${task} + 1 + endif +# echo "tcx1 reset to $task $thrd $mblck" + endif + endif + + # check max pes, reduce tasks + if ($?ICE_MACHINE_MAXPES) then + @ pesreq = ${task} * ${thrd} + if (${pesreq} > ${ICE_MACHINE_MAXPES}) then + @ task = ${ICE_MACHINE_MAXPES} / ${thrd} +# echo "tcx2 reset to $task $thrd $mblck" + endif + endif + + # check max blocks and adjust as needed + if ($mblck > 0) then + @ mblck = ${blkreq} / ${task} + @ blkchk = ${task} * ${mblck} + if (${blkreq} > ${blkchk}) then + @ mblck = $mblck + 1 + endif +# echo "tcx3 reset to $task $thrd $mblck" + endif + + # update pesx based on use defined settings and machine limits to reflect actual value + set pesx = ${task}x${thrd}x${blckx}x${blcky}x${mblck} + if (${mblck} == 0) then + set pesx = ${task}x${thrd}x${blckx}x${blcky} + endif + if (${blckx} == 0 || ${blcky} == 0) then + set pesx = ${task}x${thrd} + endif + set testname_noid = ${spval} # create case for test cases if (${docase} == 0) then @@ -533,7 +635,7 @@ EOF endif # from basic script dir to case - foreach file (cice.build cice.settings Makefile ice_in makdep.c setup_run_dirs.csh) + foreach file (cice.build cice.settings Makefile ice_in makdep.c setup_run_dirs.csh timeseries.csh timeseries.py) if !(-e ${ICE_SCRIPTS}/$file) then echo "${0}: ERROR, ${ICE_SCRIPTS}/$file not found" exit -1 @@ -560,7 +662,6 @@ EOF end cd ${casedir} - source ./env.${machcomp} -nomodules || exit 2 set quietmode = false if ($?ICE_MACHINE_QUIETMODE) then @@ -586,36 +687,6 @@ EOF #------------------------------------------------------------ # Compute a default blocksize - set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'` - if (${chck} == OK) then - set task = `echo ${pesx} | cut -d x -f 1` - set thrd = `echo ${pesx} | cut -d x -f 2` - set blckx = `echo ${pesx} | cut -d x -f 3` - set blcky = `echo ${pesx} | cut -d x -f 4` - set mblck = `echo ${pesx} | cut -d x -f 5` - else - set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*x[0-9][0-9]*$/OK/'` - if (${chck} == OK) then - set task = `echo ${pesx} | cut -d x -f 1` - set thrd = `echo ${pesx} | cut -d x -f 2` - set blckx = `echo ${pesx} | cut -d x -f 3` - set blcky = `echo ${pesx} | cut -d x -f 4` - set mblck = 0 - else - set chck = `echo ${pesx} | sed 's/^[0-9][0-9]*x[0-9][0-9]*$/OK/'` - if (${chck} == OK) then - set task = `echo ${pesx} | cut -d x -f 1` - set thrd = `echo ${pesx} | cut -d x -f 2` - set blckx = 0 - set blcky = 0 - set mblck = 0 - else - echo "${0}: ERROR in -p argument, ${pesx}, must be [m]x[n], [m]x[n]x[bx]x[by], or [m]x[n]x[bx]x[by]x[mb] " - exit -1 - endif - endif - endif - setenv ICE_DECOMP_GRID ${grid} setenv ICE_DECOMP_NTASK ${task} setenv ICE_DECOMP_NTHRD ${thrd} @@ -847,7 +918,7 @@ echo "${testname_base}" cd ${testname_base} source ./cice.settings set ciceexe = "../ciceexe.\${ICE_COMPILER}.\${ICE_COMMDIR}.\${ICE_BLDDEBUG}.\${ICE_THREADED}.\${ICE_IOTYPE}" -./cice.build \${ciceexe} +./cice.build --exe \${ciceexe} if !(-e \${ciceexe}) cp -p \${ICE_RUNDIR}/cice \${ciceexe} EOF end diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 index b1f162967..0cb5d9102 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90 @@ -1,9 +1,11 @@ ! ice_dyn_evp_1d ! -! Contains 3 Fortran modules, +! Contained 3 Fortran modules, ! * dmi_omp ! * bench_v2 ! * ice_dyn_evp_1d +! These were merged into one module, ice_dyn_evp_1d to support some +! coupled build systems. ! ! Modules used for: ! * convert 2D arrays into 1D vectors @@ -11,9 +13,9 @@ ! * convert 1D vectors into 2D matrices ! ! Call from ice_dyn_evp.F90: -! call evp_copyin(...) -! call evp_kernel() -! call evp_copyout(...) +! call ice_dyn_evp_1d_copyin(...) +! call ice_dyn_evp_1d_kernel() +! call ice_dyn_evp_1d_copyout(...) ! ! * REAL4 internal version: ! mv evp_kernel1d.F90 evp_kernel1d_r8.F90 @@ -27,7 +29,9 @@ !=============================================================================== !=============================================================================== -module dmi_omp + +!-- One dimension representation of EVP 2D arrays used for EVP kernels +module ice_dyn_evp_1d use ice_kinds_mod use ice_fileunits, only: nu_diag @@ -35,32 +39,89 @@ module dmi_omp implicit none private - public :: domp_init, domp_get_domain, domp_get_thread_no + public :: ice_dyn_evp_1d_copyin, ice_dyn_evp_1d_copyout, ice_dyn_evp_1d_kernel + + interface ice_dyn_evp_1d_copyin +! module procedure evp_copyin_v1 + module procedure evp_copyin_v2 + end interface + + interface ice_dyn_evp_1d_kernel +! module procedure evp_kernel_v1 + module procedure evp_kernel_v2 + end interface + + interface ice_dyn_evp_1d_copyout + module procedure evp_copyout + end interface + + interface convert_2d_1d +! module procedure convert_2d_1d_v1 + module procedure convert_2d_1d_v2 + end interface + + integer(kind=int_kind) :: & + NA_len, NAVEL_len + logical(kind=log_kind), dimension(:), allocatable :: & + skipucell + integer(kind=int_kind), dimension(:), allocatable :: & + ee,ne,se,nw,sw,sse,indi,indj,indij , halo_parent + real (kind=dbl_kind), dimension(:), allocatable :: & + cdn_ocn,aiu,uocn,vocn,forcex,forcey,Tbu,tarear, & + umassdti,fm,uarear,strintx,strinty,uvel_init,vvel_init + real (kind=dbl_kind), dimension(:), allocatable :: & + strength,uvel,vvel,dxt,dyt, & +!v1 dxhy,dyhx,cyp,cxp,cym,cxm,tinyarea, & +!v1 waterx,watery, & + stressp_1, stressp_2, stressp_3, stressp_4, & + stressm_1, stressm_2, stressm_3, stressm_4, & + stress12_1,stress12_2,stress12_3,stress12_4, & + divu,rdg_conv,rdg_shear,shear,taubx,tauby + real (kind=DBL_KIND), dimension(:), allocatable :: & + str1, str2, str3, str4, str5, str6, str7, str8 + real (kind=dbl_kind), dimension(:), allocatable :: & + HTE,HTN, & + HTEm1,HTNm1 + logical(kind=log_kind),parameter :: dbug = .false. + +!--- dmi_omp --------------------------- interface domp_get_domain module procedure domp_get_domain_rlu end interface INTEGER, PARAMETER :: JPIM = SELECTED_INT_KIND(9) - integer(int_kind), private :: domp_iam, domp_nt + integer(int_kind) :: domp_iam, domp_nt #if defined (_OPENMP) ! Please note, this constant will create a compiler info for a constant ! expression in IF statements: - real(kind=dbl_kind), private :: rdomp_iam, rdomp_nt + real(kind=dbl_kind) :: rdomp_iam, rdomp_nt !$OMP THREADPRIVATE(domp_iam,domp_nt,rdomp_iam,rdomp_nt) #endif +!--- dmi_omp --------------------------- + +!--- bench_v2 -------------------------- + interface evp1d_stress + module procedure stress_i + module procedure stress_l + end interface + interface evp1d_stepu + module procedure stepu_iter + module procedure stepu_last + end interface +!--- bench_v2 -------------------------- -contains + contains -!---------------------------------------------------------------------------- +!=============================================================================== +!former module dmi_omp subroutine domp_init(nt_out) #if defined (_OPENMP) use omp_lib, only : omp_get_thread_num, omp_get_num_threads #endif - use ice_forcing, only : dbug integer(int_kind), intent(out) :: nt_out @@ -70,9 +131,9 @@ subroutine domp_init(nt_out) !$OMP PARALLEL DEFAULT(none) #if defined (_OPENMP) domp_iam = omp_get_thread_num() - rdomp_iam = real(domp_iam,8) + rdomp_iam = real(domp_iam,dbl_kind) domp_nt = omp_get_num_threads() - rdomp_nt = real(domp_nt,8) + rdomp_nt = real(domp_nt,dbl_kind) #else domp_iam = 0 domp_nt = 1 @@ -111,8 +172,8 @@ subroutine domp_get_domain_rlu(lower,upper,d_lower,d_upper) #if defined (_OPENMP) use omp_lib, only : omp_in_parallel -#endif use ice_constants, only: p5 +#endif integer(KIND=JPIM), intent(in) :: lower,upper integer(KIND=JPIM), intent(out) :: d_lower,d_upper @@ -120,7 +181,6 @@ subroutine domp_get_domain_rlu(lower,upper,d_lower,d_upper) #if defined (_OPENMP) !-- local variables real(kind=dbl_kind) :: dlen - integer(int_kind) :: lr, ur #endif character(len=*), parameter :: subname = '(domp_get_domain_rlu)' @@ -139,9 +199,9 @@ subroutine domp_get_domain_rlu(lower,upper,d_lower,d_upper) #if defined (_OPENMP) if (omp_in_parallel()) then - dlen = real(upper-lower+1, 8) - d_lower = lower + floor((rdomp_iam*dlen+p5)/rdomp_nt, 4) - d_upper = lower -1 + floor((rdomp_iam*dlen+dlen+p5)/rdomp_nt, 4) + dlen = real(upper-lower+1, dbl_kind) + d_lower = lower + floor((rdomp_iam*dlen+p5)/rdomp_nt, JPIM) + d_upper = lower -1 + floor((rdomp_iam*dlen+dlen+p5)/rdomp_nt, JPIM) endif #endif @@ -166,30 +226,11 @@ end subroutine domp_get_thread_no !---------------------------------------------------------------------------- -end module dmi_omp +!former end module dmi_omp -!=============================================================================== !=============================================================================== -module bench_v2 - - use ice_fileunits, only: nu_diag - use ice_exit, only: abort_ice - - implicit none - private - public :: evp1d_stress, evp1d_stepu, evp1d_halo_update - - interface evp1d_stress - module procedure stress_i - module procedure stress_l - end interface - interface evp1d_stepu - module procedure stepu_iter - module procedure stepu_last - end interface - - contains +!former module bench_v2 !---------------------------------------------------------------------------- @@ -202,7 +243,6 @@ subroutine stress_i(NA_len, & str6,str7,str8) use ice_kinds_mod - use dmi_omp, only : domp_get_domain use ice_constants, only: p027, p055, p111, p166, p222, p25, p333, p5, c1p5, c1 use icepack_parameters, only: puny use ice_dyn_shared, only: ecci, denom1, arlx1i, Ktens, revp @@ -468,7 +508,6 @@ subroutine stress_l(NA_len, tarear, & str1,str2,str3,str4,str5,str6,str7,str8 ) use ice_kinds_mod - use dmi_omp, only : domp_get_domain use ice_constants, only: p027, p055, p111, p166, p222, p25, p333, p5, c1p5, c0, c1 use icepack_parameters, only: puny use ice_dyn_shared, only: ecci, denom1, arlx1i, Ktens, revp @@ -737,7 +776,6 @@ subroutine stepu_iter(NA_len,rhow, & str1,str2,str3,str4,str5,str6,str7,str8, nw,sw,se,skipme) use ice_kinds_mod - use dmi_omp, only : domp_get_domain use ice_dyn_shared, only: brlx, revp use ice_constants, only: c0, c1 @@ -828,7 +866,6 @@ subroutine stepu_last(NA_len, rhow, & str1,str2,str3,str4,str5,str6,str7,str8, nw,sw,se,skipme) use ice_kinds_mod - use dmi_omp, only : domp_get_domain use ice_constants, only: c0, c1 use icepack_parameters, only: puny use ice_dyn_shared, only: brlx, revp, basalstress @@ -921,7 +958,6 @@ end subroutine stepu_last subroutine evp1d_halo_update(NAVEL_len,lb,ub,uvel,vvel, halo_parent) use ice_kinds_mod - use dmi_omp, only : domp_get_domain implicit none @@ -958,59 +994,9 @@ end subroutine evp1d_halo_update !---------------------------------------------------------------------------- -end module bench_v2 +!former end module bench_v2 !=============================================================================== -!=============================================================================== - -!-- One dimension representation of EVP 2D arrays used for EVP kernels -module ice_dyn_evp_1d - - use ice_kinds_mod - use ice_fileunits, only: nu_diag - use ice_exit, only: abort_ice - !-- BEGIN: specific for the KERNEL - use ice_dyn_shared, only: revp, ecci, denom1, arlx1i, brlx - !-- END: specific for the KERNEL - - implicit none - private - public :: evp_copyin, evp_copyout, evp_kernel_v2 - - interface evp_copyin -! module procedure evp_copyin_v1 - module procedure evp_copyin_v2 - end interface - interface convert_2d_1d -! module procedure convert_2d_1d_v1 - module procedure convert_2d_1d_v2 - end interface - - integer(kind=int_kind) :: & - NA_len, NAVEL_len - logical(kind=log_kind), dimension(:), allocatable :: & - skipucell - integer(kind=int_kind), dimension(:), allocatable :: & - ee,ne,se,nw,sw,sse,indi,indj,indij , halo_parent - real (kind=dbl_kind), dimension(:), allocatable :: & - cdn_ocn,aiu,uocn,vocn,forcex,forcey,Tbu,tarear, & - umassdti,fm,uarear,strintx,strinty,uvel_init,vvel_init - real (kind=dbl_kind), dimension(:), allocatable :: & - strength,uvel,vvel,dxt,dyt, & -!v1 dxhy,dyhx,cyp,cxp,cym,cxm,tinyarea, & -!v1 waterx,watery, & - stressp_1, stressp_2, stressp_3, stressp_4, & - stressm_1, stressm_2, stressm_3, stressm_4, & - stress12_1,stress12_2,stress12_3,stress12_4, & - divu,rdg_conv,rdg_shear,shear,taubx,tauby - real (kind=DBL_KIND), dimension(:), allocatable :: & - str1, str2, str3, str4, str5, str6, str7, str8 - real (kind=dbl_kind), dimension(:), allocatable :: & - HTE,HTN, & - HTEm1,HTNm1 - - contains - !---------------------------------------------------------------------------- subroutine alloc1d(na) @@ -1150,7 +1136,6 @@ subroutine evp_copyin_v2(nx,ny,nblk,nx_glob,ny_glob, use ice_gather_scatter, only: gather_global_ext use ice_domain, only: distrb_info use ice_communicate, only: my_task, master_task - use ice_constants, only: c0,c1,p5 implicit none @@ -1286,9 +1271,8 @@ subroutine evp_copyout(nx,ny,nblk,nx_glob,ny_glob, & I_stress12_1,I_stress12_2,I_stress12_3,I_stress12_4, & I_divu,I_rdg_conv,I_rdg_shear,I_shear,I_taubx,I_tauby ) - use ice_constants, only : c0, field_loc_center, field_loc_NEcorner, & - field_type_scalar, field_type_vector - use ice_gather_scatter, only: scatter_global_ext, scatter_global + use ice_constants, only : c0 + use ice_gather_scatter, only: scatter_global_ext use ice_domain, only: distrb_info use ice_communicate, only: my_task, master_task @@ -1413,14 +1397,12 @@ subroutine evp_kernel_v2 use ice_constants, only : c0 use ice_dyn_shared, only: ndte - use bench_v2, only : evp1d_stress, evp1d_stepu, evp1d_halo_update - use dmi_omp, only : domp_init use icepack_intfc, only: icepack_query_parameters use ice_communicate, only: my_task, master_task implicit none real(kind=dbl_kind) :: rhow - integer (kind=int_kind) :: ierr, lun, i, nthreads + integer (kind=int_kind) :: i, nthreads integer (kind=int_kind) :: na,nb,navel character(len=*), parameter :: subname = '(evp_kernel_v2)' @@ -2049,7 +2031,6 @@ end subroutine findXinY_halo subroutine numainit(l,u,uu) - use dmi_omp, only : domp_get_domain use ice_constants, only: c0 implicit none @@ -2133,6 +2114,7 @@ subroutine numainit(l,u,uu) end subroutine numainit !---------------------------------------------------------------------------- +!=============================================================================== end module ice_dyn_evp_1d diff --git a/cicecore/cicedynB/general/ice_forcing.F90 b/cicecore/cicedynB/general/ice_forcing.F90 old mode 100644 new mode 100755 index 4baa88fff..53eb0ec49 --- a/cicecore/cicedynB/general/ice_forcing.F90 +++ b/cicecore/cicedynB/general/ice_forcing.F90 @@ -30,9 +30,9 @@ module ice_forcing use ice_timers, only: ice_timer_start, ice_timer_stop, timer_readwrite, & timer_bound use ice_arrays_column, only: oceanmixed_ice, restore_bgc - use ice_constants, only: c0, c1, c2, c3, c4, c5, c10, c12, c20, & - c180, c365, c1000, c3600 - use ice_constants, only: p001, p01, p1, p25, p5, p6 + use ice_constants, only: c0, c1, c2, c3, c4, c5, c8, c10, c12, c15, c20, & + c180, c360, c365, c1000, c3600 + use ice_constants, only: p001, p01, p1, p2, p25, p5, p6 use ice_constants, only: cm_to_m use ice_constants, only: field_loc_center, field_type_scalar, & field_type_vector, field_loc_NEcorner @@ -115,7 +115,8 @@ module ice_forcing atm_data_format, & ! 'bin'=binary or 'nc'=netcdf ocn_data_format, & ! 'bin'=binary or 'nc'=netcdf atm_data_type, & ! 'default', 'monthly', 'ncar', - ! 'LYq' or 'hadgem' or 'oned' + ! 'LYq' or 'hadgem' or 'oned' or + ! 'JRA55' bgc_data_type, & ! 'default', 'clim' ocn_data_type, & ! 'default', 'clim', 'ncar', 'oned', ! 'hadgem_sst' or 'hadgem_sst_uvocn' @@ -211,6 +212,8 @@ subroutine init_forcing_atmo ! Determine the current and final year of the forcing cycle based on ! namelist input; initialize the atmospheric forcing data filenames. + use ice_calendar, only: use_leap_years + character(len=*), parameter :: subname = '(init_forcing_atmo)' ! Allocate forcing arrays @@ -235,6 +238,14 @@ subroutine init_forcing_atmo file=__FILE__, line=__LINE__) endif + if (use_leap_years .and. (trim(atm_data_type) /= 'JRA55' .and. & + trim(atm_data_type) /= 'default' .and. & + trim(atm_data_type) /= 'box2001')) then + write(nu_diag,*) 'use_leap_years option is currently only supported for' + write(nu_diag,*) 'JRA55, default , and box2001 atmospheric data' + call abort_ice(error_message=subname, file=__FILE__, line=__LINE__) + endif + !------------------------------------------------------------------- ! Get filenames for input forcing data !------------------------------------------------------------------- @@ -244,6 +255,8 @@ subroutine init_forcing_atmo call NCAR_files(fyear) elseif (trim(atm_data_type) == 'LYq') then call LY_files(fyear) + elseif (trim(atm_data_type) == 'JRA55') then + call JRA55_files(fyear) elseif (trim(atm_data_type) == 'hadgem') then call hadgem_files(fyear) elseif (trim(atm_data_type) == 'monthly') then @@ -414,11 +427,11 @@ subroutine init_forcing_ocn(dt) if (trim(ocn_data_type) == 'hadgem_sst' .or. & trim(ocn_data_type) == 'hadgem_sst_uvocn') then - diag = .true. ! write diagnostic information + diag = .true. ! write diagnostic information sst_file = trim (ocn_data_dir)//'/MONTHLY/sst.1997.nc' - if (my_task == master_task) then + if (my_task == master_task) then write (nu_diag,*) ' ' write (nu_diag,*) 'Initial SST file:', trim(sst_file) @@ -539,6 +552,8 @@ subroutine get_forcing_atmo call ncar_data elseif (trim(atm_data_type) == 'LYq') then call LY_data + elseif (trim(atm_data_type) == 'JRA55') then + call JRA55_data(fyear) elseif (trim(atm_data_type) == 'hadgem') then call hadgem_data elseif (trim(atm_data_type) == 'monthly') then @@ -1376,6 +1391,10 @@ subroutine file_year (data_file, yr) i = index(data_file,'.nc') - 5 tmpname = data_file write(data_file,'(a,i4.4,a)') tmpname(1:i), yr, '.nc' + elseif (trim(atm_data_type) == 'JRA55') then ! netcdf + i = index(data_file,'.nc') - 5 + tmpname = data_file + write(data_file,'(a,i4.4,a)') tmpname(1:i), yr, '.nc' else ! LANL/NCAR naming convention i = index(data_file,'.dat') - 5 tmpname = data_file @@ -2002,6 +2021,22 @@ subroutine LY_files (yr) endif ! master_task end subroutine LY_files + subroutine JRA55_files(yr) +! + integer (kind=int_kind), intent(in) :: & + yr ! current forcing year + + character(len=*), parameter :: subname = '(JRA55_files)' + + uwind_file = & + trim(atm_data_dir)//'/8XDAILY/JRA55_03hr_forcing_2005.nc' + call file_year(uwind_file,yr) + if (my_task == master_task) then + write (nu_diag,*) ' ' + write (nu_diag,*) 'Atmospheric data files:' + write (nu_diag,*) trim(uwind_file) + endif + end subroutine JRA55_files !======================================================================= ! @@ -2225,6 +2260,239 @@ subroutine LY_data end subroutine LY_data + subroutine JRA55_data (yr) + + use ice_blocks, only: block, get_block + use ice_global_reductions, only: global_minval, global_maxval + use ice_domain, only: nblocks, distrb_info, blocks_ice + use ice_flux, only: fsnow, Tair, uatm, vatm, Qa, fsw, flw + use ice_grid, only: hm, tlon, tlat, tmask, umask + use ice_state, only: aice + use ice_calendar, only: days_per_year, use_leap_years + + + integer (kind=int_kind) :: & + ncid , & ! netcdf file id + i, j , & + ixm,ixx,ixp , & ! record numbers for neighboring months + recnum , & ! record number + maxrec , & ! maximum record number + recslot , & ! spline slot for current record + midmonth , & ! middle day of month + dataloc , & ! = 1 for data located in middle of time interval + ! = 2 for date located at end of time interval + iblk , & ! block index + ilo,ihi,jlo,jhi, & ! beginning and end of physical domain + yr ! current forcing year + + real (kind=dbl_kind) :: & + sec3hr , & ! number of seconds in 3 hours + secday , & ! number of seconds in day + Tffresh , & + vmin, vmax + + logical (kind=log_kind) :: readm, read6,debug_n_d + + type (block) :: & + this_block ! block information for current block + + character(len=64) :: fieldname !netcdf field name + character(len=*), parameter :: subname = '(JRA55_data)' + + debug_n_d = .false. !usually false + + call icepack_query_parameters(Tffresh_out=Tffresh) + call icepack_query_parameters(secday_out=secday) + call icepack_warnings_flush(nu_diag) + if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & + file=__FILE__, line=__LINE__) + + !------------------------------------------------------------------- + ! 3-hourly data + ! + ! Assume that the 3-hourly value is located at the end of the + ! 3-hour period. This is the convention for NCEP reanalysis data. + ! E.g. record 1 gives conditions at 3 am GMT on 1 January. + !------------------------------------------------------------------- + + dataloc = 2 ! data located at end of interval + sec3hr = secday/c8 ! seconds in 3 hours + !maxrec = 2920 ! 365*8; for leap years = 366*8 + + if(use_leap_years) days_per_year = 366 !overrides setting of 365 in ice_calendar + maxrec = days_per_year*8 + + if(days_per_year == 365 .and. (mod(yr, 4) == 0)) then + call abort_ice('days_per_year should be set to 366 for leap years') + end if + + ! current record number + recnum = 8*int(yday) - 7 + int(real(sec,kind=dbl_kind)/sec3hr) + + ! Compute record numbers for surrounding data (2 on each side) + + ixm = mod(recnum+maxrec-2,maxrec) + 1 + ixx = mod(recnum-1, maxrec) + 1 + + ! Compute interpolation coefficients + ! If data is located at the end of the time interval, then the + ! data value for the current record goes in slot 2 + + recslot = 2 + ixp = -99 + call interp_coeff (recnum, recslot, sec3hr, dataloc) + + ! Read + read6 = .false. + if (istep==1 .or. oldrecnum .ne. recnum) read6 = .true. + !------------------------------------------------------------------- + ! File is NETCDF with winds in NORTH and EAST direction + ! file variable names are: + ! glbrad (shortwave W/m^2) + ! dlwsfc (longwave W/m^2) + ! wndewd (eastward wind m/s) + ! wndnwd (northward wind m/s) + ! airtmp (air temperature K) + ! spchmd (specific humidity kg/kg) + ! ttlpcp (precipitation kg/m s-1) + !------------------------------------------------------------------- + call ice_open_nc(uwind_file,ncid) + + fieldname = 'airtmp' + call ice_read_nc(ncid,recnum,fieldname,Tair_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,Tair_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'wndewd' + call ice_read_nc(ncid,recnum,fieldname,uatm_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,uatm_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'wndnwd' + call ice_read_nc(ncid,recnum,fieldname,vatm_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,vatm_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'spchmd' + call ice_read_nc(ncid,recnum,fieldname,Qa_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,Qa_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'glbrad' + call ice_read_nc(ncid,recnum,fieldname,fsw_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,fsw_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'dlwsfc' + call ice_read_nc(ncid,recnum,fieldname,flw_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,flw_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + fieldname = 'ttlpcp' + call ice_read_nc(ncid,recnum,fieldname,fsnow_data(:,:,1,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + call ice_read_nc(ncid,recnum,fieldname,fsnow_data(:,:,2,:),debug_n_d, & + field_loc=field_loc_center, & + field_type=field_type_scalar) + + call ice_close_nc(ncid) + + + ! Interpolate + call interpolate_data (Tair_data, Tair) + call interpolate_data (uatm_data, uatm) + call interpolate_data (vatm_data, vatm) + call interpolate_data (Qa_data, Qa) + call interpolate_data (fsw_data, fsw) + call interpolate_data (flw_data, flw) + call interpolate_data (fsnow_data, fsnow) + + !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block) + do iblk = 1, nblocks + ! limit summer Tair values where ice is present + do j = 1, ny_block + do i = 1, nx_block + if (aice(i,j,iblk) > p1) Tair(i,j,iblk) = min(Tair(i,j,iblk), Tffresh+p1) + enddo + enddo + + do j = 1, ny_block + do i = 1, nx_block + Qa (i,j,iblk) = Qa (i,j,iblk) * hm(i,j,iblk) + Tair(i,j,iblk) = Tair(i,j,iblk) * hm(i,j,iblk) + uatm(i,j,iblk) = uatm(i,j,iblk) * hm(i,j,iblk) + vatm(i,j,iblk) = vatm(i,j,iblk) * hm(i,j,iblk) + fsw (i,j,iblk) = fsw (i,j,iblk) * hm(i,j,iblk) + flw (i,j,iblk) = flw (i,j,iblk) * hm(i,j,iblk) + fsnow(i,j,iblk) = fsnow (i,j,iblk) * hm(i,j,iblk) + enddo + enddo + + enddo ! iblk + !$OMP END PARALLEL DO + + ! Save record number + oldrecnum = recnum + + if (dbug) then + if (my_task == master_task) write (nu_diag,*) 'JRA55_bulk_data' + vmin = global_minval(fsw,distrb_info,tmask) + + vmax = global_maxval(fsw,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'fsw',vmin,vmax + vmin = global_minval(flw,distrb_info,tmask) + vmax = global_maxval(flw,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'flw',vmin,vmax + vmin =global_minval(fsnow,distrb_info,tmask) + vmax =global_maxval(fsnow,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'fsnow',vmin,vmax + vmin = global_minval(Tair,distrb_info,tmask) + vmax = global_maxval(Tair,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'Tair',vmin,vmax + vmin = global_minval(uatm,distrb_info,umask) + vmax = global_maxval(uatm,distrb_info,umask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'uatm',vmin,vmax + vmin = global_minval(vatm,distrb_info,umask) + vmax = global_maxval(vatm,distrb_info,umask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'vatm',vmin,vmax + vmin = global_minval(Qa,distrb_info,tmask) + vmax = global_maxval(Qa,distrb_info,tmask) + if (my_task.eq.master_task) & + write (nu_diag,*) 'Qa',vmin,vmax + if (my_task.eq.master_task) & + write (nu_diag,*) 'maxrec',maxrec + write (nu_diag,*) 'days_per_year', days_per_year + + endif ! dbug + + end subroutine JRA55_data + + !======================================================================= ! ! AOMIP shortwave forcing @@ -2260,6 +2528,7 @@ subroutine compute_shortwave(nx_block, ny_block, & sw0 , & secday , & pi , & + lontmp , & deg2rad integer (kind=int_kind) :: & @@ -2275,8 +2544,18 @@ subroutine compute_shortwave(nx_block, ny_block, & do j=jlo,jhi do i=ilo,ihi deg2rad = pi/c180 +! solar_time = mod(real(sec,kind=dbl_kind),secday)/c3600 & +! + c12*sin(p5*TLON(i,j)) + +! Convert longitude to range of -180 to 180 for LST calculation + + lontmp = mod(TLON(i,j)/deg2rad,c360) + if (lontmp .gt. c180) lontmp = lontmp - c360 + if (lontmp .lt. -c180) lontmp = lontmp + c360 + solar_time = mod(real(sec,kind=dbl_kind),secday)/c3600 & - + c12*sin(p5*TLON(i,j)) + + lontmp/c15 + if (solar_time .ge. 24._dbl_kind) solar_time = solar_time - 24._dbl_kind hour_angle = (c12 - solar_time)*pi/c12 declin = 23.44_dbl_kind*cos((172._dbl_kind-yday) & * c2*pi/c365)*deg2rad ! use dayyr instead of c365??? @@ -3325,6 +3604,7 @@ subroutine ocn_data_ncar_init 'T', 'S', 'hblt', 'U', 'V', & 'dhdx', 'dhdy', 'qdp' / +#ifdef ncdf integer (kind=int_kind) :: & fid , & ! file id dimid ! dimension id @@ -3333,6 +3613,7 @@ subroutine ocn_data_ncar_init status , & ! status flag nlat , & ! number of longitudes of data nlon ! number of latitudes of data +#endif real (kind=dbl_kind), dimension (nx_block,ny_block,max_blocks) :: & work1 @@ -3466,6 +3747,7 @@ subroutine ocn_data_ncar_init_3D use netcdf #endif +#ifdef ncdf integer (kind=int_kind) :: & n , & ! field index m , & ! month index @@ -3488,6 +3770,7 @@ subroutine ocn_data_ncar_init_3D real (kind=dbl_kind), dimension (nx_block,ny_block,max_blocks) :: & work1, work2 +#endif character(len=*), parameter :: subname = '(ocn_data_ncar_init_3D)' @@ -3804,8 +4087,8 @@ subroutine ocn_data_hadgem(dt) ! Reads in HadGEM ocean forcing data as required from netCDF files ! Current options (selected by ocn_data_type) -! hadgem_sst: read in sst only -! hadgem_sst_uvocn: read in sst plus uocn and vocn +! hadgem_sst: read in sst only +! hadgem_sst_uvocn: read in sst plus uocn and vocn ! authors: Ann Keen, Met Office @@ -3831,10 +4114,10 @@ subroutine ocn_data_hadgem(dt) logical (kind=log_kind) :: readm character (char_len) :: & - fieldname ! field name in netcdf file + fieldname ! field name in netcdf file character (char_len_long) :: & - filename ! name of netCDF file + filename ! name of netCDF file character(len=*), parameter :: subname = '(ocn_data_hadgem)' @@ -3887,7 +4170,7 @@ subroutine ocn_data_hadgem(dt) ! ----------------------------------------------------------- ! SST ! ----------------------------------------------------------- - sst_file = trim(ocn_data_dir)//'/MONTHLY/sst.1997.nc' + sst_file = trim(ocn_data_dir)//'/MONTHLY/sst.1997.nc' fieldname='sst' call read_data_nc (readm, 0, fyear, ixm, month, ixp, & maxrec, sst_file, fieldname, sst_data, & @@ -3921,23 +4204,23 @@ subroutine ocn_data_hadgem(dt) if (trim(ocn_data_type)=='hadgem_sst_uvocn') then - filename = trim(ocn_data_dir)//'/MONTHLY/uocn.1997.nc' - fieldname='uocn' - call read_data_nc (readm, 0, fyear, ixm, month, ixp, & + filename = trim(ocn_data_dir)//'/MONTHLY/uocn.1997.nc' + fieldname='uocn' + call read_data_nc (readm, 0, fyear, ixm, month, ixp, & maxrec, filename, fieldname, uocn_data, & field_loc_center, field_type_vector) - ! Interpolate to current time step - call interpolate_data (uocn_data, uocn) + ! Interpolate to current time step + call interpolate_data (uocn_data, uocn) - filename = trim(ocn_data_dir)//'/MONTHLY/vocn.1997.nc' - fieldname='vocn' - call read_data_nc (readm, 0, fyear, ixm, month, ixp, & + filename = trim(ocn_data_dir)//'/MONTHLY/vocn.1997.nc' + fieldname='vocn' + call read_data_nc (readm, 0, fyear, ixm, month, ixp, & maxrec, filename, fieldname, vocn_data, & field_loc_center, field_type_vector) - ! Interpolate to current time step - call interpolate_data (vocn_data, vocn) + ! Interpolate to current time step + call interpolate_data (vocn_data, vocn) !----------------------------------------------------------------- ! Rotate zonal/meridional vectors to local coordinates, @@ -3959,9 +4242,9 @@ subroutine ocn_data_hadgem(dt) uocn(i,j,iblk) = uocn(i,j,iblk) * cm_to_m vocn(i,j,iblk) = vocn(i,j,iblk) * cm_to_m - enddo ! i - enddo ! j - enddo ! nblocks + enddo ! i + enddo ! j + enddo ! nblocks !$OMP END PARALLEL DO !----------------------------------------------------------------- @@ -3983,7 +4266,6 @@ subroutine ocn_data_hycom_init ! + rename/link file use ice_blocks, only: nx_block, ny_block use ice_domain, only: nblocks - use ice_domain_size, only: max_blocks use ice_flux, only: sss, sst, Tf #ifdef ncdf use netcdf @@ -4442,13 +4724,11 @@ subroutine ISPOL_data !local parameters +#ifdef ncdf character (char_len_long) :: & met_file, & ! netcdf filename fieldname ! field name in netcdf file - integer (kind=int_kind) :: & - status ! status flag - real (kind=dbl_kind), dimension(2), save :: & Tair_data_p , & ! air temperature (K) for interpolation Qa_data_p, fsnow_data_p, & @@ -4468,30 +4748,33 @@ subroutine ISPOL_data ! for interpolation of hourly data integer (kind=int_kind) :: & - ixm,ixx,ixp , & ! record numbers for neighboring months - recnum , & ! record number - recnum4X , & ! record number - maxrec , & ! maximum record number - recslot , & ! spline slot for current record - dataloc ! = 1 for data located in middle of time interval + ixm,ixx,ixp , & ! record numbers for neighboring months + maxrec , & ! maximum record number + recslot , & ! spline slot for current record + dataloc ! = 1 for data located in middle of time interval ! = 2 for date located at end of time interval - real (kind=dbl_kind) :: & + real (kind=dbl_kind) :: & secday , & Qa_pnt real (kind=dbl_kind) :: & - sec1hr ! number of seconds in 1 hour + sec1hr ! number of seconds in 1 hour logical (kind=log_kind) :: read1 +#endif + + integer (kind=int_kind) :: & + recnum , & ! record number + recnum4X ! record number character(len=*), parameter :: subname = '(ISPOL_data)' +#ifdef ncdf call icepack_query_parameters(secday_out=secday) call icepack_warnings_flush(nu_diag) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & file=__FILE__, line=__LINE__) -#ifdef ncdf if (trim(atm_data_format) == 'nc') then ! read nc file !------------------------------------------------------------------- @@ -4636,9 +4919,9 @@ subroutine ISPOL_data ! fixed data ! May not be needed !----------------------------- - rhoa (:,:,:) = 1.3_dbl_kind ! air density (kg/m^3) - cldf(:,:,:) = c1 !0.25_dbl_kind ! cloud fraction - frain(:,:,:) = c0 ! this is available in hourlymet_rh file + rhoa (:,:,:) = 1.3_dbl_kind ! air density (kg/m^3) + cldf(:,:,:) = c1 !0.25_dbl_kind ! cloud fraction + frain(:,:,:) = c0 ! this is available in hourlymet_rh file ! Save record number for next time step oldrecnum = recnum @@ -4675,6 +4958,7 @@ subroutine ocn_data_ispol_init use netcdf #endif +#ifdef ncdf integer (kind=int_kind) :: & n , & ! field index m ! month index @@ -4686,13 +4970,14 @@ subroutine ocn_data_ispol_init 'dhdx', 'dhdy', 'qdp' / real (kind=dbl_kind) :: & - work + work integer (kind=int_kind) :: & fid ! file id integer (kind=int_kind) :: & status ! status flag +#endif character(len=*), parameter :: subname = '(ocn_data_ispol_init)' @@ -4755,10 +5040,8 @@ subroutine box2001_data ! authors: Elizabeth Hunke, LANL use ice_domain, only: nblocks - use ice_constants, only: c0, c1, c2, c3, c4, c5, p2 use ice_blocks, only: nx_block, ny_block, nghost use ice_flux, only: uocn, vocn, uatm, vatm, wind, rhoa, strax, stray - use ice_fileunits, only: nu_diag, nu_forcing use ice_grid, only: uvm ! local parameters @@ -4767,7 +5050,7 @@ subroutine box2001_data iblk, i,j ! loop indices real (kind=dbl_kind) :: & - secday, pi , c10, c12, c20, puny, period, pi2, tau + secday, pi , puny, period, pi2, tau call icepack_query_parameters(pi_out=pi, pi2_out=pi2, puny_out=puny) call icepack_query_parameters(secday_out=secday) diff --git a/cicecore/cicedynB/infrastructure/ice_grid.F90 b/cicecore/cicedynB/infrastructure/ice_grid.F90 index 35c3003a4..7aa967a6c 100644 --- a/cicecore/cicedynB/infrastructure/ice_grid.F90 +++ b/cicecore/cicedynB/infrastructure/ice_grid.F90 @@ -15,18 +15,21 @@ module ice_grid use ice_kinds_mod + use ice_broadcast, only: broadcast_scalar, broadcast_array use ice_boundary, only: ice_HaloUpdate, ice_HaloExtrapolate use ice_communicate, only: my_task, master_task use ice_blocks, only: block, get_block, nx_block, ny_block, nghost use ice_domain_size, only: nx_global, ny_global, max_blocks use ice_domain, only: blocks_ice, nblocks, halo_info, distrb_info, & ew_boundary_type, ns_boundary_type, init_domain_distribution - use ice_fileunits, only: nu_diag, nu_grid, nu_kmt + use ice_fileunits, only: nu_diag, nu_grid, nu_kmt, & + get_fileunit, release_fileunit use ice_gather_scatter, only: gather_global, scatter_global use ice_read_write, only: ice_read, ice_read_nc, ice_read_global, & ice_read_global_nc, ice_open, ice_open_nc, ice_close_nc use ice_timers, only: timer_bound, ice_timer_start, ice_timer_stop use ice_exit, only: abort_ice + use ice_global_reductions, only: global_minval, global_maxval use icepack_intfc, only: icepack_warnings_flush, icepack_warnings_aborted use icepack_intfc, only: icepack_query_parameters @@ -540,7 +543,11 @@ subroutine init_grid2 ! bathymetry !----------------------------------------------------------------- +#ifdef RASM_MODS + call get_bathymetry_popfile +#else call get_bathymetry +#endif !---------------------------------------------------------------- ! Corner coordinates for CF compliant history files @@ -1662,7 +1669,6 @@ subroutine Tlatlon use ice_constants, only: c0, c1, c2, c4, & field_loc_center, field_type_scalar - use ice_global_reductions, only: global_minval, global_maxval integer (kind=int_kind) :: & i, j, iblk , & ! horizontal indices @@ -2357,6 +2363,98 @@ subroutine get_bathymetry end subroutine get_bathymetry +!======================================================================= +! with use_bathymetry = false, vertical depth profile generated for max KMT +! with use_bathymetry = true, expects to read in pop vert_grid file + + subroutine get_bathymetry_popfile + + integer (kind=int_kind) :: & + i, j, k, iblk ! loop indices + + integer (kind=int_kind) :: & + ntmp, nlevel , & ! number of levels (max KMT) + k1 , & ! levels + ierr , & ! error tag + fid ! fid unit number + + real (kind=dbl_kind), dimension(:),allocatable :: & + depth , & ! total depth, m + thick ! layer thickness, cm -> m + + character(len=*), parameter :: subname = '(get_bathymetry_popfile)' + + ntmp = maxval(KMT) + nlevel = global_maxval(ntmp,distrb_info) + + if (my_task==master_task) then + write(nu_diag,*) subname,' KMT max = ',nlevel + endif + + allocate(depth(nlevel),thick(nlevel)) + thick = -999999. + depth = -999999. + + if (use_bathymetry) then + + write (nu_diag,*) subname,' Bathymetry file = ', trim(bathymetry_file) + if (my_task == master_task) then + call get_fileunit(fid) + open(fid,file=bathymetry_file,form='formatted',iostat=ierr) + if (ierr/=0) call abort_ice(subname//' open error') + do k = 1,nlevel + read(fid,*,iostat=ierr) thick(k) + if (ierr/=0) call abort_ice(subname//' read error') + enddo + call release_fileunit(fid) + endif + + call broadcast_array(thick,master_task) + + else + + ! create thickness profile + k1 = min(5,nlevel) + do k = 1,k1 + thick(k) = max(10000._dbl_kind/float(nlevel),500.) + enddo + do k = k1+1,nlevel + thick(k) = min(thick(k-1)*1.2_dbl_kind,20000._dbl_kind) + enddo + + endif + + ! convert thick from cm to m + thick = thick / 100._dbl_kind + + ! convert to total depth + depth(1) = thick(1) + do k = 2, nlevel + depth(k) = depth(k-1) + thick(k) + if (depth(k) < 0.) call abort_ice(subname//' negative depth error') + enddo + + if (my_task==master_task) then + do k = 1,nlevel + write(nu_diag,'(2a,i6,2f13.7)') subname,' k, thick(m), depth(m) = ',k,thick(k),depth(k) + enddo + endif + + bathymetry = 0._dbl_kind + do iblk = 1, nblocks + do j = 1, ny_block + do i = 1, nx_block + k = kmt(i,j,iblk) + if (k > nlevel) call abort_ice(subname//' kmt/nlevel error') + if (k > 0) bathymetry(i,j,iblk) = depth(k) + enddo + enddo + enddo + + deallocate(depth,thick) + + end subroutine get_bathymetry_popfile + !======================================================================= ! Read bathymetry data for basal stress calculation (grounding scheme for diff --git a/configuration/scripts/machines/Macros.cheyenne_intel b/configuration/scripts/machines/Macros.cheyenne_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.hobart_intel b/configuration/scripts/machines/Macros.hobart_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.hobart_nag b/configuration/scripts/machines/Macros.hobart_nag old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.phase2_intel b/configuration/scripts/machines/Macros.phase2_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.phase3_intel b/configuration/scripts/machines/Macros.phase3_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/Macros.testmachine_intel b/configuration/scripts/machines/Macros.testmachine_intel old mode 100755 new mode 100644 diff --git a/configuration/scripts/machines/env.cesium_intel b/configuration/scripts/machines/env.cesium_intel old mode 100644 new mode 100755 diff --git a/configuration/scripts/machines/env.millikan_intel b/configuration/scripts/machines/env.millikan_intel old mode 100644 new mode 100755 diff --git a/doc/source/user_guide/ug_running.rst b/doc/source/user_guide/ug_running.rst index b22e79628..4abd52d0a 100644 --- a/doc/source/user_guide/ug_running.rst +++ b/doc/source/user_guide/ug_running.rst @@ -7,6 +7,71 @@ Running CICE Quick-start instructions are provided in the :ref:`quickstart` section. +.. _software: + +Software Requirements +---------------------- + +To run stand-alone, CICE requires + +- gmake (GNU Make) +- Fortran and C compilers (Intel, PGI, GNU, Cray, and NAG have been tested) +- NetCDF +- MPI (this is actually optional but without it you can only run on 1 processor) + +Below are lists of software versions that the Consortium has tested at some point. There is no +guarantee that all compiler versions work with all CICE model versions. At any given +point, the Consortium is regularly testing on several different compilers, but not +necessarily on all possible versions or combinations. A CICE goal is to be relatively portable +across different hardware, compilers, and other software. As a result, the coding +implementation tends to be on the conservative side at times. If there are problems +porting to a particular system, please let the Consortium know. + +The Consortium has tested the following compilers at some point, + +- Intel 15.0.3.187 +- Intel 16.0.1.150 +- Intel 17.0.1.132 +- Intel 17.0.2.174 +- Intel 17.0.5.239 +- Intel 18.0.1.163 +- Intel 19.0.2 +- Intel 19.0.3.199 +- PGI 16.10.0 +- GNU 6.3.0 +- GNU 7.2.0 +- GNU 7.3.0 +- Cray 8.5.8 +- Cray 8.6.4 +- NAG 6.2 + +The Consortium has tested the following mpi versions, + +- MPICH 7.3.2 +- MPICH 7.5.3 +- MPICH 7.6.2 +- MPICH 7.6.3 +- MPICH 7.7.6 +- Intel MPI 18.0.1 +- MPT 2.14 +- MPT 2.17 +- MPT 2.18 +- MPT 2.19 +- OpenMPI 1.6.5 + +The NetCDF implementation is relatively general and should work with any version of NetCDF 3 or 4. The Consortium has tested + +- NetCDF 4.3.0 +- NetCDF 4.3.2 +- NetCDF 4.4.0 +- NetCDF 4.4.1.1.32 +- NetCDF 4.4.1.1 +- NetCDF 4.4.2 +- NetCDF 4.5.0 +- NetCDF 4.6.1.3 + +Please email the Consortium if this list can be extended. + .. _scripts: Scripts @@ -56,7 +121,7 @@ Once a case/test is created, several files are placed in the case directory - **makdep.c** is a tool that will automatically generate the make dependencies - **Macros.[machine]** defines the Makefile macros - **Makefile** is the makefile used to build the model -- **cice.build** is a script that builds and compiles the model +- **cice.build** is a script that calls the Makefile and compiles the model - **ice\_in** is the namelist input file - **setup\_run\_dirs.csh** is a script that will create the run directories. This will be called automatically from the **cice.run** script if the user does not invoke it. - **cice.run** is a batch run script @@ -82,7 +147,7 @@ Some hints: - To change batch settings, manually edit the top of the **cice.run** or **cice.test** (if running a test) file - To turn on the debug compiler flags, set ``ICE_BLDDEBUG`` in **cice.setttings** to true. It is also possible to use the ``debug`` option (``-s debug``) when creating the case with **cice.setup** to set this option automatically. - To change compiler options, manually edit the Macros file -- To clean the build before each compile, set ``ICE_CLEANBUILD`` in **cice.settings** to true (this is the default value), or use the ``buildclean`` option (``-s buildclean``) when creating the case with **cice.setup**. To not clean before the build, set ``ICE_CLEANBUILD`` in **cice.settings** to false, or use the ``buildincremental`` option (``-s buildincremental``) when creating the case with **cice.setup**. +- To clean the build before each compile, set ``ICE_CLEANBUILD`` in **cice.settings** to true (this is the default value), or use the ``buildclean`` option (``-s buildclean``) when creating the case with **cice.setup**. To not clean before the build, set ``ICE_CLEANBUILD`` in **cice.settings** to false, or use the ``buildincremental`` option (``-s buildincremental``) when creating the case with **cice.setup**. It is recommended that the ``ICE_CLEANBUILD`` be set to true if there are any questions about whether the build is proceeding properly. To build and run:: @@ -96,8 +161,8 @@ case directory, NOT the run directory. .. _case_options: -Command Line Options -~~~~~~~~~~~~~~~~~~~~ +**cice.setup** Command Line Options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``cice.setup -h`` provides a summary of the command line options. There are three different modes, ``--case``, ``--test``, and ``--suite``. This section provides details about the relevant options for setting up cases with examples. Testing will be described in greater detail in the :ref:`testing` section. @@ -153,8 +218,8 @@ by doing ``cice.setup --help``. The default CICE namelist and CICE settings are specified in the files **configuration/scripts/ice_in** and -**configuration/scripts/cice.settings** respectively. When picking a -preset setting (option), the set_env.setting and set_nml.setting will be used to +**configuration/scripts/cice.settings** respectively. When picking +settings (options), the set_env.setting and set_nml.setting will be used to change the defaults. This is done as part of the ``cice.setup`` and the modifications are resolved in the **cice.settings** and **ice_in** file placed in the case directory. If multiple options are chosen and then conflict, then the last @@ -202,6 +267,84 @@ To add some optional settings, one might do:: Once the cases are created, users are free to modify the cice.settings and ice_in namelist to further modify their setup. +.. _cicebuild: + +More about **cice.build** +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**cice.build** is copied into the case directory and should be run interactively from the +case directory to build the model. CICE is built with make and there is a generic +Makefile and a machine specific Macros file in the case directory. **cice.build** +is a wrapper for a call to make that includes several other features. + +CICE is built as follows. First, the makdep binary is created by compiling a small +C program. The makdep binary is then run and dependency files are created. The dependency +files are included into the Makefile automatically. As a result, make dependencies do not +need to be explicitly defined by the user. In the next step, make compiles the CICE +code and generates the cice binary. + +The standard and recommended way to run is with +no arguments +:: + + cice.build + +However, **cice.build** does support a couple other use modes. +:: + + cice.build [-h|--help] + +provides a summary of the usage. +:: + + cice.build [make arguments] [target] + +turns off most of the features of the cice.build script and turns it into a wrapper +for the make call. The arguments and/or target are passed to make and invoked more +or less like make [make arguments] [target]. This will be the case if either or +both the arguments or target are passed to cice.build. Some examples of that are +:: + + cice.build --version + +which will pass --version to make. +:: + + cice.build targets + +is a valid target of the CICE Makefile and simply echos all the valid +targets of the Makefile. +:: + + cice.build cice + +or :: + + cice.build all + +are largely equivalent to running **cice.build** without an argument, +although as noted earlier, many of the extra features of the cice.build script +are turned off when calling cice.build with a target or an argument. Any of the +full builds will compile makdep, generate the source code dependencies, and +compile the source code. +:: + + cice.build [clean|realclean] + cice.build [db_files|db_flags] + cice.build [makdep|depends] + +are other valid options for cleaning the build, writing out information about +the Makefile setup, and building just the makdep tool or the dependency file. +It is also possible to target a particular CICE object file. + +Finally, there is one important parameter in **cice.settings**. The ``ICE_CLEANBUILD`` +variable defines whether the model is cleaned before a build is carried out. By +default, this variable is true which means each invokation of **cice.build** will +automatically clean the prior build. If incremental builds are desired to save +time during development, the ``ICE_CLEANBUILD`` setting in **cice.settings** should +be modified. + + .. _porting: Porting @@ -235,6 +378,36 @@ directory back to **configuration/scripts/machines/** and update the **configuration/scripts/cice.batch.csh** file, retest, and then add and commit the updated machine files to the repository. +.. _machvars: + +Machine variables +~~~~~~~~~~~~~~~~~~~~~~ + +There are several machine specific variables defined in the **env.$[machine]**. These +variables are used to generate working cases for a given machine, compiler, and batch +system. Some variables are optional. + +.. csv-table:: *Machine Settings* + :header: "variable", "format", "description" + :widths: 15, 15, 25 + + "ICE_MACHINE_ENVNAME", "string", "machine name" + "ICE_MACHINE_COMPILER", "string", "compiler" + "ICE_MACHINE_MAKE", "string", "make command" + "ICE_MACHINE_WKDIR", "string", "root work directory" + "ICE_MACHINE_INPUTDATA", "string", "root input data directory" + "ICE_MACHINE_BASELINE", "string", "root regression baseline directory" + "ICE_MACHINE_SUBMIT", "string", "batch job submission command" + "ICE_MACHINE_TPNODE", "integer", "machine maximum MPI tasks per node" + "ICE_MACHINE_MAXPES", "integer", "machine maximum total processors per job (optional)" + "ICE_MACHINE_MAXTHREADS", "integer", "machine maximum threads per mpi task (optional)" + "ICE_MACHINE_MAXRUNLENGTH", "integer", "batch wall time limit in hours (optional)" + "ICE_MACHINE_ACCT", "string", "batch default account" + "ICE_MACHINE_QUEUE", "string", "batch default queue" + "ICE_MACHINE_BLDTHRDS", "integer", "number of threads used during build" + "ICE_MACHINE_QSTAT", "string", "batch job status command (optional)" + "ICE_MACHINE_QUIETMODE", "true/false", "flag to reduce build output (optional)" + .. _cross_compiling: Cross-compiling @@ -305,3 +478,94 @@ should be rebuilt before being resubmitted. It is always recommended that users modify the scripts and input settings in the case directory, NOT the run directory. In general, files in the run directory are overwritten by versions in the case directory when the model is built, submitted, and run. + +.. _timeseries: + +Timeseries Plotting +------------------- + +The CICE scripts include two scripts that will generate timeseries figures from a +diagnostic output file, a Python version (``timeseries.py``) and a csh version +(``timeseries.csh``). Both scripts create the same set of plots, but the Python +script has more capabilities, and it's likely that the csh +script will be removed in the future. + +To use the ``timeseries.py`` script, the following requirements must be met: + +* Python v2.7 or later +* numpy Python package +* matplotlib Python package +* datetime Python package + +See :ref:`CodeCompliance` for additional information about how to setup the Python +environment, but we recommend using ``pip`` as follows: :: + + pip install --user numpy + pip install --user matplotlib + pip install --user datetime + +When creating a case or test via ``cice.setup``, the ``timeseries.csh`` and +``timeseries.py`` scripts are automatically copied to the case directory. +Alternatively, the plotting scripts can be found in ``./configuration/scripts``, and can be +run from any directory. + +The Python script can be passed a directory, a specific log file, or no directory at all: + + - If a directory is passed, the script will look either in that directory or in + directory/logs for a filename like cice.run*. As such, users can point the script + to either a case directory or the ``logs`` directory directly. The script will use + the file with the most recent creation time. + - If a specific file is passed the script parses that file, assuming that the file + matches the same form of cice.run* files. + - If nothing is passed, the script will look for log files or a ``logs`` directory in the + directory from where the script was run. + +For example: + +Run the timeseries script on the desired case. :: + +$ python timeseries.py /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/ + +or :: + +$ python timeseries.py /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/logs + +The output figures are placed in the directory where the ``timeseries.py`` script is run. + +The plotting script will plot the following variables by default, but you can also select +specific plots to create via the optional command line arguments. + + - total ice area (:math:`km^2`) + - total ice extent (:math:`km^2`) + - total ice volume (:math:`m^3`) + - total snow volume (:math:`m^3`) + - RMS ice speed (:math:`m/s`) + +For example, to plot only total ice volume and total snow volume :: + +$ python timeseries.py /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/ --volume --snw_vol + +To generate plots for all of the cases within a suite with a testid, create and run a script such as :: + + #!/bin/csh + foreach dir (`ls -1 | grep testid`) + echo $dir + python timeseries.py $dir + end + +Plots are only made for a single output file at a time. The ability to plot output from +a series of cice.run* files is not currently possible, but may be added in the future. +However, using the ``--bdir`` option will plot two datasets (from log files) on the +same figure. + +For the latest help information for the script, run :: + +$ python timeseries.py -h + +The ``timeseries.csh`` script works basically the same way as the Python version, however it +does not include all of the capabilities present in the Python version. + +To use the C-Shell version of the script, :: + +$ ./timeseries.csh /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/ + From 68615cdca8ccc59a29bdd224bdff8c3b583fa7c4 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 23:14:53 +0000 Subject: [PATCH 43/54] retrieve hera --- .../scripts/machines/Macros.hera_intel | 60 +++++++++++++++++++ configuration/scripts/machines/env.hera_intel | 26 ++++++++ 2 files changed, 86 insertions(+) create mode 100644 configuration/scripts/machines/Macros.hera_intel create mode 100755 configuration/scripts/machines/env.hera_intel diff --git a/configuration/scripts/machines/Macros.hera_intel b/configuration/scripts/machines/Macros.hera_intel new file mode 100644 index 000000000..38cc62ed8 --- /dev/null +++ b/configuration/scripts/machines/Macros.hera_intel @@ -0,0 +1,60 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpiifort +MPIFC:= mpiifort +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := + + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.hera_intel b/configuration/scripts/machines/env.hera_intel new file mode 100755 index 000000000..ef3afdb7a --- /dev/null +++ b/configuration/scripts/machines/env.hera_intel @@ -0,0 +1,26 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +#module list +module purge +module load intel/18.0.5.274 +module load impi/2018.0.4 +module load esmf/7.1.0r +module load hdf5/1.10.4 netcdf/4.6.1 +module load wgrib wgrib2 +#echo renewed modules: +#module list + +setenv ICE_MACHINE_ENVNAME hera +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/rgdev/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/rgdev/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/rgdev/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "batch" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true From 74659f2314eb06feba26caba4743ff5271ed70a5 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 23:17:30 +0000 Subject: [PATCH 44/54] docs and cicecore sync to master --- .github/PULL_REQUEST_TEMPLATE | 46 +++--- .gitmodules | 4 - .../cicedynB/analysis/ice_diagnostics.F90 | 2 + .../cicedynB/analysis/ice_history_bgc.F90 | 12 +- .../cicedynB/analysis/ice_history_shared.F90 | 16 +-- cicecore/cicedynB/dynamics/ice_dyn_eap.F90 | 8 +- cicecore/cicedynB/dynamics/ice_dyn_evp.F90 | 15 +- cicecore/cicedynB/dynamics/ice_dyn_shared.F90 | 59 +++----- .../cicedynB/dynamics/ice_transport_remap.F90 | 2 +- cicecore/cicedynB/general/ice_forcing_bgc.F90 | 60 ++++---- cicecore/cicedynB/general/ice_init.F90 | 25 ++-- cicecore/cicedynB/general/ice_step_mod.F90 | 6 - .../comm/mpi/ice_gather_scatter.F90 | 1 - .../infrastructure/comm/mpi/ice_reprosum.F90 | 2 +- .../comm/serial/ice_boundary.F90 | 3 +- .../comm/serial/ice_reprosum.F90 | 2 +- .../cicedynB/infrastructure/ice_domain.F90 | 6 +- .../infrastructure/ice_read_write.F90 | 30 ++-- .../cicedynB/infrastructure/ice_restoring.F90 | 2 +- .../io/io_binary/ice_history_write.F90 | 2 +- .../io/io_binary/ice_restart.F90 | 15 +- .../io/io_netcdf/ice_restart.F90 | 13 +- .../io/io_pio/ice_history_write.F90 | 8 +- cicecore/drivers/cesm/ice_comp_esmf.F90 | 12 +- cicecore/drivers/cesm/ice_comp_mct.F90 | 14 +- cicecore/drivers/cesm/ice_prescribed_mod.F90 | 2 +- cicecore/drivers/cesm/ice_scam.F90 | 1 - cicecore/drivers/cice/CICE_InitMod.F90 | 4 +- cicecore/drivers/cice/CICE_RunMod.F90 | 2 +- cicecore/shared/ice_arrays_column.F90 | 5 +- cicecore/shared/ice_calendar.F90 | 8 +- cicecore/shared/ice_constants.F90 | 2 +- cicecore/shared/ice_init_column.F90 | 17 +-- cicecore/shared/ice_restart_column.F90 | 3 - cicecore/shared/ice_spacecurve.F90 | 84 +++++------ cicecore/version.txt | 2 +- doc/source/conf.py | 4 +- doc/source/developer_guide/dg_scripts.rst | 26 +++- doc/source/intro/quickstart.rst | 2 + doc/source/master_list.bib | 17 +++ doc/source/science_guide/sg_dynamics.rst | 10 +- doc/source/user_guide/ug_case_settings.rst | 3 +- doc/source/user_guide/ug_implementation.rst | 134 +++++++++--------- doc/source/user_guide/ug_testing.rst | 69 ++------- doc/source/user_guide/ug_troubleshooting.rst | 3 + 45 files changed, 355 insertions(+), 408 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 6256f9bc6..83509033c 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -1,29 +1,19 @@ +[Remove this and add a short summary line]: -For detailed information about submitting Pull Requests (PRs) to the CICE-Consortium, -please refer to: - - -## PR checklist -- [ ] Short (1 sentence) summary of your PR: - ENTER INFORMATION HERE -- [ ] Developer(s): - ENTER INFORMATION HERE -- [ ] Suggest PR reviewers from list in the column to the right. -- [ ] Please copy the PR test results link or provide a summary of testing completed below. - ENTER INFORMATION HERE -- How much do the PR code changes differ from the unmodified code? - - [ ] bit for bit - - [ ] different at roundoff level - - [ ] more substantial -- Does this PR create or have dependencies on Icepack or any other models? - - [ ] Yes - - [ ] No -- Does this PR add any new test cases? - - [ ] Yes - - [ ] No -- Is the documentation being updated? ("Documentation" includes information on the wiki or in the .rst files from doc/source/, which are used to create the online technical docs at https://readthedocs.org/projects/cice-consortium-cice/. A test build of the technical docs will be performed as part of the PR testing.) - - [ ] Yes - - [ ] No, does the documentation need to be updated at a later time? - - [ ] Yes - - [ ] No -- [ ] Please provide any additional information or relevant details below: +- Developer(s): + +- Please suggest code Pull Request reviewers in the column at right. + +- Are the code changes bit for bit, different at roundoff level, or more substantial? + +- Please include the link to test results or paste the summary block from the bottom of the testing output below. + +- Does this PR create or have dependencies on Icepack or any other models? + +- Is the documentation being updated with this PR? (Y/N) +If not, does the documentation need to be updated separately at a later time? (Y/N) + +Note: "Documentation" includes information on the wiki and .rst files in doc/source/, +which are used to create the online technical docs at https://readthedocs.org/projects/cice-consortium-cice/. + +- Other Relevant Details: diff --git a/.gitmodules b/.gitmodules index 9e463fcec..e69de29bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +0,0 @@ -[submodule "icepack"] - path = icepack - url = https://github.com/cice-consortium/icepack.git - branch = master diff --git a/cicecore/cicedynB/analysis/ice_diagnostics.F90 b/cicecore/cicedynB/analysis/ice_diagnostics.F90 index 4557b4075..5f8c56264 100644 --- a/cicecore/cicedynB/analysis/ice_diagnostics.F90 +++ b/cicecore/cicedynB/analysis/ice_diagnostics.F90 @@ -124,6 +124,7 @@ subroutine runtime_diags (dt) use ice_global_reductions, only: global_sum, global_sum_prod, global_maxval use ice_grid, only: lmask_n, lmask_s, tarean, tareas use ice_state ! everything +! tcraig, this is likely to cause circular dependency because ice_prescribed_mod is high level routine #ifdef CESMCOUPLED use ice_prescribed_mod, only: prescribed_ice #endif @@ -839,6 +840,7 @@ subroutine runtime_diags (dt) if (print_global) then ! global diags for conservations checks +! tcraig, this is likely to cause circular dependency because ice_prescribed_mod is high level routine #ifdef CESMCOUPLED if (prescribed_ice) then write (nu_diag,*) '----------------------------' diff --git a/cicecore/cicedynB/analysis/ice_history_bgc.F90 b/cicecore/cicedynB/analysis/ice_history_bgc.F90 index ae5bccb2f..c27683423 100644 --- a/cicecore/cicedynB/analysis/ice_history_bgc.F90 +++ b/cicecore/cicedynB/analysis/ice_history_bgc.F90 @@ -112,10 +112,10 @@ module ice_history_bgc f_peakval = 'x', f_bgc_Fed_ml = 'x', & f_bgc_Fep_ml = 'x', f_bgc_hum_ml = 'x', & f_bgc_N_cat1 = 'x', f_bgc_DOC_cat1 = 'x', & - f_bgc_DIC_cat1 = 'x', f_bgc_Nit_cat1 = 'x', & + f_bgc_DIC_cat1 = 'x', f_bgc_Nit_cat1 = 'x', & f_bgc_Am_cat1 = 'x', f_bgc_Sil_cat1 = 'x', & f_bgc_DMSPd_cat1= 'x', f_bgc_DMS_cat1 = 'x', & - f_bgc_DON_cat1 = 'x', f_bgc_Fed_cat1 = 'x', & + f_bgc_DON_cat1 = 'x', f_bgc_Fed_cat1 = 'x', & f_bgc_hum_cat1 = 'x', f_bgc_Fep_cat1 = 'x', & f_bgc_PON_cat1 = 'x' @@ -226,11 +226,11 @@ module ice_history_bgc n_upNO , n_upNH , & n_bTin , n_bphi , & n_iDi , n_iki , & - n_bgc_PON , n_bgc_PON_ml , & + n_bgc_PON , & n_fbri , n_hbri , & n_zfswin , n_Nitnet , & n_Amnet , n_Silnet , & - n_humnet , & + n_humnet , & n_DMSPpnet , n_DMSPdnet , & n_DMSnet , n_PONnet , & n_Nitsnow , n_Amsnow , & @@ -238,8 +238,8 @@ module ice_history_bgc n_DMSPpsnow , n_DMSPdsnow , & n_DMSsnow , n_PONsnow , & n_Nitfrac , n_Amfrac , & - n_Silfrac , n_zbgc_frac , & - n_humfrac , & + n_Silfrac , & + n_humfrac , & n_DMSPpfrac , n_DMSPdfrac , & n_DMSfrac , n_PONfrac , & n_grownet , n_PPnet , & diff --git a/cicecore/cicedynB/analysis/ice_history_shared.F90 b/cicecore/cicedynB/analysis/ice_history_shared.F90 index d8335e1f7..64a82e9ae 100644 --- a/cicecore/cicedynB/analysis/ice_history_shared.F90 +++ b/cicecore/cicedynB/analysis/ice_history_shared.F90 @@ -452,7 +452,7 @@ module ice_history_shared f_e22, & f_s11, f_s12, & f_s22, & - f_yieldstress11, & + f_yieldstress11, & f_yieldstress12, & f_yieldstress22 @@ -608,13 +608,13 @@ module ice_history_shared n_keffn_top , & n_Tinz , n_Sinz , & n_Tsnz , & - n_a11 , n_a12 , & - n_e11 , n_e12 , & - n_e22 , & - n_s11 , n_s12 , & - n_s22 , & - n_yieldstress11, n_yieldstress12, & - n_yieldstress22 + n_a11 , n_a12 , & + n_e11 , n_e12 , & + n_e22 , & + n_s11 , n_s12 , & + n_s22 , & + n_yieldstress11, n_yieldstress12, & + n_yieldstress22 interface accum_hist_field ! generic interface module procedure accum_hist_field_2D, & diff --git a/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 b/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 index 944d80607..bc42e4855 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_eap.F90 @@ -119,8 +119,8 @@ subroutine eap (dt) use ice_constants, only: field_loc_center, field_loc_NEcorner, & field_type_scalar, field_type_vector use ice_domain, only: nblocks, blocks_ice, halo_info, maskhalo_dyn - use ice_dyn_shared, only: fcor_blk, ndte, dtei, a_min, m_min, & - cosw, sinw, denom1, uvel_init, vvel_init, arlx1i, & + use ice_dyn_shared, only: fcor_blk, ndte, dtei, & + denom1, uvel_init, vvel_init, arlx1i, & dyn_prep1, dyn_prep2, stepu, dyn_finish, & basal_stress_coeff, basalstress use ice_flux, only: rdg_conv, strairxT, strairyT, & @@ -1615,9 +1615,9 @@ subroutine update_stress_rdg (ksub, ndte, divu, tension, & invstressconviso, & Angle_denom_gamma, Angle_denom_alpha, & Tany_1, Tany_2, & - gamma, alpha, x, y, dx, dy, da, & + x, y, dx, dy, da, & invdx, invdy, invda, invsin, & - invleng, dtemp1, dtemp2, atempprime, & + dtemp1, dtemp2, atempprime, & kxw, kyw, kaw, & puny, pi, pi2, piq, pih diff --git a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 index c1371db7a..002341667 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_evp.F90 @@ -37,8 +37,8 @@ module ice_dyn_evp use ice_communicate, only: my_task use ice_constants, only: field_loc_center, field_loc_NEcorner, & field_type_scalar, field_type_vector - use ice_constants, only: c0, c4, p027, p055, p111, p166, & - p2, p222, p25, p333, p5, c1 + use ice_constants, only: c0, p027, p055, p111, p166, & + p222, p25, p333, p5, c1 use ice_dyn_shared, only: stepu, dyn_prep1, dyn_prep2, dyn_finish, & ndte, yield_curve, ecci, denom1, arlx1i, fcor_blk, uvel_init, & vvel_init, basal_stress_coeff, basalstress, Ktens, revp @@ -92,7 +92,8 @@ subroutine evp (dt) aice_init, aice0, aicen, vicen, strength use ice_timers, only: timer_dynamics, timer_bound, & ice_timer_start, ice_timer_stop, timer_evp_1d, timer_evp_2d - use ice_dyn_evp_1d + use ice_dyn_evp_1d, only: ice_dyn_evp_1d_copyin, ice_dyn_evp_1d_kernel, & + ice_dyn_evp_1d_copyout use ice_dyn_shared, only: kevp_kernel real (kind=dbl_kind), intent(in) :: & @@ -353,7 +354,7 @@ subroutine evp (dt) call abort_ice(trim(subname)//' & & Kernel not tested on tripole grid. Set kevp_kernel=0') endif - call evp_copyin( & + call ice_dyn_evp_1d_copyin( & nx_block,ny_block,nblocks,nx_global+2*nghost,ny_global+2*nghost, & HTE,HTN, & !v1 dxhy,dyhx,cyp,cxp,cym,cxm,tinyarea, & @@ -367,7 +368,7 @@ subroutine evp (dt) stress12_1,stress12_2,stress12_3,stress12_4 ) if (kevp_kernel == 2) then call ice_timer_start(timer_evp_1d) - call evp_kernel_v2() + call ice_dyn_evp_1d_kernel() call ice_timer_stop(timer_evp_1d) !v1 else if (kevp_kernel == 1) then !v1 call evp_kernel_v1() @@ -375,7 +376,7 @@ subroutine evp (dt) if (my_task == 0) write(nu_diag,*) subname,' ERROR: kevp_kernel = ',kevp_kernel call abort_ice(subname//' kevp_kernel not supported.') endif - call evp_copyout( & + call ice_dyn_evp_1d_copyout( & nx_block,ny_block,nblocks,nx_global+2*nghost,ny_global+2*nghost,& !strocn uvel,vvel, strocnx,strocny, strintx,strinty, & uvel,vvel, strintx,strinty, & @@ -641,7 +642,7 @@ subroutine stress (nx_block, ny_block, & tensionne, tensionnw, tensionse, tensionsw, & ! tension shearne, shearnw, shearse, shearsw , & ! shearing Deltane, Deltanw, Deltase, Deltasw , & ! Delt - puny , & ! puny +! puny , & ! puny c0ne, c0nw, c0se, c0sw , & ! useful combinations c1ne, c1nw, c1se, c1sw , & ssigpn, ssigps, ssigpe, ssigpw , & diff --git a/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 b/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 index 4c479171e..fec4c6823 100644 --- a/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 +++ b/cicecore/cicedynB/dynamics/ice_dyn_shared.F90 @@ -11,7 +11,8 @@ module ice_dyn_shared use ice_kinds_mod use ice_communicate, only: my_task, master_task - use ice_constants, only: c0, c1, p01, p001 + use ice_constants, only: c0, c1, c2, p01, p001 + use ice_constants, only: omega, spval_dbl, p5, c4 use ice_blocks, only: nx_block, ny_block use ice_domain_size, only: max_blocks use ice_fileunits, only: nu_diag @@ -115,7 +116,6 @@ end subroutine alloc_dyn_shared subroutine init_evp (dt) use ice_blocks, only: nx_block, ny_block - use ice_constants, only: c0, c2, omega use ice_domain, only: nblocks use ice_domain_size, only: max_blocks use ice_flux, only: rdg_conv, rdg_shear, iceumask, & @@ -205,12 +205,6 @@ end subroutine init_evp subroutine set_evp_parameters (dt) - use ice_communicate, only: my_task, master_task - use ice_constants, only: p25, c1, c2, p5 - use ice_domain, only: distrb_info - use ice_global_reductions, only: global_minval - use ice_grid, only: dxt, dyt, tmask - real (kind=dbl_kind), intent(in) :: & dt ! time step @@ -275,8 +269,6 @@ subroutine dyn_prep1 (nx_block, ny_block, & strairx, strairy, & tmass, icetmask) - use ice_constants, only: c0 - integer (kind=int_kind), intent(in) :: & nx_block, ny_block, & ! block dimensions ilo,ihi,jlo,jhi ! beginning and end of physical domain @@ -409,8 +401,6 @@ subroutine dyn_prep2 (nx_block, ny_block, & uvel, vvel, & Tbu) - use ice_constants, only: c0, c1 - integer (kind=int_kind), intent(in) :: & nx_block, ny_block, & ! block dimensions ilo,ihi,jlo,jhi ! beginning and end of physical domain @@ -503,20 +493,7 @@ subroutine dyn_prep2 (nx_block, ny_block, & taubx (i,j) = c0 tauby (i,j) = c0 - if (revp==1) then ! revised evp - stressp_1 (i,j) = c0 - stressp_2 (i,j) = c0 - stressp_3 (i,j) = c0 - stressp_4 (i,j) = c0 - stressm_1 (i,j) = c0 - stressm_2 (i,j) = c0 - stressm_3 (i,j) = c0 - stressm_4 (i,j) = c0 - stress12_1(i,j) = c0 - stress12_2(i,j) = c0 - stress12_3(i,j) = c0 - stress12_4(i,j) = c0 - else if (icetmask(i,j)==0) then ! classic evp + if (icetmask(i,j)==0) then stressp_1 (i,j) = c0 stressp_2 (i,j) = c0 stressp_3 (i,j) = c0 @@ -529,7 +506,7 @@ subroutine dyn_prep2 (nx_block, ny_block, & stress12_2(i,j) = c0 stress12_3(i,j) = c0 stress12_4(i,j) = c0 - endif ! revp + endif enddo ! i enddo ! j @@ -890,8 +867,6 @@ subroutine basal_stress_coeff (nx_block, ny_block, & vice, aice, & hwater, Tbu) - use ice_constants, only: c0, c1 - integer (kind=int_kind), intent(in) :: & nx_block, ny_block, & ! block dimensions icellu ! no. of cells where icetmask = 1 @@ -913,8 +888,10 @@ subroutine basal_stress_coeff (nx_block, ny_block, & hu, & ! volume per unit area of ice at u location (mean thickness) hwu, & ! water depth at u location hcu, & ! critical thickness at u location - k2 = 15.0_dbl_kind , & ! second free parameter (N/m^3) for landfast parametrization - alphab = 20.0_dbl_kind ! alphab=Cb factor in Lemieux et al 2015 + k2 = 15.0_dbl_kind , & ! second free parameter (N/m^3) for landfast parametrization + alphab = 20.0_dbl_kind, & ! alphab=Cb factor in Lemieux et al 2015 + threshold_hw = 30.0_dbl_kind ! max water depth for grounding + ! see keel data from Amundrud et al. 2004 (JGR) integer (kind=int_kind) :: & i, j, ij @@ -926,15 +903,21 @@ subroutine basal_stress_coeff (nx_block, ny_block, & j = indxuj(ij) ! convert quantities to u-location - au = max(aice(i,j),aice(i+1,j),aice(i,j+1),aice(i+1,j+1)) + hwu = min(hwater(i,j),hwater(i+1,j),hwater(i,j+1),hwater(i+1,j+1)) - hu = max(vice(i,j),vice(i+1,j),vice(i,j+1),vice(i+1,j+1)) - ! 1- calculate critical thickness - hcu = au * hwu / k1 + if (hwu < threshold_hw) then + + au = max(aice(i,j),aice(i+1,j),aice(i,j+1),aice(i+1,j+1)) + hu = max(vice(i,j),vice(i+1,j),vice(i,j+1),vice(i+1,j+1)) - ! 2- calculate basal stress factor - Tbu(i,j) = k2 * max(c0,(hu - hcu)) * exp(-alphab * (c1 - au)) + ! 1- calculate critical thickness + hcu = au * hwu / k1 + + ! 2- calculate basal stress factor + Tbu(i,j) = k2 * max(c0,(hu - hcu)) * exp(-alphab * (c1 - au)) + + endif enddo ! ij @@ -953,8 +936,6 @@ subroutine principal_stress(nx_block, ny_block, & sig1, sig2, & sigP) - use ice_constants, only: spval_dbl, p5, c4 - integer (kind=int_kind), intent(in) :: & nx_block, ny_block ! block dimensions diff --git a/cicecore/cicedynB/dynamics/ice_transport_remap.F90 b/cicecore/cicedynB/dynamics/ice_transport_remap.F90 index bf9d0f373..b641104ed 100644 --- a/cicecore/cicedynB/dynamics/ice_transport_remap.F90 +++ b/cicecore/cicedynB/dynamics/ice_transport_remap.F90 @@ -260,7 +260,7 @@ subroutine init_remap ! xxxav, xxyav, xyyav, yyyav integer (kind=int_kind) :: & - i, j, iblk ! standard indices + i, j, iblk ! standard indices character(len=*), parameter :: subname = '(init_remap)' diff --git a/cicecore/cicedynB/general/ice_forcing_bgc.F90 b/cicecore/cicedynB/general/ice_forcing_bgc.F90 index 0b34accb1..6e543a056 100644 --- a/cicecore/cicedynB/general/ice_forcing_bgc.F90 +++ b/cicecore/cicedynB/general/ice_forcing_bgc.F90 @@ -77,7 +77,7 @@ subroutine get_forcing_bgc use ice_domain, only: nblocks, blocks_ice use ice_arrays_column, only: ocean_bio_all use ice_calendar, only: yday - use ice_flux, only: sss +! use ice_flux, only: sss use ice_flux_bgc, only: sil, nit use ice_forcing, only: trestore, trest, fyear, & read_clim_data_nc, interpolate_data, & @@ -202,7 +202,7 @@ subroutine get_forcing_bgc if (istep == 1 .or. .NOT. restore_bgc) then !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -210,8 +210,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi sil(i,j,iblk) = sildat(i,j,iblk) ks = 2*icepack_max_algae + icepack_max_doc + 3 + icepack_max_dic @@ -224,7 +224,7 @@ subroutine get_forcing_bgc elseif (restore_bgc) then !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -232,8 +232,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi sil(i,j,iblk) = sil(i,j,iblk) & + (sildat(i,j,iblk)-sil(i,j,iblk))*dt/trest @@ -246,7 +246,7 @@ subroutine get_forcing_bgc endif !restore elseif (tr_bgc_Sil) then ! bgc_data_type /= 'clim' !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -254,10 +254,10 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi - sil(i,j,iblk) = 25.0_dbl_kind + sil(i,j,iblk) = 25.0_dbl_kind ks = 2*icepack_max_algae + icepack_max_doc + 3 + icepack_max_dic ocean_bio_all(i,j,ks,iblk) = sil(i,j,iblk) !Sil enddo @@ -283,7 +283,7 @@ subroutine get_forcing_bgc if (istep == 1 .or. .NOT. restore_bgc) then !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -291,10 +291,10 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi - nit(i,j,iblk) = nitdat(i,j,iblk) + nit(i,j,iblk) = nitdat(i,j,iblk) ks = icepack_max_algae + 1 ocean_bio_all(i,j,ks,iblk) = nit(i,j,iblk) !nit ks = 2*icepack_max_algae + icepack_max_doc + 7 + icepack_max_dic @@ -305,7 +305,7 @@ subroutine get_forcing_bgc !$OMP END PARALLEL DO elseif (restore_bgc ) then !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -313,8 +313,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi nit(i,j,iblk) = nit(i,j,iblk) & + (nitdat(i,j,iblk)-nit(i,j,iblk))*dt/trest @@ -330,7 +330,7 @@ subroutine get_forcing_bgc ! elseif (trim(nit_data_type) == 'sss' .AND. tr_bgc_Nit) then ! !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) -! do iblk = 1, nblocks +! do iblk = 1, nblocks ! this_block = get_block(blocks_ice(iblk),iblk) ! ilo = this_block%ilo @@ -353,7 +353,7 @@ subroutine get_forcing_bgc elseif (tr_bgc_Nit) then ! bgc_data_type /= 'clim' !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -361,8 +361,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi nit(i,j,iblk) = 12.0_dbl_kind ks = icepack_max_algae + 1 @@ -444,7 +444,7 @@ subroutine get_forcing_bgc endif !$OMP PARALLEL DO PRIVATE(iblk,ilo,ihi,jlo,jhi,this_block) - do iblk = 1, nblocks + do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) ilo = this_block%ilo @@ -452,8 +452,8 @@ subroutine get_forcing_bgc jlo = this_block%jlo jhi = this_block%jhi - do j = jlo, jhi - do i = ilo, ihi + do j = jlo, jhi + do i = ilo, ihi ks = 2*icepack_max_algae + icepack_max_doc + 3 + icepack_max_dic ocean_bio_all(i,j,ks,iblk) = sil(i,j,iblk) !Sil @@ -791,7 +791,7 @@ subroutine init_bgc_data (fed1,fep1) !------------------------------------------------------------------- if (trim(fe_data_type) == 'clim') then - diag = .true. ! write diagnostic information + diag = .true. ! write diagnostic information iron_file = trim(bgc_data_dir)//'dFe_50m_annual_Tagliabue_gx1.nc' if (my_task == master_task) then @@ -808,7 +808,7 @@ subroutine init_bgc_data (fed1,fep1) if (my_task == master_task) call ice_close_nc(fid) - diag = .true. ! write diagnostic information + diag = .true. ! write diagnostic information iron_file = trim(bgc_data_dir)//'pFe_bathy_gx1.nc' if (my_task == master_task) then @@ -858,6 +858,9 @@ subroutine faero_optics ! local parameters + logical (kind=log_kind) :: modal_aero + +#ifdef ncdf integer (kind=int_kind) :: & varid , & ! variable id status , & ! status output from netcdf routines @@ -869,11 +872,10 @@ subroutine faero_optics integer (kind=int_kind) :: & fid ! file id for netCDF file - logical (kind=log_kind) :: modal_aero - character (char_len_long) :: & optics_file, & ! netcdf filename fieldname ! field name in netcdf file +#endif character(len=*), parameter :: subname = '(faero_optics)' diff --git a/cicecore/cicedynB/general/ice_init.F90 b/cicecore/cicedynB/general/ice_init.F90 index 9b48fed51..1d3aedf6e 100644 --- a/cicecore/cicedynB/general/ice_init.F90 +++ b/cicecore/cicedynB/general/ice_init.F90 @@ -19,14 +19,14 @@ module ice_init use ice_fileunits, only: nu_nml, nu_diag, nml_filename, diag_type, & ice_stdout, get_fileunit, release_fileunit, bfbflag, flush_fileunit, & ice_IOUnitsMinUnit, ice_IOUnitsMaxUnit +#ifdef CESMCOUPLED use ice_fileunits, only: inst_suffix +#endif use icepack_intfc, only: icepack_warnings_flush, icepack_warnings_aborted use icepack_intfc, only: icepack_aggregate use icepack_intfc, only: icepack_init_trcr use icepack_intfc, only: icepack_init_parameters use icepack_intfc, only: icepack_init_tracer_flags - use icepack_intfc, only: icepack_init_tracer_numbers - use icepack_intfc, only: icepack_init_tracer_indices use icepack_intfc, only: icepack_query_tracer_flags use icepack_intfc, only: icepack_query_tracer_numbers use icepack_intfc, only: icepack_query_tracer_indices @@ -107,8 +107,6 @@ subroutine input_data nml_error, & ! namelist i/o error flag n ! loop index - character (len=6) :: chartmp - logical :: exists real (kind=dbl_kind) :: ustar_min, albicev, albicei, albsnowv, albsnowi, & @@ -127,8 +125,6 @@ subroutine input_data logical (kind=log_kind) :: tr_iage, tr_FY, tr_lvl, tr_pond, tr_aero logical (kind=log_kind) :: tr_pond_cesm, tr_pond_lvl, tr_pond_topo - integer (kind=int_kind) :: nt_Tsfc, nt_sice, nt_qice, nt_qsno, nt_iage, nt_FY - integer (kind=int_kind) :: nt_alvl, nt_vlvl, nt_apnd, nt_hpnd, nt_ipnd, nt_aero integer (kind=int_kind) :: numin, numax ! unit number limits integer (kind=int_kind) :: rpcesm, rplvl, rptopo @@ -1886,7 +1882,7 @@ subroutine boxslotcyl_data_aice(aicen, i, j, & ! Geometric configuration of the slotted cylinder diam = p3 *dxrect*(nx_global-1) center_x = p5 *dxrect*(nx_global-1) - center_y = p75*dxrect*(ny_global-1) + center_y = p75*dyrect*(ny_global-1) radius = p5*diam width = p166*diam length = c5*p166*diam @@ -1899,11 +1895,11 @@ subroutine boxslotcyl_data_aice(aicen, i, j, & ! check if grid point is inside slotted cylinder in_slot = (dxrect*real(iglob(i)-1, kind=dbl_kind) >= slot_x(1)) .and. & (dxrect*real(iglob(i)-1, kind=dbl_kind) <= slot_x(2)) .and. & - (dxrect*real(jglob(j)-1, kind=dbl_kind) >= slot_y(1)) .and. & - (dxrect*real(jglob(j)-1, kind=dbl_kind) <= slot_y(2)) + (dyrect*real(jglob(j)-1, kind=dbl_kind) >= slot_y(1)) .and. & + (dyrect*real(jglob(j)-1, kind=dbl_kind) <= slot_y(2)) in_cyl = sqrt((dxrect*real(iglob(i)-1, kind=dbl_kind) - center_x)**c2 + & - (dxrect*real(jglob(j)-1, kind=dbl_kind) - center_y)**c2) <= radius + (dyrect*real(jglob(j)-1, kind=dbl_kind) - center_y)**c2) <= radius in_slotted_cyl = in_cyl .and. .not. in_slot @@ -1927,9 +1923,10 @@ subroutine boxslotcyl_data_vel(i, j, & iglob, jglob, & uvel, vvel) - use ice_constants, only: c1, c4, c2, c12, p5, cm_to_m + use ice_constants, only: c2, c12, p5, cm_to_m use ice_domain_size, only: nx_global, ny_global use ice_grid, only: dxrect + use icepack_parameters, only: secday, pi integer (kind=int_kind), intent(in) :: & i, j, & ! local indices @@ -1946,15 +1943,11 @@ subroutine boxslotcyl_data_vel(i, j, & max_vel , & ! max velocity domain_length , & ! physical domain length period ! rotational period - - real (kind=dbl_kind), parameter :: & - pi = c4*atan(c1), & ! pi - days_to_s = 86400._dbl_kind character(len=*), parameter :: subname = '(boxslotcyl_data_vel)' domain_length = dxrect*cm_to_m*nx_global - period = c12*days_to_s ! 12 days rotational period + period = c12*secday ! 12 days rotational period max_vel = pi*domain_length/period uvel(i,j) = c2*max_vel*(real(jglob(j), kind=dbl_kind) - p5) & diff --git a/cicecore/cicedynB/general/ice_step_mod.F90 b/cicecore/cicedynB/general/ice_step_mod.F90 index fb759cb27..4bbf0aa3b 100644 --- a/cicecore/cicedynB/general/ice_step_mod.F90 +++ b/cicecore/cicedynB/general/ice_step_mod.F90 @@ -947,8 +947,6 @@ subroutine ocean_mixed_layer (dt, iblk) use ice_arrays_column, only: Cdn_atm, Cdn_atm_ratio use ice_blocks, only: nx_block, ny_block - use ice_blocks, only: block, get_block - use ice_domain, only: blocks_ice use ice_flux, only: sst, Tf, Qa, uatm, vatm, wind, potT, rhoa, zlvl, & frzmlt, fhocn, fswthru, flw, flwout_ocn, fsens_ocn, flat_ocn, evap_ocn, & alvdr_ocn, alidr_ocn, alvdf_ocn, alidf_ocn, swidf, swvdf, swidr, swvdr, & @@ -970,7 +968,6 @@ subroutine ocean_mixed_layer (dt, iblk) frzmlt_max = c1000 ! max magnitude of frzmlt (W/m^2) integer (kind=int_kind) :: & - ilo,ihi,jlo,jhi, & ! beginning and end of physical domain i, j , & ! horizontal indices ij ! combined ij index @@ -986,9 +983,6 @@ subroutine ocean_mixed_layer (dt, iblk) integer (kind=int_kind), dimension(nx_block*ny_block) :: & indxi, indxj ! compressed indices for ocean cells - type (block) :: & - this_block ! block information for current block - character(len=*), parameter :: subname = '(ocn_mixed_layer)' !----------------------------------------------------------------- diff --git a/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90 b/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90 index afdfa2dfd..e9eb49db0 100644 --- a/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90 +++ b/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90 @@ -2820,7 +2820,6 @@ subroutine scatter_global_ext(ARRAY, ARRAY_G, src_task, dst_dist) iblk, jblk, &! block indices iglb, jglb, &! global indices nrecvs, &! actual number of messages received - isrc, jsrc, &! source addresses dst_block, &! location of block in dst array ierr ! MPI error flag diff --git a/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 b/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 index ab1c5969a..8f2992eb4 100644 --- a/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 +++ b/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90 @@ -60,7 +60,7 @@ MODULE ice_reprosum !- include statements -------------------------------------------------- !----------------------------------------------------------------------- #ifndef SERIAL_REMOVE_MPI -#include +include 'mpif.h' #endif !----------------------------------------------------------------------- diff --git a/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90 b/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90 index 9775a9dad..9c2cfd9fc 100644 --- a/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90 +++ b/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90 @@ -3434,8 +3434,7 @@ subroutine ice_HaloUpdate_stress(array1, array2, halo, & ! local variables integer (int_kind) :: & - i,j,n,nmsg, &! dummy loop indices - ierr, &! error or status flag for MPI,alloc + nmsg, &! dummy loop indices nxGlobal, &! global domain size in x (tripole) iSrc,jSrc, &! source addresses for message iDst,jDst, &! dest addresses for message diff --git a/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 b/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 index 490b16b14..d52e58add 100644 --- a/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 +++ b/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90 @@ -61,7 +61,7 @@ MODULE ice_reprosum !- include statements -------------------------------------------------- !----------------------------------------------------------------------- #ifndef SERIAL_REMOVE_MPI -#include +include 'mpif.h' #endif !----------------------------------------------------------------------- diff --git a/cicecore/cicedynB/infrastructure/ice_domain.F90 b/cicecore/cicedynB/infrastructure/ice_domain.F90 index b4b4c4ab2..ae20e5855 100644 --- a/cicecore/cicedynB/infrastructure/ice_domain.F90 +++ b/cicecore/cicedynB/infrastructure/ice_domain.F90 @@ -300,9 +300,11 @@ subroutine init_domain_distribution(KMTG,ULATG) i,j,n ,&! dummy loop indices ig,jg ,&! global indices work_unit ,&! size of quantized work unit +#ifdef ncdf fid ,&! file id varid ,&! var id status ,&! netcdf return code +#endif tblocks_tmp ,&! total number of blocks nblocks_tmp ,&! temporary value of nblocks nblocks_max ! max blocks on proc @@ -478,12 +480,12 @@ subroutine init_domain_distribution(KMTG,ULATG) if (this_block%j_glob(j) > 0) then do i=this_block%ilo,this_block%ihi if (this_block%i_glob(i) > 0) then - ig = this_block%i_glob(i) + ig = this_block%i_glob(i) jg = this_block%j_glob(j) if (KMTG(ig,jg) > puny .and. & (ULATG(ig,jg) < shlat/rad_to_deg .or. & ULATG(ig,jg) > nhlat/rad_to_deg) ) & - nocn(n) = nocn(n) + flat(ig,jg) + nocn(n) = nocn(n) + flat(ig,jg) endif end do endif diff --git a/cicecore/cicedynB/infrastructure/ice_read_write.F90 b/cicecore/cicedynB/infrastructure/ice_read_write.F90 index ff05f4fe3..22d3f8260 100644 --- a/cicecore/cicedynB/infrastructure/ice_read_write.F90 +++ b/cicecore/cicedynB/infrastructure/ice_read_write.F90 @@ -1516,12 +1516,10 @@ subroutine ice_read_nc_z(fid, nrec, varname, work, diag, & ! local variables +#ifdef ncdf real (kind=dbl_kind), dimension(:), allocatable :: & - work_z - - character(len=*), parameter :: subname = '(ice_read_nc_z)' + work_z -#ifdef ncdf ! netCDF file diagnostics: integer (kind=int_kind) :: & varid, & ! netcdf id for field @@ -1532,6 +1530,11 @@ subroutine ice_read_nc_z(fid, nrec, varname, work, diag, & character (char_len) :: & dimname ! dimension name +#endif + + character(len=*), parameter :: subname = '(ice_read_nc_z)' + +#ifdef ncdf allocate(work_z(nilyr)) @@ -1625,8 +1628,8 @@ subroutine ice_write_nc_xy(fid, nrec, varid, work, diag, & amin, amax, asum ! min, max values and sum of input array character (char_len) :: & - lvarname, & ! variable name - dimname ! dimension name + lvarname ! variable name +! dimname ! dimension name real (kind=dbl_kind), dimension(:,:), allocatable :: & work_g1 @@ -1745,8 +1748,8 @@ subroutine ice_write_nc_xyz(fid, nrec, varid, work, diag, & amin, amax, asum ! min, max values and sum of input array character (char_len) :: & - lvarname, & ! variable name - dimname ! dimension name + lvarname ! variable name +! dimname ! dimension name real (kind=dbl_kind), dimension(:,:,:), allocatable :: & work_g1 @@ -2136,15 +2139,11 @@ subroutine ice_read_vec_nc (fid, nrec, varname, work_g, diag) ! netCDF file diagnostics: integer (kind=int_kind) :: & varid, & ! netcdf id for field - status, & ! status output from netcdf routines - nvar ! sizes of netcdf vector + status ! status output from netcdf routines real (kind=dbl_kind) :: & amin, amax ! min, max values of input vector - character (char_len) :: & - dimname ! dimension name -! work_g(:) = c0 if (my_task == master_task) then @@ -2194,10 +2193,15 @@ subroutine ice_get_ncvarsize(fid,varname,recsize) varname ! field name in netcdf file integer (kind=int_kind), intent(out) :: & recsize ! Number of records in file + + ! local variables + +#ifdef ncdf integer (kind=int_kind) :: & ndims, i, status character (char_len) :: & cvar +#endif character(len=*), parameter :: subname = '(ice_get_ncvarsize)' #ifdef ncdf diff --git a/cicecore/cicedynB/infrastructure/ice_restoring.F90 b/cicecore/cicedynB/infrastructure/ice_restoring.F90 index 799b263d6..a7851ef07 100644 --- a/cicecore/cicedynB/infrastructure/ice_restoring.F90 +++ b/cicecore/cicedynB/infrastructure/ice_restoring.F90 @@ -57,7 +57,7 @@ subroutine ice_HaloRestore_init use ice_domain, only: ew_boundary_type, ns_boundary_type, & nblocks, blocks_ice use ice_grid, only: tmask, hm - use ice_flux, only: sst, Tf, Tair, salinz, Tmltz + use ice_flux, only: Tf, Tair, salinz, Tmltz use ice_restart_shared, only: restart_ext integer (int_kind) :: & diff --git a/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90 b/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90 index 73540f5a9..d1564fab4 100644 --- a/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90 @@ -56,7 +56,7 @@ subroutine ice_write_hist(ns) ! local variables - integer (kind=int_kind) :: i,k,n,nn,nrec,nbits + integer (kind=int_kind) :: k,n,nn,nrec,nbits character (char_len) :: title character (char_len_long) :: ncfile(max_nstrm), hdrfile diff --git a/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90 b/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90 index a9aaef525..4d13540e5 100644 --- a/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90 @@ -636,12 +636,10 @@ subroutine read_restart_field(nu,nrec,work,atype,vname,ndim3, & ! local variables integer (kind=int_kind) :: & - n, & ! number of dimensions for variable - varid, & ! variable id - status ! status variable from netCDF routine + n ! number of dimensions for variable real (kind=dbl_kind), dimension(nx_block,ny_block,max_blocks) :: & - work2 ! input array (real, 8-byte) + work2 ! input array (real, 8-byte) character(len=*), parameter :: subname = '(read_restart_field)' @@ -699,12 +697,10 @@ subroutine write_restart_field(nu,nrec,work,atype,vname,ndim3,diag) ! local variables integer (kind=int_kind) :: & - n, & ! dimension counter - varid, & ! variable id - status ! status variable from netCDF routine + n ! dimension counter real (kind=dbl_kind), dimension(nx_block,ny_block,max_blocks) :: & - work2 ! input array (real, 8-byte) + work2 ! input array (real, 8-byte) character(len=*), parameter :: subname = '(write_restart_field)' @@ -735,8 +731,7 @@ subroutine final_restart() tr_pond_topo, tr_pond_lvl, tr_brine integer (kind=int_kind) :: & - nbtrcr, & ! number of bgc tracers - status + nbtrcr ! number of bgc tracers character(len=*), parameter :: subname = '(final_restart)' diff --git a/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 b/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 index 64cde26cf..29efa67f9 100644 --- a/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90 @@ -134,9 +134,9 @@ subroutine init_restart_write(filename_spec) tr_bgc_hum integer (kind=int_kind) :: & - k, n, & ! index + k, n, & ! index nx, ny, & ! global array size - iyear, imonth, iday, & ! year, month, day + iyear, & ! year nbtrcr ! number of bgc tracers character(len=char_len_long) :: filename @@ -149,7 +149,6 @@ subroutine init_restart_write(filename_spec) dimid_ncat, & ! iflag, & ! netCDF creation flag status ! status variable from netCDF routine - integer (kind=int_kind) :: oldMode character (len=3) :: nchar, ncharb @@ -602,9 +601,6 @@ subroutine init_restart_write(filename_spec) endif !z_tracers deallocate(dims) -! From Denise Worthen for theia 11 Oct 2018: - !Prevent prefilling of arrays with _FillValue - status = nf90_set_fill(ncid, nf90_nofill, oldMode) status = nf90_enddef(ncid) write(nu_diag,*) 'Writing ',filename(1:lenstr(filename)) @@ -647,11 +643,6 @@ subroutine read_restart_field(nu,nrec,work,atype,vname,ndim3, & ! local variables - integer (kind=int_kind) :: & - n, & ! number of dimensions for variable - varid, & ! variable id - status ! status variable from netCDF routine - real (kind=dbl_kind), dimension(nx_block,ny_block,max_blocks) :: & work2 ! input array (real, 8-byte) diff --git a/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 b/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 index ff3d2d6fd..43e5dab74 100644 --- a/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 +++ b/cicecore/cicedynB/infrastructure/io/io_pio/ice_history_write.F90 @@ -57,7 +57,7 @@ subroutine ice_write_hist (ns) use ice_restart_shared, only: runid, lcdf64 use netcdf #endif - use ice_pio + use ice_pio use pio integer (kind=int_kind), intent(in) :: ns @@ -161,7 +161,7 @@ subroutine ice_write_hist (ns) File%fh=-1 call ice_pio_init(mode='write', filename=trim(filename), File=File, & - clobber=.true., cdf64=lcdf64) + clobber=.true., cdf64=lcdf64) call ice_pio_initdecomp(iodesc=iodesc2d) call ice_pio_initdecomp(ndim3=ncat_hist, iodesc=iodesc3dc) @@ -349,7 +349,7 @@ subroutine ice_write_hist (ns) dimidex(4)=kmtidb dimidex(5)=kmtida - do i = 1, nvarz + do i = 1, nvarz if (igrdz(i)) then status = pio_def_var(File, trim(var_nz(i)%short_name), pio_real, & (/dimidex(i)/), varid) @@ -782,7 +782,7 @@ subroutine ice_write_hist (ns) ! Extra dimensions (NCAT, VGRD*) - do i = 1, nvarz + do i = 1, nvarz if (igrdz(i)) then status = pio_inq_varid(File, var_nz(i)%short_name, varid) SELECT CASE (var_nz(i)%short_name) diff --git a/cicecore/drivers/cesm/ice_comp_esmf.F90 b/cicecore/drivers/cesm/ice_comp_esmf.F90 index 8f682ebe5..8ae80abdc 100644 --- a/cicecore/drivers/cesm/ice_comp_esmf.F90 +++ b/cicecore/drivers/cesm/ice_comp_esmf.F90 @@ -16,7 +16,7 @@ module ice_comp_esmf use shr_sys_mod, only : shr_sys_abort, shr_sys_flush ! use shr_mem_mod, only : shr_get_memusage, shr_init_memusage use shr_file_mod, only : shr_file_getlogunit, shr_file_getloglevel, & - shr_file_setloglevel, shr_file_setlogunit + shr_file_setloglevel, shr_file_setlogunit use mct_mod #ifdef USE_ESMF_LIB use esmf @@ -26,10 +26,10 @@ module ice_comp_esmf use seq_flds_mod use seq_infodata_mod,only : seq_infodata_start_type_cont, & - seq_infodata_start_type_brnch, seq_infodata_start_type_start + seq_infodata_start_type_brnch, seq_infodata_start_type_start use seq_timemgr_mod, only : seq_timemgr_eclockgetdata, & seq_timemgr_restartalarmison, & - seq_timemgr_eclockdateinsync, & + seq_timemgr_eclockdateinsync, & seq_timemgr_stopalarmison use seq_comm_mct, only : seq_comm_suffix, seq_comm_inst, seq_comm_name use perf_mod, only : t_startf, t_stopf, t_barrierf @@ -44,12 +44,12 @@ module ice_comp_esmf use ice_domain, only : nblocks, blocks_ice, halo_info, distrb_info use ice_blocks, only : block, get_block, nx_block, ny_block use ice_grid, only : tlon, tlat, tarea, tmask, anglet, hm, & - grid_type, t2ugrid_vector, gridcpl_file, ocn_gridcell_frac + grid_type, t2ugrid_vector, gridcpl_file, ocn_gridcell_frac use ice_constants, only : c0, c1, spval_dbl, rad_to_deg, radius, secday use ice_communicate, only : my_task, master_task, MPI_COMM_ICE use ice_calendar, only : istep, istep1, force_restart_now, write_ic,& idate, idate0, mday, time, month, daycal, & - sec, dt, dt_dyn, calendar, & + sec, dt, dt_dyn, calendar, & calendar_type, nextsw_cday, days_per_year, & nyr, new_year, time2sec, year_init use icepack_orbital, only : eccen, obliqr, lambm0, mvelpp @@ -694,7 +694,7 @@ subroutine ice_run_esmf(comp, import_state, export_state, EClock, rc) force_restart_now = seq_timemgr_RestartAlarmIsOn(EClock) - if (calendar_type .eq. "GREGORIAN") then + if (calendar_type .eq. "GREGORIAN") then nyrp = nyr nyr = (curr_ymd/10000)+1 ! integer year of basedate if (nyr /= nyrp) then diff --git a/cicecore/drivers/cesm/ice_comp_mct.F90 b/cicecore/drivers/cesm/ice_comp_mct.F90 index da86f91be..7162d6397 100644 --- a/cicecore/drivers/cesm/ice_comp_mct.F90 +++ b/cicecore/drivers/cesm/ice_comp_mct.F90 @@ -14,7 +14,7 @@ module ice_comp_mct use shr_sys_mod, only : shr_sys_abort, shr_sys_flush ! use shr_mem_mod, only : shr_get_memusage, shr_init_memusage use shr_file_mod, only : shr_file_getlogunit, shr_file_getloglevel, & - shr_file_setloglevel, shr_file_setlogunit + shr_file_setloglevel, shr_file_setlogunit use shr_const_mod use mct_mod #ifdef USE_ESMF_LIB @@ -26,11 +26,11 @@ module ice_comp_mct use seq_flds_mod use seq_cdata_mod, only : seq_cdata, seq_cdata_setptrs use seq_infodata_mod,only : seq_infodata_type, seq_infodata_getdata, & - seq_infodata_putdata, seq_infodata_start_type_cont, & - seq_infodata_start_type_brnch, seq_infodata_start_type_start + seq_infodata_putdata, seq_infodata_start_type_cont, & + seq_infodata_start_type_brnch, seq_infodata_start_type_start use seq_timemgr_mod, only : seq_timemgr_eclockgetdata, & seq_timemgr_restartalarmison, & - seq_timemgr_eclockdateinsync, & + seq_timemgr_eclockdateinsync, & seq_timemgr_stopalarmison use seq_comm_mct, only : seq_comm_suffix, seq_comm_inst, seq_comm_name use perf_mod, only : t_startf, t_stopf, t_barrierf @@ -42,13 +42,13 @@ module ice_comp_mct use ice_domain, only : nblocks, blocks_ice, halo_info, distrb_info use ice_blocks, only : block, get_block, nx_block, ny_block use ice_grid, only : tlon, tlat, tarea, tmask, anglet, hm, & - grid_type, t2ugrid_vector, gridcpl_file, ocn_gridcell_frac + grid_type, t2ugrid_vector, gridcpl_file, ocn_gridcell_frac use ice_constants, only : c0, c1, spval_dbl, radius use ice_constants, only : ice_init_constants use ice_communicate, only : my_task, master_task, MPI_COMM_ICE use ice_calendar, only : istep, istep1, force_restart_now, write_ic,& idate, idate0, mday, time, month, daycal, & - sec, dt, dt_dyn, calendar, & + sec, dt, dt_dyn, calendar, & calendar_type, nextsw_cday, days_per_year, & nyr, new_year, time2sec, year_init use ice_timers @@ -579,7 +579,7 @@ subroutine ice_run_mct( EClock, cdata_i, x2i_i, i2x_i ) force_restart_now = seq_timemgr_RestartAlarmIsOn(EClock) -! if (calendar_type .eq. "GREGORIAN") then +! if (calendar_type .eq. "GREGORIAN") then ! nyrp = nyr ! nyr = (curr_ymd/10000)+1 ! integer year of basedate ! if (nyr /= nyrp) then diff --git a/cicecore/drivers/cesm/ice_prescribed_mod.F90 b/cicecore/drivers/cesm/ice_prescribed_mod.F90 index 10c71cc84..eeb217577 100644 --- a/cicecore/drivers/cesm/ice_prescribed_mod.F90 +++ b/cicecore/drivers/cesm/ice_prescribed_mod.F90 @@ -138,7 +138,7 @@ subroutine ice_prescribed_init(compid, gsmap, dom) namelist /ice_prescribed_nml/ & prescribed_ice, & model_year_align, & - stream_year_first , & + stream_year_first , & stream_year_last , & stream_fldVarName , & stream_fldFileName, & diff --git a/cicecore/drivers/cesm/ice_scam.F90 b/cicecore/drivers/cesm/ice_scam.F90 index 86a56d19c..f5280b259 100644 --- a/cicecore/drivers/cesm/ice_scam.F90 +++ b/cicecore/drivers/cesm/ice_scam.F90 @@ -12,4 +12,3 @@ module ice_scam end module ice_scam - diff --git a/cicecore/drivers/cice/CICE_InitMod.F90 b/cicecore/drivers/cice/CICE_InitMod.F90 index 1acbcb873..ddc966b55 100644 --- a/cicecore/drivers/cice/CICE_InitMod.F90 +++ b/cicecore/drivers/cice/CICE_InitMod.F90 @@ -72,12 +72,12 @@ subroutine cice_init use ice_flux, only: init_coupler_flux, init_history_therm, & init_history_dyn, init_flux_atm, init_flux_ocn, alloc_flux use ice_forcing, only: init_forcing_ocn, init_forcing_atmo, & - get_forcing_atmo, get_forcing_ocn, alloc_forcing + get_forcing_atmo, get_forcing_ocn use ice_forcing_bgc, only: get_forcing_bgc, get_atm_bgc, & faero_default, faero_optics, alloc_forcing_bgc use ice_grid, only: init_grid1, init_grid2, alloc_grid use ice_history, only: init_hist, accum_hist - use ice_restart_shared, only: restart, runid, runtype + use ice_restart_shared, only: restart, runtype use ice_init, only: input_data, init_state use ice_init_column, only: init_thermo_vertical, init_shortwave, init_zbgc, input_zbgc, count_tracers use ice_kinds_mod diff --git a/cicecore/drivers/cice/CICE_RunMod.F90 b/cicecore/drivers/cice/CICE_RunMod.F90 index a9f5a9707..af3e681b9 100644 --- a/cicecore/drivers/cice/CICE_RunMod.F90 +++ b/cicecore/drivers/cice/CICE_RunMod.F90 @@ -44,7 +44,7 @@ module CICE_RunMod subroutine CICE_Run use ice_calendar, only: istep, istep1, time, dt, stop_now, calendar - use ice_forcing, only: get_forcing_atmo, get_forcing_ocn, atm_data_type + use ice_forcing, only: get_forcing_atmo, get_forcing_ocn use ice_forcing_bgc, only: get_forcing_bgc, get_atm_bgc, & faero_default use ice_flux, only: init_flux_atm, init_flux_ocn diff --git a/cicecore/shared/ice_arrays_column.F90 b/cicecore/shared/ice_arrays_column.F90 index 71d03ed94..e6917f51f 100644 --- a/cicecore/shared/ice_arrays_column.F90 +++ b/cicecore/shared/ice_arrays_column.F90 @@ -268,7 +268,7 @@ module ice_arrays_column real (kind=dbl_kind), dimension(:), allocatable, public :: & R_C2N , & ! algal C to N (mole/mole) R_chl2N , & ! 3 algal chlorophyll to N (mg/mmol) - R_Si2N ! silica to nitrogen mole ratio for algal groups + R_Si2N ! silica to nitrogen mole ratio for algal groups !======================================================================= @@ -279,13 +279,12 @@ module ice_arrays_column subroutine alloc_arrays_column ! Allocate column arrays use ice_exit, only: abort_ice - integer (int_kind) :: nspint, max_nbtrcr, max_algae, max_aero, & + integer (int_kind) :: max_nbtrcr, max_algae, max_aero, & nmodal1, nmodal2, max_don integer (int_kind) :: ierr, ntrcr character(len=*),parameter :: subname='(alloc_arrays_column)' -! call icepack_query_parameters(nspint_out=nspint) call icepack_query_tracer_numbers(ntrcr_out=ntrcr) call icepack_query_tracer_sizes( max_nbtrcr_out=max_nbtrcr, & max_algae_out=max_algae, max_aero_out=max_aero, & diff --git a/cicecore/shared/ice_calendar.F90 b/cicecore/shared/ice_calendar.F90 index a767bdfd7..e7107f42a 100644 --- a/cicecore/shared/ice_calendar.F90 +++ b/cicecore/shared/ice_calendar.F90 @@ -54,10 +54,10 @@ module ice_calendar data daycal366/ 0,31, 60, 91,121,152,182,213,244,274,305,335,366/ real (kind=dbl_kind), parameter :: & - days_per_4c = 146097.0_dbl_kind, & - days_per_c = 36524.0_dbl_kind, & - days_per_4y = 1461.0_dbl_kind, & - days_per_y = 365.0_dbl_kind + days_per_4c = 146097.0_dbl_kind, & + days_per_c = 36524.0_dbl_kind, & + days_per_4y = 1461.0_dbl_kind, & + days_per_y = 365.0_dbl_kind integer (kind=int_kind), public :: & istep , & ! local step counter for time loop diff --git a/cicecore/shared/ice_constants.F90 b/cicecore/shared/ice_constants.F90 index c49732e35..15581e787 100644 --- a/cicecore/shared/ice_constants.F90 +++ b/cicecore/shared/ice_constants.F90 @@ -60,7 +60,7 @@ module ice_constants c180 = 180.0_dbl_kind, & c360 = 360.0_dbl_kind, & c365 = 365.0_dbl_kind, & - c400 = 400.0_dbl_kind, & + c400 = 400.0_dbl_kind, & c1000= 1000.0_dbl_kind, & c3600= 3600.0_dbl_kind, & p001 = 0.001_dbl_kind, & diff --git a/cicecore/shared/ice_init_column.F90 b/cicecore/shared/ice_init_column.F90 index 3f3a5bfe6..f86984195 100644 --- a/cicecore/shared/ice_init_column.F90 +++ b/cicecore/shared/ice_init_column.F90 @@ -444,11 +444,11 @@ subroutine init_shortwave !---------------------------------------------------------------- if (runtype == 'initial' .and. .not. restart) then scale_factor(i,j,iblk) = & - swvdr(i,j,iblk)*(c1 - alvdr_ai(i,j,iblk)) & - + swvdf(i,j,iblk)*(c1 - alvdf_ai(i,j,iblk)) & - + swidr(i,j,iblk)*(c1 - alidr_ai(i,j,iblk)) & - + swidf(i,j,iblk)*(c1 - alidf_ai(i,j,iblk)) - endif + swvdr(i,j,iblk)*(c1 - alvdr_ai(i,j,iblk)) & + + swvdf(i,j,iblk)*(c1 - alvdf_ai(i,j,iblk)) & + + swidr(i,j,iblk)*(c1 - alidr_ai(i,j,iblk)) & + + swidf(i,j,iblk)*(c1 - alidf_ai(i,j,iblk)) + endif enddo ! i enddo ! j @@ -895,8 +895,7 @@ subroutine input_zbgc integer (kind=int_kind) :: & nml_error, & ! namelist i/o error flag - k, mm , & ! loop index - ierr, abort_flag + abort_flag character(len=*), parameter :: subname='(input_zbgc)' @@ -1626,7 +1625,6 @@ subroutine count_tracers ! local variables integer (kind=int_kind) :: & - n , & ! loop index k, mm , & ! loop index nk , & ! layer index nk_bgc ! layer index @@ -2116,7 +2114,6 @@ end subroutine count_tracers subroutine init_zbgc - use ice_broadcast, only: broadcast_scalar use ice_state, only: trcr_base, trcr_depend, n_trcr_strata, & nt_strata use ice_arrays_column, only: R_C2N, R_chl2N, R_C2N_DON, R_Si2N @@ -2186,7 +2183,7 @@ subroutine init_zbgc tr_bgc_DMS, tr_bgc_PON, & tr_bgc_N, tr_bgc_C, tr_bgc_chl, & tr_bgc_DON, tr_bgc_Fe, tr_zaero, & - tr_bgc_hum, tr_aero + tr_bgc_hum real (kind=dbl_kind) :: & initbio_frac, & diff --git a/cicecore/shared/ice_restart_column.F90 b/cicecore/shared/ice_restart_column.F90 index 63d095f11..b46e6fdcd 100644 --- a/cicecore/shared/ice_restart_column.F90 +++ b/cicecore/shared/ice_restart_column.F90 @@ -786,9 +786,6 @@ subroutine write_restart_bgc() type (block) :: & this_block ! block information for current block - integer (kind=int_kind) :: & - ipoint - character(len=*),parameter :: subname='(write_restart_bgc)' call icepack_query_parameters(skl_bgc_out=skl_bgc, solve_zsal_out=solve_zsal) diff --git a/cicecore/shared/ice_spacecurve.F90 b/cicecore/shared/ice_spacecurve.F90 index 78b256b8f..ed108e1c7 100644 --- a/cicecore/shared/ice_spacecurve.F90 +++ b/cicecore/shared/ice_spacecurve.F90 @@ -31,7 +31,7 @@ module ice_spacecurve ! !PUBLIC MEMBER FUNCTIONS: public :: GenSpaceCurve, & - IsLoadBalanced + IsLoadBalanced public :: Factor, & IsFactorable, & @@ -41,24 +41,24 @@ module ice_spacecurve ! !PRIVATE MEMBER FUNCTIONS: - private :: map, & - PeanoM, & - Hilbert, & - Cinco, & + private :: map, & + PeanoM, & + Hilbert, & + Cinco, & GenCurve private :: FirstFactor, & FindandMark integer(int_kind), dimension(:,:), allocatable :: & - dir, &! direction to move along each level + dir, &! direction to move along each level ordered ! the ordering integer(int_kind), dimension(:), allocatable :: & - pos ! position along each of the axes + pos ! position along each of the axes integer(int_kind) :: & - maxdim, &! dimensionality of entire space - vcnt ! visitation count + maxdim, &! dimensionality of entire space + vcnt ! visitation count logical :: verbose=.FALSE. @@ -80,7 +80,7 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) ! !DESCRIPTION: ! This subroutine implements a Cinco space-filling curve. ! Cinco curves connect a Nb x Nb block of points where -! +! ! Nb = 5^p ! ! !REVISION HISTORY: @@ -91,12 +91,12 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) ! !INPUT PARAMETERS integer(int_kind), intent(in) :: & - l, & ! level of the space-filling curve + l, & ! level of the space-filling curve type, & ! type of SFC curve - ma, & ! Major axis [0,1] - md, & ! direction of major axis [-1,1] - ja, & ! joiner axis [0,1] - jd ! direction of joiner axis [-1,1] + ma, & ! Major axis [0,1] + md, & ! direction of major axis [-1,1] + ja, & ! joiner axis [0,1] + jd ! direction of joiner axis [-1,1] ! !OUTPUT PARAMETERS @@ -111,12 +111,12 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) !----------------------------------------------------------------------- integer(int_kind) :: & - lma, &! local major axis (next level) - lmd, &! local major direction (next level) - lja, &! local joiner axis (next level) - ljd, &! local joiner direction (next level) - ltype, &! type of SFC on next level - ll ! next level down + lma, &! local major axis (next level) + lmd, &! local major direction (next level) + lja, &! local joiner axis (next level) + ljd, &! local joiner direction (next level) + ltype, &! type of SFC on next level + ll ! next level down logical :: debug = .FALSE. @@ -963,8 +963,8 @@ function IncrementCurve(ja,jd) result(ierr) ! !INPUT PARAMETERS: integer(int_kind) :: & - ja, &! axis to increment - jd ! direction along axis + ja, &! axis to increment + jd ! direction along axis ! !OUTPUT PARAMETERS: integer(int_kind) :: ierr ! error return code @@ -975,7 +975,7 @@ function IncrementCurve(ja,jd) result(ierr) ! mark the newly visited point !----------------------------- ordered(pos(0)+1,pos(1)+1) = vcnt - + !------------------------------------ ! increment curve and update position !------------------------------------ @@ -1064,12 +1064,12 @@ function IsLoadBalanced(nelem,npart) ! !INTPUT PARAMETERS: integer(int_kind), intent(in) :: & - nelem, & ! number of blocks/elements to partition - npart ! size of partition + nelem, & ! number of blocks/elements to partition + npart ! size of partition ! !OUTPUT PARAMETERS: logical :: IsLoadBalanced ! .TRUE. if a perfectly load balanced - ! partition is possible + ! partition is possible !EOP !BOC !----------------------------------------------------------------------- @@ -1077,7 +1077,7 @@ function IsLoadBalanced(nelem,npart) ! local variables ! !----------------------------------------------------------------------- - + integer(int_kind) :: tmp1 ! temporary int character(len=*),parameter :: subname='(IsLoadBalanced)' @@ -1085,10 +1085,10 @@ function IsLoadBalanced(nelem,npart) !----------------------------------------------------------------------- tmp1 = nelem/npart - if (npart*tmp1 == nelem ) then - IsLoadBalanced=.TRUE. + if(npart*tmp1 == nelem ) then + IsLoadBalanced=.TRUE. else - IsLoadBalanced=.FALSE. + IsLoadBalanced=.FALSE. endif !EOP @@ -1285,7 +1285,7 @@ function Factor(num) result(res) !----------------------------------------------------------------------- integer(int_kind) :: & - tmp,tmp2,tmp3,tmp5 ! tempories for the factorization algorithm + tmp,tmp2,tmp3,tmp5 ! tempories for the factorization algorithm integer(int_kind) :: i,n ! loop tempories logical :: found ! logical temporary character(len=*),parameter :: subname='(Factor)' @@ -1437,9 +1437,9 @@ subroutine map(l) !----------------------------------------------------------------------- integer(int_kind) :: & - d, & ! dimension of curve only 2D is supported - type, & ! type of space-filling curve to start off - ierr ! error return code + d, & ! dimension of curve only 2D is supported + type, & ! type of space-filling curve to start off + ierr ! error return code character(len=*),parameter :: subname='(map)' d = SIZE(pos) @@ -1484,8 +1484,8 @@ subroutine PrintCurve(Mesh) ! !----------------------------------------------------------------------- integer(int_kind) :: & - gridsize, &! order of space-filling curve - i ! loop temporary + gridsize, &! order of space-filling curve + i ! loop temporary character(len=*),parameter :: subname='(PrintCurve)' !----------------------------------------------------------------------- @@ -1523,7 +1523,7 @@ subroutine PrintCurve(Mesh) write (*,*) "------------------------------------------" do i=1,gridsize write(*,6) Mesh(1,i),Mesh(2,i),Mesh(3,i), & - Mesh(4,i),Mesh(5,i),Mesh(6,i) + Mesh(4,i),Mesh(5,i),Mesh(6,i) enddo else if(gridsize == 8) then write (*,*) "A Level 3 Hilbert Curve:" @@ -1615,7 +1615,7 @@ subroutine PrintCurve(Mesh) Mesh(13,i),Mesh(14,i),Mesh(15,i),Mesh(16,i), & Mesh(17,i),Mesh(18,i),Mesh(19,i),Mesh(20,i), & Mesh(21,i),Mesh(22,i),Mesh(23,i),Mesh(24,i), & - Mesh(25,i) + Mesh(25,i) enddo else if(gridsize == 27) then write (*,*) "A Level 3 Peano Meandering Curve:" @@ -1683,7 +1683,7 @@ subroutine GenSpaceCurve(Mesh) ! !INPUT/OUTPUT PARAMETERS: integer(int_kind), target,intent(inout) :: & - Mesh(:,:) ! The SFC ordering in 2D array + Mesh(:,:) ! The SFC ordering in 2D array !EOP !BOC @@ -1694,8 +1694,8 @@ subroutine GenSpaceCurve(Mesh) !----------------------------------------------------------------------- integer(int_kind) :: & - level, &! Level of space-filling curve - dim ! dimension of SFC... currently limited to 2D + level, &! Level of space-filling curve + dim ! dimension of SFC... currently limited to 2D integer(int_kind) :: gridsize ! number of points on a side diff --git a/cicecore/version.txt b/cicecore/version.txt index de34d9d31..c6a84bfc7 100644 --- a/cicecore/version.txt +++ b/cicecore/version.txt @@ -1 +1 @@ -CICE 6.0.1 +CICE 6.0.2 diff --git a/doc/source/conf.py b/doc/source/conf.py index f9b0b7b68..4404ecbb1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -62,9 +62,9 @@ # built documents. # # The short X.Y version. -version = u'6.0.1' +version = u'6.0.2' # The full version, including alpha/beta/rc tags. -version = u'6.0.1' +version = u'6.0.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/source/developer_guide/dg_scripts.rst b/doc/source/developer_guide/dg_scripts.rst index 185616eab..da5ef7d24 100755 --- a/doc/source/developer_guide/dg_scripts.rst +++ b/doc/source/developer_guide/dg_scripts.rst @@ -31,6 +31,8 @@ The directory structure under configure/scripts is as follows. | **parse_settings.sh** replaces settings with command-line configuration | **setup_run_dirs.csh** creates the case run directories | **set_version_number.csh** updates the model version number from the **cice.setup** command line +| **timeseries.csh** generates PNG timeseries plots from output files, using GNUPLOT +| **timeseries.py** generates PNG timeseries plots from output files, using Python | **tests/** scripts for configuring and running basic tests .. _dev_strategy: @@ -78,6 +80,25 @@ with appropriate names and syntax. The set_nml file syntax is the same as namel syntax and the set_env files are consistent with csh setenv syntax. See other files for examples of the syntax. +.. _build: + +Build Scripts +-------------- + +CICE uses GNU Make to build the model. There is a common **Makefile** for all machines. +Each machine provides a Macros file to define some Makefile variables +and and an env file to specify the modules/software stack for each compiler. +The machine is built by the cice.build script which invokes Make. +There is a special trap for circular dependencies in the cice.build script to +highlight this error when it occurs. + +The **cice.build** script has some additional features including the ability to +pass a Makefile target. This is documented in :ref:`cicebuild`. In addition, there +is a hidden feature in the **cice.build** script that allows for reuse of +executables. This is used by the test suites to significantly reduce cost of +building the model. It is invoked with the ``--exe`` argument to **cice.build** +and should not be invoked by users interactively. + .. _dev_machines: Machines @@ -122,9 +143,8 @@ setup the various tests, such as smoke and restart tests (**test_smoke.script**, and the files that describe with options files are needed for each test (ie. **test_smoke.files**, **test_restart.files**). A baseline test script (**baseline.script**) is also there to setup the general regression and comparison testing. That directory also contains the preset test suites -(ie. **base_suite.ts**) and a file that supports post-processing on the model -output (**timeseries.csh**). There is also a script **report_results.csh** that pushes results -from test suites back to the CICE-Consortium test results wiki page. +(ie. **base_suite.ts**) and a script (**report_results.csh**) that pushes results from +test suites back to the CICE-Consortium test results wiki page. To add a new test (for example newtest), several files may be needed, diff --git a/doc/source/intro/quickstart.rst b/doc/source/intro/quickstart.rst index deded30a5..547d6ef20 100755 --- a/doc/source/intro/quickstart.rst +++ b/doc/source/intro/quickstart.rst @@ -14,6 +14,8 @@ found in the `CICE Git and Workflow Guide `_ or :ref:`force`. +Software requirements are noted in this :ref:`software` section. + From your main CICE directory, execute:: ./cice.setup -c ~/mycase1 -g gx3 -m testmachine -s diag1,thread -p 8x1 diff --git a/doc/source/master_list.bib b/doc/source/master_list.bib index b80b36d41..11bb3e887 100644 --- a/doc/source/master_list.bib +++ b/doc/source/master_list.bib @@ -694,6 +694,14 @@ @Article{Hunke09 issue = {CB}, url = {http://dx.doi.org/10.1029/2008JC005186} } +@Article{Large09 + author = "W.G. Large and S.G. Yeager", + title = "{The global climatology of an interannually varying air-sea flux data set}", + journal = OM, + year = {2009}, + volume = {33}, + url = {http://dx.doi.org/10.1007/s00382-008-0441-3} +} @Article{Weiss09 author = "J. Weiss and E.M. Schulson", title = "{Coulombic faulting from the grain scale to the geophysical scale: lessons from ice}", @@ -920,6 +928,15 @@ @Article{Roberts18 volume = {376}, url = {http://dx.doi.org/10.1098/rsta.2017.0344} } +@Article{Tsujino18, + author = "H. Tsujino and S. Urakawa and R.J. Small and W.M. Kim and S.G. Yeager and et al.", + title = "{JRA‐55 based surface dataset for driving ocean–sea‐ice models (JRA55‐do)}", + journal = OM, + year = {2018}, + volume = {130}, + pages = {79-139}, + url = {http://dx.doi.org/10.1016/j.ocemod.2018.07.002} +} % ********************************************** % For new entries, see example entry in BIB_TEMPLATE.txt % ********************************************** diff --git a/doc/source/science_guide/sg_dynamics.rst b/doc/source/science_guide/sg_dynamics.rst index f5db205c6..8117fd58d 100644 --- a/doc/source/science_guide/sg_dynamics.rst +++ b/doc/source/science_guide/sg_dynamics.rst @@ -190,7 +190,11 @@ above hydrostatic balance and the value of :math:`k_2`. It is, however, the para The value of :math:`k_1` can be changed at runtime using the namelist variable ``k1``. The grounding scheme can be turned on or off using the namelist logical basalstress. Note that the user must provide a bathymetry field for using this grounding -scheme. Grounding occurs up to water depth of ~25 m. It is suggested to have a bathymetry field with water depths larger than 5 m that represents well shallow water regions such as the Laptev Sea and the East Siberian Sea. +scheme. It is suggested to have a bathymetry field with water depths larger than +5 m that represents well shallow water regions such as the Laptev Sea and the +East Siberian Sea. To prevent unrealistic grounding, :math:`T_b` is set to zero when :math:`h_{wu}` +is larger than 30 m. This maximum value is chosen based on observations of large +keels in the Arctic Ocean :cite:`Amundrud04`. .. _internal-stress: @@ -553,7 +557,7 @@ Introducing another numerical parameter :math:`\alpha=2T \Delta t_e ^{-1}` :cite where as opposed to the classic EVP, the second term in each equation is at iteration :math:`k` :cite:`Bouillon13`. Also, as opposed to the classic EVP, :math:`\Delta t_e` times the number of subcycles (or iterations) does not need to be equal to the advective time step :math:`\Delta t`. -A last difference between the classic EVP and the revised approach is that the latter one initializes the stresses to 0 at the beginning of each time step, -while the classic EVP approach uses the previous time level value. The revised EVP is activated by setting the namelist parameter `revised\_evp` = true. +Finally, as with the classic EVP approach, the stresses are initialized using the previous time level values. +The revised EVP is activated by setting the namelist parameter `revised\_evp` = true. In the code :math:`\alpha = arlx` and :math:`\beta = brlx`. The values of :math:`arlx` and :math:`brlx` can be set in the namelist. It is recommended to use large values of these parameters and to set :math:`arlx=brlx` :cite:`Kimmritz15`. diff --git a/doc/source/user_guide/ug_case_settings.rst b/doc/source/user_guide/ug_case_settings.rst index 8986bc9d8..c0ee9331d 100755 --- a/doc/source/user_guide/ug_case_settings.rst +++ b/doc/source/user_guide/ug_case_settings.rst @@ -337,7 +337,8 @@ Table of namelist options "\*","``trestore``", "integer", "sst restoring time scale (days)", "" "","``restore_ice``", "true/false", "restore ice state along lateral boundaries", "" "\*","``atm_data_type``", "``default``", "constant values defined in the code", "" - "","", "``LYq``", "AOMIP/Large-Yeager forcing data", "" + "","", "``LYq``", "COREII Large-Yeager (AOMIP) forcing data", ":cite:`Large09`" + "","", "``JRA55``", "JRA55 forcing data :cite:`Tsujino18`", "" "","", "``monthly``", "monthly forcing data", "" "","", "``ncar``", "NCAR bulk forcing data", "" "","", "``box2001``", "forcing data for :cite:`Hunke01` box problem", "" diff --git a/doc/source/user_guide/ug_implementation.rst b/doc/source/user_guide/ug_implementation.rst index e2a144ab3..eea7d8310 100644 --- a/doc/source/user_guide/ug_implementation.rst +++ b/doc/source/user_guide/ug_implementation.rst @@ -106,15 +106,15 @@ Big Endian files. In CESM, the sea ice model may exchange coupling fluxes using a different grid than the computational grid. This functionality is -activated using the namelist variable ``gridcpl\_file``. +activated using the namelist variable ``gridcpl_file``. *********************** Grid domains and blocks *********************** In general, the global gridded domain is -``nx\_global`` :math:`\times`\ ``ny\_global``, while the subdomains used in the -block distribution are ``nx\_block`` :math:`\times`\ ``ny\_block``. The +``nx_global`` :math:`\times`\ ``ny_global``, while the subdomains used in the +block distribution are ``nx_block`` :math:`\times`\ ``ny_block``. The physical portion of a subdomain is indexed as [``ilo:ihi``, ``jlo:jhi``], with nghost “ghost” or “halo" cells outside the domain used for boundary conditions. These parameters are illustrated in :ref:`fig-grid` in one @@ -141,14 +141,14 @@ Figure :ref:`fig-grid` shows the grid parameters for a sample one-dimensional, 2 global domain decomposed into four local subdomains. Each local domain has one ghost (halo) cell on each side, and the physical portion of the local domains are labeled ``ilo:ihi``. The parameter -``nx\_block`` is the total number of cells in the local domain, including +``nx_block`` is the total number of cells in the local domain, including ghost cells, and the same numbering system is applied to each of the four subdomains. The user sets the ``NTASKS`` and ``NTHRDS`` settings in **cice.settings** -and chooses a block size ``block\_size\_x`` :math:`\times`\ ``block\_size\_y``, -``max\_blocks``, and decomposition information ``distribution\_type``, ``processor\_shape``, -and ``distribution\_type`` in **ice\_in**. That information is used to +and chooses a block size ``block_size_x`` :math:`\times`\ ``block_size_y``, +``max_blocks``, and decomposition information ``distribution_type``, ``processor_shape``, +and ``distribution_type`` in **ice\_in**. That information is used to determine how the blocks are distributed across the processors, and how the processors are distributed across the grid domain. Recommended combinations of these @@ -159,8 +159,8 @@ but the user can overwrite the defaults by manually changing the values in information to the log file, and if the block size or max blocks is inconsistent with the task and thread size, the model will abort. The code will also print a warning if the maximum number of blocks is too large. -Although this is not fatal, it does use extra memory. If ``max\_blocks`` is -set to -1, the code will compute a ``max\_blocks`` on the fly. +Although this is not fatal, it does use extra memory. If ``max_blocks`` is +set to -1, the code will compute a ``max_blocks`` on the fly. A loop at the end of routine *create\_blocks* in module **ice\_blocks.F90** will print the locations for all of the blocks on @@ -173,10 +173,10 @@ manually set in the code in each case (independently of the dbug flag in **ice\_in**), as there may be hundreds or thousands of blocks to print and this information should be needed only rarely. This information is much easier to look at using a debugger such as Totalview. There is also -an output field that can be activated in `icefields\_nml`, ``f\_blkmask``, +an output field that can be activated in `icefields\_nml`, ``f_blkmask``, that prints out the variable ``blkmask`` to the history file and which labels the blocks in the grid decomposition according to ``blkmask = -my\_task + iblk/100``. +my_task + iblk/100``. ************* Tripole grids @@ -198,11 +198,11 @@ poles and the cells between them can be grid T cells, making a “T-fold.” Both of these options are also supported by the OPA/NEMO ocean model, which calls the U-fold an “f-fold” (because it uses the Arakawa C-grid in which U cells are on T-rows). The choice of tripole grid is given by -the namelist variable ``ns\_boundary\_type``, ‘tripole’ for the U-fold and +the namelist variable ``ns_boundary_type``, ‘tripole’ for the U-fold and ‘tripoleT’ for the T-fold grid. In the U-fold tripole grid, the poles have U-index -:math:`{\tt nx\_global}/2` and ``nx\_global`` on the top U-row of the +:math:`{\tt nx\_global}/2` and ``nx_global`` on the top U-row of the physical grid, and points with U-index i and :math:`{\tt nx\_global-i}` are coincident. Let the fold have U-row index :math:`n` on the global grid; this will also be the T-row index of the T-row to the south of the @@ -267,7 +267,7 @@ masked by land, periodic conditions wrap the domain around the globe. CICE can be run on regional grids with open boundary conditions; except for variables describing grid lengths, non-land halo cells along the grid edge must be filled by restoring them to specified values. The -namelist variable ``restore\_ice`` turns this functionality on and off; the +namelist variable ``restore_ice`` turns this functionality on and off; the restoring timescale ``trestore`` may be used (it is also used for restoring ocean sea surface temperature in stand-alone ice runs). This implementation is only intended to provide the “hooks" for a more @@ -279,7 +279,7 @@ allow Neumann boundary conditions, which must be set explicitly. This has been done in an unreleased branch of the code; contact Elizabeth for more information. -For exact restarts using restoring, set ``restart\_ext`` = true in namelist +For exact restarts using restoring, set ``restart_ext`` = true in namelist to use the extended-grid subroutines. On tripole grids, the order of operations used for calculating elements @@ -308,27 +308,27 @@ The logical masks ``tmask`` and ``umask`` (which correspond to the real masks In addition to the land masks, two other masks are implemented in *dyn\_prep* in order to reduce the dynamics component’s work on a global -grid. At each time step the logical masks ``ice\_tmask`` and ``ice\_umask`` are +grid. At each time step the logical masks ``ice_tmask`` and ``ice_umask`` are determined from the current ice extent, such that they have the value “true” wherever ice exists. They also include a border of cells around the ice pack for numerical purposes. These masks are used in the dynamics component to prevent unnecessary calculations on grid points where there is no ice. They are not used in the thermodynamics component, so that ice may form in previously ice-free cells. Like the -land masks ``hm`` and ``uvm``, the ice extent masks ``ice\_tmask`` and ``ice\_umask`` +land masks ``hm`` and ``uvm``, the ice extent masks ``ice_tmask`` and ``ice_umask`` are for T cells and U cells, respectively. Improved parallel performance may result from utilizing halo masks for boundary updates of the full ice state, incremental remapping transport, or for EVP or EAP dynamics. These options are accessed through the -logical namelist flags ``maskhalo\_bound``, ``maskhalo\_remap``, and -``maskhalo\_dyn``, respectively. Only the halo cells containing needed +logical namelist flags ``maskhalo_bound``, ``maskhalo_remap``, and +``maskhalo_dyn``, respectively. Only the halo cells containing needed information are communicated. -Two additional masks are created for the user’s convenience: ``lmask\_n`` -and ``lmask\_s`` can be used to compute or write data only for the northern +Two additional masks are created for the user’s convenience: ``lmask_n`` +and ``lmask_s`` can be used to compute or write data only for the northern or southern hemispheres, respectively. Special constants (``spval`` and -``spval\_dbl``, each equal to :math:`10^{30}`) are used to indicate land +``spval_dbl``, each equal to :math:`10^{30}`) are used to indicate land points in the history files and diagnostics. @@ -338,13 +338,13 @@ points in the history files and diagnostics. Performance *************** -Namelist options (*domain\_nml*) provide considerable flexibility for +Namelist options (*domain_nml*) provide considerable flexibility for finding efficient processor and block configuration. Some of these choices are illustrated in :ref:`fig-distrb`. Users have control -of many aspects of the decomposition such as the block size (``block\_size\_x``, -``block\_size\_y``), the ``distribution\_type``, the ``distribution\_wght``, -the ``distribution\_wght\_file`` (when ``distribution\_type`` = ``wghtfile``), -and the ``processor\_shape`` (when ``distribution\_type`` = ``cartesian``). +of many aspects of the decomposition such as the block size (``block_size_x``, +``block_size_y``), the ``distribution_type``, the ``distribution_wght``, +the ``distribution_wght_file`` (when ``distribution_type`` = ``wghtfile``), +and the ``processor_shape`` (when ``distribution_type`` = ``cartesian``). The user specifies the total number of tasks and threads in **cice.settings** and the block size and decompostion in the namelist file. The main trades @@ -361,7 +361,7 @@ volume-to-surface ratio important for communication cost. Often 3 to 8 blocks per processor provide the decompositions flexiblity to create reasonable load balance configurations. -The ``distribution\_type`` options allow standard cartesian distributions +The ``distribution_type`` options allow standard cartesian distributions of blocks, redistribution via a ‘rake’ algorithm for improved load balancing across processors, and redistribution based on space-filling curves. There are also additional distribution types @@ -395,7 +395,7 @@ Figure :ref:`fig-distrbB` shows sample decompositions for (a) spiral center and (b) wghtfile for an Arctic polar grid. (c) is the weight field in the input file use to drive the decompostion in (b). -``processor\_shape`` is used with the ``distribution\_type`` cartesian option, +``processor_shape`` is used with the ``distribution_type`` cartesian option, and it allocates blocks to processors in various groupings such as tall, thin processor domains (``slenderX1`` or ``slenderX2``, often better for sea ice simulations on global grids where nearly all of @@ -405,14 +405,14 @@ which maximize the volume to surface ratio (and therefore on-processor computations to message passing, if there were ice in every grid cell). In cases where the number of processors is not a perfect square (4, 9, 16...), the -``processor\_shape`` namelist variable allows the user to choose how the +``processor_shape`` namelist variable allows the user to choose how the processors are arranged. Here again, it is better in the sea ice model to have more processors in x than in y, for example, 8 processors arranged 4x2 (``square-ice``) rather than 2x4 (``square-pop``). The latter option is offered for direct-communication compatibility with POP, in which this is the default. -``distribution\_wght`` chooses how the work-per-block estimates are +``distribution_wght`` chooses how the work-per-block estimates are weighted. The ‘block’ option is the default in POP and it weights each block equally. This is useful in POP which always has work in each block and is written with a lot of @@ -422,7 +422,7 @@ direct-communication compatibility with POP. The ‘latitude’ option weights the blocks based on latitude and the number of ocean grid cells they contain. Many of the non-cartesian decompositions support automatic land block elimination and provide alternative ways to -decompose blocks without needing the ``distribution\_wght``. +decompose blocks without needing the ``distribution_wght``. The rake distribution type is initialized as a standard, Cartesian distribution. Using the work-per-block estimates, blocks are “raked" @@ -549,7 +549,7 @@ layers and the ice thickness distribution defined by ``kcatbound`` = 0. Restart information for some tracers is also included in the netCDF restart files. -Three namelist variables control model initialization, ``ice\_ic``, ``runtype``, +Three namelist variables control model initialization, ``ice_ic``, ``runtype``, and ``restart``, as described in :ref:`tab-ic`. It is possible to do an initial run from a file **filename** in two ways: (1) set runtype = ‘initial’, restart = true and ice\_ic = **filename**, or (2) runtype = @@ -562,7 +562,7 @@ true or false, depending on whether the tracer restart data exist. With the second option, tracer restart flags are set to ‘continue’ for all active tracers. -An additional namelist option, ``restart\_ext`` specifies whether halo cells +An additional namelist option, ``restart_ext`` specifies whether halo cells are included in the restart files. This option is useful for tripole and regional grids, but can not be used with PIO. @@ -577,8 +577,8 @@ her own routines. Whether the code is to be run in stand-alone or coupled mode is determined at compile time, as described below. Table :ref:`tab-ic` shows ice initial state resulting from combinations of -``ice\_ic``, ``runtype`` and ``restart``. :math:`^a`\ If false, restart is reset to -true. :math:`^b`\ restart is reset to false. :math:`^c`\ ice\_ic is +``ice_ic``, ``runtype`` and ``restart``. :math:`^a`\ If false, restart is reset to +true. :math:`^b`\ restart is reset to false. :math:`^c`\ ice_ic is reset to ‘none.’ .. _tab-ic: @@ -607,9 +607,9 @@ The time step is chosen based on stability of the transport component (both horizontal and in thickness space) and on resolution of the physical forcing. CICE allows the dynamics, advection and ridging portion of the code to be run with a shorter timestep, -:math:`\Delta t_{dyn}` (``dt\_dyn``), than the thermodynamics timestep +:math:`\Delta t_{dyn}` (``dt_dyn``), than the thermodynamics timestep :math:`\Delta t` (``dt``). In this case, ``dt`` and the integer ndtd are -specified, and ``dt\_dyn`` = ``dt/ndtd``. +specified, and ``dt_dyn`` = ``dt/ndtd``. A conservative estimate of the horizontal transport time step bound, or CFL condition, under remapping yields @@ -632,8 +632,8 @@ As discussed in :cite:`Lipscomb07`, the maximum time step in practice is usually determined by the time scale for large changes in the ice strength (which depends in part on wind strength). Using the strength parameterization of :cite:`Rothrock75`, limits the time step to :math:`\sim`\ 30 -minutes for the old ridging scheme (``krdg\_partic`` = 0), and to -:math:`\sim`\ 2 hours for the new scheme (``krdg\_partic`` = 1), assuming +minutes for the old ridging scheme (``krdg_partic`` = 0), and to +:math:`\sim`\ 2 hours for the new scheme (``krdg_partic`` = 1), assuming :math:`\Delta x` = 10 km. Practical limits may be somewhat less, depending on the strength of the atmospheric winds. @@ -646,7 +646,7 @@ growth rate. For the 5-category ice thickness distribution used as the default in this distribution, this is not a stringent limitation: :math:`\Delta t < 19.4` hr, assuming :math:`\max(f) = 40` cm/day. -In the classic EVP or EAP approach (``kdyn`` = 1 or 2, ``revised\_evp`` = false), +In the classic EVP or EAP approach (``kdyn`` = 1 or 2, ``revised_evp`` = false), the dynamics component is subcycled ndte (:math:`N`) times per dynamics time step so that the elastic waves essentially disappear before the next time step. The subcycling time step (:math:`\Delta @@ -657,7 +657,7 @@ t_e`) is thus A second parameter, :math:`E_\circ` (``eyc``), defines the elastic wave damping timescale :math:`T`, described in Section :ref:`dynam`, as -``eyc\ * dt\_dyn``. The forcing terms are not updated during the subcycling. +``eyc * dt_dyn``. The forcing terms are not updated during the subcycling. Given the small step (``dte``) at which the EVP dynamics model is subcycled, the elastic parameter :math:`E` is also limited by stability constraints, as discussed in :cite:`Hunke97`. Linear stability @@ -683,7 +683,7 @@ temperature :math:`T_{sfc}` is computed internally. The numerical constraint on the thermodynamics time step is associated with the transport scheme rather than the thermodynamic solver. -For the revised EVP approach (``kdyn`` = 1, ``revised\_evp`` = true), the +For the revised EVP approach (``kdyn`` = 1, ``revised_evp`` = true), the relaxation parameter ``arlx1i`` effectively sets the damping timescale in the problem, and ``brlx`` represents the effective subcycling :cite:`Bouillon13` (see Section :ref:`revp`). @@ -699,16 +699,16 @@ History files ************* Model output data is averaged over the period(s) given by ``histfreq`` and -``histfreq\_n``, and written to binary or netCDF files prepended by ``history\_file`` -in **ice\_in**. These settings for history files are set in the -**setup\_nml** section of **ice\_in** (see :ref:`tabnamelist`). -If ``history\_file`` = ‘iceh’ then the +``histfreq_n``, and written to binary or netCDF files prepended by ``history_file`` +in **ice_in**. These settings for history files are set in the +**setup_nml** section of **ice_in** (see :ref:`tabnamelist`). +If ``history_file`` = ‘iceh’ then the filenames will have the form **iceh.[timeID].nc** or **iceh.[timeID].da**, depending on the output file format chosen in **cice.settings** (set -``ICE\_IOTYPE``). The netCDF history files are CF-compliant; header information for +``ICE_IOTYPE``). The netCDF history files are CF-compliant; header information for data contained in the netCDF files is displayed with the command ``ncdump -h filename.nc``. Parallel netCDF output is available using the PIO library; the -attribute ``io\_flavor`` distinguishes output files written with PIO from +attribute ``io_flavor`` distinguishes output files written with PIO from those written with standard netCDF. With binary files, a separate header file is written with equivalent information. Standard fields are output according to settings in the **icefields\_nml** section of **ice\_in** @@ -735,7 +735,7 @@ monthly) via its namelist flag, `f\_` :math:`\left<{var}\right>`, which is now a character string corresponding to ``histfreq`` or ‘x’ for none. (Grid variable flags are still logicals, since they are written to all files, no matter what the frequency is.) If there are no namelist flags -with a given ``histfreq`` value, or if an element of ``histfreq\_n`` is 0, then +with a given ``histfreq`` value, or if an element of ``histfreq_n`` is 0, then no file will be written at that frequency. The output period can be discerned from the filenames. @@ -766,7 +766,7 @@ The history variable names must be unique for netCDF, so in cases where a variable is written at more than one frequency, the variable name is appended with the frequency in files after the first one. In the example above, ``meltb`` is called ``meltb`` in the monthly file (for backward -compatibility with the default configuration) and ``meltb\_h`` in the +compatibility with the default configuration) and ``meltb_h`` in the 6-hourly file. Using the same frequency twice in ``histfreq`` will have unexpected @@ -774,7 +774,7 @@ consequences and currently will cause the code to abort. It is not possible at the moment to output averages once a month and also once every 3 months, for example. -If ``write\_ic`` is set to true in **ice\_in**, a snapshot of the same set +If ``write_ic`` is set to true in **ice\_in**, a snapshot of the same set of history fields at the start of the run will be written to the history directory in **iceh\_ic.[timeID].nc(da)**. Several history variables are hard-coded for instantaneous output regardless of the averaging flag, at @@ -811,16 +811,16 @@ Diagnostic files Like ``histfreq``, the parameter ``diagfreq`` can be used to regulate how often output is written to a log file. The log file unit to which diagnostic -output is written is set in **ice\_fileunits.F90**. If ``diag\_type`` = +output is written is set in **ice\_fileunits.F90**. If ``diag_type`` = ‘stdout’, then it is written to standard out (or to **ice.log.[ID]** if you redirect standard out as in **cice.run**); otherwise it is written -to the file given by ``diag\_file``. In addition to the standard diagnostic +to the file given by ``diag_file``. In addition to the standard diagnostic output (maximum area-averaged thickness, velocity, average albedo, total ice area, and total ice and snow volumes), the namelist options -``print\_points`` and ``print\_global`` cause additional diagnostic information -to be computed and written. ``print\_global`` outputs global sums that are +``print_points`` and ``print_global`` cause additional diagnostic information +to be computed and written. ``print_global`` outputs global sums that are useful for checking global conservation of mass and energy. -``print\_points`` writes data for two specific grid points. Currently, one +``print_points`` writes data for two specific grid points. Currently, one point is near the North Pole and the other is in the Weddell Sea; these may be changed in **ice\_in**. @@ -888,27 +888,27 @@ Restart files ************* CICE provides restart data in binary unformatted or netCDF formats, via -the ``ICE\_IOTYPE`` flag in **cice.settings** and namelist variable -``restart\_format``. Restart and history files must use the same format. As +the ``ICE_IOTYPE`` flag in **cice.settings** and namelist variable +``restart_format``. Restart and history files must use the same format. As with the history output, there is also an option for writing parallel netCDF restart files using PIO. The restart files created by CICE contain all of the variables needed for a full, exact restart. The filename begins with the character string ‘iced.’, and the restart dump frequency is given by the namelist -variables ``dumpfreq`` and ``dumpfreq\_n``. The pointer to the filename from +variables ``dumpfreq`` and ``dumpfreq_n``. The pointer to the filename from which the restart data is to be read for a continuation run is set in -``pointer\_file``. The code assumes that auxiliary binary tracer restart +``pointer_file``. The code assumes that auxiliary binary tracer restart files will be identified using the same pointer and file name prefix, but with an additional character string in the file name that is associated with each tracer set. All variables are included in netCDF restart files. Additional namelist flags provide further control of restart behavior. -``dump\_last`` = true causes a set of restart files to be written at the end +``dump_last`` = true causes a set of restart files to be written at the end of a run when it is otherwise not scheduled to occur. The flag -``use\_restart\_time`` enables the user to choose to use the model date -provided in the restart files. If ``use\_restart\_time`` = false then the +``use_restart_time`` enables the user to choose to use the model date +provided in the restart files. If ``use_restart_time`` = false then the initial model date stamp is determined from the namelist parameters. lcdf64 = true sets 64-bit netCDF output, allowing larger file sizes. @@ -917,7 +917,7 @@ of the “extended" global grid, including the physical domain and ghost (halo) cells around the outer edges, allow exact restarts on regional grids with open boundary conditions, and they will also simplify restarts on the various tripole grids. They are accessed by setting -``restart\_ext`` = true in namelist. Extended grid restarts are not +``restart_ext`` = true in namelist. Extended grid restarts are not available when using PIO; in this case extra halo update calls fill ghost cells for tripole grids (do not use PIO for regional grids). @@ -929,5 +929,5 @@ initialized with no ice. The gx3 case was run for 1 year using the 1997 forcing data provided with the code. The gx1 case was run for 20 years, so that the date of restart in the file is 1978-01-01. Note that the restart dates provided in the restart files can be overridden using the -namelist variables ``use\_restart\_time``, ``year\_init`` and ``istep0``. The -forcing time can also be overridden using ``fyear\_init``. +namelist variables ``use_restart_time``, ``year_init`` and ``istep0``. The +forcing time can also be overridden using ``fyear_init``. diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 27393e3de..7a17e67ea 100644 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -819,6 +819,7 @@ hemispheres, and must exceed a critical value nominally set to test and the Two-Stage test described in the previous section are provided in :cite:`Hunke18`. +.. _CodeCompliance: Code Compliance Testing Procedure ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -846,7 +847,7 @@ To install the necessary Python packages, the ``pip`` Python utility can be used pip install --user matplotlib To run the compliance test, setup a baseline run with the original baseline model and then -a perturbation run based on recent model changes. Use ``--sets qc`` in both runs in addition +a perturbation run based on recent model changes. Use ``--set qc`` in both runs in addition to other settings needed. Then use the QC script to compare history output, .. code-block:: bash @@ -866,15 +867,17 @@ The script will produce output similar to: Additionally, the exit code from the test (``echo $?``) will be 0 if the test passed, and 1 if the test failed. -Implementation notes: 1) Provide a pass/fail on each of the confidence -intervals, 2) Facilitate output of a bitmap for each test so that -locations of failures can be identified. - -The cice.t-test.py requires memory to store multiple two-dimensional fields spanning +The ``cice.t-test.py`` requires memory to store multiple two-dimensional fields spanning 1825 unique timesteps, a total of several GB. An appropriate resource is needed to run the script. If the script runs out of memory on an interactive resource, try logging into a batch resource or finding a large memory node. +The ``cice.t-test.py`` script will also attempt to generate plots of the mean ice thickness +for both the baseline and test cases. Additionally, if the 2-stage test fails then the +script will attempt to plot a map showing the grid cells that failed the test. For a +full list of options, run ``python cice.t-test.py -h``. + + End-To-End Testing Procedure ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -888,7 +891,7 @@ Below is an example of a step-by-step procedure for testing a code change that m # Create a baseline dataset (only necessary if no baseline exists on the system) # git clone the baseline code - ./cice.setup -m onyx -e intel --suite base_suite --testid base0 -bgen cice.my.baseline + ./cice.setup -m onyx -e intel --suite base_suite --testid base0 --bgen cice.my.baseline # Run the test suite with the new code # git clone the new code @@ -900,6 +903,8 @@ Below is an example of a step-by-step procedure for testing a code change that m cd testsuite.test0 ./results.csh + # Note which tests failed and determine which namelist options are responsible for the failures + .. If the regression comparisons fail, then you may want to run the QC test, @@ -913,6 +918,7 @@ If the regression comparisons fail, then you may want to run the QC test, ./cice.setup -m onyx -e intel --test smoke -g gx1 -p 44x1 --testid qc_base -s qc,medium cd onyx_intel_smoke_gx1_44x1_medium_qc.qc_base + # modify ice_in to activate the namelist options that were determined above ./cice.build ./cice.submit @@ -921,6 +927,7 @@ If the regression comparisons fail, then you may want to run the QC test, ./cice.setup -m onyx -e intel --test smoke -g gx1 -p 44x1 -testid qc_test -s qc,medium cd onyx_intel_smoke_gx1_44x1_medium_qc.qc_test + # modify ice_in to activate the namelist options that were determined above ./cice.build ./cice.submit @@ -940,51 +947,3 @@ If the regression comparisons fail, then you may want to run the QC test, INFO:__main__:Quality Control Test PASSED -.. _testplotting: - -Test Plotting ----------------- - -The CICE scripts include a script (``timeseries.csh``) that will generate timeseries -figures from a diagnostic output file. -When running a test suite, the ``timeseries.csh`` script is automatically copied to the suite directory. -If the ``timeseries.csh`` script is to be used on a test or case that is not a part of a test suite, -users will need to run the ``timeseries.csh`` script from the tests directory -(``./configuration/scripts/tests/timeseries.csh ./path/``), or copy it to a local directory. -When used with the test suites or given a path, it needs to be run in the directory -above the particular case being plotted, but it can also be run on isolated log files in the same directory, -without a path. - -For example: - -Run the test suite. :: - -$ ./cice.setup -m conrad -e intel --suite base_suite --testid t00 - -Wait for suite to finish then go to the directory. :: - -$ cd testsuite.t00 - -Run the timeseries script on the desired case. :: - -$ ./timeseries.csh /p/work1/turner/CICE_RUNS/conrad_intel_smoke_col_1x1_diag1_run1year.t00/ - -The output figures are placed in the directory where the ``timeseries.csh`` script is run. - -To generate plots for all of the cases within a suite with a testid, create and run a script such as :: - - #!/bin/csh - foreach dir (`ls -1 | grep testid`) - echo $dir - timeseries.csh $dir - end - - -This plotting script can be used to plot the following variables: - - - total ice area (:math:`km^2`) - - total ice extent (:math:`km^2`) - - total ice volume (:math:`m^3`) - - total snow volume (:math:`m^3`) - - RMS ice speed (:math:`m/s`) - diff --git a/doc/source/user_guide/ug_troubleshooting.rst b/doc/source/user_guide/ug_troubleshooting.rst index 7998eef6b..e34d1261b 100644 --- a/doc/source/user_guide/ug_troubleshooting.rst +++ b/doc/source/user_guide/ug_troubleshooting.rst @@ -190,6 +190,9 @@ Known bugs - Latitude and longitude fields in the history output may be wrong when using padding. +- History and restart files will not be written on the first timestep in + some cases. + Interpretation of albedos ---------------------------------------- From 97d662597c5faf70f49dd9ca3cd5d2bc5886d418 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 23:24:26 +0000 Subject: [PATCH 45/54] machine syncing --- configuration/scripts/machines/Macros.phase3_intel | 1 + configuration/scripts/machines/Macros.theia_intel | 2 ++ configuration/scripts/machines/env.high_Sierra_gnu | 2 +- configuration/scripts/machines/env.hobart_intel | 2 +- configuration/scripts/machines/env.phase3_intel | 13 +++++++++---- configuration/scripts/machines/env.travisCI_gnu | 4 +++- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/configuration/scripts/machines/Macros.phase3_intel b/configuration/scripts/machines/Macros.phase3_intel index 6bfea6b19..c5a8b7a7e 100644 --- a/configuration/scripts/machines/Macros.phase3_intel +++ b/configuration/scripts/machines/Macros.phase3_intel @@ -4,6 +4,7 @@ CPP := fpp CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} + #CFLAGS := -c -O2 -fp-model precise -xHost CFLAGS := -c -O2 diff --git a/configuration/scripts/machines/Macros.theia_intel b/configuration/scripts/machines/Macros.theia_intel index 614e098bd..38cc62ed8 100644 --- a/configuration/scripts/machines/Macros.theia_intel +++ b/configuration/scripts/machines/Macros.theia_intel @@ -38,6 +38,8 @@ else endif LIB_MPI := + + SCC := icc SFC := ifort MPICC := mpiicc diff --git a/configuration/scripts/machines/env.high_Sierra_gnu b/configuration/scripts/machines/env.high_Sierra_gnu index cabd66944..db8bc01a3 100755 --- a/configuration/scripts/machines/env.high_Sierra_gnu +++ b/configuration/scripts/machines/env.high_Sierra_gnu @@ -4,7 +4,7 @@ setenv ICE_MACHINE_ENVNAME high_Sierra setenv ICE_MACHINE_COMPILER gnu setenv ICE_MACHINE_MAKE make setenv ICE_MACHINE_WKDIR /Volumes/ncep/CICE_RUNS -setenv ICE_MACHINE_INPUTDATA /Volumes/Data/CICE_data +setenv ICE_MACHINE_INPUTDATA /Volumes/Data/ setenv ICE_MACHINE_BASELINE /Volumes/ncep/CICE_BASELINE setenv ICE_MACHINE_SUBMIT " " setenv ICE_MACHINE_TPNODE 4 diff --git a/configuration/scripts/machines/env.hobart_intel b/configuration/scripts/machines/env.hobart_intel index 76f4e5e55..80092297a 100755 --- a/configuration/scripts/machines/env.hobart_intel +++ b/configuration/scripts/machines/env.hobart_intel @@ -15,7 +15,7 @@ module load compiler/intel/18.0.3 endif setenv ICE_MACHINE_ENVNAME hobart -setenv ICE_MACHINE_COMPILER ifort +setenv ICE_MACHINE_COMPILER intel setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /scratch/cluster/$user/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /fs/cgd/csm/inputdata diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel index 750a72330..4c8f65eee 100755 --- a/configuration/scripts/machines/env.phase3_intel +++ b/configuration/scripts/machines/env.phase3_intel @@ -1,8 +1,12 @@ -#!/bin/csh -f --login +#!/bin/csh -f -source /etc/profile.d/lmod.csh +#. $MODULESHOME/init/csh +source $MODULESHOME/init/csh +echo done with init.csh +module use -a $MODULESHOME/modulefiles module purge +module load lsf/10.1 module load ips/18.0.1.163 module load impi/18.0.1 module load NetCDF/4.5.0 @@ -15,9 +19,10 @@ setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /u/Robert.Grumbine/noscrub/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /u/Robert.Grumbine/noscrub/ setenv ICE_MACHINE_BASELINE /u/Robert.Grumbine/noscrub/CICE_BASELINE -setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_SUBMIT "bsub < " setenv ICE_MACHINE_TPNODE 4 setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_BLDTHRDS 1 -setenv ICE_MACHINE_QSTAT "qstat " +setenv ICE_MACHINE_QSTAT "bjobs " + diff --git a/configuration/scripts/machines/env.travisCI_gnu b/configuration/scripts/machines/env.travisCI_gnu index 687c4ba07..f3c9da26b 100755 --- a/configuration/scripts/machines/env.travisCI_gnu +++ b/configuration/scripts/machines/env.travisCI_gnu @@ -7,7 +7,9 @@ setenv ICE_MACHINE_WKDIR ~/CICE_RUNS setenv ICE_MACHINE_INPUTDATA ~ setenv ICE_MACHINE_BASELINE ~/CICE_BASELINE setenv ICE_MACHINE_SUBMIT " " -setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_TPNODE 4 # maximum tasks per node +setenv ICE_MACHINE_MAXPES 4 # maximum total pes (tasks * threads) available +setenv ICE_MACHINE_MAXRUNLENGTH 1 # maximum batch wall time limit in hours (integer) setenv ICE_MACHINE_ACCT P0000000 setenv ICE_MACHINE_QUEUE "default" setenv ICE_MACHINE_BLDTHRDS 1 From a6b5c9c799957d0ada7e4bfdce04207520b32dd3 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 23:33:03 +0000 Subject: [PATCH 46/54] couple more --- configuration/scripts/cice.batch.csh | 83 ++++++++++++++-------------- configuration/scripts/cice.settings | 4 +- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index d6098aef5..c4b6654ca 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -28,24 +28,17 @@ if (${taskpernodelimit} > ${ntasks}) set taskpernodelimit = ${ntasks} set ptile = $taskpernode if ($ptile > ${maxtpn} / 2) @ ptile = ${maxtpn} / 2 -set runlength = ${ICE_RUNLENGTH} -if ($?ICE_MACHINE_MAXRUNLENGTH) then - if (${runlength} > ${ICE_MACHINE_MAXRUNLENGTH}) then - set runlength = ${ICE_MACHINE_MAXRUNLENGTH} - endif -endif - set queue = "${ICE_QUEUE}" set batchtime = "00:15:00" -if (${runlength} == 0) set batchtime = "00:29:00" -if (${runlength} == 1) set batchtime = "00:59:00" -if (${runlength} == 2) set batchtime = "2:00:00" -if (${runlength} == 3) set batchtime = "3:00:00" -if (${runlength} == 4) set batchtime = "4:00:00" -if (${runlength} == 5) set batchtime = "5:00:00" -if (${runlength} == 6) set batchtime = "6:00:00" -if (${runlength} == 7) set batchtime = "7:00:00" -if (${runlength} >= 8) set batchtime = "8:00:00" +if (${ICE_RUNLENGTH} > 0) set batchtime = "00:29:00" +if (${ICE_RUNLENGTH} == 1) set batchtime = "00:59:00" +if (${ICE_RUNLENGTH} == 2) set batchtime = "2:00:00" +if (${ICE_RUNLENGTH} == 3) set batchtime = "3:00:00" +if (${ICE_RUNLENGTH} == 4) set batchtime = "4:00:00" +if (${ICE_RUNLENGTH} == 5) set batchtime = "5:00:00" +if (${ICE_RUNLENGTH} == 6) set batchtime = "6:00:00" +if (${ICE_RUNLENGTH} == 7) set batchtime = "7:00:00" +if (${ICE_RUNLENGTH} >= 8) set batchtime = "8:00:00" set shortcase = `echo ${ICE_CASENAME} | cut -c1-15` @@ -79,18 +72,6 @@ cat >> ${jobfile} << EOFB #PBS -l nodes=1:ppn=24 EOFB -else if (${ICE_MACHINE} =~ izumi*) then -if (${runlength} > 2) set queue = "medium" -cat >> ${jobfile} << EOFB -#PBS -j oe -###PBS -m ae -#PBS -V -#PBS -q ${queue} -#PBS -N ${ICE_CASENAME} -#PBS -l nodes=${nnodes}:ppn=${taskpernode} -#PBS -l walltime=${batchtime} -EOFB - else if (${ICE_MACHINE} =~ thunder* || ${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad* || ${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr*) then cat >> ${jobfile} << EOFB #PBS -N ${shortcase} @@ -179,7 +160,8 @@ EOFB else if (${ICE_MACHINE} =~ phase3*) then cat >> ${jobfile} << EOFB # nothing to do -======= +EOFB + else if (${ICE_MACHINE} =~ millikan*) then cat >> ${jobfile} << EOFB #SBATCH -J ${ICE_CASENAME} @@ -191,27 +173,31 @@ cat >> ${jobfile} << EOFB ###SBATCH --mail-type END,FAIL ###SBATCH --mail-user=amelie.bouchat@canada.ca #SBATCH --qos=standby - EOFB -else if (${ICE_MACHINE} =~ brooks*) then +else if (${ICE_MACHINE} =~ theia*) then cat >> ${jobfile} << EOFB -#PBS -N ${ICE_CASENAME} -#PBS -j oe -#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds} -#PBS -l walltime=${batchtime} +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t ${batchtime} +#SBATCH -q batch +#SBATCH -A marine-cpu +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +#SBATCH --mail-type FAIL +#SBATCH --mail-user=robert.grumbine@noaa.gov EOFB -else if (${ICE_MACHINE} =~ theia*) then +else if (${ICE_MACHINE} =~ hera*) then cat >> ${jobfile} << EOFB #SBATCH -J ${ICE_CASENAME} -#SBATCH -t ${batchtime} +#SBATCH -t `echo ${batchtime} | cut -f1-2 -d:` #SBATCH -q batch #SBATCH -A marine-cpu #SBATCH -N ${nnodes} #SBATCH -e slurm%j.err #SBATCH -o slurm%j.out -#SBATCH --mail-type END,FAIL +#SBATCH --mail-type FAIL #SBATCH --mail-user=robert.grumbine@noaa.gov EOFB @@ -221,8 +207,25 @@ cat >> ${jobfile} << EOFB EOFB else if (${ICE_MACHINE} =~ phase3*) then -cat >> ${jobfile} << EOFB -# nothing to do +if ( ${nnodes} > 15) then + setenv p3tile 16 + setenv mem `expr 100 \* 1024 / $nnodes` +else + setenv p3tile ${nnodes} + setenv mem 8192 +endif +echo mem = ${mem} nnodes and p3tiles ${nnodes} ${p3tile} p3tile must be le nnodes +cat >> ${jobfile} << EOFB +#BSUB -J ${ICE_CASENAME} +#BSUB -q "dev_shared" +#BSUB -P RTO-T2O +#BSUB -W `echo ${batchtime} | cut -f1-2 -d:` +#BSUB -n ${nnodes} +#BSUB -R "affinity[core]" +#BSUB -R "span[ptile=${p3tile}]" +#BSUB -R "rusage[mem=${mem}]" +#BSUB -o /u/Robert.Grumbine/${ICE_CASENAME}.out.%J +#BSUB -e /u/Robert.Grumbine/${ICE_CASENAME}.err.%J EOFB else if (${ICE_MACHINE} =~ high_Sierra*) then diff --git a/configuration/scripts/cice.settings b/configuration/scripts/cice.settings index 18c8082ce..5f6388a2e 100755 --- a/configuration/scripts/cice.settings +++ b/configuration/scripts/cice.settings @@ -25,9 +25,9 @@ setenv ICE_TESTID undefined setenv ICE_BASELINE undefined setenv ICE_BASEGEN undefined setenv ICE_BASECOM undefined -setenv ICE_BFBCOMP BFB +setenv ICE_BFBCOMP undefined setenv ICE_SPVAL undefined -setenv ICE_RUNLENGTH -1 +setenv ICE_RUNLENGTH 0 setenv ICE_ACCOUNT undefined setenv ICE_QUEUE undefined From 4083514e710c05f600e2aee9515c3cdc036a9b2c Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 23:39:12 +0000 Subject: [PATCH 47/54] some more --- .github/PULL_REQUEST_TEMPLATE | 46 +++++++++++-------- .gitmodules | 4 ++ .../cicedynB/infrastructure/ice_domain.F90 | 6 +-- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 83509033c..5195feed8 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -1,19 +1,29 @@ -[Remove this and add a short summary line]: -- Developer(s): - -- Please suggest code Pull Request reviewers in the column at right. - -- Are the code changes bit for bit, different at roundoff level, or more substantial? - -- Please include the link to test results or paste the summary block from the bottom of the testing output below. - -- Does this PR create or have dependencies on Icepack or any other models? - -- Is the documentation being updated with this PR? (Y/N) -If not, does the documentation need to be updated separately at a later time? (Y/N) - -Note: "Documentation" includes information on the wiki and .rst files in doc/source/, -which are used to create the online technical docs at https://readthedocs.org/projects/cice-consortium-cice/. - -- Other Relevant Details: +For detailed information about submitting Pull Requests (PRs) to the CICE-Consortium, +please refer to: + + +## PR checklist +- [ ] Short (1 sentence) summary of your PR: + ENTER INFORMATION HERE +- [ ] Developer(s): + ENTER INFORMATION HERE +- [ ] Suggest PR reviewers from list in the column to the right. +- [ ] Please copy the PR test results link or provide a summary of testing completed below. + ENTER INFORMATION HERE +- How much do the PR code changes differ from the unmodified code? + - [ ] bit for bit + - [ ] different at roundoff level + - [ ] more substantial +- Does this PR create or have dependencies on Icepack or any other models? + - [ ] Yes + - [ ] No +- Does this PR add any new test cases? + - [ ] Yes + - [ ] No +- Is the documentation being updated? ("Documentation" includes information on the wiki or in the .rst files from doc/source/, which are used to create the online technical docs at https://readthedocs.org/projects/cice-consortium-cice/.) + - [ ] Yes + - [ ] No, does the documentation need to be updated at a later time? + - [ ] Yes + - [ ] No +- [ ] Please provide any additional information or relevant details below: diff --git a/.gitmodules b/.gitmodules index e69de29bb..9e463fcec 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "icepack"] + path = icepack + url = https://github.com/cice-consortium/icepack.git + branch = master diff --git a/cicecore/cicedynB/infrastructure/ice_domain.F90 b/cicecore/cicedynB/infrastructure/ice_domain.F90 index ae20e5855..b4b4c4ab2 100644 --- a/cicecore/cicedynB/infrastructure/ice_domain.F90 +++ b/cicecore/cicedynB/infrastructure/ice_domain.F90 @@ -300,11 +300,9 @@ subroutine init_domain_distribution(KMTG,ULATG) i,j,n ,&! dummy loop indices ig,jg ,&! global indices work_unit ,&! size of quantized work unit -#ifdef ncdf fid ,&! file id varid ,&! var id status ,&! netcdf return code -#endif tblocks_tmp ,&! total number of blocks nblocks_tmp ,&! temporary value of nblocks nblocks_max ! max blocks on proc @@ -480,12 +478,12 @@ subroutine init_domain_distribution(KMTG,ULATG) if (this_block%j_glob(j) > 0) then do i=this_block%ilo,this_block%ihi if (this_block%i_glob(i) > 0) then - ig = this_block%i_glob(i) + ig = this_block%i_glob(i) jg = this_block%j_glob(j) if (KMTG(ig,jg) > puny .and. & (ULATG(ig,jg) < shlat/rad_to_deg .or. & ULATG(ig,jg) > nhlat/rad_to_deg) ) & - nocn(n) = nocn(n) + flat(ig,jg) + nocn(n) = nocn(n) + flat(ig,jg) endif end do endif From 322d839b5d42603039dbb051bb4ff1a9dfcee74f Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Sat, 19 Oct 2019 23:46:18 +0000 Subject: [PATCH 48/54] finally the right template --- .github/PULL_REQUEST_TEMPLATE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 5195feed8..6256f9bc6 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -21,7 +21,7 @@ please refer to: Date: Sat, 19 Oct 2019 23:47:20 +0000 Subject: [PATCH 49/54] finally correct template --- .github/PULL_REQUEST_TEMPLATE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 5195feed8..6256f9bc6 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -21,7 +21,7 @@ please refer to: Date: Sun, 20 Oct 2019 00:02:44 +0000 Subject: [PATCH 50/54] typography in master --- .../cicedynB/infrastructure/ice_domain.F90 | 2 + cicecore/shared/ice_arrays_column.F90 | 2 +- cicecore/shared/ice_constants.F90 | 2 +- cicecore/shared/ice_spacecurve.F90 | 84 +++++++++---------- 4 files changed, 46 insertions(+), 44 deletions(-) diff --git a/cicecore/cicedynB/infrastructure/ice_domain.F90 b/cicecore/cicedynB/infrastructure/ice_domain.F90 index b4b4c4ab2..1776ff906 100644 --- a/cicecore/cicedynB/infrastructure/ice_domain.F90 +++ b/cicecore/cicedynB/infrastructure/ice_domain.F90 @@ -300,9 +300,11 @@ subroutine init_domain_distribution(KMTG,ULATG) i,j,n ,&! dummy loop indices ig,jg ,&! global indices work_unit ,&! size of quantized work unit +#ifdef ncdf fid ,&! file id varid ,&! var id status ,&! netcdf return code +#endif tblocks_tmp ,&! total number of blocks nblocks_tmp ,&! temporary value of nblocks nblocks_max ! max blocks on proc diff --git a/cicecore/shared/ice_arrays_column.F90 b/cicecore/shared/ice_arrays_column.F90 index e6917f51f..2ccd2d754 100644 --- a/cicecore/shared/ice_arrays_column.F90 +++ b/cicecore/shared/ice_arrays_column.F90 @@ -268,7 +268,7 @@ module ice_arrays_column real (kind=dbl_kind), dimension(:), allocatable, public :: & R_C2N , & ! algal C to N (mole/mole) R_chl2N , & ! 3 algal chlorophyll to N (mg/mmol) - R_Si2N ! silica to nitrogen mole ratio for algal groups + R_Si2N ! silica to nitrogen mole ratio for algal groups !======================================================================= diff --git a/cicecore/shared/ice_constants.F90 b/cicecore/shared/ice_constants.F90 index 15581e787..c49732e35 100644 --- a/cicecore/shared/ice_constants.F90 +++ b/cicecore/shared/ice_constants.F90 @@ -60,7 +60,7 @@ module ice_constants c180 = 180.0_dbl_kind, & c360 = 360.0_dbl_kind, & c365 = 365.0_dbl_kind, & - c400 = 400.0_dbl_kind, & + c400 = 400.0_dbl_kind, & c1000= 1000.0_dbl_kind, & c3600= 3600.0_dbl_kind, & p001 = 0.001_dbl_kind, & diff --git a/cicecore/shared/ice_spacecurve.F90 b/cicecore/shared/ice_spacecurve.F90 index ed108e1c7..78b256b8f 100644 --- a/cicecore/shared/ice_spacecurve.F90 +++ b/cicecore/shared/ice_spacecurve.F90 @@ -31,7 +31,7 @@ module ice_spacecurve ! !PUBLIC MEMBER FUNCTIONS: public :: GenSpaceCurve, & - IsLoadBalanced + IsLoadBalanced public :: Factor, & IsFactorable, & @@ -41,24 +41,24 @@ module ice_spacecurve ! !PRIVATE MEMBER FUNCTIONS: - private :: map, & - PeanoM, & - Hilbert, & - Cinco, & + private :: map, & + PeanoM, & + Hilbert, & + Cinco, & GenCurve private :: FirstFactor, & FindandMark integer(int_kind), dimension(:,:), allocatable :: & - dir, &! direction to move along each level + dir, &! direction to move along each level ordered ! the ordering integer(int_kind), dimension(:), allocatable :: & - pos ! position along each of the axes + pos ! position along each of the axes integer(int_kind) :: & - maxdim, &! dimensionality of entire space - vcnt ! visitation count + maxdim, &! dimensionality of entire space + vcnt ! visitation count logical :: verbose=.FALSE. @@ -80,7 +80,7 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) ! !DESCRIPTION: ! This subroutine implements a Cinco space-filling curve. ! Cinco curves connect a Nb x Nb block of points where -! +! ! Nb = 5^p ! ! !REVISION HISTORY: @@ -91,12 +91,12 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) ! !INPUT PARAMETERS integer(int_kind), intent(in) :: & - l, & ! level of the space-filling curve + l, & ! level of the space-filling curve type, & ! type of SFC curve - ma, & ! Major axis [0,1] - md, & ! direction of major axis [-1,1] - ja, & ! joiner axis [0,1] - jd ! direction of joiner axis [-1,1] + ma, & ! Major axis [0,1] + md, & ! direction of major axis [-1,1] + ja, & ! joiner axis [0,1] + jd ! direction of joiner axis [-1,1] ! !OUTPUT PARAMETERS @@ -111,12 +111,12 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) !----------------------------------------------------------------------- integer(int_kind) :: & - lma, &! local major axis (next level) - lmd, &! local major direction (next level) - lja, &! local joiner axis (next level) - ljd, &! local joiner direction (next level) - ltype, &! type of SFC on next level - ll ! next level down + lma, &! local major axis (next level) + lmd, &! local major direction (next level) + lja, &! local joiner axis (next level) + ljd, &! local joiner direction (next level) + ltype, &! type of SFC on next level + ll ! next level down logical :: debug = .FALSE. @@ -963,8 +963,8 @@ function IncrementCurve(ja,jd) result(ierr) ! !INPUT PARAMETERS: integer(int_kind) :: & - ja, &! axis to increment - jd ! direction along axis + ja, &! axis to increment + jd ! direction along axis ! !OUTPUT PARAMETERS: integer(int_kind) :: ierr ! error return code @@ -975,7 +975,7 @@ function IncrementCurve(ja,jd) result(ierr) ! mark the newly visited point !----------------------------- ordered(pos(0)+1,pos(1)+1) = vcnt - + !------------------------------------ ! increment curve and update position !------------------------------------ @@ -1064,12 +1064,12 @@ function IsLoadBalanced(nelem,npart) ! !INTPUT PARAMETERS: integer(int_kind), intent(in) :: & - nelem, & ! number of blocks/elements to partition - npart ! size of partition + nelem, & ! number of blocks/elements to partition + npart ! size of partition ! !OUTPUT PARAMETERS: logical :: IsLoadBalanced ! .TRUE. if a perfectly load balanced - ! partition is possible + ! partition is possible !EOP !BOC !----------------------------------------------------------------------- @@ -1077,7 +1077,7 @@ function IsLoadBalanced(nelem,npart) ! local variables ! !----------------------------------------------------------------------- - + integer(int_kind) :: tmp1 ! temporary int character(len=*),parameter :: subname='(IsLoadBalanced)' @@ -1085,10 +1085,10 @@ function IsLoadBalanced(nelem,npart) !----------------------------------------------------------------------- tmp1 = nelem/npart - if(npart*tmp1 == nelem ) then - IsLoadBalanced=.TRUE. + if (npart*tmp1 == nelem ) then + IsLoadBalanced=.TRUE. else - IsLoadBalanced=.FALSE. + IsLoadBalanced=.FALSE. endif !EOP @@ -1285,7 +1285,7 @@ function Factor(num) result(res) !----------------------------------------------------------------------- integer(int_kind) :: & - tmp,tmp2,tmp3,tmp5 ! tempories for the factorization algorithm + tmp,tmp2,tmp3,tmp5 ! tempories for the factorization algorithm integer(int_kind) :: i,n ! loop tempories logical :: found ! logical temporary character(len=*),parameter :: subname='(Factor)' @@ -1437,9 +1437,9 @@ subroutine map(l) !----------------------------------------------------------------------- integer(int_kind) :: & - d, & ! dimension of curve only 2D is supported - type, & ! type of space-filling curve to start off - ierr ! error return code + d, & ! dimension of curve only 2D is supported + type, & ! type of space-filling curve to start off + ierr ! error return code character(len=*),parameter :: subname='(map)' d = SIZE(pos) @@ -1484,8 +1484,8 @@ subroutine PrintCurve(Mesh) ! !----------------------------------------------------------------------- integer(int_kind) :: & - gridsize, &! order of space-filling curve - i ! loop temporary + gridsize, &! order of space-filling curve + i ! loop temporary character(len=*),parameter :: subname='(PrintCurve)' !----------------------------------------------------------------------- @@ -1523,7 +1523,7 @@ subroutine PrintCurve(Mesh) write (*,*) "------------------------------------------" do i=1,gridsize write(*,6) Mesh(1,i),Mesh(2,i),Mesh(3,i), & - Mesh(4,i),Mesh(5,i),Mesh(6,i) + Mesh(4,i),Mesh(5,i),Mesh(6,i) enddo else if(gridsize == 8) then write (*,*) "A Level 3 Hilbert Curve:" @@ -1615,7 +1615,7 @@ subroutine PrintCurve(Mesh) Mesh(13,i),Mesh(14,i),Mesh(15,i),Mesh(16,i), & Mesh(17,i),Mesh(18,i),Mesh(19,i),Mesh(20,i), & Mesh(21,i),Mesh(22,i),Mesh(23,i),Mesh(24,i), & - Mesh(25,i) + Mesh(25,i) enddo else if(gridsize == 27) then write (*,*) "A Level 3 Peano Meandering Curve:" @@ -1683,7 +1683,7 @@ subroutine GenSpaceCurve(Mesh) ! !INPUT/OUTPUT PARAMETERS: integer(int_kind), target,intent(inout) :: & - Mesh(:,:) ! The SFC ordering in 2D array + Mesh(:,:) ! The SFC ordering in 2D array !EOP !BOC @@ -1694,8 +1694,8 @@ subroutine GenSpaceCurve(Mesh) !----------------------------------------------------------------------- integer(int_kind) :: & - level, &! Level of space-filling curve - dim ! dimension of SFC... currently limited to 2D + level, &! Level of space-filling curve + dim ! dimension of SFC... currently limited to 2D integer(int_kind) :: gridsize ! number of points on a side From f3540e1a7d2a52d79b63ca7df1aa8b3cf43eb19a Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Mon, 21 Oct 2019 15:21:16 +0000 Subject: [PATCH 51/54] restore working batch.csh --- configuration/scripts/cice.batch.csh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index c4b6654ca..30fe08714 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -152,16 +152,6 @@ cat >> ${jobfile} << EOFB #SBATCH --qos=standby EOFB -else if (${ICE_MACHINE} =~ phase2*) then -cat >> ${jobfile} << EOFB -# nothing to do -EOFB - -else if (${ICE_MACHINE} =~ phase3*) then -cat >> ${jobfile} << EOFB -# nothing to do -EOFB - else if (${ICE_MACHINE} =~ millikan*) then cat >> ${jobfile} << EOFB #SBATCH -J ${ICE_CASENAME} From b68acf498ab1f53d55b7744c72e0b190ab5c5785 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Mon, 21 Oct 2019 16:26:40 +0000 Subject: [PATCH 52/54] towards matching consortium master --- configuration/scripts/cice.batch.csh | 37 +++++++++++++++++++++------- configuration/scripts/cice.settings | 2 +- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 30fe08714..95f01f78c 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -28,17 +28,24 @@ if (${taskpernodelimit} > ${ntasks}) set taskpernodelimit = ${ntasks} set ptile = $taskpernode if ($ptile > ${maxtpn} / 2) @ ptile = ${maxtpn} / 2 +set runlength = ${ICE_RUNLENGTH} +if ($?ICE_MACHINE_MAXRUNLENGTH) then + if (${runlength} > ${ICE_MACHINE_MAXRUNLENGTH}) then + set runlength = ${ICE_MACHINE_MAXRUNLENGTH} + endif +endif + set queue = "${ICE_QUEUE}" set batchtime = "00:15:00" -if (${ICE_RUNLENGTH} > 0) set batchtime = "00:29:00" -if (${ICE_RUNLENGTH} == 1) set batchtime = "00:59:00" -if (${ICE_RUNLENGTH} == 2) set batchtime = "2:00:00" -if (${ICE_RUNLENGTH} == 3) set batchtime = "3:00:00" -if (${ICE_RUNLENGTH} == 4) set batchtime = "4:00:00" -if (${ICE_RUNLENGTH} == 5) set batchtime = "5:00:00" -if (${ICE_RUNLENGTH} == 6) set batchtime = "6:00:00" -if (${ICE_RUNLENGTH} == 7) set batchtime = "7:00:00" -if (${ICE_RUNLENGTH} >= 8) set batchtime = "8:00:00" +if (${runlength} == 0) set batchtime = "00:29:00" +if (${runlength} == 1) set batchtime = "00:59:00" +if (${runlength} == 2) set batchtime = "2:00:00" +if (${runlength} == 3) set batchtime = "3:00:00" +if (${runlength} == 4) set batchtime = "4:00:00" +if (${runlength} == 5) set batchtime = "5:00:00" +if (${runlength} == 6) set batchtime = "6:00:00" +if (${runlength} == 7) set batchtime = "7:00:00" +if (${runlength} >= 8) set batchtime = "8:00:00" set shortcase = `echo ${ICE_CASENAME} | cut -c1-15` @@ -72,6 +79,18 @@ cat >> ${jobfile} << EOFB #PBS -l nodes=1:ppn=24 EOFB +else if (${ICE_MACHINE} =~ izumi*) then +if (${runlength} > 2) set queue = "medium" +cat >> ${jobfile} << EOFB +#PBS -j oe +###PBS -m ae +#PBS -V +#PBS -q ${queue} +#PBS -N ${ICE_CASENAME} +#PBS -l nodes=${nnodes}:ppn=${taskpernode} +#PBS -l walltime=${batchtime} +EOFB + else if (${ICE_MACHINE} =~ thunder* || ${ICE_MACHINE} =~ gordon* || ${ICE_MACHINE} =~ conrad* || ${ICE_MACHINE} =~ gaffney* || ${ICE_MACHINE} =~ koehr*) then cat >> ${jobfile} << EOFB #PBS -N ${shortcase} diff --git a/configuration/scripts/cice.settings b/configuration/scripts/cice.settings index 5f6388a2e..0513f1dd1 100755 --- a/configuration/scripts/cice.settings +++ b/configuration/scripts/cice.settings @@ -27,7 +27,7 @@ setenv ICE_BASEGEN undefined setenv ICE_BASECOM undefined setenv ICE_BFBCOMP undefined setenv ICE_SPVAL undefined -setenv ICE_RUNLENGTH 0 +setenv ICE_RUNLENGTH -1 setenv ICE_ACCOUNT undefined setenv ICE_QUEUE undefined From 432c29d603554f050baabb46e9a897f246cee262 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Mon, 21 Oct 2019 16:29:20 +0000 Subject: [PATCH 53/54] last? add for syncing --- configuration/scripts/cice.batch.csh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 95f01f78c..df003cb89 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -184,6 +184,14 @@ cat >> ${jobfile} << EOFB #SBATCH --qos=standby EOFB +else if (${ICE_MACHINE} =~ brooks*) then +cat >> ${jobfile} << EOFB +#PBS -N ${ICE_CASENAME} +#PBS -j oe +#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds} +#PBS -l walltime=${batchtime} +EOFB + else if (${ICE_MACHINE} =~ theia*) then cat >> ${jobfile} << EOFB #SBATCH -J ${ICE_CASENAME} From 162946a7a95b85c95eba52d14cb24303ec32ee82 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Mon, 21 Oct 2019 18:57:41 -0400 Subject: [PATCH 54/54] Revert icepack to 1.1.2 --- icepack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icepack b/icepack index 51ea326f1..5af464922 160000 --- a/icepack +++ b/icepack @@ -1 +1 @@ -Subproject commit 51ea326f12578355d4d51b0cea29b45cc9890cd1 +Subproject commit 5af4649222c6f1529288b721ddaf62ef0e935375