From 1a1524d45bc8c8604e0fa31e4811654a48b9775c Mon Sep 17 00:00:00 2001 From: apcraig Date: Thu, 30 May 2019 22:21:15 +0000 Subject: [PATCH] add noaa machines and minor cleanup from rg --- .../cicedynB/infrastructure/ice_domain.F90 | 4 +- cicecore/shared/ice_arrays_column.F90 | 2 +- cicecore/shared/ice_constants.F90 | 2 +- cicecore/shared/ice_spacecurve.F90 | 84 +++++++++---------- configuration/scripts/cice.batch.csh | 28 +++++++ configuration/scripts/cice.launch.csh | 27 ++++++ .../scripts/machines/Macros.high_Sierra_gnu | 47 +++++++++++ .../scripts/machines/Macros.phase2_intel | 57 +++++++++++++ .../scripts/machines/Macros.phase3_intel | 58 +++++++++++++ .../scripts/machines/Macros.theia_gnu | 48 +++++++++++ .../scripts/machines/Macros.theia_intel | 60 +++++++++++++ .../scripts/machines/Macros.theia_pgi | 48 +++++++++++ .../scripts/machines/env.high_Sierra_gnu | 15 ++++ .../scripts/machines/env.phase2_intel | 14 ++++ .../scripts/machines/env.phase3_intel | 24 ++++++ configuration/scripts/machines/env.theia_gnu | 15 ++++ .../scripts/machines/env.theia_intel | 30 +++++++ configuration/scripts/machines/env.theia_pgi | 27 ++++++ configuration/scripts/tests/rgvers_suite.ts | 18 ++++ 19 files changed, 562 insertions(+), 46 deletions(-) create mode 100644 configuration/scripts/machines/Macros.high_Sierra_gnu create mode 100755 configuration/scripts/machines/Macros.phase2_intel create mode 100755 configuration/scripts/machines/Macros.phase3_intel create mode 100644 configuration/scripts/machines/Macros.theia_gnu create mode 100644 configuration/scripts/machines/Macros.theia_intel create mode 100644 configuration/scripts/machines/Macros.theia_pgi create mode 100755 configuration/scripts/machines/env.high_Sierra_gnu create mode 100755 configuration/scripts/machines/env.phase2_intel create mode 100755 configuration/scripts/machines/env.phase3_intel create mode 100755 configuration/scripts/machines/env.theia_gnu create mode 100755 configuration/scripts/machines/env.theia_intel create mode 100755 configuration/scripts/machines/env.theia_pgi create mode 100644 configuration/scripts/tests/rgvers_suite.ts diff --git a/cicecore/cicedynB/infrastructure/ice_domain.F90 b/cicecore/cicedynB/infrastructure/ice_domain.F90 index 236f6f179..b4b4c4ab2 100644 --- a/cicecore/cicedynB/infrastructure/ice_domain.F90 +++ b/cicecore/cicedynB/infrastructure/ice_domain.F90 @@ -478,12 +478,12 @@ subroutine init_domain_distribution(KMTG,ULATG) if (this_block%j_glob(j) > 0) then do i=this_block%ilo,this_block%ihi if (this_block%i_glob(i) > 0) then - ig = this_block%i_glob(i) + ig = this_block%i_glob(i) jg = this_block%j_glob(j) if (KMTG(ig,jg) > puny .and. & (ULATG(ig,jg) < shlat/rad_to_deg .or. & ULATG(ig,jg) > nhlat/rad_to_deg) ) & - nocn(n) = nocn(n) + flat(ig,jg) + nocn(n) = nocn(n) + flat(ig,jg) endif end do endif diff --git a/cicecore/shared/ice_arrays_column.F90 b/cicecore/shared/ice_arrays_column.F90 index c430ddd13..71d03ed94 100644 --- a/cicecore/shared/ice_arrays_column.F90 +++ b/cicecore/shared/ice_arrays_column.F90 @@ -268,7 +268,7 @@ module ice_arrays_column real (kind=dbl_kind), dimension(:), allocatable, public :: & R_C2N , & ! algal C to N (mole/mole) R_chl2N , & ! 3 algal chlorophyll to N (mg/mmol) - R_Si2N ! silica to nitrogen mole ratio for algal groups + R_Si2N ! silica to nitrogen mole ratio for algal groups !======================================================================= diff --git a/cicecore/shared/ice_constants.F90 b/cicecore/shared/ice_constants.F90 index 15581e787..c49732e35 100644 --- a/cicecore/shared/ice_constants.F90 +++ b/cicecore/shared/ice_constants.F90 @@ -60,7 +60,7 @@ module ice_constants c180 = 180.0_dbl_kind, & c360 = 360.0_dbl_kind, & c365 = 365.0_dbl_kind, & - c400 = 400.0_dbl_kind, & + c400 = 400.0_dbl_kind, & c1000= 1000.0_dbl_kind, & c3600= 3600.0_dbl_kind, & p001 = 0.001_dbl_kind, & diff --git a/cicecore/shared/ice_spacecurve.F90 b/cicecore/shared/ice_spacecurve.F90 index ed108e1c7..78b256b8f 100644 --- a/cicecore/shared/ice_spacecurve.F90 +++ b/cicecore/shared/ice_spacecurve.F90 @@ -31,7 +31,7 @@ module ice_spacecurve ! !PUBLIC MEMBER FUNCTIONS: public :: GenSpaceCurve, & - IsLoadBalanced + IsLoadBalanced public :: Factor, & IsFactorable, & @@ -41,24 +41,24 @@ module ice_spacecurve ! !PRIVATE MEMBER FUNCTIONS: - private :: map, & - PeanoM, & - Hilbert, & - Cinco, & + private :: map, & + PeanoM, & + Hilbert, & + Cinco, & GenCurve private :: FirstFactor, & FindandMark integer(int_kind), dimension(:,:), allocatable :: & - dir, &! direction to move along each level + dir, &! direction to move along each level ordered ! the ordering integer(int_kind), dimension(:), allocatable :: & - pos ! position along each of the axes + pos ! position along each of the axes integer(int_kind) :: & - maxdim, &! dimensionality of entire space - vcnt ! visitation count + maxdim, &! dimensionality of entire space + vcnt ! visitation count logical :: verbose=.FALSE. @@ -80,7 +80,7 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) ! !DESCRIPTION: ! This subroutine implements a Cinco space-filling curve. ! Cinco curves connect a Nb x Nb block of points where -! +! ! Nb = 5^p ! ! !REVISION HISTORY: @@ -91,12 +91,12 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) ! !INPUT PARAMETERS integer(int_kind), intent(in) :: & - l, & ! level of the space-filling curve + l, & ! level of the space-filling curve type, & ! type of SFC curve - ma, & ! Major axis [0,1] - md, & ! direction of major axis [-1,1] - ja, & ! joiner axis [0,1] - jd ! direction of joiner axis [-1,1] + ma, & ! Major axis [0,1] + md, & ! direction of major axis [-1,1] + ja, & ! joiner axis [0,1] + jd ! direction of joiner axis [-1,1] ! !OUTPUT PARAMETERS @@ -111,12 +111,12 @@ recursive function Cinco(l,type,ma,md,ja,jd) result(ierr) !----------------------------------------------------------------------- integer(int_kind) :: & - lma, &! local major axis (next level) - lmd, &! local major direction (next level) - lja, &! local joiner axis (next level) - ljd, &! local joiner direction (next level) - ltype, &! type of SFC on next level - ll ! next level down + lma, &! local major axis (next level) + lmd, &! local major direction (next level) + lja, &! local joiner axis (next level) + ljd, &! local joiner direction (next level) + ltype, &! type of SFC on next level + ll ! next level down logical :: debug = .FALSE. @@ -963,8 +963,8 @@ function IncrementCurve(ja,jd) result(ierr) ! !INPUT PARAMETERS: integer(int_kind) :: & - ja, &! axis to increment - jd ! direction along axis + ja, &! axis to increment + jd ! direction along axis ! !OUTPUT PARAMETERS: integer(int_kind) :: ierr ! error return code @@ -975,7 +975,7 @@ function IncrementCurve(ja,jd) result(ierr) ! mark the newly visited point !----------------------------- ordered(pos(0)+1,pos(1)+1) = vcnt - + !------------------------------------ ! increment curve and update position !------------------------------------ @@ -1064,12 +1064,12 @@ function IsLoadBalanced(nelem,npart) ! !INTPUT PARAMETERS: integer(int_kind), intent(in) :: & - nelem, & ! number of blocks/elements to partition - npart ! size of partition + nelem, & ! number of blocks/elements to partition + npart ! size of partition ! !OUTPUT PARAMETERS: logical :: IsLoadBalanced ! .TRUE. if a perfectly load balanced - ! partition is possible + ! partition is possible !EOP !BOC !----------------------------------------------------------------------- @@ -1077,7 +1077,7 @@ function IsLoadBalanced(nelem,npart) ! local variables ! !----------------------------------------------------------------------- - + integer(int_kind) :: tmp1 ! temporary int character(len=*),parameter :: subname='(IsLoadBalanced)' @@ -1085,10 +1085,10 @@ function IsLoadBalanced(nelem,npart) !----------------------------------------------------------------------- tmp1 = nelem/npart - if(npart*tmp1 == nelem ) then - IsLoadBalanced=.TRUE. + if (npart*tmp1 == nelem ) then + IsLoadBalanced=.TRUE. else - IsLoadBalanced=.FALSE. + IsLoadBalanced=.FALSE. endif !EOP @@ -1285,7 +1285,7 @@ function Factor(num) result(res) !----------------------------------------------------------------------- integer(int_kind) :: & - tmp,tmp2,tmp3,tmp5 ! tempories for the factorization algorithm + tmp,tmp2,tmp3,tmp5 ! tempories for the factorization algorithm integer(int_kind) :: i,n ! loop tempories logical :: found ! logical temporary character(len=*),parameter :: subname='(Factor)' @@ -1437,9 +1437,9 @@ subroutine map(l) !----------------------------------------------------------------------- integer(int_kind) :: & - d, & ! dimension of curve only 2D is supported - type, & ! type of space-filling curve to start off - ierr ! error return code + d, & ! dimension of curve only 2D is supported + type, & ! type of space-filling curve to start off + ierr ! error return code character(len=*),parameter :: subname='(map)' d = SIZE(pos) @@ -1484,8 +1484,8 @@ subroutine PrintCurve(Mesh) ! !----------------------------------------------------------------------- integer(int_kind) :: & - gridsize, &! order of space-filling curve - i ! loop temporary + gridsize, &! order of space-filling curve + i ! loop temporary character(len=*),parameter :: subname='(PrintCurve)' !----------------------------------------------------------------------- @@ -1523,7 +1523,7 @@ subroutine PrintCurve(Mesh) write (*,*) "------------------------------------------" do i=1,gridsize write(*,6) Mesh(1,i),Mesh(2,i),Mesh(3,i), & - Mesh(4,i),Mesh(5,i),Mesh(6,i) + Mesh(4,i),Mesh(5,i),Mesh(6,i) enddo else if(gridsize == 8) then write (*,*) "A Level 3 Hilbert Curve:" @@ -1615,7 +1615,7 @@ subroutine PrintCurve(Mesh) Mesh(13,i),Mesh(14,i),Mesh(15,i),Mesh(16,i), & Mesh(17,i),Mesh(18,i),Mesh(19,i),Mesh(20,i), & Mesh(21,i),Mesh(22,i),Mesh(23,i),Mesh(24,i), & - Mesh(25,i) + Mesh(25,i) enddo else if(gridsize == 27) then write (*,*) "A Level 3 Peano Meandering Curve:" @@ -1683,7 +1683,7 @@ subroutine GenSpaceCurve(Mesh) ! !INPUT/OUTPUT PARAMETERS: integer(int_kind), target,intent(inout) :: & - Mesh(:,:) ! The SFC ordering in 2D array + Mesh(:,:) ! The SFC ordering in 2D array !EOP !BOC @@ -1694,8 +1694,8 @@ subroutine GenSpaceCurve(Mesh) !----------------------------------------------------------------------- integer(int_kind) :: & - level, &! Level of space-filling curve - dim ! dimension of SFC... currently limited to 2D + level, &! Level of space-filling curve + dim ! dimension of SFC... currently limited to 2D integer(int_kind) :: gridsize ! number of points on a side diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 4fc083703..0417c43a3 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -165,6 +165,34 @@ cat >> ${jobfile} << EOFB #SBATCH --qos=standby EOFB +else if (${ICE_MACHINE} =~ theia*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH -t ${batchtime} +#SBATCH -q batch +#SBATCH -A marine-cpu +#SBATCH -N ${nnodes} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +#SBATCH --mail-type END,FAIL +#SBATCH --mail-user=robert.grumbine@noaa.gov +EOFB + +else if (${ICE_MACHINE} =~ phase2*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + +else if (${ICE_MACHINE} =~ high_Sierra*) then +cat >> ${jobfile} << EOFB +# nothing to do +EOFB + else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFB # nothing to do diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh index c3aafba21..ce328f2a6 100755 --- a/configuration/scripts/cice.launch.csh +++ b/configuration/scripts/cice.launch.csh @@ -153,6 +153,33 @@ mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE EOFR endif +#======= +else if (${ICE_MACHINE} =~ theia*) then +cat >> ${jobfile} << EOFR +#mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +srun -n ${ntasks} -c ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +EOFR +#======= +else if (${ICE_MACHINE} =~ high_Sierra*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +EOFR + +#======= +else if (${ICE_MACHINE} =~ phase2*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +EOFR + +#======= +else if (${ICE_MACHINE} =~ phase3*) then +cat >> ${jobfile} << EOFR +mpirun -np ${ntasks} ./cice >&! \$ICE_RUNLOG_FILE +#./cice >&! \$ICE_RUNLOG_FILE +EOFR #======= else if (${ICE_MACHINE} =~ testmachine*) then cat >> ${jobfile} << EOFR diff --git a/configuration/scripts/machines/Macros.high_Sierra_gnu b/configuration/scripts/machines/Macros.high_Sierra_gnu new file mode 100644 index 000000000..960779da1 --- /dev/null +++ b/configuration/scripts/machines/Macros.high_Sierra_gnu @@ -0,0 +1,47 @@ +#============================================================================== +# Makefile macros for Travis-CI - GCC and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FFLAGS := -O2 -ffree-line-length-none -fconvert=big-endian -finit-real=nan +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + #FFLAGS += -O0 -g -Wextra -fbacktrace -fbounds-check -ffpe-trap=zero,overflow + FFLAGS += -O0 -g -std=f2008 -fbacktrace -fbounds-check -ffpe-trap=zero,overflow +else + FFLAGS += -O2 +endif + +FC := mpif90 + +MPICC:= + +MPIFC:= mpif90 +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:=gcc +SFC:= + +#ifeq ($(ICE_THREADED), true) +# LDFLAGS += -fopenmp +# CFLAGS += -fopenmp +# FFLAGS += -fopenmp +#endif diff --git a/configuration/scripts/machines/Macros.phase2_intel b/configuration/scripts/machines/Macros.phase2_intel new file mode 100755 index 000000000..6bfea6b19 --- /dev/null +++ b/configuration/scripts/machines/Macros.phase2_intel @@ -0,0 +1,57 @@ +#============================================================================== +# Makefile macros for generic test machine, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +#CFLAGS := -c -O2 -fp-model precise -xHost +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) $(NETCDF_INCLUDE) + +LIB_NETCDF := $(NETCDF_PATH)/lib +LIB_MPI := $(IMPILIBDIR) + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(compile_threaded), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +### if using parallel I/O, load all 3 libraries. PIO must be first! +#ifeq ($(IO_TYPE), pio) +# PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib +# INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include +# SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof +#endif diff --git a/configuration/scripts/machines/Macros.phase3_intel b/configuration/scripts/machines/Macros.phase3_intel new file mode 100755 index 000000000..c5a8b7a7e --- /dev/null +++ b/configuration/scripts/machines/Macros.phase3_intel @@ -0,0 +1,58 @@ +#============================================================================== +# Makefile macros for generic test machine, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} + +#CFLAGS := -c -O2 -fp-model precise -xHost +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +else + FFLAGS += -O2 +endif + +SCC := icc +SFC := ifort +MPICC := mpicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) $(NETCDF_INCLUDE) + +LIB_NETCDF := $(NETCDF_PATH)/lib +LIB_MPI := $(IMPILIBDIR) + +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(compile_threaded), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + +### if using parallel I/O, load all 3 libraries. PIO must be first! +#ifeq ($(IO_TYPE), pio) +# PIO_PATH:=/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib +# INCLDIR += -I/glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/include +# SLIBS := $(SLIBS) -L$(PIO_PATH) -lpiof +#endif diff --git a/configuration/scripts/machines/Macros.theia_gnu b/configuration/scripts/machines/Macros.theia_gnu new file mode 100644 index 000000000..63cce4b99 --- /dev/null +++ b/configuration/scripts/machines/Macros.theia_gnu @@ -0,0 +1,48 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FFLAGS := -fconvert=big-endian +#FFLAGS := -h bytwswapio +#FFLAGS := +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpif90 + +MPICC:= + +MPIFC:= mpif90 +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:= +SFC:= + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.theia_intel b/configuration/scripts/machines/Macros.theia_intel new file mode 100644 index 000000000..38cc62ed8 --- /dev/null +++ b/configuration/scripts/machines/Macros.theia_intel @@ -0,0 +1,60 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +#Note that on theia mpif90 refers to gfortran even though it is in the intel bin +FC := mpiifort +MPIFC:= mpiifort +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := + + +SCC := icc +SFC := ifort +MPICC := mpiicc +MPIFC := mpiifort + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/Macros.theia_pgi b/configuration/scripts/machines/Macros.theia_pgi new file mode 100644 index 000000000..83bcc264d --- /dev/null +++ b/configuration/scripts/machines/Macros.theia_pgi @@ -0,0 +1,48 @@ +#============================================================================== +# Makefile macros for theia - intel and openmpi compilers +#============================================================================== + +CPP := cpp +CPPDEFS := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS} +CFLAGS := -c -O2 -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost + +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +#RG: this looks more like gfortran options: +## FFLAGS += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow + FFLAGS += -O0 -g -check-bounds -implicit-none -fpe3 +else + FFLAGS += -O2 +endif + +FC := mpif90 +MPICC:= +MPIFC:= mpif90 +LD:= $(MPIFC) + +NETCDF_PATH := $(NETCDF) + +ifeq ($(ICE_IOTYPE), netcdf) + NETCDF_PATH := $(shell nc-config --prefix) + INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include + LIB_NETCDF := $(NETCDF_PATH)/lib + LIB_PNETCDF := + SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff +else + SLIBS := +endif + +LIB_MPI := +SCC:= +SFC:= + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif diff --git a/configuration/scripts/machines/env.high_Sierra_gnu b/configuration/scripts/machines/env.high_Sierra_gnu new file mode 100755 index 000000000..cabd66944 --- /dev/null +++ b/configuration/scripts/machines/env.high_Sierra_gnu @@ -0,0 +1,15 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME high_Sierra +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /Volumes/ncep/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /Volumes/Data/CICE_data +setenv ICE_MACHINE_BASELINE /Volumes/ncep/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +setenv ICE_MACHINE_QUIETMODE false +setenv ICE_MACHINE_QUEUE " " diff --git a/configuration/scripts/machines/env.phase2_intel b/configuration/scripts/machines/env.phase2_intel new file mode 100755 index 000000000..2457d4721 --- /dev/null +++ b/configuration/scripts/machines/env.phase2_intel @@ -0,0 +1,14 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME phase2 +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR ~/noscrub/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA ~/noscrub/ +setenv ICE_MACHINE_BASELINE ~/noscrub/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.phase3_intel b/configuration/scripts/machines/env.phase3_intel new file mode 100755 index 000000000..03aa7083f --- /dev/null +++ b/configuration/scripts/machines/env.phase3_intel @@ -0,0 +1,24 @@ +#!/bin/csh -f --login + +source /etc/profile.d/lmod.csh + +module purge +module load ips/18.0.1.163 +module load impi/18.0.1 +module load NetCDF/4.5.0 +module load ESMF/7_1_0r +module list + +setenv ICE_MACHINE_ENVNAME phase3 +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /u/Robert.Grumbine/noscrub/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /u/Robert.Grumbine/noscrub/ +setenv ICE_MACHINE_BASELINE /u/Robert.Grumbine/noscrub/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "qstat " + diff --git a/configuration/scripts/machines/env.theia_gnu b/configuration/scripts/machines/env.theia_gnu new file mode 100755 index 000000000..974be35e0 --- /dev/null +++ b/configuration/scripts/machines/env.theia_gnu @@ -0,0 +1,15 @@ +#!/bin/csh -f + +setenv ICE_MACHINE_ENVNAME theia +setenv ICE_MACHINE_COMPILER gnu +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /home/Robert.Grumbine/save/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.theia_intel b/configuration/scripts/machines/env.theia_intel new file mode 100755 index 000000000..0438238d9 --- /dev/null +++ b/configuration/scripts/machines/env.theia_intel @@ -0,0 +1,30 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +#module list +module purge +#module load intel/18.1.163 Works, at least w. nc 4.4.0, +#14.0.2 w nc4.3.0 does not module load intel/18.1.163 +# ok w nc4.3.0: module load intel/16.0.1.150 +# ok w nc4.3.0: module load intel/15.3.187 +module load intel/17.0.5.239 +module load impi +module load esmf +module load hdf5 netcdf/4.3.0 +module load wgrib wgrib2 +#echo renewed modules: +#module list + +setenv ICE_MACHINE_ENVNAME theia +setenv ICE_MACHINE_COMPILER intel +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /scratch3/NCEPDEV/marine/save/Robert.Grumbine/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "batch" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/machines/env.theia_pgi b/configuration/scripts/machines/env.theia_pgi new file mode 100755 index 000000000..2c4229c0e --- /dev/null +++ b/configuration/scripts/machines/env.theia_pgi @@ -0,0 +1,27 @@ +#!/bin/csh -f + +source /etc/profile.d/modules.csh +module list +module purge +module load pgi +module load impi +module load esmf +module load hdf5 netcdf/4.4.0 +module load wgrib wgrib2 +echo renewed modules: +module list + + +setenv ICE_MACHINE_ENVNAME theia +setenv ICE_MACHINE_COMPILER pgi +setenv ICE_MACHINE_MAKE make +setenv ICE_MACHINE_WKDIR /home/Robert.Grumbine/save/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /scratch3/NCEPDEV/marine/save/Robert.Grumbine/ +setenv ICE_MACHINE_BASELINE /home/Robert.Grumbine/save/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT " " +setenv ICE_MACHINE_TPNODE 4 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "default" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT " " +#setenv ICE_MACHINE_QUIETMODE true diff --git a/configuration/scripts/tests/rgvers_suite.ts b/configuration/scripts/tests/rgvers_suite.ts new file mode 100644 index 000000000..dc85613db --- /dev/null +++ b/configuration/scripts/tests/rgvers_suite.ts @@ -0,0 +1,18 @@ +# Test Grid PEs Sets BFB-compare +#restart gx1 4x4 droundrobin,short +#restart gx1 8x4 droundrobin,short +#restart gx1 12x4 droundrobin,short +#restart gx1 16x4 droundrobin,short +#restart gx1 20x4 droundrobin,short +#restart gx1 24x4 droundrobin,short +#restart gx1 28x4 droundrobin,short +#restart gx1 32x4 droundrobin,short +restart gx1 32x2 droundrobin,short +restart gx1 32x3 droundrobin,short +restart gx1 32x1 droundrobin,short + +restart gx1 33x4 droundrobin,short +restart gx1 34x4 droundrobin,short +restart gx1 35x4 droundrobin,short +restart gx1 36x4 droundrobin,short +#restart gx1 40x4 droundrobin,short