From 93c2a00178cac96ebfdb97cd675b07d1efaf2e51 Mon Sep 17 00:00:00 2001 From: Tony Craig Date: Fri, 14 Jul 2023 21:36:02 -0700 Subject: [PATCH 1/3] Update Onyx port, Add Derecho intel port (#442) * Initial port to derecho intel * Update derecho port * Update port to Onyx * update onyx port, fix cice/icepack references --- configuration/scripts/icepack.batch.csh | 15 ++++++ .../scripts/machines/Macros.derecho_intel | 49 ++++++++++++++++++ .../scripts/machines/Macros.onyx_intel | 4 +- .../scripts/machines/env.derecho_intel | 51 +++++++++++++++++++ configuration/scripts/machines/env.onyx_cray | 13 ++--- configuration/scripts/machines/env.onyx_gnu | 13 ++--- configuration/scripts/machines/env.onyx_intel | 13 ++--- 7 files changed, 138 insertions(+), 20 deletions(-) create mode 100644 configuration/scripts/machines/Macros.derecho_intel create mode 100644 configuration/scripts/machines/env.derecho_intel diff --git a/configuration/scripts/icepack.batch.csh b/configuration/scripts/icepack.batch.csh index 80d19f4fa..b821bf5c3 100755 --- a/configuration/scripts/icepack.batch.csh +++ b/configuration/scripts/icepack.batch.csh @@ -49,6 +49,21 @@ cat >> ${jobfile} << EOFB #PBS -l walltime=${ICE_RUNLENGTH} EOFB +else if (${ICE_MACHINE} =~ derecho*) then +cat >> ${jobfile} << EOFB +#PBS -q ${ICE_MACHINE_QUEUE} +#PBS -l job_priority=regular +#PBS -N ${ICE_CASENAME} +#PBS -A ${acct} +#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds} +#PBS -l walltime=${ICE_RUNLENGTH} +#PBS -j oe +#PBS -W umask=022 +#PBS -o ${ICE_CASEDIR} +###PBS -M username@domain.com +###PBS -m be +EOFB + else if (${ICE_MACHINE} =~ hobart*) then cat >> ${jobfile} << EOFB #PBS -j oe diff --git a/configuration/scripts/machines/Macros.derecho_intel b/configuration/scripts/machines/Macros.derecho_intel new file mode 100644 index 000000000..27c9a118c --- /dev/null +++ b/configuration/scripts/machines/Macros.derecho_intel @@ -0,0 +1,49 @@ +#============================================================================== +# Makefile macros for NCAR cheyenne, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise -march=core-avx2 + +FIXEDFLAGS := -fixed -132 +FREEFLAGS := -free +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2 +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08 +# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg +else + FFLAGS += -O2 +endif + +SCC := icx +SFC := ifort +CC := $(SCC) +FC := $(SFC) +LD := $(FC) + +NETCDF_PATH := $(NETCDF) + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib + +INCLDIR := $(INCLDIR) + +LIB_NETCDF := $(NETCDF)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) + +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/Macros.onyx_intel b/configuration/scripts/machines/Macros.onyx_intel index 649298e90..0efc3f5de 100644 --- a/configuration/scripts/machines/Macros.onyx_intel +++ b/configuration/scripts/machines/Macros.onyx_intel @@ -4,11 +4,11 @@ CPP := fpp CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} -CFLAGS := -c -O2 -fp-model precise -xHost +CFLAGS := -c -O2 -fp-model precise FIXEDFLAGS := -132 FREEFLAGS := -FR -FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -xHost +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) diff --git a/configuration/scripts/machines/env.derecho_intel b/configuration/scripts/machines/env.derecho_intel new file mode 100644 index 000000000..6067556e8 --- /dev/null +++ b/configuration/scripts/machines/env.derecho_intel @@ -0,0 +1,51 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +module --force purge +module load ncarenv/23.06 +module load craype +module load intel/2023.0.0 +module load ncarcompilers +#module load cray-mpich/8.1.25 +#module load hdf5/1.12.2 +module load netcdf/4.9.2 +module load cray-libsci/23.02.1.1 + +# For perftools with mpiexec +# module load perftools-base +# module load perftools +#setenv PALS_TRANSFER FALSE + +endif + +limit coredumpsize unlimited +limit stacksize unlimited +setenv PALS_QUIET TRUE + +# May be needed for OpenMP memory +setenv OMP_STACKSIZE 64M +# OMP runtime diagnostics +#setenv OMP_DISPLAY_ENV TRUE + +setenv ICE_MACHINE_MACHNAME derecho +setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11" +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, netcdf4.9.2" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/ICEPACK_RUNS +setenv ICE_MACHINE_INPUTDATA /glade/p/cesm/pcwg_dev +setenv ICE_MACHINE_BASELINE /glade/derecho/scratch/$user/ICEPACK_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub" +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "main" +setenv ICE_MACHINE_TPNODE 128 +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "qstat " diff --git a/configuration/scripts/machines/env.onyx_cray b/configuration/scripts/machines/env.onyx_cray index 8080bc0e0..e6290d1d9 100755 --- a/configuration/scripts/machines/env.onyx_cray +++ b/configuration/scripts/machines/env.onyx_cray @@ -13,14 +13,14 @@ module unload PrgEnv-cray module unload PrgEnv-gnu module unload PrgEnv-intel module unload PrgEnv-pgi -module load PrgEnv-cray/6.0.9 +module load PrgEnv-cray/6.0.10 module unload cce -module load cce/11.0.2 +module load cce/14.0.3 module unload cray-mpich module unload cray-mpich2 -module load cray-mpich/7.7.16 +module load cray-mpich/7.7.20 module unload netcdf module unload cray-netcdf @@ -28,8 +28,8 @@ module unload cray-hdf5 module unload cray-hdf5-parallel module unload cray-netcdf-hdf5parallel module unload cray-parallel-netcdf -module load cray-netcdf/4.7.4.0 -module load cray-hdf5/1.12.0.0 +module load cray-netcdf/4.8.1.3 +module load cray-hdf5/1.12.1.3 module unload cray-libsci module unload craype-hugepages2M @@ -39,13 +39,14 @@ module load craype-broadwell setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 64M endif setenv ICE_MACHINE_MACHNAME onyx setenv ICE_MACHINE_MACHINFO "Cray XC40/50 Xeon E5-2699v4 Broadwell" setenv ICE_MACHINE_ENVNAME cray -setenv ICE_MACHINE_ENVINFO "Cray cce/11.0.2" +setenv ICE_MACHINE_ENVINFO "Cray cce/14.0.3, netcdf/4.8.1.3" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR $WORKDIR/ICEPACK_RUNS setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium diff --git a/configuration/scripts/machines/env.onyx_gnu b/configuration/scripts/machines/env.onyx_gnu index efa93a3b4..2fb347e6d 100755 --- a/configuration/scripts/machines/env.onyx_gnu +++ b/configuration/scripts/machines/env.onyx_gnu @@ -13,14 +13,14 @@ module unload PrgEnv-cray module unload PrgEnv-gnu module unload PrgEnv-intel module unload PrgEnv-pgi -module load PrgEnv-gnu/6.0.9 +module load PrgEnv-gnu/6.0.10 module unload gcc -module load gcc/10.2.0 +module load gcc/12.1.0 module unload cray-mpich module unload cray-mpich2 -module load cray-mpich/7.7.16 +module load cray-mpich/7.7.20 module unload netcdf module unload cray-netcdf @@ -28,8 +28,8 @@ module unload cray-hdf5 module unload cray-hdf5-parallel module unload cray-netcdf-hdf5parallel module unload cray-parallel-netcdf -module load cray-netcdf/4.7.4.0 -module load cray-hdf5/1.12.0.0 +module load cray-netcdf/4.8.1.3 +module load cray-hdf5/1.12.1.3 module unload cray-libsci module unload craype-hugepages2M @@ -39,13 +39,14 @@ module load craype-broadwell setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 64M endif setenv ICE_MACHINE_MACHNAME onyx setenv ICE_MACHINE_MACHINFO "Cray XC40/50 Xeon E5-2699v4 Broadwell" setenv ICE_MACHINE_ENVNAME gnu -setenv ICE_MACHINE_ENVINFO "GNU Fortran (GCC) 10.2.0" +setenv ICE_MACHINE_ENVINFO "GNU Fortran (GCC) 12.1.0, netcdf/4.8.1.3" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR $WORKDIR/ICEPACK_RUNS setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium diff --git a/configuration/scripts/machines/env.onyx_intel b/configuration/scripts/machines/env.onyx_intel index ae452081d..a0245171b 100755 --- a/configuration/scripts/machines/env.onyx_intel +++ b/configuration/scripts/machines/env.onyx_intel @@ -13,14 +13,14 @@ module unload PrgEnv-cray module unload PrgEnv-gnu module unload PrgEnv-intel module unload PrgEnv-pgi -module load PrgEnv-intel/6.0.9 +module load PrgEnv-intel/6.0.10 module unload intel -module load intel/19.1.3.304 +module load intel/2021.4.0 module unload cray-mpich module unload cray-mpich2 -module load cray-mpich/7.7.16 +module load cray-mpich/7.7.20 module unload netcdf module unload cray-netcdf @@ -28,8 +28,8 @@ module unload cray-hdf5 module unload cray-hdf5-parallel module unload cray-netcdf-hdf5parallel module unload cray-parallel-netcdf -module load cray-netcdf/4.7.4.0 -module load cray-hdf5/1.12.0.0 +module load cray-netcdf/4.8.1.3 +module load cray-hdf5/1.12.1.3 module unload cray-libsci module unload craype-hugepages2M @@ -39,13 +39,14 @@ module load craype-broadwell setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited limit stacksize unlimited +setenv OMP_STACKSIZE 64M endif setenv ICE_MACHINE_MACHNAME onyx setenv ICE_MACHINE_MACHINFO "Cray XC40/50 Xeon E5-2699v4 Broadwell" setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_ENVINFO "ifort 19.1.3.304" +setenv ICE_MACHINE_ENVINFO "ifort 2021.4.0, netcdf/4.8.1.3" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR $WORKDIR/ICEPACK_RUNS setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium From 0ae5792aaa08a591ccd8f4d02b888d86c806b737 Mon Sep 17 00:00:00 2001 From: Tony Craig Date: Tue, 18 Jul 2023 13:35:06 -0700 Subject: [PATCH 2/3] Update Icepack optional arguments attribute (#443) * Update the optional arguments implementation. Found that passing non-scalar optional arguments down the calling tree without the optional attribute resulted in seg faults on some machines. Updated the implementation and documentation to reflect this change. Cleaned up the rest of the intent lines, so they are not on a new line. * Update documentation --- columnphysics/icepack_aerosol.F90 | 12 +++----- columnphysics/icepack_age.F90 | 3 +- columnphysics/icepack_algae.F90 | 12 +++----- columnphysics/icepack_atmo.F90 | 9 ++---- columnphysics/icepack_brine.F90 | 21 +++++--------- columnphysics/icepack_firstyear.F90 | 6 ++-- columnphysics/icepack_flux.F90 | 3 +- columnphysics/icepack_itd.F90 | 3 +- columnphysics/icepack_mechred.F90 | 3 +- columnphysics/icepack_snow.F90 | 12 +++----- columnphysics/icepack_therm_itd.F90 | 32 +++++++++++++--------- columnphysics/icepack_therm_vertical.F90 | 15 ++++------ columnphysics/icepack_zbgc.F90 | 15 ++++------ columnphysics/icepack_zsalinity.F90 | 12 +++----- doc/source/developer_guide/dg_col_phys.rst | 16 ++++++----- doc/source/user_guide/interfaces.include | 3 +- 16 files changed, 71 insertions(+), 106 deletions(-) diff --git a/columnphysics/icepack_aerosol.F90 b/columnphysics/icepack_aerosol.F90 index 68b8d0a7a..8e6a4933c 100644 --- a/columnphysics/icepack_aerosol.F90 +++ b/columnphysics/icepack_aerosol.F90 @@ -61,12 +61,10 @@ subroutine update_aerosol(dt, & vice_old, & vsno_old - real (kind=dbl_kind), dimension(:), & - intent(in) :: & + real (kind=dbl_kind), dimension(:), intent(in) :: & faero_atm ! aerosol deposition rate (W/m^2 s) - real (kind=dbl_kind), dimension(:), & - intent(inout) :: & + real (kind=dbl_kind), dimension(:), intent(inout) :: & faero_ocn ! aerosol flux to ocean (W/m^2 s) real (kind=dbl_kind), dimension(:,:), intent(inout) :: & @@ -473,12 +471,10 @@ subroutine update_snow_bgc (dt, nblyr, & zbgc_atm, & ! and atm to ice concentration * volume (kg or mmol/m^3*m) flux_bio ! total ocean tracer flux (mmol/m^2/s) - real (kind=dbl_kind), dimension(nbtrcr), & - intent(in) :: & + real (kind=dbl_kind), dimension(nbtrcr), intent(in) :: & flux_bio_atm ! aerosol deposition rate (kg or mmol/m^2 s) - real (kind=dbl_kind), dimension(ntrcr), & - intent(inout) :: & + real (kind=dbl_kind), dimension(ntrcr), intent(inout) :: & trcrn ! ice/snow tracer array ! local variables diff --git a/columnphysics/icepack_age.F90 b/columnphysics/icepack_age.F90 index e3eedf27b..224756936 100644 --- a/columnphysics/icepack_age.F90 +++ b/columnphysics/icepack_age.F90 @@ -26,8 +26,7 @@ subroutine increment_age (dt, iage) real (kind=dbl_kind), intent(in) :: & dt ! time step - real (kind=dbl_kind), & - intent(inout) :: & + real (kind=dbl_kind), intent(inout) :: & iage character(len=*),parameter :: subname='(increment_age)' diff --git a/columnphysics/icepack_algae.F90 b/columnphysics/icepack_algae.F90 index 4a3bce7ac..0ffd880a2 100644 --- a/columnphysics/icepack_algae.F90 +++ b/columnphysics/icepack_algae.F90 @@ -170,12 +170,10 @@ subroutine zbio (dt, nblyr, & icgrid , & ! CICE interface coordinate fswthrul ! visible short wave radiation on icgrid (W/m^2) - real (kind=dbl_kind), dimension(nbtrcr), & - intent(in) :: & + real (kind=dbl_kind), dimension(nbtrcr), intent(in) :: & flux_bio_atm ! aerosol/bgc deposition rate (mmol/m^2 s) - real (kind=dbl_kind), dimension(ntrcr), & - intent(inout) :: & + real (kind=dbl_kind), dimension(ntrcr), intent(inout) :: & trcrn real (kind=dbl_kind), dimension (nblyr+1), intent(inout) :: & @@ -2198,8 +2196,7 @@ subroutine compute_FCT_matrix (C_in, sbdiag, dt, nblyr, & bphin_N, & ! Porosity with min condition on igrid bgrid - real (kind=dbl_kind), dimension (nblyr+1), & - intent(out) :: & + real (kind=dbl_kind), dimension (nblyr+1), intent(out) :: & sbdiag , & ! sub-diagonal matrix elements diag , & ! diagonal matrix elements spdiag , & ! super-diagonal matrix elements @@ -2385,8 +2382,7 @@ subroutine compute_FCT_corr (C_in, C_low, dt, nblyr, & real (kind=dbl_kind), intent(in) :: & dt ! time step - real (kind=dbl_kind), dimension (nblyr+1), & - intent(in) :: & + real (kind=dbl_kind), dimension (nblyr+1), intent(in) :: & D_sbdiag , & ! sub-diagonal artificial diffusion matrix elements ML , & ! Lumped mass diagonal matrix elements D_spdiag ! super-diagonal artificial diffusion matrix elements diff --git a/columnphysics/icepack_atmo.F90 b/columnphysics/icepack_atmo.F90 index 388b848af..b4a97e0a4 100644 --- a/columnphysics/icepack_atmo.F90 +++ b/columnphysics/icepack_atmo.F90 @@ -90,8 +90,7 @@ subroutine atmo_boundary_layer (sfctype, & real (kind=dbl_kind), intent(inout) :: & Cdn_atm_ratio_n ! ratio drag coeff / neutral drag coeff - real (kind=dbl_kind), & - intent(inout) :: & + real (kind=dbl_kind), intent(inout) :: & strx , & ! x surface stress (N) stry ! y surface stress (N) @@ -545,8 +544,7 @@ subroutine neutral_drag_coeffs (apnd, hpnd, & aicen , & ! concentration of ice vicen ! volume per unit area of ice (m) - real (kind=dbl_kind), & - intent(out) :: & + real (kind=dbl_kind), intent(out) :: & hfreebd , & ! freeboard (m) hdraft , & ! draught of ice + snow column (Stoessel1993) hridge , & ! ridge height @@ -863,8 +861,7 @@ subroutine icepack_atm_boundary(sfctype, & Cdn_atm , & ! neutral drag coefficient Cdn_atm_ratio_n ! ratio drag coeff / neutral drag coeff - real (kind=dbl_kind), & - intent(inout) :: & + real (kind=dbl_kind), intent(inout) :: & strx , & ! x surface stress (N) stry ! y surface stress (N) diff --git a/columnphysics/icepack_brine.F90 b/columnphysics/icepack_brine.F90 index 28e2301c6..952b326f5 100644 --- a/columnphysics/icepack_brine.F90 +++ b/columnphysics/icepack_brine.F90 @@ -160,14 +160,12 @@ subroutine compute_microS_mushy (nilyr, nblyr, & real (kind=dbl_kind), dimension (nilyr+1), intent(in) :: & cgrid ! CICE vertical coordinate - real (kind=dbl_kind), & - intent(in) :: & + real (kind=dbl_kind), intent(in) :: & hice_old , & ! previous timestep ice height (m) sss , & ! ocean salinity (ppt) sst ! ocean temperature (C) - real (kind=dbl_kind), dimension(ntrcr), & - intent(in) :: & + real (kind=dbl_kind), dimension(ntrcr), intent(in) :: & trcrn real (kind=dbl_kind), intent(out) :: & @@ -177,19 +175,16 @@ subroutine compute_microS_mushy (nilyr, nblyr, & real (kind=dbl_kind), intent(inout) :: & hbr_old ! previous timestep brine height (m) - real (kind=dbl_kind), dimension (nblyr+1), & - intent(inout) :: & + real (kind=dbl_kind), dimension (nblyr+1), intent(inout) :: & iDin ! tracer diffusivity/h^2 (1/s) includes gravity drainage/molecular - real (kind=dbl_kind), dimension (nblyr+1), & - intent(inout) :: & + real (kind=dbl_kind), dimension (nblyr+1), intent(inout) :: & iphin , & ! porosity on the igrid ibrine_rho , & ! brine rho on interface ibrine_sal , & ! brine sal on interface iTin ! Temperature on the igrid (oC) - real (kind=dbl_kind), dimension (nblyr+2), & - intent(inout) :: & + real (kind=dbl_kind), dimension (nblyr+2), intent(inout) :: & bSin , & ! bulk salinity (ppt) on bgrid brine_sal , & ! equilibrium brine salinity (ppt) brine_rho ! internal brine density (kg/m^3) @@ -313,14 +308,12 @@ subroutine prepare_hbrine (nblyr, & integer (kind=int_kind), intent(in) :: & nblyr ! number of bio layers - real (kind=dbl_kind), dimension (:), & - intent(in) :: & + real (kind=dbl_kind), dimension (:), intent(in) :: & bSin , & ! salinity of ice layers on bio grid (ppt) bTin , & ! temperature of ice layers on bio grid for history (C) i_grid ! biology grid interface points - real (kind=dbl_kind), dimension (:), & - intent(inout) :: & + real (kind=dbl_kind), dimension (:), intent(inout) :: & brine_sal , & ! equilibrium brine salinity (ppt) brine_rho , & ! internal brine density (kg/m^3) ibrine_rho , & ! brine density on interface (kg/m^3) diff --git a/columnphysics/icepack_firstyear.F90 b/columnphysics/icepack_firstyear.F90 index ea2c87c51..02e5c8c17 100644 --- a/columnphysics/icepack_firstyear.F90 +++ b/columnphysics/icepack_firstyear.F90 @@ -42,12 +42,10 @@ subroutine update_FYarea (dt, & dt , & ! time step yday ! day of the year - logical (kind=log_kind), & - intent(in) :: & + logical (kind=log_kind), intent(in) :: & nhmask, shmask - real (kind=dbl_kind), & - intent(inout) :: & + real (kind=dbl_kind), intent(inout) :: & FYarea character(len=*),parameter :: subname='(update_FYarea)' diff --git a/columnphysics/icepack_flux.F90 b/columnphysics/icepack_flux.F90 index 54e33b8ad..5be93277d 100644 --- a/columnphysics/icepack_flux.F90 +++ b/columnphysics/icepack_flux.F90 @@ -251,8 +251,7 @@ subroutine set_sfcflux (aicen, & fcondtopn) ! ice state variables - real (kind=dbl_kind), & - intent(in) :: & + real (kind=dbl_kind), intent(in) :: & aicen , & ! concentration of ice flatn_f , & ! latent heat flux (W/m^2) fsensn_f , & ! sensible heat flux (W/m^2) diff --git a/columnphysics/icepack_itd.F90 b/columnphysics/icepack_itd.F90 index 3c18a4e13..d93970715 100644 --- a/columnphysics/icepack_itd.F90 +++ b/columnphysics/icepack_itd.F90 @@ -681,8 +681,7 @@ subroutine column_sum (nsum, xin, xout) integer (kind=int_kind), intent(in) :: & nsum ! number of categories/layers - real (kind=dbl_kind), dimension (nsum), & - intent(in) :: & + real (kind=dbl_kind), dimension (nsum), intent(in) :: & xin ! input field real (kind=dbl_kind), intent(out) :: & diff --git a/columnphysics/icepack_mechred.F90 b/columnphysics/icepack_mechred.F90 index ea049761f..ecba9f587 100644 --- a/columnphysics/icepack_mechred.F90 +++ b/columnphysics/icepack_mechred.F90 @@ -695,8 +695,7 @@ subroutine ridge_prep (dt, & real (kind=dbl_kind), intent(inout):: & asum ! sum of ice and open water area - real (kind=dbl_kind), & - intent(out):: & + real (kind=dbl_kind), intent(out):: & closing_net, & ! net rate at which area is removed (1/s) divu_adv , & ! divu as implied by transport scheme (1/s) opning ! rate of opening due to divergence/shear diff --git a/columnphysics/icepack_snow.F90 b/columnphysics/icepack_snow.F90 index f661c2882..65a16d016 100644 --- a/columnphysics/icepack_snow.F90 +++ b/columnphysics/icepack_snow.F90 @@ -925,13 +925,11 @@ subroutine snow_dry_metamorph (nslyr,nilyr, dt, rsnw, drsnw_dry, zqsn, & real (kind=dbl_kind), intent(in) :: & dt ! time step (s) - real (kind=dbl_kind), dimension(nslyr), & - intent(in) :: & + real (kind=dbl_kind), dimension(nslyr), intent(in) :: & rsnw, & ! snow grain radius (10^-6 m) zqsn ! snow enthalpy (J m-3) - real (kind=dbl_kind), dimension(nslyr), & - intent(inout) :: & + real (kind=dbl_kind), dimension(nslyr), intent(inout) :: & drsnw_dry ! change due to snow aging (10^-6 m) real (kind=dbl_kind), intent(in) :: & @@ -1190,12 +1188,10 @@ subroutine drain_snow (nslyr, vsnon, aicen, & real (kind=dbl_kind), intent(inout) :: & meltsliq ! total liquid content (kg/m^2) - real (kind=dbl_kind), dimension(nslyr), & - intent(in) :: & + real (kind=dbl_kind), dimension(nslyr), intent(in) :: & massice ! mass of ice in snow (kg/m^2) - real (kind=dbl_kind), dimension(nslyr), & - intent(inout) :: & + real (kind=dbl_kind), dimension(nslyr), intent(inout) :: & massliq ! mass of liquid in snow (kg/m^2) ! local temporary variables diff --git a/columnphysics/icepack_therm_itd.F90 b/columnphysics/icepack_therm_itd.F90 index 97c24c2ee..868d0b8ff 100644 --- a/columnphysics/icepack_therm_itd.F90 +++ b/columnphysics/icepack_therm_itd.F90 @@ -897,13 +897,15 @@ subroutine lateral_melt (dt, ncat, & integer (kind=int_kind), intent(in) :: & ncat , & ! number of thickness categories - nfsd , & ! number of floe size categories nilyr , & ! number of ice layers nblyr , & ! number of bio layers nslyr , & ! number of snow layers n_aero , & ! number of aerosol tracers nbtrcr ! number of bio tracers + integer (kind=int_kind), intent(in), optional :: & + nfsd ! number of floe size categories + real (kind=dbl_kind), dimension (:), intent(inout) :: & aicen , & ! concentration of ice vicen , & ! volume per unit area of ice (m) @@ -913,7 +915,9 @@ subroutine lateral_melt (dt, ncat, & trcrn ! tracer array real (kind=dbl_kind), intent(in) :: & - rside , & ! fraction of ice that melts laterally + rside ! fraction of ice that melts laterally + + real (kind=dbl_kind), intent(in), optional :: & wlat ! lateral melt rate (m/s) real (kind=dbl_kind), intent(inout) :: & @@ -933,14 +937,14 @@ subroutine lateral_melt (dt, ncat, & real (kind=dbl_kind), dimension(:), intent(inout) :: & faero_ocn ! aerosol flux to ocean (kg/m^2/s) - real (kind=dbl_kind), dimension(:), intent(inout) :: & + real (kind=dbl_kind), dimension(:), intent(inout), optional :: & fiso_ocn ! isotope flux to ocean (kg/m^2/s) - real (kind=dbl_kind), dimension (:), intent(in) :: & + real (kind=dbl_kind), dimension (:), intent(in), optional :: & floe_rad_c , & ! fsd size bin centre in m (radius) floe_binwidth ! fsd size bin width in m (radius) - real (kind=dbl_kind), dimension (:), intent(out) :: & + real (kind=dbl_kind), dimension (:), intent(out), optional :: & d_afsd_latm ! change in fsd due to lateral melt (m) ! local variables @@ -1327,7 +1331,6 @@ subroutine add_new_ice (ncat, nilyr, & integer (kind=int_kind), intent(in) :: & ncat , & ! number of thickness categories - nfsd , & ! number of floe size categories nilyr , & ! number of ice layers nblyr , & ! number of bio layers ntrcr , & ! number of tracers @@ -1335,6 +1338,9 @@ subroutine add_new_ice (ncat, nilyr, & n_aero, & ! number of aerosol tracers ktherm ! type of thermodynamics (-1 none, 1 BL99, 2 mushy) + integer (kind=int_kind), intent(in), optional :: & + nfsd ! number of floe size categories + real (kind=dbl_kind), dimension(0:ncat), intent(in) :: & hin_max ! category boundaries (m) @@ -1402,30 +1408,30 @@ subroutine add_new_ice (ncat, nilyr, & ! water isotopes - real (kind=dbl_kind), dimension(:), intent(inout) :: & + real (kind=dbl_kind), dimension(:), intent(inout), optional :: & fiso_ocn ! isotope flux to ocean (kg/m^2/s) - real (kind=dbl_kind), intent(in) :: & + real (kind=dbl_kind), intent(in), optional :: & HDO_ocn , & ! ocean concentration of HDO (kg/kg) H2_16O_ocn , & ! ocean concentration of H2_16O (kg/kg) H2_18O_ocn ! ocean concentration of H2_18O (kg/kg) ! floe size distribution - real (kind=dbl_kind), intent(in) :: & + real (kind=dbl_kind), intent(in), optional :: & wave_sig_ht ! significant height of waves globally (m) - real (kind=dbl_kind), dimension(:), intent(in) :: & + real (kind=dbl_kind), dimension(:), intent(in), optional :: & wave_spectrum ! ocean surface wave spectrum, E(f) (m^2 s) - real(kind=dbl_kind), dimension(:), intent(in) :: & + real(kind=dbl_kind), dimension(:), intent(in), optional :: & wavefreq, & ! wave frequencies (s^-1) dwavefreq ! wave frequency bin widths (s^-1) - real (kind=dbl_kind), dimension (:), intent(in) :: & + real (kind=dbl_kind), dimension (:), intent(in), optional :: & floe_rad_c , & ! fsd size bin centre in m (radius) floe_binwidth ! fsd size bin width in m (radius) - real (kind=dbl_kind), dimension(:), intent(out) :: & + real (kind=dbl_kind), dimension(:), intent(out), optional :: & ! change in thickness distribution (area) d_afsd_latg , & ! due to fsd lateral growth d_afsd_newi ! new ice formation diff --git a/columnphysics/icepack_therm_vertical.F90 b/columnphysics/icepack_therm_vertical.F90 index 3e6aea082..f4796d553 100644 --- a/columnphysics/icepack_therm_vertical.F90 +++ b/columnphysics/icepack_therm_vertical.F90 @@ -141,8 +141,7 @@ subroutine thermo_vertical (nilyr, nslyr, & massliq ! liquid water mass in snow (kg/m^2) ! input from atmosphere - real (kind=dbl_kind), & - intent(in) :: & + real (kind=dbl_kind), intent(in) :: & flw , & ! incoming longwave radiation (W/m^2) potT , & ! air potential temperature (K) Qa , & ! specific humidity (kg/kg) @@ -151,8 +150,7 @@ subroutine thermo_vertical (nilyr, nslyr, & shcoef , & ! transfer coefficient for sensible heat lhcoef ! transfer coefficient for latent heat - real (kind=dbl_kind), & - intent(inout) :: & + real (kind=dbl_kind), intent(inout) :: & fswsfc , & ! SW absorbed at ice/snow surface (W m-2) fswint , & ! SW absorbed in ice interior, below surface (W m-2) fpond ! fresh water flux to ponds (kg/m^2/s) @@ -695,8 +693,7 @@ subroutine init_vertical_profile(nilyr, nslyr, & real (kind=dbl_kind), dimension (:), intent(in) :: & zSin ! internal ice layer salinities - real (kind=dbl_kind), dimension (:), & - intent(inout) :: & + real (kind=dbl_kind), dimension (:), intent(inout) :: & zqsn , & ! snow enthalpy zTsn ! snow temperature @@ -1775,14 +1772,12 @@ subroutine freeboard (nslyr, & ! real (kind=dbl_kind), intent(in) :: & ! dt ! time step - real (kind=dbl_kind), & - intent(inout) :: & + real (kind=dbl_kind), intent(inout) :: & snoice , & ! snow-ice formation (m/step-->cm/day) dsnow ! change in snow thickness after snow-ice formation (m) ! iage ! ice age (s) - real (kind=dbl_kind), & - intent(inout) :: & + real (kind=dbl_kind), intent(inout) :: & hin , & ! ice thickness (m) hsn ! snow thickness (m) diff --git a/columnphysics/icepack_zbgc.F90 b/columnphysics/icepack_zbgc.F90 index 964efd442..715a2c514 100644 --- a/columnphysics/icepack_zbgc.F90 +++ b/columnphysics/icepack_zbgc.F90 @@ -98,8 +98,7 @@ subroutine add_new_ice_bgc (dt, nblyr, & real (kind=dbl_kind), intent(in) :: & dt ! time step (s) - real (kind=dbl_kind), dimension (:), & - intent(in) :: & + real (kind=dbl_kind), dimension (:), intent(in) :: & aicen_init , & ! initial concentration of ice vicen_init , & ! intiial volume per unit area of ice (m) aicen , & ! concentration of ice @@ -108,8 +107,7 @@ subroutine add_new_ice_bgc (dt, nblyr, & real (kind=dbl_kind), intent(in) :: & vsnon1 ! category 1 snow volume per unit area (m) - real (kind=dbl_kind), dimension (:,:), & - intent(inout) :: & + real (kind=dbl_kind), dimension (:,:), intent(inout) :: & trcrn ! ice tracers real (kind=dbl_kind), intent(in) :: & @@ -122,12 +120,10 @@ subroutine add_new_ice_bgc (dt, nblyr, & real (kind=dbl_kind), intent(in) :: & hsurp ! thickness of new ice added to each cat - real (kind=dbl_kind), dimension (:), & - intent(inout) :: & + real (kind=dbl_kind), dimension (:), intent(inout) :: & flux_bio ! tracer flux to ocean from biology (mmol/m^2/s) - real (kind=dbl_kind), dimension (:), & - intent(in) :: & + real (kind=dbl_kind), dimension (:), intent(in) :: & ocean_bio ! ocean concentration of biological tracer ! local @@ -398,8 +394,7 @@ subroutine adjust_tracer_profile (nbtrcr, dt, ntrcr, & real (kind=dbl_kind), dimension (nilyr+1), intent(in) :: & cgrid ! CICE grid - real (kind=dbl_kind), dimension (ntrcr), & - intent(inout) :: & + real (kind=dbl_kind), dimension (ntrcr), intent(inout) :: & trcrn ! ice tracers ! local variables diff --git a/columnphysics/icepack_zsalinity.F90 b/columnphysics/icepack_zsalinity.F90 index 16de63bd0..8d52b022f 100644 --- a/columnphysics/icepack_zsalinity.F90 +++ b/columnphysics/icepack_zsalinity.F90 @@ -119,12 +119,10 @@ subroutine zsalinity (n_cat, dt, & brine_sal , & ! brine salinity (ppt) brine_rho ! brine density (kg/m^3) - real (kind=dbl_kind), dimension (nblyr), & - intent(inout) :: & + real (kind=dbl_kind), dimension (nblyr), intent(inout) :: & trcrn_S ! salinity tracer ppt (on bio grid) - real (kind=dbl_kind), dimension (nilyr), & - intent(inout) :: & + real (kind=dbl_kind), dimension (nilyr), intent(inout) :: & trcrn_q , & ! enthalpy tracer trcrn_Si ! salinity on CICE grid @@ -258,12 +256,10 @@ subroutine solve_zsalinity (nilyr, nblyr, & brine_sal , & ! brine salinity (ppt) brine_rho ! brine density (kg/m^3) - real (kind=dbl_kind), dimension (nblyr), & - intent(inout) :: & + real (kind=dbl_kind), dimension (nblyr), intent(inout) :: & trcrn_S ! salinity tracer ppt (on bio grid) - real (kind=dbl_kind), dimension (nilyr), & - intent(inout) :: & + real (kind=dbl_kind), dimension (nilyr), intent(inout) :: & trcrn_q , & ! enthalpy tracer trcrn_Si ! salinity on CICE grid diff --git a/doc/source/developer_guide/dg_col_phys.rst b/doc/source/developer_guide/dg_col_phys.rst index c144a91f3..7fd5389ea 100755 --- a/doc/source/developer_guide/dg_col_phys.rst +++ b/doc/source/developer_guide/dg_col_phys.rst @@ -129,7 +129,9 @@ Overall, columnphysics changes in the Icepack model should include the following * Icepack is a simple serial code. Global flags and parameters should be set identically on all tasks/threads that call into Icepack. Icepack has no ability to reconcile or identify inconsistencies between different tasks/threads. All aspects of correct parallel implementation is managed by the driver code. - * Optional arguments are encouraged in the public Icepack interfaces. They provide backwards compatibility in the public interfaces and allow future extensions. Argument that are not always required should ultimately be made optional. There are several ways optional arguments can be passed down the calling tree in Icepack. Two options, copying into local data or copying into module data are viable. But the recommended approach is to + * Optional arguments are encouraged in the public Icepack interfaces. They provide backwards compatibility in the public interfaces and allow future extensions. Arguments that are not always required should ultimately be made optional. There are several ways optional arguments can be passed down the calling tree in Icepack. Two options, copying into local data or copying into module data are viable. But the recommended approach is to pass optional arguments down the calling tree, + + * Optional arguments can be used as calls are made down the calling tree without regard to whether they are present or not. * Use universal flags and parameters to turn on/off features. Avoid having features triggered by the presence of optional arguments. @@ -137,7 +139,7 @@ Overall, columnphysics changes in the Icepack model should include the following * Leverage the icepack subroutine ``icepack_checkoptargflags`` which controls how often to check the optional arguments. The ``argcheck`` namelist setting controls when to do the checks, 'never', 'first', or 'always' are valid settings - * Pass all optional arguments down the calling tree as needed. Optional arguments can be passed down a calling tree as non-optional as long as they are not used unless they have been passed in above. This is the recommended method for optional arguments within Icepack. Sometimes the optional attribute needs to be defined in lower level routines for a variable if that variable needs to be checked by Fortran's present. That's OK, but should generally be avoided if possible. + * Pass optional arguments down the calling tree within Icepack as needed. In Fortran, the present attribute is carried down the calling tree automatically, but the ``optional`` attribute should also be defined in lower level subroutines. This is not strictly required in cases where the subroutine is always called with the optional arguments, but it's good practice. * An example of how this might look is @@ -182,8 +184,8 @@ Overall, columnphysics changes in the Icepack model should include the following subroutine some_columnphysics_subroutine(arg1, arg2, arg3, ...) real (kind=dbl_kind), intent(inout) :: arg1 - real (kind=dbl_kind), dimension(:), intent(inout) :: arg2 - real (kind=dbl_kind), intent(inout) :: arg3 + real (kind=dbl_kind), optional, dimension(:), intent(inout) :: arg2 + real (kind=dbl_kind), optional, intent(inout) :: arg3 if (flag_arg2) then arg2(:) = ... @@ -199,7 +201,7 @@ Overall, columnphysics changes in the Icepack model should include the following subroutine someother_columnphysics_subroutine(arg3) - real (kind=dbl_kind), intent(inout) :: arg3 + real (kind=dbl_kind), optional, intent(inout) :: arg3 arg3 = ... @@ -210,7 +212,7 @@ Overall, columnphysics changes in the Icepack model should include the following * If optional arguments are passed but not needed, this is NOT an error. If optional argument are not passed but needed, this is an error. - * If checking and implementation are done properly, optional arguments that are not needed will never be referenced anywhere in Icepack at that timestep. Optional arguments should be matched with the appropriate flags at the first entry into Icepack. + * If checking and implementation are done properly, optional arguments that are not needed will never be referenced anywhere in Icepack at that timestep. Optional arguments should be matched with the appropriate flags at the first entry into Icepack as much as possible. - * There is a unit test in CICE to verify robustness of this approach. + * There is a unit test (optarg) in CICE to verify optional argument passing. There is also a unit test (opticep) in CICE that checks that NOT passing the optional arguments from CICE is robust. diff --git a/doc/source/user_guide/interfaces.include b/doc/source/user_guide/interfaces.include index b55be984c..2c641afa4 100644 --- a/doc/source/user_guide/interfaces.include +++ b/doc/source/user_guide/interfaces.include @@ -42,8 +42,7 @@ icepack_atm_boundary Cdn_atm , & ! neutral drag coefficient Cdn_atm_ratio_n ! ratio drag coeff / neutral drag coeff - real (kind=dbl_kind), & - intent(inout) :: & + real (kind=dbl_kind), intent(inout) :: & strx , & ! x surface stress (N) stry ! y surface stress (N) From 4728746ea2926bf10acc5de354b3eae16d418af5 Mon Sep 17 00:00:00 2001 From: Tony Craig Date: Tue, 18 Jul 2023 13:35:32 -0700 Subject: [PATCH 3/3] Fix hsn_new(1) one-line fix. (#444) See https://github.com/CICE-Consortium/Icepack/issues/441 --- columnphysics/icepack_snow.F90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/columnphysics/icepack_snow.F90 b/columnphysics/icepack_snow.F90 index 65a16d016..94a0f1251 100644 --- a/columnphysics/icepack_snow.F90 +++ b/columnphysics/icepack_snow.F90 @@ -689,6 +689,8 @@ subroutine snow_redist(dt, nslyr, ncat, wind, ain, vin, vsn, zqsn, & hslyr, hsn_new(n), & zqsn(:,n)) if (icepack_warnings_aborted(subname)) return + else + hsn_new(1) = hsn_new(1) + dhsn endif ! nslyr > 1 endif ! |dhsn| > puny endif ! ain > puny