Update S4 environment and module files (#1303)

Updates the S4 module file and environment file to keep up to date with the current develop branch. Fixes #1297. Addresses one bug in #1195. This also increases the memory request for the ediag job. During testing, that job initially failed. For the 5 cycles run between 2022051500 - 2022051600, the largest memory footprint was ~26.5GB.
NOAA-EMC · Feb 14, 2023 · 5ac6836 · 5ac6836
1 parent 1040216
commit 5ac6836
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 85 deletions.
diff --git a/env/S4.env b/env/S4.env
@@ -1,10 +1,11 @@
-#!/usr/bin/env bash
+#! /usr/bin/env bash
 
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
     echo "argument can be any one of the following:"
     echo "atmanalrun atmensanalrun"
+    echo "aeroanlrun"
     echo "anal sfcanl fcst post vrfy metp"
     echo "eobs eupd ecen efcs epos"
     echo "postsnd awips gempak"
@@ -30,264 +31,256 @@ export NTHSTACK=1024000000
 ulimit -s unlimited
 ulimit -a
 
-if [[ ${step} = "prep" || ${step} = "prepbufr" ]]; then
+if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
 
-    npe_node_prep=${npe_node_prep:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_prep))
 
     export POE="NO"
     export BACK="NO"
     export sys_tp="S4"
     export launcher_PREP="srun"
 
-elif [[ ${step} = "waveinit" || ${step} = "waveprep" || ${step} = "wavepostsbs" || ${step} = "wavepostbndpnt" || ${step} = "wavepostbndpntbll" || ${step} = "wavepostpnt" ]]; then
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export CFP_MP="YES"
-    if [[ ${step} = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
     export wavempexec=${launcher}
     export wave_mpmd=${mpmd_opt}
 
-elif [[ ${step} = "atmanalrun" ]]; then
+elif [[ "${step}" = "atmanalrun" ]]; then
 
     export CFP_MP=${CFP_MP:-"YES"}
     export USE_CFP=${USE_CFP:-"YES"}
-    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
-    npe_node_atmanalrun=${npe_node_atmanalrun:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_atmanalrun))
 
     export NTHREADS_ATMANAL=${nth_atmanalrun:-${nth_max}}
     [[ ${NTHREADS_ATMANAL} -gt ${nth_max} ]] && export NTHREADS_ATMANAL=${nth_max}
-    export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun:-0}"
+    export APRUN_ATMANAL="${launcher} -n ${npe_atmanalrun}"
 
-elif [[ ${step} = "atmensanalrun" ]]; then
+elif [[ "${step}" = "atmensanalrun" ]]; then
 
     export CFP_MP=${CFP_MP:-"YES"}
     export USE_CFP=${USE_CFP:-"YES"}
-    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
-    npe_node_atmensanalrun=${npe_node_atmensanalrun:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_atmensanalrun))
 
     export NTHREADS_ATMENSANAL=${nth_atmensanalrun:-${nth_max}}
     [[ ${NTHREADS_ATMENSANAL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANAL=${nth_max}
-    export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun:-0}"
+    export APRUN_ATMENSANAL="${launcher} -n ${npe_atmensanalrun}"
 
-elif [[ ${step} = "aeroanlrun" ]]; then
+elif [[ "${step}" = "aeroanlrun" ]]; then
 
-    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
-    npe_node_aeroanlrun=${npe_node_aeroanlrun:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_aeroanlrun))
 
     export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}}
     [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
-    export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun:-0}"
+    export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}"
 
-elif [[ ${step} = "anal" || ${step} = "analcalc" ]]; then
+elif [[ "${step}" = "ocnanalbmat" ]]; then
+     echo "WARNING: ${step} is not enabled on S4!"
+
+elif [[ "${step}" = "ocnanalrun" ]]; then
+     echo "WARNING: ${step} is not enabled on S4!"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO
 
     export CFP_MP=${CFP_MP:-"YES"}
     export USE_CFP=${USE_CFP:-"YES"}
-    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
-    npe_node_anal=${npe_node_anal:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_anal))
 
     export NTHREADS_GSI=${nth_anal:-${nth_max}}
     [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
-    export APRUN_GSI=${launcher}
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}}"
 
     export NTHREADS_CALCINC=${nth_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
-    export APRUN_CALCINC=${launcher}
+    export APRUN_CALCINC="${launcher} \$ncmd"
 
     export NTHREADS_CYCLE=${nth_cycle:-12}
     [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
     npe_cycle=${ntiles:-6}
-    export APRUN_CYCLE="${launcher} -n ${npe_cycle:-0}"
+    export APRUN_CYCLE="${launcher} -n ${npe_cycle}"
 
 
     export NTHREADS_GAUSFCANL=1
     npe_gausfcanl=${npe_gausfcanl:-1}
-    export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl:-0}"
+    export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}"
 
-elif [[ ${step} = "sfcanl" ]]; then
-    npe_node_sfcanl=${npe_node_sfcanl:-${npe_node_max}}
+elif [[ "${step}" = "sfcanl" ]]; then
     nth_max=$((npe_node_max / npe_node_sfcanl))
 
     export NTHREADS_CYCLE=${nth_sfcanl:-14}
     [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
     npe_sfcanl=${ntiles:-6}
-    export APRUN_CYCLE="${launcher} -n ${npe_sfcanl:-0}"
+    export APRUN_CYCLE="${launcher} -n ${npe_sfcanl}"
 
-elif [[ ${step} = "gldas" ]]; then
+elif [[ "${step}" = "gldas" ]]; then
+
+    export USE_CFP="NO"
+    export CFP_MP="YES"
 
-    npe_node_gldas=${npe_node_gldas:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_gldas))
 
     export NTHREADS_GLDAS=${nth_gldas:-${nth_max}}
     [[ ${NTHREADS_GLDAS} -gt ${nth_max} ]] && export NTHREADS_GLDAS=${nth_max}
-    export APRUN_GLDAS="${launcher} -n ${npe_gldas:-0}"
+    export APRUN_GLDAS="${launcher} -n ${npe_gldas}"
 
     export NTHREADS_GAUSSIAN=${nth_gaussian:-1}
     [[ ${NTHREADS_GAUSSIAN} -gt ${nth_max} ]] && export NTHREADS_GAUSSIAN=${nth_max}
-    export APRUN_GAUSSIAN="${launcher} -n ${npe_gaussian:-0}"
+    export APRUN_GAUSSIAN="${launcher} -n ${npe_gaussian}"
 
 # Must run data processing with exactly the number of tasks as time
 # periods being processed.
 
-    gldas_spinup_hours=${gldas_spinup_hours:-0}
     npe_gldas_data_proc=$((gldas_spinup_hours + 12))
-    export APRUN_GLDAS_DATA_PROC="${launcher} -n ${npe_gldas_data_proc} --multi-prog"
+    export APRUN_GLDAS_DATA_PROC="${launcher} -n ${npe_gldas_data_proc} ${mpmd_opt}"
 
-elif [[ ${step} = "eobs" ]]; then
+elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO
 
-    export CFP_MP=${CFP_MP:-"YES"}
-    export USE_CFP=${USE_CFP:-"YES"}
-    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
-
-    npe_node_eobs=${npe_node_eobs:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_eobs))
 
     export NTHREADS_GSI=${nth_eobs:-${nth_max}}
     [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
-    export APRUN_GSI=${launcher}
-
-elif [[ ${step} = "eupd" ]]; then
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_eobs}}"
 
     export CFP_MP=${CFP_MP:-"YES"}
     export USE_CFP=${USE_CFP:-"YES"}
-    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+elif [[ "${step}" = "eupd" ]]; then
 
-    npe_node_eupd=${npe_node_eupd:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_eupd))
 
     export NTHREADS_ENKF=${nth_eupd:-${nth_max}}
     [[ ${NTHREADS_ENKF} -gt ${nth_max} ]] && export NTHREADS_ENKF=${nth_max}
-    export APRUN_ENKF=${launcher}
+    export APRUN_ENKF="${launcher} -n ${npe_enkf:-${npe_eupd}}"
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
-elif [[ ${step} = "fcst" ]]; then
+elif [[ "${step}" = "fcst" ]]; then
 
     #PEs and PEs/node can differ for GFS and GDAS forecasts if threading differs
-    if [[ ${CDUMP:-gdas} == "gfs" ]]; then
-        npe_fcst=${npe_fcst_gfs:-0}
-        npe_node_fcst=${npe_node_fcst_gfs:-${npe_node_max}}
-        nth_fv3=${nth_fv3_gfs:-1}
+    if [[ ${CDUMP} == "gfs" ]]; then
+        npe_fcst=${npe_fcst_gfs}
+        npe_node_fcst=${npe_node_fcst_gfs}
+        nth_fv3=${nth_fv3_gfs}
     fi
 
-    npe_node_fcst=${npe_node_fcst:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_fcst))
 
     export NTHREADS_FV3=${nth_fv3:-${nth_max}}
     [[ ${NTHREADS_FV3} -gt ${nth_max} ]] && export NTHREADS_FV3=${nth_max}
     export cores_per_node=${npe_node_max}
-    export APRUN_FV3="${launcher} -n ${npe_fcst:-0}"
+    export APRUN_FV3="${launcher} -n ${npe_fcst}"
 
     export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
     [[ ${NTHREADS_REGRID_NEMSIO} -gt ${nth_max} ]] && export NTHREADS_REGRID_NEMSIO=${nth_max}
-    export APRUN_REGRID_NEMSIO=${launcher}
+    export APRUN_REGRID_NEMSIO="${launcher} -n ${LEVS}"
 
     export NTHREADS_REMAP=${nth_remap:-2}
     [[ ${NTHREADS_REMAP} -gt ${nth_max} ]] && export NTHREADS_REMAP=${nth_max}
-    export APRUN_REMAP=${launcher}
+    export APRUN_REMAP="${launcher} -n ${npe_remap:-${npe_fcst}}"
     export I_MPI_DAPL_UD="enable"
 
-elif [[ ${step} = "efcs" ]]; then
+elif [[ "${step}" = "efcs" ]]; then
 
-    npe_node_efcs=${npe_node_efcs:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_efcs))
 
     export NTHREADS_FV3=${nth_efcs:-${nth_max}}
     [[ ${NTHREADS_FV3} -gt ${nth_max} ]] && export NTHREADS_FV3=${nth_max}
     export cores_per_node=${npe_node_max}
-    export APRUN_FV3="${launcher} -n ${npe_efcs:-0}"
+    export APRUN_FV3="${launcher} -n ${npe_efcs}"
 
     export NTHREADS_REGRID_NEMSIO=${nth_regrid_nemsio:-1}
     [[ ${NTHREADS_REGRID_NEMSIO} -gt ${nth_max} ]] && export NTHREADS_REGRID_NEMSIO=${nth_max}
-    export APRUN_REGRID_NEMSIO="${launcher} ${LEVS:-128}"
+    export APRUN_REGRID_NEMSIO="${launcher} -n ${LEVS}"
 
-elif [[ ${step} = "post" ]]; then
+elif [[ "${step}" = "post" ]]; then
 
-    npe_node_post=${npe_node_post:-npe_node_max}
     nth_max=$((npe_node_max / npe_node_post))
 
     export NTHREADS_NP=${nth_np:-1}
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
-    export APRUN_NP=${launcher}
+    export APRUN_NP="${launcher} -n ${npe_post}"
 
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
-    export APRUN_DWN=${launcher}
+    export APRUN_DWN="${launcher} -n ${npe_dwn}"
 
-elif [[ ${step} = "ecen" ]]; then
+elif [[ "${step}" = "ecen" ]]; then
 
-    npe_node_ecen=${npe_node_ecen:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_ecen))
 
     export NTHREADS_ECEN=${nth_ecen:-${nth_max}}
     [[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max}
-    export APRUN_ECEN=${launcher}
+    export APRUN_ECEN="${launcher} -n ${npe_ecen}"
 
     export NTHREADS_CHGRES=${nth_chgres:-12}
     [[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max}
     export APRUN_CHGRES="time"
 
     export NTHREADS_CALCINC=${nth_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
-    export APRUN_CALCINC=${launcher}
+    export APRUN_CALCINC="${launcher} -n ${npe_ecen}"
 
-elif [[ ${step} = "esfc" ]]; then
+elif [[ "${step}" = "esfc" ]]; then
 
-    npe_node_esfc=${npe_node_esfc:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_esfc))
 
     export NTHREADS_ESFC=${nth_esfc:-${nth_max}}
     [[ ${NTHREADS_ESFC} -gt ${nth_max} ]] && export NTHREADS_ESFC=${nth_max}
-    export APRUN_ESFC="${launcher} -n ${npe_esfc:-0}"
+    export APRUN_ESFC="${launcher} -n ${npe_esfc}"
 
     export NTHREADS_CYCLE=${nth_cycle:-14}
     [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
-    export APRUN_CYCLE="${launcher} -n ${npe_esfc:-0}"
+    export APRUN_CYCLE="${launcher} -n ${npe_esfc}"
 
-elif [[ ${step} = "epos" ]]; then
+elif [[ "${step}" = "epos" ]]; then
 
-    npe_node_epos=${npe_node_epos:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_epos))
 
     export NTHREADS_EPOS=${nth_epos:-${nth_max}}
     [[ ${NTHREADS_EPOS} -gt ${nth_max} ]] && export NTHREADS_EPOS=${nth_max}
-    export APRUN_EPOS=${launcher}
+    export APRUN_EPOS="${launcher} -n ${npe_epos}"
 
-elif [[ ${step} = "init" ]]; then
+elif [[ "${step}" = "init" ]]; then
 
-    export APRUN=${launcher}
+    export APRUN="${launcher} -n ${npe_init}"
 
-elif [[ ${step} = "postsnd" ]]; then
+elif [[ "${step}" = "postsnd" ]]; then
 
     export CFP_MP="YES"
 
-    npe_node_postsnd=${npe_node_postsnd:-${npe_node_max}}
     nth_max=$((npe_node_max / npe_node_postsnd))
 
     export NTHREADS_POSTSND=${nth_postsnd:-1}
     [[ ${NTHREADS_POSTSND} -gt ${nth_max} ]] && export NTHREADS_POSTSND=${nth_max}
-    export APRUN_POSTSND=${launcher}
+    export APRUN_POSTSND="${launcher} -n ${npe_postsnd}"
 
     export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
     [[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max}
-    export APRUN_POSTSNDCFP=${launcher}
+    export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsndcfp} ${mpmd_opt}"
 
-elif [[ ${step} = "awips" ]]; then
+elif [[ "${step}" = "awips" ]]; then
 
      echo "WARNING: ${step} is not enabled on S4!"
 
-elif [[ ${step} = "gempak" ]]; then
+elif [[ "${step}" = "gempak" ]]; then
 
      echo "WARNING: ${step} is not enabled on S4!"
 fi
diff --git a/modulefiles/module_base.s4.lua b/modulefiles/module_base.s4.lua
@@ -24,8 +24,8 @@ load(pathJoin("png", "1.6.35"))
 
 load(pathJoin("hdf5", "1.10.6"))
 load(pathJoin("netcdf", "4.7.4"))
-load(pathJoin("pio", "2.5.2"))
-load(pathJoin("esmf", "8.2.1b04"))
+load(pathJoin("pio", "2.5.7"))
+load(pathJoin("esmf", "8.3.0b09"))
 load(pathJoin("fms", "2021.03"))
 
 load(pathJoin("bacio", "2.4.1"))

diff --git a/parm/config/config.resources b/parm/config/config.resources
@@ -729,7 +729,7 @@ elif [ ${step} = "ediag" ]; then
     export npe_ediag=48
     export nth_ediag=1
     export npe_node_ediag=$(echo "${npe_node_max} / ${nth_ediag}" | bc)
-    export memory_ediag="24GB"
+    export memory_ediag="30GB"
 
 elif [ ${step} = "eupd" ]; then
 

diff --git a/scripts/exgdas_enkf_earc.sh b/scripts/exgdas_enkf_earc.sh
@@ -48,7 +48,7 @@ if (( 10#${ENSGRP} > 0 )) && [[ ${HPSSARCH} = "YES" || ${LOCALARCH} = "YES" ]];
    SAVEWARMICB="NO"
    mm=$(echo "${CDATE}"|cut -c 5-6)
    dd=$(echo "${CDATE}"|cut -c 7-8)
-   nday=$(( (mm-1)*30+dd ))
+   nday=$(( (10#${mm}-1)*30+10#${dd} ))
    mod=$((nday % ARCH_WARMICFREQ))
    if [ "${CDATE}" -eq "${firstday}" ] && [ "${cyc}" -eq "${EARCINC_CYC}" ]; then SAVEWARMICA="YES" ; fi
    if [ "${CDATE}" -eq "${firstday}" ] && [ "${cyc}" -eq "${EARCICS_CYC}" ]; then SAVEWARMICB="YES" ; fi