[develop] Add restart capability to forecast task

Externals.cfg

@@ -58,7 +58,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/AQM-utils
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = b066931
+hash = b58ba99
 local_path = sorc/AQM-utils
 required = True
 

jobs/JREGIONAL_BIAS_CORRECTION_O3

@@ -65,10 +65,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
 export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"
 
 TMP_STDAY=`${NDATE} -8760 ${PDY}${cyc} | cut -c1-8`  # 1 year back
-# The earliest day of training data is August 1 2020
-if [ ${TMP_STDAY} -le 20220810 ]; then 
-  TMP_STDAY=20220810 
-fi
 export BC_STDAY=${BC_STDAY:-${TMP_STDAY}}
 #
 #-----------------------------------------------------------------------

jobs/JREGIONAL_BIAS_CORRECTION_PM25

@@ -65,10 +65,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
 export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"
 
 TMP_STDAY=`${NDATE} -8760 ${PDY}${cyc} | cut -c1-8`  # 1 year back
-# The earliest day of training data is August 1 2020
-if [ ${TMP_STDAY} -le 20220810 ]; then 
-  TMP_STDAY=20220810 
-fi
 export BC_STDAY=${BC_STDAY:-${TMP_STDAY}}
 #
 #-----------------------------------------------------------------------

parm/FV3.input.yml

@@ -315,11 +315,20 @@ FV3_GFS_v15_thompson_mynn_lam3km:
   gfs_physics_nml:
     avg_max_length: 3600.0
     cdmbgwd: [0.88, 0.04]
+    debug: True
     do_deep: False
+    do_gsl_drag_ls_bl: False
+    do_gsl_drag_ss: True
+    do_gsl_drag_tofd: True
     do_mynnsfclay: True
+    do_tofd: False
     do_ugwp: False
+    do_ugwp_v0: False
+    do_ugwp_v0_nst_only: False
+    do_ugwp_v0_orog_only: False
     fhswr: 900.0
     fhlwr: 900.0
+    gwd_opt: 2
     iaer: 1011
     iccn: 2
     icliq_sw: 2
@@ -337,16 +346,20 @@ FV3_GFS_v15_thompson_mynn_lam3km:
     iopt_snf: 4
     iopt_stc: 1
     iopt_tbot: 2
-    iopt_trs: 2
+    iopt_trs: !!python/none
     iovr: 3
     ldiag_ugwp: False
     lgfdlmprad: False
     lsm: 1
+    lsoil: !!python/none
     lsoil_lsm: !!python/none
     ltaerosol: False    
+    print_diff_pgr: True
+    sfclay_compute_flux: !!python/none
     xkzminv: 0.3
     xkzm_m: 1.0
     xkzm_h: 1.0
+  surf_map_nml: !!python/none
 
 FV3_GFS_v16:
   cires_ugwp_nml:
@@ -356,6 +369,8 @@ FV3_GFS_v16:
     agrid_vel_rst: False
     d2_bg_k1: 0.2
     d2_bg_k2: 0.0
+    delt_max: 0.002
+    dz_min: 6
     fv_sg_adj: 450
     hord_dp: -5
     hord_mt: 5
@@ -395,7 +410,6 @@ FV3_GFS_v16:
     lndp_type: !!python/none
     lsoil: 4
     n_var_lndp: !!python/none
-    nstf_name: [2, 1, 0, 0, 0]
     prautco: [0.00015, 0.00015]
     psautco: [0.0008, 0.0005]
     satmedmf: True
@@ -506,7 +520,6 @@ FV3_GFS_v17_p8:
     min_lakeice: 0.15
     min_seaice: 0.15
     nsfullradar_diag: !!python/none
-    nstf_name: [2, 0, 0, 0, 0]
     prautco: [0.00015, 0.00015]
     psautco: [0.0008, 0.0005]
     qdiag3d: False

parm/FV3LAM_wflow.xml

@@ -839,7 +839,6 @@ MODULES_RUN_TASK_FP script.
     <native>{{ native_run_fcst }}</native>
   {%- elif machine in ["WCOSS2"]  %}
     <nodes>{{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}:tpp={{ omp_num_threads_run_fcst }}</nodes>
-    <native>&SCHED_NATIVE_CMD;</native>
     <nodesize>&NCORES_PER_NODE;</nodesize>
   {%- else %}
     <nodes>{{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }}</nodes>
@@ -1387,7 +1386,11 @@ it, we'd have to include more jinja if-statements here.
   <task name="&TN_BIAS_CORRECTION_O3;" cycledefs="forecast" maxtries="{{ maxtries_bias_correction_o3 }}">
     &RSRV_DEFAULT;
     <command>&LOAD_MODULES_RUN_TASK_FP; "&TN_BIAS_CORRECTION_O3;" "&JOBSdir;/JREGIONAL_BIAS_CORRECTION_O3"</command>
+{%- if machine in ["WCOSS2"]  %}
+    <nodes>{{ nnodes_bias_correction_o3 }}:ppn={{ ppn_bias_correction_o3 }}:tpp={{ omp_num_threads_bias_correction_o3 }}</nodes>
+{%- else %}
     <nodes>{{ nnodes_bias_correction_o3 }}:ppn={{ ppn_bias_correction_o3 }}</nodes>
+{%- endif %}
 {%- if machine not in ["GAEA"]  %}
     <memory>{{ mem_bias_correction_o3 }}</memory>
 {%- endif %}
@@ -1418,7 +1421,11 @@ it, we'd have to include more jinja if-statements here.
   <task name="&TN_BIAS_CORRECTION_PM25;" cycledefs="forecast" maxtries="{{ maxtries_bias_correction_pm25 }}">
     &RSRV_DEFAULT;
     <command>&LOAD_MODULES_RUN_TASK_FP; "&TN_BIAS_CORRECTION_PM25;" "&JOBSdir;/JREGIONAL_BIAS_CORRECTION_PM25"</command>
+{%- if machine in ["WCOSS2"]  %}
+    <nodes>{{ nnodes_bias_correction_pm25 }}:ppn={{ ppn_bias_correction_pm25 }}:tpp={{ omp_num_threads_bias_correction_pm25 }}</nodes>
+{%- else %}
     <nodes>{{ nnodes_bias_correction_pm25 }}:ppn={{ ppn_bias_correction_pm25 }}</nodes>
+{%- endif %}
 {%- if machine not in ["GAEA"]  %}
     <memory>{{ mem_bias_correction_pm25 }}</memory>
 {%- endif %}
@@ -1443,7 +1450,6 @@ it, we'd have to include more jinja if-statements here.
 {%- endif %}
 
 
-
 {#-
 Tasks for combining (adding) hourly APCP (accumulated precipitation) from
 forecasts to obtain APCP obs for longer accumulation periods (3 hours,

parm/model_configure

@@ -1,23 +1,26 @@
-total_member:            1
-PE_MEMBER01:             {{ PE_MEMBER01 }}
 start_year:              {{ start_year }}
 start_month:             {{ start_month }}
 start_day:               {{ start_day }}
 start_hour:              {{ start_hour }}
 start_minute:            0
 start_second:            0
 nhours_fcst:             {{ nhours_fcst }}
+fhrot:                   {{ fhrot }}
 RUN_CONTINUE:            .false.
 ENS_SPS:                 .false.
 dt_atmos:                {{ dt_atmos }}
 calendar:                'julian'
 memuse_verbose:          .false.
-atmos_nthreads:          {{ atmos_nthreads }}
 restart_interval:        {{ restart_interval }}
 output_1st_tstep_rst:    .false.
 write_dopost:            {{ write_dopost }}
 ideflate:                0
 nbits:                   0
+ichunk2d:                -1
+jchunk2d:                -1
+ichunk3d:                -1
+jchunk3d:                -1
+kchunk3d:                -1
 quilting:                {{ quilting }}
 {% if quilting  %}
 #

parm/nems.configure

@@ -2,10 +2,13 @@
 ####  NEMS Run-Time Configuration File  #####
 #############################################
 
-{% if not print_esmf %}
 # ESMF #
+{%- if print_esmf %}
+logKindFlag:            ESMF_LOGKIND_MULTI
+{%- else %}
 logKindFlag:            ESMF_LOGKIND_MULTI_ON_ERROR
-{% endif %}
+{%- endif %}
+globalResourceControl:  true
 
 {% if cpl_aqm  %}
 # EARTH #
@@ -40,9 +43,23 @@ runSeq::
   @
 ::
 {% else %}
+# EARTH #
 EARTH_component_list: ATM
- ATM_model:            fv3
- runSeq::
-   ATM
- ::
+EARTH_attributes::
+  Verbosity = 0
+::
+
+# ATM #
+ATM_model:                      fv3
+ATM_petlist_bounds:             0 {{ pe_member01_m1 }}
+ATM_omp_num_threads:            {{ atm_omp_num_threads }}
+ATM_attributes::
+  Verbosity = 0
+  Diagnostic = 0
+::
+
+# Run Sequence #
+runSeq::
+  ATM
+::
 {% endif %}

scripts/exregional_bias_correction_o3.sh

@@ -50,9 +50,11 @@ This is the ex-script for the task that runs BIAS-CORRECTION-O3.
 #
 #-----------------------------------------------------------------------
 #
+set -x
 export KMP_AFFINITY=${KMP_AFFINITY_BIAS_CORRECTION_O3}
 export OMP_NUM_THREADS=${OMP_NUM_THREADS_BIAS_CORRECTION_O3}
 export OMP_STACKSIZE=${OMP_STACKSIZE_BIAS_CORRECTION_O3}
+export OMP_PLACES=cores
 #
 #-----------------------------------------------------------------------
 #
@@ -84,8 +86,6 @@ rm_vrfy -rf "$DATA"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
 
-set -x
-
 yyyy=${PDY:0:4}
 yyyymm=${PDY:0:6}
 yyyy_m1=${PDYm1:0:4}
@@ -211,7 +211,7 @@ fi
 
 mkdir_vrfy -p ${DATA}/data/sites
 cp_vrfy ${PARMaqm_utils}/bias_correction/config.ozone.bias_corr_${id_domain}.${cyc}z ${DATA}
- 
+
 PREP_STEP
 eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_correct config.ozone.bias_corr_${id_domain}.${cyc}z ${cyc}z ${BC_STDAY} ${PDY} ${REDIRECT_OUT_ERR} || print_err_msg_exit "Call to executable to run AQM_BIAS_CORRECT returned with nonzero exit code."
 POST_STEP

scripts/exregional_bias_correction_pm25.sh

@@ -50,9 +50,11 @@ This is the ex-script for the task that runs BIAS-CORRECTION-PM25.
 #
 #-----------------------------------------------------------------------
 #
+set -x
 export KMP_AFFINITY=${KMP_AFFINITY_BIAS_CORRECTION_PM25}
 export OMP_NUM_THREADS=${OMP_NUM_THREADS_BIAS_CORRECTION_PM25}
 export OMP_STACKSIZE=${OMP_STACKSIZE_BIAS_CORRECTION_PM25}
+export OMP_PLACES=cores
 #
 #-----------------------------------------------------------------------
 #
@@ -84,8 +86,6 @@ rm_vrfy -rf $DATA
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
 
-set -x
-
 yyyy=${PDY:0:4}
 yyyymm=${PDY:0:6}
 yyyy_m1=${PDYm1:0:4}

scripts/exregional_make_orog.sh

@@ -244,7 +244,8 @@ mv_vrfy "${raw_orog_fp_orig}" "${raw_orog_fp}"
 #
 #-----------------------------------------------------------------------
 #
-if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ] || [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v17_p8" ]; then
+suites=( "FV3_HRRR" "FV3_GFS_v15_thompson_mynn_lam3km" "FV3_GFS_v17_p8" )
+if [[ ${suites[@]} =~ "${CCPP_PHYS_SUITE}" ]] ; then
   DATA="${DATA:-${OROG_DIR}/temp_orog_data}"
   mkdir_vrfy -p ${DATA}
   cd_vrfy ${DATA}