[develop] Add restart capability to forecast task

Externals.cfg

@@ -58,7 +58,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/AQM-utils
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = b066931
+hash = b58ba99
 local_path = sorc/AQM-utils
 required = True
 

parm/FV3.input.yml

@@ -290,6 +290,7 @@ FV3_GFS_v15p2:
     lsoil: !!python/none
     lsoil_lsm: !!python/none
     ltaerosol: !!python/none
+    nstf_name: [2, 0, 0, 0, 0]
     shal_cnv: True
     shinhong: False
     ttendlim: !!python/none
@@ -315,11 +316,20 @@ FV3_GFS_v15_thompson_mynn_lam3km:
   gfs_physics_nml:
     avg_max_length: 3600.0
     cdmbgwd: [0.88, 0.04]
+    debug: True
     do_deep: False
+    do_gsl_drag_ls_bl: False
+    do_gsl_drag_ss: True
+    do_gsl_drag_tofd: True
     do_mynnsfclay: True
+    do_tofd: False
     do_ugwp: False
+    do_ugwp_v0: False
+    do_ugwp_v0_nst_only: False
+    do_ugwp_v0_orog_only: False
     fhswr: 900.0
     fhlwr: 900.0
+    gwd_opt: 2
     iaer: 1011
     iccn: 2
     icliq_sw: 2
@@ -337,16 +347,21 @@ FV3_GFS_v15_thompson_mynn_lam3km:
     iopt_snf: 4
     iopt_stc: 1
     iopt_tbot: 2
-    iopt_trs: 2
+    iopt_trs: !!python/none
     iovr: 3
     ldiag_ugwp: False
     lgfdlmprad: False
     lsm: 1
+    lsoil: !!python/none
     lsoil_lsm: !!python/none
     ltaerosol: False    
+    nstf_name: [2, 0, 0, 0, 0]
+    print_diff_pgr: True
+    sfclay_compute_flux: !!python/none
     xkzminv: 0.3
     xkzm_m: 1.0
     xkzm_h: 1.0
+  surf_map_nml: !!python/none
 
 FV3_GFS_v16:
   cires_ugwp_nml:
@@ -356,6 +371,8 @@ FV3_GFS_v16:
     agrid_vel_rst: False
     d2_bg_k1: 0.2
     d2_bg_k2: 0.0
+    delt_max: 0.002
+    dz_min: 6
     fv_sg_adj: 450
     hord_dp: -5
     hord_mt: 5
@@ -395,7 +412,7 @@ FV3_GFS_v16:
     lndp_type: !!python/none
     lsoil: 4
     n_var_lndp: !!python/none
-    nstf_name: [2, 1, 0, 0, 0]
+    nstf_name: [2, 0, 0, 0, 0]
     prautco: [0.00015, 0.00015]
     psautco: [0.0008, 0.0005]
     satmedmf: True

parm/model_configure

@@ -1,23 +1,26 @@
-total_member:            1
-PE_MEMBER01:             {{ PE_MEMBER01 }}
 start_year:              {{ start_year }}
 start_month:             {{ start_month }}
 start_day:               {{ start_day }}
 start_hour:              {{ start_hour }}
 start_minute:            0
 start_second:            0
 nhours_fcst:             {{ nhours_fcst }}
+fhrot:                   {{ fhrot }}
 RUN_CONTINUE:            .false.
 ENS_SPS:                 .false.
 dt_atmos:                {{ dt_atmos }}
 calendar:                'julian'
 memuse_verbose:          .false.
-atmos_nthreads:          {{ atmos_nthreads }}
 restart_interval:        {{ restart_interval }}
 output_1st_tstep_rst:    .false.
 write_dopost:            {{ write_dopost }}
 ideflate:                0
 nbits:                   0
+ichunk2d:                -1
+jchunk2d:                -1
+ichunk3d:                -1
+jchunk3d:                -1
+kchunk3d:                -1
 quilting:                {{ quilting }}
 {% if quilting  %}
 #

parm/nems.configure

@@ -2,10 +2,13 @@
 ####  NEMS Run-Time Configuration File  #####
 #############################################
 
-{% if not print_esmf %}
 # ESMF #
+{%- if print_esmf %}
+logKindFlag:            ESMF_LOGKIND_MULTI
+{%- else %}
 logKindFlag:            ESMF_LOGKIND_MULTI_ON_ERROR
-{% endif %}
+{%- endif %}
+globalResourceControl:  true
 
 {% if cpl_aqm  %}
 # EARTH #
@@ -40,9 +43,23 @@ runSeq::
   @
 ::
 {% else %}
+# EARTH #
 EARTH_component_list: ATM
- ATM_model:            fv3
- runSeq::
-   ATM
- ::
+EARTH_attributes::
+  Verbosity = 0
+::
+
+# ATM #
+ATM_model:                      fv3
+ATM_petlist_bounds:             0 {{ pe_member01_m1 }}
+ATM_omp_num_threads:            {{ atm_omp_num_threads }}
+ATM_attributes::
+  Verbosity = 0
+  Diagnostic = 0
+::
+
+# Run Sequence #
+runSeq::
+  ATM
+::
 {% endif %}

scripts/exregional_make_orog.sh

@@ -244,7 +244,8 @@ mv_vrfy "${raw_orog_fp_orig}" "${raw_orog_fp}"
 #
 #-----------------------------------------------------------------------
 #
-if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ] || [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v17_p8" ]; then
+suites=( FV3_HRRR FV3_GFS_v15_thompson_mynn_lam3km  FV3_GFS_v17_p8 )
+if [[ ${suites[@]} =~ "${CCPP_PHYS_SUITE}" ]] ; then
   DATA="${DATA:-${OROG_DIR}/temp_orog_data}"
   mkdir_vrfy -p ${DATA}
   cd_vrfy ${DATA}

scripts/exregional_run_fcst.sh

@@ -48,13 +48,19 @@ specified cycle.
 #
 #-----------------------------------------------------------------------
 #
-# Set OpenMP variables.
+# Set environment variables.
 #
 #-----------------------------------------------------------------------
 #
 export KMP_AFFINITY=${KMP_AFFINITY_RUN_FCST}
 export OMP_NUM_THREADS=${OMP_NUM_THREADS_RUN_FCST}
 export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_FCST}
+export MPI_TYPE_DEPTH=20
+export ESMF_RUNTIME_COMPLIANCECHECK=OFF:depth=4
+if [ "${PRINT_ESMF}" = "TRUE" ]; then
+  export ESMF_RUNTIME_PROFILE=ON
+  export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
+fi
 #
 #-----------------------------------------------------------------------
 #
@@ -204,18 +210,16 @@ create_symlink_to_file target="$target" symlink="$symlink" \
 # that the FV3 model is hardcoded to recognize, and those are the names 
 # we use below.
 #
-if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ] || [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v17_p8" ]; then
-
-  fileids=( "ss" "ls" )
-  for fileid in "${fileids[@]}"; do
-    target="${FIXlam}/${CRES}${DOT_OR_USCORE}oro_data_${fileid}.tile${TILE_RGNL}.halo${NH0}.nc"
-    symlink="oro_data_${fileid}.nc"
+suites=( FV3_HRRR FV3_GFS_v15_thompson_mynn_lam3km  FV3_GFS_v17_p8 )
+if [[ ${suites[@]} =~ "${CCPP_PHYS_SUITE}" ]] ; then
+  file_ids=( "ss" "ls" )
+  for file_id in "${file_ids[@]}"; do
+    target="${FIXlam}/${CRES}${DOT_OR_USCORE}oro_data_${file_id}.tile${TILE_RGNL}.halo${NH0}.nc"
+    symlink="oro_data_${file_id}.nc"
     create_symlink_to_file target="$target" symlink="$symlink" \
                            relative="${relative_link_flag}"
   done
-
 fi
-
 #
 #-----------------------------------------------------------------------
 #
@@ -449,18 +453,96 @@ cat > itag <<EOF
 /
 EOF
 fi
+#
+#-----------------------------------------------------------------------
+#
+# Update or copy the FV3 input.nml file
+#
+#-----------------------------------------------------------------------
+#
+if [ "${DO_ENSEMBLE}" = TRUE ] && ([ "${DO_SPP}" = TRUE ] || [ "${DO_SPPT}" = TRUE ] || [ "${DO_SHUM}" = TRUE ] || \
+   [ "${DO_SKEB}" = TRUE ] || [ "${DO_LSM_SPP}" =  TRUE ]); then
+  python3 $USHdir/set_FV3nml_ens_stoch_seeds.py \
+      --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \
+      --cdate "$CDATE" || print_err_msg_exit "\
+Call to function to create the ensemble-based namelist for the current
+cycle's (cdate) run directory (DATA) failed:
+  cdate = \"${CDATE}\"
+  DATA = \"${DATA}\""
+else
+  cp_vrfy "${FV3_NML_FP}" "${DATA}/${FV3_NML_FN}"
+fi
+#
+#-----------------------------------------------------------------------
+#
+# Replace parameter values for restart in FV3 input.nml and model_configure.
+# Add restart files to INPUT directory.
+#
+#-----------------------------------------------------------------------
+#
+flag_fcst_restart="FALSE"
+if [ "${DO_FCST_RESTART}" = "TRUE" ] && [ "$(ls -A ${DATA}/RESTART )" ]; then
+  cp_vrfy input.nml input.nml_orig
+  cp_vrfy model_configure model_configure_orig
+  if [ "${CPL_AQM}" = "TRUE" ]; then
+    cp_vrfy aqm.rc aqm.rc_orig
+  fi
+  relative_link_flag="FALSE"
+  flag_fcst_restart="TRUE"
 
-if [ "${CPL_AQM}" = "TRUE" ]; then
+  # Update FV3 input.nml for restart
+  python3 $USHdir/update_restart_input_nml_file.py \
+    --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \
+    --run_dir "${DATA}" \
+    || print_err_msg_exit "\
+Call to function to update the FV3 input.nml file for restart for the 
+current cycle's (cdate) run directory (DATA) failed:
+  cdate = \"${CDATE}\"
+  DATA = \"${DATA}\""
+
+  # Search for available restart files and find the latest hour
+  fnm_rst_pdy=()
+  fnm_rst_hms=() 
+  for fnm_rst in "${DATA}/RESTART"/*; do
+    fnm_rst_base=$( basename "${fnm_rst}" )
+    fnm_rst_pdy+=( $( echo "${fnm_rst_base}" | awk -F"." '{ print $1 }' ) )
+    fnm_rst_hms+=( $( echo "${fnm_rst_base}" | awk -F"." '{ print $2 }' ) )
+  done
+
+  # Find latest restart hour
+  fnm_rst_pdy_max=$( printf '%s\n' "${fnm_rst_pdy[@]}" | sort -nu | tail -1 )
+  fnm_rst_hms_max=$( printf '%s\n' "${fnm_rst_hms[@]}" | sort -nu | tail -1 )
+
+  # Compute start hour of restart run for model_configure
+  cdate_org=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC" "+%s" )
+  cdate_res=$( $DATE_UTIL --utc --date "${fnm_rst_pdy_max} ${fnm_rst_hms_max:0:2} UTC" "+%s" )
+  FHROT=$(( ( cdate_res - cdate_org )/(60*60) ))
+
+  # Create soft-link of restart files in INPUT directory
+  cd_vrfy ${DATA}/INPUT
+
+  file_ids=( "coupler.res" "fv_core.res.nc" "fv_core.res.tile1.nc" "fv_srf_wnd.res.tile1.nc" "fv_tracer.res.tile1.nc" "phy_data.nc" "sfc_data.nc" )
+  for file_id in "${file_ids[@]}"; do
+    rm_vrfy "${file_id}"
+    target="${DATA}/RESTART/${fnm_rst_pdy_max}.${fnm_rst_hms_max}.${file_id}"
+    symlink="${file_id}"
+    create_symlink_to_file target="$target" symlink="$symlink" relative="${relative_link_flag}"
+  done
+
+  cd_vrfy ${DATA}   
+fi
 #
 #-----------------------------------------------------------------------
 #
 # Setup air quality model cold/warm start
 #
 #-----------------------------------------------------------------------
 #
-  init_concentrations="false"
-  if [ "${COLDSTART}" = "TRUE" ] && [ "${PDY}${cyc}" = "${DATE_FIRST_CYCL:0:10}" ]; then
+if [ "${CPL_AQM}" = "TRUE" ]; then
+  if [ "${COLDSTART}" = "TRUE" ] && [ "${PDY}${cyc}" = "${DATE_FIRST_CYCL:0:10}" ] && [ "${flag_fcst_restart}" = "FALSE" ]; then
     init_concentrations="true"
+  else
+    init_concentrations="false"
   fi
 #
 #-----------------------------------------------------------------------
@@ -474,31 +556,14 @@ if [ "${CPL_AQM}" = "TRUE" ]; then
     --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \
     --cdate "$CDATE" \
     --run-dir "${DATA}" \
-    --init-concentration "${init_concentrations}" \
+    --init_concentrations "${init_concentrations}" \
     || print_err_msg_exit "\
 Call to function to create an aqm.rc file for the current
 cycle's (cdate) run directory (DATA) failed:
   cdate = \"${CDATE}\"
   DATA = \"${DATA}\""
 fi
-#
-#-----------------------------------------------------------------------
-#
 
-if [ "${DO_ENSEMBLE}" = TRUE ] && ([ "${DO_SPP}" = TRUE ] || [ "${DO_SPPT}" = TRUE ] || [ "${DO_SHUM}" = TRUE ] || \
-   [ "${DO_SKEB}" = TRUE ] || [ "${DO_LSM_SPP}" =  TRUE ]); then
-  python3 $USHdir/set_FV3nml_ens_stoch_seeds.py \
-      --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \
-      --cdate "$CDATE" || print_err_msg_exit "\
-Call to function to create the ensemble-based namelist for the current
-cycle's (cdate) run directory (DATA) failed:
-  cdate = \"${CDATE}\"
-  DATA = \"${DATA}\""
-else
-  create_symlink_to_file target="${FV3_NML_FP}" \
-                         symlink="${DATA}/${FV3_NML_FN}" \
-                         relative="${relative_link_flag}"
-fi
 #
 #-----------------------------------------------------------------------
 #
@@ -511,6 +576,7 @@ python3 $USHdir/create_model_configure_file.py \
   --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \
   --cdate "$CDATE" \
   --fcst_len_hrs "${FCST_LEN_HRS}" \
+  --fhrot "${FHROT}" \
   --run-dir "${DATA}" \
   --sub-hourly-post "${SUB_HOURLY_POST}" \
   --dt-subhourly-post-mnts "${DT_SUBHOURLY_POST_MNTS}" \

ush/config.aqm.community.yaml

@@ -59,8 +59,10 @@ task_run_fcst:
   WTIME_RUN_FCST: 02:00:00
   QUILTING: true
   PRINT_ESMF: false
+  DO_FCST_RESTART: false
 task_run_post:
   POST_OUTPUT_DOMAIN_NAME: 793
+  USE_CUSTOM_POST_CONFIG_FILE: false
 global:
   DO_ENSEMBLE: false
   NUM_ENS_MEMBERS: 2

ush/config.aqm.nco.realtime.yaml

@@ -70,13 +70,15 @@ task_run_fcst:
   LAYOUT_X: 50
   LAYOUT_Y: 34
   BLOCKSIZE: 16
-  RESTART_INTERVAL: 6 18
+  RESTART_INTERVAL: 6 24 36 48 60
   WTIME_RUN_FCST: 04:00:00
   OMP_NUM_THREADS_RUN_FCST: 1
   QUILTING: true
   PRINT_ESMF: false
+  DO_FCST_RESTART: false
 task_run_post:
   POST_OUTPUT_DOMAIN_NAME: 793
+  USE_CUSTOM_POST_CONFIG_FILE: false
 task_aqm_lbcs:
   WTIME_AQM_LBCS: 01:00:00
 global: