RRFS_B ensemble retro (NOAA-EMC#447)

* Update scripts for running ensemble retros on Hera. * changes for running RRFSE retro * Update retro data path on Jet * Use spinup cycles for ensemble initialization * Update JREGIONAL_RUN_FCST * Update JREGIONAL_RUN_PREPSTART * Update JREGIONAL_SAVE_RESTART * Updating config.sh for CONUS_13km * Update model_configure to have different output frequency * Updating config.sh for CONUS_3km Co-authored-by: Ming.Hu <[email protected]>
EricJames-NOAA · Nov 16, 2022 · b7387a8 · b7387a8
1 parent da620e0
commit b7387a8
Show file tree

Hide file tree

Showing 19 changed files with 850 additions and 399 deletions.
diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST
@@ -66,10 +66,12 @@ the specified cycle.
 #   others(prod): this job is for product cycle
 #
 CYCLE_TYPE=${CYCLE_TYPE:-prod}
+CYCLE_SUBTYPE=${CYCLE_SUBTYPE:-empty}
 if [ ${CYCLE_TYPE} == "spinup" ]; then
   run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_spinup"
-elif  [ ${CYCLE_TYPE} == "ensinit" ]; then
-  run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit"
+  if  [ ${CYCLE_SUBTYPE} == "ensinit" ]; then
+    run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit"
+  fi
 elif  [ ${CYCLE_TYPE} == "ensfcst" ]; then
   run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_long"
 else
@@ -93,6 +95,7 @@ mkdir_vrfy -p ${gridspec_dir}
 $SCRIPTSDIR/exregional_run_fcst.sh \
   cdate="${CDATE}" \
   cycle_type="${CYCLE_TYPE}" \
+  cycle_subtype="${CYCLE_SUBTYPE}" \
   cycle_dir="${run_dir}" \
   gridspec_dir="${gridspec_dir}" \
   ensmem_indx="${ENSMEM_INDX}" \

diff --git a/jobs/JREGIONAL_RUN_PREPSTART b/jobs/JREGIONAL_RUN_PREPSTART
@@ -67,19 +67,14 @@ the specified cycle.
 #   others(prod): this job is for product cycle
 #
 CYCLE_TYPE=${CYCLE_TYPE:-prod}
+CYCLE_SUBTYPE=${CYCLE_SUBTYPE:-empty}
 if [ ${CYCLE_TYPE} == "spinup" ]; then
   modelinputdir=${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_spinup/INPUT
-elif [ ${CYCLE_TYPE} == "ensinit" ]; then
-  modelinputdir=${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam/INPUT
+  if [ ${DO_ENSINIT} == "TRUE" ] && [ ${CYCLE_SUBTYPE} == "ensinit" ]; then
+     modelinputdir=${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit/INPUT
+  fi
 else
   modelinputdir=${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam/INPUT
-  if [ ${DO_ENSINIT} == "TRUE" ]; then
-    for cyc_start in "${CYCL_HRS_ENSINIT[@]}"; do
-      if [ ${HH} -eq ${cyc_start} ]; then 
-        modelinputdir=${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit/INPUT
-      fi
-    done
-  fi
 fi
 
 rm -fr ${modelinputdir}
@@ -99,6 +94,7 @@ gridspec_dir=${NWGES_BASEDIR}/grid_spec
 $SCRIPTSDIR/exregional_run_prepstart.sh \
              cycle_dir="${CYCLE_DIR}" \
              cycle_type="${CYCLE_TYPE}" \
+             cycle_subtype="${CYCLE_SUBTYPE}" \
              gridspec_dir="${gridspec_dir}" \
              modelinputdir="${modelinputdir}" \
              lbcs_root="${LBCS_ROOT}" \

diff --git a/jobs/JREGIONAL_SAVE_RESTART b/jobs/JREGIONAL_SAVE_RESTART
@@ -66,12 +66,14 @@ on the output files corresponding to a specified forecast hour.
 # NWGES_DIR is the directory in which the model restart file will be saved
 #
 CYCLE_TYPE=${CYCLE_TYPE:-prod}
+CYCLE_SUBTYPE=${CYCLE_SUBTYPE:-empty}
 if [ ${CYCLE_TYPE} == "spinup" ]; then
   run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_spinup"
   nwges_dir="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_spinup"
-elif [ ${CYCLE_TYPE} == "ensinit" ]; then
-  run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit"
-  nwges_dir="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit"
+  if [ ${CYCLE_SUBTYPE} == "ensinit" ]; then
+    run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit"
+    nwges_dir="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit"
+  fi
 else
   run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam"
   nwges_dir="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam"
@@ -109,7 +111,8 @@ $SCRIPTSDIR/exregional_save_restart.sh \
   run_dir="${run_dir}" \
   nwges_dir="${nwges_dir}" \
   fhr="${fhr}" \
-  cycle_type="${CYCLE_TYPE}" || \
+  cycle_type="${CYCLE_TYPE}"  \
+  cycle_subtype="${CYCLE_SUBTYPE}" || \
 print_err_msg_exit "\
 Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed."
 #

diff --git a/scripts/exregional_run_analysis.sh b/scripts/exregional_run_analysis.sh
@@ -215,6 +215,11 @@ if  [[ ${regional_ensemble_option:-1} -eq 5 ]]; then
     restart_prefix="${YYYYMMDD}.${HH}0000."
     slash_ensmem_subdir=$memchar
     bkpathmem=${rrfse_fg_root}/${YYYYMMDDHHmInterv}/${slash_ensmem_subdir}/fcst_fv3lam/RESTART
+    for cycl_hrs in ${CYCL_HRS_PRODSTART_ENS}; do
+     if [ $HH == ${cycl_hrs} ]; then
+       bkpathmem=${rrfse_fg_root}/${YYYYMMDDHHmInterv}/${slash_ensmem_subdir}/fcst_fv3lam_spinup/RESTART
+     fi
+    done
 
     dynvarfile=${bkpathmem}/${restart_prefix}fv_core.res.tile1.nc
     tracerfile=${bkpathmem}/${restart_prefix}fv_tracer.res.tile1.nc

diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh
@@ -67,6 +67,7 @@ specified cycle.
 valid_args=( \
 "cdate" \
 "cycle_type" \
+"cycle_subtype" \
 "cycle_dir" \
 "gridspec_dir" \
 "ensmem_indx" \
@@ -563,6 +564,7 @@ fi
 create_model_configure_file \
   cdate="$cdate" \
   cycle_type="$cycle_type" \
+  cycle_subtype="$cycle_subtype" \
   nthreads=${OMP_NUM_THREADS:-1} \
   run_dir="${run_dir}" || print_err_msg_exit "\
 Call to function to create a model configuration file for the current

diff --git a/scripts/exregional_run_prepstart.sh b/scripts/exregional_run_prepstart.sh
@@ -55,7 +55,7 @@ specified cycle.
 #
 #-----------------------------------------------------------------------
 #
-valid_args=( "cycle_dir" "cycle_type" "gridspec_dir" "modelinputdir" "lbcs_root" "fg_root")
+valid_args=( "cycle_dir" "cycle_type" "cycle_subtype" "gridspec_dir" "modelinputdir" "lbcs_root" "fg_root")
 process_args valid_args "$@"
 #
 #-----------------------------------------------------------------------
@@ -172,13 +172,10 @@ if [ ${cycle_type} == "spinup" ]; then
       BKTYPE=1
     fi
   done
-elif [ ${cycle_type} == "ensinit" ]; then
-  echo "ensinit cycle - warm start from 1 timestep restart files"
-  for cyc_start in "${CYCL_HRS_ENSINIT[@]}"; do
-    if [ ${HH} -eq ${cyc_start} ]; then
-      BKTYPE=0
-    fi
-  done
+  if [ ${cycle_subtype} == "spinup" ]; then
+    echo "ensinit cycle - warm start from 1 timestep restart files"
+    BKTYPE=0
+  fi
 else
   echo " product cycle"
   for cyc_start in "${CYCL_HRS_PRODSTART[@]}"; do
@@ -260,7 +257,7 @@ else
 #
   restart_prefix="${YYYYMMDD}.${HH}0000."
 
-  if [ ${cycle_type} == "ensinit" ] ; then
+  if [ ${cycle_subtype} == "spinup" ] ; then
 # point to the 0-h cycle for the warm start from the 1 timestep restart files
     fg_restart_dirname=fcst_fv3lam_ensinit
     bkpath=${fg_root}/${YYYYMMDDHH}${SLASH_ENSMEM_SUBDIR}/${fg_restart_dirname}/RESTART  # cycling, use background from RESTART
@@ -330,7 +327,7 @@ else
         done
       done
     fi
-    if [ ${cycle_type} == "ensinit" ] ; then
+    if [ ${cycle_subtype} == "spinup" ] ; then
       cp_vrfy ${fg_root}/${YYYYMMDDHH}${SLASH_ENSMEM_SUBDIR}/${fg_restart_dirname}/INPUT/gfs_ctrl.nc  gfs_ctrl.nc
     else
       cp_vrfy ${fg_root}/${YYYYMMDDHHmInterv}${SLASH_ENSMEM_SUBDIR}/${fg_restart_dirname}/INPUT/gfs_ctrl.nc  gfs_ctrl.nc
@@ -357,7 +354,7 @@ else
     ncatted -a checksum,,d,, fv_core.res.nc
 
 # generate coupler.res with right date
-    if [ ${cycle_type} == "ensinit" ]; then
+    if [ ${cycle_subtype} == "spinup" ]; then
 # from the 1 timestep restart files
       head -2 bk_coupler.res > coupler.res
       head -2 bk_coupler.res | tail -1 >> coupler.res

diff --git a/scripts/exregional_run_recenter.sh b/scripts/exregional_run_recenter.sh
@@ -157,7 +157,11 @@ cd_vrfy ${recenterdir}
 # loop through ensemble members to link all the member files
 #
 
-fg_restart_dirname=fcst_fv3lam
+if [ ${cycle_type} == "spinup" ]; then
+  fg_restart_dirname=fcst_fv3lam_spinup
+else
+  fg_restart_dirname=fcst_fv3lam
+fi
 
 imem=1
 for imem in  $(seq 1 $nens)
@@ -201,14 +205,14 @@ dynvarfile_control=${ENSCTRL_CYCLE_DIR}/fcst_fv3lam/INPUT/fv_core.res.tile1.nc
 tracerfile_control=${ENSCTRL_CYCLE_DIR}/fcst_fv3lam/INPUT/fv_tracer.res.tile1.nc
 dynvarfile_control_spinup=${ENSCTRL_CYCLE_DIR}/fcst_fv3lam_spinup/INPUT/fv_core.res.tile1.nc
 tracerfile_control_spinup=${ENSCTRL_CYCLE_DIR}/fcst_fv3lam_spinup/INPUT/fv_tracer.res.tile1.nc
-if [ -r "${dynvarfile_control}" ] && [ -r "${tracerfile_control}" ] ; then
-  ln -sf ${ENSCTRL_CYCLE_DIR}/fcst_fv3lam/INPUT/fv_core.res.tile1.nc  ./control_dynvar
-  ln -sf ${ENSCTRL_CYCLE_DIR}/fcst_fv3lam/INPUT/fv_tracer.res.tile1.nc   ./control_tracer
-  ln -sf ${ENSCTRL_CYCLE_DIR}/fcst_fv3lam/INPUT/sfc_data.nc  ./control_sfcvar
-elif [ -r "${dynvarfile_control_spinup}" ] && [ -r "${tracerfile_control_spinup}" ] ; then
+if [ -r "${dynvarfile_control_spinup}" ] && [ -r "${tracerfile_control_spinup}" ] ; then
   ln -sf ${ENSCTRL_CYCLE_DIR}/fcst_fv3lam_spinup/INPUT/fv_core.res.tile1.nc  ./control_dynvar
   ln -sf ${ENSCTRL_CYCLE_DIR}/fcst_fv3lam_spinup/INPUT/fv_tracer.res.tile1.nc   ./control_tracer
   ln -sf ${ENSCTRL_CYCLE_DIR}/fcst_fv3lam_spinup/INPUT/sfc_data.nc  ./control_sfcvar
+elif [ -r "${dynvarfile_control}" ] && [ -r "${tracerfile_control}" ] ; then
+  ln -sf ${ENSCTRL_CYCLE_DIR}/fcst_fv3lam/INPUT/fv_core.res.tile1.nc  ./control_dynvar
+  ln -sf ${ENSCTRL_CYCLE_DIR}/fcst_fv3lam/INPUT/fv_tracer.res.tile1.nc   ./control_tracer
+  ln -sf ${ENSCTRL_CYCLE_DIR}/fcst_fv3lam/INPUT/sfc_data.nc  ./control_sfcvar
 else
   print_err_msg_exit "Error: cannot find background: ${dynvarfile_control} or ${dynvarfile_control_spinup}"
 fi
@@ -234,6 +238,7 @@ cat << EOF > namelist.ens
   varlist(3)="t2m q2m f10m tsea smois tsea tsfc tsfcl alnsf alnwf alvsf alvwf emis_ice emis_lnd"
   l_write_mean=.false.
   l_recenter=.true.
+  beta=${beta_recenter},
 /
 EOF
 

diff --git a/scripts/exregional_save_restart.sh b/scripts/exregional_save_restart.sh
@@ -61,6 +61,7 @@ valid_args=( \
 "nwges_dir" \
 "fhr" \
 "cycle_type" \
+"cycle_subtype" \
 )
 process_args valid_args "$@"
 #
@@ -166,7 +167,7 @@ else
   print_info_msg "$VERBOSE" " The forecast length for cycle (\"${hh}\") is
                  ( \"${FCST_LEN_HRS_thiscycle}\") "
 
-  if [ -r "$run_dir/RESTART/coupler.res" ] && ([ ${fhr} -eq ${FCST_LEN_HRS_thiscycle} ] || [ ${cycle_type} == "ensinit" ]) ; then
+  if [ -r "$run_dir/RESTART/coupler.res" ] && ([ ${fhr} -eq ${FCST_LEN_HRS_thiscycle} ] || [ ${cycle_subtype} == "ensinit" ]) ; then
     if [ "${IO_LAYOUT_Y}" == "1" ]; then
       for file in ${filelistn}; do
         mv_vrfy $run_dir/RESTART/${file} ${nwges_dir}/RESTART/${restart_prefix}.${file}