Skip to content

Commit

Permalink
Enhancements v038 (NOAA-EMC#487)
Browse files Browse the repository at this point in the history
* Update code for RRFS_A v0.3.8 on wocss2:
1. Use ens_mean_recenter_P2DIO.exe for receneter and ensmean (parallel IO through levels)
2. change clean hours: save 5 days of the prodcut but purgen run directory in 12-h
3. Use 6 nodes to run ensemble DA: it take about 10 mintues to finish
4. Turn on SPP in ensembnle forecast
5. Use "FV3LAM_wflow_enkf.log" for ensemble DA and forecast
6. remov "-l" from launch_FV3LAM_wflow.sh

* Update code for rrfs_b on JET:
1) add HPSS_ACCOUNT
2) add configure for RRFS_B real-time run

---------

Co-authored-by: ming hu <[email protected]>
  • Loading branch information
hu5970 and ming hu authored Feb 21, 2023
1 parent d628af9 commit 338df6d
Show file tree
Hide file tree
Showing 10 changed files with 196 additions and 45 deletions.
5 changes: 3 additions & 2 deletions scripts/exregional_calc_ensmean.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ case $MACHINE in
export OMP_STACKSIZE=500M
export OMP_NUM_THREADS=1
ncores=$(( NNODES_RUN_RECENTER*PPN_RUN_RECENTER ))
APRUN="mpiexec -n 93 -ppn ${PPN_RUN_RECENTER} --cpu-bind core --depth ${OMP_NUM_THREADS}"
APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_RECENTER} --cpu-bind core --depth ${OMP_NUM_THREADS}"
;;
#
"THEIA")
Expand Down Expand Up @@ -226,7 +226,8 @@ EOF
#

echo pwd is `pwd`
ENSMEAN_EXEC=${EXECDIR}/gen_ensmean_recenter.exe
#ENSMEAN_EXEC=${EXECDIR}/gen_ensmean_recenter.exe
ENSMEAN_EXEC=${EXECDIR}/ens_mean_recenter_P2DIO.exe

if [ -f ${ENSMEAN_EXEC} ]; then
print_info_msg "$VERBOSE" "
Expand Down
5 changes: 3 additions & 2 deletions scripts/exregional_run_recenter.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ case $MACHINE in
export OMP_STACKSIZE=500M
export OMP_NUM_THREADS=1
ncores=$(( NNODES_RUN_RECENTER*PPN_RUN_RECENTER ))
APRUN="mpiexec -n 93 -ppn ${PPN_RUN_RECENTER} --cpu-bind core --depth ${OMP_NUM_THREADS}"
APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_RECENTER} --cpu-bind core --depth ${OMP_NUM_THREADS}"
;;
#
"THEIA")
Expand Down Expand Up @@ -249,7 +249,8 @@ EOF
#

echo pwd is `pwd`
ENSMEAN_EXEC=${EXECDIR}/gen_ensmean_recenter.exe
#ENSMEAN_EXEC=${EXECDIR}/gen_ensmean_recenter.exe
ENSMEAN_EXEC=${EXECDIR}/ens_mean_recenter_P2DIO.exe

if [ -f ${ENSMEAN_EXEC} ]; then
print_info_msg "$VERBOSE" "
Expand Down
4 changes: 2 additions & 2 deletions ush/config.sh.rrfs_a_c3
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
MACHINE="wcoss2"
version="v0.3.6"
version="v0.3.8"
ACCOUNT="RRFS_DEV"
#RESERVATION="rrfsdet"
EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/$version/"
Expand Down Expand Up @@ -93,7 +93,7 @@ EXTRN_MDL_NAME_LBCS="FV3GFS"
envir="para"

NET="rrfs_a"
TAG="c3v36"
TAG="c3v38"

ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam"
NCL_REGION="conus"
Expand Down
113 changes: 113 additions & 0 deletions ush/config.sh.rrfs_b_c3
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
MACHINE="jet"
version="v0.3.6"
ACCOUNT="nrtrr"
RESERVATION="rrfsdet"
EXPT_BASEDIR="/home/rtrr/RRFS/rrfs.${version}"
EXPT_SUBDIR="RRFS_CONUS_3km"

PREDEF_GRID_NAME=RRFS_CONUS_3km
. set_rrfs_config_general.sh

DO_DACYCLE="TRUE"
DO_SURFACE_CYCLE="TRUE"
DO_SPINUP="TRUE"
DO_POST_SPINUP="FALSE"
DO_POST_PROD="TRUE"
DO_RETRO="FALSE"
DO_NONVAR_CLDANAL="TRUE"
DO_ENVAR_RADAR_REF="FALSE"
DO_SMOKE_DUST="TRUE"
DO_REFL2TTEN="FALSE"
RADARREFL_TIMELEVEL=(0)
FH_DFI_RADAR="0.0,0.25,0.5"
DO_SOIL_ADJUST="TRUE"
DO_RADDA="TRUE"
DO_BUFRSND="FALSE"

EXTRN_MDL_ICS_OFFSET_HRS="3"
LBC_SPEC_INTVL_HRS="1"
EXTRN_MDL_LBCS_OFFSET_HRS="6"
BOUNDARY_LEN_HRS="18"
BOUNDARY_PROC_GROUP_NUM="3"

# avaialble retro period:
# 20210511-20210531; 20210718-20210801
DATE_FIRST_CYCL="20230201"
DATE_LAST_CYCL="20230331"
CYCL_HRS=( "00" "12" )
CYCL_HRS_SPINSTART=("03" "15")
CYCL_HRS_PRODSTART=("09" "21")
CYCLEMONTH="02,03"
CYCLEDAY="*"

STARTYEAR=${DATE_FIRST_CYCL:0:4}
STARTMONTH=${DATE_FIRST_CYCL:4:2}
STARTDAY=${DATE_FIRST_CYCL:6:2}
STARTHOUR="00"
ENDYEAR=${DATE_LAST_CYCL:0:4}
ENDMONTH=${DATE_LAST_CYCL:4:2}
ENDDAY=${DATE_LAST_CYCL:6:2}
ENDHOUR="23"

PREEXISTING_DIR_METHOD="upgrade" # "rename"
INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}0300 ${DATE_LAST_CYCL}2300 12:00:00"
BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
PROD_CYCLEDEF="00 01-11,13-23 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *"
PRODLONG_CYCLEDEF="00 00,12 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *"
ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1400 ${DATE_LAST_CYCL}2300 24:00:00"
if [[ $DO_SPINUP == "TRUE" ]] ; then
SPINUP_CYCLEDEF="00 03-08,15-20 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *"
fi

FCST_LEN_HRS="3"
FCST_LEN_HRS_SPINUP="1"
POSTPROC_LEN_HRS="3"
POSTPROC_LONG_LEN_HRS="12"
#FCST_LEN_HRS_CYCLES=(48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18)
for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=3; done
for i in {0..23..12}; do FCST_LEN_HRS_CYCLES[$i]=12; done
DA_CYCLE_INTERV="1"
RESTART_INTERVAL="1 2 3"
RESTART_INTERVAL_LONG="1 2 3 4"

USE_RRFSE_ENS="FALSE"
CYCL_HRS_HYB_FV3LAM_ENS=("19" "20" "21" "22" "23" "00")

SST_update_hour=01
GVF_update_hour=04
SNOWICE_update_hour=00
netcdf_diag=.true.
binary_diag=.false.

regional_ensemble_option=1

WTIME_RUN_FCST="00:45:00"
WTIME_RUN_FCST_LONG="03:45:00"
WTIME_MAKE_LBCS="02:00:00"
WTIME_RUN_ANAL="00:50:00"
WTIME_RUN_PREPSTART="00:20:00"

EXTRN_MDL_NAME_ICS="FV3GFS"
EXTRN_MDL_NAME_LBCS="FV3GFS"

NET="RRFS_CONUS"
TAG="RRFS_CONUS_3km"

ARCHIVEDIR="/1year/BMC/wrfruc/rrfs_b"
NCL_REGION="conus"
MODEL="RRFS_B"

RUN="RRFS_conus_3km"

. set_rrfs_config.sh

STMP="/lfs4/BMC/nrtrr/NCO_dirs/${version}/stmp" # Path to directory STMP that mostly contains input files.
PTMP="/lfs4/BMC/nrtrr/NCO_dirs/${version}/com" # Path to directory STMP that mostly contains input files.
NWGES="/lfs4/BMC/nrtrr/NCO_dirs/${version}/nwges/${envir}/${NET}" # Path to directory NWGES that save boundary, cold initial, restart files
if [[ ${regional_ensemble_option} == "5" ]]; then
RRFSE_NWGES="/lfs4/BMC/wrfruc/RRFSE/NCO_dirs/${version}" # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid.
NUM_ENS_MEMBERS=30 # FV3LAM ensemble size for GSI hybrid analysis
CYCL_HRS_PRODSTART_ENS=( "07" "19" )
DO_ENVAR_RADAR_REF="TRUE"
fi

40 changes: 26 additions & 14 deletions ush/config.sh_rrfs_a_enkf_c3
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
MACHINE="wcoss2"
version="v0.3.6"
version="v0.3.8"
ACCOUNT="RRFS_DEV"
#RESERVATION="rrfsens"
#RESERVATION_POST="rrfsdet"
Expand Down Expand Up @@ -52,7 +52,9 @@ CYCL_HRS=( "00" "12" )
CYCL_HRS=( "18" )
CYCL_HRS_SPINSTART=("06" "18")
CYCL_HRS_PRODSTART=("07" "19")
#CYCL_HRS_STOCH=("00" "12")
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
CYCL_HRS_STOCH=("00" "12")
fi
#CYCL_HRS_RECENTER=("19")
CYCLEMONTH="02,03"
CYCLEDAY="*"
Expand Down Expand Up @@ -124,7 +126,7 @@ FV3GFS_FILE_FMT_LBCS="grib2"

envir="para"
NET="rrfs_a"
TAG="cenkf36"
TAG="cenkf38"
ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam"
NCL_REGION="conus"
MODEL="rrfs_b"
Expand All @@ -139,6 +141,22 @@ if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then
DO_ENKF_RADAR_REF="TRUE"
DO_ENSPOST="FALSE"
DO_ENSINIT="TRUE"

RADAR_REF_THINNING="2"
ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens"
CLEAN_OLDFCST_HRS="12"
CLEAN_OLDSTMPPOST_HRS="12"
cld_bld_hgt=0.0
l_precip_clear_only=.true.
write_diag_2=.true.

START_TIME_SPINUP="00:30:00"
LAYOUT_X="11"
LAYOUT_Y="32"
NNODES_RUN_FCST="6"
NNODES_RUN_RECENTER="10"
PPN_RUN_RECENTER="128"

NUM_ENS_MEMBERS_FCST=9
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
NUM_ENS_MEMBERS=${NUM_ENS_MEMBERS_FCST}
Expand All @@ -147,18 +165,12 @@ if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then
# SPPT_MAG=0.5
WTIME_RUN_FCST="04:45:00"
WTIME_MAKE_LBCS="01:30:00"
TAG="cefcst36"
TAG="cefcst38"

LAYOUT_X="31"
LAYOUT_Y="32"
NNODES_RUN_FCST="16"
fi
RADAR_REF_THINNING="2"
ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens"
NNODES_RUN_RECENTER="3"
PPN_RUN_RECENTER="128"
CLEAN_OLDFCST_HRS="48"
CLEAN_OLDSTMPPOST_HRS="48"
cld_bld_hgt=0.0
l_precip_clear_only=.true.
write_diag_2=.true.
START_TIME_SPINUP="00:30:00"
fi

RUN="rrfs_a"
Expand Down
5 changes: 5 additions & 0 deletions ush/config_defaults.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ version="0.1.0"
# The account under which to submit non-reservation jobs to the queue.
# Defaults to ACCOUNT if not set.
#
# HPSS_ACCOUNT:
# The account under which to submit non-reservation jobs to the queue.
# Defaults to SERVICE_ACCOUNT if not set.
#
# SCHED:
# The job scheduler to use (e.g. slurm). Set this to an empty string in
# order for the experiment generation script to set it depending on the
Expand Down Expand Up @@ -143,6 +147,7 @@ version="0.1.0"
MACHINE="BIG_COMPUTER"
ACCOUNT="project_name"
SERVICE_ACCOUNT=""
HPSS_ACCOUNT=""
RESERVATION=""
RESERVATION_POST=""
SCHED=""
Expand Down
1 change: 1 addition & 0 deletions ush/generate_FV3LAM_wflow.sh
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ settings="\
#
'account': $ACCOUNT
'service_account': ${SERVICE_ACCOUNT:-$ACCOUNT}
'hpss_account': ${HPSS_ACCOUNT:-$SERVICE_ACCOUNT}
'reservation': $RESERVATION
'reservation_post': $RESERVATION_POST
'sched': $SCHED
Expand Down
2 changes: 1 addition & 1 deletion ush/launch_FV3LAM_wflow.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash -l
#!/bin/bash

#
#-----------------------------------------------------------------------
Expand Down
57 changes: 35 additions & 22 deletions ush/set_rrfs_config_general.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
if [[ $MACHINE == "jet" ]] ; then
if [[ -n $RESERVATION ]] ; then
ACCOUNT=rtrr
SERVICE_ACCOUNT=nrtrr
SERVICE_ACCOUNT=rtrr
HPSS_ACCOUNT=nrtrr
PARTITION_DEFAULT=kjet
PARTITION_FCST=kjet
PARTITION_GRAPHICS=kjet
Expand Down Expand Up @@ -47,6 +48,7 @@ fi

if [[ $MACHINE == "wcoss2" ]] ; then
ACCOUNT=RRFS-DEV
HPSS_ACCOUNT="RRFS-DEV"
QUEUE_DEFAULT="devhigh"
QUEUE_ANALYSIS="devmax"
QUEUE_FCST="devmax"
Expand Down Expand Up @@ -82,29 +84,29 @@ HYBENSMEM_NMIN=66
IO_LAYOUT_Y=1
PPN_RUN_REF2TTEN=1
PPN_RUN_NONVARCLDANL=${IO_LAYOUT_Y}
MAXTRIES_GET_EXTRN_ICS="3"
MAXTRIES_GET_EXTRN_LBCS="3"
MAXTRIES_MAKE_ICS="3"
MAXTRIES_MAKE_LBCS="3"
MAXTRIES_RUN_PREPSTART="3"
MAXTRIES_RUN_FCST="3"
MAXTRIES_ANAL_GSI="3"
MAXTRIES_POSTANAL="3"
MAXTRIES_ANAL_ENKF="3"
MAXTRIES_RUN_ANAL="3"
MAXTRIES_CLDANL_NONVAR="3"
MAXTRIES_SAVE_DA_OUTPUT="3"
MAXTRIES_SAVE_RESTART="3"
MAXTRIES_RUN_POST="3"
MAXTRIES_RUN_PRDGEN="3"
MAXTRIES_GET_EXTRN_ICS="2"
MAXTRIES_GET_EXTRN_LBCS="2"
MAXTRIES_MAKE_ICS="2"
MAXTRIES_MAKE_LBCS="2"
MAXTRIES_RUN_PREPSTART="2"
MAXTRIES_RUN_FCST="2"
MAXTRIES_ANAL_GSI="2"
MAXTRIES_POSTANAL="2"
MAXTRIES_ANAL_ENKF="2"
MAXTRIES_RUN_ANAL="2"
MAXTRIES_CLDANL_NONVAR="2"
MAXTRIES_SAVE_DA_OUTPUT="2"
MAXTRIES_SAVE_RESTART="2"
MAXTRIES_RUN_POST="2"
MAXTRIES_RUN_PRDGEN="2"

WTIME_RUN_FCST="00:45:00"
WTIME_RUN_FCST_LONG="03:45:00"
WTIME_MAKE_LBCS="02:00:00"
WTIME_RUN_ANAL="00:50:00"
WTIME_RUN_PREPSTART="00:20:00"

START_TIME_SPINUP="02:10:00"
START_TIME_SPINUP="01:30:00"
START_TIME_PROD="01:40:00"
START_TIME_LATE_ANALYSIS="01:40:00"
START_TIME_CONVENTIONAL_SPINUP="00:50:00"
Expand All @@ -126,8 +128,9 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then
PPN_MAKE_ICS="20"
NNODES_MAKE_LBCS="3"
PPN_MAKE_LBCS="20"
NNODES_RUN_POST="1"
NNODES_RUN_POST="2"
PPN_RUN_POST="40"
PPN_PROC_RADAR="20"
fi
fi

Expand Down Expand Up @@ -167,6 +170,7 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then
NNODES_RUN_FCST="16"
PPN_RUN_POST="64"
NNODES_RUN_POST="2"
TPP_RUN_POST="2"
PPN_PROC_RADAR="128"
NNODES_PROC_RADAR="1"
PPN_RUN_ENKF="32"
Expand All @@ -193,7 +197,7 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then
MAXTRIES_ANAL_GSI="2"

WTIME_RUN_FCST="00:55:00"
WTIME_RUN_FCST_LONG="02:15:00"
WTIME_RUN_FCST_LONG="03:15:00"
WTIME_RUN_FCST_SPINUP="00:25:00"
WTIME_MAKE_ICS="00:20:00"
WTIME_MAKE_LBCS="02:00:00"
Expand All @@ -207,6 +211,13 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then
FV3GFS_FILE_FMT_ICS="netcdf"
FV3GFS_FILE_FMT_LBCS="netcdf"

CLEAN_OLDPROD_HRS="120"
CLEAN_OLDLOG_HRS="120"
CLEAN_OLDRUN_HRS="12"
CLEAN_OLDFCST_HRS="24"
CLEAN_OLDSTMPPOST_HRS="12"
CLEAN_NWGES_HRS="28"

fi

if [[ $MACHINE == "orion" ]] ; then
Expand Down Expand Up @@ -408,10 +419,12 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_NA_3km" ]] ; then
WTIME_RUN_FCST_LONG="04:30:00"
WTIME_MAKE_LBCS="02:30:00"

CLEAN_OLDRUN_HRS="24"
CLEAN_OLDFCST_HRS="12"
CLEAN_OLDPROD_HRS="120"
CLEAN_OLDLOG_HRS="120"
CLEAN_OLDRUN_HRS="12"
CLEAN_OLDFCST_HRS="24"
CLEAN_OLDSTMPPOST_HRS="12"
CLEAN_NWGES_HRS="48"
CLEAN_NWGES_HRS="28"
fi

fi
Expand Down
Loading

0 comments on commit 338df6d

Please sign in to comment.