Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Resource adjustments and updated obsproc/prepobs packages for tcvitals bug fix #862

Merged
merged 6 commits into from
Jun 16, 2022
Merged

Resource adjustments and updated obsproc/prepobs packages for tcvitals bug fix #862

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
eb5262f
Resource adjustments for memory and dev
KateFriedman-NOAA Jun 15, 2022
63b31eb
Memory adjustments in ecf scripts
KateFriedman-NOAA Jun 15, 2022
53ff95e
Update obsproc and prepobs versions in version files
KateFriedman-NOAA Jun 15, 2022
31f1452
Update C768 block in emc.dyn with nco.static values
KateFriedman-NOAA Jun 16, 2022
b1b806c
Remove additional wtime_fcst_gfs in fcst block
KateFriedman-NOAA Jun 16, 2022
8dc703f
Add check of CASE=768 to fcst npe_node_fcst values
KateFriedman-NOAA Jun 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:06:00
#PBS -l select=1:mpiprocs=80:ompthreads=1:ncpus=80:mem=60GB
#PBS -l select=1:mpiprocs=80:ompthreads=1:ncpus=80:mem=80GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:15:00
#PBS -l select=1:ncpus=2:mpiprocs=2:mem=4GB
#PBS -l select=1:ncpus=2:mpiprocs=2:mem=20GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak_meta_ncdc.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:30:00
#PBS -l select=1:ncpus=1:mem=1GB
#PBS -l select=1:ncpus=1:mem=20GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=03:00:00
#PBS -l select=1:ncpus=28:mpiprocs=28:mem=2GB
#PBS -l select=1:ncpus=28:mpiprocs=28:mem=200GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_meta.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=03:00:00
#PBS -l select=1:ncpus=23:mpiprocs=23:mem=2GB
#PBS -l select=1:ncpus=23:mpiprocs=23:mem=200GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/atmos/post_processing/awips_20km_1p0/jgfs_atmos_awips_master.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:10:00
#PBS -l select=1:ncpus=1:mem=3GB
#PBS -l select=1:ncpus=1:mem=10GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/atmos/post_processing/awips_g2/jgfs_atmos_awips_g2_master.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:05:00
#PBS -l select=1:ncpus=1:mem=3GB
#PBS -l select=1:ncpus=1:mem=10GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_blending_0p25.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:30:00
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=1GB
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=5GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:30:00
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=5GB
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=60GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/atmos/post_processing/grib2_wafs/jgfs_atmos_wafs_grib2_0p25.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:30:00
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=1GB
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=10GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/atmos/post_processing/grib_wafs/jgfs_atmos_wafs_master.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:05:00
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=1GB
#PBS -l select=1:mpiprocs=1:ompthreads=1:ncpus=1:mem=5GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/wave/gempak/jgfs_wave_gempak.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=02:00:00
#PBS -l select=1:ncpus=1:mem=1GB
#PBS -l select=1:ncpus=1:mem=10GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/wave/post/jgfs_wave_postsbs.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=03:00:00
#PBS -l select=1:mpiprocs=8:ompthreads=1:ncpus=8:mem=10GB
#PBS -l select=1:mpiprocs=8:ompthreads=1:ncpus=8:mem=40GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
2 changes: 1 addition & 1 deletion ecf/scripts/gfs/wave/prep/jgfs_wave_prep.ecf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
#PBS -q %QUEUE%
#PBS -A %PROJ%-%PROJENVIR%
#PBS -l walltime=00:10:00
#PBS -l select=1:mpiprocs=65:ompthreads=1:ncpus=65:mem=150GB
#PBS -l select=1:mpiprocs=65:ompthreads=1:ncpus=65:mem=200GB
#PBS -l place=vscatter
#PBS -l debug=true

Expand Down
4 changes: 2 additions & 2 deletions parm/config/config.fv3.emc.dyn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ case $case_in in
export DELTIM=150
export layout_x=8
export layout_y=12
export layout_x_gfs=16
export layout_x_gfs=12
export layout_y_gfs=24
export npe_wav=140
export npe_wav_gfs=448
Expand All @@ -127,7 +127,7 @@ case $case_in in
export WRTTASK_PER_GROUP=64
if [[ "$WRTTASK_PER_GROUP" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP=$npe_node_max ; fi
export WRITE_GROUP_GFS=8
export WRTTASK_PER_GROUP_GFS=48
export WRTTASK_PER_GROUP_GFS=64
if [[ "$WRTTASK_PER_GROUP_GFS" -gt "$npe_node_max" ]]; then export WRTTASK_PER_GROUP_GFS=$npe_node_max ; fi
export WRTIOBUF="32M"
;;
Expand Down
38 changes: 22 additions & 16 deletions parm/config/config.resources.emc.dyn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
eval "export npe_$step=4"
eval "export npe_node_$step=2"
eval "export nth_$step=1"
eval "export memory_$step=40G"
if [[ "$machine" != "WCOSS2" ]]; then
eval "export memory_$step=40G"
fi
KateFriedman-NOAA marked this conversation as resolved.
Show resolved Hide resolved

elif [ $step = "waveinit" ]; then

Expand All @@ -62,7 +64,8 @@ elif [ $step = "waveprep" ]; then
export nth_waveprep=1
export npe_node_waveprep=$npe_waveprep
export npe_node_waveprep_gfs=$npe_waveprep_gfs
export memory_waveprep="150GB"
export memory_waveprep="100GB"
export memory_waveprep_gfs="200GB"
export NTASKS=$npe_waveprep
export NTASKS_gfs=$npe_waveprep_gfs

Expand All @@ -74,7 +77,7 @@ elif [ $step = "wavepostsbs" ]; then
export nth_wavepostsbs=1
export npe_node_wavepostsbs=$npe_wavepostsbs
export memory_wavepostsbs="10GB"
export memory_wavepostsbs_gfs="10GB"
export memory_wavepostsbs_gfs="40GB"
export NTASKS=$npe_wavepostsbs

elif [ $step = "wavepostbndpnt" ]; then
Expand Down Expand Up @@ -108,7 +111,7 @@ elif [ $step = "wavegempak" ]; then
export nth_wavegempak=1
export npe_node_wavegempak=$npe_wavegempak
export NTASKS=$npe_wavegempak
export memory_wavegempak="1GB"
export memory_wavegempak="10GB"

elif [ $step = "waveawipsbulls" ]; then

Expand Down Expand Up @@ -191,16 +194,16 @@ elif [ $step = "fcst" ]; then
export nth_fcst_gfs=${nth_fv3_gfs:-2}
export npe_node_fcst=$(echo "$npe_node_max / $nth_fcst" | bc)
export npe_node_fcst_gfs=$(echo "$npe_node_max / $nth_fcst_gfs" | bc)
if [[ "$machine" == "WCOSS2" ]]; then
if [[ "$machine" == "WCOSS2" && "$CASE" = "C768" ]]; then
export npe_node_fcst=32
export npe_node_fcst_gfs=42
export npe_node_fcst_gfs=24
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How do the values on L200 and L199 differ from the calculations on L195 and L196?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

L195 = 128/3 = 42.6, which rounds up to 43 in the xml
L196 = 128/5 = 25.6, which rounds up to 26 in the xml

Neither value lays nicely across the WCOSS2 nodes.

George V and I did a lot of testing to get the values on L199 and L200. We want users using these values for C768 on WCOSS2. May also want them for lower resolutions too but haven't fully vetted those resolutions on WCOSS2 yet.

This reminded me I forgot to adjust the C768 block in config.fv3.emc.dyn with the tested values for layout_x_gfs and WRTTASK_PER_GROUP_GFS from config.fv3.nco.static. Please see that change in this PR now too.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok. So these values are specific to C768.
May be then please add a note to that effect.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done. I added a check for CASE=768 to make sure those values aren't used outside of high res on WCOSS2.

fi

elif [ $step = "post" ]; then

export wtime_post="00:12:00"
export wtime_post_gfs="01:00:00"
export npe_post=112
export npe_post=126
export nth_post=1
export npe_node_post=$npe_post
export npe_node_post_gfs=$npe_post
Expand All @@ -215,7 +218,7 @@ elif [ $step = "wafs" ]; then
export npe_wafs=1
export npe_node_wafs=1
export nth_wafs=1
export memory_wafs="1GB"
export memory_wafs="5GB"

elif [ $step = "wafsgcip" ]; then

Expand All @@ -231,7 +234,7 @@ elif [ $step = "wafsgrib2" ]; then
export npe_wafsgrib2=1
export npe_node_wafsgrib2=1
export nth_wafsgrib2=1
export memory_wafsgrib2="5GB"
export memory_wafsgrib2="60GB"

elif [ $step = "wafsblending" ]; then

Expand All @@ -247,15 +250,15 @@ elif [ $step = "wafsgrib20p25" ]; then
export npe_wafsgrib20p25=1
export npe_node_wafsgrib20p25=1
export nth_wafsgrib20p25=1
export memory_wafsgrib20p25="1GB"
export memory_wafsgrib20p25="10GB"

elif [ $step = "wafsblending0p25" ]; then

export wtime_wafsblending0p25="00:30:00"
export npe_wafsblending0p25=1
export npe_node_wafsblending0p25=1
export nth_wafsblending0p25=1
export memory_wafsblending0p25="1GB"
export memory_wafsblending0p25="5GB"

elif [ $step = "vrfy" ]; then

Expand Down Expand Up @@ -298,7 +301,7 @@ elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then
eval "export npe_$step=1"
eval "export npe_node_$step=1"
eval "export nth_$step=1"
eval "export memory_$step=2048M"
eval "export memory_$step=50GB"

elif [ $step = "eobs" -o $step = "eomg" ]; then

Expand Down Expand Up @@ -340,6 +343,9 @@ elif [ $step = "eupd" ]; then
if [[ "$machine" = "WCOSS_DELL_P3" ]]; then
export npe_eupd=960
export nth_eupd=7
elif [[ "$machine" = "WCOSS2" ]]; then
export npe_eupd=315
export nth_eupd=14
fi
elif [ $CASE = "C384" ]; then
export npe_eupd=270
Expand Down Expand Up @@ -376,7 +382,7 @@ elif [ $step = "esfc" ]; then
export nth_esfc=1
export nth_cycle=$nth_esfc
export npe_node_cycle=$(echo "$npe_node_max / $nth_cycle" | bc)
export memory_esfc="40GB"
export memory_esfc="80GB"

elif [ $step = "efcs" ]; then

Expand Down Expand Up @@ -413,7 +419,7 @@ elif [ $step = "awips" ]; then
export npe_awips=1
export npe_node_awips=1
export nth_awips=1
export memory_awips="3GB"
export memory_awips="10GB"
if [[ "$machine" == "WCOSS_DELL_P3" ]]; then
export npe_awips=2
export npe_node_awips=2
Expand All @@ -428,8 +434,8 @@ elif [ $step = "gempak" ]; then
export npe_node_gempak=2
export npe_node_gempak_gfs=28
export nth_gempak=1
export memory_gempak="4GB"
export memory_gempak_gfs="2GB"
export memory_gempak="20GB"
export memory_gempak_gfs="200GB"

else

Expand Down
25 changes: 13 additions & 12 deletions parm/config/config.resources.nco.static
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ elif [ $step = "waveprep" ]; then
export nth_waveprep=1
export npe_node_waveprep=$npe_waveprep
export npe_node_waveprep_gfs=$npe_waveprep_gfs
export memory_waveprep="150GB"
export memory_waveprep="100GB"
export memory_waveprep_gfs="200GB"
export NTASKS=$npe_waveprep
export NTASKS_gfs=$npe_waveprep_gfs

Expand All @@ -60,7 +61,7 @@ elif [ $step = "wavepostsbs" ]; then
export nth_wavepostsbs=1
export npe_node_wavepostsbs=$npe_wavepostsbs
export memory_wavepostsbs="10GB"
export memory_wavepostsbs_gfs="10GB"
export memory_wavepostsbs_gfs="40GB"
export NTASKS=$npe_wavepostsbs

elif [ $step = "wavepostbndpnt" ]; then
Expand Down Expand Up @@ -94,7 +95,7 @@ elif [ $step = "wavegempak" ]; then
export nth_wavegempak=1
export npe_node_wavegempak=$npe_wavegempak
export NTASKS=$npe_wavegempak
export memory_wavegempak="1GB"
export memory_wavegempak="10GB"

elif [ $step = "waveawipsbulls" ]; then

Expand Down Expand Up @@ -179,7 +180,7 @@ elif [ $step = "wafs" ]; then
export npe_wafs=1
export npe_node_wafs=1
export nth_wafs=1
export memory_wafs="1GB"
export memory_wafs="5GB"

elif [ $step = "wafsgcip" ]; then

Expand All @@ -195,7 +196,7 @@ elif [ $step = "wafsgrib2" ]; then
export npe_wafsgrib2=1
export npe_node_wafsgrib2=1
export nth_wafsgrib2=1
export memory_wafsgrib2="5GB"
export memory_wafsgrib2="60GB"

elif [ $step = "wafsblending" ]; then

Expand All @@ -211,15 +212,15 @@ elif [ $step = "wafsgrib20p25" ]; then
export npe_wafsgrib20p25=1
export npe_node_wafsgrib20p25=1
export nth_wafsgrib20p25=1
export memory_wafsgrib20p25="1GB"
export memory_wafsgrib20p25="10GB"

elif [ $step = "wafsblending0p25" ]; then

export wtime_wafsblending0p25="00:30:00"
export npe_wafsblending0p25=1
export npe_node_wafsblending0p25=1
export nth_wafsblending0p25=1
export memory_wafsblending0p25="1GB"
export memory_wafsblending0p25="5GB"

elif [ $step = "vrfy" ]; then

Expand Down Expand Up @@ -255,7 +256,7 @@ elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then
eval "export npe_$step=1"
eval "export npe_node_$step=1"
eval "export nth_$step=1"
eval "export memory_$step=2048M"
eval "export memory_$step=50GB"

elif [ $step = "eobs" -o $step = "eomg" ]; then

Expand Down Expand Up @@ -298,7 +299,7 @@ elif [ $step = "esfc" ]; then
export nth_esfc=1
export nth_cycle=$nth_esfc
export npe_node_cycle=$(echo "$npe_node_max / $nth_cycle" | bc)
export memory_esfc="40GB"
export memory_esfc="80GB"

elif [ $step = "efcs" ]; then

Expand Down Expand Up @@ -329,7 +330,7 @@ elif [ $step = "awips" ]; then
export npe_awips=1
export npe_node_awips=1
export nth_awips=1
export memory_awips="3GB"
export memory_awips="10GB"

elif [ $step = "gempak" ]; then

Expand All @@ -339,8 +340,8 @@ elif [ $step = "gempak" ]; then
export npe_node_gempak=2
export npe_node_gempak_gfs=28
export nth_gempak=1
export memory_gempak="4GB"
export memory_gempak_gfs="2GB"
export memory_gempak="20GB"
export memory_gempak_gfs="200GB"

else

Expand Down
4 changes: 2 additions & 2 deletions versions/hera.ver
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ export hpc_ver=1.2.0
export hpc_intel_ver=18.0.5.274
export hpc_impi_ver=2018.0.4

export obsproc_run_ver=1.0.0-rd
export prepobs_run_ver=1.0.0-rd
export obsproc_run_ver=1.0.2-rd
export prepobs_run_ver=1.0.1-rd

export hpss_ver=hpss
export prod_util_ver=1.2.2
Expand Down
Loading