From cb1bb10ec51fe5683e064ac3546e8c7984b4a592 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Fri, 1 Nov 2024 18:29:08 -0400 Subject: [PATCH 01/14] revise distribution: add slurm-array; rev stats --- .../NCEP_enkf/scripts/gmao/atmens_ntasks.pl | 49 +++++ .../scripts/gmao/atmens_recenter.csh | 119 +++++++--- .../NCEP_enkf/scripts/gmao/atmens_stats.csh | 80 ++++--- .../NCEP_enkf/scripts/gmao/atmos_eaod.csh | 10 +- .../NCEP_enkf/scripts/gmao/atmos_enkf.csh | 10 +- .../NCEP_enkf/scripts/gmao/atmos_ens2gcm.csh | 133 ++++++++---- .../scripts/gmao/etc/AtmEnsConfig.csh | 5 +- .../scripts/gmao/etc/AtmOSEConfig.csh | 1 - .../scripts/gmao/etc/R21C/AtmEnsConfig.csh | 7 +- .../scripts/gmao/etc/R21C/mp_stats_NP.rc | 22 -- .../scripts/gmao/etc/R21C/mp_stats_NZ.rc | 22 -- .../NCEP_enkf/scripts/gmao/etc/mp_stats.rc | 2 +- .../NCEP_enkf/scripts/gmao/gcm_ensemble.csh | 124 +++++++---- .../NCEP_enkf/scripts/gmao/gen_ensbkg.csh | 4 +- .../NCEP_Etc/NCEP_enkf/scripts/gmao/jobgen.pl | 78 +++++-- .../NCEP_enkf/scripts/gmao/obsvr_ensemble.csh | 126 +++++++---- .../NCEP_enkf/scripts/gmao/post_egcm.csh | 205 +++++++++++++++--- .../NCEP_enkf/scripts/gmao/setobsvr.csh | 7 + .../NCEP_enkf/scripts/gmao/setperts.csh | 4 +- .../NCEP_enkf/scripts/gmao/ut_atmens_stats.j | 1 - .../NCEP_enkf/scripts/gmao/ut_rec_stats.j | 1 - 21 files changed, 670 insertions(+), 340 deletions(-) create mode 100755 src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_ntasks.pl delete mode 100644 src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/mp_stats_NP.rc delete mode 100644 src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/mp_stats_NZ.rc diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_ntasks.pl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_ntasks.pl new file mode 100755 index 00000000..939019a2 --- /dev/null +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_ntasks.pl @@ -0,0 +1,49 @@ +#!/usr/bin/env perl +# +use POSIX qw/ceil/; +use POSIX qw/floor/; +use strict; +use warnings; + +my $MYNAME = "atmens_ntasks.pl"; +my ($this_ntasks_per_node, $thisnode, $myntasks, $ncores_needed, $nexecs_fit, $myntasks_per_node); + +my ($MYTASK_NCPUS, $ipoe) = @ARGV; +if (not defined $MYTASK_NCPUS) { + die "Need NCPUS\n"; +} + +if (not defined $ipoe) { + die "${MYNAME}: Need DST\n"; +} + +$this_ntasks_per_node = `facter processorcount`; +$this_ntasks_per_node = $this_ntasks_per_node - 2; +$myntasks = $ipoe * $MYTASK_NCPUS; +$ncores_needed = ceil($myntasks / $this_ntasks_per_node + 0.1); +$nexecs_fit = floor($this_ntasks_per_node / $MYTASK_NCPUS ); +if ( $nexecs_fit == 0 ) {$nexecs_fit = 1}; +$myntasks = $ncores_needed * $this_ntasks_per_node; +$myntasks_per_node = $nexecs_fit * $MYTASK_NCPUS; + +# the above does not give a decent partition of the borgs +# so let me wire a mildly wasteful way of doing this +# basically the following grabs as many nodes as instances +# of a exec being distributed within a single job; the +# following also accomodates the case when an instance of +# an exec needs more than a single node to run. +# The code above was aimed at doing this a little more efficiently +# and less wastefully, but I can quite get it to work, so ... +$ncores_needed = ceil($MYTASK_NCPUS / $this_ntasks_per_node + 0.1); +$myntasks = $ipoe * $ncores_needed * $this_ntasks_per_node; +$myntasks_per_node = -1; + +my $JOBGEN_NCPUS = $myntasks; +my $JOBGEN_NCPUS_PER_NODE = $myntasks_per_node; + +#print "per-node = $this_ntasks_per_node \n"; +#print "cores = $ncores_needed \n"; +#print "fit = $nexecs_fit \n"; +print "$JOBGEN_NCPUS \n"; +print "$JOBGEN_NCPUS_PER_NODE \n"; + diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_recenter.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_recenter.csh index fe054ddf..448675ad 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_recenter.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_recenter.csh @@ -20,6 +20,7 @@ # 21Feb2020 Todling Allow for high freq bkg (up to 1mn) # 04Jun2020 Todling Revise parallelization strategy # 23Jun2020 Todling Redef meaning of ATMENSLOC +# 25Oct2024 Todling Implement slurm array distribution opt #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then setenv ATMENS_VERBOSE 0 @@ -72,7 +73,9 @@ if ( $#argv < 8 ) then echo " ADDINF_FACTOR - inflation factor (such as 0.25)" echo " AENS_ADDINFLATION - when set, apply additive inflation to analyzed members" echo " AENS_DONORECENTER - allow bypassing recentering" - echo " AENS_RECENTER_DSTJOB - distribute multiple works within smaller jobs" + echo " AENS_RECENTER_ARRAY - distribute multiple using slurm array capability" + echo " AENS_RECENTER_DSTJOB - distribute multiple works within smaller jobs" + echo " AENS_RECENTER_PACKL - distribute multiple using slurm packable capability" echo " ASYNBKG - background frequency (when adaptive inflation on)" echo " CENTRAL_BLEND - 0 or 1=to blend members with central (def: 1)" echo " FVHOME - location of experiment " @@ -82,7 +85,7 @@ if ( $#argv < 8 ) then echo " ENSRECENTER_NCPUS - when parallel ens on, this sets NCPUS for recentering" echo " (NOTE: required when ENSPARALLEL is on)" echo " RECENTER_WALLCLOCK - wall clock time to run dyn_recenter, default 0:10:00 " - echo " RECENTER_QNAME - name of queue (default: NULL, that is, let pbs pick) " + echo " RECENTER_QNAME - name of queue (default: NULL, that is, let batch pick) " echo " " echo " REMARKS " echo " " @@ -90,7 +93,7 @@ if ( $#argv < 8 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 08Apr2013 by: R. Todling" + echo " Last modified: 30Oct2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -104,24 +107,30 @@ if ( !($?FVROOT) ) setenv FAILED 1 if ( !($?AENS_ADDINFLATION) ) setenv AENS_ADDINFLATION 0 if ( !($?AENS_DONORECENTER) ) setenv AENS_DONORECENTER 0 +if ( !($?AENS_RECENTER_ARRAY) ) setenv AENS_RECENTER_ARRAY 0 if ( !($?AENS_RECENTER_DSTJOB) ) setenv AENS_RECENTER_DSTJOB 0 +if ( !($?AENS_RECENTER_PACKL) ) setenv AENS_RECENTER_PACKL 0 if ( !($?CENTRAL_BLEND) ) setenv CENTRAL_BLEND 1 if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 if ( !($?ENSPARALLEL) ) setenv ENSPARALLEL 0 if ( !($?RECENTER_WALLCLOCK) ) setenv RECENTER_WALLCLOCK 0:10:00 +if ( !($?JOBGEN_PFXNAME) ) then + set pfxname = "" +else + set pfxname = ${JOBGEN_PFXNAME}_ +endif + if ( $ENSPARALLEL ) then if ( !($?RECENTER_QNAME) ) then echo "${MYNAME}: error, env var RECENTER_QNAME not defined" setenv FAILED 1 endif - if ( ! $AENS_RECENTER_DSTJOB ) then - setenv JOBGEN_NCPUS_PER_NODE 2 - endif if ( !($?ENSRECENTER_NCPUS) ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $ENSRECENTER_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif endif @@ -142,6 +151,11 @@ set infloc = $8 set hh = `echo $nhms | cut -c1-2` set hhmn = `echo $nhms | cut -c1-4` set yyyymmddhh = ${nymd}${hh} +set yyyy = `echo $nymd | cut -c1-4` +set mm = `echo $nymd | cut -c5-6` +set dd = `echo $nymd | cut -c7-8` +set ddmmyyyy = ${dd}${mm}${yyyy} +set hhzddmmyyyy = ${hh}Z${ddmmyyyy} # used in jobname (easier to see cycle date/time) if ( -e $ENSWORK/.DONE_${MYNAME}_${ftype1}_${ftype2}.$yyyymmddhh ) then echo " ${MYNAME}: already done" @@ -167,6 +181,11 @@ else endif endif +set packable = "" +if ( $AENS_RECENTER_PACKL ) then + set packable = "-packable" +endif + if ( $CENTRAL_BLEND ) then set blend = "-damp" else @@ -279,7 +298,7 @@ while ( $ic < $nmem ) if ( $AENS_RECENTER_DSTJOB != 0 ) then # case of multiple jobs within few larger ones - set cmdline = "serial_run $cmdline" + if ( $AENS_RECENTER_DSTJOB == 0 ) set cmdline = "serial_run $cmdline" # collect multiple recenter calls into jumbo file if ( $ipoe < $AENS_RECENTER_DSTJOB ) then # nmem better devide by AENS_PERTS_DSTJOB @@ -305,48 +324,80 @@ while ( $ic < $nmem ) $ENSWORK/.DONE_MEM${memtag}_${MYNAME}_${ftype1}_${ftype2}.$yyyymmddhh \ "Recenter ANA Failed" - if ( $AENS_RECENTER_DSTJOB != 0 ) then if ( -e recenter_mem${memtag}.j ) then chmod +x recenter_mem${memtag}.j else - echo " ${MYNAME}: Recenter Failed to generate PBS jobs for Member ${memtag}, Aborting ... " + echo " ${MYNAME}: Recenter Failed to generate batch jobs for Member ${memtag}, Aborting ... " touch $ENSWORK/.FAILED exit(1) endif + if ( $AENS_RECENTER_ARRAY ) then - if ( ($ipoe == $AENS_RECENTER_DSTJOB) || (($fpoe == $ntodo ) && ($ipoe < $AENS_RECENTER_DSTJOB) ) ) then - @ myncpus = $ipoe - setenv JOBGEN_NCPUS $myncpus - jobgen.pl \ - -q $RECENTER_QNAME \ - recenter_dst${npoe} \ - $GID \ - $RECENTER_WALLCLOCK \ - "job_distributor.csh -machfile $ENSWORK/recenter_machfile$npoe -usrcmd $ENSWORK/recenter_poe.$npoe -usrntask $ENSRECENTER_NCPUS -njobs $ipoe " \ - $ENSWORK \ - $MYNAME \ - $ENSWORK/.DONE_POE${npoe}_${MYNAME}.$yyyymmddhh \ - "Recenter Failed for Member ${npoe}" - if (! -e recenter_dst${npoe}.j ) then - echo " ${MYNAME}: Recenter Failed to generate DST PBS jobs for Member ${memtag}, Aborting ... " - touch $ENSWORK/.FAILED - exit(1) - endif - /bin/mv recenter_dst${npoe}.j $ENSWORK/ - $ATMENS_BATCHSUB $ENSWORK/recenter_dst${npoe}.j - touch .SUBMITTED - @ ipoe = 0 # reset counter @ npoe++ - endif + if ( $npoe == $nmem ) then # time to launch slurm ARRAY job + + cd $ensloc + + jobgen.pl \ + -egress DYNRECENTER_EGRESS \ + -q $RECENTER_QNAME $packable \ + ${pfxname}recenter_array_${ftype1}_${ftype2}.$hhzddmmyyyy \ + $GID \ + -array "1-${nmem}%${AENS_RECENTER_DSTJOB}" -ncc \ + $RECENTER_WALLCLOCK \ + recenter_mem\${memtag}.j \ + $ensloc/mem\$memtag \ + $MYNAME \ + $ENSWORK/.DONE_ARRAY_${MYNAME}_${ftype1}_${ftype2}.$hhzddmmyyyy \ + "Recenter ANA Array Job Failed" + + if ( -e $ensloc/${pfxname}recenter_array_${ftype1}_${ftype2}.$hhzddmmyyyy.j ) then + $ATMENS_BATCHSUB $ensloc/${pfxname}recenter_array_${ftype1}_${ftype2}.$hhzddmmyyyy.j + else + echo " ${MYNAME}: Failed to generate array batch jobs for Recentering ANA, Aborting ... " + touch $ensloc/.FAILED + exit(1) + endif + endif + + else + + if ( ($ipoe == $AENS_RECENTER_DSTJOB) || (($fpoe == $ntodo ) && ($ipoe < $AENS_RECENTER_DSTJOB) ) ) then + @ myncpus = $ipoe + setenv JOBGEN_NCPUS $myncpus + jobgen.pl \ + -q $RECENTER_QNAME \ + recenter_dst${npoe} \ + $GID \ + $RECENTER_WALLCLOCK \ + "job_distributor.csh -machfile $ENSWORK/recenter_machfile$npoe -usrcmd $ENSWORK/recenter_poe.$npoe -usrntask $ENSRECENTER_NCPUS -njobs $ipoe " \ + $ENSWORK \ + $MYNAME \ + $ENSWORK/.DONE_POE${npoe}_${MYNAME}.$yyyymmddhh \ + "Recenter Failed for Member ${npoe}" + if (! -e recenter_dst${npoe}.j ) then + echo " ${MYNAME}: Recenter Failed to generate DST batch jobs for Member ${memtag}, Aborting ... " + touch $ENSWORK/.FAILED + exit(1) + endif + /bin/mv recenter_dst${npoe}.j $ENSWORK/ + $ATMENS_BATCHSUB $ENSWORK/recenter_dst${npoe}.j + touch .SUBMITTED + @ ipoe = 0 # reset counter + @ npoe++ + + endif + + endif # else if ( -e recenter_mem${memtag}.j ) then $ATMENS_BATCHSUB recenter_mem${memtag}.j else - echo " ${MYNAME}: Failed to generate PBS jobs for Recentering ANA, Aborting ... " + echo " ${MYNAME}: Failed to generate batch jobs for Recentering ANA, Aborting ... " touch $ensloc/.FAILED exit(1) endif @@ -391,6 +442,7 @@ while ( $ic < $nmem ) endif # cd - end +cd $ensloc # Monitor status of ongoing jobs # ------------------------------ @@ -418,6 +470,7 @@ while ( $ic < $nmem + 1 ) @ ic = $ic + 1 end /bin/rm $ENSWORK/recenter_poe.* +/bin/rm $ENSWORK/*recenter_array*output* #/bin/rm $ENSWORK/recenter_poe*.j if ($failed) then diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh index 514b4603..00d1e4a8 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh @@ -15,6 +15,7 @@ # 25Mar2013 Todling Allow mp_stats to run under mpi # 21Feb2020 Todling Allow for high freq bkg (up to 1mn) # 02May2020 Todling Allow for user-spec freq of bkg stat calc +# 01Nov2024 Todling Design fix: hidden files belong to parent dir #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then @@ -90,7 +91,7 @@ if ( $#argv < 5 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 02May2020 by: R. Todling" + echo " Last modified: 01Nov2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -108,11 +109,11 @@ if ( !($?EXPID) ) setenv FAILED 1 if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 if ( $ENSPARALLEL ) then - setenv JOBGEN_NCPUS_PER_NODE 2 if ( !($?AENSTAT_NCPUS) ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $AENSTAT_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif if ( !($?GID) ) setenv FAILED 1 if ( !($?AENSTAT_QNAME) ) setenv FAILED 1 @@ -151,13 +152,9 @@ if ($?ATMENS_BKGSTATFRQ) then setenv BKGFREQ $ATMENS_BKGSTATFRQ endif -set yyyy = `echo $nymd | cut -c1-4` -set mm = `echo $nymd | cut -c5-6` -set dd = `echo $nymd | cut -c7-8` set hh = `echo $nhms | cut -c1-2` set hhmn = `echo $nhms | cut -c1-4` set yyyymmddhhmn = ${nymd}${hhmn} -set timetagz = ${nymd}_${hhmn}z @ bkgfreq_hr = $BKGFREQ / 60 @ bkgfreq_mn = $BKGFREQ - $bkgfreq_hr * 60 set bkgfreq_hh = `echo $bkgfreq_hr |awk '{printf "%02d", $1}'` @@ -176,20 +173,17 @@ if ( ("$ftype" == "ana.eta" ) ) then endif endif +# Inquire from HISTORY +# -------------------- +set ttemplate = `echorc.x -rc $ATMENSETC/HISTAENS.rc.tmpl -template dummy $nymd ${hh}0000 $ftype.template` +set timetagz = `echo $ttemplate | cut -d. -f1` + +###set lmtype = `echorc.x -rc HISTAENS.rc.tmpl $ftype` set lmtype = `echo $ftype | cut -d_ -f6` if (("$lmtype" == "p48") || ("$lmtype" == "z17") || ("$lmtype" == "slv")) then - set timetagz = ${yyyy}-${mm}-${dd}T${hhmn}Z - setenv myloc $ensloc/ensdiag + setenv MYLOC $ensloc/ensdiag else - setenv myloc $ensloc -endif - -if ("$lmtype" == "p48") then - set statsrc = "mp_stats_NP.rc" -else if ("$lmtype" == "z17") then - set statsrc = "mp_stats_NZ.rc" -else - set statsrc = "mp_stats.rc" + setenv MYLOC $ensloc endif set etag = "NULL" @@ -197,7 +191,7 @@ set etag = "NULL" # get positioned ... # ------------------ #cd $ensloc/ -cd $myloc/ +cd $MYLOC/ if( !($?ENSWORK) ) then setenv ENSWORK $ensloc endif @@ -205,46 +199,45 @@ endif # if new stat calculation ... # --------------------------- if( ($?ATMENSETC) ) then - if ( -e $ATMENSETC/$statsrc ) then + if ( -e $ATMENSETC/mp_stats.rc ) then if ( !($?AENSTAT_MPIRUN) ) then echo " ${MYNAME}: env(AENSTAT_MPIRUN) not defined, aborting ..." exit 1 endif - if(! -d ensmean ) mkdir -p $myloc/ensmean - if(! -d ensrms ) mkdir -p $myloc/ensrms - cd mem001 + if(! -d ensmean ) mkdir -p $MYLOC/ensmean + if(! -d ensrms ) mkdir -p $MYLOC/ensrms + cd $MYLOC/mem001 set alltype = `ls *.${ftype}.*${timetagz}.$NCSUFFIX` + cd - foreach fn ( $alltype ) -# set my_date = `echo $fn | cut -d. -f4 | cut -c1-8` -# set my_hhmn = `echo $fn | cut -d. -f4 | cut -c10-13` - set my_date = $nymd - set my_hhmn = $hhmn - set mopt = "-o $myloc/ensmean/$fn" - set sopt = "-stdv $myloc/ensrms/$fn" + set mopt = "-o $MYLOC/ensmean/$fn" + set sopt = "-stdv $MYLOC/ensrms/$fn" set eopt = "" if ("$ftype" == "bkg.eta" || "$ftype" == "ana.eta" || "$ftype" == "prog.eta" ) then if("$ftype" == "bkg.eta" ) set etype = "bene.err" if("$ftype" == "ana.eta" ) set etype = "aene.err" if("$ftype" == "prog.eta") set etype = "pene.err" - set eopt = "-ene ../ensrms/$EXPID.${etype}.${my_date}_${my_hhmn}z.$NCSUFFIX" + set eopt = "-ene ensrms/$EXPID.${etype}.${timetagz}.$NCSUFFIX" endif if (("$lmtype" == "p48") || ("$lmtype" == "z17") || ("$lmtype" == "slv")) then - if(! -d ensvar ) mkdir -p $myloc/ensvar - set sopt = "-variance $myloc/ensvar/$fn" + if(! -d ensvar ) mkdir -p $MYLOC/ensvar + set sopt = "-variance $MYLOC/ensvar/$fn" + endif + set dims = (`getgfiodim.x mem001/$fn` ) + setenv MP_STATS_LM $dims[3] + if ( $MP_STATS_LM == 0 ) then + setenv MP_STATS_LM 1 endif - if(! -e .MP_STATS_EGRESS_${ftype}_${my_date}${my_hhmn} ) then - $dry_run $AENSTAT_MPIRUN -rc $ATMENSETC/$statsrc $mopt $sopt $eopt -inc ${bkgfreq_hhmn}00 \ - -egress .MP_STATS_EGRESS_${ftype}_${my_date}${my_hhmn} ../mem*/$fn + vED -env $ATMENSETC/mp_stats.rc -o mp_stats.${ftype}.${timetagz}.rc + if(! -e .MP_STATS_EGRESS_${ftype}_${timetagz} ) then + $dry_run $AENSTAT_MPIRUN -rc mp_stats.${ftype}.${timetagz}.rc $mopt $sopt $eopt -inc ${bkgfreq_hhmn}00 \ + -egress .MP_STATS_EGRESS_${ftype}_${timetagz} mem*/$fn endif end # make sure all is successfully done foreach fn ( $alltype ) - set my_date = $nymd - set my_hhmn = $hhmn -# set my_date = `echo $fn | cut -d. -f4 | cut -c1-8` -# set my_hhmn = `echo $fn | cut -d. -f4 | cut -c10-13` - if (! -e .MP_STATS_EGRESS_${ftype}_${my_date}${my_hhmn} ) then - echo " ${MYNAME}: Failed to calculate stats (mp_stats.x) for ${ftype}_${my_date}${my_hhmn}, Aborting ... " + if (! -e .MP_STATS_EGRESS_${ftype}_${timetagz} ) then + echo " ${MYNAME}: Failed to calculate stats (mp_stats.x) for ${ftype}_${timetagz}, Aborting ... " touch $ENSWORK/.FAILED exit(1) endif @@ -252,6 +245,7 @@ if( ($?ATMENSETC) ) then exit (0) endif endif +/bin/rm mp_stats.*.rc setenv NCPUS 1 # NOTE: for now since there is a memory issue @@ -264,17 +258,17 @@ if ( $ATMENS_DOMEAN ) then if(! -d ens$this ) mkdir -p $ensloc/ens$this cd mem001 set alltype = `ls *.${ftype}.*${timetagz}.$NCSUFFIX` + cd - foreach fn ( $alltype ) set my_date = `echo $fn | cut -d. -f4 | cut -c1-8` set my_hhmn = `echo $fn | cut -d. -f4 | cut -c10-13` if ( "$this" == "mean" && $FAKEMEAN ) then - $dry_run /bin/cp $fn ../ens$this/$fn + $dry_run /bin/cp $fn ens$this/$fn else - $dry_run GFIO_mean_r4.x -o ../ens$this/$fn $opt -date $my_date -time ${my_hhmn}00 -inc ${bkgfreq_hhmn}00 ../mem*/$fn & + $dry_run GFIO_mean_r4.x -o ens$this/$fn $opt -date $my_date -time ${my_hhmn}00 -inc ${bkgfreq_hhmn}00 mem*/$fn & endif end wait - cd ../ end endif # diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_eaod.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_eaod.csh index 19b9c216..ba0d1f53 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_eaod.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_eaod.csh @@ -99,7 +99,7 @@ if ( $#argv < 4 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 26Mar2017 by: R. Todling" + echo " Last modified: 29Oct2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -122,7 +122,6 @@ if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 if ( !($?ENSPARALLEL) ) setenv ENSPARALLEL 0 if ( !($?ATMENS_DEBUG) ) setenv ATMENS_DEBUG 0 -if ( !($?ATMENKFAERO_MPIPROCS) ) setenv ATMENKFAERO_MPIPROCS 0 if ( !($?ATMENKFAERO_WALLCLOCK)) setenv ATMENKFAERO_WALLCLOCK 1:00:00 if ( !($?ATMENKFAERO_QNAME) ) setenv ATMENKFAERO_QNAME NULL @@ -596,14 +595,9 @@ if ( $AENS_GAAS_OPT > 1 ) then echo "${MYNAME}: running EnKF-aero analysis at ${eyyyymmddhh} ..." if( $ENSPARALLEL ) then - if ( $ATMENKFAERO_MPIPROCS ) then - set mpiprocs = "-mpiprocs $ATMENKFAERO_MPIPROCS" - else - set mpiprocs = "" - endif jobgen.pl \ -egress aenkfaero${xtag}.$eyyyymmddhh.log \ - -q $ATMENKFAERO_QNAME $mpiprocs \ + -q $ATMENKFAERO_QNAME \ aenkfaero${xtag} \ $GID \ $ATMENKFAERO_WALLCLOCK \ diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_enkf.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_enkf.csh index 3427c929..492f5b62 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_enkf.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_enkf.csh @@ -85,7 +85,7 @@ if ( $#argv < 3 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 03Mar2017 by: R. Todling" + echo " Last modified: 29Oct2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -102,7 +102,6 @@ if ( !($?MPIRUN_ATMENKF) ) setenv FAILED 1 if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 if ( !($?ENSPARALLEL) ) setenv ENSPARALLEL 0 if ( !($?ATMENS_DEBUG) ) setenv ATMENS_DEBUG 0 -if ( !($?ATMENKF_MPIPROCS) ) setenv ATMENKF_MPIPROCS 0 if ( !($?ATMENKF_WALLCLOCK)) setenv ATMENKF_WALLCLOCK 1:00:00 if ( !($?ATMENKF_QNAME) ) setenv ATMENKF_QNAME NULL @@ -230,14 +229,9 @@ else # ------------ if( $ENSPARALLEL ) then - if ( $ATMENKF_MPIPROCS ) then - set mpiprocs = "-mpiprocs $ATMENKF_MPIPROCS" - else - set mpiprocs = "" - endif jobgen.pl \ -egress enkf.log \ - -q $ATMENKF_QNAME $mpiprocs \ + -q $ATMENKF_QNAME \ aenkf \ $GID \ $ATMENKF_WALLCLOCK \ diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_ens2gcm.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_ens2gcm.csh index 04823663..16028009 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_ens2gcm.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_ens2gcm.csh @@ -10,6 +10,7 @@ # 07May2017 Todling Allow own config of mkiau when running GEPS # 03May2020 Todling Logic not to over-subscribe node # 21Jun2020 Todling Add ability to handle a control case +# 25Oct2024 Todling Implement slurm array distribution opt #------------------------------------------------------------------ # if ( !($?ATMENS_VERBOSE) ) then @@ -60,19 +61,20 @@ if ( $#argv < 3 ) then echo " " echo " ATMGEPS - set when running atmopheric GEOS EPS" echo " AENS_IAU_DSTJOB- distribute multiple works within smaller jobs" + echo " AENS_IAU_ARRAY - let slurm array control job distribution" echo " NCSUFFIX - suffix of hdf/netcdf files (default: nc4)" echo " ENSPARALLEL - when set, runs all ensemble components in parallel " echo " (default: off)" echo " ENSIAU_NCPUS - when parallel ens on, this sets NCPUS for IAU calculation" echo " IAU_WALLCLOCK - wall clock time to run makeiau, default 0:10:00 " - echo " IAU_QNAME - name of queue (default: NULL, that is, let pbs pick) " + echo " IAU_QNAME - name of queue (default: NULL, that is, let batch pick) " echo " MPIRUN_ENSIAU - specifies mprun command line, needed when ENSPARALLEL is on" echo " " echo " SEE ALSO" echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 07May2017 by: R. Todling" + echo " Last modified: 29Oct2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -89,17 +91,25 @@ if ( !($?GID) ) setenv FAILED 1 if ( !($?ATMGEPS) ) setenv ATMGEPS 0 if ( !($?AENS_IAU_DSTJOB) ) setenv AENS_IAU_DSTJOB 0 +if ( !($?AENS_IAU_ARRAY) ) setenv AENS_IAU_ARRAY 0 if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 if ( !($?ENSPARALLEL) ) setenv ENSPARALLEL 0 if ( !($?IAU_WALLCLOCK) ) setenv IAU_WALLCLOCK 0:10:00 if ( !($?IAU_QNAME) ) setenv IAU_QNAME NULL +if ( !($?JOBGEN_PFXNAME) ) then + set pfxname = "" +else + set pfxname = ${JOBGEN_PFXNAME}_ +endif + if ( $ENSPARALLEL ) then if ( !($?MPIRUN_ENSIAU) ) setenv FAILED 1 if ( !($?ENSIAU_NCPUS) ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $ENSIAU_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif endif @@ -114,6 +124,11 @@ set nymd = $2 set nhms = $3 set hh = `echo $nhms | cut -c1-2` set yyyymmddhh = ${nymd}${hh} +set yyyy = `echo $nymd | cut -c1-4` +set mm = `echo $nymd | cut -c5-6` +set dd = `echo $nymd | cut -c7-8` +set ddmmyyyy = ${dd}${mm}${yyyy} +set hhzddmmyyyy = ${hh}Z${ddmmyyyy} # used in jobname (easier to see cycle date/time) setenv ENSWORK $FVWORK if ( -e $ENSWORK/.DONE_${MYNAME}.$yyyymmddhh ) then @@ -184,7 +199,7 @@ if ( -d $ENSWORK/ensctrl ) then if ( -e iau_ensctrl.j ) then $ATMENS_BATCHSUB iau_ensctrl.j else - echo " ${MYNAME}: Failed to generate PBS jobs for makeiau (ctrl), Aborting ... " + echo " ${MYNAME}: Failed to generate batch jobs for makeiau (ctrl), Aborting ... " touch $ENSWORK/.FAILED exit(1) endif @@ -255,6 +270,7 @@ while ( $ic < $nmem ) @ fpoe++ + set machfile = "" if ( $AENS_IAU_DSTJOB != 0 ) then # case of multiple jobs within few larger ones # collect multiple iau calls into jumbo file if ( $ipoe < $AENS_IAU_DSTJOB ) then # nmem better devide by AENS_IAU_DSTJOB @@ -263,9 +279,9 @@ while ( $ic < $nmem ) echo $this_script_name >> $ENSWORK/iau_poe.$npoe chmod +x $ENSWORK/iau_poe.$npoe endif - set machfile = "-machfile $ENSWORK/iau_machfile$npoe.$ipoe" - else - set machfile = "" + if ( $AENS_IAU_ARRAY == 0 ) then + set machfile = "-machfile $ENSWORK/iau_machfile$npoe.$ipoe" + endif endif jobgen.pl \ @@ -284,57 +300,76 @@ while ( $ic < $nmem ) if ( -e iau_mem${memtag}.j ) then chmod +x iau_mem${memtag}.j else - echo " ${MYNAME}: Failed to generate IAU PBS jobs for Member ${memtag}, Aborting ... " + echo " ${MYNAME}: Failed to generate IAU batch jobs for Member ${memtag}, Aborting ... " touch $ENSWORK/.FAILED exit(1) endif - if ( ($ipoe == $AENS_IAU_DSTJOB) || (($fpoe == $ntodo ) && ($ipoe < $AENS_IAU_DSTJOB) ) ) then - set this_ntasks_per_node = `facter processorcount` - @ ncores_needed = $ENSIAU_NCPUS / $this_ntasks_per_node - if ( $ncores_needed == 0 ) then - @ myncpus = $this_ntasks_per_node - else - if ( $ENSIAU_NCPUS == $ncores_needed * $this_ntasks_per_node ) then - @ myncpus = $ENSIAU_NCPUS - else - @ myncpus = $ENSIAU_NCPUS / $this_ntasks_per_node - @ module = $myncpus * $this_ntasks_per_node - $ENSIAU_NCPUS - if ( $module != 0 ) @ myncpus = $myncpus + 1 - @ myncpus = $myncpus * $this_ntasks_per_node - endif - endif - @ myncpus = $ipoe * $myncpus - #_ @ myncpus = $AENS_IAU_DSTJOB * $ENSIAU_NCPUS - setenv JOBGEN_NCPUS $myncpus - jobgen.pl \ - -egress AIAU_EGRESS -q $IAU_QNAME \ - iau_dst${npoe} \ - $GID \ - $IAU_WALLCLOCK \ - "job_distributor.csh -machfile $ENSWORK/iau_machfile$npoe -usrcmd $ENSWORK/iau_poe.$npoe -usrntask $ENSIAU_NCPUS -njobs $ipoe" \ - $ENSWORK \ - $MYNAME \ - $ENSWORK/.DONE_POE${npoe}_${MYNAME}.$yyyymmddhh \ - "IAU Failed for Member ${npoe}" - /bin/mv iau_dst${npoe}.j $ENSWORK/ - $ATMENS_BATCHSUB $ENSWORK/iau_dst${npoe}.j - touch .SUBMITTED - @ ipoe = 0 # reset counter + if ( $AENS_IAU_ARRAY ) then + @ npoe++ - endif + if ( ($npoe == $nmem) || ($fpoe == $ntodo) ) then + + cd $ENSWORK + jobgen.pl \ + -egress IAU_EGRESS \ + -q $IAU_QNAME \ + ${pfxname}iau_array.$hhzddmmyyyy \ + -array "1-${nmem}%${AENS_IAU_DSTJOB}" -ncc \ + $GID \ + $IAU_WALLCLOCK \ + iau_mem\${memtag}.j \ + $ENSWORK/mem\${memtag} \ + $MYNAME \ + $ENSWORK/.DONE_ARRAY_${MYNAME}.$yyyymmddhh \ + "IAU Array Job Failed" + + if ( -e ${pfxname}iau_array.$hhzddmmyyyy.j ) then + $ATMENS_BATCHSUB ${pfxname}iau_array.$hhzddmmyyyy.j + else + echo " ${MYNAME}: Failed to generate array atch job for makeiau, Aborting ... " + touch $ENSWORK/.FAILED + exit(1) + endif - else + endif + + else # old style distribution + + if ( ($ipoe == $AENS_IAU_DSTJOB) || (($fpoe == $ntodo ) && ($ipoe < $AENS_IAU_DSTJOB) ) ) then + set mydist = (`atmens_ntasks.pl $ENSIAU_NCPUS $ipoe`) + setenv JOBGEN_NCPUS $mydist[1] + setenv JOBGEN_NCPUS_PER_NODE $mydist[2] + jobgen.pl \ + -egress AIAU_EGRESS -q $IAU_QNAME \ + iau_dst${npoe} \ + $GID \ + $IAU_WALLCLOCK \ + "job_distributor.csh -machfile $ENSWORK/iau_machfile$npoe -usrcmd $ENSWORK/iau_poe.$npoe -usrntask $ENSIAU_NCPUS -njobs $ipoe" \ + $ENSWORK \ + $MYNAME \ + $ENSWORK/.DONE_POE${npoe}_${MYNAME}.$yyyymmddhh \ + "IAU Failed for Member ${npoe}" + /bin/mv iau_dst${npoe}.j $ENSWORK/ + $ATMENS_BATCHSUB $ENSWORK/iau_dst${npoe}.j + touch .SUBMITTED + @ ipoe = 0 # reset counter + @ npoe++ + endif - if ( -e iau_mem${memtag}.j ) then - $ATMENS_BATCHSUB iau_mem${memtag}.j else - echo " ${MYNAME}: Failed to generate PBS jobs for makeiau, Aborting ... " - touch $ENSWORK/.FAILED - exit(1) - endif + + if ( -e iau_mem${memtag}.j ) then + $ATMENS_BATCHSUB iau_mem${memtag}.j + else + echo " ${MYNAME}: Failed to generate batch jobs for makeiau, Aborting ... " + touch $ENSWORK/.FAILED + exit(1) + endif + + endif # - endif # + endif # else # do serial work @@ -352,6 +387,7 @@ while ( $ic < $nmem ) endif # check for termination of individual case end +cd $ENSWORK # Monitor status of ongoing jobs # ------------------------------ @@ -379,6 +415,7 @@ if( -e $ENSWORK/.FAILED ) then else /bin/rm iau_dst* /bin/rm iau_poe* + /bin/rm *iau_array*output* endif touch .DONE_${MYNAME}.$yyyymmddhh diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmEnsConfig.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmEnsConfig.csh index a552b6a4..701703b2 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmEnsConfig.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmEnsConfig.csh @@ -68,7 +68,8 @@ setenv ADDINF_FACTOR_SPPT 0.2 # additive inflation for when SPPT is used setenv RECENTER_WALLCLOCK 1:00:00 setenv ENSRECENTER_NCPUS 1 setenv RECENTER_QNAME $ATMENS_QNAME -setenv AENS_RECENTER_DSTJOB 4 +setenv AENS_RECENTER_DSTJOB 16 +setenv AENS_RECENTER_ARRAY 1 # ensemble GAAS and AERO EnKF # --------------------------- @@ -151,7 +152,7 @@ setenv AENSTAT_QNAME $ATMENS_QNAME # post-egcm calculations # ---------------------- -setenv PEGCM_NCPUS @STAT_CPUS +setenv PEGCM_ARRAY 1 setenv PEGCM_WALLCLOCK 1:00:00 setenv PEGCM_QNAME $ATMENS_QNAME diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmOSEConfig.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmOSEConfig.csh index 7d135734..6b40549c 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmOSEConfig.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/AtmOSEConfig.csh @@ -136,7 +136,6 @@ setenv AENSTAT_QNAME $ATMENS_QNAME # post-egcm calculations # ---------------------- -setenv PEGCM_NCPUS @STAT_CPUS setenv PEGCM_WALLCLOCK 1:00:00 setenv PEGCM_QNAME $ATMENS_QNAME diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh index 7ca31843..9a25ab55 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh @@ -68,7 +68,8 @@ setenv ADDINF_FACTOR_SPPT 0.2 # additive inflation for when SPPT is used setenv RECENTER_WALLCLOCK 1:00:00 setenv ENSRECENTER_NCPUS 1 setenv RECENTER_QNAME $ATMENS_QNAME -setenv AENS_RECENTER_DSTJOB 4 +setenv AENS_RECENTER_DSTJOB 16 +setenv AENS_RECENTER_ARRAY 1 # ensemble GAAS and AERO EnKF # --------------------------- @@ -151,12 +152,12 @@ setenv AENSTAT_QNAME $ATMENS_QNAME # post-egcm calculations # ---------------------- -setenv PEGCM_NCPUS @STAT_CPUS +setenv PEGCM_ARRAY 1 setenv PEGCM_WALLCLOCK 1:00:00 setenv PEGCM_QNAME $ATMENS_QNAME # NOTES: -setenv OBSCLASS1 "r21c_igra2_raob_pre-qc_bufr,r21c_cdas_conv_pre-qc_bufr,r21c_prof_pre-qc_bufr,r21c_avhrrwnd_pre-qc_bufr,r21c_nmodis_pre-qc_bufr,r21c_goes_pre-qc_prep_bufr,r21c_metsat_pre-qc_prep_bufr,r21c_jma_pre-qc_prep_bufr,r21c_ascat_pre-qc_bufr,r21c_repro_ers2_pre-qc_bufr,r21c_qscat_jpl_pre-qc_bufr,r21c_loon_bufr,r21c_gmao_mlst_bufr,r21c_prep_bufr,r21c_acftpfl_bufr,r21c_satwnd_bufr,r21c_avhrr_satwnd_bufr,r21c_ncep_tcvitals,r21c_tmi_bufr,r21c_gpsro_bufr,r21c_sevcsr_bufr,r21c_1bamua_bufr,r21c_1bamub_bufr,r21c_1bhrs2_bufr,r21c_1bhrs3_bufr,r21c_1bhrs4_bufr,r21c_1bmsu_bufr,r21c_1bmhs_bufr,r21c_1bssu_bufr,r21c_eosairs_bufr,r21c_eosamsua_bufr,r21c_mtiasi_bufr,r21c_atms_bufr,r21c_ssmit11_bufr,r21c_ssmit13_bufr,r21c_ssmit14_bufr,r21c_ssmit15_bufr,r21c_amsre_bufr,r21c_osbuv8_bufr,r21c_mls_nc,r21c_npp_ompslp_nc,r21c_aura_omieff_nc,r21c_npp_ompsnmeff_nc,r21c_avcsam_bufr,r21c_avcspm_bufr,r21c_amsr2_bufr,r21c_crisfsr_bufr,r21c_gmi_bufr,r21c_prep_bufr,r21c_acftpfl_bufr" +setenv OBSCLASS1 "r21c_loon_bufr,r21c_gmao_mlst_bufr,r21c_prep_bufr,r21c_acftpfl_bufr,r21c_satwnd_bufr,r21c_avhrr_satwnd_bufr,r21c_ncep_tcvitals,r21c_tmi_bufr,r21c_gpsro_bufr,r21c_sevcsr_bufr,r21c_1bamua_bufr,r21c_1bamub_bufr,r21c_1bhrs2_bufr,r21c_1bhrs3_bufr,r21c_1bhrs4_bufr,r21c_1bmsu_bufr,r21c_1bmhs_bufr,r21c_1bssu_bufr,r21c_eosairs_bufr,r21c_eosamsua_bufr,r21c_mtiasi_bufr,r21c_atms_bufr,r21c_ssmit11_bufr,r21c_ssmit13_bufr,r21c_ssmit14_bufr,r21c_ssmit15_bufr,r21c_amsre_bufr,r21c_osbuv8_bufr,r21c_mls_nc,r21c_npp_ompslp_nc,r21c_aura_omieff_nc,r21c_npp_ompsnmeff_nc,r21c_avcsam_bufr,r21c_avcspm_bufr,r21c_amsr2_bufr,r21c_crisfsr_bufr,r21c_gmi_bufr,r21c_prep_bufr,r21c_acftpfl_bufr" setenv OBSCLASS "gmao_prep_bufr,gmao_acftpfl_bufr,$OBSCLASS1" diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/mp_stats_NP.rc b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/mp_stats_NP.rc deleted file mode 100644 index b5364509..00000000 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/mp_stats_NP.rc +++ /dev/null @@ -1,22 +0,0 @@ -# MAPL required variables -# ----------------------- -RUN_DT: 21600 # for now, must be set to 6-hrs -MAPL_ENABLE_TIMERS: NO -PRINTSPEC: 0 # (0: OFF, 1: IMPORT & EXPORT, 2: IMPORT, 3: EXPORT) - - -# Model Resolution and Timestep Parameters -# 4x4 is usually enough for any resolution -# ---------------------------------------- -NX: @NX -NY: @NY - -#MP_STATS_IM: 1152 -#MP_STATS_JM: 721 -MP_STATS_IM: @MP_STATS_IM -MP_STATS_JM: @MP_STATS_JM -MP_STATS_LM: 48 - -VNORM: YES -EPS_EER: 1.0 - diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/mp_stats_NZ.rc b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/mp_stats_NZ.rc deleted file mode 100644 index 26b4c211..00000000 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/mp_stats_NZ.rc +++ /dev/null @@ -1,22 +0,0 @@ -# MAPL required variables -# ----------------------- -RUN_DT: 21600 # for now, must be set to 6-hrs -MAPL_ENABLE_TIMERS: NO -PRINTSPEC: 0 # (0: OFF, 1: IMPORT & EXPORT, 2: IMPORT, 3: EXPORT) - - -# Model Resolution and Timestep Parameters -# 4x4 is usually enough for any resolution -# ---------------------------------------- -NX: @NX -NY: @NY - -#MP_STATS_IM: 1152 -#MP_STATS_JM: 721 -MP_STATS_IM: @MP_STATS_IM -MP_STATS_JM: @MP_STATS_JM -MP_STATS_LM: 17 - -VNORM: YES -EPS_EER: 1.0 - diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/mp_stats.rc b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/mp_stats.rc index ab0e2249..d179514f 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/mp_stats.rc +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/mp_stats.rc @@ -15,7 +15,7 @@ NY: @NY #MP_STATS_JM: 721 MP_STATS_IM: @MP_STATS_IM MP_STATS_JM: @MP_STATS_JM -MP_STATS_LM: @MP_STATS_LM +MP_STATS_LM: $MP_STATS_LM VNORM: YES EPS_EER: 1.0 diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gcm_ensemble.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gcm_ensemble.csh index bcc13cac..633ab964 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gcm_ensemble.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gcm_ensemble.csh @@ -29,6 +29,7 @@ # 21Feb2020 Todling Allow for high freq bkg (up to 1mn) # 03May2020 Todling Logic not to over-subscribe node # 22Jun2020 Todling Add ability to run a control member (also cleaned up) +# 26Oct2024 Todling Implement slurm array distribution opt #------------------------------------------------------------------------------------- if ( !($?ATMENS_VERBOSE) ) then @@ -98,6 +99,7 @@ if ( $#argv < 6 ) then echo " ENSGCM_NCPUS - when parallel ens on, this sets NCPUS for AGCM integration" echo " ENSCTRLONLY - allows running control member only" echo " AENS_GCM_DSTJOB- distribute multiple works within smaller jobs" + echo " AENS_GCM_ARRAY - let slurm array control job distribution" echo " AGCM_WALLCLOCK - wall clock time to run agcm, default 1:00:00 " echo " AGCM_QNAME - name of queue (default: NULL, that is, let BATCH pick) " echo " ATMENS_DO4DIAU - trigger to run 4DIAU " @@ -122,7 +124,7 @@ if ( $#argv < 6 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 20Apr2017 by: R. Todling" + echo " Last modified: 29Oct2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -149,22 +151,26 @@ if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 if ( !($?ENSPARALLEL) ) setenv ENSPARALLEL 0 if ( !($?ENSCTRLONLY) ) setenv ENSCTRLONLY 0 if ( !($?AENS_GCM_DSTJOB) ) setenv AENS_GCM_DSTJOB 0 +if ( !($?AENS_GCM_ARRAY) ) setenv AENS_GCM_ARRAY 0 if ( !($?AGCM_WALLCLOCK)) setenv AGCM_WALLCLOCK 1:00:00 if ( !($?AGCM_QNAME) ) setenv AGCM_QNAME NULL if ( !($?ATMENS_DO4DIAU)) setenv ATMENS_DO4DIAU 0 if ( !($?ATMENS_IGNORE_CHKPNT)) setenv ATMENS_IGNORE_CHKPNT 0 +if ( !($?JOBGEN_PFXNAME) ) then + set pfxname = "" +else + set pfxname = ${JOBGEN_PFXNAME}_ +endif + if ( $ENSPARALLEL ) then if ( !($?MPIRUN_ENSGCM) ) setenv FAILED 1 if ( !($?ENSGCM_NCPUS) ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $ENSGCM_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif - if ( !($?ENSGCM_NCPUS_PER_NODE) ) then - if ( $ENSGCM_NCPUS_PER_NODE > 0 ) then - setenv JOBGEN_NCPUS_PER_NODE $ENSGCM_NCPUS_PER_NODE - endif endif if ( $FAILED ) then @@ -181,6 +187,11 @@ set nlons = $5 set nlats = $6 set hhb = `echo $nhmsb | cut -c1-2` set yyyymmddhh = ${nymdb}${hhb} +set yyyy = `echo $nymd | cut -c1-4` +set mm = `echo $nymd | cut -c5-6` +set dd = `echo $nymd | cut -c7-8` +set ddmmyyyy = ${dd}${mm}${yyyy} +set hhzddmmyyyy = ${hhb}Z${ddmmyyyy} # used in jobname (easier to see cycle date/time) setenv ENSWORK $FVWORK if (-e $ENSWORK/.DONE_${MYNAME}.$yyyymmddhh ) then @@ -435,6 +446,7 @@ if(! -e .DONE_ENSFCST ) then if( $ENSPARALLEL ) then @ fpoe++ + set machfile = "" if ( $AENS_GCM_DSTJOB != 0 ) then # case of multiple jobs within few larger ones # collect multiple gcm calls into jumbo file if ( $ipoe < $AENS_GCM_DSTJOB ) then # nmem better devide by AENS_GCM_DSTJOB @@ -443,9 +455,9 @@ if(! -e .DONE_ENSFCST ) then echo $this_script_name >> $ENSWORK/agcm_poe.$npoe chmod +x $ENSWORK/agcm_poe.$npoe endif - set machfile = "-machfile $ENSWORK/agcm_machfile$npoe.$ipoe" - else - set machfile = "" + if ( $AENS_GCM_ARRAY == 0 ) then + set machfile = "-machfile $ENSWORK/agcm_machfile$npoe.$ipoe" + endif endif jobgen.pl \ @@ -470,46 +482,62 @@ if(! -e .DONE_ENSFCST ) then exit(1) endif - if ( ($ipoe == $AENS_GCM_DSTJOB) || (($fpoe == $ntodo) && ($ipoe < $AENS_GCM_DSTJOB) ) ) then - set this_ntasks_per_node = `facter processorcount` - @ ncores_needed = $ENSGCM_NCPUS / $this_ntasks_per_node - if ( $ncores_needed == 0 ) then - @ myncpus = $this_ntasks_per_node - else - if ( $ENSGCM_NCPUS == $ncores_needed * $this_ntasks_per_node ) then - @ myncpus = $ENSGCM_NCPUS - else - @ myncpus = $ENSGCM_NCPUS / $this_ntasks_per_node - @ module = $myncpus * $this_ntasks_per_node - $ENSGSI_NCPUS - if ( $module != 0 ) @ myncpus = $myncpus + 1 - @ myncpus = $myncpus * $this_ntasks_per_node - endif - endif - @ myncpus = $ipoe * $myncpus - #_ @ myncpus = $ipoe * $ENSGCM_NCPUS - setenv JOBGEN_NCPUS $myncpus - jobgen.pl \ - -q $AGCM_QNAME \ - agcm_dst${npoe} \ - $GID \ - $AGCM_WALLCLOCK \ - "job_distributor.csh -machfile $ENSWORK/agcm_machfile$npoe -usrcmd $ENSWORK/agcm_poe.$npoe -usrntask $ENSGCM_NCPUS -njobs $ipoe" \ - $ENSWORK \ - $MYNAME \ - $ENSWORK/.DONE_POE${npoe}_${MYNAME}.$yyyymmddhh \ - "AGCM Failed for Member ${npoe}" - if (! -e agcm_dst${npoe}.j ) then - echo " ${MYNAME}: AGCM Failed to generate DST BATCH jobs for Member ${memtag}, Aborting ... " - touch $ENSWORK/.FAILED - exit(1) - endif - /bin/mv agcm_dst${npoe}.j $ENSWORK/ - # this job is really not monitored; the real work done by agcm_mem${memtag}.j is monitored - $ATMENS_BATCHSUB $ENSWORK/agcm_dst${npoe}.j - touch .SUBMITTED - @ ipoe = 0 # reset counter + if ( $AENS_GCM_ARRAY ) then + @ npoe++ - endif + if ( ($npoe == $nmem) || ($fpoe == $ntodo) ) then + cd $ENSWORK + jobgen.pl \ + -egress EGRESS -q $AGCM_QNAME \ + ${pfxname}agcm_array.$hhzddmmyyyy \ + $GID \ + -array "1-$nmem%${AENS_GCM_DSTJOB}" -ncc \ + $AGCM_WALLCLOCK \ + agcm_mem\${memtag}.j \ + $ENSWORK/mem\${memtag} \ + $MYNAME \ + $ENSWORK/.DONE_ARRAY_${MYNAME}.$yyyymmddhh \ + "Atmos GCM Array Job Failed" + + if ( -e ${pfxname}agcm_array.$hhzddmmyyyy.j ) then + $ATMENS_BATCHSUB ${pfxname}agcm_array.$hhzddmmyyyy.j + else + echo " ${MYNAME}: AGCM Failed to generate BATCH ARRAY jobs, Aborting ... " + touch $ENSWORK/.FAILED + exit(1) + endif + + endif + + else + if ( ($ipoe == $AENS_GCM_DSTJOB) || (($fpoe == $ntodo) && ($ipoe < $AENS_GCM_DSTJOB) ) ) then + set mydist = (`atmens_ntasks.pl $ENSGCM_NCPUS $ipoe`) + setenv JOBGEN_NCPUS $mydist[1] + setenv JOBGEN_NCPUS_PER_NODE $mydist[2] + jobgen.pl \ + -q $AGCM_QNAME \ + agcm_dst${npoe} \ + $GID \ + $AGCM_WALLCLOCK \ + "job_distributor.csh -machfile $ENSWORK/agcm_machfile$npoe -usrcmd $ENSWORK/agcm_poe.$npoe -usrntask $ENSGCM_NCPUS -njobs $ipoe" \ + $ENSWORK \ + $MYNAME \ + $ENSWORK/.DONE_POE${npoe}_${MYNAME}.$yyyymmddhh \ + "AGCM Failed for Member ${npoe}" + if (! -e agcm_dst${npoe}.j ) then + echo " ${MYNAME}: AGCM Failed to generate DST BATCH jobs for Member ${memtag}, Aborting ... " + touch $ENSWORK/.FAILED + exit(1) + endif + /bin/mv agcm_dst${npoe}.j $ENSWORK/ + # this job is really not monitored; the real work done by agcm_mem${memtag}.j is monitored + $ATMENS_BATCHSUB $ENSWORK/agcm_dst${npoe}.j + touch .SUBMITTED + @ ipoe = 0 # reset counter + @ npoe++ + endif # + + endif # else if ( -e agcm_mem${memtag}.j ) then $ATMENS_BATCHSUB agcm_mem${memtag}.j @@ -544,6 +572,7 @@ if(! -e .DONE_ENSFCST ) then cd ../ end # end loop over members + cd $ENSWORK # Monitor status of ongoing jobs # ------------------------------ @@ -627,6 +656,7 @@ endif # recycling of RSTs # -------- /bin/rm agcm_dst* /bin/rm agcm_poe* +/bin/rm *agcm_array*output* # made it down here, all done # --------------------------- diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensbkg.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensbkg.csh index 66372526..ed2c534a 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensbkg.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensbkg.csh @@ -74,7 +74,7 @@ if ( $#argv < 4 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 08Apr2013 by: R. Todling" + echo " Last modified: 30Oct2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -92,7 +92,7 @@ if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 # --------------------------------------------------------------- if ( !($?ATMENS_DOSTATS) ) setenv ATMENS_DOSTATS 1 -setenv JOBGEN_NCPUS_PER_NODE 2 +setenv JOBGEN_NCPUS_PER_NODE -1 if ( !($?ENSRECENTER_NCPUS) ) then setenv FAILED 1 else diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/jobgen.pl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/jobgen.pl index fa408d7b..1fa2d9c5 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/jobgen.pl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/jobgen.pl @@ -10,6 +10,7 @@ # MPT we must prevent this. # 30Mar2017 Todling Hack to fix env that looks for missing lib under messed up NCCS batch system # 21Feb2020 Todling Allow for high freq bkg (up to 1mn) +# 25Oct2024 Todling Handling slurm array (and packable jobs) #----------------------------------------------------------------------------------------------------- use Env; # make env vars readily available @@ -28,13 +29,15 @@ # Command line options - GetOptions ( "egress=s", + GetOptions ( "array=s", + "egress=s", "expid=s", "q=s", "proc=s", - "mpiprocs=s", "machfile=s", "xc=s", + "ncc", + "packable", "h" ); usage() if $opt_h; @@ -94,6 +97,15 @@ sub init { $proc = ""; } + if ( $opt_packable ) { + if ( ! $opt_array ) { + print "$0: must specify array opt when using packable jobs \n\n"; + exit(1); + } + } + + $check_completion = 1; + if ( $opt_ncc ) {$check_completion = 0;} # FVROOT is where the binaries have been installed # ------------------------------------------------ @@ -112,17 +124,6 @@ sub init { print "$0: failed due undefined or zero ncpus_per_node \n\n"; exit(1); } - $nodes = $ncpus / $ncpus_per_node; - - if ( $opt_mpiprocs ) { - $mpiprocs = $opt_mpiprocs; - if ( $mpiprocs > $ncpus_per_node ) { - print "$0: failed due to invalid mpiprocs \n\n"; - exit(1); - } - } else { - $mpiprocs = $ncpus_per_node; - } # allow overwrite of job name $newjobname = $jobname; @@ -133,6 +134,11 @@ sub init { $newjobname = $newjobname . "_" . $ENV{JOBGEN_SFXNAME}; } + $forcerun = 0; + if ( $ENV{JOBGEN_FORCERUN} ) { + $forcerun = 1; + } + # The following is a tricky one: replace mpirun w/ mpiexec if ( $opt_machfile ) { @@ -176,16 +182,24 @@ sub gen { if ( $ENV{JOBGEN_QOS} ) { if ( $opt_q ne "datamove" ) { + if ( ! $opt_packable ) { # RT: until unfil NCCS allows dastest to run packable print SCRIPT <<"EOF"; #SBATCH --qos=$ENV{JOBGEN_QOS} EOF + } } } if ( $ENV{JOBGEN_PARTITION} ) { if ( $opt_q ne "datamove" ) { + if ( $opt_packable ) { + print SCRIPT <<"EOF"; +#SBATCH --partition=packable +EOF + } else { print SCRIPT <<"EOF"; #SBATCH --partition=$ENV{JOBGEN_PARTITION} EOF + } } } if ( $ENV{JOBGEN_RESERVATION} ) { @@ -203,14 +217,21 @@ sub gen { } print SCRIPT <<"EOF"; #SBATCH --ntasks=${ncpus} -#_SBATCH --ntasks-per-node=${ncpus_per_node} EOF - } - - if ( $opt_q ne "datamove" ) { + if ( $ncpus_per_node > 0 ) { + print SCRIPT <<"EOF"; +#SBATCH --ntasks-per-node=${ncpus_per_node} +EOF + } if ( $ENV{JOBGEN_STREAM} ) { print SCRIPT <<"EOF"; #SBATCH --constraint=$ENV{JOBGEN_STREAM} +EOF + } + if ( $opt_array ) { + print SCRIPT <<"EOF"; +#SBATCH --array=$opt_array +#SBATCH -o ${jobname}_output.%A_%a EOF } } @@ -250,8 +271,13 @@ sub gen { if ( $fvroot ) { print SCRIPT <<"EOF"; setenv FVROOT $fvroot + setenv OSVERSION `basename \$FVROOT` source \$FVROOT/bin/g5_modules set path = ( . \$FVHOME/run \$FVROOT/bin \$path ) + if ( \$OSVERSION == "install-SLES15" ) then + setenv I_MPI_FABRICS shm:ofi + setenv I_MPI_OFI_PROVIDER psm3 + endif EOF } @@ -261,6 +287,11 @@ sub gen { EOF } + if( $opt_array ) { + print SCRIPT <<"EOF"; + set memtag = `echo \${SLURM_ARRAY_TASK_ID} |awk '{printf "%03d", \$1}'` +EOF + } print SCRIPT <<"EOF"; # These env vars are here because the batch system is messed up @@ -274,8 +305,14 @@ sub gen { /bin/rm .SUBMITTED touch .RUNNING EOF + if( $opt_egress ) { print SCRIPT <<"EOF"; + if ( $check_completion ) then + if ( (! $forcerun) ) then + if ( -e $file2touch ) exit 0 + endif + endif if ( -e $opt_egress ) then /bin/rm $opt_egress endif @@ -449,8 +486,11 @@ sub usage { OPTIONS - -egress specify file to watch for completion of job (e.g., EGRESS for GCM) - -expid experiment name + -array X slurm array distribution (w/ or w/o packable) + -egress X specify file to watch for completion of job (e.g., EGRESS for GCM) + -expid X experiment name + -ncc no check completion (run regardless whether completed before) + -packable slurm packable distribution (must include array opt) -q specify pbs queue (e.g., datamove when archiving) -h prints this usage notice diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/obsvr_ensemble.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/obsvr_ensemble.csh index 7e78a819..970ca9ad 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/obsvr_ensemble.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/obsvr_ensemble.csh @@ -26,6 +26,7 @@ # 31Mar2020 Todling Jobmonitor to protect against faulty batch-block # 03May2020 Todling Logic not to over-subscribe node # 23Jun2020 Todling Refef meaning of ATMENSLOC +# 26Oct2024 Todling Implement slurm array distribution opt #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then @@ -99,6 +100,7 @@ if ( $#argv < 4 ) then echo " (default: off)" echo " ENSGSI_NCPUS - when parallel ens on, this sets NCPUS for Observer calculation" echo " AENS_OBSVR_DSTJOB- distribute multiple works within smaller jobs" + echo " AENS_OBSVR_ARRAY - let slurm array control job distribution" echo " OBSVR_WALLCLOCK - wall clock time to run observer, default 1:00:00 " echo " OBSVR_QNAME - name of queue (default: NULL, that is, let BATCH pick) " echo " " @@ -109,7 +111,7 @@ if ( $#argv < 4 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 08Apr2013 by: R. Todling" + echo " Last modified: 30Oct2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -131,6 +133,7 @@ if ( !($?STAGE4HYBGSI) ) setenv FAILED 1 # TBD hack if ( !($?AENS_ADDINFLATION) ) setenv AENS_ADDINFLATION 0 if ( !($?AENS_OBSVR_DSTJOB) ) setenv AENS_OBSVR_DSTJOB 0 +if ( !($?AENS_OBSVR_ARRAY) ) setenv AENS_OBSVR_ARRAY 0 if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 if ( !($?ENSMEANONLY) ) setenv ENSMEANONLY 0 # used for testing this script only (not to be global) if ( !($?ENSPARALLEL) ) setenv ENSPARALLEL 0 @@ -138,11 +141,18 @@ if ( !($?OBSVR_WALLCLOCK))setenv OBSVR_WALLCLOCK 1:00:00 if ( !($?OBSVR_QNAME)) setenv OBSVR_QNAME NULL if ( !($?STRICT) ) setenv STRICT 1 +if ( !($?JOBGEN_PFXNAME) ) then + set pfxname = "" +else + set pfxname = ${JOBGEN_PFXNAME}_ +endif + if ( $ENSPARALLEL ) then if ( !($?ENSGSI_NCPUS) ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $ENSGSI_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif endif @@ -158,6 +168,11 @@ set nymd = $3 set nhms = $4 set hh0 = `echo $nhms | cut -c1-2` set yyyymmddhh = ${nymd}${hh0} +set yyyy = `echo $nymd | cut -c1-4` +set mm = `echo $nymd | cut -c5-6` +set dd = `echo $nymd | cut -c7-8` +set ddmmyyyy = ${dd}${mm}${yyyy} +set hhzddmmyyyy = ${hh0}Z${ddmmyyyy} # used in jobname (easier to see cycle date/time) setenv ENSWORK $FVWORK if ( -e $ENSWORK/.DONE_${MYNAME}.$yyyymmddhh ) then @@ -539,6 +554,7 @@ while ( $n < $nmem ) @ fpoe++ + set machfile = "" if ( $AENS_OBSVR_DSTJOB != 0 ) then # case of multiple jobs within few larger ones # collect multiple observer calls into jumbo file if ( $ipoe < $AENS_OBSVR_DSTJOB ) then # nmem better devide by AENS_OBSVR_DSTJOB @@ -547,9 +563,9 @@ while ( $n < $nmem ) echo $this_script_name >> $ENSWORK/obsvr_poe.$npoe chmod +x $ENSWORK/obsvr_poe.$npoe endif - set machfile = "-machfile $ENSWORK/obsvr_machfile$npoe.$ipoe" - else - set machfile = "" + if ( $AENS_OBSVR_ARRAY == 0 ) then + set machfile = "-machfile $ENSWORK/obsvr_machfile$npoe.$ipoe" + endif endif jobgen.pl \ @@ -565,6 +581,7 @@ while ( $n < $nmem ) "Observer Failed for Member ${nnn}" if ( $AENS_OBSVR_DSTJOB != 0 ) then + if ( -e obs_mem${nnn}.j ) then chmod +x obs_mem${nnn}.j else @@ -573,46 +590,67 @@ while ( $n < $nmem ) exit(1) endif - if ( ($ipoe == $AENS_OBSVR_DSTJOB) || (($fpoe == $ntodo ) && ($ipoe < $AENS_OBSVR_DSTJOB) ) ) then - set this_ntasks_per_node = `facter processorcount` - @ ncores_needed = $ENSGSI_NCPUS / $this_ntasks_per_node - if ( $ncores_needed == 0 ) then - @ myncpus = $this_ntasks_per_node - else - if ( $ENSGSI_NCPUS == $ncores_needed * $this_ntasks_per_node ) then - @ myncpus = $ENSGSI_NCPUS - else - @ myncpus = $ENSGSI_NCPUS / $this_ntasks_per_node - @ module = $myncpus * $this_ntasks_per_node - $ENSGSI_NCPUS - if ( $module != 0 ) @ myncpus = $myncpus + 1 - @ myncpus = $myncpus * $this_ntasks_per_node - endif - endif - @ myncpus = $ipoe * $myncpus - #_ @ myncpus = $ipoe * $ENSGSI_NCPUS - setenv JOBGEN_NCPUS $myncpus - jobgen.pl \ - -q $OBSVR_QNAME \ - obsvr_dst${npoe} \ - $GID \ - $OBSVR_WALLCLOCK \ - "job_distributor.csh -machfile $ENSWORK/obsvr_machfile$npoe -usrcmd $ENSWORK/obsvr_poe.$npoe -usrntask $ENSGSI_NCPUS -njobs $ipoe " \ - $ENSWORK \ - $MYNAME \ - $ENSWORK/.DONE_POE${npoe}_${MYNAME}.$yyyymmddhh \ - "Observer Failed for Member ${npoe}" - if (! -e obsvr_dst${npoe}.j ) then - echo " ${MYNAME}: Observer Failed to generate DST BATCH jobs for Member ${nnn}, Aborting ... " - touch $ENSWORK/.FAILED - exit(1) - endif - /bin/mv obsvr_dst${npoe}.j $ENSWORK/ - $ATMENS_BATCHSUB $ENSWORK/obsvr_dst${npoe}.j - touch .SUBMITTED - @ ipoe = 0 # reset counter + if ( $AENS_OBSVR_ARRAY ) then + @ npoe++ - endif - else + if ( ($npoe == $nmem) || ($fpoe == $ntodo) ) then + + cd $ENSWORK + jobgen.pl \ + -egress GSI_EGRESS -q $OBSVR_QNAME \ + ${pfxname}obsvr_array.$hhzddmmyyyy \ + -array "1-$nmem%${AENS_OBSVR_DSTJOB}" -ncc \ + $GID \ + $OBSVR_WALLCLOCK \ + obs_mem\${memtag}.j \ + $ENSWORK/mem\${memtag} \ + $MYNAME \ + $ENSWORK/.DONE_ARRAY_${MYNAME}.$yyyymmddhh \ + "Observer Array Job Failed" + + if ( -e ${pfxname}obsvr_array.$hhzddmmyyyy.j ) then + $ATMENS_BATCHSUB ${pfxname}obsvr_array.$hhzddmmyyyy.j + else + echo " ${MYNAME}: Observer Failed to generate ARRAY BATCH jobs, Aborting ... " + touch $ENSWORK/.FAILED + exit(1) + endif + + endif + + else # old style distribution + + if ( ($ipoe == $AENS_OBSVR_DSTJOB) || (($fpoe == $ntodo ) && ($ipoe < $AENS_OBSVR_DSTJOB) ) ) then + set mydist = (`atmens_ntasks.pl $ENSGSI_NCPUS $ipoe`) + setenv JOBGEN_NCPUS $mydist[1] + setenv JOBGEN_NCPUS_PER_NODE $mydist[2] + jobgen.pl \ + -q $OBSVR_QNAME \ + obsvr_dst${npoe} \ + $GID \ + $OBSVR_WALLCLOCK \ + "job_distributor.csh -machfile $ENSWORK/obsvr_machfile$npoe -usrcmd $ENSWORK/obsvr_poe.$npoe -usrntask $ENSGSI_NCPUS -njobs $ipoe " \ + $ENSWORK \ + $MYNAME \ + $ENSWORK/.DONE_POE${npoe}_${MYNAME}.$yyyymmddhh \ + "Observer Failed for Member ${npoe}" + if (! -e obsvr_dst${npoe}.j ) then + echo " ${MYNAME}: Observer Failed to generate DST BATCH jobs for Member ${nnn}, Aborting ... " + touch $ENSWORK/.FAILED + exit(1) + endif + /bin/mv obsvr_dst${npoe}.j $ENSWORK/ + $ATMENS_BATCHSUB $ENSWORK/obsvr_dst${npoe}.j + touch .SUBMITTED + @ ipoe = 0 # reset counter + @ npoe++ + + endif # + + endif # + + else + if ( -e obs_mem${nnn}.j ) then $ATMENS_BATCHSUB obs_mem${nnn}.j touch .SUBMITTED @@ -647,6 +685,7 @@ while ( $n < $nmem ) cd ../ endif # DONE_MEM end +cd $ENSWORK # Monitor status of ongoing jobs # ------------------------------ @@ -676,6 +715,7 @@ while ( $n < $nmem ) cd - /bin/rm obsvr_dst* /bin/rm obsvr_poe* + /bin/rm *obsvr_array*output* else sleep 20 # allow for system-delay if (! -e .DONE_MEM${nnn}_${MYNAME}.$yyyymmddhh ) then # check for file one more time before giving up diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh index 3d9301c2..37ed5250 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh @@ -9,6 +9,13 @@ # streams to work on and do stats for # 21Apr2014 Todling Implement parallelization of stats calculation # 21Feb2020 Todling Allow for high freq bkg (up to 1mn) +# 30Oct2024 Todling - Bug fix: job monitor was waiting for last +# of date/time set of jobs to complete, but +# there is a chance some of those complete +# sooner than some from early date/time; now +# job-monitor works at date/time level. +# - Add option to pack jobs with slurm arrays +# and possibly packable. #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then @@ -71,6 +78,10 @@ if ( $#argv < 5 ) then echo " OPTIONAL ENVIRONMENT VARIABLES" echo " " echo " NCSUFFIX - suffix of hdf/netcdf files (default: nc4)" + echo " PEGCM_ARRAY - let slurm control distribution of jobs " + echo " PEGCM_PACKL - let arrays combined with packable jobs " + echo " PEGCM_ALLPARALLEL - parallelize all streams (a little aggressive)" + echo " (default: parallize by date/time)" echo " " echo " OPTIONAL RESOURCE FILES" echo " " @@ -82,7 +93,7 @@ if ( $#argv < 5 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 08Apr2013 by: R. Todling" + echo " Last modified: 31Oct2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -98,18 +109,35 @@ if ( !($?TIMEINC) ) setenv FAILED 1 if ( !($?NCSUFFIX) ) setenv NCSUFFIX nc4 +if ( !($?PEGCM_ALLPARALLEL) ) setenv PEGCM_ALLPARALLEL 0 +if ( !($?PEGCM_ARRAY) ) setenv PEGCM_ARRAY 0 +if ( !($?PEGCM_PACKL) ) setenv PEGCM_PACKL 0 if ( !($?PEGCM_QNAME) ) setenv PEGCM_QNAME NULL if ( !($?PEGCM_WALLCLOCK)) setenv PEGCM_WALLCLOCK NULL + +setenv JOBGEN_NCPUS_PER_NODE -1 + if ( $PEGCM_QNAME == "NULL" || $PEGCM_WALLCLOCK == "NULL" ) then setenv PEGCM_SERIAL 1 + setenv PEGCM_ALLPARALLEL 0 # override user specs else - if ( !($?PEGCM_NCPUS) ) then - echo "${MYNAME}: must define PEGCM_NCPUS" + if ( !($?AENSTAT_NCPUS) ) then + echo "${MYNAME}: must define AENSTAT_NCPUS" + setenv FAILED 1 + else + setenv JOBGEN_NCPUS $AENSTAT_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif setenv PEGCM_SERIAL 0 endif +if ( !($?JOBGEN_PFXNAME) ) then + set pfxname = "" +else + set pfxname = ${JOBGEN_PFXNAME}_ +endif + if ( $FAILED ) then env echo " ${MYNAME}: not all required env vars defined" @@ -139,6 +167,11 @@ if ( -e $ENSWORK/.FAILED ) then exit(1) endif +set packable = "" +if ( $PEGCM_PACKL ) then + set packable = "-packable" +endif + #source $FVROOT/bin/g5_modules set path = ( . $FVHOME/run $FVROOT/bin $path ) @@ -152,7 +185,7 @@ touch .no_archiving # Calculate mean/rms of newly generated ensemble # ---------------------------------------------- -if (! -e $ENSWORK/.DONE_redone_allbkgstat_$MYNAME.$yyyymmddhhmn ) then +if (! -e $ENSWORK/.DONE_redone_allstat_$MYNAME.$yyyymmddhhmn ) then cd $ENSWORK @@ -182,18 +215,21 @@ if (! -e $ENSWORK/.DONE_redone_allbkgstat_$MYNAME.$yyyymmddhhmn ) then @ anafreq_sec = $TIMEINC * 60 @ toffset_sec = $toffset * 60 @ nt = $anafreq_sec / $bkgfreq_sec + 1 -# @ nall = $nt * $ntyps - @ nall = 0 + @ ntotal = $nt * $ntyps set adate = ( `tick $nymdb $nhmsb $toffset_sec` ) - @ n = 0 - @ m = 0 + @ n = 0; @ idx = 0 while ( $n < $nt ) @ n++ set this_nymd = $adate[1] set this_nhms = $adate[2] set this_hhmn = `echo $this_nhms | cut -c1-4` + set this_mm = `echo $this_nymd | cut -c5-6` + set this_dd = `echo $this_nymd | cut -c7-8` + set this_hh = `echo $this_nhms | cut -c1-2` + set this_hhzddmm = ${this_hh}Z${this_dd}${this_mm} set this_yyyymmddhhmn = ${this_nymd}${this_hhmn} - if (! -e $ENSWORK/.DONE_redone_bkgstat_$MYNAME.$this_yyyymmddhhmn ) then + if (! -e $ENSWORK/.DONE_redone_stat_$MYNAME.$this_yyyymmddhhmn ) then + @ m = 0 foreach outkind ( $alltyps ) @ m++ set mmm = `echo $m | awk '{printf "%03d", $1}'` @@ -209,50 +245,151 @@ if (! -e $ENSWORK/.DONE_redone_allbkgstat_$MYNAME.$yyyymmddhhmn ) then else # submit stat calls as independent jobs - setenv JOBGEN_NCPUS_PER_NODE 1 - setenv JOBGEN_NCPUS $PEGCM_NCPUS + if ( $PEGCM_ALLPARALLEL ) then + @ idx = $idx + 1 + set idx = `echo $idx | awk '{printf "%03d", $1}'` + set tagA = $yyyymmddhhmn + set tagB = $yyyymmddhhmn + else + set idx = $mmm + set tagA = $this_hhzddmm + set tagB = $this_yyyymmddhhmn + endif + setenv JOBGEN_NCPUS $AENSTAT_NCPUS jobgen.pl \ -q $PEGCM_QNAME \ - pegcm_$mmm \ + pegcm_${idx}.${tagA} \ $GID \ $PEGCM_WALLCLOCK \ "atmens_stats.csh $nmem $outkind $ENSWORK $this_nymd $this_nhms |& tee -a $ENSWORK/pegcm_${outkind}.$this_yyyymmddhhmn.log"\ $ENSWORK \ $MYNAME \ - $ENSWORK/.DONE_MEM${mmm}_${MYNAME}.$yyyymmddhhmn \ + $ENSWORK/.DONE_MEM${idx}_${MYNAME}.$tagB \ "PEGCM Failed" - if ( -e pegcm_${mmm}.j ) then - $ATMENS_BATCHSUB pegcm_${mmm}.j - touch .SUBMITTED + if ( -e pegcm_${idx}.${tagA}.j ) then + chmod +x pegcm_${idx}.${tagA}.j + if ( ! $PEGCM_ARRAY ) then + $ATMENS_BATCHSUB pegcm_${idx}.${tagA}.j + touch .SUBMITTED + endif else - echo " ${MYNAME}: PostEGCM Failed to generate PBS jobs for Member ${mmm}, Aborting ... " + echo " ${MYNAME}: PostEGCM Failed to generate job for ${mmm}_${this_hhzddmm}, Aborting ... " touch $ENSWORK/.FAILED exit(1) endif endif # parallel jobs - @ nall = $nall + 1 + end # - endif + touch $ENSWORK/.DONE_redone_stat_$MYNAME.$this_yyyymmddhhmn + + # In case of parallel jobs ... + # ---------------------------- + if( ! $PEGCM_SERIAL ) then + if ( ! $PEGCM_ALLPARALLEL ) then + + # If slurm arrays, launch before monitoring ... + # --------------------------------------------- + if ( $PEGCM_ARRAY ) then + # Note: the parameter called "memtag" in the job-name line below is + # is a parameter if jobgen.pl - not of the present program; + # the name of the var in jobgen is "memtag", and is properly + # set internally in jobgen. + jobgen.pl \ + -q $PEGCM_QNAME $packable \ + ${pfxname}pegcm_array.$this_hhzddmm \ + $GID \ + -array "1-${ntyps}" -ncc \ + $PEGCM_WALLCLOCK \ + pegcm_\${memtag}.${this_hhzddmm}.j \ + $ENSWORK \ + $MYNAME \ + $ENSWORK/.DONE_ARRAY_${MYNAME}_\${memtag}.$this_yyyymmddhhmn \ + "PEGCM Array Job Failed" + + if ( -e $ensloc/${pfxname}pegcm_array.$this_hhzddmm.j ) then + $ATMENS_BATCHSUB $ensloc/${pfxname}pegcm_array.$this_hhzddmm.j + else + echo " ${MYNAME}: Failed to generate array batch PEGCM jobs, Aborting ... " + touch $ensloc/.FAILED + exit(1) + endif + endif # + + # Monitor batch jobs + # ------------------ + jobmonitor.csh $ntyps ${MYNAME} $ENSWORK $this_yyyymmddhhmn + if ($status) then + echo "${MYNAME}: cannot complete due to failed jobmonitor, aborting" + exit(1) + endif + + # clean up + # -------- + /bin/rm $ENSWORK/pegcm_*.j + /bin/rm $ENSWORK/pegcm_*.j.* + /bin/rm $ENSWORK/*pegcm_*.log + + endif # <.not.PEGCM_ALLPARALLEL> + endif # <.not.SERIAL> + + endif # + touch $ENSWORK/.DONE_redone_allstat_$MYNAME.$yyyymmddhhmn + + # Increment date/time + # ------------------- set adate = (`tick $this_nymd $this_nhms $bkgfreq_sec`) - end -endif + end # + + # In case of doing separated jobs, monitor their completion + # --------------------------------------------------------- + if( $PEGCM_ALLPARALLEL ) then + + # If slurm arrays, launch before monitoring ... + # --------------------------------------------- + if ( $PEGCM_ARRAY ) then + # Note: the parameter called "memtag" in the job-name line below is + # is a parameter if jobgen.pl - not of the present program; + # the name of the var in jobgen is "memtag", and is properly + # set internally in jobgen. + jobgen.pl \ + -q $PEGCM_QNAME $packable \ + ${pfxname}pegcm_array.$yyyymmddhhmn \ + $GID \ + -array "1-${ntotal}" -ncc \ + $PEGCM_WALLCLOCK \ + pegcm_\${memtag}.${yyyymmddhhmn}.j \ + $ENSWORK \ + $MYNAME \ + $ENSWORK/.DONE_ARRAY_${MYNAME}_\${memtag}.$yyyymmddhhmn \ + "PEGCM Array Job Failed" + + if ( -e $ensloc/${pfxname}pegcm_array.$yyyymmddhhmn.j ) then + $ATMENS_BATCHSUB $ensloc/${pfxname}pegcm_array.$yyyymmddhhmn.j + else + echo " ${MYNAME}: Failed to generate array batch PEGCM jobs, Aborting ... " + touch $ensloc/.FAILED + exit(1) + endif + endif # + + # Monitor batch jobs + # ------------------ + jobmonitor.csh $ntotal ${MYNAME} $ENSWORK $yyyymmddhhmn + if ($status) then + echo "${MYNAME}: cannot complete due to failed jobmonitor, aborting" + exit(1) + endif + + # clean up + # -------- + /bin/rm $ENSWORK/pegcm_*.j + /bin/rm $ENSWORK/pegcm_*.j.* + /bin/rm $ENSWORK/*pegcm_*.log + endif # -# In case of doing separated jobs, monitor their completion -# --------------------------------------------------------- -if( ! $PEGCM_SERIAL ) then - jobmonitor.csh $nall $MYNAME $ENSWORK $yyyymmddhhmn - if ($status) then - echo "${MYNAME}: cannot complete due to failed jobmonitor, aborting" - exit(1) - endif - # clean up - # -------- - /bin/rm pegcm_*.j pegcm_*.j.* - /bin/rm pegcm_*.log endif -touch $ENSWORK/.DONE_redone_allbkgstat_$MYNAME.$yyyymmddhhmn # made it down here, all done # --------------------------- diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setobsvr.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setobsvr.csh index 29ee8a46..226ab94a 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setobsvr.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setobsvr.csh @@ -18,6 +18,7 @@ # 16Apr2018 Todling Revise access to Y. Zhu sat-bias-correction files # 15Feb2020 Todling Allow acquire to work as non-batch call # 20Jun2020 Todling Minor changes for flexible location of RC files +# 29Oct2024 Todling Edit obsys.rc and fill in env vars #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then @@ -182,6 +183,12 @@ /bin/cp $ATMENSETC/GSI_GridComp.rc.tmpl . /bin/cp $ATMENSETC/satbias.acq . +# obsys.rc might have env vars to be filled in +# -------------------------------------------- + if (-e obsys.rc ) then + /bin/mv obsys.rc obsys.rc.env + vED -env obsys.rc.env -o obsys.rc + endif # acquire initial conditions diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setperts.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setperts.csh index f804597b..29d41cb6 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setperts.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setperts.csh @@ -74,7 +74,7 @@ if ( $#argv < 6 ) then echo " " echo " AUTHOR" echo " Amal El Akkraoui (Amal.ElAkkraoui@nasa.gov), NASA/GMAO " - echo " Last modified: 08Apr2013 by: R. Todling" + echo " Last modified: 30Oct2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -116,7 +116,7 @@ setenv DO_DMGET 1 # force dmget to be used when acquire retrieve files setenv dry_run if ( $ENSPARALLEL ) then - setenv JOBGEN_NCPUS_PER_NODE 2 + setenv JOBGEN_NCPUS_PER_NODE -1 if ( !($?ENSRECENTER_NCPUS) ) then setenv FAILED 1 else diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_atmens_stats.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_atmens_stats.j index 3d786634..91e90329 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_atmens_stats.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_atmens_stats.j @@ -11,7 +11,6 @@ setenv JOBGEN_QOS advda setenv JOBGEN_CONSTRAINT hasw # to run pegcm stats in parallel -setenv PEGCM_NCPUS 4 setenv PEGCM_WALLCLOCK 1:30:00 setenv PEGCM_QNAME compute diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_rec_stats.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_rec_stats.j index 6e5eb05e..e59e1848 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_rec_stats.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_rec_stats.j @@ -10,7 +10,6 @@ setenv GET_TAR_BATCH 1 setenv ATMENS_BATCHSUB sbatch # to run pegcm stats in parallel -setenv PEGCM_NCPUS 4 setenv PEGCM_WALLCLOCK 0:30:00 setenv PEGCM_QNAME nccs2 From 528d39e0fb61f5f8ce74b4a7c7723c74d6d28698 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Fri, 1 Nov 2024 18:49:47 -0400 Subject: [PATCH 02/14] full parallelization w/ slurm array --- .../NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh index 9a25ab55..dae2eef1 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh @@ -152,6 +152,7 @@ setenv AENSTAT_QNAME $ATMENS_QNAME # post-egcm calculations # ---------------------- +setenv PEGCM_ALLPARALLEL 1 setenv PEGCM_ARRAY 1 setenv PEGCM_WALLCLOCK 1:00:00 setenv PEGCM_QNAME $ATMENS_QNAME From 5bdd164aa2e63a3d4b933c43879938ca2ca76459 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Sat, 2 Nov 2024 08:02:26 -0400 Subject: [PATCH 03/14] minor fixes --- .../NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh | 12 ++++++++++-- .../NCEP_enkf/scripts/gmao/etc/CMakeLists.txt | 2 -- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh index 00d1e4a8..a9f3b9a4 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh @@ -175,8 +175,16 @@ endif # Inquire from HISTORY # -------------------- -set ttemplate = `echorc.x -rc $ATMENSETC/HISTAENS.rc.tmpl -template dummy $nymd ${hh}0000 $ftype.template` -set timetagz = `echo $ttemplate | cut -d. -f1` +set timetagz = ${nymd}_${hhmm}z +if ( $ftype != "ana.eta" && $ftype != "inc.eta" ) then # these types are not in HISTORY + set hist = (`ls $ATMENSETC/HIST*.rc.tmpl`) + if ( $#hist != 1 ) then + echo " ${MYNAME}: should only find single HIST*rc.tmpl in $ATMENSETC, Aborting ..." + exit 1 + endif + set ttemplate = `echorc.x -rc $hist[1] -template dummy $nymd ${hh}0000 $ftype.template` + set timetagz = `echo $ttemplate | cut -d. -f1` +endif ###set lmtype = `echorc.x -rc HISTAENS.rc.tmpl $ftype` set lmtype = `echo $ftype | cut -d_ -f6` diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt index 4bfaae1e..fb2aecae 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt @@ -46,8 +46,6 @@ set (R21C_files R21C/obs1gsi_member.rc R21C/HISTAENS.rc.tmpl R21C/post_egcm.rc - R21C/mp_stats_NZ.rc - R21C/mp_stats_NP.rc R21C/CAP.rc.tmpl R21C/atmens_storage.arc R21C/AtmEnsConfig.csh From 892e10339394b482320c3f0151154bfab8896868 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Mon, 4 Nov 2024 11:14:04 -0500 Subject: [PATCH 04/14] revise HISTORY handling of gcm diag fields (during corrector); and handling of post_egcm --- .../NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j | 17 +++++- .../NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ose.j | 6 +- .../NCEP_enkf/scripts/gmao/atmens_efsens.csh | 22 ++------ .../NCEP_enkf/scripts/gmao/atmens_stats.csh | 51 ++++++++--------- .../NCEP_enkf/scripts/gmao/atmens_vtrack.csh | 1 + .../NCEP_enkf/scripts/gmao/atmos_eaod.csh | 2 +- .../NCEP_enkf/scripts/gmao/atmos_egsi.csh | 22 ++------ .../NCEP_enkf/scripts/gmao/atmos_enkf.csh | 1 + .../NCEP_enkf/scripts/gmao/etc/CAP.rc.tmpl | 4 +- .../NCEP_enkf/scripts/gmao/etc/CMakeLists.txt | 16 +++--- .../scripts/gmao/etc/R21C/AtmEnsConfig.csh | 7 ++- .../scripts/gmao/etc/R21C/CAP.rc.tmpl | 21 ------- .../scripts/gmao/etc/R21C/HISTAENS.rc.tmpl | 44 +++++++-------- .../scripts/gmao/etc/R21C/post_egcm_diag.rc | 14 +++++ .../NCEP_enkf/scripts/gmao/gen_ensbkg.csh | 5 +- .../NCEP_enkf/scripts/gmao/gen_ensrst.csh | 2 +- .../NCEP_enkf/scripts/gmao/obsvr_ensfinal.csh | 1 + .../NCEP_enkf/scripts/gmao/post_eana.csh | 16 ++++-- .../NCEP_enkf/scripts/gmao/post_efso.csh | 1 + .../NCEP_enkf/scripts/gmao/post_egcm.csh | 45 ++++++--------- .../NCEP_enkf/scripts/gmao/ut_atmens_stats.j | 55 ++++++------------- 21 files changed, 158 insertions(+), 195 deletions(-) delete mode 100644 src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/CAP.rc.tmpl create mode 100644 src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm_diag.rc diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j index 7885b2e7..7de642d3 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j @@ -511,7 +511,7 @@ # ------------------------------------------- if ( $RUN_PEANA || $DO_ATM_ENS ) then zeit_ci.x post_eana - post_eana.csh $EXPID $anymd $anhms |& tee -a atm_ens.log + post_eana.csh $EXPID $anymd $anhms spread |& tee -a atm_ens.log if( $status) then echo "post_eana failed" exit(1) @@ -595,11 +595,22 @@ set arch_nhms = $nhmsb if( $RUN_AENSFCST || $DO_ATM_ENS ) then zeit_ci.x post_egcm - post_egcm.csh $EXPID $nymdb $nhmsb $TIMEINC $FVHOME/atmens + set myrc = $ATMENSETC/post_egcm.rc + if ( -e $ATMENSETC/post_egcm_${hhb}.rc ) set myrc = $ATMENSETC/post_egcm_${hhb}.rc + post_egcm.csh $EXPID $nymdb $nhmsb $TIMEINC spread $myrc $FVHOME/atmens if ($status) then - echo "post_egcm failed" + echo "post_egcm (bkg) failed" exit(1) endif + set myrc = $ATMENSETC/post_egcm_diag.rc + if ( -e $ATMENSETC/post_egcm_diag_${hhb}.rc ) set myrc = $ATMENSETC/post_egcm_diag_${hhb}.rc + if ( -e $myrc ) then + post_egcm.csh $EXPID $nymdb $nhmsb 0 variance $myrc $FVHOME/atmens/ensdiag + if ($status) then + echo "post_egcm (diag) failed" + exit(1) + endif + endif zeit_co.x post_egcm endif diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ose.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ose.j index 6c2704dc..d7e31ca4 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ose.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ose.j @@ -393,9 +393,11 @@ set arch_nhms = $nhmsb if( ($RUN_POSTEGCM && $RUN_AENSFCST) || $DO_ATM_ENS ) then zeit_ci.x post_egcm - post_egcm.csh $EXPID $nymdb $nhmsb $TIMEINC $FVHOME/atmens + set myrc = $ATMENSETC/post_egcm.rc + if ( -e $ATMENSETC/post_egcm_${hhb}.rc ) set myrc = $ATMENSETC/post_egcm_${hhb}.rc + post_egcm.csh $EXPID $nymdb $nhmsb $TIMEINC spread $myrc $FVHOME/atmens if ($status) then - echo "post_egcm failed" + echo "post_egcm (bkg) failed" exit(1) endif zeit_co.x post_egcm diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_efsens.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_efsens.csh index cb497d29..e4f34a67 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_efsens.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_efsens.csh @@ -11,6 +11,7 @@ setenv DRYRUN # # 23Apr2017 Todling Initial script # 03May2020 Todling Logic not to over-subscribe node +# 03Nov2024 Todling Revised job distribution #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then @@ -97,6 +98,7 @@ if ( $ENSPARALLEL ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $ENSGCMADJ_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif endif @@ -289,23 +291,9 @@ set nmem = $members[1] endif if ( ($ipoe == $AENS_GCMADJ_DSTJOB) || (($fpoe == $ntodo) && ($ipoe < $AENS_GCMADJ_DSTJOB) ) ) then - set this_ntasks_per_node = `facter processorcount` - @ ncores_needed = $ENSGCMADJ_NCPUS / $this_ntasks_per_node - if ( $ncores_needed == 0 ) then - @ myncpus = $this_ntasks_per_node - else - if ( $ENSGCMADJ_NCPUS == $ncores_needed * $this_ntasks_per_node ) then - @ myncpus = $ENSGCMADJ_NCPUS - else - @ myncpus = $ENSGCMADJ_NCPUS / $this_ntasks_per_node - @ module = $myncpus * $this_ntasks_per_node - $ENSGCMADJ_NCPUS - if ( $module != 0 ) @ myncpus = $myncpus + 1 - @ myncpus = $myncpus * $this_ntasks_per_node - endif - endif - @ myncpus = $ipoe * $myncpus - #_ @ myncpus = $ipoe * $ENSGCMADJ_NCPUS - setenv JOBGEN_NCPUS $myncpus + set mydist = (`atmens_ntasks.pl $ENSGCMADJ_NCPUS $ipoe`) + setenv JOBGEN_NCPUS $mydist[1] + setenv JOBGEN_NCPUS_PER_NODE $mydist[2] jobgen.pl \ -egress AGCMADJ_DST_EGRESS -q $AGCMADJ_QNAME \ agcmadj_dst${npoe} \ diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh index a9f3b9a4..ab528ee8 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh @@ -15,7 +15,8 @@ # 25Mar2013 Todling Allow mp_stats to run under mpi # 21Feb2020 Todling Allow for high freq bkg (up to 1mn) # 02May2020 Todling Allow for user-spec freq of bkg stat calc -# 01Nov2024 Todling Design fix: hidden files belong to parent dir +# 01Nov2024 Todling - Design fix: hidden files belong to parent dir +# - Enhanced flexibility in handling mp_stats.rc #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then @@ -28,7 +29,7 @@ setenv MYNAME atmens_stats.csh # need usage here # --------------- -if ( $#argv < 5 ) then +if ( $#argv < 6 ) then echo " " echo " \\begin{verbatim} " echo " " @@ -43,6 +44,7 @@ if ( $#argv < 5 ) then echo " where" echo " nmem - number of members to be created" echo " ftype - file type (e.g., bkg.eta, bkg.sfc, ana.eta)" + echo " xstat - extra statistics (e.g., spread or variance" echo " ensloc - location to place generated ensemble" echo " nymd - date of members to calc stats for (as YYYYMMDD)" echo " nhms - time of members to calc stats for (as HHMMSS)" @@ -59,7 +61,7 @@ if ( $#argv < 5 ) then echo " of RMS; and finally performing (iv) calculation of energy-based spread." echo " " echo " Example of valid command line:" - echo " $MYNAME 10 /archive/u/$user/u000_c72/atmens 20111201 210000" + echo " $MYNAME 10 bkg.eta spread /archive/u/$user/u000_c72/atmens 20111201 210000" echo " " echo " REQUIRED RESOURCE FILES" echo " " @@ -81,6 +83,7 @@ if ( $#argv < 5 ) then echo " AENSTAT_NCPUS - number of cpus to use for this procedure" echo " (NOTE: required when ENSPARALLEL is on)" echo " ATMENS_BKGSTATFRQ - specific bkg freq for stats calculation" + echo " " echo " SEE ALSO" echo " mp_stats.x - program to calculate statistics from fields in SDF files" echo " dyn_diff.x - program to calculate difference between dyn-vector files" @@ -143,9 +146,10 @@ endif # ----------------------- set nmem = $1 # number of ensemble members set ftype = $2 # file type (e.g, bkg.eta) -set ensloc = $3 # root location for members and mean -set nymd = $4 # date of members to calc stats for (YYYYMMDD) -set nhms = $5 # time of members to calc stats for (HHMMSS) +set xstat = $3 # file type (e.g, bkg.eta) +set ensloc = $4 # root location for members and mean +set nymd = $5 # date of members to calc stats for (YYYYMMDD) +set nhms = $6 # time of members to calc stats for (HHMMSS) setenv BKGFREQ $ASYNBKG if ($?ATMENS_BKGSTATFRQ) then @@ -175,7 +179,7 @@ endif # Inquire from HISTORY # -------------------- -set timetagz = ${nymd}_${hhmm}z +set timetagz = ${nymd}_${hhmn}z if ( $ftype != "ana.eta" && $ftype != "inc.eta" ) then # these types are not in HISTORY set hist = (`ls $ATMENSETC/HIST*.rc.tmpl`) if ( $#hist != 1 ) then @@ -186,20 +190,11 @@ if ( $ftype != "ana.eta" && $ftype != "inc.eta" ) then # these types are not in set timetagz = `echo $ttemplate | cut -d. -f1` endif -###set lmtype = `echorc.x -rc HISTAENS.rc.tmpl $ftype` -set lmtype = `echo $ftype | cut -d_ -f6` -if (("$lmtype" == "p48") || ("$lmtype" == "z17") || ("$lmtype" == "slv")) then - setenv MYLOC $ensloc/ensdiag -else - setenv MYLOC $ensloc -endif - set etag = "NULL" # get positioned ... # ------------------ -#cd $ensloc/ -cd $MYLOC/ +cd $ensloc/ if( !($?ENSWORK) ) then setenv ENSWORK $ensloc endif @@ -212,14 +207,20 @@ if( ($?ATMENSETC) ) then echo " ${MYNAME}: env(AENSTAT_MPIRUN) not defined, aborting ..." exit 1 endif - if(! -d ensmean ) mkdir -p $MYLOC/ensmean - if(! -d ensrms ) mkdir -p $MYLOC/ensrms - cd $MYLOC/mem001 + if(! -d ensmean ) mkdir -p $ensloc/ensmean + if( $xstat == "spread" && (! -d ensrms) ) mkdir -p $ensloc/ensrms + if( $xstat == "variance" && (! -d ensvar) ) mkdir -p $ensloc/ensvar + cd $ensloc/mem001 set alltype = `ls *.${ftype}.*${timetagz}.$NCSUFFIX` cd - foreach fn ( $alltype ) - set mopt = "-o $MYLOC/ensmean/$fn" - set sopt = "-stdv $MYLOC/ensrms/$fn" + set mopt = "-o $ensloc/ensmean/$fn" + if ( -d $ensloc/ensrms ) then + set sopt = "-stdv $ensloc/ensrms/$fn" + endif + if ( -d $ensloc/ensvar ) then + set sopt = "-variance $ensloc/ensvar/$fn" + endif set eopt = "" if ("$ftype" == "bkg.eta" || "$ftype" == "ana.eta" || "$ftype" == "prog.eta" ) then if("$ftype" == "bkg.eta" ) set etype = "bene.err" @@ -227,10 +228,6 @@ if( ($?ATMENSETC) ) then if("$ftype" == "prog.eta") set etype = "pene.err" set eopt = "-ene ensrms/$EXPID.${etype}.${timetagz}.$NCSUFFIX" endif - if (("$lmtype" == "p48") || ("$lmtype" == "z17") || ("$lmtype" == "slv")) then - if(! -d ensvar ) mkdir -p $MYLOC/ensvar - set sopt = "-variance $MYLOC/ensvar/$fn" - endif set dims = (`getgfiodim.x mem001/$fn` ) setenv MP_STATS_LM $dims[3] if ( $MP_STATS_LM == 0 ) then @@ -253,7 +250,7 @@ if( ($?ATMENSETC) ) then exit (0) endif endif -/bin/rm mp_stats.*.rc +/bin/rm $ensloc/mp_stats.*.rc setenv NCPUS 1 # NOTE: for now since there is a memory issue diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_vtrack.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_vtrack.csh index 8901adf4..46bc8a4f 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_vtrack.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_vtrack.csh @@ -108,6 +108,7 @@ if ( $ENSPARALLEL ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $ENSVTRK_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif endif diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_eaod.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_eaod.csh index ba0d1f53..c3a0e878 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_eaod.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_eaod.csh @@ -212,7 +212,7 @@ if ( "$doing_gaas" == ".TRUE." ) then # Launch acquire job to retrieve pre-existing ensemble # ---------------------------------------------------- #setenv JOBGEN_NCPUS 1 - #setenv JOBGEN_NCPUS_PER_NODE 1 + setenv JOBGEN_NCPUS_PER_NODE -1 jobgen.pl \ -expid $expid \ acq_aodana \ diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_egsi.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_egsi.csh index 9b6f09f7..84c700a5 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_egsi.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_egsi.csh @@ -9,6 +9,7 @@ # 24Nov2011 Todling Initial script # 20Jun2020 Todling Revampped (based on observer) # 23Jun2020 Todling Redef meaning of ATMENSLOC +# 03Nov2024 Todling Revised job distribution #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then @@ -125,6 +126,7 @@ if ( $ENSPARALLEL ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $ENSGSI_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif endif @@ -605,23 +607,9 @@ while ( $n < $nmem ) endif if ( ($ipoe == $AENS_OBSVR_DSTJOB) || (($fpoe == $ntodo ) && ($ipoe < $AENS_OBSVR_DSTJOB) ) ) then - set this_ntasks_per_node = `facter processorcount` - @ ncores_needed = $ENSGSI_NCPUS / $this_ntasks_per_node - if ( $ncores_needed == 0 ) then - @ myncpus = $this_ntasks_per_node - else - if ( $ENSGSI_NCPUS == $ncores_needed * $this_ntasks_per_node ) then - @ myncpus = $ENSGSI_NCPUS - else - @ myncpus = $ENSGSI_NCPUS / $this_ntasks_per_node - @ module = $myncpus * $this_ntasks_per_node - $ENSGSI_NCPUS - if ( $module != 0 ) @ myncpus = $myncpus + 1 - @ myncpus = $myncpus * $this_ntasks_per_node - endif - endif - @ myncpus = $ipoe * $myncpus - #_ @ myncpus = $ipoe * $ENSGSI_NCPUS - setenv JOBGEN_NCPUS $myncpus + set mydist = (`atmens_ntasks.pl $ENSGSI_NCPUS $ipoe`) + setenv JOBGEN_NCPUS $mydist[1] + setenv JOBGEN_NCPUS_PER_NODE $mydist[2] jobgen.pl \ -q $OBSVR_QNAME \ egsi_dst${npoe} \ diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_enkf.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_enkf.csh index 492f5b62..bd7499e6 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_enkf.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmos_enkf.csh @@ -110,6 +110,7 @@ if ( $ENSPARALLEL ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $AENKF_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif endif diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CAP.rc.tmpl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CAP.rc.tmpl index efad01f3..d5148ae3 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CAP.rc.tmpl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CAP.rc.tmpl @@ -1,6 +1,6 @@ -MAPLROOT_COMPNAME: GCS - ROOT_NAME: GCS +MAPLROOT_COMPNAME: GCM + ROOT_NAME: GCM ROOT_CF: AGCM.rc HIST_CF: HISTORY.rc diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt index fb2aecae..4a54371f 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt @@ -41,14 +41,14 @@ install ( ) set (R21C_files - R21C/atmos_enkf.nml.tmpl - R21C/obs1gsi_mean.rc - R21C/obs1gsi_member.rc - R21C/HISTAENS.rc.tmpl - R21C/post_egcm.rc - R21C/CAP.rc.tmpl - R21C/atmens_storage.arc - R21C/AtmEnsConfig.csh + R21C/AtmEnsConfig.csh + R21C/atmens_storage.arc + R21C/atmos_enkf.nml.tmpl + R21C/HISTAENS.rc.tmpl + R21C/obs1gsi_mean.rc + R21C/obs1gsi_member.rc + R21C/post_egcm_diag.rc + R21C/post_egcm.rc ) install ( diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh index dae2eef1..4d5e5fce 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh @@ -65,7 +65,7 @@ setenv AENS_DONORECENTER 0 # do not recenter ensemble setenv ADDINF_FACTOR 0.35 # additive inflation coeff (seems small after mass-div fix) setenv ADDINF_FACTOR_SPPT 0.2 # additive inflation for when SPPT is used -setenv RECENTER_WALLCLOCK 1:00:00 +setenv RECENTER_WALLCLOCK 0:30:00 setenv ENSRECENTER_NCPUS 1 setenv RECENTER_QNAME $ATMENS_QNAME setenv AENS_RECENTER_DSTJOB 16 @@ -137,7 +137,7 @@ setenv MPIRUN_ENSANA "$ATMENS_MPIRUN -np $ENSGSI_NCPUS GSIsa.x" # esma_mpir # setup_perts.csh #---------------- -setenv AENS_PERTS_DSTJOB 8 +# setenv AENS_PERTS_DSTJOB 8 setenv PERTS_QNAME $ATMENS_QNAME setenv PERTS_WALLCLOCK 1:00:00 setenv PERTS_NCPUS 24 @@ -160,5 +160,8 @@ setenv PEGCM_QNAME $ATMENS_QNAME # NOTES: setenv OBSCLASS1 "r21c_loon_bufr,r21c_gmao_mlst_bufr,r21c_prep_bufr,r21c_acftpfl_bufr,r21c_satwnd_bufr,r21c_avhrr_satwnd_bufr,r21c_ncep_tcvitals,r21c_tmi_bufr,r21c_gpsro_bufr,r21c_sevcsr_bufr,r21c_1bamua_bufr,r21c_1bamub_bufr,r21c_1bhrs2_bufr,r21c_1bhrs3_bufr,r21c_1bhrs4_bufr,r21c_1bmsu_bufr,r21c_1bmhs_bufr,r21c_1bssu_bufr,r21c_eosairs_bufr,r21c_eosamsua_bufr,r21c_mtiasi_bufr,r21c_atms_bufr,r21c_ssmit11_bufr,r21c_ssmit13_bufr,r21c_ssmit14_bufr,r21c_ssmit15_bufr,r21c_amsre_bufr,r21c_osbuv8_bufr,r21c_mls_nc,r21c_npp_ompslp_nc,r21c_aura_omieff_nc,r21c_npp_ompsnmeff_nc,r21c_avcsam_bufr,r21c_avcspm_bufr,r21c_amsr2_bufr,r21c_crisfsr_bufr,r21c_gmi_bufr,r21c_prep_bufr,r21c_acftpfl_bufr" + +# NONSENS +#setenv OBSCLASS1 "r21c_igra2_raob_pre-qc_bufr,r21c_cdas_conv_pre-qc_bufr,r21c_prof_pre-qc_bufr,r21c_avhrrwnd_pre-qc_bufr,r21c_nmodis_pre-qc_bufr,r21c_goes_pre-qc_prep_bufr,r21c_metsat_pre-qc_prep_bufr,r21c_jma_pre-qc_prep_bufr,r21c_ascat_pre-qc_bufr,r21c_repro_ers2_pre-qc_bufr,r21c_qscat_jpl_pre-qc_bufr,r21c_loon_bufr,r21c_gmao_mlst_bufr,r21c_prep_bufr,r21c_acftpfl_bufr,r21c_satwnd_bufr,r21c_avhrr_satwnd_bufr,r21c_ncep_tcvitals,r21c_tmi_bufr,r21c_gpsro_bufr,r21c_sevcsr_bufr,r21c_1bamua_bufr,r21c_1bamub_bufr,r21c_1bhrs2_bufr,r21c_1bhrs3_bufr,r21c_1bhrs4_bufr,r21c_1bmsu_bufr,r21c_1bmhs_bufr,r21c_1bssu_bufr,r21c_eosairs_bufr,r21c_eosamsua_bufr,r21c_mtiasi_bufr,r21c_atms_bufr,r21c_ssmit11_bufr,r21c_ssmit13_bufr,r21c_ssmit14_bufr,r21c_ssmit15_bufr,r21c_amsre_bufr,r21c_osbuv8_bufr,r21c_mls_nc,r21c_npp_ompslp_nc,r21c_aura_omieff_nc,r21c_npp_ompsnmeff_nc,r21c_avcsam_bufr,r21c_avcspm_bufr,r21c_amsr2_bufr,r21c_crisfsr_bufr,r21c_gmi_bufr,r21c_prep_bufr,r21c_acftpfl_bufr" setenv OBSCLASS "gmao_prep_bufr,gmao_acftpfl_bufr,$OBSCLASS1" diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/CAP.rc.tmpl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/CAP.rc.tmpl deleted file mode 100644 index d5148ae3..00000000 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/CAP.rc.tmpl +++ /dev/null @@ -1,21 +0,0 @@ - -MAPLROOT_COMPNAME: GCM - ROOT_NAME: GCM -ROOT_CF: AGCM.rc -HIST_CF: HISTORY.rc - -BEG_DATE: 17760704 210000 -END_DATE: 21130921 030000 -JOB_SGMT: 0 120000 -NUM_SGMT: 9999 -HEARTBEAT_DT: 450 - -USE_SHMEM: 0 - -#CoresPerNode: 8 - -MAPL_ENABLE_TIMERS: NO -MAPL_ENABLE_MEMUTILS: NO -PRINTSPEC: 0 # (0: OFF, 1: IMPORT & EXPORT, 2: IMPORT, 3: EXPORT) - -USE_EXTDATA2G: .TRUE. diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl index d5606d71..2ef140bf 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl @@ -58,8 +58,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' asm_inst_6hr_glo_L288x181_p48.grid_label: PC288x181-DC , asm_inst_6hr_glo_L288x181_p48.frequency: 060000 , asm_inst_6hr_glo_L288x181_p48.duration: 060000 , - asm_inst_6hr_glo_L288x181_p48.end_date: >>>IOEDATE<<< , - asm_inst_6hr_glo_L288x181_p48.end_time: >>>IOETIME<<< , + asm_inst_6hr_glo_L288x181_p48.end_date: >>>IOBBKGD<<< , + asm_inst_6hr_glo_L288x181_p48.end_time: >>>IOBBKGT<<< , asm_inst_6hr_glo_L288x181_p48.vscale: 100.0 , asm_inst_6hr_glo_L288x181_p48.vunit: 'hPa' , asm_inst_6hr_glo_L288x181_p48.vvars: 'log(PLE)' , 'DYN' , @@ -92,8 +92,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' slv_inst_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , slv_inst_6hr_glo_L288x181_slv.frequency: 060000 , slv_inst_6hr_glo_L288x181_slv.duration: 060000 , - slv_inst_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , - slv_inst_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , + slv_inst_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , + slv_inst_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , slv_inst_6hr_glo_L288x181_slv.fields: 'SLP' , 'DYN' , 'PS' , 'DYN' , 'U850;V850' , 'DYN' , @@ -152,8 +152,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' flx_tavg_6hr_glo_L288x181_slv.frequency: 060000 , flx_tavg_6hr_glo_L288x181_slv.duration: 060000 , flx_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , - flx_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , - flx_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , + flx_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , + flx_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , flx_tavg_6hr_glo_L288x181_slv.fields: 'LHFX' , 'SURFACE' , 'EFLUX' , 'EVAP' , 'SURFACE' , 'SH' , 'SURFACE' , 'HFLUX' , @@ -211,8 +211,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' rad_tavg_6hr_glo_L288x181_slv.frequency: 060000 , rad_tavg_6hr_glo_L288x181_slv.duration: 060000 , rad_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , - rad_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , - rad_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , + rad_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , + rad_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , rad_tavg_6hr_glo_L288x181_slv.fields: 'EMIS' , 'SURFACE' , 'TS' , 'SURFACE' , 'ALBEDO' , 'SOLAR' , @@ -266,8 +266,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' lnd_tavg_6hr_glo_L288x181_slv.frequency: 060000 , lnd_tavg_6hr_glo_L288x181_slv.duration: 060000 , lnd_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , - lnd_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , - lnd_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , + lnd_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , + lnd_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , lnd_tavg_6hr_glo_L288x181_slv.fields: 'GRN' , 'SURFACE' , 'LAI' , 'SURFACE' , 'WET3' , 'SURFACE' , 'GWETPROF' , @@ -332,8 +332,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' lfo_tavg_6hr_glo_L288x181_slv.frequency: 060000 , lfo_tavg_6hr_glo_L288x181_slv.duration: 060000 , lfo_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , - lfo_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , - lfo_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , + lfo_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , + lfo_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , lfo_tavg_6hr_glo_L288x181_slv.fields: 'SLRSF' , 'SOLAR' , 'SWGDN' , 'LWS' , 'IRRAD' , 'LWGAB' , 'PCU' , 'SURFACE' , 'PRECRAINCUCORR' , @@ -356,8 +356,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' glc_tavg_6hr_glo_L288x181_slv.regrid_name: 'PE90x540-CF' , glc_tavg_6hr_glo_L288x181_slv.regrid_exch: 'tile.bin' , glc_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , - glc_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , - glc_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , + glc_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , + glc_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , glc_tavg_6hr_glo_L288x181_slv.fields: 'SNOWDP_GL' , 'LANDICE' , 'SNOWDPGLC' , 'SNOMAS_GL' , 'LANDICE' , 'SNOMASGLC' , 'ASNOW_GL' , 'LANDICE' , 'ASNOWGLC' , @@ -394,8 +394,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' int_inst_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , int_inst_6hr_glo_L288x181_slv.frequency: 060000 , int_inst_6hr_glo_L288x181_slv.duration: 060000 , - int_inst_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , - int_inst_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , + int_inst_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , + int_inst_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , int_inst_6hr_glo_L288x181_slv.fields: 'TQV' , 'AGCM' , 'TQI' , 'AGCM' , 'TQL' , 'AGCM' , @@ -416,8 +416,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' int_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , int_tavg_6hr_glo_L288x181_slv.frequency: 060000 , int_tavg_6hr_glo_L288x181_slv.duration: 060000 , - int_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , - int_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , + int_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , + int_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , int_tavg_6hr_glo_L288x181_slv.fields: 'UCPT' , 'DYN' , 'UFLXCPT' , 'VCPT' , 'DYN' , 'VFLXCPT' , 'UPHI' , 'DYN' , 'UFLXPHI' , @@ -442,8 +442,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' aer_inst_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , aer_inst_6hr_glo_L288x181_slv.frequency: 060000 , aer_inst_6hr_glo_L288x181_slv.duration: 060000 , - aer_inst_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , - aer_inst_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , + aer_inst_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , + aer_inst_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , aer_inst_6hr_glo_L288x181_slv.splitField: 1 , aer_inst_6hr_glo_L288x181_slv.fields: 'TOTEXTTAU' , 'GOCART2G' , 'TOTSCATAU' , 'GOCART2G' , @@ -462,8 +462,8 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' pbl_inst_6hr_glo_L288x181_z17.grid_label: PC288x181-DC , pbl_inst_6hr_glo_L288x181_z17.frequency: 060000 , pbl_inst_6hr_glo_L288x181_z17.duration: 060000 , - pbl_inst_6hr_glo_L288x181_z17.end_date: >>>IOEDATE<<< , - pbl_inst_6hr_glo_L288x181_z17.end_time: >>>IOETIME<<< , + pbl_inst_6hr_glo_L288x181_z17.end_date: >>>IOBBKGD<<< , + pbl_inst_6hr_glo_L288x181_z17.end_time: >>>IOBBKGT<<< , pbl_inst_6hr_glo_L288x181_z17.vunit: 'm' , pbl_inst_6hr_glo_L288x181_z17.vvars: 'ZLES' , 'TURBULENCE' , pbl_inst_6hr_glo_L288x181_z17.levels: 100 200 300 400 600 800 1000 1200 1400 1600 1800 2000 2300 2600 3000 3500 4000 , diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm_diag.rc b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm_diag.rc new file mode 100644 index 00000000..86f1699f --- /dev/null +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm_diag.rc @@ -0,0 +1,14 @@ + +COLLECTIONS:: + 'asm_inst_6hr_glo_L288x181_p48' + 'slv_inst_6hr_glo_L288x181_slv' + 'flx_tavg_6hr_glo_L288x181_slv' + 'rad_tavg_6hr_glo_L288x181_slv' + 'lnd_tavg_6hr_glo_L288x181_slv' + 'lfo_tavg_6hr_glo_L288x181_slv' + 'glc_tavg_6hr_glo_L288x181_slv' + 'aer_tavg_6hr_glo_L288x181_slv' + 'int_inst_6hr_glo_L288x181_slv' + 'int_tavg_6hr_glo_L288x181_slv' + 'pbl_inst_6hr_glo_L288x181_z17' + :: diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensbkg.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensbkg.csh index ed2c534a..547e2a3e 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensbkg.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensbkg.csh @@ -11,6 +11,7 @@ # NMC-like perturbations to them # 20Oct2012 Todling Update API of stats script # 12Mar2014 Todling Update interface to main re-center program +# 02Nov2024 Todling Update API of stats script #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then @@ -74,7 +75,7 @@ if ( $#argv < 4 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 30Oct2024 by: R. Todling" + echo " Last modified: 02Nov2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -314,7 +315,7 @@ if ( $ATMENS_DOSTATS ) then set nymd = `echo $bkgeta | cut -d. -f3 | cut -c1-8` set hh = `echo $bkgeta | cut -d. -f3 | cut -c10-11` set nhms = ${hh}0000 - atmens_stats.csh $nmem bkg.eta $ensloc $nymd $nhms + atmens_stats.csh $nmem bkg.eta spread $ensloc $nymd $nhms end else echo " ${MYNAME}: by request, no statistics calculated" diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensrst.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensrst.csh index 23c0c218..2d386301 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensrst.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/gen_ensrst.csh @@ -149,7 +149,7 @@ if ( $RECENTERBKG ) then # Now re-calculate statistics for ensemble (mean and rms) # ------------------------------------------------------- - atmens_stats.csh $nmem bkg.eta $ensloc $this_nymd $this_nhms + atmens_stats.csh $nmem bkg.eta spread $ensloc $this_nymd $this_nhms end end diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/obsvr_ensfinal.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/obsvr_ensfinal.csh index 49a117a6..33c43aa1 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/obsvr_ensfinal.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/obsvr_ensfinal.csh @@ -130,6 +130,7 @@ if ( $ENSPARALLEL ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $ENSGSI_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif endif diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_eana.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_eana.csh index ff808465..a73c4055 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_eana.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_eana.csh @@ -9,6 +9,7 @@ # 05Feb2012 Todling Strip from original location # 07Mar2012 El Akkraoui Remove redundant stats calculation # 20Oct2012 Todling Update API of stats script +# 02Nov2024 Todling Update API of stats script, and of self #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then setenv ATMENS_VERBOSE 0 @@ -23,7 +24,7 @@ setenv skipTRANSF # no transform needed for ana-sensitivity exec setenv skipSOLVER # need to run the analysis sensitivity solver setenv skipSATBIAS "-skipSATBIAS" # no need to worry about running satellite bias correction -if ( $#argv < 3 ) then +if ( $#argv < 4 ) then echo " " echo " \\begin{verbatim} " echo " " @@ -33,12 +34,13 @@ if ( $#argv < 3 ) then echo " " echo " SYNOPSIS " echo " " - echo " $MYNAME expid nymd nhms " + echo " $MYNAME expid nymd nhms xstat" echo " " echo " where" echo " expid - usual experiment name, e.g., b541iau" echo " nymd - date of analysis, as in YYYYMMDD" echo " nhms - time of analysis, as in HHMMSS" + echo " xstat - extra statistic (e.g., spread or variance)" echo " " echo " DESCRIPTION" echo " " @@ -48,7 +50,7 @@ if ( $#argv < 3 ) then echo " analysis is available." echo " " echo " Example of valid command line:" - echo " $MYNAME b541iau 20091019 000000 " + echo " $MYNAME b541iau 20091019 000000 spread" echo " " echo " REQUIRED ENVIRONMENT VARIABLES" echo " " @@ -71,7 +73,7 @@ if ( $#argv < 3 ) then echo " " echo " AUTHOR" echo " Ricardo Todling (Ricardo.Todling@nasa.gov), NASA/GMAO " - echo " Last modified: 08Apr2013 by: R. Todling" + echo " Last modified: 02Nov2024 by: R. Todling" echo " \\end{verbatim} " echo " \\clearpage " exit(0) @@ -112,6 +114,8 @@ endif set expid = $1 set nymd = $2 set nhms = $3 +set xstat = $4 + set hh = `echo $nhms | cut -c1-2` set yyyymmddhh = ${nymd}${hh} @@ -134,13 +138,13 @@ touch .no_archiving # Calculate analysis ensemble mean # -------------------------------- - atmens_stats.csh $nmem ana.eta $ENSWORK/updated_ens $nymd $nhms + atmens_stats.csh $nmem ana.eta $xstat $ENSWORK/updated_ens $nymd $nhms if ( $status ) then echo " ${MYNAME}: error in calculating ensemble mean analysis, aborting ..." exit(1) endif if ( -e $ENSWORK/.FOUNDINC ) then - atmens_stats.csh $nmem inc.eta $ENSWORK/updated_ens $nymd $nhms + atmens_stats.csh $nmem inc.eta $xstat $ENSWORK/updated_ens $nymd $nhms if ( $status ) then echo " ${MYNAME}: error in calculating ensemble mean ana increment, aborting ..." exit(1) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_efso.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_efso.csh index 0492f52b..c2223002 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_efso.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_efso.csh @@ -90,6 +90,7 @@ if ( $ENSPARALLEL ) then setenv FAILED 1 else setenv JOBGEN_NCPUS $AENKF_NCPUS + setenv JOBGEN_NCPUS_PER_NODE -1 endif endif diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh index 37ed5250..1436355d 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh @@ -16,6 +16,7 @@ # job-monitor works at date/time level. # - Add option to pack jobs with slurm arrays # and possibly packable. +# 02Nov2024 Todling Revise handling of ensdiag and stats opt (API) #------------------------------------------------------------------ if ( !($?ATMENS_VERBOSE) ) then @@ -26,7 +27,7 @@ endif setenv MYNAME post_egcm.csh -if ( $#argv < 5 ) then +if ( $#argv < 7 ) then echo " " echo " \\begin{verbatim} " echo " " @@ -36,13 +37,15 @@ if ( $#argv < 5 ) then echo " " echo " SYNOPSIS" echo " " - echo " $MYNAME expid nymd nhms toffset ensloc" + echo " $MYNAME expid nymd nhms toffset xstat rc ensloc" echo " " echo " where" echo " expid - usual experiment name, e.g., b541iau" echo " nymd - initial date of forecast, as in YYYYMMDD " echo " nhms - initial time of forecast, as HHMMSS" echo " toffset - time offset to start calculating stats (min)" + echo " xstat - addition statics (beyond mean, e.g. spread/variance" + echo " rc - post_egcm resource file" echo " ensloc - location of ensemble members" echo " " echo " DESCRIPTION" @@ -65,7 +68,7 @@ if ( $#argv < 5 ) then echo " it is also possible to have a files like post_egcm_21.rc." echo " " echo " Example of valid command line:" - echo " $MYNAME b541iau 20091018 210000 360 FVWORK/updated_ens" + echo " $MYNAME b541iau 20091018 210000 360 spread post_egcm.rc FVWORK/updated_ens" echo " " echo " REQUIRED ENVIRONMENT VARIABLES" echo " " @@ -85,8 +88,6 @@ if ( $#argv < 5 ) then echo " " echo " OPTIONAL RESOURCE FILES" echo " " - echo " post_egcm.rc - user specific collection subset" - echo " " echo " SEE ALSO " echo " " echo " atmens_stats.csh - calculates required/desired statistics from ensemble" @@ -148,14 +149,16 @@ set expid = $1 set nymdb = $2 set nhmsb = $3 set toffset = $4 -set ensloc = $5 +set xstat = $5 +set egcmrc = $6 +set ensloc = $7 set hhb = `echo $nhmsb | cut -c1-2` set hhmnb = `echo $nhmsb | cut -c1-4` set yyyymmddhhmn = ${nymdb}${hhmnb} setenv ENSWORK $ensloc -if (-e $ENSWORK/.DONE_${MYNAME}.$yyyymmddhhmn ) then +if (-e $ENSWORK/.DONE_${MYNAME}_${xstat}.$yyyymmddhhmn ) then echo " ${MYNAME}: already done" exit(0) endif @@ -185,27 +188,13 @@ touch .no_archiving # Calculate mean/rms of newly generated ensemble # ---------------------------------------------- -if (! -e $ENSWORK/.DONE_redone_allstat_$MYNAME.$yyyymmddhhmn ) then +if (! -e $ENSWORK/.DONE_redone_allstat_${MYNAME}_${xstat}.$yyyymmddhhmn ) then cd $ENSWORK # determine history selections to handle # -------------------------------------- - if ( -e $ATMENSETC/post_egcm.rc || -e $ATMENSETC/post_egcm_${hhb}.rc ) then - if ( -e $ATMENSETC/post_egcm_${hhb}.rc ) then - set this_histrc = $ATMENSETC/post_egcm_${hhb}.rc - else - set this_histrc = $ATMENSETC/post_egcm.rc - endif - set alltyps = (`echorc.x -rc $this_histrc COLLECTIONS`) - else - if ( -e $ATMENSETC/HISTAENS_${hhb}.rc.tmpl ) then - set this_histrc = $ATMENSETC/HISTAENS_${hhb}.rc.tmpl - else - set this_histrc = $ATMENSETC/HISTAENS.rc.tmpl - endif - set alltyps = (`edhist.pl -q 3 -list inc -i $this_histrc`) - endif + set alltyps = (`echorc.x -rc $egcmrc COLLECTIONS`) set nn = `echo $alltyps | wc` set ntyps = $nn[2] # number of types to handle @@ -235,7 +224,7 @@ if (! -e $ENSWORK/.DONE_redone_allstat_$MYNAME.$yyyymmddhhmn ) then set mmm = `echo $m | awk '{printf "%03d", $1}'` if ( $PEGCM_SERIAL ) then - atmens_stats.csh $nmem $outkind $ENSWORK $this_nymd $this_nhms + atmens_stats.csh $nmem $outkind $xstat $ENSWORK $this_nymd $this_nhms if ($status) then echo " ${MYNAME}: trouble calculating stats for $this_nymd $this_nhms, aborting ..." exit(1) @@ -261,7 +250,7 @@ if (! -e $ENSWORK/.DONE_redone_allstat_$MYNAME.$yyyymmddhhmn ) then pegcm_${idx}.${tagA} \ $GID \ $PEGCM_WALLCLOCK \ - "atmens_stats.csh $nmem $outkind $ENSWORK $this_nymd $this_nhms |& tee -a $ENSWORK/pegcm_${outkind}.$this_yyyymmddhhmn.log"\ + "atmens_stats.csh $nmem $outkind $xstat $ENSWORK $this_nymd $this_nhms |& tee -a $ENSWORK/pegcm_${outkind}.$this_yyyymmddhhmn.log"\ $ENSWORK \ $MYNAME \ $ENSWORK/.DONE_MEM${idx}_${MYNAME}.$tagB \ @@ -329,13 +318,14 @@ if (! -e $ENSWORK/.DONE_redone_allstat_$MYNAME.$yyyymmddhhmn ) then # -------- /bin/rm $ENSWORK/pegcm_*.j /bin/rm $ENSWORK/pegcm_*.j.* + /bin/rm $ENSWORK/pegcm_*output* /bin/rm $ENSWORK/*pegcm_*.log endif # <.not.PEGCM_ALLPARALLEL> endif # <.not.SERIAL> endif # - touch $ENSWORK/.DONE_redone_allstat_$MYNAME.$yyyymmddhhmn + touch $ENSWORK/.DONE_redone_allstat_${MYNAME}_${xstat}.$yyyymmddhhmn # Increment date/time # ------------------- @@ -386,6 +376,7 @@ if (! -e $ENSWORK/.DONE_redone_allstat_$MYNAME.$yyyymmddhhmn ) then # -------- /bin/rm $ENSWORK/pegcm_*.j /bin/rm $ENSWORK/pegcm_*.j.* + /bin/rm $ENSWORK/pegcm_*output* /bin/rm $ENSWORK/*pegcm_*.log endif # @@ -393,6 +384,6 @@ endif # made it down here, all done # --------------------------- -touch $ENSWORK/.DONE_${MYNAME}.$yyyymmddhhmn +touch $ENSWORK/.DONE_${MYNAME}_${xstat}.$yyyymmddhhmn echo " ${MYNAME}: Complete " exit(0) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_atmens_stats.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_atmens_stats.j index 91e90329..e1012e29 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_atmens_stats.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/ut_atmens_stats.j @@ -4,25 +4,26 @@ setenv dry_run #echo setenv SIMULATE_ENSEMBLE 0 setenv ATMENS_VERBOSE 1 setenv DOANA 0 -setenv NEW 1 # when set to 1 will submit little job to queue; other do work on interactve queue setenv JOBGEN_QOS advda #setenv JOBGEN_PARTITION preops setenv JOBGEN_CONSTRAINT hasw # to run pegcm stats in parallel +setenv PEGCM_ALLPARALLEL 1 +setenv PEGCM_ARRAY 1 setenv PEGCM_WALLCLOCK 1:30:00 setenv PEGCM_QNAME compute -setenv EXPID f513a_rt -setenv FVHOME /gpfsm/dnb02/projects/p61/$user/$EXPID +setenv EXPID m21c_j98 +setenv FVHOME /gpfsm/dnb05/projects/p139/$user/M21C/$EXPID setenv FVROOT `cat $FVHOME/.FVROOT` setenv ENSWORK $FVHOME setenv ATMENSETC $FVHOME/run/atmens setenv TIMEINC 360 setenv ASYNBKG 180 -setenv ATMENS_BATCHSUB qsub +setenv ATMENS_BATCHSUB sbatch setenv GID g0613 setenv ENSPARALLEL 1 setenv AENSTAT_NCPUS 4 @@ -49,43 +50,23 @@ if ( $DOANA ) then set nymda = 20120401 set nhmsa = 000000 - $dry_run atmens_stats.csh $nmem ana.eta $ENSWORK/atmens $nymda $nhmsa + $dry_run atmens_stats.csh $nmem ana.eta spread $ENSWORK/atmens $nymda $nhmsa else - set nymdb = 20151211 - set nhmsb = 090000 + set nymdb = 19971231 + set nhmsb = 210000 set hhb = `echo $nhmsb | cut -c1-2` - if ( $NEW ) then - post_egcm.csh $EXPID $nymdb $nhmsb 0 $FVHOME/atmens - else - - @ bkgfreq_sec = $ASYNBKG * 60 - @ anafreq_sec = $TIMEINC * 60 - @ iaufreq_sec = 0 #$iaumn * 60 - @ nt = $anafreq_sec / $bkgfreq_sec + 1 - set adate = ( `tick $nymdb $nhmsb $iaufreq_sec` ) - @ n = 0 - while ( $n < $nt ) - @ n++ - set this_nymd = $adate[1] - set this_nhms = $adate[2] - set this_hh = `echo $this_nhms | cut -c1-2` - set this_yyyymmddhh = ${this_nymd}${this_hh} - if (! -e $ENSWORK/.DONE_redone_bkgstat_$MYNAME.$this_yyyymmddhh ) then - foreach ftype ( bkg.eta bkg.sfc cbkg.eta abkg.eta gaas_bkg.sfc ) - $dry_run atmens_stats.csh $nmem $ftype $ENSWORK/atmens $this_nymd $this_nhms - if($status) then - echo "Failed" - exit(1) - else - $dry_run touch $ENSWORK/.DONE_redone_bkgstat_$MYNAME.$this_yyyymmddhh - endif - end # foreach - endif - set adate = (`tick $this_nymd $this_nhms $bkgfreq_sec`) - end - endif # NEW + post_egcm.csh $EXPID $nymdb $nhmsb $TIMEINC spread $ATMENSETC/post_egcm.rc $FVHOME/atmens + if ($status) then + echo "post_egcm (bkg) failed" + exit(1) + endif + post_egcm.csh $EXPID $nymdb $nhmsb 0 variance $ATMENSETC/post_egcm_diag.rc $FVHOME/atmens/ensdiag + if ($status) then + echo "post_egcm (diag) failed" + exit(1) + endif endif # From 9a8a0b9fc908be7f06f1c1cc9383d4b2b5267bb7 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Mon, 4 Nov 2024 11:26:09 -0500 Subject: [PATCH 05/14] update GMAO_Shared - will tag soon --- components.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components.yaml b/components.yaml index 4efbd48f..7f8092c9 100644 --- a/components.yaml +++ b/components.yaml @@ -28,7 +28,7 @@ NCEP_Shared: GMAO_Shared: local: ./src/Shared/@GMAO_Shared remote: ../GMAO_Shared.git - tag: v1.7.2+R21C_v1.0.1 + branch: feature/rtodling/general_gmao_preqc_bfr develop: R21C GEOS_Util: From 3d2d6f773b9ca391d85aa4cb663ee8bf391ada8b Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Mon, 4 Nov 2024 13:35:42 -0500 Subject: [PATCH 06/14] add helpful input file for m21c and minor fixed in two repos --- components.yaml | 2 +- .../GEOSdas_App/testsuites/m21c_j98.input | 246 ++++++++++++++++++ 2 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 src/Applications/GEOSdas_App/testsuites/m21c_j98.input diff --git a/components.yaml b/components.yaml index 7f8092c9..a1c48037 100644 --- a/components.yaml +++ b/components.yaml @@ -52,7 +52,7 @@ FMS: GEOSana_GridComp: local: ./src/Components/@GEOSana_GridComp remote: ../GEOSana_GridComp.git - tag: v1.5.5-R21C-4 + branch: bugfix/rtodling/typo_pound_better_than_dollar develop: R21C GEOSgcm_GridComp: diff --git a/src/Applications/GEOSdas_App/testsuites/m21c_j98.input b/src/Applications/GEOSdas_App/testsuites/m21c_j98.input new file mode 100644 index 00000000..dbb22c87 --- /dev/null +++ b/src/Applications/GEOSdas_App/testsuites/m21c_j98.input @@ -0,0 +1,246 @@ +#--------------- +# m21c_j98.input +#--------------- + +codeID: 271edc7 +description: m21c_j98__agrid_C360__ogrid_CS +fvsetupID: + +---ENDHEADERS--- + +Remote account for Intranet plots? [rtodling@train] +> + +Is this a MERRA2 (1), GEOSIT (2), R21C (3) experiment? [0] +> 3 + +AGCM Horizontal Resolution? [C360] +> C360 + +AGCM Vertical Resolution? [72] +> + +OGCM Resolution? [C] +> CS + +EXPID? [u000_C360] +> $expid + +Check for previous use of expid (y/n)? [y] +> n + +EXPDSC? [m21c_j98__agrid_C360__ogrid_CS] +> + +Land Boundary Conditions? [Icarus-NLv3] +> + +Catchment Model choice? [1] +> + +FVHOME? [/discover/nobackup/rtodling/m21c_j98] +> /discover/nobackup/projects/gmao/dadev/rtodling/M21C/$expid + +The directory /discover/nobackup/projects/gmao/dadev/rtodling/M21C/m21c_j98 does not exist. Create it now? [y] +> + +Processing nodes (1:Westmere, 2:SandyBridge, 3:Ivy Bridge, 4:Haswell, 5:Skylake, 6:Cascase)? [4] +> 6 + +Which case of variational analysis? [1] +> + +Window of the variational analysis (min)? [360] +> + +FVINPUT? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar] +> + +REAL TIME BCS? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/g5gcm/bcs/realtime/OSTIA_REYNOLDS] +> + +agcmpert? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/agcmpert] +> + +chemistry? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/chemistry] +> + +g5chem? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/g5chem] +> + +g5gcm? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/g5gcm] +> + +PIESA? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/PIESA] +> + +MERRA2? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/MERRA2] +> + +AeroCom? [/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/AeroCom] +> + +FVICS? [/archive/u/jstassi/restarts/GEOSadas-5_24_0] +> /home/dao_ops/e5303_m21c_jan98/run/.../archive/rs/Y1997/M12/e5303_m21c_jan98.rst.19971230_21z.tar + +Run model-adjoint-related applications (0=no,1=yes)? [0] +> 0 + +Run analysis-sensitivity applications (0=no,1=yes)? [0] +> + +Ending year-month-day? [20190119] +> 19980206 + +Length of FORECAST run segments (in hours)? [123] +> + +Number of one-day DAS segments per PBS job? [1] +> + +Number of PEs in the zonal direction (NX)? [8] +> + +Number of PEs in the meridional direction (NY)? [48] +> + +Job nickname? [g5das] +> m21cA + +Run in split executable mode (1=yes;0=no)? [1] +> + +Frequency of background fields (min)? [60] +> + +Triangular spectral truncation? [254] +> + +Analysis vertical levels (sig))? [72] +> + +GSI grid resolution? [NA] +> + +GEOS grid resolution instead? [d] +> + +Ensemble Hybrid (<0;3=3dHyb;4=Hyb4d)? [4] +> + +Number of procs in the zonal direction (NX)? [12] +> + +Number of procs in the meridional direction (NY)? [20] +> + +Which main class of ObsSys (1: NRT; 2: MERRA; 3: MERRA-2; 4: GEOS-IT; 5: R21C)? [5] +> + +OBSERVING SYSTEM CLASSES? +> r21c_igra2_raob_pre-qc_bufr,r21c_cdas_conv_pre-qc_bufr,r21c_prof_pre-qc_bufr,r21c_avhrrwnd_pre-qc_bufr,r21c_nmodis_pre-qc_bufr,r21c_goes_pre-qc_prep_bufr,r21c_metsat_pre-qc_prep_bufr,r21c_jma_pre-qc_prep_bufr,r21c_ascat_pre-qc_bufr,r21c_repro_ers2_pre-qc_bufr,r21c_qscat_jpl_pre-qc_bufr,r21c_loon_bufr,r21c_gmao_mlst_bufr,r21c_prep_bufr,r21c_acftpfl_bufr,r21c_satwnd_bufr,r21c_avhrr_satwnd_bufr,r21c_ncep_tcvitals,r21c_tmi_bufr,r21c_gpsro_bufr,r21c_sevcsr_bufr,r21c_1bamua_bufr,r21c_1bamub_bufr,r21c_1bhrs2_bufr,r21c_1bhrs3_bufr,r21c_1bhrs4_bufr,r21c_1bmsu_bufr,r21c_1bmhs_bufr,r21c_1bssu_bufr,r21c_eosairs_bufr,r21c_eosamsua_bufr,r21c_mtiasi_bufr,r21c_atms_bufr,r21c_ssmit11_bufr,r21c_ssmit13_bufr,r21c_ssmit14_bufr,r21c_ssmit15_bufr,r21c_amsre_bufr,r21c_osbuv8_bufr,r21c_mls_nc,r21c_npp_ompslp_nc,r21c_aura_omieff_nc,r21c_npp_ompsnmeff_nc,r21c_avcsam_bufr,r21c_avcspm_bufr,r21c_amsr2_bufr,r21c_crisfsr_bufr,r21c_gmi_bufr,r21c_prep_bufr,r21c_acftpfl_bufr + +CHECKING OBSYSTEM? [2] +> 1 + +Which RADCOR option? [HAIMB_HRAD] +> + +Use sat channel-correlated observation errors (y/n)? [y] +> + +Use aircraft bias correction (y/n)? [y] +> + +Use unified radiance bias correction (y/n)? [y] +> + +Land DAS Analysis (y/n)? [n] +> + +Frequency (in days) for writing restarts? [0] +> + +Frequency for PROGNOSTIC fields? [010000] +> + +Frequency for surface (2D) DIAGNOSTIC fields? [010000] +> + +Frequency for upper air (3D) DIAGNOSTIC fields? [030000] +> + +Dimension of output in zonal direction? [1152] +> + +Dimension of output in meridional direction? [721] +> + +Would you like 2D diagnostics? [y] +> + +Would you like 3D diagnostics? [y] +> + +Would you like to compress diagnostics output files? [n] +> + +Include GOCART tracers (CO,CO2,aerosols,etc) (y/n)? [y] +> + +Select GOCART Emission Files to use: [AMIP] +> + +Do Aerosol Analysis (y/n)? [y] +> + +AOD OBSERVING CLASSES [or type 'none']? +> patmosx_ods,mod04_061_his,myd04_061_his,aeronet_obs + +Enable GAAS feedback to model (y/n)? [y] +> + +Which template? [HISTORY_R21C.rc.tmpl] +> + +Which template? [GCMPROG.rc.tmpl] +> + +Output Restart TYPE (bin or nc4) [nc4] +> + +Select group: [g2538] +> g0613 + +Replayed Ensemble? [no] +> + +Use SPPT-scheme for Ensemble? [yes] +> + +Use Precip Correction for Ensemble? [yes] +> + +Ensemble Resolution? [C90] +> + +Ensemble Vertical Levels? [72] +> + +Experiment archive directory for ensemble restarts or 'later': [/archive/u/rtodling/m21c_j98] +> /gpfsm/dnb08/projects/p253/archive/e5303_m21c_jan98 + +Continue without missing resource files? [y] +> + +Edit COLLECTIONS list in run/HISTORY.rc.tmpl (y/n)? [n] +> + +Edit COLLECTIONS list in fcst/HISTORY.rc.tmpl (y/n)? [n] +> + +Edit files in run directory for CERES configuration (y/n)? [n] +> + +Which? [Q] +> From ff5c2796f7b293042d32810e58c8cef2ef674383 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Mon, 4 Nov 2024 15:43:51 -0500 Subject: [PATCH 07/14] tagged two repos w/ add minor changes --- components.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components.yaml b/components.yaml index a1c48037..484660e8 100644 --- a/components.yaml +++ b/components.yaml @@ -28,7 +28,7 @@ NCEP_Shared: GMAO_Shared: local: ./src/Shared/@GMAO_Shared remote: ../GMAO_Shared.git - branch: feature/rtodling/general_gmao_preqc_bfr + tag: v1.7.2+R21C_v1.0.2 develop: R21C GEOS_Util: @@ -52,7 +52,7 @@ FMS: GEOSana_GridComp: local: ./src/Components/@GEOSana_GridComp remote: ../GEOSana_GridComp.git - branch: bugfix/rtodling/typo_pound_better_than_dollar + tag: v1.5.5-R21C-5 develop: R21C GEOSgcm_GridComp: From 27145203ae006d13d875b4e854d60c8b8e58fddc Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Mon, 4 Nov 2024 18:11:15 -0500 Subject: [PATCH 08/14] minor fixes to the ensdiag, see PR --- .../NCEP_enkf/scripts/gmao/atmens_recenter.csh | 2 +- .../NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh | 2 +- .../NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh | 4 +--- .../NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl | 1 + .../NCEP_enkf/scripts/gmao/etc/R21C/post_egcm.rc | 11 ----------- .../scripts/gmao/etc/R21C/post_egcm_diag.rc | 2 +- .../NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh | 12 ++++++------ .../NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl | 6 ++---- 8 files changed, 13 insertions(+), 27 deletions(-) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_recenter.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_recenter.csh index 448675ad..e50d794f 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_recenter.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_recenter.csh @@ -470,7 +470,7 @@ while ( $ic < $nmem + 1 ) @ ic = $ic + 1 end /bin/rm $ENSWORK/recenter_poe.* -/bin/rm $ENSWORK/*recenter_array*output* +/bin/rm $ENSWORK/*recenter_array* #/bin/rm $ENSWORK/recenter_poe*.j if ($failed) then diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh index ab528ee8..c9722beb 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atmens_stats.csh @@ -238,6 +238,7 @@ if( ($?ATMENSETC) ) then $dry_run $AENSTAT_MPIRUN -rc mp_stats.${ftype}.${timetagz}.rc $mopt $sopt $eopt -inc ${bkgfreq_hhmn}00 \ -egress .MP_STATS_EGRESS_${ftype}_${timetagz} mem*/$fn endif + if ( -e .MP_STATS_EGRESS_${ftype}_${timetagz} ) /bin/rm mp_stats.${ftype}.${timetagz}.rc end # make sure all is successfully done foreach fn ( $alltype ) @@ -250,7 +251,6 @@ if( ($?ATMENSETC) ) then exit (0) endif endif -/bin/rm $ensloc/mp_stats.*.rc setenv NCPUS 1 # NOTE: for now since there is a memory issue diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh index 4d5e5fce..24ae3b07 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/AtmEnsConfig.csh @@ -137,7 +137,7 @@ setenv MPIRUN_ENSANA "$ATMENS_MPIRUN -np $ENSGSI_NCPUS GSIsa.x" # esma_mpir # setup_perts.csh #---------------- -# setenv AENS_PERTS_DSTJOB 8 +#setenv AENS_PERTS_DSTJOB 8 setenv PERTS_QNAME $ATMENS_QNAME setenv PERTS_WALLCLOCK 1:00:00 setenv PERTS_NCPUS 24 @@ -161,7 +161,5 @@ setenv PEGCM_QNAME $ATMENS_QNAME # NOTES: setenv OBSCLASS1 "r21c_loon_bufr,r21c_gmao_mlst_bufr,r21c_prep_bufr,r21c_acftpfl_bufr,r21c_satwnd_bufr,r21c_avhrr_satwnd_bufr,r21c_ncep_tcvitals,r21c_tmi_bufr,r21c_gpsro_bufr,r21c_sevcsr_bufr,r21c_1bamua_bufr,r21c_1bamub_bufr,r21c_1bhrs2_bufr,r21c_1bhrs3_bufr,r21c_1bhrs4_bufr,r21c_1bmsu_bufr,r21c_1bmhs_bufr,r21c_1bssu_bufr,r21c_eosairs_bufr,r21c_eosamsua_bufr,r21c_mtiasi_bufr,r21c_atms_bufr,r21c_ssmit11_bufr,r21c_ssmit13_bufr,r21c_ssmit14_bufr,r21c_ssmit15_bufr,r21c_amsre_bufr,r21c_osbuv8_bufr,r21c_mls_nc,r21c_npp_ompslp_nc,r21c_aura_omieff_nc,r21c_npp_ompsnmeff_nc,r21c_avcsam_bufr,r21c_avcspm_bufr,r21c_amsr2_bufr,r21c_crisfsr_bufr,r21c_gmi_bufr,r21c_prep_bufr,r21c_acftpfl_bufr" -# NONSENS -#setenv OBSCLASS1 "r21c_igra2_raob_pre-qc_bufr,r21c_cdas_conv_pre-qc_bufr,r21c_prof_pre-qc_bufr,r21c_avhrrwnd_pre-qc_bufr,r21c_nmodis_pre-qc_bufr,r21c_goes_pre-qc_prep_bufr,r21c_metsat_pre-qc_prep_bufr,r21c_jma_pre-qc_prep_bufr,r21c_ascat_pre-qc_bufr,r21c_repro_ers2_pre-qc_bufr,r21c_qscat_jpl_pre-qc_bufr,r21c_loon_bufr,r21c_gmao_mlst_bufr,r21c_prep_bufr,r21c_acftpfl_bufr,r21c_satwnd_bufr,r21c_avhrr_satwnd_bufr,r21c_ncep_tcvitals,r21c_tmi_bufr,r21c_gpsro_bufr,r21c_sevcsr_bufr,r21c_1bamua_bufr,r21c_1bamub_bufr,r21c_1bhrs2_bufr,r21c_1bhrs3_bufr,r21c_1bhrs4_bufr,r21c_1bmsu_bufr,r21c_1bmhs_bufr,r21c_1bssu_bufr,r21c_eosairs_bufr,r21c_eosamsua_bufr,r21c_mtiasi_bufr,r21c_atms_bufr,r21c_ssmit11_bufr,r21c_ssmit13_bufr,r21c_ssmit14_bufr,r21c_ssmit15_bufr,r21c_amsre_bufr,r21c_osbuv8_bufr,r21c_mls_nc,r21c_npp_ompslp_nc,r21c_aura_omieff_nc,r21c_npp_ompsnmeff_nc,r21c_avcsam_bufr,r21c_avcspm_bufr,r21c_amsr2_bufr,r21c_crisfsr_bufr,r21c_gmi_bufr,r21c_prep_bufr,r21c_acftpfl_bufr" setenv OBSCLASS "gmao_prep_bufr,gmao_acftpfl_bufr,$OBSCLASS1" diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl index 2ef140bf..de39e025 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl @@ -416,6 +416,7 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' int_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , int_tavg_6hr_glo_L288x181_slv.frequency: 060000 , int_tavg_6hr_glo_L288x181_slv.duration: 060000 , + int_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , int_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , int_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , int_tavg_6hr_glo_L288x181_slv.fields: 'UCPT' , 'DYN' , 'UFLXCPT' , diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm.rc b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm.rc index 00f68e0b..545dea01 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm.rc +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm.rc @@ -5,15 +5,4 @@ COLLECTIONS:: 'cbkg.eta' 'abkg.eta' 'gaas_bkg.sfc' - 'asm_inst_6hr_glo_L288x181_p48' - 'slv_inst_6hr_glo_L288x181_slv' - 'flx_tavg_6hr_glo_L288x181_slv' - 'rad_tavg_6hr_glo_L288x181_slv' - 'lnd_tavg_6hr_glo_L288x181_slv' - 'lfo_tavg_6hr_glo_L288x181_slv' - 'glc_tavg_6hr_glo_L288x181_slv' - 'aer_tavg_6hr_glo_L288x181_slv' - 'int_inst_6hr_glo_L288x181_slv' - 'int_tavg_6hr_glo_L288x181_slv' - 'pbl_inst_6hr_glo_L288x181_z17' :: diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm_diag.rc b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm_diag.rc index 86f1699f..423bb778 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm_diag.rc +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/post_egcm_diag.rc @@ -7,7 +7,7 @@ COLLECTIONS:: 'lnd_tavg_6hr_glo_L288x181_slv' 'lfo_tavg_6hr_glo_L288x181_slv' 'glc_tavg_6hr_glo_L288x181_slv' - 'aer_tavg_6hr_glo_L288x181_slv' + 'aer_inst_6hr_glo_L288x181_slv' 'int_inst_6hr_glo_L288x181_slv' 'int_tavg_6hr_glo_L288x181_slv' 'pbl_inst_6hr_glo_L288x181_z17' diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh index 1436355d..059e07a0 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/post_egcm.csh @@ -316,9 +316,9 @@ if (! -e $ENSWORK/.DONE_redone_allstat_${MYNAME}_${xstat}.$yyyymmddhhmn ) then # clean up # -------- - /bin/rm $ENSWORK/pegcm_*.j - /bin/rm $ENSWORK/pegcm_*.j.* - /bin/rm $ENSWORK/pegcm_*output* + /bin/rm $ENSWORK/*pegcm_*.j + /bin/rm $ENSWORK/*pegcm_*.j.* + /bin/rm $ENSWORK/*pegcm_*output* /bin/rm $ENSWORK/*pegcm_*.log endif # <.not.PEGCM_ALLPARALLEL> @@ -374,9 +374,9 @@ if (! -e $ENSWORK/.DONE_redone_allstat_${MYNAME}_${xstat}.$yyyymmddhhmn ) then # clean up # -------- - /bin/rm $ENSWORK/pegcm_*.j - /bin/rm $ENSWORK/pegcm_*.j.* - /bin/rm $ENSWORK/pegcm_*output* + /bin/rm $ENSWORK/*pegcm_*.j + /bin/rm $ENSWORK/*pegcm_*.j.* + /bin/rm $ENSWORK/*pegcm_*output* /bin/rm $ENSWORK/*pegcm_*.log endif # diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl index 9ae07475..1abb375e 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl @@ -375,12 +375,10 @@ sub init { obs1gsi_mean.rc obs1gsi_member.rc HISTAENS.rc.tmpl - mp_stats_NP.rc - mp_stats_NZ.rc post_egcm.rc + post_egcm_diag.rc AtmEnsConfig.csh - atmens_storage.arc - CAP.rc.tmpl); + atmens_storage.arc); # location where ensemble RC files reside $AENSHOME = "$FVHOME/run/atmens"; From a5bda7604c70d8e4096539a682732c70ceb1b3cc Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Mon, 4 Nov 2024 18:42:22 -0500 Subject: [PATCH 09/14] no more diffs in mp_stats for prog/diag --- .../NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl index 1abb375e..d5970953 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl @@ -723,12 +723,8 @@ sub ed_stat_rc { my($acq); -# $tmprc = "$mydir/tmp.rc"; -# $thisrc = "$mydir/mp_stats.rc"; if ( $opt_r21c ) { - @estat_files = qw ( mp_stats.rc - mp_stats_NP.rc - mp_stats_NZ.rc); + @estat_files = qw ( mp_stats.rc); } else { @estat_files = qw ( mp_stats.rc); } From f004018c4ce72ef170deb77ab6f9b2f0a00bd7d8 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Tue, 5 Nov 2024 08:17:27 -0500 Subject: [PATCH 10/14] minor --- src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j index 7de642d3..8990d814 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j @@ -5,6 +5,7 @@ #SBATCH --ntasks=96 #SBATCH --ntasks-per-node=>>>NCPUS_PER_NODE<<< #SBATCH --time=6:00:00 +#SBATCH --no-requeue # #SBATCH --job-name=atm_ens #SBATCH --output=atm_ens.log.o%j @@ -66,7 +67,11 @@ if( `uname -m` != "ia64" ) then setenv FORT90L -Wl,-T endif +# NOTE: if user has access to TSE: +# (i) comment out line below +# (ii) uncomment and adjust following line setenv FVWORK $FVHOME/../enswork.$BIGNAME +# setenv FVWORK /discover/nobackup/projects/gmao/dadev/TSE_staging/$user/enswork.$BIGNAME if ($?kidwork) then # this case, overwrite FVWORK with user-specific setenv FVWORK $FVHOME/../$kidwork endif From eed85f6ed73e1bfbc1355cb04a72aef6af81e318 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Tue, 5 Nov 2024 12:04:46 -0500 Subject: [PATCH 11/14] minor - add diag post for regular DAS --- .../NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt | 3 ++- .../NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm.rc | 5 +---- .../NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm_diag.rc | 6 ++++++ .../NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl | 3 ++- 4 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm_diag.rc diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt index 4a54371f..1efc2b0c 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/CMakeLists.txt @@ -28,6 +28,7 @@ set (ALLETC mp_stats.rc nmcperts.rc post_egcm.rc + post_egcm_diag.rc odsstats_ktonly.rc obs_aodmean.rc obs_aodmember.rc @@ -47,8 +48,8 @@ set (R21C_files R21C/HISTAENS.rc.tmpl R21C/obs1gsi_mean.rc R21C/obs1gsi_member.rc - R21C/post_egcm_diag.rc R21C/post_egcm.rc + R21C/post_egcm_diag.rc ) install ( diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm.rc b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm.rc index cb8fa457..1d0f4d05 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm.rc +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm.rc @@ -3,10 +3,7 @@ COLLECTIONS:: 'bkg.eta' 'bkg.sfc' 'cbkg.eta' - 'abkg.eta' +# 'abkg.eta' 'gaas_bkg.sfc' # 'prog.eta' -# 'tavg1_2d_slv_Nx' -# 'inst1_2d_asm_Nx' - 'inst1_2d_ocn_Nx' :: diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm_diag.rc b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm_diag.rc new file mode 100644 index 00000000..fa2c64e8 --- /dev/null +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/post_egcm_diag.rc @@ -0,0 +1,6 @@ + +COLLECTIONS:: +# 'tavg1_2d_slv_Nx' +# 'inst1_2d_asm_Nx' + 'inst1_2d_ocn_Nx' + :: diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl index d5970953..49a03d40 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/setup_atmens.pl @@ -341,7 +341,8 @@ sub init { mp_stats_perts.rc nmcperts.rc odsstats_ktonly.rc - post_egcm.rc ); + post_egcm.rc + post_egcm_diag.rc ); @osercs = qw ( AGCM.rc.tmpl AtmOSEConfig.csh From 4de8683cc2742dc49a99d12d210280ce97a5ea5f Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Tue, 5 Nov 2024 16:10:38 -0500 Subject: [PATCH 12/14] ens diag back to processing predictor part of gcm integration --- .../NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j | 2 +- .../scripts/gmao/etc/R21C/HISTAENS.rc.tmpl | 72 +++++++++++-------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j index 8990d814..9c3a34b7 100755 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/atm_ens.j @@ -610,7 +610,7 @@ set myrc = $ATMENSETC/post_egcm_diag.rc if ( -e $ATMENSETC/post_egcm_diag_${hhb}.rc ) set myrc = $ATMENSETC/post_egcm_diag_${hhb}.rc if ( -e $myrc ) then - post_egcm.csh $EXPID $nymdb $nhmsb 0 variance $myrc $FVHOME/atmens/ensdiag + post_egcm.csh $EXPID $nymdb $nhmsb $TIMEINC variance $myrc $FVHOME/atmens/ensdiag if ($status) then echo "post_egcm (diag) failed" exit(1) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl index de39e025..5d756e44 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl @@ -58,8 +58,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' asm_inst_6hr_glo_L288x181_p48.grid_label: PC288x181-DC , asm_inst_6hr_glo_L288x181_p48.frequency: 060000 , asm_inst_6hr_glo_L288x181_p48.duration: 060000 , - asm_inst_6hr_glo_L288x181_p48.end_date: >>>IOBBKGD<<< , - asm_inst_6hr_glo_L288x181_p48.end_time: >>>IOBBKGT<<< , + asm_inst_6hr_glo_L288x181_p48.ref_date: >>>IOBBKGD<<< , + asm_inst_6hr_glo_L288x181_p48.ref_time: >>>IOBBKGT<<< , + asm_inst_6hr_glo_L288x181_p48.end_date: >>>IOEDATE<<< , + asm_inst_6hr_glo_L288x181_p48.end_time: >>>IOETIME<<< , asm_inst_6hr_glo_L288x181_p48.vscale: 100.0 , asm_inst_6hr_glo_L288x181_p48.vunit: 'hPa' , asm_inst_6hr_glo_L288x181_p48.vvars: 'log(PLE)' , 'DYN' , @@ -92,8 +94,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' slv_inst_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , slv_inst_6hr_glo_L288x181_slv.frequency: 060000 , slv_inst_6hr_glo_L288x181_slv.duration: 060000 , - slv_inst_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , - slv_inst_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , + slv_inst_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , + slv_inst_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , + slv_inst_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , + slv_inst_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , slv_inst_6hr_glo_L288x181_slv.fields: 'SLP' , 'DYN' , 'PS' , 'DYN' , 'U850;V850' , 'DYN' , @@ -151,9 +155,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' flx_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , flx_tavg_6hr_glo_L288x181_slv.frequency: 060000 , flx_tavg_6hr_glo_L288x181_slv.duration: 060000 , - flx_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , - flx_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , - flx_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , + flx_tavg_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , + flx_tavg_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , + flx_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , + flx_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , flx_tavg_6hr_glo_L288x181_slv.fields: 'LHFX' , 'SURFACE' , 'EFLUX' , 'EVAP' , 'SURFACE' , 'SH' , 'SURFACE' , 'HFLUX' , @@ -210,9 +215,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' rad_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , rad_tavg_6hr_glo_L288x181_slv.frequency: 060000 , rad_tavg_6hr_glo_L288x181_slv.duration: 060000 , - rad_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , - rad_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , - rad_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , + rad_tavg_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , + rad_tavg_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , + rad_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , + rad_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , rad_tavg_6hr_glo_L288x181_slv.fields: 'EMIS' , 'SURFACE' , 'TS' , 'SURFACE' , 'ALBEDO' , 'SOLAR' , @@ -265,9 +271,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' lnd_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , lnd_tavg_6hr_glo_L288x181_slv.frequency: 060000 , lnd_tavg_6hr_glo_L288x181_slv.duration: 060000 , - lnd_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , - lnd_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , - lnd_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , + lnd_tavg_6hr_glo_L288x181_slv.ref_date: >>IOBBKGD<<< , + lnd_tavg_6hr_glo_L288x181_slv.ref_time: >>IOBBKGT<<< , + lnd_tavg_6hr_glo_L288x181_slv.end_date: >>IOEDATE<<< , + lnd_tavg_6hr_glo_L288x181_slv.end_time: >>IOETIME<<< , lnd_tavg_6hr_glo_L288x181_slv.fields: 'GRN' , 'SURFACE' , 'LAI' , 'SURFACE' , 'WET3' , 'SURFACE' , 'GWETPROF' , @@ -331,9 +338,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' lfo_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , lfo_tavg_6hr_glo_L288x181_slv.frequency: 060000 , lfo_tavg_6hr_glo_L288x181_slv.duration: 060000 , - lfo_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , - lfo_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , - lfo_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , + lfo_tavg_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , + lfo_tavg_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , + lfo_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , + lfo_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , lfo_tavg_6hr_glo_L288x181_slv.fields: 'SLRSF' , 'SOLAR' , 'SWGDN' , 'LWS' , 'IRRAD' , 'LWGAB' , 'PCU' , 'SURFACE' , 'PRECRAINCUCORR' , @@ -352,12 +360,13 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' glc_tavg_6hr_glo_L288x181_slv.mode: 'time-averaged' , glc_tavg_6hr_glo_L288x181_slv.frequency: 060000 , glc_tavg_6hr_glo_L288x181_slv.duration: 060000 , - glc_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , glc_tavg_6hr_glo_L288x181_slv.regrid_name: 'PE90x540-CF' , glc_tavg_6hr_glo_L288x181_slv.regrid_exch: 'tile.bin' , glc_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , - glc_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , - glc_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , + glc_tavg_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , + glc_tavg_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , + glc_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , + glc_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , glc_tavg_6hr_glo_L288x181_slv.fields: 'SNOWDP_GL' , 'LANDICE' , 'SNOWDPGLC' , 'SNOMAS_GL' , 'LANDICE' , 'SNOMASGLC' , 'ASNOW_GL' , 'LANDICE' , 'ASNOWGLC' , @@ -394,8 +403,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' int_inst_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , int_inst_6hr_glo_L288x181_slv.frequency: 060000 , int_inst_6hr_glo_L288x181_slv.duration: 060000 , - int_inst_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , - int_inst_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , + int_inst_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , + int_inst_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , + int_inst_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , + int_inst_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , int_inst_6hr_glo_L288x181_slv.fields: 'TQV' , 'AGCM' , 'TQI' , 'AGCM' , 'TQL' , 'AGCM' , @@ -416,9 +427,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' int_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , int_tavg_6hr_glo_L288x181_slv.frequency: 060000 , int_tavg_6hr_glo_L288x181_slv.duration: 060000 , - int_tavg_6hr_glo_L288x181_slv.ref_time: 030000 , - int_tavg_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , - int_tavg_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , + int_tavg_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , + int_tavg_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , + int_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , + int_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , int_tavg_6hr_glo_L288x181_slv.fields: 'UCPT' , 'DYN' , 'UFLXCPT' , 'VCPT' , 'DYN' , 'VFLXCPT' , 'UPHI' , 'DYN' , 'UFLXPHI' , @@ -443,8 +455,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' aer_inst_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , aer_inst_6hr_glo_L288x181_slv.frequency: 060000 , aer_inst_6hr_glo_L288x181_slv.duration: 060000 , - aer_inst_6hr_glo_L288x181_slv.end_date: >>>IOBBKGD<<< , - aer_inst_6hr_glo_L288x181_slv.end_time: >>>IOBBKGT<<< , + aer_inst_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , + aer_inst_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , + aer_inst_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , + aer_inst_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , aer_inst_6hr_glo_L288x181_slv.splitField: 1 , aer_inst_6hr_glo_L288x181_slv.fields: 'TOTEXTTAU' , 'GOCART2G' , 'TOTSCATAU' , 'GOCART2G' , @@ -463,8 +477,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' pbl_inst_6hr_glo_L288x181_z17.grid_label: PC288x181-DC , pbl_inst_6hr_glo_L288x181_z17.frequency: 060000 , pbl_inst_6hr_glo_L288x181_z17.duration: 060000 , - pbl_inst_6hr_glo_L288x181_z17.end_date: >>>IOBBKGD<<< , - pbl_inst_6hr_glo_L288x181_z17.end_time: >>>IOBBKGT<<< , + pbl_inst_6hr_glo_L288x181_z17.ref_date: >>>IOBBKGD<<< , + pbl_inst_6hr_glo_L288x181_z17.ref_time: >>>IOBBKGT<<< , + pbl_inst_6hr_glo_L288x181_z17.end_date: >>>IOEDATE<<< , + pbl_inst_6hr_glo_L288x181_z17.end_time: >>>IOETIME<<< , pbl_inst_6hr_glo_L288x181_z17.vunit: 'm' , pbl_inst_6hr_glo_L288x181_z17.vvars: 'ZLES' , 'TURBULENCE' , pbl_inst_6hr_glo_L288x181_z17.levels: 100 200 300 400 600 800 1000 1200 1400 1600 1800 2000 2300 2600 3000 3500 4000 , From 5a7f62d9c970b49556dca8d4df8a9a0db6ed3668 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Thu, 7 Nov 2024 08:39:40 -0500 Subject: [PATCH 13/14] all work except lnd_tavg which comes out 3 hr off time! --- .../NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl index 5d756e44..8c21ced7 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl @@ -58,8 +58,6 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' asm_inst_6hr_glo_L288x181_p48.grid_label: PC288x181-DC , asm_inst_6hr_glo_L288x181_p48.frequency: 060000 , asm_inst_6hr_glo_L288x181_p48.duration: 060000 , - asm_inst_6hr_glo_L288x181_p48.ref_date: >>>IOBBKGD<<< , - asm_inst_6hr_glo_L288x181_p48.ref_time: >>>IOBBKGT<<< , asm_inst_6hr_glo_L288x181_p48.end_date: >>>IOEDATE<<< , asm_inst_6hr_glo_L288x181_p48.end_time: >>>IOETIME<<< , asm_inst_6hr_glo_L288x181_p48.vscale: 100.0 , @@ -94,8 +92,6 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' slv_inst_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , slv_inst_6hr_glo_L288x181_slv.frequency: 060000 , slv_inst_6hr_glo_L288x181_slv.duration: 060000 , - slv_inst_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , - slv_inst_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , slv_inst_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , slv_inst_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , slv_inst_6hr_glo_L288x181_slv.fields: 'SLP' , 'DYN' , @@ -403,8 +399,6 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' int_inst_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , int_inst_6hr_glo_L288x181_slv.frequency: 060000 , int_inst_6hr_glo_L288x181_slv.duration: 060000 , - int_inst_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , - int_inst_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , int_inst_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , int_inst_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , int_inst_6hr_glo_L288x181_slv.fields: 'TQV' , 'AGCM' , @@ -455,8 +449,6 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' aer_inst_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , aer_inst_6hr_glo_L288x181_slv.frequency: 060000 , aer_inst_6hr_glo_L288x181_slv.duration: 060000 , - aer_inst_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , - aer_inst_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , aer_inst_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , aer_inst_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , aer_inst_6hr_glo_L288x181_slv.splitField: 1 , @@ -477,8 +469,6 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' pbl_inst_6hr_glo_L288x181_z17.grid_label: PC288x181-DC , pbl_inst_6hr_glo_L288x181_z17.frequency: 060000 , pbl_inst_6hr_glo_L288x181_z17.duration: 060000 , - pbl_inst_6hr_glo_L288x181_z17.ref_date: >>>IOBBKGD<<< , - pbl_inst_6hr_glo_L288x181_z17.ref_time: >>>IOBBKGT<<< , pbl_inst_6hr_glo_L288x181_z17.end_date: >>>IOEDATE<<< , pbl_inst_6hr_glo_L288x181_z17.end_time: >>>IOETIME<<< , pbl_inst_6hr_glo_L288x181_z17.vunit: 'm' , From 62f73af37fbfe69c4675d32e2d7f31a72d327444 Mon Sep 17 00:00:00 2001 From: Ricardo Todling Date: Thu, 7 Nov 2024 08:45:47 -0500 Subject: [PATCH 14/14] found issue w/ lnd_tavg output - minor typo --- .../NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl index 8c21ced7..8984078e 100644 --- a/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl +++ b/src/Applications/NCEP_Etc/NCEP_enkf/scripts/gmao/etc/R21C/HISTAENS.rc.tmpl @@ -267,10 +267,10 @@ COLLECTIONS: 'asm_inst_6hr_glo_L288x181_p48' lnd_tavg_6hr_glo_L288x181_slv.grid_label: PC288x181-DC , lnd_tavg_6hr_glo_L288x181_slv.frequency: 060000 , lnd_tavg_6hr_glo_L288x181_slv.duration: 060000 , - lnd_tavg_6hr_glo_L288x181_slv.ref_date: >>IOBBKGD<<< , - lnd_tavg_6hr_glo_L288x181_slv.ref_time: >>IOBBKGT<<< , - lnd_tavg_6hr_glo_L288x181_slv.end_date: >>IOEDATE<<< , - lnd_tavg_6hr_glo_L288x181_slv.end_time: >>IOETIME<<< , + lnd_tavg_6hr_glo_L288x181_slv.ref_date: >>>IOBBKGD<<< , + lnd_tavg_6hr_glo_L288x181_slv.ref_time: >>>IOBBKGT<<< , + lnd_tavg_6hr_glo_L288x181_slv.end_date: >>>IOEDATE<<< , + lnd_tavg_6hr_glo_L288x181_slv.end_time: >>>IOETIME<<< , lnd_tavg_6hr_glo_L288x181_slv.fields: 'GRN' , 'SURFACE' , 'LAI' , 'SURFACE' , 'WET3' , 'SURFACE' , 'GWETPROF' ,