From 0b0a799167aaf54b2cd1ba42ee8380d6ea7d9571 Mon Sep 17 00:00:00 2001 From: David Huber Date: Sun, 14 Jan 2024 11:11:52 -0600 Subject: [PATCH 1/3] Apply fixes to archiving job. #2076 --- jobs/JGLOBAL_ARCHIVE | 2 +- ush/hpssarch_gen.sh | 34 +++++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/jobs/JGLOBAL_ARCHIVE b/jobs/JGLOBAL_ARCHIVE index e6c016e703..66f6dfa8dc 100755 --- a/jobs/JGLOBAL_ARCHIVE +++ b/jobs/JGLOBAL_ARCHIVE @@ -19,7 +19,7 @@ YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM_ATMO COM_OCEAN_HISTORY COM_OCEAN_INPUT COM_OCEAN_RESTART COM_OCEAN_XSECT COM_OCEAN_2D COM_OCEAN_3D \ COM_OCEAN_ANALYSIS \ COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION \ - COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON + COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON COM_CONF for grid in "0p25" "0p50" "1p00"; do YMD=${PDY} HH=${cyc} GRID=${grid} generate_com -rx "COM_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL" diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index c34fff1a84..2c5606773d 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -92,7 +92,7 @@ if [[ ${type} = "gfs" ]]; then # This uses the bash extended globbing option { echo "./logs/${PDY}${cyc}/gfs!(arch).log" - echo "${COM_ATMOS_HISTORY/${ROTDIR}\//}/input.nml" + echo "${COM_CONF/${ROTDIR}\//}/ufs.input.nml" if [[ ${MODE} = "cycled" ]]; then if [[ -s "${COM_ATMOS_ANALYSIS}/${head}gsistat" ]]; then @@ -169,7 +169,8 @@ if [[ ${type} = "gfs" ]]; then if [[ ${ARCH_GAUSSIAN} = "YES" ]]; then { echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2" - echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" + # TODO Create sflux idx files so they can be archived + #echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" } >> "${DATA}/gfs_flux.txt" { @@ -368,13 +369,16 @@ if [[ ${type} == "gdas" ]]; then echo "./logs/${PDY}${cyc}/gdas${fstep}.log" fi done - echo "./logs/${PDY}${cyc}/gdaspost*.log" + echo "./logs/${PDY}${cyc}/gdas*prod*.log" + if [[ "${WRITE_DOPOST}" == ".false." ]]; then + echo "./logs/${PDY}${cyc}/gdas*upp*.log" + fi fh=0 while [[ ${fh} -le 9 ]]; do fhr=$(printf %03i "${fh}") echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2" - echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" + #echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.f${fhr}" echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.f${fhr}.idx" echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.f${fhr}" @@ -407,10 +411,13 @@ if [[ ${type} == "gdas" ]]; then fi subtyplist="gome_metop-b omi_aura ompslp_npp ompsnp_n20 ompsnp_npp ompstc8_n20 ompstc8_npp sbuv2_n19" for subtype in ${subtyplist}; do - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.${PDY}${cyc}.ieee_d${suffix}" - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.ctl" - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.${PDY}${cyc}.ieee_d${suffix}" - echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.ctl" + # On occassion, data is not available for some of these satellites. Check for existence. + if [[ -s "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.${PDY}${cyc}.ieee_d${suffix}" ]]; then + echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.${PDY}${cyc}.ieee_d${suffix}" + echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.ctl" + echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.${PDY}${cyc}.ieee_d${suffix}" + echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.ctl" + fi done echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/stdout.${type}.tar.gz" done @@ -612,10 +619,19 @@ if [[ ${type} == "enkfgdas" || ${type} == "enkfgfs" ]]; then fi fi done # loop over FHR - for fstep in eobs ecen esfc eupd efcs epos ; do + for fstep in efcs epos ; do echo "logs/${PDY}${cyc}/${RUN}${fstep}*.log" done + # eobs, ecen, esfc, and eupd are not run on the first cycle + for fstep in eobs ecen esfc eupd ; do + for log in "${ROTDIR}/logs/${PDY}${cyc}/${RUN}${fstep}"*".log"; do + if [[ -s "${log}" ]]; then + echo "logs/${PDY}${cyc}/${RUN}${fstep}*.log" + fi + done + done + # eomg* are optional jobs for log in "${ROTDIR}/logs/${PDY}${cyc}/${RUN}eomg"*".log"; do if [[ -s "${log}" ]]; then From 8730330d3e2a318960209e26b4d84328849b51cc Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 16 Jan 2024 08:54:29 -0600 Subject: [PATCH 2/3] Generate sflux master grib2 index files. --- scripts/exglobal_atmos_products.sh | 8 ++++++-- ush/hpssarch_gen.sh | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/exglobal_atmos_products.sh b/scripts/exglobal_atmos_products.sh index 5f0b1db6cf..43c94bd6d0 100755 --- a/scripts/exglobal_atmos_products.sh +++ b/scripts/exglobal_atmos_products.sh @@ -167,14 +167,18 @@ done # for (( nset=1 ; nset <= downset ; nset++ )) #--------------------------------------------------------------- +# Create the index file for the sflux master, if it exists. +FLUX_FILE="${COM_ATMOS_MASTER}/${PREFIX}sfluxgrb${fhr3}.grib2" +if [[ -s "${FLUX_FILE}" ]]; then + ${WGRIB2} -s "${FLUX_FILE}" > "${FLUX_FILE}.idx" +fi + # Section creating slfux grib2 interpolated products # Create 1-degree sflux grib2 output # move to COM and index it if [[ "${FLXGF:-}" == "YES" ]]; then # Files needed by ${INTERP_ATMOS_SFLUXSH} - FLUX_FILE="${COM_ATMOS_MASTER}/${PREFIX}sfluxgrb${fhr3}.grib2" - input_file="${FLUX_FILE}" output_file_prefix="sflux_${fhr3}" grid_string="1p00" diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index 2c5606773d..f337a6d177 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -169,8 +169,7 @@ if [[ ${type} = "gfs" ]]; then if [[ ${ARCH_GAUSSIAN} = "YES" ]]; then { echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2" - # TODO Create sflux idx files so they can be archived - #echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" + echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" } >> "${DATA}/gfs_flux.txt" { From 9bf552500be413cea144167c5c3efcb53578a238 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 16 Jan 2024 09:57:07 -0600 Subject: [PATCH 3/3] Archive GFS master GRIB2 sflux files. --- ush/hpssarch_gen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh index f337a6d177..db39c2eac5 100755 --- a/ush/hpssarch_gen.sh +++ b/ush/hpssarch_gen.sh @@ -377,7 +377,7 @@ if [[ ${type} == "gdas" ]]; then while [[ ${fh} -le 9 ]]; do fhr=$(printf %03i "${fh}") echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2" - #echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" + echo "${COM_ATMOS_MASTER/${ROTDIR}\//}/${head}sfluxgrbf${fhr}.grib2.idx" echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.f${fhr}" echo "${COM_ATMOS_GRIB_0p25/${ROTDIR}\//}/${head}pgrb2.0p25.f${fhr}.idx" echo "${COM_ATMOS_GRIB_1p00/${ROTDIR}\//}/${head}pgrb2.1p00.f${fhr}"