Apply fixes to archiving jobs (NOAA-EMC#2221)

Add checks for potentially missing log/data files and update log names for archiving.

Missing sflux index files are also added to the products script.

Resolves NOAA-EMC#2076

jobs/JGLOBAL_ARCHIVE

@@ -19,7 +19,7 @@ YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM_ATMO
   COM_OCEAN_HISTORY COM_OCEAN_INPUT COM_OCEAN_RESTART COM_OCEAN_XSECT COM_OCEAN_2D COM_OCEAN_3D \
   COM_OCEAN_ANALYSIS \
   COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION \
-  COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON
+  COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON COM_CONF
 
 for grid in "0p25" "0p50" "1p00"; do
   YMD=${PDY} HH=${cyc} GRID=${grid} generate_com -rx "COM_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL"

scripts/exglobal_atmos_products.sh

@@ -167,14 +167,18 @@ done  # for (( nset=1 ; nset <= downset ; nset++ ))
 
 #---------------------------------------------------------------
 
+# Create the index file for the sflux master, if it exists.
+FLUX_FILE="${COM_ATMOS_MASTER}/${PREFIX}sfluxgrb${fhr3}.grib2"
+if [[ -s "${FLUX_FILE}" ]]; then
+  ${WGRIB2} -s "${FLUX_FILE}" > "${FLUX_FILE}.idx"
+fi
+
 # Section creating slfux grib2 interpolated products
 # Create 1-degree sflux grib2 output
 # move to COM and index it
 if [[ "${FLXGF:-}" == "YES" ]]; then
 
   # Files needed by ${INTERP_ATMOS_SFLUXSH}
-  FLUX_FILE="${COM_ATMOS_MASTER}/${PREFIX}sfluxgrb${fhr3}.grib2"
-
   input_file="${FLUX_FILE}"
   output_file_prefix="sflux_${fhr3}"
   grid_string="1p00"

ush/hpssarch_gen.sh

@@ -92,7 +92,7 @@ if [[ ${type} = "gfs" ]]; then
   #  This uses the bash extended globbing option
   {
     echo "./logs/${PDY}${cyc}/gfs!(arch).log"
-    echo "${COM_ATMOS_HISTORY/${ROTDIR}\//}/input.nml"
+    echo "${COM_CONF/${ROTDIR}\//}/ufs.input.nml"
 
     if [[ ${MODE} = "cycled" ]]; then
       if [[ -s "${COM_ATMOS_ANALYSIS}/${head}gsistat" ]]; then
@@ -368,7 +368,10 @@ if [[ ${type} == "gdas" ]]; then
         echo "./logs/${PDY}${cyc}/gdas${fstep}.log"
       fi
     done
-    echo "./logs/${PDY}${cyc}/gdaspost*.log"
+    echo "./logs/${PDY}${cyc}/gdas*prod*.log"
+    if [[ "${WRITE_DOPOST}" == ".false." ]]; then
+       echo "./logs/${PDY}${cyc}/gdas*upp*.log"
+    fi
 
     fh=0
     while [[ ${fh} -le 9 ]]; do
@@ -407,10 +410,13 @@ if [[ ${type} == "gdas" ]]; then
         fi
         subtyplist="gome_metop-b omi_aura ompslp_npp ompsnp_n20 ompsnp_npp ompstc8_n20 ompstc8_npp sbuv2_n19"
         for subtype in ${subtyplist}; do
-          echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.${PDY}${cyc}.ieee_d${suffix}"
-          echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.ctl"
-          echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.${PDY}${cyc}.ieee_d${suffix}"
-          echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.ctl"
+          # On occassion, data is not available for some of these satellites.  Check for existence.
+          if [[ -s "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.${PDY}${cyc}.ieee_d${suffix}" ]]; then
+             echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.${PDY}${cyc}.ieee_d${suffix}"
+             echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.anl.ctl"
+             echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.${PDY}${cyc}.ieee_d${suffix}"
+             echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/${subtype}.ges.ctl"
+          fi
         done
         echo "${COM_ATMOS_OZNMON/${ROTDIR}\//}/${type}/stdout.${type}.tar.gz"
       done
@@ -612,10 +618,19 @@ if [[ ${type} == "enkfgdas" || ${type} == "enkfgfs" ]]; then
         fi
       fi
     done # loop over FHR
-    for fstep in eobs ecen esfc eupd efcs epos ; do
+    for fstep in efcs epos ; do
      echo "logs/${PDY}${cyc}/${RUN}${fstep}*.log"
     done
 
+  # eobs, ecen, esfc, and eupd are not run on the first cycle
+    for fstep in eobs ecen esfc eupd ; do
+       for log in "${ROTDIR}/logs/${PDY}${cyc}/${RUN}${fstep}"*".log"; do
+          if [[ -s "${log}" ]]; then
+             echo "logs/${PDY}${cyc}/${RUN}${fstep}*.log"
+          fi
+       done
+    done
+
   # eomg* are optional jobs
     for log in "${ROTDIR}/logs/${PDY}${cyc}/${RUN}eomg"*".log"; do
        if [[ -s "${log}" ]]; then