[develop] Add workflow for radar reflectivity processing.

jobs/JREGIONAL_GET_DA_OBS

@@ -54,7 +54,9 @@ This script retrieves observation data for RRFS data assimilation tasks.
 #-----------------------------------------------------------------------
 #
 export DATA="${COMIN}/obs"
+export RADAR_DATA="${COMIN}/radar"
 mkdir_vrfy -p "${DATA}"
+mkdir_vrfy -p "${RADAR_DATA}"
 
 # Set needed date/time variables
 export START_DATE=$(echo "${PDY} ${cyc}")

parm/data_locations.yml

@@ -88,7 +88,7 @@ FV3GFS:
           - gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
           - com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
           - com_gfs_v16.2_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
-          - com_gfs_v16.3_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar            
+          - com_gfs_v16.3_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
         fcst:
           - gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
           - com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_pgrb2.tar
@@ -111,7 +111,7 @@ FV3GFS:
           - ['gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'gpfs_dell1_nco_ops_com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar']
           - ['com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'com_gfs_prod_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar']
           - ['com_gfs_v16.2_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'com_gfs_v16.2_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar']
-          - ['com_gfs_v16.3_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'com_gfs_v16.3_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar']            
+          - ['com_gfs_v16.3_gfs.{yyyymmdd}_{hh}.gfs_nca.tar', 'com_gfs_v16.3_gfs.{yyyymmdd}_{hh}.gfs_ncb.tar']
     file_names:
       <<: *gfs_file_names
   aws:
@@ -316,6 +316,28 @@ GFS_obs:
         obs:
           - "{yy}{jjj}{hh}00.gfs.t{hh}z.syndata.tcvitals.tm00"
 
+NSSL_mrms:
+
+  hpss:
+    protocol: htar
+    archive_format: tar
+    archive_path:
+      - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd}
+      - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd}
+      - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd}
+      - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd}
+    archive_internal_dir:
+      - ./upperair/mrms/conus/MergedReflectivityQC
+    archive_file_names:
+      - dcom_ldmdata_obs.tar
+      - dcom_prod_ldmdata_obs.tar
+      - ldmdata.tide.{yyyymmdd}.tar
+      - ldmdata.gyre.{yyyymmdd}.tar
+    file_names:
+      obs:
+        - "*MergedReflectivityQC_*_{yyyymmdd}-{hh}{min}*.grib2*"
+
+
 RAP_obs:
   hpss:
     protocol: htar
@@ -367,6 +389,7 @@ RAP_obs:
         - "{yyyymmddhh}.rap_e.t{hh}z.lgycld.tm00.bufr_d"
         - "{yyyymmddhh}.rap_e.t{hh}z.nexrad.tm00.bufr_d"
         - "{yyyymmddhh}.rap_e.t{hh}z.satwnd.tm00.bufr_d"
+        - "{yyyymmddhh}.rap_e.t{hh}z.lghtng.tm00.bufr_d"
   aws:
     protocol: download
     url: https://noaa-rap-pds.s3.amazonaws.com/rap.{yyyymmdd}

parm/wflow/da_data_preproc.yaml

@@ -14,6 +14,7 @@ default_data_preproc_task: &default_preproc
     subcyc: !cycstr "@M"
     LOGDIR: !cycstr "&LOGDIR;"
     CYCLE_TYPE: '#cycle_type#'
+    nprocs: '{{ parent.nnodes * parent.ppn }}'
   native: '{{ platform.SCHED_NATIVE_CMD }}'
   nodes: '{{ nnodes }}:ppn={{ ppn }}'
   nnodes: 1
@@ -44,6 +45,7 @@ metatask_process_obs_cycle_type:
   task_process_radarref_#cycle_type#:
     <<: *default_preproc
     command: '&LOAD_MODULES_RUN_TASK_FP; "process_obs" "&JOBSdir;/JREGIONAL_PROCESS_RADARREF"'
+    nnodes: 2
     ppn: 24
     join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
     dependency:

scripts/exregional_get_da_obs.sh

@@ -8,7 +8,8 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
-source_config_for_task "task_get_da_obs" ${GLOBAL_VAR_DEFNS_FP}
+. $USHdir/get_mrms_files.sh
+source_config_for_task "task_get_da_obs|task_process_radarref" ${GLOBAL_VAR_DEFNS_FP}
 #
 #-----------------------------------------------------------------------
 #
@@ -120,9 +121,9 @@ The command was:
 ${cmd}
 "
 # Link to GSI-expected filenames
-mv_vrfy "${DATA}/${template_arr[0]}" "${DATA}/lghtngbufr"
-mv_vrfy "${DATA}/${template_arr[1]}" "${DATA}/lgycld.bufr_d"
-mv_vrfy "${DATA}/${template_arr[2]}" "${DATA}/prepbufr"
+ln_vrfy "${DATA}/${template_arr[0]}" "${DATA}/lghtngbufr"
+ln_vrfy "${DATA}/${template_arr[1]}" "${DATA}/lgycld.bufr_d"
+ln_vrfy "${DATA}/${template_arr[2]}" "${DATA}/prepbufr"
 
 #
 #-----------------------------------------------------------------------
@@ -184,6 +185,66 @@ if [ "${NLDN_NEEDED:-}" = "TRUE" ]; then
   done
 
 fi
+#
+#-----------------------------------------------------------------------
+#
+# retrieve NSSL Mosaic Reflectivity observations
+#
+#-----------------------------------------------------------------------
+#
+
+
+# If files are available on disk, copy them here in the code. This
+# approach is used here because of its efficiency compared to repeated
+# calls to retrieve_data.py for this observation file type.
+
+#
+#-----------------------------------------------------------------------
+#
+# Check for files on disk first
+#
+#-----------------------------------------------------------------------
+#
+mrms="MergedReflectivityQC"
+
+  for timelevel in ${RADARREFL_TIMELEVEL[@]}; do
+    echo "timelevel = ${timelevel}"
+    timelevel=$( printf %2.2i $timelevel )
+    radar_output_path=${RADAR_DATA}/${timelevel}
+    mkdir -p $radar_output_path
+
+    #-----------------------------------------------------------------------
+    # copy observation files to staging location
+    #-----------------------------------------------------------------------
+    if [ -n "${NSSLMOSAIC:-}" ] ; then
+        get_mrms_files $timelevel $radar_output_path $mrms
+    fi
+
+    # Check to see if files were retrieved from disk
+    # Try other resources, if not
+    if [ ! -s ${radar_output_path}/filelist_mrms ]; then
+
+      cmd="
+      python3 -u ${USHdir}/retrieve_data.py \
+        --debug \
+        --file_set obs \
+        --config ${PARMdir}/data_locations.yml \
+        --cycle_date ${PDY}${cyc}${timelevel} \
+        --data_stores ${EXTRN_MDL_DATA_STORES} \
+        --data_type NSSL_mrms \
+        --output_path ${radar_output_path}
+      "
+
+      $cmd || print_err_msg_exit "\
+      Call to retrieve_data.py failed with a non-zero exit status.
+
+      The command was:
+      ${cmd}
+      "
+      ls $radar_output_path/*${mrms}* > $radar_output_path/filelist_mrms
+    fi
+  done
+
 
 #
 #-----------------------------------------------------------------------

scripts/exregional_process_radarref.sh

@@ -8,6 +8,7 @@
 #-----------------------------------------------------------------------
 #
 . $USHdir/source_util_funcs.sh
+. $USHdir/get_mrms_files.sh
 source_config_for_task "task_process_radarref|task_run_fcst" ${GLOBAL_VAR_DEFNS_FP}
 #
 #-----------------------------------------------------------------------
@@ -53,8 +54,6 @@ with FV3 for the specified cycle.
 #
 eval ${PRE_TASK_CMDS}
 
-nprocs=$(( NNODES_PROCESS_RADARREF*PPN_PROCESS_RADARREF))
-
 #
 #-----------------------------------------------------------------------
 #
@@ -80,17 +79,6 @@ DD=${YYYYMMDDHH:6:2}
 #-----------------------------------------------------------------------
 #
 BKTYPE=0
-if [ ${DO_SPINUP} == "TRUE" ]; then
-  if [ ${CYCLE_TYPE} == "spinup" ]; then
-    if [[ ${CYCL_HRS_SPINSTART[@]} =~ "$cyc" ]] ; then
-      BKTYPE=1
-    fi
-  fi
-else
-  if [[ ${CYCL_HRS_PRODSTART[@]} =~ "$cyc" ]] ; then
-    BKTYPE=1
-  fi
-fi
 
 n_iolayouty=$(($IO_LAYOUT_Y-1))
 
@@ -103,84 +91,62 @@ n_iolayouty=$(($IO_LAYOUT_Y-1))
 #-----------------------------------------------------------------------
 #
 print_info_msg "$VERBOSE" "
-Getting into working directory for radar reflectivity process ..."
+Getting into working directory for radar reflectivity process ... ${DATA}"
+
+pregen_grid_dir=$DOMAIN_PREGEN_BASEDIR/${PREDEF_GRID_NAME}
+print_info_msg "$VERBOSE" "pregen_grid_dir is $pregen_grid_dir"
 
 for timelevel in ${RADARREFL_TIMELEVEL[@]}; do
+  echo "timelevel = ${timelevel}"
   timelevel=$( printf %2.2i $timelevel )
-  mkdir_vrfy ${DATA}/${timelevel}
-  cd ${DATA}/${timelevel}
-
-  pregen_grid_dir=$DOMAIN_PREGEN_BASEDIR/${PREDEF_GRID_NAME}
-
-  print_info_msg "$VERBOSE" "pregen_grid_dir is $pregen_grid_dir"
+  mkdir_vrfy -p ${DATA}/${timelevel}
+  cd_vrfy ${DATA}/${timelevel}
 
-#
-#-----------------------------------------------------------------------
-#
-# link or copy background files
-#
-#-----------------------------------------------------------------------
+  #
+  #-----------------------------------------------------------------------
+  #
+  # copy background files
+  #
+  #-----------------------------------------------------------------------
 
   if [ ${BKTYPE} -eq 1 ]; then
-    cp_vrfy ${pregen_grid_dir}/fv3_grid_spec fv3sar_grid_spec.nc
+    cp -f ${pregen_grid_dir}/fv3_grid_spec fv3sar_grid_spec.nc
   else
     if [ "${IO_LAYOUT_Y}" == "1" ]; then
-      cp_vrfy ${pregen_grid_dir}/fv3_grid_spec fv3sar_grid_spec.nc
+      cp -f ${pregen_grid_dir}/fv3_grid_spec fv3sar_grid_spec.nc
     else
       for iii in $(seq -w 0 $(printf %4.4i $n_iolayouty))
       do
-        cp_vrfy ${pregen_grid_dir}/fv3_grid_spec.${iii} fv3sar_grid_spec.nc.${iii}
+        cp -f ${pregen_grid_dir}/fv3_grid_spec.${iii} fv3sar_grid_spec.nc.${iii}
       done
     fi
   fi
 
-#
-#-----------------------------------------------------------------------
-#
-# link/copy observation files to working directory 
-#
-#-----------------------------------------------------------------------
-
-  NSSL=${OBSPATH_NSSLMOSIAC}
+  #
+  #-----------------------------------------------------------------------
+  #
+  # copy observation files to working directory, if running in real
+  # time. otherwise, the get_da_obs data task will do this for you.
+  #
+  #-----------------------------------------------------------------------
 
   mrms="MergedReflectivityQC"
-
-# Link to the MRMS operational data
-  echo "timelevel = ${timelevel}"
-  echo "RADARREFL_MINS = ${RADARREFL_MINS[@]}"
-
-# Link to the MRMS operational data
-# This loop finds files closest to the given "timelevel"
-  for min in ${RADARREFL_MINS[@]}
-  do
-    min=$( printf %2.2i $((timelevel+min)) )
-    echo "Looking for data valid:"${YYYY}"-"${MM}"-"${DD}" "${cyc}":"${min}
-    sec=0
-    while [[ $sec -le 59 ]]; do
-      ss=$(printf %2.2i ${sec})
-      nsslfile=${NSSL}/*${mrms}_00.50_${YYYY}${MM}${DD}-${cyc}${min}${ss}.${OBS_SUFFIX}
-      if [ -s $nsslfile ]; then
-        echo 'Found '${nsslfile}
-        nsslfile1=*${mrms}_*_${YYYY}${MM}${DD}-${cyc}${min}*.${OBS_SUFFIX}
-        numgrib2=$(ls ${NSSL}/${nsslfile1} | wc -l)
-        echo 'Number of GRIB-2 files: '${numgrib2}
-        if [ ${numgrib2} -ge 10 ] && [ ! -e filelist_mrms ]; then
-          cp ${NSSL}/${nsslfile1} . 
-          ls ${nsslfile1} > filelist_mrms 
-          echo 'Creating links for ${YYYY}${MM}${DD}-${cyc}${min}'
-        fi
-      fi
-      ((sec+=1))
-    done
-  done
+  if [ "${DO_REAL_TIME}" = true ] ; then
+    get_mrms_files $timelevel "./" $mrms
+  else
+    # The data was staged by the get_da_obs task, so copy from COMIN.
+    # Use copy here so that we can unzip if necessary.
+    cp_vrfy ${COMIN}/radar/${timelevel}/* .
+  fi # DO_REAL_TIME
 
   if [ -s filelist_mrms ]; then
 
-     if [ ${OBS_SUFFIX} == "grib2.gz" ]; then
-        gzip -d *.gz
-        mv filelist_mrms filelist_mrms_org
-        ls MergedReflectivityQC_*_${YYYY}${MM}${DD}-${cyc}????.grib2 > filelist_mrms
-     fi
+    # Unzip files, if that's needed and update filelist_mrms
+    if [ $(ls *.gz 2> /dev/null | wc -l) -gt 0 ]; then
+       gzip -d *.gz
+       mv filelist_mrms filelist_mrms_org
+       ls ${mrms}_*_${YYYY}${MM}${DD}-${cyc}????.grib2 > filelist_mrms
+    fi
 
      numgrib2=$(more filelist_mrms | wc -l)
      print_info_msg "$VERBOSE" "Using radar data from: `head -1 filelist_mrms | cut -c10-15`"
@@ -189,7 +155,7 @@ for timelevel in ${RADARREFL_TIMELEVEL[@]}; do
      # remove filelist_mrms if zero bytes
      rm -f filelist_mrms
 
-     echo "WARNING: Not enough radar reflectivity files available for loop ${timelevel}."
+     echo "WARNING: Not enough radar reflectivity files available for timelevel ${timelevel}."
      continue
   fi
 

tests/WE2E/test_configs/data_assimilation/config.process_obs.yaml

@@ -9,7 +9,9 @@ workflow:
   PREDEF_GRID_NAME: RRFS_CONUS_3km
   DATE_FIRST_CYCL: '2022072000'
   DATE_LAST_CYCL: '2022072000'
-
+  PREEXISTING_DIR_METHOD: rename
+task_process_radarref:
+  RADARREFL_TIMELEVEL: [0]
 rocoto:
   entities:
     START_TIME_NSSLMOSAIC: "00:45:00"
@@ -20,5 +22,4 @@ rocoto:
   tasks:
     taskgroups: '{{ ["parm/wflow/da_data_preproc.yaml"]|include }}'
     metatask_process_obs_cycle_type:
-      task_process_radarref_#cycle_type#:
       task_process_lightning_#cycle_type#:

ush/config.da_cycling.yaml

@@ -41,7 +41,6 @@ rocoto:
   tasks:
     taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/da_data_preproc.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml"]|include }}'
     metatask_process_obs_cycle_type:
-      task_process_radarref_#cycle_type#:
       task_process_lightning_#cycle_type#:
     metatask_run_ensemble:
       task_run_fcst_mem#mem#: