Changes obsproc to obsprep within ocean obs prep task (#858)

Shifts internal references to obsproc from that to obsprep, so as not to
confuse the ocean obs prep task with the process that delivers
observations files to the dmpdir.

Partially addresses #857
concurrent with mutually dependent global-workflow PR
NOAA-EMC/global-workflow#2236

CI will fail until associated g-w PR is merged

parm/soca/obsprep/obsprep_config.yaml

@@ -0,0 +1,41 @@
+observations:
+- obs space:
+    name: sss_smap
+    dmpdir subdir: SSS
+    dmpdir regex: SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5
+    provider: SMAP
+    output file: sss_smap.ioda.nc
+- obs space:
+    name: sss_smos
+    provider: SMOS
+    dmpdir subdir: SSS
+    output file: sss_smos.ioda.nc
+    dmpdir regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc
+- obs space:
+    name: adt_rads_all
+    dmpdir subdir: ADT
+    dmpdir regex: rads_adt_??_???????.nc
+    provider: RADS
+    output file: adt_rads_all.nc4
+- obs space:
+    name: icec_amsr2_north
+    provider: AMSR2
+    dmpdir subdir: icec
+    output file: icec_amsr2_north.ioda.nc
+    dmpdir regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc
+- obs space:
+    name: icec_amsr2_south
+    provider: AMSR2
+    dmpdir subdir: icec
+    output file: icec_amsr2_south.ioda.nc
+    dmpdir regex: AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc
+- obs space:
+    name: sst_metopb_l3u
+    provider: GHRSST
+    dmpdir subdir: 'sst'
+    output file: sst_metopb_l3u.ioda.nc
+    dmpdir regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc'
+    bounds:
+      units: C
+      min: -3.0
+      max: 50.0

scripts/exglobal_prep_ocean_obs.py

@@ -30,8 +30,13 @@
 # present in OBS_YAML_DIR
 obsConfig = YAMLFile(OBS_YAML)
 
-OBSPROC_YAML = os.getenv('OBSPROC_YAML')
-obsprocConfig = YAMLFile(OBSPROC_YAML)
+OBSPREP_YAML = os.getenv('OBSPREP_YAML')
+
+if os.path.exists(OBSPREP_YAML):
+    obsprepConfig = YAMLFile(OBSPREP_YAML)
+else:
+    print(f"CRITICAL: OBSPREP_YAML file {OBSPREP_YAML} does not exist")
+    raise FileNotFoundError
 
 filesToSave = []
 
@@ -48,41 +53,41 @@
             print("WARNING: Ill-formed observer yaml file, skipping")
             continue  # to next observer
 
-# ...look through the observations in OBSPROC_YAML...
-        for observation in obsprocConfig['observations']:
+# ...look through the observations in OBSPREP_YAML...
+        for observation in obsprepConfig['observations']:
 
-            obsprocSpace = observation['obs space']
-            obsprocSpaceName = obsprocSpace['name']
+            obsprepSpace = observation['obs space']
+            obsprepSpaceName = obsprepSpace['name']
 
 # ...for a matching name, and process the observation source
-            if obsprocSpaceName == obsSpaceName:
+            if obsprepSpaceName == obsSpaceName:
 
-                print(f"obsprocSpaceName: {obsSpaceName}")
+                print(f"obsprepSpaceName: {obsSpaceName}")
 
                 # fetch the obs files from DMPDIR to RUNDIR
-                matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace)
+                matchingFiles = prep_marine_obs.obs_fetch(obsprepSpace)
 
                 if not matchingFiles:
                     print("WARNING: No files found for obs source , skipping")
                     break  # to next observation source in OBS_YAML
 
-                obsprocSpace['input files'] = matchingFiles
-                obsprocSpace['window begin'] = windowBegin
-                obsprocSpace['window end'] = windowEnd
+                obsprepSpace['input files'] = matchingFiles
+                obsprepSpace['window begin'] = windowBegin
+                obsprepSpace['window end'] = windowEnd
                 outputFilename = f"gdas.t{cyc}z.{obsSpaceName}.{PDY}{cyc}.nc4"
-                obsprocSpace['output file'] = outputFilename
+                obsprepSpace['output file'] = outputFilename
 
-                iodaYamlFilename = obsprocSpaceName + '2ioda.yaml'
-                save_as_yaml(obsprocSpace, iodaYamlFilename)
+                iodaYamlFilename = obsprepSpaceName + '2ioda.yaml'
+                save_as_yaml(obsprepSpace, iodaYamlFilename)
 
                 subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True)
 
-                filesToSave.append([obsprocSpace['output file'],
-                                    os.path.join(COMOUT_OBS, obsprocSpace['output file'])])
+                filesToSave.append([obsprepSpace['output file'],
+                                    os.path.join(COMOUT_OBS, obsprepSpace['output file'])])
                 filesToSave.append([iodaYamlFilename,
                                     os.path.join(COMOUT_OBS, iodaYamlFilename)])
 except TypeError:
-    print("CRITICAL: Ill-formed OBS_YAML file, exiting")
+    print("CRITICAL: Ill-formed OBS_YAML or OBSPREP_YAML file, exiting")
     raise
 
 if not os.path.exists(COMOUT_OBS):

test/soca/gw/CMakeLists.txt

@@ -33,12 +33,12 @@ add_test(NAME test_gdasapp_soca_run_clean
 file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/apps_scratch)
 
 # Create scratch for obs prep task
-file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc)
+file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep)
 
-# Set up obsproc, based on test_gdasapp_util_prepdata
-add_test(NAME test_gdasapp_soca_setup_obsproc
-         COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsproc.sh ${PROJECT_SOURCE_DIR}
-         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc)
+# Set up obsprep, based on test_gdasapp_util_prepdata
+add_test(NAME test_gdasapp_soca_setup_obsprep
+         COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsprep.sh ${PROJECT_SOURCE_DIR}
+         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep)
 
 # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_*
 set(jjob_list "JGLOBAL_PREP_OCEAN_OBS"

test/soca/gw/run_jjobs.yaml.test

@@ -55,8 +55,8 @@ setup_expt config:
     NICAS_GRID_SIZE: 150
   prepoceanobs:
     SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
-    OBSPROC_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml
-    DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsproc
+    OBSPREP_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml
+    DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsprep
 
 job options:
   account: da-cpu

test/soca/gw/setup_obsproc.sh → test/soca/gw/setup_obsprep.sh

@@ -3,7 +3,7 @@ set -ex
 
 project_source_dir=$1
 
-# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc, set in ctest command
+# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep, set in ctest command
 test_dmpdir="gdas.20180415/12"
 
 rm -rf ${test_dmpdir}

ush/soca/prep_marine_obs.py

@@ -63,10 +63,10 @@
 }
 
 
-def obs_fetch(obsprocSpace):
+def obs_fetch(obsprepSpace):
 
-    subDir = obsprocSpace['obsproc subdir']
-    filepattern = obsprocSpace['obsproc regex']
+    subDir = obsprepSpace['dmpdir subdir']
+    filepattern = obsprepSpace['dmpdir regex']
 
     dataDir = os.path.join(cycDir, subDir)
     # TODO: check the existence of this