From 6044d099626f568a1a671ea86398e2aedfaf72f4 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Thu, 18 Jan 2024 09:25:57 -0500 Subject: [PATCH] Changes obsproc to obsprep within ocean obs prep task (#858) Shifts internal references to obsproc from that to obsprep, so as not to confuse the ocean obs prep task with the process that delivers observations files to the dmpdir. Partially addresses https://github.com/NOAA-EMC/GDASApp/issues/857 concurrent with mutually dependent global-workflow PR https://github.com/NOAA-EMC/global-workflow/pull/2236 CI will fail until associated g-w PR is merged --- parm/soca/obsprep/obsprep_config.yaml | 41 +++++++++++++++++++ parm/soca/obsproc/obsproc_config.yaml | 41 ------------------- scripts/exglobal_prep_ocean_obs.py | 41 +++++++++++-------- test/soca/gw/CMakeLists.txt | 10 ++--- test/soca/gw/run_jjobs.yaml.test | 4 +- .../gw/{setup_obsproc.sh => setup_obsprep.sh} | 2 +- ush/soca/prep_marine_obs.py | 6 +-- 7 files changed, 75 insertions(+), 70 deletions(-) create mode 100644 parm/soca/obsprep/obsprep_config.yaml delete mode 100644 parm/soca/obsproc/obsproc_config.yaml rename test/soca/gw/{setup_obsproc.sh => setup_obsprep.sh} (90%) diff --git a/parm/soca/obsprep/obsprep_config.yaml b/parm/soca/obsprep/obsprep_config.yaml new file mode 100644 index 000000000..1be4c45d3 --- /dev/null +++ b/parm/soca/obsprep/obsprep_config.yaml @@ -0,0 +1,41 @@ +observations: +- obs space: + name: sss_smap + dmpdir subdir: SSS + dmpdir regex: SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5 + provider: SMAP + output file: sss_smap.ioda.nc +- obs space: + name: sss_smos + provider: SMOS + dmpdir subdir: SSS + output file: sss_smos.ioda.nc + dmpdir regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc +- obs space: + name: adt_rads_all + dmpdir subdir: ADT + dmpdir regex: rads_adt_??_???????.nc + provider: RADS + output file: adt_rads_all.nc4 +- obs space: + name: icec_amsr2_north + provider: AMSR2 + dmpdir subdir: icec + output file: icec_amsr2_north.ioda.nc + dmpdir regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc +- obs space: + name: icec_amsr2_south + provider: AMSR2 + dmpdir subdir: icec + output file: icec_amsr2_south.ioda.nc + dmpdir regex: AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc +- obs space: + name: sst_metopb_l3u + provider: GHRSST + dmpdir subdir: 'sst' + output file: sst_metopb_l3u.ioda.nc + dmpdir regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc' + bounds: + units: C + min: -3.0 + max: 50.0 diff --git a/parm/soca/obsproc/obsproc_config.yaml b/parm/soca/obsproc/obsproc_config.yaml deleted file mode 100644 index 5102ac9a3..000000000 --- a/parm/soca/obsproc/obsproc_config.yaml +++ /dev/null @@ -1,41 +0,0 @@ -observations: -- obs space: - name: sss_smap - obsproc subdir: SSS - obsproc regex: SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5 - provider: SMAP - output file: sss_smap.ioda.nc -- obs space: - name: sss_smos - provider: SMOS - obsproc subdir: SSS - output file: sss_smos.ioda.nc - obsproc regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc -- obs space: - name: adt_rads_all - obsproc subdir: ADT - obsproc regex: rads_adt_??_???????.nc - provider: RADS - output file: adt_rads_all.nc4 -- obs space: - name: icec_amsr2_north - provider: AMSR2 - obsproc subdir: icec - output file: icec_amsr2_north.ioda.nc - obsproc regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc -- obs space: - name: icec_amsr2_south - provider: AMSR2 - obsproc subdir: icec - output file: icec_amsr2_south.ioda.nc - obsproc regex: AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc -- obs space: - name: sst_metopb_l3u - provider: GHRSST - obsproc subdir: 'sst' - output file: sst_metopb_l3u.ioda.nc - obsproc regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc' - bounds: - units: C - min: -3.0 - max: 50.0 diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py index 437f18234..a8ab59d6b 100755 --- a/scripts/exglobal_prep_ocean_obs.py +++ b/scripts/exglobal_prep_ocean_obs.py @@ -30,8 +30,13 @@ # present in OBS_YAML_DIR obsConfig = YAMLFile(OBS_YAML) -OBSPROC_YAML = os.getenv('OBSPROC_YAML') -obsprocConfig = YAMLFile(OBSPROC_YAML) +OBSPREP_YAML = os.getenv('OBSPREP_YAML') + +if os.path.exists(OBSPREP_YAML): + obsprepConfig = YAMLFile(OBSPREP_YAML) +else: + print(f"CRITICAL: OBSPREP_YAML file {OBSPREP_YAML} does not exist") + raise FileNotFoundError filesToSave = [] @@ -48,41 +53,41 @@ print("WARNING: Ill-formed observer yaml file, skipping") continue # to next observer -# ...look through the observations in OBSPROC_YAML... - for observation in obsprocConfig['observations']: +# ...look through the observations in OBSPREP_YAML... + for observation in obsprepConfig['observations']: - obsprocSpace = observation['obs space'] - obsprocSpaceName = obsprocSpace['name'] + obsprepSpace = observation['obs space'] + obsprepSpaceName = obsprepSpace['name'] # ...for a matching name, and process the observation source - if obsprocSpaceName == obsSpaceName: + if obsprepSpaceName == obsSpaceName: - print(f"obsprocSpaceName: {obsSpaceName}") + print(f"obsprepSpaceName: {obsSpaceName}") # fetch the obs files from DMPDIR to RUNDIR - matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace) + matchingFiles = prep_marine_obs.obs_fetch(obsprepSpace) if not matchingFiles: print("WARNING: No files found for obs source , skipping") break # to next observation source in OBS_YAML - obsprocSpace['input files'] = matchingFiles - obsprocSpace['window begin'] = windowBegin - obsprocSpace['window end'] = windowEnd + obsprepSpace['input files'] = matchingFiles + obsprepSpace['window begin'] = windowBegin + obsprepSpace['window end'] = windowEnd outputFilename = f"gdas.t{cyc}z.{obsSpaceName}.{PDY}{cyc}.nc4" - obsprocSpace['output file'] = outputFilename + obsprepSpace['output file'] = outputFilename - iodaYamlFilename = obsprocSpaceName + '2ioda.yaml' - save_as_yaml(obsprocSpace, iodaYamlFilename) + iodaYamlFilename = obsprepSpaceName + '2ioda.yaml' + save_as_yaml(obsprepSpace, iodaYamlFilename) subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True) - filesToSave.append([obsprocSpace['output file'], - os.path.join(COMOUT_OBS, obsprocSpace['output file'])]) + filesToSave.append([obsprepSpace['output file'], + os.path.join(COMOUT_OBS, obsprepSpace['output file'])]) filesToSave.append([iodaYamlFilename, os.path.join(COMOUT_OBS, iodaYamlFilename)]) except TypeError: - print("CRITICAL: Ill-formed OBS_YAML file, exiting") + print("CRITICAL: Ill-formed OBS_YAML or OBSPREP_YAML file, exiting") raise if not os.path.exists(COMOUT_OBS): diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index bd6616744..25d290571 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -33,12 +33,12 @@ add_test(NAME test_gdasapp_soca_run_clean file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/apps_scratch) # Create scratch for obs prep task -file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc) +file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep) -# Set up obsproc, based on test_gdasapp_util_prepdata -add_test(NAME test_gdasapp_soca_setup_obsproc - COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsproc.sh ${PROJECT_SOURCE_DIR} - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc) +# Set up obsprep, based on test_gdasapp_util_prepdata +add_test(NAME test_gdasapp_soca_setup_obsprep + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/gw/setup_obsprep.sh ${PROJECT_SOURCE_DIR} + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep) # Test JGDAS_GLOBAL_OCEAN_ANALYSIS_* set(jjob_list "JGLOBAL_PREP_OCEAN_OBS" diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index c4f4e12f3..dbd2fcd2f 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -55,8 +55,8 @@ setup_expt config: NICAS_GRID_SIZE: 150 prepoceanobs: SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml - OBSPROC_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml - DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsproc + OBSPREP_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml + DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/gdas/test/soca/gw/obsprep job options: account: da-cpu diff --git a/test/soca/gw/setup_obsproc.sh b/test/soca/gw/setup_obsprep.sh similarity index 90% rename from test/soca/gw/setup_obsproc.sh rename to test/soca/gw/setup_obsprep.sh index 9aa7f0583..f64bc7a54 100755 --- a/test/soca/gw/setup_obsproc.sh +++ b/test/soca/gw/setup_obsprep.sh @@ -3,7 +3,7 @@ set -ex project_source_dir=$1 -# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsproc, set in ctest command +# working directory should be ${PROJECT_BINARY_DIR}/test/soca/gw/obsprep, set in ctest command test_dmpdir="gdas.20180415/12" rm -rf ${test_dmpdir} diff --git a/ush/soca/prep_marine_obs.py b/ush/soca/prep_marine_obs.py index 6bae569fd..5f1ec429f 100755 --- a/ush/soca/prep_marine_obs.py +++ b/ush/soca/prep_marine_obs.py @@ -63,10 +63,10 @@ } -def obs_fetch(obsprocSpace): +def obs_fetch(obsprepSpace): - subDir = obsprocSpace['obsproc subdir'] - filepattern = obsprocSpace['obsproc regex'] + subDir = obsprepSpace['dmpdir subdir'] + filepattern = obsprepSpace['dmpdir regex'] dataDir = os.path.join(cycDir, subDir) # TODO: check the existence of this