-
Notifications
You must be signed in to change notification settings - Fork 120
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[develop] Introduce DA data preprocessing to workflow (#778)
This is a set of contributions as part of the RRFS/dev merge. The major capabilities include: * A new task (get_da_obs) that retrieves and stages observation files for the subsequent data processing tasks. Most of this logic is existing consolidated logic that has been moved out of the process* tasks. * Add workflow entries for running get_da_obs, process_radarref, process_lightning, and process_bufrobs tasks * Add a new config.rrfs.yaml file for RRFS-specific options; this will be expanded over time as more capabilities are added. * Add a new test (process_obs) to test these initial observation processing tasks. Note that in order to run this test, the app must be built with the rrfs_utils and gsi builds, which is not part of the default build at this time. * More verbose errors for run_WE2E_tests.py * Improvements to retrieve_data.py 1) Rename --file_type and --external_model arguments to --file_fmt and --data_type, respectively. These are less confusing option names since both model data and obs can be retrieved with this script. 2) Fix bug where script was not looking for files in additional locations if they exist 3) Fix bug where script failed if any files are missing from zip archive; this should not be a failure condition unless all are missing 4) Make some arguments optional if it's possible to continue without them 5) Give more helpful/verbose error messages if an invalid/unavailable data store was selected 6) Add new unit tests "test_rap_obs_from_hpss" and "test_rap_e_obs_from_hpss" --------- Co-authored-by: Christina.Holt <[email protected]>
- Loading branch information
1 parent
ab11918
commit 4892952
Showing
27 changed files
with
741 additions
and
166 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| #!/bin/bash | ||
|
|
||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| # Source the variable definitions file and the bash utility functions. | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| . $USHdir/source_util_funcs.sh | ||
| source_config_for_task "task_run_anl|task_run_enkf" ${GLOBAL_VAR_DEFNS_FP} | ||
| . $USHdir/job_preamble.sh | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| # Save current shell options (in a global array). Then set new options | ||
| # for this script/function. | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| { save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| # Get the full path to the file in which this script/function is located | ||
| # (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in | ||
| # which the file is located (scrfunc_dir). | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) | ||
| scrfunc_fn=$( basename "${scrfunc_fp}" ) | ||
| scrfunc_dir=$( dirname "${scrfunc_fp}" ) | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| # Print message indicating entry into script. | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| print_info_msg " | ||
| ======================================================================== | ||
| Entering script: \"${scrfunc_fn}\" | ||
| In directory: \"${scrfunc_dir}\" | ||
| This script retrieves observation data for RRFS data assimilation tasks. | ||
| ========================================================================" | ||
|
|
||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| # Create the directory where the GSI obs files should be stored | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| export DATA="${COMIN}/obs" | ||
| mkdir_vrfy -p "${DATA}" | ||
|
|
||
| # Set needed date/time variables | ||
| export START_DATE=$(echo "${PDY} ${cyc}") | ||
| export YYYYMMDDHH=$(date +%Y%m%d%H -d "${START_DATE}") | ||
|
|
||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| # Call the ex-script for this J-job | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| $SCRIPTSdir/exregional_get_da_obs.sh || \ | ||
| print_err_msg_exit "\ | ||
| Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| # Run job postamble. | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| job_postamble | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| # Restore the shell options saved at the beginning of this script/function. | ||
| # | ||
| #----------------------------------------------------------------------- | ||
| # | ||
| { restore_shell_opts; } > /dev/null 2>&1 | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| load("hpss") | ||
| load("python_srw") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| load("hpss") | ||
| load("python_srw") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -213,4 +213,3 @@ metatask_run_ensemble: | |
| taskdep: | ||
| attrs: | ||
| task: aqm_lbcs | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| # This group contains all the tasks needed for preprocessing tasks for | ||
| # RRFS DA. | ||
|
|
||
| default_data_preproc_task: &default_preproc | ||
| account: '&ACCOUNT;' | ||
| attrs: | ||
| cycledefs: #cycledefs_type# | ||
| maxtries: '1' | ||
| envars: &default_envars | ||
| GLOBAL_VAR_DEFNS_FP: '&GLOBAL_VAR_DEFNS_FP;' | ||
| USHdir: '&USHdir;' | ||
| PDY: !cycstr "@Y@m@d" | ||
| cyc: !cycstr "@H" | ||
| subcyc: !cycstr "@M" | ||
| LOGDIR: !cycstr "&LOGDIR;" | ||
| CYCLE_TYPE: '#cycle_type#' | ||
| native: '{{ platform.SCHED_NATIVE_CMD }}' | ||
| nodes: '{{ nnodes }}:ppn={{ ppn }}' | ||
| nnodes: 1 | ||
| nodesize: "&NCORES_PER_NODE;" | ||
| ppn: 1 | ||
| partition: '{% if platform.get("PARTITION_DEFAULT") %}&PARTITION_DEFAULT;{% else %}None{% endif %}' | ||
| queue: '&QUEUE_DEFAULT;' | ||
| walltime: 00:25:00 | ||
|
|
||
|
|
||
| task_get_da_obs: | ||
| <<: *default_preproc | ||
| command: '&LOAD_MODULES_RUN_TASK_FP; "get_da_obs" "&JOBSdir;/JREGIONAL_GET_DA_OBS"' | ||
| attrs: | ||
| cycledefs: forecast | ||
| maxtries: '1' | ||
| join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' | ||
| partition: '&PARTITION_HPSS;' | ||
| queue: '&QUEUE_HPSS;' | ||
| dependency: | ||
| timedep: '<cyclestr offset="&START_TIME_CONVENTIONAL;">@Y@m@d@H@M00</cyclestr>' | ||
|
|
||
| metatask_process_obs_cycle_type: | ||
| var: | ||
| cycledefs_type: forecast,long_forecast | ||
| cycle_type: prod | ||
|
|
||
| task_process_radarref_#cycle_type#: | ||
| <<: *default_preproc | ||
| command: '&LOAD_MODULES_RUN_TASK_FP; "process_obs" "&JOBSdir;/JREGIONAL_PROCESS_RADARREF"' | ||
| ppn: 24 | ||
| join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' | ||
| dependency: | ||
| or: | ||
| taskdep: | ||
| attrs: | ||
| task: get_da_obs | ||
| and: | ||
| not: | ||
| taskvalid: | ||
| attrs: | ||
| task: get_da_obs | ||
| streq: | ||
| left: do_real_time | ||
| right: '{% if workflow.DO_REAL_TIME %}do_real_time{% endif %}' | ||
| timedep: '<cyclestr offset="&START_TIME_NSSLMOSAIC;">@Y@m@d@H@M00</cyclestr>' | ||
|
|
||
| task_process_lightning_#cycle_type#: | ||
| <<: *default_preproc | ||
| command: '&LOAD_MODULES_RUN_TASK_FP; "process_obs" "&JOBSdir;/JREGIONAL_PROCESS_LIGHTNING"' | ||
| join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' | ||
| dependency: | ||
| or: | ||
| taskdep: | ||
| attrs: | ||
| task: get_da_obs | ||
| and: | ||
| not: | ||
| taskvalid: | ||
| attrs: | ||
| task: get_da_obs | ||
| streq: | ||
| left: do_real_time | ||
| right: '{% if workflow.DO_REAL_TIME %}do_real_time{% endif %}' | ||
| timedep: '<cyclestr offset="&START_TIME_LIGHTNING;">@Y@m@d@H@M00</cyclestr>' | ||
|
|
||
|
|
||
| task_process_bufrobs_#cycle_type#: | ||
| <<: *default_preproc | ||
| command: '&LOAD_MODULES_RUN_TASK_FP; "process_obs" "&JOBSdir;/JREGIONAL_PROCESS_BUFROBS"' | ||
| join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' | ||
| dependency: | ||
| or: | ||
| taskdep: | ||
| attrs: | ||
| task: get_da_obs | ||
| and: | ||
| not: | ||
| taskvalid: | ||
| attrs: | ||
| task: get_da_obs | ||
| streq: | ||
| left: do_real_time | ||
| right: '{% if workflow.DO_REAL_TIME %}do_real_time{% endif %}' | ||
| timedep: '<cyclestr offset="&START_TIME_CONVENTIONAL;">@Y@m@d@H@M00</cyclestr>' | ||
|
|
Oops, something went wrong.