From a72dc80629b80644a7790f23882b4806f5d05160 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:58:54 -0600 Subject: [PATCH] per #2578, added documentation and tests for setting -input_thresh and -vld_thresh --- docs/Users_Guide/glossary.rst | 26 +++++++ docs/Users_Guide/wrappers.rst | 4 + .../pcp_combine/test_pcp_combine_wrapper.py | 78 ++++++++++++------- .../PCPCombine/PCPCombine_add.conf | 3 + .../PCPCombine/PCPCombine_bucket.conf | 3 + .../PCPCombine/PCPCombine_derive.conf | 3 + .../PCPCombine/PCPCombine_loop_custom.conf | 3 + .../PCPCombine_python_embedding.conf | 3 + .../PCPCombine/PCPCombine_subtract.conf | 2 + .../PCPCombine/PCPCombine_sum.conf | 3 + 10 files changed, 101 insertions(+), 27 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 29890ccfcf..f351df3f53 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -11979,3 +11979,29 @@ METplus Configuration Glossary There is no default, so a value must be specified. All runtime frequencies are supported. | *Used by:* UserScript + + FCST_PCP_COMBINE_INPUT_THRESH + Specify the value for the command line argument '-input_thresh' for the + forecast run of PCPCombine, e.g. :term:`FCST_PCP_COMBINE_RUN` is True. + Not used when :term:`FCST_PCP_COMBINE_METHOD` is SUBTRACT or USER_DEFINED. + + | *Used by:* PCPCombine + + OBS_PCP_COMBINE_INPUT_THRESH + Specify the value for the command line argument '-input_thresh' for the + observation run of PCPCombine, e.g. :term:`OBS_PCP_COMBINE_RUN` is True. + Not used when :term:`OBS_PCP_COMBINE_METHOD` is SUBTRACT or USER_DEFINED. + + | *Used by:* PCPCombine + + FCST_PCP_COMBINE_VLD_THRESH + Specify the value for the command line argument '-vld_thresh' for the + forecast run of PCPCombine, e.g. :term:`FCST_PCP_COMBINE_RUN` is True. + + | *Used by:* PCPCombine + + OBS_PCP_COMBINE_VLD_THRESH + Specify the value for the command line argument '-vld_thresh' for the + observation run of PCPCombine, e.g. :term:`OBS_PCP_COMBINE_RUN` is True. + + | *Used by:* PCPCombine diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index cb24c991dd..870ccf672d 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -6027,6 +6027,10 @@ METplus Configuration | :term:`PCP_COMBINE_INC_VALID_TIMES` | :term:`PCP_COMBINE_SKIP_INIT_TIMES` | :term:`PCP_COMBINE_INC_INIT_TIMES` +| :term:`FCST_PCP_COMBINE_INPUT_THRESH` +| :term:`FCST_PCP_COMBINE_VLD_THRESH` +| :term:`OBS_PCP_COMBINE_INPUT_THRESH` +| :term:`OBS_PCP_COMBINE_VLD_THRESH` | .. warning:: **DEPRECATED:** diff --git a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py index 89609abbf8..316e1c2eed 100644 --- a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py +++ b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py @@ -706,7 +706,7 @@ def test_get_extra_fields(metplus_config, names, levels, expected_args): wrapper = PCPCombineWrapper(config) - wrapper._handle_extra_field_arguments(data_src) + wrapper.set_command_line_arguments(data_src) wrapper._handle_name_argument('', data_src) for index, expected_arg in enumerate(expected_args): assert wrapper.args[index] == expected_arg @@ -858,42 +858,66 @@ def test_subtract_method_zero_accum(metplus_config): @pytest.mark.parametrize( - 'thresh, success', [ - (None, False), - (0.6, True), - (1.0, False), + 'input_thresh, vld_thresh, success', [ + (None, None, False), + (0.6, None, True), + (1.0, None, False), + (None, 0.2, False), + (0.6, 0.2, True), + (1.0, 0.2, False), ] ) @pytest.mark.wrapper -def test_add_method_missing_input(metplus_config, thresh, success): +def test_add_method_missing_input(metplus_config, get_test_data_dir, input_thresh, vld_thresh, success): data_src = "OBS" - lookback = 6 * 3600 - task_info = { - 'valid': datetime.strptime("2016090415", '%Y%m%d%H') - } - time_info = ti_calculate(task_info) + input_dir = get_test_data_dir('accum') config = metplus_config set_minimum_config_settings(config, data_src) - if thresh is not None: - config.set('config', f'{data_src}_PCP_COMBINE_INPUT_THRESH', thresh) + config.set('config', 'LOOP_BY', "VALID") + config.set('config', 'VALID_TIME_FMT', "%Y%m%d%H") + config.set('config', 'VALID_BEG', "2016090415") + config.set('config', 'VALID_END', "2016090415") + config.set('config', 'VALID_INCREMENT', "1d") + config.set('config', f'{data_src}_PCP_COMBINE_INPUT_DIR', input_dir) + config.set('config', f'{data_src}_PCP_COMBINE_OUTPUT_ACCUM', '6H') + config.set('config', f'{data_src}_PCP_COMBINE_INPUT_ACCUMS', '1H') + if input_thresh is not None: + config.set('config', f'{data_src}_PCP_COMBINE_INPUT_THRESH', input_thresh) + if vld_thresh is not None: + config.set('config', f'{data_src}_PCP_COMBINE_VLD_THRESH', vld_thresh) wrapper = PCPCombineWrapper(config) - input_dir = get_test_data_dir(wrapper.config, subdir='accum') - files_found = wrapper.setup_add_method(time_info, lookback, data_src) + assert wrapper.isOK + + all_cmds = wrapper.run_all_times() if not success: - assert not files_found + assert len(all_cmds) == 0 return - assert files_found + field_name = wrapper.config.get('config', f'{data_src}_PCP_COMBINE_INPUT_NAMES') + field_info = f"'name=\"{field_name}\";'" - in_files = [item[0] for item in files_found] - print(f"Input files: {in_files}") - assert (len(in_files) == 6 and - f"{input_dir}/20160904/file.2016090415.01h" in in_files and - f"{input_dir}/20160904/file.2016090414.01h" in in_files and - f"{input_dir}/20160904/file.2016090413.01h" in in_files and - f"{input_dir}/20160904/file.2016090412.01h" in in_files and - f"MISSING{input_dir}/20160904/file.2016090411.01h" in in_files and - f"MISSING{input_dir}/20160904/file.2016090410.01h" in in_files - ) + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + out_dir = wrapper.c_dict.get(f'{data_src}_OUTPUT_DIR') + extra_args = '' + if input_thresh: + extra_args += f' -input_thresh {input_thresh}' + if vld_thresh: + extra_args += f' -vld_thresh {vld_thresh}' + expected_cmds = [ + f"{app_path} {verbosity} -add" + f" {input_dir}/20160904/file.2016090415.01h {field_info}" + f" {input_dir}/20160904/file.2016090414.01h {field_info}" + f" {input_dir}/20160904/file.2016090413.01h {field_info}" + f" {input_dir}/20160904/file.2016090412.01h {field_info}" + f" MISSING{input_dir}/20160904/file.2016090411.01h {field_info}" + f" MISSING{input_dir}/20160904/file.2016090410.01h {field_info}" + f"{extra_args} {out_dir}/20160904/outfile.2016090415_A06h" + ] + assert len(all_cmds) == len(expected_cmds) + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf index 529e6999ca..5c41b22184 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf @@ -70,3 +70,6 @@ FCST_PCP_COMBINE_INPUT_LEVELS = Surface FCST_PCP_COMBINE_OUTPUT_ACCUM = 15M FCST_PCP_COMBINE_OUTPUT_NAME = A001500 + +#FCST_PCP_COMBINE_INPUT_THRESH = +#FCST_PCP_COMBINE_VLD_THRESH = diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf index bd8ca11286..6325df895b 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf @@ -66,3 +66,6 @@ FCST_PCP_COMBINE_INPUT_ACCUMS = {lead} FCST_PCP_COMBINE_OUTPUT_ACCUM = 15H FCST_PCP_COMBINE_OUTPUT_NAME = APCP + +#FCST_PCP_COMBINE_INPUT_THRESH = +#FCST_PCP_COMBINE_VLD_THRESH = diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_derive.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_derive.conf index b2f8d6f637..831f636ba7 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_derive.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_derive.conf @@ -79,3 +79,6 @@ FCST_PCP_COMBINE_OUTPUT_NAME = #FCST_PCP_COMBINE_EXTRA_NAMES = #FCST_PCP_COMBINE_EXTRA_LEVELS = #FCST_PCP_COMBINE_EXTRA_OUTPUT_NAMES = + +#FCST_PCP_COMBINE_INPUT_THRESH = +#FCST_PCP_COMBINE_VLD_THRESH = diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf index 64d449d8ba..720d6ccb43 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf @@ -70,3 +70,6 @@ FCST_PCP_COMBINE_INPUT_ACCUMS = 24H FCST_PCP_COMBINE_OUTPUT_ACCUM = 24H FCST_PCP_COMBINE_OUTPUT_NAME = APCP + +#FCST_PCP_COMBINE_INPUT_THRESH = +#FCST_PCP_COMBINE_VLD_THRESH = diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf index d9cd56f96e..ebb7a0127e 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf @@ -64,6 +64,9 @@ OBS_PCP_COMBINE_INPUT_DATATYPE = PYTHON_NUMPY OBS_PCP_COMBINE_INPUT_ACCUMS = 6 OBS_PCP_COMBINE_INPUT_NAMES = {PARM_BASE}/use_cases/met_tool_wrapper/PCPCombine/sum_IMERG_V06_HDF5.py {OBS_PCP_COMBINE_INPUT_DIR} IRprecipitation {valid?fmt=%Y%m%d%H} 02 +#OBS_PCP_COMBINE_INPUT_THRESH = +#OBS_PCP_COMBINE_VLD_THRESH = + [user_env_vars] # uncomment and change this to the path of a version of python that has the h5py package installed #MET_PYTHON_EXE = /path/to/python/with/h5-py/and/numpy/packages/bin/python diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf index caf0890409..59174513d1 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf @@ -69,3 +69,5 @@ FCST_PCP_COMBINE_OUTPUT_ACCUM = 3H FCST_PCP_COMBINE_OUTPUT_NAME = APCP_03 FCST_PCP_COMBINE_USE_ZERO_ACCUM = False + +#FCST_PCP_COMBINE_VLD_THRESH = diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_sum.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_sum.conf index bdfa337ebd..9012e7fb3b 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_sum.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_sum.conf @@ -66,3 +66,6 @@ FCST_PCP_COMBINE_INPUT_LEVELS = Surface FCST_PCP_COMBINE_OUTPUT_ACCUM = 15M FCST_PCP_COMBINE_OUTPUT_NAME = A001500 + +#FCST_PCP_COMBINE_INPUT_THRESH = +#FCST_PCP_COMBINE_VLD_THRESH =